From 9df65ad7e2e0bd675afb8f5deb570c025c8127b6 Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Tue, 23 Jul 2024 11:38:16 -0400
Subject: [PATCH 001/128] OrderByCostVisitor sorts alphabetically within a
field when sorting by field cardinality (#2470)
---
.../nodes/FieldOrTermNodeCostComparator.java | 5 +-
.../query/jexl/nodes/NodeCostComparator.java | 11 +++-
...estLimitReturnedGroupsToHitTermGroups.java | 66 +++++++++++++++++++
.../order/OrderByCostVisitorTest.java | 34 ++++++++++
4 files changed, 114 insertions(+), 2 deletions(-)
diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/nodes/FieldOrTermNodeCostComparator.java b/warehouse/query-core/src/main/java/datawave/query/jexl/nodes/FieldOrTermNodeCostComparator.java
index 4531df30f4..4e66d8e959 100644
--- a/warehouse/query-core/src/main/java/datawave/query/jexl/nodes/FieldOrTermNodeCostComparator.java
+++ b/warehouse/query-core/src/main/java/datawave/query/jexl/nodes/FieldOrTermNodeCostComparator.java
@@ -41,12 +41,15 @@ int getCostIndex(JexlNode node) {
sum += getCostIndex(node.jjtGetChild(i));
}
return sum;
+ } else if (QueryPropertyMarker.findInstance(node).isAnyType()) {
+ return Integer.MAX_VALUE;
} else if (node instanceof ASTAndNode) {
int lowest = Integer.MAX_VALUE;
for (int i = 0; i < node.jjtGetNumChildren(); i++) {
int cost = getCostIndex(node.jjtGetChild(i));
- if (cost < lowest)
+ if (cost < lowest) {
lowest = cost;
+ }
}
return lowest;
} else {
diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/nodes/NodeCostComparator.java b/warehouse/query-core/src/main/java/datawave/query/jexl/nodes/NodeCostComparator.java
index cc6d40b9ef..a238e5c600 100644
--- a/warehouse/query-core/src/main/java/datawave/query/jexl/nodes/NodeCostComparator.java
+++ b/warehouse/query-core/src/main/java/datawave/query/jexl/nodes/NodeCostComparator.java
@@ -4,6 +4,8 @@
import org.apache.commons.jexl3.parser.JexlNode;
+import datawave.query.jexl.visitors.JexlStringBuildingVisitor;
+
/**
* Compare nodes based on arbitrary cost.
*
@@ -15,7 +17,14 @@ public abstract class NodeCostComparator implements Comparator {
public int compare(JexlNode left, JexlNode right) {
int leftCost = getCostIndex(left);
int rightCost = getCostIndex(right);
- return Integer.compare(leftCost, rightCost);
+
+ int result = Integer.compare(leftCost, rightCost);
+ if (result == 0) {
+ // if comparing by field cost (same field) provide an opportunity to sort alphabetically
+ result = JexlStringBuildingVisitor.buildQuery(left).compareTo(JexlStringBuildingVisitor.buildQuery(right));
+ }
+
+ return result;
}
// Evaluate OR nodes last, then And nodes, then nodes by node id
diff --git a/warehouse/query-core/src/test/java/datawave/query/TestLimitReturnedGroupsToHitTermGroups.java b/warehouse/query-core/src/test/java/datawave/query/TestLimitReturnedGroupsToHitTermGroups.java
index b72bc23eee..57871ef451 100644
--- a/warehouse/query-core/src/test/java/datawave/query/TestLimitReturnedGroupsToHitTermGroups.java
+++ b/warehouse/query-core/src/test/java/datawave/query/TestLimitReturnedGroupsToHitTermGroups.java
@@ -284,10 +284,76 @@ public void testGroupWithRegexAndMatchesInGroup() throws Exception {
// definitely should NOT include group 3, 12, or 13. fox are crow are included because matchesInGroup cares about fieldname and subgroup, not group
Set goodResults = Sets.newHashSet("BIRD.WILD.2:crow", "CANINE.WILD.2:fox", "CANINE.PET.2:chihuahua", "BIRD.PET.2:parrot");
+ // added because sorting query causes 'basset' to evaluate first
+ goodResults.addAll(Sets.newHashSet("CANINE.PET.1:basset", "BIRD.PET.1:canary"));
runTestQuery(queryString, format.parse("20091231"), format.parse("20150101"), extraParameters, goodResults);
}
+ @Test
+ public void testGroupWithExpandedRegexNaturalOrderAndMatchesInGroup() throws Exception {
+ Map extraParameters = new HashMap<>();
+ extraParameters.put("include.grouping.context", "true");
+ extraParameters.put("hit.list", "true");
+ extraParameters.put("limit.fields", "CANINE=-1,BIRD=-1");
+ extraParameters.put("return.fields", "CANINE,BIRD");
+
+ // this is the same query as above with terms in natural order after regex expansion
+ String queryString = "(CANINE == 'chihuahua' || CANINE == 'beagle' || CANINE == 'dachshund' || CANINE == 'basset') && grouping:matchesInGroup(CANINE, '.*a.*', BIRD, '.*o.*')";
+
+ // definitely should NOT include group 3, 12, or 13. fox are crow are included because matchesInGroup cares about fieldname and subgroup, not group
+ Set goodResults = Sets.newHashSet("BIRD.WILD.2:crow", "CANINE.WILD.2:fox", "CANINE.PET.2:chihuahua", "BIRD.PET.2:parrot");
+ // added because sorting query causes 'basset' to evaluate first
+ goodResults.addAll(Sets.newHashSet("CANINE.PET.1:basset", "BIRD.PET.1:canary"));
+
+ runTestQuery(queryString, format.parse("20091231"), format.parse("20150101"), extraParameters, goodResults);
+ }
+
+ @Test
+ public void testGroupWithExpandedRegexAlphabeticalOrderAndMatchesInGroup() throws Exception {
+ Map extraParameters = new HashMap<>();
+ extraParameters.put("include.grouping.context", "true");
+ extraParameters.put("hit.list", "true");
+ extraParameters.put("limit.fields", "CANINE=-1,BIRD=-1");
+ extraParameters.put("return.fields", "CANINE,BIRD");
+
+ // this is the same query as above but union terms are ordered alphabetically
+ String queryString = "(CANINE == 'basset' || CANINE == 'beagle' || CANINE == 'chihuahua' || CANINE == 'dachshund') && grouping:matchesInGroup(CANINE, '.*a.*', BIRD, '.*o.*')";
+
+ // definitely should NOT include group 3, 12, or 13. fox are crow are included because matchesInGroup cares about fieldname and subgroup, not group
+ Set goodResults = Sets.newHashSet("BIRD.WILD.2:crow", "CANINE.WILD.2:fox", "CANINE.PET.2:chihuahua", "BIRD.PET.2:parrot");
+
+ // extra groups when 'basset' sorts first
+ // 'chihuahua' is from group 2, so when LimitFields reduces the document it only returns fields from group 2
+ // when 'basset' sorts first it is from group 1, so LimitFields reduces the document fields to groups 1 and 2
+ goodResults.addAll(Sets.newHashSet("CANINE.PET.1:basset", "BIRD.PET.1:canary"));
+
+ runTestQuery(queryString, format.parse("20091231"), format.parse("20150101"), extraParameters, goodResults);
+ }
+
+ @Test
+ public void testGroupWithExpandedRegexAlphabeticalOrderAndMatchesInGroupPartTwo() throws Exception {
+ Map extraParameters = new HashMap<>();
+ extraParameters.put("include.grouping.context", "true");
+ extraParameters.put("hit.list", "true");
+ extraParameters.put("limit.fields", "CANINE=-1,BIRD=-1");
+ extraParameters.put("return.fields", "CANINE,BIRD");
+
+ // this is the same query as above but 'beagle' is sorted first on purpose to demonstrate group 0 influencing return fields
+ String queryString = "(CANINE == 'beagle' || CANINE == 'basset' || CANINE == 'chihuahua' || CANINE == 'dachshund') && grouping:matchesInGroup(CANINE, '.*a.*', BIRD, '.*o.*')";
+
+ // definitely should NOT include group 3, 12, or 13. fox are crow are included because matchesInGroup cares about fieldname and subgroup, not group
+ Set goodResults = Sets.newHashSet("BIRD.WILD.2:crow", "CANINE.WILD.2:fox", "CANINE.PET.2:chihuahua", "BIRD.PET.2:parrot");
+
+ // similar to above, when 'beagle' sorts first group 0 is added to the HIT_TERM and LimitFields will return those fields
+ goodResults.addAll(Sets.newHashSet("BIRD.PET.0:parakeet", "CANINE.PET.0:beagle"));
+
+ // disable just for this test to prove group 0 can be returned
+ logic.setSortQueryBeforeGlobalIndex(false);
+ runTestQuery(queryString, format.parse("20091231"), format.parse("20150101"), extraParameters, goodResults);
+ logic.setSortQueryBeforeGlobalIndex(true);
+ }
+
@Test
public void testMultipleIncludeGroupingFalse() throws Exception {
Map extraParameters = new HashMap<>();
diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/order/OrderByCostVisitorTest.java b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/order/OrderByCostVisitorTest.java
index e407983f13..cde4e603d7 100644
--- a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/order/OrderByCostVisitorTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/order/OrderByCostVisitorTest.java
@@ -231,6 +231,40 @@ private String buildRandomQuery(List nodes, String joiner) {
return sb.toString();
}
+ @Test
+ public void testModelExpansion() throws Exception {
+ // F == '1' || F == '2', where F maps to A, B
+ String query = "A == '1' || B == '1' || A == '2' || B == '2'";
+ String expected = "A == '1' || A == '2' || B == '1' || B == '2'";
+ testDefaultOrdering(query, expected);
+ }
+
+ @Test
+ public void testSortingByNodeTypeWithinSameField() throws ParseException {
+ String query = "A == '1' || B == '1' || A =~ 'ba.*' || B =~ 'ba.*'";
+ String expected = "B == '1' || B =~ 'ba.*' || A == '1' || A =~ 'ba.*'";
+ testFieldOrdering(query, expected);
+
+ // with marker
+ query = "A == '1' || B == '1' || ((_Delayed_ = true) && (A =~ 'ba.*')) || A =~ 'ba.*' || B =~ 'ba.*'";
+ expected = "B == '1' || B =~ 'ba.*' || A == '1' || A =~ 'ba.*' || ((_Delayed_ = true) && (A =~ 'ba.*'))";
+ testFieldOrdering(query, expected);
+ }
+
+ @Test
+ public void testSortSameFieldTermAndMarker() throws ParseException {
+ String query = "((_Delayed_ = true) && (A =~ 'ba.*')) || A == '1'";
+ String expected = "A == '1' || ((_Delayed_ = true) && (A =~ 'ba.*'))";
+ testFieldOrdering(query, expected);
+ }
+
+ @Test
+ public void testSortDuplicateTerms() throws ParseException {
+ String query = "A == '1' && B == '2' && A == '1' && B == '2'";
+ testDefaultOrdering(query, "A == '1' && A == '1' && B == '2' && B == '2'");
+ testFieldOrdering(query, "B == '2' && B == '2' && A == '1' && A == '1'");
+ }
+
private void testDefaultOrdering(String query, String expected) throws ParseException {
ASTJexlScript script = JexlASTHelper.parseAndFlattenJexlQuery(query);
ASTJexlScript ordered = OrderByCostVisitor.order(script);
From 9c1c708a6d8ae502ca213bb9404eb728a68cf30c Mon Sep 17 00:00:00 2001
From: Whitney O'Meara
Date: Tue, 23 Jul 2024 19:39:24 +0000
Subject: [PATCH 002/128] updated query starter submodule
---
microservices/starters/query | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/microservices/starters/query b/microservices/starters/query
index 62cd38fbb1..8f87712829 160000
--- a/microservices/starters/query
+++ b/microservices/starters/query
@@ -1 +1 @@
-Subproject commit 62cd38fbb1445586b5ada3132d4db0268cb5da5f
+Subproject commit 8f87712829151dd688ef69e6c8a39d43c1a18332
From b871440825762c33e84c66aab68779e861456dab Mon Sep 17 00:00:00 2001
From: Whitney O'Meara
Date: Tue, 23 Jul 2024 19:46:10 +0000
Subject: [PATCH 003/128] updated cached results starter submodule
---
microservices/starters/cached-results | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/microservices/starters/cached-results b/microservices/starters/cached-results
index 57f73a6299..e3448f1148 160000
--- a/microservices/starters/cached-results
+++ b/microservices/starters/cached-results
@@ -1 +1 @@
-Subproject commit 57f73a6299282af35a12ec8f94fbd50ddfb66d83
+Subproject commit e3448f11487b8f3233f8c1473d02cf9c4fe7dedf
From e4084510c5aa47b271775fff931e929c23a80a99 Mon Sep 17 00:00:00 2001
From: Whitney O'Meara
Date: Tue, 23 Jul 2024 19:51:15 +0000
Subject: [PATCH 004/128] updated mapreduce query submodule
---
microservices/services/mapreduce-query | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/microservices/services/mapreduce-query b/microservices/services/mapreduce-query
index c97b2d80fd..3f0be3320b 160000
--- a/microservices/services/mapreduce-query
+++ b/microservices/services/mapreduce-query
@@ -1 +1 @@
-Subproject commit c97b2d80fd2b3de4dc7cd667c4877435095e5695
+Subproject commit 3f0be3320b529e112b99c16347361dff5612af30
From c2a94bb8529e711c484d8b923b449f3921f76122 Mon Sep 17 00:00:00 2001
From: Whitney O'Meara
Date: Tue, 23 Jul 2024 19:58:47 +0000
Subject: [PATCH 005/128] updated query service submodule
---
microservices/services/query | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/microservices/services/query b/microservices/services/query
index ca874129da..72dbe97516 160000
--- a/microservices/services/query
+++ b/microservices/services/query
@@ -1 +1 @@
-Subproject commit ca874129da250b2db3515477526959e2c4f41b35
+Subproject commit 72dbe975169528fc5d62a831f627e66ed88ce8b9
From 907f0f7149fb068276c0f667de260956b13c9eb9 Mon Sep 17 00:00:00 2001
From: Whitney O'Meara
Date: Tue, 23 Jul 2024 20:03:38 +0000
Subject: [PATCH 006/128] updated query executor submodule
---
microservices/services/query-executor | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/microservices/services/query-executor b/microservices/services/query-executor
index 6bc95e4b0b..b44cb68f2a 160000
--- a/microservices/services/query-executor
+++ b/microservices/services/query-executor
@@ -1 +1 @@
-Subproject commit 6bc95e4b0b343cd3527389e009c8e62ecc5d20c2
+Subproject commit b44cb68f2a379803e0ecad4d6dec99fa9575f00e
From c58b9a543897bda2bd2194d4695aa8ae1cba1280 Mon Sep 17 00:00:00 2001
From: Whitney O'Meara
Date: Tue, 23 Jul 2024 20:48:35 +0000
Subject: [PATCH 007/128] updated query submodule
---
microservices/services/query | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/microservices/services/query b/microservices/services/query
index 72dbe97516..d5b014aeed 160000
--- a/microservices/services/query
+++ b/microservices/services/query
@@ -1 +1 @@
-Subproject commit 72dbe975169528fc5d62a831f627e66ed88ce8b9
+Subproject commit d5b014aeedd7a6c65cad565dfb006ee2bbdf31d4
From 440c62e57381e09277cf3663d9f9ec76a54cecff Mon Sep 17 00:00:00 2001
From: Moon Moon
Date: Thu, 25 Jul 2024 13:50:38 -0400
Subject: [PATCH 008/128] Updating with proposed changes from Composite
Timestamp PR (#2356)
Co-authored-by: palindrome <31748527+hlgp@users.noreply.github.com>
---
.../mapreduce/job/TableConfigurationUtil.java | 13 ++++++++++++-
.../config/AbstractTableConfigHelper.java | 11 ++++++-----
.../table/config/ShardTableConfigHelper.java | 2 +-
.../config/AbstractTableConfigHelperTest.java | 18 +++++++++---------
4 files changed, 28 insertions(+), 16 deletions(-)
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/TableConfigurationUtil.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/TableConfigurationUtil.java
index e59e6e882b..b9fa2c2f98 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/TableConfigurationUtil.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/TableConfigurationUtil.java
@@ -1,5 +1,7 @@
package datawave.ingest.mapreduce.job;
+import static datawave.ingest.table.config.AbstractTableConfigHelper.DISABLE_VERSIONING_ITERATOR;
+
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -27,8 +29,10 @@
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.NamespaceOperations;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.constraints.DefaultKeySizeConstraint;
import org.apache.accumulo.core.iterators.Combiner;
import org.apache.accumulo.core.iterators.IteratorUtil;
import org.apache.hadoop.conf.Configuration;
@@ -259,7 +263,14 @@ protected void createAndConfigureTablesIfNecessary(Set tableNames, Table
// If the tables don't exist, then create them.
try {
if (!tops.exists(table)) {
- tops.create(table);
+ boolean disableVersioning = conf != null && conf.getBoolean(table + DISABLE_VERSIONING_ITERATOR, false);
+ if (disableVersioning) {
+ tops.create(table, new NewTableConfiguration().withoutDefaultIterators());
+ // withoutDefaultIterators will also skip the default table constraint, so set that
+ tops.setProperty(table, Property.TABLE_CONSTRAINT_PREFIX + "1", DefaultKeySizeConstraint.class.getName());
+ } else {
+ tops.create(table);
+ }
}
} catch (TableExistsException te) {
// in this case, somebody else must have created the table after our existence check
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/AbstractTableConfigHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/AbstractTableConfigHelper.java
index 670b0924d4..479a98d2a8 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/AbstractTableConfigHelper.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/AbstractTableConfigHelper.java
@@ -25,7 +25,7 @@
public abstract class AbstractTableConfigHelper implements TableConfigHelper {
- private static final String DISABLE_VERSIONING_ITERATOR = ".disable.versioning.iterator";
+ public static final String DISABLE_VERSIONING_ITERATOR = ".disable.versioning.iterator";
protected Configuration config;
protected AbstractTableConfigHelper() {}
@@ -92,9 +92,9 @@ public static void setPropertyIfNecessary(String tableName, String propertyName,
* @throws TableNotFoundException
* if the table is not found
*/
- protected void setAggregatorConfigurationIfNecessary(String tableName, List aggregators, TableOperations tops, Logger log)
- throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
- if (areAggregatorsConfigured(tableName, aggregators, tops)) {
+ protected void setAggregatorConfigurationIfNecessary(String tableName, List aggregators, TableOperations tops, Configuration config,
+ Logger log) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ if (areAggregatorsConfigured(tableName, aggregators, tops, config)) {
log.debug(tableName + " appears to have its aggregators configured already.");
return;
}
@@ -148,7 +148,8 @@ public static Map generateInitialTableProperties(Configuration co
* @throws TableNotFoundException
* if the table is not found
*/
- protected boolean areAggregatorsConfigured(String tableName, List aggregators, TableOperations tops) throws TableNotFoundException {
+ protected boolean areAggregatorsConfigured(String tableName, List aggregators, TableOperations tops, Configuration config)
+ throws TableNotFoundException {
boolean aggregatorsConfigured = false;
Map props = generateInitialTableProperties(config, tableName);
props.putAll(generateAggTableProperties(aggregators));
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ShardTableConfigHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ShardTableConfigHelper.java
index 2014e3d8d6..49ba804d89 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ShardTableConfigHelper.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/ShardTableConfigHelper.java
@@ -170,7 +170,7 @@ protected void configureShardTable(TableOperations tops) throws AccumuloExceptio
// Set a text index aggregator on the "tf" (Term Frequency) column family
CombinerConfiguration tfConf = new CombinerConfiguration(new Column("tf"),
new IteratorSetting(10, "TF", datawave.ingest.table.aggregator.TextIndexAggregator.class.getName()));
- setAggregatorConfigurationIfNecessary(tableName, Collections.singletonList(tfConf), tops, log);
+ setAggregatorConfigurationIfNecessary(tableName, Collections.singletonList(tfConf), tops, conf, log);
if (markingsSetupIteratorEnabled) {
for (IteratorScope scope : IteratorScope.values()) {
diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/table/config/AbstractTableConfigHelperTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/table/config/AbstractTableConfigHelperTest.java
index 12561ed04a..dcbe818fce 100644
--- a/warehouse/ingest-core/src/test/java/datawave/ingest/table/config/AbstractTableConfigHelperTest.java
+++ b/warehouse/ingest-core/src/test/java/datawave/ingest/table/config/AbstractTableConfigHelperTest.java
@@ -253,7 +253,7 @@ public void exposeAreAggregatorsConfigured() throws AssertionError, TableNotFoun
areAggregatorsConfiguredCalled = false;
AbstractTableConfigHelperTest.GET_PROPERTIES_THROWS_ACCUMULO_EXCEPTION = false;
- boolean results = this.areAggregatorsConfigured(tableName, aggregators, tops);
+ boolean results = this.areAggregatorsConfigured(tableName, aggregators, tops, config);
Assert.assertTrue("AreAggregatorsConfigured called", areAggregatorsConfiguredCalled);
Assert.assertFalse("AreAggregatorsConfigured returned and unexpected results", results);
@@ -277,7 +277,7 @@ public void exposeAreAggregatorsConfigured() throws AssertionError, TableNotFoun
parent.tableProperties.put(key, value);
}
- results = this.areAggregatorsConfigured(tableName, aggregators, tops);
+ results = this.areAggregatorsConfigured(tableName, aggregators, tops, config);
Assert.assertTrue("AreAggregatorsConfigured called", areAggregatorsConfiguredCalled);
Assert.assertFalse("AreAggregatorsConfigured returned and unexpected results", results);
@@ -288,7 +288,7 @@ public void exposeAreAggregatorsConfigured() throws AssertionError, TableNotFoun
parent.tableProperties.putAll(props);
- results = this.areAggregatorsConfigured(tableName, aggregators, tops);
+ results = this.areAggregatorsConfigured(tableName, aggregators, tops, config);
Assert.assertTrue("AreAggregatorsConfigured called", areAggregatorsConfiguredCalled);
Assert.assertTrue("AreAggregatorsConfigured returned and unexpected results", results);
@@ -301,7 +301,7 @@ public void exposeAreAggregatorsConfigured() throws AssertionError, TableNotFoun
tableName = AbstractTableConfigHelperTest.BAD_TABLE_NAME;
- this.areAggregatorsConfigured(tableName, aggregators, tops);
+ this.areAggregatorsConfigured(tableName, aggregators, tops, config);
Assert.fail("AreAggregratorsConfigured failed to throw the expected exception.");
@@ -322,7 +322,7 @@ public void exposeAreAggregatorsConfigured() throws AssertionError, TableNotFoun
tableName = AbstractTableConfigHelperTest.TABLE_NAME;
- this.areAggregatorsConfigured(tableName, aggregators, tops);
+ this.areAggregatorsConfigured(tableName, aggregators, tops, config);
Assert.fail("AreAggregratorsConfigured failed to throw the expected exception.");
@@ -357,7 +357,7 @@ public void exposeSetCombinerConfigurationIfNecessaryForTest()
resultsForOverridenAreAggregatorsConfigured = true;
areAggregatorsConfiguredCalled = false;
- this.setAggregatorConfigurationIfNecessary(tableName, aggregators, tops, log);
+ this.setAggregatorConfigurationIfNecessary(tableName, aggregators, tops, config, log);
Assert.assertTrue("SetCombinerConfigurationIfNecessary() failed to call areAggregatorsConfigured", areAggregatorsConfiguredCalled);
Assert.assertEquals("SetCombinerConfigurationIfNecessary() failed to generate the expected number of debug messages.", 1, debugMessages.size());
@@ -368,7 +368,7 @@ public void exposeSetCombinerConfigurationIfNecessaryForTest()
resultsForOverridenAreAggregatorsConfigured = false;
areAggregatorsConfiguredCalled = false;
- this.setAggregatorConfigurationIfNecessary(tableName, aggregators, tops, log);
+ this.setAggregatorConfigurationIfNecessary(tableName, aggregators, tops, config, log);
Assert.assertTrue("SetCombinerConfigurationIfNecessary() failed to call areAggregatorsConfigured", areAggregatorsConfiguredCalled);
Assert.assertEquals("SetCombinerConfigurationIfNecessary() failed to generate the expected number of debug messages.", 0, debugMessages.size());
@@ -411,7 +411,7 @@ public void exposeSetCombinerConfigurationIfNecessaryForTest()
public boolean resultsForSetLocalityGroupsConfigured = false;
@Override
- protected boolean areAggregatorsConfigured(String tableName, List aggregators, TableOperations tops)
+ protected boolean areAggregatorsConfigured(String tableName, List aggregators, TableOperations tops, Configuration config)
throws TableNotFoundException {
boolean results = false;
@@ -423,7 +423,7 @@ protected boolean areAggregatorsConfigured(String tableName, List
Date: Thu, 25 Jul 2024 18:32:04 +0000
Subject: [PATCH 009/128] Update pom's for 7.3.0-SNAPSHOT
---
common-test/pom.xml | 2 +-
contrib/datawave-quickstart/docker/pom.xml | 2 +-
core/cached-results/pom.xml | 2 +-
core/common-util/pom.xml | 2 +-
core/common/pom.xml | 2 +-
core/connection-pool/pom.xml | 2 +-
core/map-reduce/pom.xml | 2 +-
core/modification/pom.xml | 2 +-
core/pom.xml | 2 +-
core/query/pom.xml | 2 +-
core/utils/pom.xml | 2 +-
docs/pom.xml | 2 +-
microservices/pom.xml | 2 +-
microservices/services/pom.xml | 2 +-
microservices/starters/pom.xml | 2 +-
pom.xml | 2 +-
warehouse/accumulo-extensions/pom.xml | 2 +-
warehouse/age-off-utils/pom.xml | 2 +-
warehouse/age-off/pom.xml | 2 +-
warehouse/assemble/datawave/pom.xml | 2 +-
warehouse/assemble/pom.xml | 2 +-
warehouse/assemble/webservice/pom.xml | 2 +-
warehouse/common/pom.xml | 2 +-
warehouse/core/pom.xml | 2 +-
warehouse/data-dictionary-core/pom.xml | 2 +-
warehouse/edge-dictionary-core/pom.xml | 2 +-
warehouse/edge-model-configuration-core/pom.xml | 2 +-
warehouse/index-stats/pom.xml | 2 +-
warehouse/ingest-configuration/pom.xml | 2 +-
warehouse/ingest-core/pom.xml | 2 +-
warehouse/ingest-csv/pom.xml | 2 +-
warehouse/ingest-json/pom.xml | 2 +-
warehouse/ingest-nyctlc/pom.xml | 2 +-
warehouse/ingest-scripts/pom.xml | 2 +-
warehouse/ingest-ssdeep/pom.xml | 2 +-
warehouse/ingest-wikipedia/pom.xml | 2 +-
warehouse/metrics-core/pom.xml | 2 +-
warehouse/ops-tools/config-compare/pom.xml | 2 +-
warehouse/ops-tools/index-validation/pom.xml | 2 +-
warehouse/ops-tools/pom.xml | 2 +-
warehouse/pom.xml | 2 +-
warehouse/query-core/pom.xml | 2 +-
warehouse/regression-testing/pom.xml | 2 +-
warehouse/ssdeep-common/pom.xml | 2 +-
web-services/accumulo/pom.xml | 2 +-
web-services/atom/pom.xml | 2 +-
web-services/cached-results/pom.xml | 2 +-
web-services/client/pom.xml | 2 +-
web-services/common-util/pom.xml | 2 +-
web-services/common/pom.xml | 2 +-
web-services/deploy/application/pom.xml | 2 +-
web-services/deploy/configuration/pom.xml | 2 +-
web-services/deploy/docs/pom.xml | 2 +-
web-services/deploy/pom.xml | 2 +-
web-services/deploy/spring-framework-integration/pom.xml | 2 +-
web-services/dictionary/pom.xml | 2 +-
web-services/examples/client-login/pom.xml | 2 +-
web-services/examples/http-client/pom.xml | 2 +-
web-services/examples/jms-client/pom.xml | 2 +-
web-services/examples/pom.xml | 2 +-
web-services/examples/query-war/pom.xml | 2 +-
web-services/map-reduce-embedded/pom.xml | 2 +-
web-services/map-reduce-status/pom.xml | 2 +-
web-services/map-reduce/pom.xml | 2 +-
web-services/metrics/pom.xml | 2 +-
web-services/model/pom.xml | 2 +-
web-services/modification/pom.xml | 2 +-
web-services/pom.xml | 2 +-
web-services/query-websocket/pom.xml | 2 +-
web-services/query/pom.xml | 2 +-
web-services/rest-api/pom.xml | 2 +-
web-services/security/pom.xml | 2 +-
web-services/web-root/pom.xml | 2 +-
73 files changed, 73 insertions(+), 73 deletions(-)
diff --git a/common-test/pom.xml b/common-test/pom.xml
index e4a3b80db0..47e5748ad4 100644
--- a/common-test/pom.xml
+++ b/common-test/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-common-test
${project.artifactId}
diff --git a/contrib/datawave-quickstart/docker/pom.xml b/contrib/datawave-quickstart/docker/pom.xml
index 505a2ba975..f09081ff2f 100644
--- a/contrib/datawave-quickstart/docker/pom.xml
+++ b/contrib/datawave-quickstart/docker/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
../../../pom.xml
quickstart
diff --git a/core/cached-results/pom.xml b/core/cached-results/pom.xml
index e81cb961ce..3c6ff891c5 100644
--- a/core/cached-results/pom.xml
+++ b/core/cached-results/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-core-cached-results
${project.artifactId}
diff --git a/core/common-util/pom.xml b/core/common-util/pom.xml
index 539e13d5fd..36d3f74ee0 100644
--- a/core/common-util/pom.xml
+++ b/core/common-util/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-core-common-util
${project.artifactId}
diff --git a/core/common/pom.xml b/core/common/pom.xml
index 076fd4243a..f0c5c146e5 100644
--- a/core/common/pom.xml
+++ b/core/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-core-common
${project.artifactId}
diff --git a/core/connection-pool/pom.xml b/core/connection-pool/pom.xml
index 57db045dc1..72ffc84b89 100644
--- a/core/connection-pool/pom.xml
+++ b/core/connection-pool/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-core-connection-pool
${project.artifactId}
diff --git a/core/map-reduce/pom.xml b/core/map-reduce/pom.xml
index 4a3ffd2714..19e6cd165b 100644
--- a/core/map-reduce/pom.xml
+++ b/core/map-reduce/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-core-map-reduce
${project.artifactId}
diff --git a/core/modification/pom.xml b/core/modification/pom.xml
index c58b3ccd75..1b673aac03 100644
--- a/core/modification/pom.xml
+++ b/core/modification/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-core-modification
${project.artifactId}
diff --git a/core/pom.xml b/core/pom.xml
index f6b1a6a294..0782a50da5 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
gov.nsa.datawave.core
datawave-core-parent
diff --git a/core/query/pom.xml b/core/query/pom.xml
index 40c9a4da44..f408dd93bc 100644
--- a/core/query/pom.xml
+++ b/core/query/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-core-query
${project.artifactId}
diff --git a/core/utils/pom.xml b/core/utils/pom.xml
index 763f20783f..9ca5cc4e5a 100644
--- a/core/utils/pom.xml
+++ b/core/utils/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
gov.nsa.datawave.core
datawave-utils-parent
diff --git a/docs/pom.xml b/docs/pom.xml
index d18da7f888..371e0eec79 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-docs
diff --git a/microservices/pom.xml b/microservices/pom.xml
index 3f934abe69..fd93c515ff 100644
--- a/microservices/pom.xml
+++ b/microservices/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
gov.nsa.datawave.microservice
datawave-microservice-build-parent
diff --git a/microservices/services/pom.xml b/microservices/services/pom.xml
index 278ed9928e..ffaa2dfd91 100644
--- a/microservices/services/pom.xml
+++ b/microservices/services/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.microservice
datawave-microservice-build-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-microservice-service-build-parent
pom
diff --git a/microservices/starters/pom.xml b/microservices/starters/pom.xml
index 369559f288..77621c90bb 100644
--- a/microservices/starters/pom.xml
+++ b/microservices/starters/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.microservice
datawave-microservice-build-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-microservice-starter-build-parent
pom
diff --git a/pom.xml b/pom.xml
index 7f1afff223..03943d8dce 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
4.0.0
gov.nsa.datawave
datawave-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
pom
DataWave
DataWave is a Java-based ingest and query framework that leverages Apache Accumulo to provide fast, secure access to your data.
diff --git a/warehouse/accumulo-extensions/pom.xml b/warehouse/accumulo-extensions/pom.xml
index ce637f4f9b..053bc39cba 100644
--- a/warehouse/accumulo-extensions/pom.xml
+++ b/warehouse/accumulo-extensions/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-accumulo-extensions
${project.artifactId}
diff --git a/warehouse/age-off-utils/pom.xml b/warehouse/age-off-utils/pom.xml
index 9a6ec52c4d..54509389b5 100644
--- a/warehouse/age-off-utils/pom.xml
+++ b/warehouse/age-off-utils/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-age-off-utils
${project.artifactId}
diff --git a/warehouse/age-off/pom.xml b/warehouse/age-off/pom.xml
index 54abf09b0d..41a1757055 100644
--- a/warehouse/age-off/pom.xml
+++ b/warehouse/age-off/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-age-off
${project.artifactId}
diff --git a/warehouse/assemble/datawave/pom.xml b/warehouse/assemble/datawave/pom.xml
index e87e0f32f3..5bfd5b87bf 100644
--- a/warehouse/assemble/datawave/pom.xml
+++ b/warehouse/assemble/datawave/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
assemble-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
assemble-datawave
jar
diff --git a/warehouse/assemble/pom.xml b/warehouse/assemble/pom.xml
index ff18b0b6b5..1d3913c969 100644
--- a/warehouse/assemble/pom.xml
+++ b/warehouse/assemble/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
assemble-parent
pom
diff --git a/warehouse/assemble/webservice/pom.xml b/warehouse/assemble/webservice/pom.xml
index 8b286b54d1..892716435e 100644
--- a/warehouse/assemble/webservice/pom.xml
+++ b/warehouse/assemble/webservice/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
assemble-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
assemble-webservice
${project.artifactId}
diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml
index ea4734cfea..e6e8b755fa 100644
--- a/warehouse/common/pom.xml
+++ b/warehouse/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-common
${project.artifactId}
diff --git a/warehouse/core/pom.xml b/warehouse/core/pom.xml
index 8020a9d24b..e7eaf1a7ab 100644
--- a/warehouse/core/pom.xml
+++ b/warehouse/core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-core
jar
diff --git a/warehouse/data-dictionary-core/pom.xml b/warehouse/data-dictionary-core/pom.xml
index 33a50a35b6..f90d17c202 100644
--- a/warehouse/data-dictionary-core/pom.xml
+++ b/warehouse/data-dictionary-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-data-dictionary-core
jar
diff --git a/warehouse/edge-dictionary-core/pom.xml b/warehouse/edge-dictionary-core/pom.xml
index b790c631fa..7b8b4823fa 100644
--- a/warehouse/edge-dictionary-core/pom.xml
+++ b/warehouse/edge-dictionary-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-edge-dictionary-core
jar
diff --git a/warehouse/edge-model-configuration-core/pom.xml b/warehouse/edge-model-configuration-core/pom.xml
index eae14b0446..4d11abecda 100644
--- a/warehouse/edge-model-configuration-core/pom.xml
+++ b/warehouse/edge-model-configuration-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-edge-model-configuration-core
jar
diff --git a/warehouse/index-stats/pom.xml b/warehouse/index-stats/pom.xml
index 1896e01ade..0c2a4bad5a 100644
--- a/warehouse/index-stats/pom.xml
+++ b/warehouse/index-stats/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-index-stats
jar
diff --git a/warehouse/ingest-configuration/pom.xml b/warehouse/ingest-configuration/pom.xml
index cb9204671e..d735715294 100644
--- a/warehouse/ingest-configuration/pom.xml
+++ b/warehouse/ingest-configuration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ingest-configuration
diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml
index e6af49811f..2009ba8827 100644
--- a/warehouse/ingest-core/pom.xml
+++ b/warehouse/ingest-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ingest-core
jar
diff --git a/warehouse/ingest-csv/pom.xml b/warehouse/ingest-csv/pom.xml
index e763a978ab..12a093182c 100644
--- a/warehouse/ingest-csv/pom.xml
+++ b/warehouse/ingest-csv/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ingest-csv
jar
diff --git a/warehouse/ingest-json/pom.xml b/warehouse/ingest-json/pom.xml
index 8513be753c..cb21fbaab8 100644
--- a/warehouse/ingest-json/pom.xml
+++ b/warehouse/ingest-json/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ingest-json
jar
diff --git a/warehouse/ingest-nyctlc/pom.xml b/warehouse/ingest-nyctlc/pom.xml
index 2757ec06e1..d9bf64ca02 100644
--- a/warehouse/ingest-nyctlc/pom.xml
+++ b/warehouse/ingest-nyctlc/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ingest-nyctlc
jar
diff --git a/warehouse/ingest-scripts/pom.xml b/warehouse/ingest-scripts/pom.xml
index dc669ab02c..88f2be3eac 100644
--- a/warehouse/ingest-scripts/pom.xml
+++ b/warehouse/ingest-scripts/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ingest-scripts
${project.artifactId}
diff --git a/warehouse/ingest-ssdeep/pom.xml b/warehouse/ingest-ssdeep/pom.xml
index 9bbf0fb81d..b0dce60368 100644
--- a/warehouse/ingest-ssdeep/pom.xml
+++ b/warehouse/ingest-ssdeep/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ingest-ssdeep
diff --git a/warehouse/ingest-wikipedia/pom.xml b/warehouse/ingest-wikipedia/pom.xml
index 40bbc8f227..2411456de0 100644
--- a/warehouse/ingest-wikipedia/pom.xml
+++ b/warehouse/ingest-wikipedia/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ingest-wikipedia
jar
diff --git a/warehouse/metrics-core/pom.xml b/warehouse/metrics-core/pom.xml
index c144d602e1..848deab6a0 100644
--- a/warehouse/metrics-core/pom.xml
+++ b/warehouse/metrics-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-metrics-core
jar
diff --git a/warehouse/ops-tools/config-compare/pom.xml b/warehouse/ops-tools/config-compare/pom.xml
index 6e435f06d1..efef8a6a16 100644
--- a/warehouse/ops-tools/config-compare/pom.xml
+++ b/warehouse/ops-tools/config-compare/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-ops-tools-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ops-tools-config-compare
diff --git a/warehouse/ops-tools/index-validation/pom.xml b/warehouse/ops-tools/index-validation/pom.xml
index 338f86878a..1a0d667827 100644
--- a/warehouse/ops-tools/index-validation/pom.xml
+++ b/warehouse/ops-tools/index-validation/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-ops-tools-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ops-tools-index-validation
jar
diff --git a/warehouse/ops-tools/pom.xml b/warehouse/ops-tools/pom.xml
index 935ea0dbeb..63dc4d43e0 100644
--- a/warehouse/ops-tools/pom.xml
+++ b/warehouse/ops-tools/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ops-tools-parent
pom
diff --git a/warehouse/pom.xml b/warehouse/pom.xml
index 78b95496a1..47b3151514 100644
--- a/warehouse/pom.xml
+++ b/warehouse/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-warehouse-parent
pom
diff --git a/warehouse/query-core/pom.xml b/warehouse/query-core/pom.xml
index 63e8b07f34..0ab62b7297 100644
--- a/warehouse/query-core/pom.xml
+++ b/warehouse/query-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-query-core
jar
diff --git a/warehouse/regression-testing/pom.xml b/warehouse/regression-testing/pom.xml
index a8ebe512bb..f0d1c5bf8b 100644
--- a/warehouse/regression-testing/pom.xml
+++ b/warehouse/regression-testing/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-regression-testing
${project.artifactId}
diff --git a/warehouse/ssdeep-common/pom.xml b/warehouse/ssdeep-common/pom.xml
index 3a17715dd9..1ee5420dc7 100644
--- a/warehouse/ssdeep-common/pom.xml
+++ b/warehouse/ssdeep-common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ssdeep-common
diff --git a/web-services/accumulo/pom.xml b/web-services/accumulo/pom.xml
index e389ff3667..c56d680f7e 100644
--- a/web-services/accumulo/pom.xml
+++ b/web-services/accumulo/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-accumulo
ejb
diff --git a/web-services/atom/pom.xml b/web-services/atom/pom.xml
index 7e4c5a00aa..3ed8cdaf74 100644
--- a/web-services/atom/pom.xml
+++ b/web-services/atom/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-atom
ejb
diff --git a/web-services/cached-results/pom.xml b/web-services/cached-results/pom.xml
index ed2e0a2024..7ba4396d91 100644
--- a/web-services/cached-results/pom.xml
+++ b/web-services/cached-results/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-cached-results
ejb
diff --git a/web-services/client/pom.xml b/web-services/client/pom.xml
index bbcde53b50..308b7324b4 100644
--- a/web-services/client/pom.xml
+++ b/web-services/client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-client
jar
diff --git a/web-services/common-util/pom.xml b/web-services/common-util/pom.xml
index 0a7aa49ce5..11053e83ca 100644
--- a/web-services/common-util/pom.xml
+++ b/web-services/common-util/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-common-util
jar
diff --git a/web-services/common/pom.xml b/web-services/common/pom.xml
index 3c707b6e75..92a83c41d1 100644
--- a/web-services/common/pom.xml
+++ b/web-services/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-common
ejb
diff --git a/web-services/deploy/application/pom.xml b/web-services/deploy/application/pom.xml
index 892b1511c3..6a57224e36 100644
--- a/web-services/deploy/application/pom.xml
+++ b/web-services/deploy/application/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-deploy-application
ear
diff --git a/web-services/deploy/configuration/pom.xml b/web-services/deploy/configuration/pom.xml
index 814f91b48a..3dfd1a7598 100644
--- a/web-services/deploy/configuration/pom.xml
+++ b/web-services/deploy/configuration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-deploy-configuration
jar
diff --git a/web-services/deploy/docs/pom.xml b/web-services/deploy/docs/pom.xml
index 894bb5f923..3143d7daa2 100644
--- a/web-services/deploy/docs/pom.xml
+++ b/web-services/deploy/docs/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-deploy-docs
war
diff --git a/web-services/deploy/pom.xml b/web-services/deploy/pom.xml
index 36cf35dda0..b03953e423 100644
--- a/web-services/deploy/pom.xml
+++ b/web-services/deploy/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
diff --git a/web-services/deploy/spring-framework-integration/pom.xml b/web-services/deploy/spring-framework-integration/pom.xml
index 42f342db50..a4da5f63c9 100644
--- a/web-services/deploy/spring-framework-integration/pom.xml
+++ b/web-services/deploy/spring-framework-integration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
spring-framework-integration
${project.artifactId}
diff --git a/web-services/dictionary/pom.xml b/web-services/dictionary/pom.xml
index 6d622bdfb2..11ce4058a4 100644
--- a/web-services/dictionary/pom.xml
+++ b/web-services/dictionary/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-dictionary
ejb
diff --git a/web-services/examples/client-login/pom.xml b/web-services/examples/client-login/pom.xml
index b66549068d..7a33c36b08 100644
--- a/web-services/examples/client-login/pom.xml
+++ b/web-services/examples/client-login/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-examples-client-login
ejb
diff --git a/web-services/examples/http-client/pom.xml b/web-services/examples/http-client/pom.xml
index 60541a929d..d57aed10ec 100644
--- a/web-services/examples/http-client/pom.xml
+++ b/web-services/examples/http-client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-examples-http-client
jar
diff --git a/web-services/examples/jms-client/pom.xml b/web-services/examples/jms-client/pom.xml
index 94eaf8b3c3..b98231ce62 100644
--- a/web-services/examples/jms-client/pom.xml
+++ b/web-services/examples/jms-client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-examples-jms-client
jar
diff --git a/web-services/examples/pom.xml b/web-services/examples/pom.xml
index 2c9ff72f5e..e528702657 100644
--- a/web-services/examples/pom.xml
+++ b/web-services/examples/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-examples-parent
pom
diff --git a/web-services/examples/query-war/pom.xml b/web-services/examples/query-war/pom.xml
index eb8a921978..55ff7a67b8 100644
--- a/web-services/examples/query-war/pom.xml
+++ b/web-services/examples/query-war/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-examples-query-war
war
diff --git a/web-services/map-reduce-embedded/pom.xml b/web-services/map-reduce-embedded/pom.xml
index 8aaa50a355..2ac62f8ee0 100644
--- a/web-services/map-reduce-embedded/pom.xml
+++ b/web-services/map-reduce-embedded/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-map-reduce-embedded
jar
diff --git a/web-services/map-reduce-status/pom.xml b/web-services/map-reduce-status/pom.xml
index 21399f53a8..b86c8c9396 100644
--- a/web-services/map-reduce-status/pom.xml
+++ b/web-services/map-reduce-status/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-map-reduce-status
ejb
diff --git a/web-services/map-reduce/pom.xml b/web-services/map-reduce/pom.xml
index db13f2b30e..3293760d53 100644
--- a/web-services/map-reduce/pom.xml
+++ b/web-services/map-reduce/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-map-reduce
ejb
diff --git a/web-services/metrics/pom.xml b/web-services/metrics/pom.xml
index d69b02c8e4..f37ccc6f73 100644
--- a/web-services/metrics/pom.xml
+++ b/web-services/metrics/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-metrics
ejb
diff --git a/web-services/model/pom.xml b/web-services/model/pom.xml
index 043fc848ce..e5ba9b1b2e 100644
--- a/web-services/model/pom.xml
+++ b/web-services/model/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-model
ejb
diff --git a/web-services/modification/pom.xml b/web-services/modification/pom.xml
index d760aa9a3d..1f6173c489 100644
--- a/web-services/modification/pom.xml
+++ b/web-services/modification/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-modification
ejb
diff --git a/web-services/pom.xml b/web-services/pom.xml
index 4ada3e522f..5bd3b7a820 100644
--- a/web-services/pom.xml
+++ b/web-services/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
gov.nsa.datawave.webservices
datawave-ws-parent
diff --git a/web-services/query-websocket/pom.xml b/web-services/query-websocket/pom.xml
index 06ed6844c4..60c860a05f 100644
--- a/web-services/query-websocket/pom.xml
+++ b/web-services/query-websocket/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-query-websocket
war
diff --git a/web-services/query/pom.xml b/web-services/query/pom.xml
index ef5a454614..9cc2da6094 100644
--- a/web-services/query/pom.xml
+++ b/web-services/query/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-query
ejb
diff --git a/web-services/rest-api/pom.xml b/web-services/rest-api/pom.xml
index 1aae0b4cec..dc50b24d32 100644
--- a/web-services/rest-api/pom.xml
+++ b/web-services/rest-api/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-rest-api
war
diff --git a/web-services/security/pom.xml b/web-services/security/pom.xml
index a33cf2b4ad..fecfc78a57 100644
--- a/web-services/security/pom.xml
+++ b/web-services/security/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-security
ejb
diff --git a/web-services/web-root/pom.xml b/web-services/web-root/pom.xml
index 5251ccc2a5..1aaea45f87 100644
--- a/web-services/web-root/pom.xml
+++ b/web-services/web-root/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.3.0-SNAPSHOT
datawave-ws-web-root
war
From 8405ea35d866996666201f160b1e7b056c1cabf4 Mon Sep 17 00:00:00 2001
From: hgklohr
Date: Thu, 25 Jul 2024 18:33:41 +0000
Subject: [PATCH 010/128] 7.2.0
---
common-test/pom.xml | 2 +-
contrib/datawave-quickstart/docker/pom.xml | 2 +-
core/cached-results/pom.xml | 2 +-
core/common-util/pom.xml | 2 +-
core/common/pom.xml | 2 +-
core/connection-pool/pom.xml | 2 +-
core/map-reduce/pom.xml | 2 +-
core/modification/pom.xml | 2 +-
core/pom.xml | 2 +-
core/query/pom.xml | 2 +-
core/utils/pom.xml | 2 +-
docs/pom.xml | 2 +-
microservices/pom.xml | 2 +-
microservices/services/pom.xml | 2 +-
microservices/starters/pom.xml | 2 +-
pom.xml | 2 +-
warehouse/accumulo-extensions/pom.xml | 2 +-
warehouse/age-off-utils/pom.xml | 2 +-
warehouse/age-off/pom.xml | 2 +-
warehouse/assemble/datawave/pom.xml | 2 +-
warehouse/assemble/pom.xml | 2 +-
warehouse/assemble/webservice/pom.xml | 2 +-
warehouse/common/pom.xml | 2 +-
warehouse/core/pom.xml | 2 +-
warehouse/data-dictionary-core/pom.xml | 2 +-
warehouse/edge-dictionary-core/pom.xml | 2 +-
warehouse/edge-model-configuration-core/pom.xml | 2 +-
warehouse/index-stats/pom.xml | 2 +-
warehouse/ingest-configuration/pom.xml | 2 +-
warehouse/ingest-core/pom.xml | 2 +-
warehouse/ingest-csv/pom.xml | 2 +-
warehouse/ingest-json/pom.xml | 2 +-
warehouse/ingest-nyctlc/pom.xml | 2 +-
warehouse/ingest-scripts/pom.xml | 2 +-
warehouse/ingest-ssdeep/pom.xml | 2 +-
warehouse/ingest-wikipedia/pom.xml | 2 +-
warehouse/metrics-core/pom.xml | 2 +-
warehouse/ops-tools/config-compare/pom.xml | 2 +-
warehouse/ops-tools/index-validation/pom.xml | 2 +-
warehouse/ops-tools/pom.xml | 2 +-
warehouse/pom.xml | 2 +-
warehouse/query-core/pom.xml | 2 +-
warehouse/regression-testing/pom.xml | 2 +-
warehouse/ssdeep-common/pom.xml | 2 +-
web-services/accumulo/pom.xml | 2 +-
web-services/atom/pom.xml | 2 +-
web-services/cached-results/pom.xml | 2 +-
web-services/client/pom.xml | 2 +-
web-services/common-util/pom.xml | 2 +-
web-services/common/pom.xml | 2 +-
web-services/deploy/application/pom.xml | 2 +-
web-services/deploy/configuration/pom.xml | 2 +-
web-services/deploy/docs/pom.xml | 2 +-
web-services/deploy/pom.xml | 2 +-
web-services/deploy/spring-framework-integration/pom.xml | 2 +-
web-services/dictionary/pom.xml | 2 +-
web-services/examples/client-login/pom.xml | 2 +-
web-services/examples/http-client/pom.xml | 2 +-
web-services/examples/jms-client/pom.xml | 2 +-
web-services/examples/pom.xml | 2 +-
web-services/examples/query-war/pom.xml | 2 +-
web-services/map-reduce-embedded/pom.xml | 2 +-
web-services/map-reduce-status/pom.xml | 2 +-
web-services/map-reduce/pom.xml | 2 +-
web-services/metrics/pom.xml | 2 +-
web-services/model/pom.xml | 2 +-
web-services/modification/pom.xml | 2 +-
web-services/pom.xml | 2 +-
web-services/query-websocket/pom.xml | 2 +-
web-services/query/pom.xml | 2 +-
web-services/rest-api/pom.xml | 2 +-
web-services/security/pom.xml | 2 +-
web-services/web-root/pom.xml | 2 +-
73 files changed, 73 insertions(+), 73 deletions(-)
diff --git a/common-test/pom.xml b/common-test/pom.xml
index e4a3b80db0..c843260a49 100644
--- a/common-test/pom.xml
+++ b/common-test/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-common-test
${project.artifactId}
diff --git a/contrib/datawave-quickstart/docker/pom.xml b/contrib/datawave-quickstart/docker/pom.xml
index 505a2ba975..4bc20dc46b 100644
--- a/contrib/datawave-quickstart/docker/pom.xml
+++ b/contrib/datawave-quickstart/docker/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
../../../pom.xml
quickstart
diff --git a/core/cached-results/pom.xml b/core/cached-results/pom.xml
index e81cb961ce..0e77b53ee0 100644
--- a/core/cached-results/pom.xml
+++ b/core/cached-results/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-core-cached-results
${project.artifactId}
diff --git a/core/common-util/pom.xml b/core/common-util/pom.xml
index 539e13d5fd..907eada1fc 100644
--- a/core/common-util/pom.xml
+++ b/core/common-util/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-core-common-util
${project.artifactId}
diff --git a/core/common/pom.xml b/core/common/pom.xml
index 076fd4243a..a32f70b44f 100644
--- a/core/common/pom.xml
+++ b/core/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-core-common
${project.artifactId}
diff --git a/core/connection-pool/pom.xml b/core/connection-pool/pom.xml
index 57db045dc1..60393a1aa7 100644
--- a/core/connection-pool/pom.xml
+++ b/core/connection-pool/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-core-connection-pool
${project.artifactId}
diff --git a/core/map-reduce/pom.xml b/core/map-reduce/pom.xml
index 4a3ffd2714..f696753415 100644
--- a/core/map-reduce/pom.xml
+++ b/core/map-reduce/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-core-map-reduce
${project.artifactId}
diff --git a/core/modification/pom.xml b/core/modification/pom.xml
index c58b3ccd75..e3c7dd122e 100644
--- a/core/modification/pom.xml
+++ b/core/modification/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-core-modification
${project.artifactId}
diff --git a/core/pom.xml b/core/pom.xml
index f6b1a6a294..b2cd4be510 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
gov.nsa.datawave.core
datawave-core-parent
diff --git a/core/query/pom.xml b/core/query/pom.xml
index 40c9a4da44..111b9244b1 100644
--- a/core/query/pom.xml
+++ b/core/query/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-core-query
${project.artifactId}
diff --git a/core/utils/pom.xml b/core/utils/pom.xml
index 763f20783f..0651ba557a 100644
--- a/core/utils/pom.xml
+++ b/core/utils/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
gov.nsa.datawave.core
datawave-utils-parent
diff --git a/docs/pom.xml b/docs/pom.xml
index d18da7f888..e239f48f30 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-docs
diff --git a/microservices/pom.xml b/microservices/pom.xml
index 3f934abe69..5a5b2d830b 100644
--- a/microservices/pom.xml
+++ b/microservices/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
gov.nsa.datawave.microservice
datawave-microservice-build-parent
diff --git a/microservices/services/pom.xml b/microservices/services/pom.xml
index 278ed9928e..d868b486ac 100644
--- a/microservices/services/pom.xml
+++ b/microservices/services/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.microservice
datawave-microservice-build-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-microservice-service-build-parent
pom
diff --git a/microservices/starters/pom.xml b/microservices/starters/pom.xml
index 369559f288..5fadb33873 100644
--- a/microservices/starters/pom.xml
+++ b/microservices/starters/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.microservice
datawave-microservice-build-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-microservice-starter-build-parent
pom
diff --git a/pom.xml b/pom.xml
index 7f1afff223..62ab380418 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
4.0.0
gov.nsa.datawave
datawave-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
pom
DataWave
DataWave is a Java-based ingest and query framework that leverages Apache Accumulo to provide fast, secure access to your data.
diff --git a/warehouse/accumulo-extensions/pom.xml b/warehouse/accumulo-extensions/pom.xml
index ce637f4f9b..6c0863c199 100644
--- a/warehouse/accumulo-extensions/pom.xml
+++ b/warehouse/accumulo-extensions/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-accumulo-extensions
${project.artifactId}
diff --git a/warehouse/age-off-utils/pom.xml b/warehouse/age-off-utils/pom.xml
index 9a6ec52c4d..f4005515b6 100644
--- a/warehouse/age-off-utils/pom.xml
+++ b/warehouse/age-off-utils/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-age-off-utils
${project.artifactId}
diff --git a/warehouse/age-off/pom.xml b/warehouse/age-off/pom.xml
index 54abf09b0d..036a8d18f5 100644
--- a/warehouse/age-off/pom.xml
+++ b/warehouse/age-off/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-age-off
${project.artifactId}
diff --git a/warehouse/assemble/datawave/pom.xml b/warehouse/assemble/datawave/pom.xml
index e87e0f32f3..c075aea9e0 100644
--- a/warehouse/assemble/datawave/pom.xml
+++ b/warehouse/assemble/datawave/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
assemble-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
assemble-datawave
jar
diff --git a/warehouse/assemble/pom.xml b/warehouse/assemble/pom.xml
index ff18b0b6b5..8204982332 100644
--- a/warehouse/assemble/pom.xml
+++ b/warehouse/assemble/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
assemble-parent
pom
diff --git a/warehouse/assemble/webservice/pom.xml b/warehouse/assemble/webservice/pom.xml
index 8b286b54d1..456bb593ff 100644
--- a/warehouse/assemble/webservice/pom.xml
+++ b/warehouse/assemble/webservice/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
assemble-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
assemble-webservice
${project.artifactId}
diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml
index ea4734cfea..f482856bc9 100644
--- a/warehouse/common/pom.xml
+++ b/warehouse/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-common
${project.artifactId}
diff --git a/warehouse/core/pom.xml b/warehouse/core/pom.xml
index 8020a9d24b..22444921a5 100644
--- a/warehouse/core/pom.xml
+++ b/warehouse/core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-core
jar
diff --git a/warehouse/data-dictionary-core/pom.xml b/warehouse/data-dictionary-core/pom.xml
index 33a50a35b6..bf686dbbcb 100644
--- a/warehouse/data-dictionary-core/pom.xml
+++ b/warehouse/data-dictionary-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-data-dictionary-core
jar
diff --git a/warehouse/edge-dictionary-core/pom.xml b/warehouse/edge-dictionary-core/pom.xml
index b790c631fa..7e0002efbd 100644
--- a/warehouse/edge-dictionary-core/pom.xml
+++ b/warehouse/edge-dictionary-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-edge-dictionary-core
jar
diff --git a/warehouse/edge-model-configuration-core/pom.xml b/warehouse/edge-model-configuration-core/pom.xml
index eae14b0446..f83d0c1745 100644
--- a/warehouse/edge-model-configuration-core/pom.xml
+++ b/warehouse/edge-model-configuration-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-edge-model-configuration-core
jar
diff --git a/warehouse/index-stats/pom.xml b/warehouse/index-stats/pom.xml
index 1896e01ade..2df5614ef3 100644
--- a/warehouse/index-stats/pom.xml
+++ b/warehouse/index-stats/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-index-stats
jar
diff --git a/warehouse/ingest-configuration/pom.xml b/warehouse/ingest-configuration/pom.xml
index cb9204671e..068f9494df 100644
--- a/warehouse/ingest-configuration/pom.xml
+++ b/warehouse/ingest-configuration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ingest-configuration
diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml
index e6af49811f..4fcab17c54 100644
--- a/warehouse/ingest-core/pom.xml
+++ b/warehouse/ingest-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ingest-core
jar
diff --git a/warehouse/ingest-csv/pom.xml b/warehouse/ingest-csv/pom.xml
index e763a978ab..060adfe2f5 100644
--- a/warehouse/ingest-csv/pom.xml
+++ b/warehouse/ingest-csv/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ingest-csv
jar
diff --git a/warehouse/ingest-json/pom.xml b/warehouse/ingest-json/pom.xml
index 8513be753c..10c9f76ea3 100644
--- a/warehouse/ingest-json/pom.xml
+++ b/warehouse/ingest-json/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ingest-json
jar
diff --git a/warehouse/ingest-nyctlc/pom.xml b/warehouse/ingest-nyctlc/pom.xml
index 2757ec06e1..53dd6bcbf3 100644
--- a/warehouse/ingest-nyctlc/pom.xml
+++ b/warehouse/ingest-nyctlc/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ingest-nyctlc
jar
diff --git a/warehouse/ingest-scripts/pom.xml b/warehouse/ingest-scripts/pom.xml
index dc669ab02c..9843ee664e 100644
--- a/warehouse/ingest-scripts/pom.xml
+++ b/warehouse/ingest-scripts/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ingest-scripts
${project.artifactId}
diff --git a/warehouse/ingest-ssdeep/pom.xml b/warehouse/ingest-ssdeep/pom.xml
index 9bbf0fb81d..4dd50f5609 100644
--- a/warehouse/ingest-ssdeep/pom.xml
+++ b/warehouse/ingest-ssdeep/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ingest-ssdeep
diff --git a/warehouse/ingest-wikipedia/pom.xml b/warehouse/ingest-wikipedia/pom.xml
index 40bbc8f227..c176c51ed5 100644
--- a/warehouse/ingest-wikipedia/pom.xml
+++ b/warehouse/ingest-wikipedia/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ingest-wikipedia
jar
diff --git a/warehouse/metrics-core/pom.xml b/warehouse/metrics-core/pom.xml
index c144d602e1..d3b07dbb44 100644
--- a/warehouse/metrics-core/pom.xml
+++ b/warehouse/metrics-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-metrics-core
jar
diff --git a/warehouse/ops-tools/config-compare/pom.xml b/warehouse/ops-tools/config-compare/pom.xml
index 6e435f06d1..e9cc6f32d0 100644
--- a/warehouse/ops-tools/config-compare/pom.xml
+++ b/warehouse/ops-tools/config-compare/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-ops-tools-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ops-tools-config-compare
diff --git a/warehouse/ops-tools/index-validation/pom.xml b/warehouse/ops-tools/index-validation/pom.xml
index 338f86878a..fa41b77806 100644
--- a/warehouse/ops-tools/index-validation/pom.xml
+++ b/warehouse/ops-tools/index-validation/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-ops-tools-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ops-tools-index-validation
jar
diff --git a/warehouse/ops-tools/pom.xml b/warehouse/ops-tools/pom.xml
index 935ea0dbeb..e1e77adc1f 100644
--- a/warehouse/ops-tools/pom.xml
+++ b/warehouse/ops-tools/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ops-tools-parent
pom
diff --git a/warehouse/pom.xml b/warehouse/pom.xml
index 78b95496a1..6e5ff77b18 100644
--- a/warehouse/pom.xml
+++ b/warehouse/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-warehouse-parent
pom
diff --git a/warehouse/query-core/pom.xml b/warehouse/query-core/pom.xml
index 63e8b07f34..254f37d6ac 100644
--- a/warehouse/query-core/pom.xml
+++ b/warehouse/query-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-query-core
jar
diff --git a/warehouse/regression-testing/pom.xml b/warehouse/regression-testing/pom.xml
index a8ebe512bb..d77bcd2f93 100644
--- a/warehouse/regression-testing/pom.xml
+++ b/warehouse/regression-testing/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-regression-testing
${project.artifactId}
diff --git a/warehouse/ssdeep-common/pom.xml b/warehouse/ssdeep-common/pom.xml
index 3a17715dd9..1e8b42370c 100644
--- a/warehouse/ssdeep-common/pom.xml
+++ b/warehouse/ssdeep-common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ssdeep-common
diff --git a/web-services/accumulo/pom.xml b/web-services/accumulo/pom.xml
index e389ff3667..3a95b0bb6f 100644
--- a/web-services/accumulo/pom.xml
+++ b/web-services/accumulo/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-accumulo
ejb
diff --git a/web-services/atom/pom.xml b/web-services/atom/pom.xml
index 7e4c5a00aa..c89f00f4a0 100644
--- a/web-services/atom/pom.xml
+++ b/web-services/atom/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-atom
ejb
diff --git a/web-services/cached-results/pom.xml b/web-services/cached-results/pom.xml
index ed2e0a2024..af14a6c051 100644
--- a/web-services/cached-results/pom.xml
+++ b/web-services/cached-results/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-cached-results
ejb
diff --git a/web-services/client/pom.xml b/web-services/client/pom.xml
index bbcde53b50..8bf58ec5fe 100644
--- a/web-services/client/pom.xml
+++ b/web-services/client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-client
jar
diff --git a/web-services/common-util/pom.xml b/web-services/common-util/pom.xml
index 0a7aa49ce5..86cf6ef59a 100644
--- a/web-services/common-util/pom.xml
+++ b/web-services/common-util/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-common-util
jar
diff --git a/web-services/common/pom.xml b/web-services/common/pom.xml
index 3c707b6e75..0068b88955 100644
--- a/web-services/common/pom.xml
+++ b/web-services/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-common
ejb
diff --git a/web-services/deploy/application/pom.xml b/web-services/deploy/application/pom.xml
index 892b1511c3..f9a937d0a8 100644
--- a/web-services/deploy/application/pom.xml
+++ b/web-services/deploy/application/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-deploy-application
ear
diff --git a/web-services/deploy/configuration/pom.xml b/web-services/deploy/configuration/pom.xml
index 814f91b48a..62f25d196e 100644
--- a/web-services/deploy/configuration/pom.xml
+++ b/web-services/deploy/configuration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-deploy-configuration
jar
diff --git a/web-services/deploy/docs/pom.xml b/web-services/deploy/docs/pom.xml
index 894bb5f923..26c7c06efe 100644
--- a/web-services/deploy/docs/pom.xml
+++ b/web-services/deploy/docs/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-deploy-docs
war
diff --git a/web-services/deploy/pom.xml b/web-services/deploy/pom.xml
index 36cf35dda0..ad635b4ab0 100644
--- a/web-services/deploy/pom.xml
+++ b/web-services/deploy/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
diff --git a/web-services/deploy/spring-framework-integration/pom.xml b/web-services/deploy/spring-framework-integration/pom.xml
index 42f342db50..194cc1492f 100644
--- a/web-services/deploy/spring-framework-integration/pom.xml
+++ b/web-services/deploy/spring-framework-integration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
spring-framework-integration
${project.artifactId}
diff --git a/web-services/dictionary/pom.xml b/web-services/dictionary/pom.xml
index 6d622bdfb2..2193ec7a45 100644
--- a/web-services/dictionary/pom.xml
+++ b/web-services/dictionary/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-dictionary
ejb
diff --git a/web-services/examples/client-login/pom.xml b/web-services/examples/client-login/pom.xml
index b66549068d..e625d191ec 100644
--- a/web-services/examples/client-login/pom.xml
+++ b/web-services/examples/client-login/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-examples-client-login
ejb
diff --git a/web-services/examples/http-client/pom.xml b/web-services/examples/http-client/pom.xml
index 60541a929d..d6840c01e6 100644
--- a/web-services/examples/http-client/pom.xml
+++ b/web-services/examples/http-client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-examples-http-client
jar
diff --git a/web-services/examples/jms-client/pom.xml b/web-services/examples/jms-client/pom.xml
index 94eaf8b3c3..8d5575daa4 100644
--- a/web-services/examples/jms-client/pom.xml
+++ b/web-services/examples/jms-client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-examples-jms-client
jar
diff --git a/web-services/examples/pom.xml b/web-services/examples/pom.xml
index 2c9ff72f5e..4ea7cae98e 100644
--- a/web-services/examples/pom.xml
+++ b/web-services/examples/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-examples-parent
pom
diff --git a/web-services/examples/query-war/pom.xml b/web-services/examples/query-war/pom.xml
index eb8a921978..475f391a9d 100644
--- a/web-services/examples/query-war/pom.xml
+++ b/web-services/examples/query-war/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-examples-query-war
war
diff --git a/web-services/map-reduce-embedded/pom.xml b/web-services/map-reduce-embedded/pom.xml
index 8aaa50a355..d31df28673 100644
--- a/web-services/map-reduce-embedded/pom.xml
+++ b/web-services/map-reduce-embedded/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-map-reduce-embedded
jar
diff --git a/web-services/map-reduce-status/pom.xml b/web-services/map-reduce-status/pom.xml
index 21399f53a8..fc875e2cc4 100644
--- a/web-services/map-reduce-status/pom.xml
+++ b/web-services/map-reduce-status/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-map-reduce-status
ejb
diff --git a/web-services/map-reduce/pom.xml b/web-services/map-reduce/pom.xml
index db13f2b30e..9740eb44d6 100644
--- a/web-services/map-reduce/pom.xml
+++ b/web-services/map-reduce/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-map-reduce
ejb
diff --git a/web-services/metrics/pom.xml b/web-services/metrics/pom.xml
index d69b02c8e4..a36c3a20a1 100644
--- a/web-services/metrics/pom.xml
+++ b/web-services/metrics/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-metrics
ejb
diff --git a/web-services/model/pom.xml b/web-services/model/pom.xml
index 043fc848ce..28f28d2f56 100644
--- a/web-services/model/pom.xml
+++ b/web-services/model/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-model
ejb
diff --git a/web-services/modification/pom.xml b/web-services/modification/pom.xml
index d760aa9a3d..ac9d22f925 100644
--- a/web-services/modification/pom.xml
+++ b/web-services/modification/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-modification
ejb
diff --git a/web-services/pom.xml b/web-services/pom.xml
index 4ada3e522f..6e161d0e97 100644
--- a/web-services/pom.xml
+++ b/web-services/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
gov.nsa.datawave.webservices
datawave-ws-parent
diff --git a/web-services/query-websocket/pom.xml b/web-services/query-websocket/pom.xml
index 06ed6844c4..66ea79b94d 100644
--- a/web-services/query-websocket/pom.xml
+++ b/web-services/query-websocket/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-query-websocket
war
diff --git a/web-services/query/pom.xml b/web-services/query/pom.xml
index ef5a454614..87cb3371c9 100644
--- a/web-services/query/pom.xml
+++ b/web-services/query/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-query
ejb
diff --git a/web-services/rest-api/pom.xml b/web-services/rest-api/pom.xml
index 1aae0b4cec..8f5a7339d6 100644
--- a/web-services/rest-api/pom.xml
+++ b/web-services/rest-api/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-rest-api
war
diff --git a/web-services/security/pom.xml b/web-services/security/pom.xml
index a33cf2b4ad..07117b77d6 100644
--- a/web-services/security/pom.xml
+++ b/web-services/security/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-security
ejb
diff --git a/web-services/web-root/pom.xml b/web-services/web-root/pom.xml
index 5251ccc2a5..381253460b 100644
--- a/web-services/web-root/pom.xml
+++ b/web-services/web-root/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0-SNAPSHOT
+ 7.2.0
datawave-ws-web-root
war
From c024131817d092b62af6e28ef3364f3f3893482d Mon Sep 17 00:00:00 2001
From: hgklohr
Date: Thu, 25 Jul 2024 18:35:31 +0000
Subject: [PATCH 011/128] 7.2.1-SNAPSHOT
---
common-test/pom.xml | 2 +-
contrib/datawave-quickstart/docker/pom.xml | 2 +-
core/cached-results/pom.xml | 2 +-
core/common-util/pom.xml | 2 +-
core/common/pom.xml | 2 +-
core/connection-pool/pom.xml | 2 +-
core/map-reduce/pom.xml | 2 +-
core/modification/pom.xml | 2 +-
core/pom.xml | 2 +-
core/query/pom.xml | 2 +-
core/utils/pom.xml | 2 +-
docs/pom.xml | 2 +-
microservices/pom.xml | 2 +-
microservices/services/pom.xml | 2 +-
microservices/starters/pom.xml | 2 +-
pom.xml | 2 +-
warehouse/accumulo-extensions/pom.xml | 2 +-
warehouse/age-off-utils/pom.xml | 2 +-
warehouse/age-off/pom.xml | 2 +-
warehouse/assemble/datawave/pom.xml | 2 +-
warehouse/assemble/pom.xml | 2 +-
warehouse/assemble/webservice/pom.xml | 2 +-
warehouse/common/pom.xml | 2 +-
warehouse/core/pom.xml | 2 +-
warehouse/data-dictionary-core/pom.xml | 2 +-
warehouse/edge-dictionary-core/pom.xml | 2 +-
warehouse/edge-model-configuration-core/pom.xml | 2 +-
warehouse/index-stats/pom.xml | 2 +-
warehouse/ingest-configuration/pom.xml | 2 +-
warehouse/ingest-core/pom.xml | 2 +-
warehouse/ingest-csv/pom.xml | 2 +-
warehouse/ingest-json/pom.xml | 2 +-
warehouse/ingest-nyctlc/pom.xml | 2 +-
warehouse/ingest-scripts/pom.xml | 2 +-
warehouse/ingest-ssdeep/pom.xml | 2 +-
warehouse/ingest-wikipedia/pom.xml | 2 +-
warehouse/metrics-core/pom.xml | 2 +-
warehouse/ops-tools/config-compare/pom.xml | 2 +-
warehouse/ops-tools/index-validation/pom.xml | 2 +-
warehouse/ops-tools/pom.xml | 2 +-
warehouse/pom.xml | 2 +-
warehouse/query-core/pom.xml | 2 +-
warehouse/regression-testing/pom.xml | 2 +-
warehouse/ssdeep-common/pom.xml | 2 +-
web-services/accumulo/pom.xml | 2 +-
web-services/atom/pom.xml | 2 +-
web-services/cached-results/pom.xml | 2 +-
web-services/client/pom.xml | 2 +-
web-services/common-util/pom.xml | 2 +-
web-services/common/pom.xml | 2 +-
web-services/deploy/application/pom.xml | 2 +-
web-services/deploy/configuration/pom.xml | 2 +-
web-services/deploy/docs/pom.xml | 2 +-
web-services/deploy/pom.xml | 2 +-
web-services/deploy/spring-framework-integration/pom.xml | 2 +-
web-services/dictionary/pom.xml | 2 +-
web-services/examples/client-login/pom.xml | 2 +-
web-services/examples/http-client/pom.xml | 2 +-
web-services/examples/jms-client/pom.xml | 2 +-
web-services/examples/pom.xml | 2 +-
web-services/examples/query-war/pom.xml | 2 +-
web-services/map-reduce-embedded/pom.xml | 2 +-
web-services/map-reduce-status/pom.xml | 2 +-
web-services/map-reduce/pom.xml | 2 +-
web-services/metrics/pom.xml | 2 +-
web-services/model/pom.xml | 2 +-
web-services/modification/pom.xml | 2 +-
web-services/pom.xml | 2 +-
web-services/query-websocket/pom.xml | 2 +-
web-services/query/pom.xml | 2 +-
web-services/rest-api/pom.xml | 2 +-
web-services/security/pom.xml | 2 +-
web-services/web-root/pom.xml | 2 +-
73 files changed, 73 insertions(+), 73 deletions(-)
diff --git a/common-test/pom.xml b/common-test/pom.xml
index c843260a49..daf1dcfa6a 100644
--- a/common-test/pom.xml
+++ b/common-test/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-common-test
${project.artifactId}
diff --git a/contrib/datawave-quickstart/docker/pom.xml b/contrib/datawave-quickstart/docker/pom.xml
index 4bc20dc46b..d5b845ae37 100644
--- a/contrib/datawave-quickstart/docker/pom.xml
+++ b/contrib/datawave-quickstart/docker/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
../../../pom.xml
quickstart
diff --git a/core/cached-results/pom.xml b/core/cached-results/pom.xml
index 0e77b53ee0..cf330a880e 100644
--- a/core/cached-results/pom.xml
+++ b/core/cached-results/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-core-cached-results
${project.artifactId}
diff --git a/core/common-util/pom.xml b/core/common-util/pom.xml
index 907eada1fc..69b3a4af4f 100644
--- a/core/common-util/pom.xml
+++ b/core/common-util/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-core-common-util
${project.artifactId}
diff --git a/core/common/pom.xml b/core/common/pom.xml
index a32f70b44f..9c28026696 100644
--- a/core/common/pom.xml
+++ b/core/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-core-common
${project.artifactId}
diff --git a/core/connection-pool/pom.xml b/core/connection-pool/pom.xml
index 60393a1aa7..69e58f532c 100644
--- a/core/connection-pool/pom.xml
+++ b/core/connection-pool/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-core-connection-pool
${project.artifactId}
diff --git a/core/map-reduce/pom.xml b/core/map-reduce/pom.xml
index f696753415..8c88d1856f 100644
--- a/core/map-reduce/pom.xml
+++ b/core/map-reduce/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-core-map-reduce
${project.artifactId}
diff --git a/core/modification/pom.xml b/core/modification/pom.xml
index e3c7dd122e..396a4d7181 100644
--- a/core/modification/pom.xml
+++ b/core/modification/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-core-modification
${project.artifactId}
diff --git a/core/pom.xml b/core/pom.xml
index b2cd4be510..b547f65996 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
gov.nsa.datawave.core
datawave-core-parent
diff --git a/core/query/pom.xml b/core/query/pom.xml
index 111b9244b1..4d2b63986b 100644
--- a/core/query/pom.xml
+++ b/core/query/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-core-query
${project.artifactId}
diff --git a/core/utils/pom.xml b/core/utils/pom.xml
index 0651ba557a..21adb9cc2e 100644
--- a/core/utils/pom.xml
+++ b/core/utils/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
gov.nsa.datawave.core
datawave-utils-parent
diff --git a/docs/pom.xml b/docs/pom.xml
index e239f48f30..e8eab01a0d 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-docs
diff --git a/microservices/pom.xml b/microservices/pom.xml
index 5a5b2d830b..86018808ed 100644
--- a/microservices/pom.xml
+++ b/microservices/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
gov.nsa.datawave.microservice
datawave-microservice-build-parent
diff --git a/microservices/services/pom.xml b/microservices/services/pom.xml
index d868b486ac..8da49b8d1e 100644
--- a/microservices/services/pom.xml
+++ b/microservices/services/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.microservice
datawave-microservice-build-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-microservice-service-build-parent
pom
diff --git a/microservices/starters/pom.xml b/microservices/starters/pom.xml
index 5fadb33873..e4bce0c87b 100644
--- a/microservices/starters/pom.xml
+++ b/microservices/starters/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.microservice
datawave-microservice-build-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-microservice-starter-build-parent
pom
diff --git a/pom.xml b/pom.xml
index 62ab380418..e6f3fdd2f4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
4.0.0
gov.nsa.datawave
datawave-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
pom
DataWave
DataWave is a Java-based ingest and query framework that leverages Apache Accumulo to provide fast, secure access to your data.
diff --git a/warehouse/accumulo-extensions/pom.xml b/warehouse/accumulo-extensions/pom.xml
index 6c0863c199..f4e3f6ea57 100644
--- a/warehouse/accumulo-extensions/pom.xml
+++ b/warehouse/accumulo-extensions/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-accumulo-extensions
${project.artifactId}
diff --git a/warehouse/age-off-utils/pom.xml b/warehouse/age-off-utils/pom.xml
index f4005515b6..422e1f7c92 100644
--- a/warehouse/age-off-utils/pom.xml
+++ b/warehouse/age-off-utils/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-age-off-utils
${project.artifactId}
diff --git a/warehouse/age-off/pom.xml b/warehouse/age-off/pom.xml
index 036a8d18f5..24015f6c31 100644
--- a/warehouse/age-off/pom.xml
+++ b/warehouse/age-off/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-age-off
${project.artifactId}
diff --git a/warehouse/assemble/datawave/pom.xml b/warehouse/assemble/datawave/pom.xml
index c075aea9e0..7a017096f1 100644
--- a/warehouse/assemble/datawave/pom.xml
+++ b/warehouse/assemble/datawave/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
assemble-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
assemble-datawave
jar
diff --git a/warehouse/assemble/pom.xml b/warehouse/assemble/pom.xml
index 8204982332..07f89c4b3d 100644
--- a/warehouse/assemble/pom.xml
+++ b/warehouse/assemble/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
assemble-parent
pom
diff --git a/warehouse/assemble/webservice/pom.xml b/warehouse/assemble/webservice/pom.xml
index 456bb593ff..13a0f272c4 100644
--- a/warehouse/assemble/webservice/pom.xml
+++ b/warehouse/assemble/webservice/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
assemble-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
assemble-webservice
${project.artifactId}
diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml
index f482856bc9..455f6e9b00 100644
--- a/warehouse/common/pom.xml
+++ b/warehouse/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-common
${project.artifactId}
diff --git a/warehouse/core/pom.xml b/warehouse/core/pom.xml
index 22444921a5..5c1c27393b 100644
--- a/warehouse/core/pom.xml
+++ b/warehouse/core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-core
jar
diff --git a/warehouse/data-dictionary-core/pom.xml b/warehouse/data-dictionary-core/pom.xml
index bf686dbbcb..18cebd0a23 100644
--- a/warehouse/data-dictionary-core/pom.xml
+++ b/warehouse/data-dictionary-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-data-dictionary-core
jar
diff --git a/warehouse/edge-dictionary-core/pom.xml b/warehouse/edge-dictionary-core/pom.xml
index 7e0002efbd..d0ff7e12cf 100644
--- a/warehouse/edge-dictionary-core/pom.xml
+++ b/warehouse/edge-dictionary-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-edge-dictionary-core
jar
diff --git a/warehouse/edge-model-configuration-core/pom.xml b/warehouse/edge-model-configuration-core/pom.xml
index f83d0c1745..53ef6c61e0 100644
--- a/warehouse/edge-model-configuration-core/pom.xml
+++ b/warehouse/edge-model-configuration-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-edge-model-configuration-core
jar
diff --git a/warehouse/index-stats/pom.xml b/warehouse/index-stats/pom.xml
index 2df5614ef3..17483ab144 100644
--- a/warehouse/index-stats/pom.xml
+++ b/warehouse/index-stats/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-index-stats
jar
diff --git a/warehouse/ingest-configuration/pom.xml b/warehouse/ingest-configuration/pom.xml
index 068f9494df..1e6c9a11b6 100644
--- a/warehouse/ingest-configuration/pom.xml
+++ b/warehouse/ingest-configuration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ingest-configuration
diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml
index 4fcab17c54..0e7c60c2ff 100644
--- a/warehouse/ingest-core/pom.xml
+++ b/warehouse/ingest-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ingest-core
jar
diff --git a/warehouse/ingest-csv/pom.xml b/warehouse/ingest-csv/pom.xml
index 060adfe2f5..4b97eb7ee3 100644
--- a/warehouse/ingest-csv/pom.xml
+++ b/warehouse/ingest-csv/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ingest-csv
jar
diff --git a/warehouse/ingest-json/pom.xml b/warehouse/ingest-json/pom.xml
index 10c9f76ea3..ac56d833cf 100644
--- a/warehouse/ingest-json/pom.xml
+++ b/warehouse/ingest-json/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ingest-json
jar
diff --git a/warehouse/ingest-nyctlc/pom.xml b/warehouse/ingest-nyctlc/pom.xml
index 53dd6bcbf3..cc6b814e92 100644
--- a/warehouse/ingest-nyctlc/pom.xml
+++ b/warehouse/ingest-nyctlc/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ingest-nyctlc
jar
diff --git a/warehouse/ingest-scripts/pom.xml b/warehouse/ingest-scripts/pom.xml
index 9843ee664e..0e4050f0ed 100644
--- a/warehouse/ingest-scripts/pom.xml
+++ b/warehouse/ingest-scripts/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ingest-scripts
${project.artifactId}
diff --git a/warehouse/ingest-ssdeep/pom.xml b/warehouse/ingest-ssdeep/pom.xml
index 4dd50f5609..a3ed96b52e 100644
--- a/warehouse/ingest-ssdeep/pom.xml
+++ b/warehouse/ingest-ssdeep/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ingest-ssdeep
diff --git a/warehouse/ingest-wikipedia/pom.xml b/warehouse/ingest-wikipedia/pom.xml
index c176c51ed5..ffaf74cf1b 100644
--- a/warehouse/ingest-wikipedia/pom.xml
+++ b/warehouse/ingest-wikipedia/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ingest-wikipedia
jar
diff --git a/warehouse/metrics-core/pom.xml b/warehouse/metrics-core/pom.xml
index d3b07dbb44..8ee6ce5486 100644
--- a/warehouse/metrics-core/pom.xml
+++ b/warehouse/metrics-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-metrics-core
jar
diff --git a/warehouse/ops-tools/config-compare/pom.xml b/warehouse/ops-tools/config-compare/pom.xml
index e9cc6f32d0..16349e90e0 100644
--- a/warehouse/ops-tools/config-compare/pom.xml
+++ b/warehouse/ops-tools/config-compare/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-ops-tools-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ops-tools-config-compare
diff --git a/warehouse/ops-tools/index-validation/pom.xml b/warehouse/ops-tools/index-validation/pom.xml
index fa41b77806..7bdd5f17f0 100644
--- a/warehouse/ops-tools/index-validation/pom.xml
+++ b/warehouse/ops-tools/index-validation/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-ops-tools-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ops-tools-index-validation
jar
diff --git a/warehouse/ops-tools/pom.xml b/warehouse/ops-tools/pom.xml
index e1e77adc1f..4c774f9c9b 100644
--- a/warehouse/ops-tools/pom.xml
+++ b/warehouse/ops-tools/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ops-tools-parent
pom
diff --git a/warehouse/pom.xml b/warehouse/pom.xml
index 6e5ff77b18..47019a161b 100644
--- a/warehouse/pom.xml
+++ b/warehouse/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-warehouse-parent
pom
diff --git a/warehouse/query-core/pom.xml b/warehouse/query-core/pom.xml
index 254f37d6ac..f34ce1eac7 100644
--- a/warehouse/query-core/pom.xml
+++ b/warehouse/query-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-query-core
jar
diff --git a/warehouse/regression-testing/pom.xml b/warehouse/regression-testing/pom.xml
index d77bcd2f93..b69ba2c74b 100644
--- a/warehouse/regression-testing/pom.xml
+++ b/warehouse/regression-testing/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-regression-testing
${project.artifactId}
diff --git a/warehouse/ssdeep-common/pom.xml b/warehouse/ssdeep-common/pom.xml
index 1e8b42370c..5a54996db8 100644
--- a/warehouse/ssdeep-common/pom.xml
+++ b/warehouse/ssdeep-common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ssdeep-common
diff --git a/web-services/accumulo/pom.xml b/web-services/accumulo/pom.xml
index 3a95b0bb6f..fd544184fb 100644
--- a/web-services/accumulo/pom.xml
+++ b/web-services/accumulo/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-accumulo
ejb
diff --git a/web-services/atom/pom.xml b/web-services/atom/pom.xml
index c89f00f4a0..aca4f7774f 100644
--- a/web-services/atom/pom.xml
+++ b/web-services/atom/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-atom
ejb
diff --git a/web-services/cached-results/pom.xml b/web-services/cached-results/pom.xml
index af14a6c051..c59a9076b0 100644
--- a/web-services/cached-results/pom.xml
+++ b/web-services/cached-results/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-cached-results
ejb
diff --git a/web-services/client/pom.xml b/web-services/client/pom.xml
index 8bf58ec5fe..4273e6bacc 100644
--- a/web-services/client/pom.xml
+++ b/web-services/client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-client
jar
diff --git a/web-services/common-util/pom.xml b/web-services/common-util/pom.xml
index 86cf6ef59a..7d03d240bc 100644
--- a/web-services/common-util/pom.xml
+++ b/web-services/common-util/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-common-util
jar
diff --git a/web-services/common/pom.xml b/web-services/common/pom.xml
index 0068b88955..c0726f4832 100644
--- a/web-services/common/pom.xml
+++ b/web-services/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-common
ejb
diff --git a/web-services/deploy/application/pom.xml b/web-services/deploy/application/pom.xml
index f9a937d0a8..b4c4de6620 100644
--- a/web-services/deploy/application/pom.xml
+++ b/web-services/deploy/application/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-deploy-application
ear
diff --git a/web-services/deploy/configuration/pom.xml b/web-services/deploy/configuration/pom.xml
index 62f25d196e..1c914de4cf 100644
--- a/web-services/deploy/configuration/pom.xml
+++ b/web-services/deploy/configuration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-deploy-configuration
jar
diff --git a/web-services/deploy/docs/pom.xml b/web-services/deploy/docs/pom.xml
index 26c7c06efe..e179b70533 100644
--- a/web-services/deploy/docs/pom.xml
+++ b/web-services/deploy/docs/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-deploy-docs
war
diff --git a/web-services/deploy/pom.xml b/web-services/deploy/pom.xml
index ad635b4ab0..32b5dca752 100644
--- a/web-services/deploy/pom.xml
+++ b/web-services/deploy/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
diff --git a/web-services/deploy/spring-framework-integration/pom.xml b/web-services/deploy/spring-framework-integration/pom.xml
index 194cc1492f..d69ba7137b 100644
--- a/web-services/deploy/spring-framework-integration/pom.xml
+++ b/web-services/deploy/spring-framework-integration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
spring-framework-integration
${project.artifactId}
diff --git a/web-services/dictionary/pom.xml b/web-services/dictionary/pom.xml
index 2193ec7a45..55b0dc11c5 100644
--- a/web-services/dictionary/pom.xml
+++ b/web-services/dictionary/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-dictionary
ejb
diff --git a/web-services/examples/client-login/pom.xml b/web-services/examples/client-login/pom.xml
index e625d191ec..0c319397a6 100644
--- a/web-services/examples/client-login/pom.xml
+++ b/web-services/examples/client-login/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-examples-client-login
ejb
diff --git a/web-services/examples/http-client/pom.xml b/web-services/examples/http-client/pom.xml
index d6840c01e6..c0382445ea 100644
--- a/web-services/examples/http-client/pom.xml
+++ b/web-services/examples/http-client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-examples-http-client
jar
diff --git a/web-services/examples/jms-client/pom.xml b/web-services/examples/jms-client/pom.xml
index 8d5575daa4..e504137e92 100644
--- a/web-services/examples/jms-client/pom.xml
+++ b/web-services/examples/jms-client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-examples-jms-client
jar
diff --git a/web-services/examples/pom.xml b/web-services/examples/pom.xml
index 4ea7cae98e..ee28cfed2e 100644
--- a/web-services/examples/pom.xml
+++ b/web-services/examples/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-examples-parent
pom
diff --git a/web-services/examples/query-war/pom.xml b/web-services/examples/query-war/pom.xml
index 475f391a9d..7e858c744e 100644
--- a/web-services/examples/query-war/pom.xml
+++ b/web-services/examples/query-war/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-examples-query-war
war
diff --git a/web-services/map-reduce-embedded/pom.xml b/web-services/map-reduce-embedded/pom.xml
index d31df28673..a0337f4b3d 100644
--- a/web-services/map-reduce-embedded/pom.xml
+++ b/web-services/map-reduce-embedded/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-map-reduce-embedded
jar
diff --git a/web-services/map-reduce-status/pom.xml b/web-services/map-reduce-status/pom.xml
index fc875e2cc4..35d7cd2753 100644
--- a/web-services/map-reduce-status/pom.xml
+++ b/web-services/map-reduce-status/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-map-reduce-status
ejb
diff --git a/web-services/map-reduce/pom.xml b/web-services/map-reduce/pom.xml
index 9740eb44d6..ba338fdaaf 100644
--- a/web-services/map-reduce/pom.xml
+++ b/web-services/map-reduce/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-map-reduce
ejb
diff --git a/web-services/metrics/pom.xml b/web-services/metrics/pom.xml
index a36c3a20a1..af3fd381ae 100644
--- a/web-services/metrics/pom.xml
+++ b/web-services/metrics/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-metrics
ejb
diff --git a/web-services/model/pom.xml b/web-services/model/pom.xml
index 28f28d2f56..d8b5c17fa9 100644
--- a/web-services/model/pom.xml
+++ b/web-services/model/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-model
ejb
diff --git a/web-services/modification/pom.xml b/web-services/modification/pom.xml
index ac9d22f925..f2874d71ca 100644
--- a/web-services/modification/pom.xml
+++ b/web-services/modification/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-modification
ejb
diff --git a/web-services/pom.xml b/web-services/pom.xml
index 6e161d0e97..6de4ad5a69 100644
--- a/web-services/pom.xml
+++ b/web-services/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
gov.nsa.datawave.webservices
datawave-ws-parent
diff --git a/web-services/query-websocket/pom.xml b/web-services/query-websocket/pom.xml
index 66ea79b94d..85b02d7c2b 100644
--- a/web-services/query-websocket/pom.xml
+++ b/web-services/query-websocket/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-query-websocket
war
diff --git a/web-services/query/pom.xml b/web-services/query/pom.xml
index 87cb3371c9..5ea1aec85f 100644
--- a/web-services/query/pom.xml
+++ b/web-services/query/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-query
ejb
diff --git a/web-services/rest-api/pom.xml b/web-services/rest-api/pom.xml
index 8f5a7339d6..b3f9db66fa 100644
--- a/web-services/rest-api/pom.xml
+++ b/web-services/rest-api/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-rest-api
war
diff --git a/web-services/security/pom.xml b/web-services/security/pom.xml
index 07117b77d6..e8c4f418b5 100644
--- a/web-services/security/pom.xml
+++ b/web-services/security/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-security
ejb
diff --git a/web-services/web-root/pom.xml b/web-services/web-root/pom.xml
index 381253460b..62ca2642e3 100644
--- a/web-services/web-root/pom.xml
+++ b/web-services/web-root/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.0
+ 7.2.1-SNAPSHOT
datawave-ws-web-root
war
From 57075c51170b902f06e0e6b1f2e7f56b1b541e11 Mon Sep 17 00:00:00 2001
From: rdhayes68 <104331030+rdhayes68@users.noreply.github.com>
Date: Thu, 25 Jul 2024 16:43:03 -0400
Subject: [PATCH 012/128] Correct DROP state for normalization bug. (#2285)
* Correct DROP state for normalization bug.
* Update code format
* Updated from code review comments
* Correct DROP state for normalization bug.
* added NormalizedContentInterface test
* Correct DROP state for normalization bug.
* Add test for ShardedDataTypeHandler
* Update test for ShardedDataTypeHandler Failure Policy.
* Update test for ShardedDataTypeHandler Failure Policy.
* Update test for ShardedDataTypeHandler Failure Policy.
* Update based on code review comments.
* Update based on code review comments.
---------
Co-authored-by: hgklohr
---
.../data/config/ingest/BaseIngestHelper.java | 8 +-
.../shard/AbstractColumnBasedHandler.java | 2 +-
.../handler/shard/ShardedDataTypeHandler.java | 138 +++----
.../csv/NormalizedContentInterfaceTest.java | 252 +++++++++++++
...ardedDataTypeHandlerFailurePolicyTest.java | 355 ++++++++++++++++++
.../config/ingest/norm-content-interface.xml | 135 +++++++
.../src/test/resources/input/my-nci.csv | 3 +
7 files changed, 821 insertions(+), 72 deletions(-)
create mode 100644 warehouse/ingest-csv/src/test/java/datawave/ingest/csv/NormalizedContentInterfaceTest.java
create mode 100644 warehouse/ingest-csv/src/test/java/datawave/ingest/csv/ShardedDataTypeHandlerFailurePolicyTest.java
create mode 100644 warehouse/ingest-csv/src/test/resources/config/ingest/norm-content-interface.xml
create mode 100644 warehouse/ingest-csv/src/test/resources/input/my-nci.csv
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java
index 4dcd86e58e..4be57bdf85 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java
@@ -985,8 +985,12 @@ protected void applyNormalizationAndAddToResults(Multimap getShardNamesAndValues(Raw
// Put the normalized field name and normalized value into the reverse
if (createGlobalReverseIndexTerms) {
- if (helper.isReverseIndexedField(e.getValue().getIndexedFieldName())) {
+ if (e.getValue().getIndexedFieldValue() != null && helper.isReverseIndexedField(e.getValue().getIndexedFieldName())) {
NormalizedContentInterface rField = (NormalizedContentInterface) (e.getValue().clone());
rField.setEventFieldValue(new StringBuilder(rField.getEventFieldValue()).reverse().toString());
rField.setIndexedFieldValue(new StringBuilder(rField.getIndexedFieldValue()).reverse().toString());
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java
index 73798508b4..dc3f27c7a9 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java
@@ -1019,50 +1019,50 @@ protected Multimap createShardFieldIndexColumn(RawRecordCon
Multimap values = HashMultimap.create();
- Text colf = new Text("fi");
- TextUtil.textAppend(colf, fieldName, replaceMalformedUTF8);
- Text unmaskedColq = new Text(fieldValue);
- TextUtil.textAppend(unmaskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
- TextUtil.textAppend(unmaskedColq, event.getId().toString(), replaceMalformedUTF8);
-
- if (value == null) {
- value = NULL_VALUE;
- }
+ if (!StringUtils.isEmpty(fieldValue)) {
+ Text colf = new Text("fi");
+ TextUtil.textAppend(colf, fieldName, replaceMalformedUTF8);
+ Text unmaskedColq = new Text(fieldValue);
+ TextUtil.textAppend(unmaskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
+ TextUtil.textAppend(unmaskedColq, event.getId().toString(), replaceMalformedUTF8);
+
+ if (value == null) {
+ value = NULL_VALUE;
+ }
- if (null != maskedFieldHelper && maskedFieldHelper.contains(fieldName)) {
- if (!StringUtils.isEmpty(fieldValue)) {
+ if (null != maskedFieldHelper && maskedFieldHelper.contains(fieldName)) {
// Put unmasked colq with original visibility
Key k = createKey(shardId, colf, unmaskedColq, visibility, event.getDate(), deleteMode);
BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
values.put(bKey, value);
- }
- // We need to use the normalized masked values
- final String normalizedMaskedValue = helper.getNormalizedMaskedValue(fieldName);
- if (!StringUtils.isEmpty(normalizedMaskedValue)) {
- Text maskedColq = new Text(normalizedMaskedValue);
- TextUtil.textAppend(maskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
- TextUtil.textAppend(maskedColq, event.getId().toString(), replaceMalformedUTF8);
+ // We need to use the normalized masked values
+ final String normalizedMaskedValue = helper.getNormalizedMaskedValue(fieldName);
+ if (!StringUtils.isEmpty(normalizedMaskedValue)) {
+ Text maskedColq = new Text(normalizedMaskedValue);
+ TextUtil.textAppend(maskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
+ TextUtil.textAppend(maskedColq, event.getId().toString(), replaceMalformedUTF8);
+
+ // Put masked colq with masked visibility
+ Key key = createKey(shardId, colf, maskedColq, maskedVisibility, event.getDate(), deleteMode);
+ BulkIngestKey bulkIngestKey = new BulkIngestKey(this.getShardTableName(), key);
+ values.put(bulkIngestKey, value);
+ }
+ } else if (!StringUtils.isEmpty(fieldValue)) {
+ /**
+ * For values that are not being masked, we use the "unmaskedValue" and the masked visibility e.g. release the value as it was in the event at
+ * the lower visibility
+ */
+ byte[] refVisibility = visibility;
+
+ if (null != maskedFieldHelper) {
+ refVisibility = maskedVisibility;
+ }
- // Put masked colq with masked visibility
- Key k = createKey(shardId, colf, maskedColq, maskedVisibility, event.getDate(), deleteMode);
+ Key k = createKey(shardId, colf, unmaskedColq, refVisibility, event.getDate(), deleteMode);
BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
values.put(bKey, value);
}
- } else if (!StringUtils.isEmpty(fieldValue)) {
- /**
- * For values that are not being masked, we use the "unmaskedValue" and the masked visibility e.g. release the value as it was in the event at the
- * lower visibility
- */
- byte[] refVisibility = visibility;
-
- if (null != maskedFieldHelper) {
- refVisibility = maskedVisibility;
- }
-
- Key k = createKey(shardId, colf, unmaskedColq, refVisibility, event.getDate(), deleteMode);
- BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
- values.put(bKey, value);
}
return values;
@@ -1097,51 +1097,51 @@ protected void createShardFieldIndexColumn(RawRecordContainer event, Multimap expectedValues = createExpectedValues();
+ // for the leave policy, clear out the exception but add
+ // a failed normalization field
+ expectedValues.put(FAILED_NORMALIZATION_FIELD, createNormalizedContent(FAILED_NORMALIZATION_FIELD, DATE_FIELD));
+ expectedValues.put(DATE_FIELD, createNormalizedContentLeave());
+
+ test(expectedValues);
+ }
+
+ @Test
+ public void testDrop() throws IOException {
+ conf.set(DATA_NAME + BaseIngestHelper.DEFAULT_FAILED_NORMALIZATION_POLICY, BaseIngestHelper.FailurePolicy.DROP.name());
+
+ Multimap expectedValues = createExpectedValues();
+ // for the drop policy, clear out the exception,
+ // clear out the indexed field value and add
+ // a failed normalization field
+ expectedValues.put(FAILED_NORMALIZATION_FIELD, createNormalizedContent(FAILED_NORMALIZATION_FIELD, DATE_FIELD));
+ expectedValues.put(DATE_FIELD, createNormalizedContentDrop());
+
+ test(expectedValues);
+ }
+
+ @Test
+ public void testFail() throws IOException {
+ conf.set(DATA_NAME + BaseIngestHelper.DEFAULT_FAILED_NORMALIZATION_POLICY, BaseIngestHelper.FailurePolicy.FAIL.name());
+
+ Multimap expectedValues = createExpectedValues();
+ // for the fail policy, leave the exception and let the
+ // caller (EventMapper) fail the event
+ SimpleThrowable exception = SimpleThrowable.create("Failed to normalize value as a Date: " + BAD_DATE_FIELD);
+ expectedValues.put(DATE_FIELD, createNormalizedContentFail(exception));
+
+ test(expectedValues);
+ }
+
+ private NormalizedContentInterface createNormalizedContentLeave() {
+ NormalizedContentInterface fieldAndValue = createNormalizedContent(DATE_FIELD, BAD_DATE_FIELD);
+ fieldAndValue.setError(null);
+ return fieldAndValue;
+ }
+
+ private NormalizedContentInterface createNormalizedContentDrop() {
+ NormalizedContentInterface fieldAndValue = createNormalizedContent(DATE_FIELD, BAD_DATE_FIELD, null);
+ fieldAndValue.setError(null);
+ return fieldAndValue;
+ }
+
+ private NormalizedContentInterface createNormalizedContentFail(Throwable throwable) {
+ NormalizedContentInterface fieldAndValue = createNormalizedContent(DATE_FIELD, BAD_DATE_FIELD);
+ fieldAndValue.setError(throwable);
+ return fieldAndValue;
+ }
+
+ private Multimap createExpectedValues() {
+ final String HEADER_DATE = "HEADER_DATE";
+ final String HEADER_NUMBER = "HEADER_NUMBER";
+ final String HEADER_ID = "HEADER_ID";
+ final String HEADER_TEXT_1 = "HEADER_TEXT_1";
+ final String HEADER_TEXT_2 = "HEADER_TEXT_2";
+
+ Multimap expectedValues = HashMultimap.create();
+
+ expectedValues.put(HEADER_DATE, createNormalizedContent(HEADER_DATE, "2024-02-29 12:01:47", "2024-02-29T12:01:47.000Z"));
+ expectedValues.put(HEADER_NUMBER, createNormalizedContent(HEADER_NUMBER, "111", "+cE1.11"));
+ expectedValues.put(HEADER_ID, createNormalizedContent(HEADER_ID, "header_one"));
+ expectedValues.put(HEADER_TEXT_1, createNormalizedContent(HEADER_TEXT_1, "text one-one"));
+ expectedValues.put(HEADER_TEXT_2, createNormalizedContent(HEADER_TEXT_2, "text two-one"));
+ expectedValues.put(DATE_FIELD, createNormalizedContent(DATE_FIELD, "2024-02-29 12:01:47", "2024-02-29T12:01:47.000Z"));
+
+ expectedValues.put(HEADER_DATE, createNormalizedContent(HEADER_DATE, "2024-02-29 17:11:43", "2024-02-29T17:11:43.000Z"));
+ expectedValues.put(HEADER_NUMBER, createNormalizedContent(HEADER_NUMBER, "222", "+cE2.22"));
+ expectedValues.put(HEADER_ID, createNormalizedContent(HEADER_ID, "header_two"));
+ expectedValues.put(HEADER_TEXT_1, createNormalizedContent(HEADER_TEXT_1, "text one-two"));
+ expectedValues.put(HEADER_TEXT_2, createNormalizedContent(HEADER_TEXT_2, "text two-two"));
+
+ expectedValues.put(HEADER_DATE, createNormalizedContent(HEADER_DATE, "2024-03-01 12:01:24", "2024-03-01T12:01:24.000Z"));
+ expectedValues.put(HEADER_NUMBER, createNormalizedContent(HEADER_NUMBER, "333", "+cE3.33"));
+ expectedValues.put(HEADER_ID, createNormalizedContent(HEADER_ID, "header_three"));
+ expectedValues.put(HEADER_TEXT_1, createNormalizedContent(HEADER_TEXT_1, "text one-three"));
+ expectedValues.put(HEADER_TEXT_2, createNormalizedContent(HEADER_TEXT_2, "text two-three"));
+ expectedValues.put(DATE_FIELD, createNormalizedContent(DATE_FIELD, "2024-03-01 12:01:24", "2024-03-01T12:01:24.000Z"));
+
+ return expectedValues;
+ }
+
+ private void test(Multimap expectedValues) throws IOException {
+ TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID());
+
+ TypeRegistry registry = TypeRegistry.getInstance(context.getConfiguration());
+ Type type = registry.get(context.getConfiguration().get(DataTypeHelper.Properties.DATA_NAME));
+ type.clearIngestHelper();
+
+ Multimap actualValues = HashMultimap.create();
+
+ IngestHelperInterface helper = type.getIngestHelper(context.getConfiguration());
+
+ CSVRecordReader reader = new CSVRecordReader();
+ reader.initialize(split, context);
+ while (reader.nextKeyValue()) {
+ Multimap fields = helper.getEventFields(reader.getEvent());
+ fields.entries().forEach(entry -> actualValues.put(entry.getKey(), entry.getValue()));
+ }
+ reader.close();
+
+ compare(actualValues, expectedValues);
+ }
+
+ private NormalizedContentInterface createNormalizedContent(String field, String value) {
+ NormalizedFieldAndValue fieldAndValue = new NormalizedFieldAndValue(field, value);
+ if (!field.equals(FAILED_NORMALIZATION_FIELD)) {
+ fieldAndValue.setMarkings(createDefaultMarkings());
+ }
+ return fieldAndValue;
+ }
+
+ private NormalizedContentInterface createNormalizedContent(String field, String value, String indexValue) {
+ NormalizedContentInterface fieldAndValue = createNormalizedContent(field, value);
+ fieldAndValue.setIndexedFieldValue(indexValue);
+ return fieldAndValue;
+ }
+
+ private Map createDefaultMarkings() {
+ Map defaultMarkings = new HashMap<>(1);
+ defaultMarkings.put("columnVisibility", "PRIVATE");
+ return defaultMarkings;
+ }
+
+ private void compare(Multimap actualValues, Multimap expectedValues) {
+ // need to modify any error values within the actualValues set in order to perform a simple comparison of error messages
+ Multimap values = HashMultimap.create();
+ actualValues.entries().forEach(entry -> {
+ NormalizedContentInterface nci;
+ if (entry.getValue().getError() == null) {
+ nci = entry.getValue();
+ } else {
+ NormalizedContentInterface temp = new NormalizedFieldAndValue(entry.getValue());
+ temp.setError(SimpleThrowable.create(temp.getError().getMessage()));
+ nci = temp;
+ }
+ values.put(entry.getKey(), nci);
+ });
+
+ Multimap actualToExpectedDiff = Multimaps.filterEntries(values,
+ entry -> !expectedValues.containsEntry(entry.getKey(), entry.getValue()));
+ Multimap expectedToActualDiff = Multimaps.filterEntries(expectedValues,
+ entry -> !values.containsEntry(entry.getKey(), entry.getValue()));
+
+ if (!actualToExpectedDiff.isEmpty()) {
+ log.error("Actual value difference exists");
+ logDiffs(actualToExpectedDiff);
+ }
+ if (!expectedToActualDiff.isEmpty()) {
+ log.error("Expected value difference exists");
+ logDiffs(expectedToActualDiff);
+ }
+
+ Assert.assertFalse("The actual normalized results do not mach expected results.", !actualToExpectedDiff.isEmpty() || !expectedToActualDiff.isEmpty());
+ }
+
+ private void logDiffs(Multimap diff) {
+ diff.entries().forEach(entry -> {
+ log.error("Key: " + entry.getKey());
+ log.error("Event Field Name: " + entry.getValue().getEventFieldName() + " Value: " + entry.getValue().getEventFieldValue());
+ log.error("Indexed Field Name: " + entry.getValue().getIndexedFieldName() + " Value: " + entry.getValue().getIndexedFieldValue());
+ if (entry.getValue().getError() != null) {
+ log.error("Error: " + entry.getValue().getError().getMessage());
+ }
+ });
+ }
+
+ private static class SimpleThrowable extends Throwable {
+ public static SimpleThrowable create(String message) {
+ return new SimpleThrowable(message, null, true, false);
+ }
+
+ protected SimpleThrowable(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+
+ @Override
+ public int hashCode() {
+ return this.getMessage().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof SimpleThrowable) {
+ return this.getMessage().equals(((SimpleThrowable) obj).getMessage());
+ }
+ return super.equals(obj);
+ }
+ }
+}
diff --git a/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/ShardedDataTypeHandlerFailurePolicyTest.java b/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/ShardedDataTypeHandlerFailurePolicyTest.java
new file mode 100644
index 0000000000..55cdc203d1
--- /dev/null
+++ b/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/ShardedDataTypeHandlerFailurePolicyTest.java
@@ -0,0 +1,355 @@
+package datawave.ingest.csv;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.junit.Before;
+import org.junit.Test;
+
+import datawave.ingest.csv.mr.handler.ColumnBasedHandlerTestUtil;
+import datawave.ingest.csv.mr.input.CSVRecordReader;
+import datawave.ingest.data.RawRecordContainer;
+import datawave.ingest.data.Type;
+import datawave.ingest.data.TypeRegistry;
+import datawave.ingest.data.config.DataTypeHelper;
+import datawave.ingest.data.config.ingest.BaseIngestHelper;
+import datawave.ingest.mapreduce.handler.DataTypeHandler;
+import datawave.ingest.mapreduce.handler.shard.AbstractColumnBasedHandler;
+
+public class ShardedDataTypeHandlerFailurePolicyTest {
+ private static final String CONFIG_FILE = "config/ingest/norm-content-interface.xml";
+ private static final String CSV_FILE = "/input/my-nci.csv";
+
+ private final String DATA_NAME = "mycsv";
+ private final Text VIS = new Text("PRIVATE");
+ private final String NB = "\u0000";
+
+ private final long FIRST_TS = 1709208107000L;
+ private final long SECOND_TS = 1709226703000L;
+ private final long THIRD_TS = 1709294484000L;
+ private final String UID = "-3bjkmz.-h4x7jj.eny3am";
+ private final Text ROW = new Text("20240229_9");
+ private final String DT_UID = NB + DATA_NAME + NB + UID;
+ private final Text C_FAM = new Text(DATA_NAME + NB + UID);
+ private final Text SHARD_ID_DT = new Text(ROW + NB + DATA_NAME);
+
+ private Configuration conf;
+ private FileSplit split;
+ private AbstractColumnBasedHandler abstractHandler;
+ private CSVRecordReader reader;
+
+ @Before
+ public void setup() {
+ conf = new Configuration();
+ conf.addResource(ClassLoader.getSystemResource(CONFIG_FILE));
+
+ split = new FileSplit(new Path(Objects.requireNonNull(getClass().getResource(CSV_FILE)).getPath()), 0, 200, new String[0]);
+ }
+
+ @Test
+ public void testLeave() throws IOException {
+ conf.set(DATA_NAME + BaseIngestHelper.DEFAULT_FAILED_NORMALIZATION_POLICY, BaseIngestHelper.FailurePolicy.LEAVE.name());
+
+ initializeTest();
+ processFirstRow();
+
+ HashMap> expectedKeys = getSecondRowExpectedKeys();
+
+ // Sharded Indexes added based upon LEAVE normalization policy
+ Set shardKeys = new HashSet<>();
+ Set shardIndexKeys = new HashSet<>();
+ Set shardReverseIndexKeys = new HashSet<>();
+
+ // for the leave policy, clear out the exception but add
+ // a failed normalization field
+
+ // event keys
+ shardKeys.add(new Key(ROW, C_FAM, new Text("DATE_FIELD\u000020240229 17:11:43"), VIS, SECOND_TS));
+ shardKeys.add(new Key(ROW, C_FAM, new Text("FAILED_NORMALIZATION_FIELD\u0000DATE_FIELD"), VIS, SECOND_TS));
+ // index keys
+ shardKeys.add(new Key(ROW, new Text("fi\u0000DATE_FIELD"), new Text("20240229 17:11:43" + DT_UID), VIS, SECOND_TS));
+ shardKeys.add(new Key(ROW, new Text("fi\u0000FAILED_NORMALIZATION_FIELD"), new Text("DATE_FIELD" + DT_UID), VIS, SECOND_TS));
+ // shardIndex keys
+ long stamp = SECOND_TS - (SECOND_TS % 86400000);
+ shardIndexKeys.add(new Key(new Text("20240229 17:11:43"), new Text("DATE_FIELD"), SHARD_ID_DT, VIS, stamp));
+ shardIndexKeys.add(new Key(new Text("DATE_FIELD"), new Text("FAILED_NORMALIZATION_FIELD"), SHARD_ID_DT, VIS, stamp));
+ // reverse shardIndex keys
+ shardReverseIndexKeys.add(new Key(new Text("34:11:71 92204202"), new Text("DATE_FIELD"), SHARD_ID_DT, VIS, stamp));
+ shardReverseIndexKeys.add(new Key(new Text("DLEIF_ETAD"), new Text("FAILED_NORMALIZATION_FIELD"), SHARD_ID_DT, VIS, stamp));
+
+ expectedKeys.get("shardKeys").addAll(shardKeys);
+ expectedKeys.get("shardIndexKeys").addAll(shardIndexKeys);
+ expectedKeys.get("shardReverseIndexKeys").addAll(shardReverseIndexKeys);
+
+ assertTrue("Can you read the second row of records?", reader.nextKeyValue());
+ RawRecordContainer event = reader.getEvent();
+ assertEquals("Do expected and actual timestamps equal?", SECOND_TS, event.getDate());
+
+ processEvent(abstractHandler, event, expectedKeys.get("shardKeys"), expectedKeys.get("shardIndexKeys"), expectedKeys.get("shardReverseIndexKeys"));
+
+ processThirdRow();
+
+ reader.close();
+ }
+
+ @Test
+ public void testDrop() throws IOException {
+ conf.set(DATA_NAME + BaseIngestHelper.DEFAULT_FAILED_NORMALIZATION_POLICY, BaseIngestHelper.FailurePolicy.DROP.name());
+
+ initializeTest();
+ processFirstRow();
+
+ HashMap> expectedKeys = getSecondRowExpectedKeys();
+
+ // Sharded Indexes added based upon LEAVE normalization policy
+ Set shardKeys = new HashSet<>();
+ Set shardIndexKeys = new HashSet<>();
+ Set shardReverseIndexKeys = new HashSet<>();
+
+ // for the drop policy, clear out the exception,
+ // clear out the indexed field value and add
+ // a failed normalization field
+
+ // event keys
+ shardKeys.add(new Key(ROW, C_FAM, new Text("DATE_FIELD\u000020240229 17:11:43"), VIS, SECOND_TS));
+ shardKeys.add(new Key(ROW, C_FAM, new Text("FAILED_NORMALIZATION_FIELD\u0000DATE_FIELD"), VIS, SECOND_TS));
+ // index keys
+ shardKeys.add(new Key(ROW, new Text("fi\u0000FAILED_NORMALIZATION_FIELD"), new Text("DATE_FIELD" + DT_UID), VIS, SECOND_TS));
+ // shardIndex keys
+ long stamp = SECOND_TS - (SECOND_TS % 86400000);
+ shardIndexKeys.add(new Key(new Text("DATE_FIELD"), new Text("FAILED_NORMALIZATION_FIELD"), SHARD_ID_DT, VIS, stamp));
+ // reverse shardIndex keys
+ shardReverseIndexKeys.add(new Key(new Text("DLEIF_ETAD"), new Text("FAILED_NORMALIZATION_FIELD"), SHARD_ID_DT, VIS, stamp));
+
+ expectedKeys.get("shardKeys").addAll(shardKeys);
+ expectedKeys.get("shardIndexKeys").addAll(shardIndexKeys);
+ expectedKeys.get("shardReverseIndexKeys").addAll(shardReverseIndexKeys);
+
+ assertTrue("Can you read the second row of records?", reader.nextKeyValue());
+ RawRecordContainer event = reader.getEvent();
+ assertEquals("Do expected and actual timestamps equal?", SECOND_TS, event.getDate());
+
+ processEvent(abstractHandler, event, expectedKeys.get("shardKeys"), expectedKeys.get("shardIndexKeys"), expectedKeys.get("shardReverseIndexKeys"));
+
+ processThirdRow();
+
+ reader.close();
+ }
+
+ @Test
+ public void testFail() throws IOException {
+ conf.set(DATA_NAME + BaseIngestHelper.DEFAULT_FAILED_NORMALIZATION_POLICY, BaseIngestHelper.FailurePolicy.FAIL.name());
+
+ initializeTest();
+ processFirstRow();
+
+ HashMap> expectedKeys = getSecondRowExpectedKeys();
+
+ // Sharded Indexes added based upon LEAVE normalization policy
+ Set shardKeys = new HashSet<>();
+ Set shardIndexKeys = new HashSet<>();
+ Set shardReverseIndexKeys = new HashSet<>();
+
+ // for the fail policy, leave the exception and let the
+ // caller (EventMapper) fail the event
+
+ // event keys
+ shardKeys.add(new Key(ROW, C_FAM, new Text("DATE_FIELD\u000020240229 17:11:43"), VIS, SECOND_TS));
+ // index keys
+ shardKeys.add(new Key(ROW, new Text("fi\u0000DATE_FIELD"), new Text("20240229 17:11:43" + DT_UID), VIS, SECOND_TS));
+ // shardIndex keys
+ long stamp = SECOND_TS - (SECOND_TS % 86400000);
+ shardIndexKeys.add(new Key(new Text("20240229 17:11:43"), new Text("DATE_FIELD"), SHARD_ID_DT, VIS, stamp));
+ // reverse shardIndex keys
+ shardReverseIndexKeys.add(new Key(new Text("34:11:71 92204202"), new Text("DATE_FIELD"), SHARD_ID_DT, VIS, stamp));
+
+ expectedKeys.get("shardKeys").addAll(shardKeys);
+ expectedKeys.get("shardIndexKeys").addAll(shardIndexKeys);
+ expectedKeys.get("shardReverseIndexKeys").addAll(shardReverseIndexKeys);
+
+ assertTrue("Can you read the second row of records?", reader.nextKeyValue());
+ RawRecordContainer event = reader.getEvent();
+ assertEquals("Do expected and actual timestamps equal?", SECOND_TS, event.getDate());
+
+ processEvent(abstractHandler, event, expectedKeys.get("shardKeys"), expectedKeys.get("shardIndexKeys"), expectedKeys.get("shardReverseIndexKeys"));
+
+ processThirdRow();
+
+ reader.close();
+ }
+
+ private void initializeTest() throws IOException {
+ TaskAttemptContextImpl context = new TaskAttemptContextImpl(conf, new TaskAttemptID());
+ TypeRegistry registry = TypeRegistry.getInstance(context.getConfiguration());
+ Type type = registry.get(context.getConfiguration().get(DataTypeHelper.Properties.DATA_NAME));
+ type.clearIngestHelper();
+
+ abstractHandler = new AbstractColumnBasedHandler<>();
+ abstractHandler.setup(context);
+
+ reader = new CSVRecordReader();
+ reader.initialize(split, context);
+ }
+
+ private void processFirstRow() throws IOException {
+ HashMap> expectedKeys = getExpectedKeysFromFirstRow();
+
+ assertTrue("Can you read the first row of records?", reader.nextKeyValue());
+ RawRecordContainer event = reader.getEvent();
+ assertEquals("Do event timestamps equal?", FIRST_TS, event.getDate());
+
+ processEvent(abstractHandler, reader.getEvent(), expectedKeys.get("shardKeys"), expectedKeys.get("shardIndexKeys"),
+ expectedKeys.get("shardReverseIndexKeys"));
+ }
+
+ private void processThirdRow() throws IOException {
+ HashMap> expectedKeys = getExpectedKeysFromThirdRow();
+
+ assertTrue("Can you read the third row of records?", reader.nextKeyValue());
+ RawRecordContainer event = reader.getEvent();
+ assertEquals("Do event timestamps equal?", THIRD_TS, event.getDate());
+
+ processEvent(abstractHandler, reader.getEvent(), expectedKeys.get("shardKeys"), expectedKeys.get("shardIndexKeys"),
+ expectedKeys.get("shardReverseIndexKeys"));
+ }
+
+ private HashMap> getExpectedKeysFromFirstRow() {
+ HashMap> expected = new HashMap<>();
+
+ Set shardKeys = new HashSet<>();
+ Set shardIndexKeys = new HashSet<>();
+ Set shardReverseIndexKeys = new HashSet<>();
+
+ String uid = "7cabjd.k6nbu6.-go983l";
+ Text row = new Text("20240229_6");
+
+ String dtUid = NB + DATA_NAME + NB + uid;
+ Text cFam = new Text(DATA_NAME + NB + uid);
+ Text shardIdDt = new Text(row + NB + DATA_NAME);
+ // event keys
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_DATE\u00002024-02-29 12:01:47"), VIS, FIRST_TS));
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_ID\u0000header_one"), VIS, FIRST_TS));
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_NUMBER\u0000111"), VIS, FIRST_TS));
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_TEXT_1\u0000text one-one"), VIS, FIRST_TS));
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_TEXT_2\u0000text two-one"), VIS, FIRST_TS));
+ shardKeys.add(new Key(row, cFam, new Text("DATE_FIELD\u00002024-02-29 12:01:47"), VIS, FIRST_TS));
+ // index keys
+ shardKeys.add(new Key(row, new Text("fi\u0000HEADER_NUMBER"), new Text("+cE1.11" + dtUid), VIS, FIRST_TS));
+ shardKeys.add(new Key(row, new Text("fi\u0000HEADER_DATE"), new Text("2024-02-29T12:01:47.000Z" + dtUid), VIS, FIRST_TS));
+ shardKeys.add(new Key(row, new Text("fi\u0000HEADER_ID"), new Text("header_one" + dtUid), VIS, FIRST_TS));
+ shardKeys.add(new Key(row, new Text("fi\u0000DATE_FIELD"), new Text("2024-02-29T12:01:47.000Z" + dtUid), VIS, FIRST_TS));
+ // shardIndex keys
+ long stamp = FIRST_TS - (FIRST_TS % 86400000);
+ shardIndexKeys.add(new Key(new Text("+cE1.11"), new Text("HEADER_NUMBER"), shardIdDt, VIS, stamp));
+ shardIndexKeys.add(new Key(new Text("2024-02-29T12:01:47.000Z"), new Text("HEADER_DATE"), shardIdDt, VIS, stamp));
+ shardIndexKeys.add(new Key(new Text("header_one"), new Text("HEADER_ID"), shardIdDt, VIS, stamp));
+ shardIndexKeys.add(new Key(new Text("2024-02-29T12:01:47.000Z"), new Text("DATE_FIELD"), shardIdDt, VIS, stamp));
+ // reverse shardIndex keys
+ shardReverseIndexKeys.add(new Key(new Text("11.1Ec+"), new Text("HEADER_NUMBER"), shardIdDt, VIS, stamp));
+ shardReverseIndexKeys.add(new Key(new Text("Z000.74:10:21T92-20-4202"), new Text("HEADER_DATE"), shardIdDt, VIS, stamp));
+ shardReverseIndexKeys.add(new Key(new Text("eno_redaeh"), new Text("HEADER_ID"), shardIdDt, VIS, stamp));
+ shardReverseIndexKeys.add(new Key(new Text("Z000.74:10:21T92-20-4202"), new Text("DATE_FIELD"), shardIdDt, VIS, stamp));
+
+ expected.put("shardKeys", shardKeys);
+ expected.put("shardIndexKeys", shardIndexKeys);
+ expected.put("shardReverseIndexKeys", shardReverseIndexKeys);
+
+ return expected;
+ }
+
+ private HashMap> getSecondRowExpectedKeys() {
+ HashMap> expected = new HashMap<>();
+
+ Set shardKeys = new HashSet<>();
+ Set shardIndexKeys = new HashSet<>();
+ Set shardReverseIndexKeys = new HashSet<>();
+
+ // event keys
+ shardKeys.add(new Key(ROW, C_FAM, new Text("HEADER_DATE\u00002024-02-29 17:11:43"), VIS, SECOND_TS));
+ shardKeys.add(new Key(ROW, C_FAM, new Text("HEADER_ID\u0000header_two"), VIS, SECOND_TS));
+ shardKeys.add(new Key(ROW, C_FAM, new Text("HEADER_NUMBER\u0000222"), VIS, SECOND_TS));
+ shardKeys.add(new Key(ROW, C_FAM, new Text("HEADER_TEXT_1\u0000text one-two"), VIS, SECOND_TS));
+ shardKeys.add(new Key(ROW, C_FAM, new Text("HEADER_TEXT_2\u0000text two-two"), VIS, SECOND_TS));
+ // index keys
+ shardKeys.add(new Key(ROW, new Text("fi\u0000HEADER_NUMBER"), new Text("+cE2.22" + DT_UID), VIS, SECOND_TS));
+ shardKeys.add(new Key(ROW, new Text("fi\u0000HEADER_DATE"), new Text("2024-02-29T17:11:43.000Z" + DT_UID), VIS, SECOND_TS));
+ shardKeys.add(new Key(ROW, new Text("fi\u0000HEADER_ID"), new Text("header_two" + DT_UID), VIS, SECOND_TS));
+ // shardIndex keys
+ long stamp = SECOND_TS - (SECOND_TS % 86400000);
+ shardIndexKeys.add(new Key(new Text("+cE2.22"), new Text("HEADER_NUMBER"), SHARD_ID_DT, VIS, stamp));
+ shardIndexKeys.add(new Key(new Text("2024-02-29T17:11:43.000Z"), new Text("HEADER_DATE"), SHARD_ID_DT, VIS, stamp));
+ shardIndexKeys.add(new Key(new Text("header_two"), new Text("HEADER_ID"), SHARD_ID_DT, VIS, stamp));
+ // reverse shardIndex keys
+ shardReverseIndexKeys.add(new Key(new Text("22.2Ec+"), new Text("HEADER_NUMBER"), SHARD_ID_DT, VIS, stamp));
+ shardReverseIndexKeys.add(new Key(new Text("Z000.34:11:71T92-20-4202"), new Text("HEADER_DATE"), SHARD_ID_DT, VIS, stamp));
+ shardReverseIndexKeys.add(new Key(new Text("owt_redaeh"), new Text("HEADER_ID"), SHARD_ID_DT, VIS, stamp));
+
+ expected.put("shardKeys", shardKeys);
+ expected.put("shardIndexKeys", shardIndexKeys);
+ expected.put("shardReverseIndexKeys", shardReverseIndexKeys);
+
+ return expected;
+ }
+
+ private HashMap> getExpectedKeysFromThirdRow() {
+ HashMap> expected = new HashMap<>();
+
+ Set shardKeys = new HashSet<>();
+ Set shardIndexKeys = new HashSet<>();
+ Set shardReverseIndexKeys = new HashSet<>();
+
+ String uid = "2a6d40.b56k6m.pmeuml";
+ Text row = new Text("20240301_2");
+
+ String dtUid = NB + DATA_NAME + NB + uid;
+ Text cFam = new Text(DATA_NAME + NB + uid);
+ Text shardIdDt = new Text(row + NB + DATA_NAME);
+ // event keys
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_DATE\u00002024-03-01 12:01:24"), VIS, THIRD_TS));
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_ID\u0000header_three"), VIS, THIRD_TS));
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_NUMBER\u0000333"), VIS, THIRD_TS));
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_TEXT_1\u0000text one-three"), VIS, THIRD_TS));
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_TEXT_2\u0000text two-three"), VIS, THIRD_TS));
+ shardKeys.add(new Key(row, cFam, new Text("DATE_FIELD\u00002024-03-01 12:01:24"), VIS, THIRD_TS));
+ // index keys
+ shardKeys.add(new Key(row, new Text("fi\u0000HEADER_NUMBER"), new Text("+cE3.33" + dtUid), VIS, THIRD_TS));
+ shardKeys.add(new Key(row, new Text("fi\u0000HEADER_DATE"), new Text("2024-03-01T12:01:24.000Z" + dtUid), VIS, THIRD_TS));
+ shardKeys.add(new Key(row, new Text("fi\u0000HEADER_ID"), new Text("header_three" + dtUid), VIS, THIRD_TS));
+ shardKeys.add(new Key(row, new Text("fi\u0000DATE_FIELD"), new Text("2024-03-01T12:01:24.000Z" + dtUid), VIS, THIRD_TS));
+ // shardIndex keys
+ long stamp = THIRD_TS - (THIRD_TS % 86400000);
+ shardIndexKeys.add(new Key(new Text("+cE3.33"), new Text("HEADER_NUMBER"), shardIdDt, VIS, stamp));
+ shardIndexKeys.add(new Key(new Text("2024-03-01T12:01:24.000Z"), new Text("HEADER_DATE"), shardIdDt, VIS, stamp));
+ shardIndexKeys.add(new Key(new Text("header_three"), new Text("HEADER_ID"), shardIdDt, VIS, stamp));
+ shardIndexKeys.add(new Key(new Text("2024-03-01T12:01:24.000Z"), new Text("DATE_FIELD"), shardIdDt, VIS, stamp));
+ // reverse shardIndex keys
+ shardReverseIndexKeys.add(new Key(new Text("33.3Ec+"), new Text("HEADER_NUMBER"), shardIdDt, VIS, stamp));
+ shardReverseIndexKeys.add(new Key(new Text("Z000.42:10:21T10-30-4202"), new Text("HEADER_DATE"), shardIdDt, VIS, stamp));
+ shardReverseIndexKeys.add(new Key(new Text("eerht_redaeh"), new Text("HEADER_ID"), shardIdDt, VIS, stamp));
+ shardReverseIndexKeys.add(new Key(new Text("Z000.42:10:21T10-30-4202"), new Text("DATE_FIELD"), shardIdDt, VIS, stamp));
+
+ expected.put("shardKeys", shardKeys);
+ expected.put("shardIndexKeys", shardIndexKeys);
+ expected.put("shardReverseIndexKeys", shardReverseIndexKeys);
+
+ return expected;
+ }
+
+ private void processEvent(DataTypeHandler handler, RawRecordContainer record, Set shardKeys, Set shardIndexKeys,
+ Set shardReverseIndexKeys) {
+ ColumnBasedHandlerTestUtil.processEvent(handler, record, shardKeys, shardIndexKeys, shardReverseIndexKeys);
+ }
+}
diff --git a/warehouse/ingest-csv/src/test/resources/config/ingest/norm-content-interface.xml b/warehouse/ingest-csv/src/test/resources/config/ingest/norm-content-interface.xml
new file mode 100644
index 0000000000..064fd78e9c
--- /dev/null
+++ b/warehouse/ingest-csv/src/test/resources/config/ingest/norm-content-interface.xml
@@ -0,0 +1,135 @@
+
+
+
+
+
+
+ data.name
+ mycsv
+
+
+
+ ingest.data.types
+ mycsv
+
+
+
+ mycsv.ingest.helper.class
+ datawave.ingest.data.config.ingest.CSVIngestHelper
+
+
+
+ mycsv.reader.class
+ datawave.ingest.csv.mr.input.CSVRecordReader
+
+
+
+ mycsv.handler.classes
+ datawave.ingest.csv.mr.handler.ContentCSVColumnBasedHandler
+
+
+
+ mycsv.ingest.policy.enforcer.class
+ datawave.policy.ExampleIngestPolicyEnforcer
+
+
+
+ mycsv.data.header
+ HEADER_DATE,HEADER_ID,HEADER_NUMBER,HEADER_TEXT_1,HEADER_TEXT_2
+
+
+
+ mycsv.data.separator
+ ,
+
+
+
+ mycsv.data.process.extra.fields
+ TRUE
+
+
+
+ mycsv.data.category.id.field
+ HEADER_ID
+
+
+
+ mycsv.data.category.uuid.fields
+ HEADER_ID
+
+
+
+ mycsv.data.category.index.disallowlist
+ HEADER_TEXT_1,HEADER_TEXT_2
+
+
+
+ mycsv.data.category.index.reverse.disallowlist
+ HEADER_TEXT_1,HEADER_TEXT_2
+
+
+
+ mycsv.data.category.marking.default
+ PRIVATE
+
+
+
+ mycsv.HEADER_DATE.data.field.type.class
+ datawave.data.type.DateType
+
+
+
+ mycsv.HEADER_ID.data.field.type.class
+ datawave.data.type.StringType
+
+
+
+ mycsv.HEADER_NUMBER.data.field.type.class
+ datawave.data.type.NumberType
+
+
+
+ mycsv.HEADER_TEXT_1.data.field.type.class
+ datawave.data.type.StringType
+
+
+
+ mycsv.HEADER_TEXT_2.data.field.type.class
+ datawave.data.type.StringType
+
+
+ mycsv.DATE_FIELD.data.field.type.class
+ datawave.data.type.DateType
+
+
+
+ num.shards
+ 20
+
+
+
+ shard.table.name
+ shard
+
+
+
+ shard.global.index.table.name
+ shardIndex
+
+
+
+ shard.global.rindex.table.name
+ shardReverseIndex
+
+
+
+ mycsv.data.category.date
+ HEADER_DATE
+
+
+
+ mycsv.data.category.date.formats
+ yyyy-MM-dd HH:mm:ss
+
+
+
\ No newline at end of file
diff --git a/warehouse/ingest-csv/src/test/resources/input/my-nci.csv b/warehouse/ingest-csv/src/test/resources/input/my-nci.csv
new file mode 100644
index 0000000000..10920e43cf
--- /dev/null
+++ b/warehouse/ingest-csv/src/test/resources/input/my-nci.csv
@@ -0,0 +1,3 @@
+2024-02-29 12:01:47,header_one,111,text one-one,text two-one,DATE_FIELD=2024-02-29 12:01:47
+2024-02-29 17:11:43,header_two,222,text one-two,text two-two,DATE_FIELD=20240229 17:11:43
+2024-03-01 12:01:24,header_three,333,text one-three,text two-three,DATE_FIELD=2024-03-01 12:01:24
\ No newline at end of file
From 6cf345f2642ab68c9e8659bc33f5853b283fb2f9 Mon Sep 17 00:00:00 2001
From: rdhayes68 <104331030+rdhayes68@users.noreply.github.com>
Date: Thu, 25 Jul 2024 16:43:03 -0400
Subject: [PATCH 013/128] Correct DROP state for normalization bug. (#2285)
* Correct DROP state for normalization bug.
* Update code format
* Updated from code review comments
* Correct DROP state for normalization bug.
* added NormalizedContentInterface test
* Correct DROP state for normalization bug.
* Add test for ShardedDataTypeHandler
* Update test for ShardedDataTypeHandler Failure Policy.
* Update test for ShardedDataTypeHandler Failure Policy.
* Update test for ShardedDataTypeHandler Failure Policy.
* Update based on code review comments.
* Update based on code review comments.
---------
Co-authored-by: hgklohr
---
.../data/config/ingest/BaseIngestHelper.java | 8 +-
.../shard/AbstractColumnBasedHandler.java | 2 +-
.../handler/shard/ShardedDataTypeHandler.java | 138 +++----
.../csv/NormalizedContentInterfaceTest.java | 252 +++++++++++++
...ardedDataTypeHandlerFailurePolicyTest.java | 355 ++++++++++++++++++
.../config/ingest/norm-content-interface.xml | 135 +++++++
.../src/test/resources/input/my-nci.csv | 3 +
7 files changed, 821 insertions(+), 72 deletions(-)
create mode 100644 warehouse/ingest-csv/src/test/java/datawave/ingest/csv/NormalizedContentInterfaceTest.java
create mode 100644 warehouse/ingest-csv/src/test/java/datawave/ingest/csv/ShardedDataTypeHandlerFailurePolicyTest.java
create mode 100644 warehouse/ingest-csv/src/test/resources/config/ingest/norm-content-interface.xml
create mode 100644 warehouse/ingest-csv/src/test/resources/input/my-nci.csv
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java
index 4dcd86e58e..4be57bdf85 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java
@@ -985,8 +985,12 @@ protected void applyNormalizationAndAddToResults(Multimap getShardNamesAndValues(Raw
// Put the normalized field name and normalized value into the reverse
if (createGlobalReverseIndexTerms) {
- if (helper.isReverseIndexedField(e.getValue().getIndexedFieldName())) {
+ if (e.getValue().getIndexedFieldValue() != null && helper.isReverseIndexedField(e.getValue().getIndexedFieldName())) {
NormalizedContentInterface rField = (NormalizedContentInterface) (e.getValue().clone());
rField.setEventFieldValue(new StringBuilder(rField.getEventFieldValue()).reverse().toString());
rField.setIndexedFieldValue(new StringBuilder(rField.getIndexedFieldValue()).reverse().toString());
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java
index 73798508b4..dc3f27c7a9 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java
@@ -1019,50 +1019,50 @@ protected Multimap createShardFieldIndexColumn(RawRecordCon
Multimap values = HashMultimap.create();
- Text colf = new Text("fi");
- TextUtil.textAppend(colf, fieldName, replaceMalformedUTF8);
- Text unmaskedColq = new Text(fieldValue);
- TextUtil.textAppend(unmaskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
- TextUtil.textAppend(unmaskedColq, event.getId().toString(), replaceMalformedUTF8);
-
- if (value == null) {
- value = NULL_VALUE;
- }
+ if (!StringUtils.isEmpty(fieldValue)) {
+ Text colf = new Text("fi");
+ TextUtil.textAppend(colf, fieldName, replaceMalformedUTF8);
+ Text unmaskedColq = new Text(fieldValue);
+ TextUtil.textAppend(unmaskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
+ TextUtil.textAppend(unmaskedColq, event.getId().toString(), replaceMalformedUTF8);
+
+ if (value == null) {
+ value = NULL_VALUE;
+ }
- if (null != maskedFieldHelper && maskedFieldHelper.contains(fieldName)) {
- if (!StringUtils.isEmpty(fieldValue)) {
+ if (null != maskedFieldHelper && maskedFieldHelper.contains(fieldName)) {
// Put unmasked colq with original visibility
Key k = createKey(shardId, colf, unmaskedColq, visibility, event.getDate(), deleteMode);
BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
values.put(bKey, value);
- }
- // We need to use the normalized masked values
- final String normalizedMaskedValue = helper.getNormalizedMaskedValue(fieldName);
- if (!StringUtils.isEmpty(normalizedMaskedValue)) {
- Text maskedColq = new Text(normalizedMaskedValue);
- TextUtil.textAppend(maskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
- TextUtil.textAppend(maskedColq, event.getId().toString(), replaceMalformedUTF8);
+ // We need to use the normalized masked values
+ final String normalizedMaskedValue = helper.getNormalizedMaskedValue(fieldName);
+ if (!StringUtils.isEmpty(normalizedMaskedValue)) {
+ Text maskedColq = new Text(normalizedMaskedValue);
+ TextUtil.textAppend(maskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
+ TextUtil.textAppend(maskedColq, event.getId().toString(), replaceMalformedUTF8);
+
+ // Put masked colq with masked visibility
+ Key key = createKey(shardId, colf, maskedColq, maskedVisibility, event.getDate(), deleteMode);
+ BulkIngestKey bulkIngestKey = new BulkIngestKey(this.getShardTableName(), key);
+ values.put(bulkIngestKey, value);
+ }
+ } else if (!StringUtils.isEmpty(fieldValue)) {
+ /**
+ * For values that are not being masked, we use the "unmaskedValue" and the masked visibility e.g. release the value as it was in the event at
+ * the lower visibility
+ */
+ byte[] refVisibility = visibility;
+
+ if (null != maskedFieldHelper) {
+ refVisibility = maskedVisibility;
+ }
- // Put masked colq with masked visibility
- Key k = createKey(shardId, colf, maskedColq, maskedVisibility, event.getDate(), deleteMode);
+ Key k = createKey(shardId, colf, unmaskedColq, refVisibility, event.getDate(), deleteMode);
BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
values.put(bKey, value);
}
- } else if (!StringUtils.isEmpty(fieldValue)) {
- /**
- * For values that are not being masked, we use the "unmaskedValue" and the masked visibility e.g. release the value as it was in the event at the
- * lower visibility
- */
- byte[] refVisibility = visibility;
-
- if (null != maskedFieldHelper) {
- refVisibility = maskedVisibility;
- }
-
- Key k = createKey(shardId, colf, unmaskedColq, refVisibility, event.getDate(), deleteMode);
- BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
- values.put(bKey, value);
}
return values;
@@ -1097,51 +1097,51 @@ protected void createShardFieldIndexColumn(RawRecordContainer event, Multimap expectedValues = createExpectedValues();
+ // for the leave policy, clear out the exception but add
+ // a failed normalization field
+ expectedValues.put(FAILED_NORMALIZATION_FIELD, createNormalizedContent(FAILED_NORMALIZATION_FIELD, DATE_FIELD));
+ expectedValues.put(DATE_FIELD, createNormalizedContentLeave());
+
+ test(expectedValues);
+ }
+
+ @Test
+ public void testDrop() throws IOException {
+ conf.set(DATA_NAME + BaseIngestHelper.DEFAULT_FAILED_NORMALIZATION_POLICY, BaseIngestHelper.FailurePolicy.DROP.name());
+
+ Multimap expectedValues = createExpectedValues();
+ // for the drop policy, clear out the exception,
+ // clear out the indexed field value and add
+ // a failed normalization field
+ expectedValues.put(FAILED_NORMALIZATION_FIELD, createNormalizedContent(FAILED_NORMALIZATION_FIELD, DATE_FIELD));
+ expectedValues.put(DATE_FIELD, createNormalizedContentDrop());
+
+ test(expectedValues);
+ }
+
+ @Test
+ public void testFail() throws IOException {
+ conf.set(DATA_NAME + BaseIngestHelper.DEFAULT_FAILED_NORMALIZATION_POLICY, BaseIngestHelper.FailurePolicy.FAIL.name());
+
+ Multimap expectedValues = createExpectedValues();
+ // for the fail policy, leave the exception and let the
+ // caller (EventMapper) fail the event
+ SimpleThrowable exception = SimpleThrowable.create("Failed to normalize value as a Date: " + BAD_DATE_FIELD);
+ expectedValues.put(DATE_FIELD, createNormalizedContentFail(exception));
+
+ test(expectedValues);
+ }
+
+ private NormalizedContentInterface createNormalizedContentLeave() {
+ NormalizedContentInterface fieldAndValue = createNormalizedContent(DATE_FIELD, BAD_DATE_FIELD);
+ fieldAndValue.setError(null);
+ return fieldAndValue;
+ }
+
+ private NormalizedContentInterface createNormalizedContentDrop() {
+ NormalizedContentInterface fieldAndValue = createNormalizedContent(DATE_FIELD, BAD_DATE_FIELD, null);
+ fieldAndValue.setError(null);
+ return fieldAndValue;
+ }
+
+ private NormalizedContentInterface createNormalizedContentFail(Throwable throwable) {
+ NormalizedContentInterface fieldAndValue = createNormalizedContent(DATE_FIELD, BAD_DATE_FIELD);
+ fieldAndValue.setError(throwable);
+ return fieldAndValue;
+ }
+
+ private Multimap createExpectedValues() {
+ final String HEADER_DATE = "HEADER_DATE";
+ final String HEADER_NUMBER = "HEADER_NUMBER";
+ final String HEADER_ID = "HEADER_ID";
+ final String HEADER_TEXT_1 = "HEADER_TEXT_1";
+ final String HEADER_TEXT_2 = "HEADER_TEXT_2";
+
+ Multimap expectedValues = HashMultimap.create();
+
+ expectedValues.put(HEADER_DATE, createNormalizedContent(HEADER_DATE, "2024-02-29 12:01:47", "2024-02-29T12:01:47.000Z"));
+ expectedValues.put(HEADER_NUMBER, createNormalizedContent(HEADER_NUMBER, "111", "+cE1.11"));
+ expectedValues.put(HEADER_ID, createNormalizedContent(HEADER_ID, "header_one"));
+ expectedValues.put(HEADER_TEXT_1, createNormalizedContent(HEADER_TEXT_1, "text one-one"));
+ expectedValues.put(HEADER_TEXT_2, createNormalizedContent(HEADER_TEXT_2, "text two-one"));
+ expectedValues.put(DATE_FIELD, createNormalizedContent(DATE_FIELD, "2024-02-29 12:01:47", "2024-02-29T12:01:47.000Z"));
+
+ expectedValues.put(HEADER_DATE, createNormalizedContent(HEADER_DATE, "2024-02-29 17:11:43", "2024-02-29T17:11:43.000Z"));
+ expectedValues.put(HEADER_NUMBER, createNormalizedContent(HEADER_NUMBER, "222", "+cE2.22"));
+ expectedValues.put(HEADER_ID, createNormalizedContent(HEADER_ID, "header_two"));
+ expectedValues.put(HEADER_TEXT_1, createNormalizedContent(HEADER_TEXT_1, "text one-two"));
+ expectedValues.put(HEADER_TEXT_2, createNormalizedContent(HEADER_TEXT_2, "text two-two"));
+
+ expectedValues.put(HEADER_DATE, createNormalizedContent(HEADER_DATE, "2024-03-01 12:01:24", "2024-03-01T12:01:24.000Z"));
+ expectedValues.put(HEADER_NUMBER, createNormalizedContent(HEADER_NUMBER, "333", "+cE3.33"));
+ expectedValues.put(HEADER_ID, createNormalizedContent(HEADER_ID, "header_three"));
+ expectedValues.put(HEADER_TEXT_1, createNormalizedContent(HEADER_TEXT_1, "text one-three"));
+ expectedValues.put(HEADER_TEXT_2, createNormalizedContent(HEADER_TEXT_2, "text two-three"));
+ expectedValues.put(DATE_FIELD, createNormalizedContent(DATE_FIELD, "2024-03-01 12:01:24", "2024-03-01T12:01:24.000Z"));
+
+ return expectedValues;
+ }
+
+ private void test(Multimap expectedValues) throws IOException {
+ TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID());
+
+ TypeRegistry registry = TypeRegistry.getInstance(context.getConfiguration());
+ Type type = registry.get(context.getConfiguration().get(DataTypeHelper.Properties.DATA_NAME));
+ type.clearIngestHelper();
+
+ Multimap actualValues = HashMultimap.create();
+
+ IngestHelperInterface helper = type.getIngestHelper(context.getConfiguration());
+
+ CSVRecordReader reader = new CSVRecordReader();
+ reader.initialize(split, context);
+ while (reader.nextKeyValue()) {
+ Multimap fields = helper.getEventFields(reader.getEvent());
+ fields.entries().forEach(entry -> actualValues.put(entry.getKey(), entry.getValue()));
+ }
+ reader.close();
+
+ compare(actualValues, expectedValues);
+ }
+
+ private NormalizedContentInterface createNormalizedContent(String field, String value) {
+ NormalizedFieldAndValue fieldAndValue = new NormalizedFieldAndValue(field, value);
+ if (!field.equals(FAILED_NORMALIZATION_FIELD)) {
+ fieldAndValue.setMarkings(createDefaultMarkings());
+ }
+ return fieldAndValue;
+ }
+
+ private NormalizedContentInterface createNormalizedContent(String field, String value, String indexValue) {
+ NormalizedContentInterface fieldAndValue = createNormalizedContent(field, value);
+ fieldAndValue.setIndexedFieldValue(indexValue);
+ return fieldAndValue;
+ }
+
+ private Map createDefaultMarkings() {
+ Map defaultMarkings = new HashMap<>(1);
+ defaultMarkings.put("columnVisibility", "PRIVATE");
+ return defaultMarkings;
+ }
+
+ private void compare(Multimap actualValues, Multimap expectedValues) {
+ // need to modify any error values within the actualValues set in order to perform a simple comparison of error messages
+ Multimap values = HashMultimap.create();
+ actualValues.entries().forEach(entry -> {
+ NormalizedContentInterface nci;
+ if (entry.getValue().getError() == null) {
+ nci = entry.getValue();
+ } else {
+ NormalizedContentInterface temp = new NormalizedFieldAndValue(entry.getValue());
+ temp.setError(SimpleThrowable.create(temp.getError().getMessage()));
+ nci = temp;
+ }
+ values.put(entry.getKey(), nci);
+ });
+
+ Multimap actualToExpectedDiff = Multimaps.filterEntries(values,
+ entry -> !expectedValues.containsEntry(entry.getKey(), entry.getValue()));
+ Multimap expectedToActualDiff = Multimaps.filterEntries(expectedValues,
+ entry -> !values.containsEntry(entry.getKey(), entry.getValue()));
+
+ if (!actualToExpectedDiff.isEmpty()) {
+ log.error("Actual value difference exists");
+ logDiffs(actualToExpectedDiff);
+ }
+ if (!expectedToActualDiff.isEmpty()) {
+ log.error("Expected value difference exists");
+ logDiffs(expectedToActualDiff);
+ }
+
+ Assert.assertFalse("The actual normalized results do not mach expected results.", !actualToExpectedDiff.isEmpty() || !expectedToActualDiff.isEmpty());
+ }
+
+ private void logDiffs(Multimap diff) {
+ diff.entries().forEach(entry -> {
+ log.error("Key: " + entry.getKey());
+ log.error("Event Field Name: " + entry.getValue().getEventFieldName() + " Value: " + entry.getValue().getEventFieldValue());
+ log.error("Indexed Field Name: " + entry.getValue().getIndexedFieldName() + " Value: " + entry.getValue().getIndexedFieldValue());
+ if (entry.getValue().getError() != null) {
+ log.error("Error: " + entry.getValue().getError().getMessage());
+ }
+ });
+ }
+
+ private static class SimpleThrowable extends Throwable {
+ public static SimpleThrowable create(String message) {
+ return new SimpleThrowable(message, null, true, false);
+ }
+
+ protected SimpleThrowable(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+
+ @Override
+ public int hashCode() {
+ return this.getMessage().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof SimpleThrowable) {
+ return this.getMessage().equals(((SimpleThrowable) obj).getMessage());
+ }
+ return super.equals(obj);
+ }
+ }
+}
diff --git a/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/ShardedDataTypeHandlerFailurePolicyTest.java b/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/ShardedDataTypeHandlerFailurePolicyTest.java
new file mode 100644
index 0000000000..55cdc203d1
--- /dev/null
+++ b/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/ShardedDataTypeHandlerFailurePolicyTest.java
@@ -0,0 +1,355 @@
+package datawave.ingest.csv;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.junit.Before;
+import org.junit.Test;
+
+import datawave.ingest.csv.mr.handler.ColumnBasedHandlerTestUtil;
+import datawave.ingest.csv.mr.input.CSVRecordReader;
+import datawave.ingest.data.RawRecordContainer;
+import datawave.ingest.data.Type;
+import datawave.ingest.data.TypeRegistry;
+import datawave.ingest.data.config.DataTypeHelper;
+import datawave.ingest.data.config.ingest.BaseIngestHelper;
+import datawave.ingest.mapreduce.handler.DataTypeHandler;
+import datawave.ingest.mapreduce.handler.shard.AbstractColumnBasedHandler;
+
+public class ShardedDataTypeHandlerFailurePolicyTest {
+ private static final String CONFIG_FILE = "config/ingest/norm-content-interface.xml";
+ private static final String CSV_FILE = "/input/my-nci.csv";
+
+ private final String DATA_NAME = "mycsv";
+ private final Text VIS = new Text("PRIVATE");
+ private final String NB = "\u0000";
+
+ private final long FIRST_TS = 1709208107000L;
+ private final long SECOND_TS = 1709226703000L;
+ private final long THIRD_TS = 1709294484000L;
+ private final String UID = "-3bjkmz.-h4x7jj.eny3am";
+ private final Text ROW = new Text("20240229_9");
+ private final String DT_UID = NB + DATA_NAME + NB + UID;
+ private final Text C_FAM = new Text(DATA_NAME + NB + UID);
+ private final Text SHARD_ID_DT = new Text(ROW + NB + DATA_NAME);
+
+ private Configuration conf;
+ private FileSplit split;
+ private AbstractColumnBasedHandler abstractHandler;
+ private CSVRecordReader reader;
+
+ @Before
+ public void setup() {
+ conf = new Configuration();
+ conf.addResource(ClassLoader.getSystemResource(CONFIG_FILE));
+
+ split = new FileSplit(new Path(Objects.requireNonNull(getClass().getResource(CSV_FILE)).getPath()), 0, 200, new String[0]);
+ }
+
+ @Test
+ public void testLeave() throws IOException {
+ conf.set(DATA_NAME + BaseIngestHelper.DEFAULT_FAILED_NORMALIZATION_POLICY, BaseIngestHelper.FailurePolicy.LEAVE.name());
+
+ initializeTest();
+ processFirstRow();
+
+ HashMap> expectedKeys = getSecondRowExpectedKeys();
+
+ // Sharded Indexes added based upon LEAVE normalization policy
+ Set shardKeys = new HashSet<>();
+ Set shardIndexKeys = new HashSet<>();
+ Set shardReverseIndexKeys = new HashSet<>();
+
+ // for the leave policy, clear out the exception but add
+ // a failed normalization field
+
+ // event keys
+ shardKeys.add(new Key(ROW, C_FAM, new Text("DATE_FIELD\u000020240229 17:11:43"), VIS, SECOND_TS));
+ shardKeys.add(new Key(ROW, C_FAM, new Text("FAILED_NORMALIZATION_FIELD\u0000DATE_FIELD"), VIS, SECOND_TS));
+ // index keys
+ shardKeys.add(new Key(ROW, new Text("fi\u0000DATE_FIELD"), new Text("20240229 17:11:43" + DT_UID), VIS, SECOND_TS));
+ shardKeys.add(new Key(ROW, new Text("fi\u0000FAILED_NORMALIZATION_FIELD"), new Text("DATE_FIELD" + DT_UID), VIS, SECOND_TS));
+ // shardIndex keys
+ long stamp = SECOND_TS - (SECOND_TS % 86400000);
+ shardIndexKeys.add(new Key(new Text("20240229 17:11:43"), new Text("DATE_FIELD"), SHARD_ID_DT, VIS, stamp));
+ shardIndexKeys.add(new Key(new Text("DATE_FIELD"), new Text("FAILED_NORMALIZATION_FIELD"), SHARD_ID_DT, VIS, stamp));
+ // reverse shardIndex keys
+ shardReverseIndexKeys.add(new Key(new Text("34:11:71 92204202"), new Text("DATE_FIELD"), SHARD_ID_DT, VIS, stamp));
+ shardReverseIndexKeys.add(new Key(new Text("DLEIF_ETAD"), new Text("FAILED_NORMALIZATION_FIELD"), SHARD_ID_DT, VIS, stamp));
+
+ expectedKeys.get("shardKeys").addAll(shardKeys);
+ expectedKeys.get("shardIndexKeys").addAll(shardIndexKeys);
+ expectedKeys.get("shardReverseIndexKeys").addAll(shardReverseIndexKeys);
+
+ assertTrue("Can you read the second row of records?", reader.nextKeyValue());
+ RawRecordContainer event = reader.getEvent();
+ assertEquals("Do expected and actual timestamps equal?", SECOND_TS, event.getDate());
+
+ processEvent(abstractHandler, event, expectedKeys.get("shardKeys"), expectedKeys.get("shardIndexKeys"), expectedKeys.get("shardReverseIndexKeys"));
+
+ processThirdRow();
+
+ reader.close();
+ }
+
+ @Test
+ public void testDrop() throws IOException {
+ conf.set(DATA_NAME + BaseIngestHelper.DEFAULT_FAILED_NORMALIZATION_POLICY, BaseIngestHelper.FailurePolicy.DROP.name());
+
+ initializeTest();
+ processFirstRow();
+
+ HashMap> expectedKeys = getSecondRowExpectedKeys();
+
+ // Sharded Indexes added based upon LEAVE normalization policy
+ Set shardKeys = new HashSet<>();
+ Set shardIndexKeys = new HashSet<>();
+ Set shardReverseIndexKeys = new HashSet<>();
+
+ // for the drop policy, clear out the exception,
+ // clear out the indexed field value and add
+ // a failed normalization field
+
+ // event keys
+ shardKeys.add(new Key(ROW, C_FAM, new Text("DATE_FIELD\u000020240229 17:11:43"), VIS, SECOND_TS));
+ shardKeys.add(new Key(ROW, C_FAM, new Text("FAILED_NORMALIZATION_FIELD\u0000DATE_FIELD"), VIS, SECOND_TS));
+ // index keys
+ shardKeys.add(new Key(ROW, new Text("fi\u0000FAILED_NORMALIZATION_FIELD"), new Text("DATE_FIELD" + DT_UID), VIS, SECOND_TS));
+ // shardIndex keys
+ long stamp = SECOND_TS - (SECOND_TS % 86400000);
+ shardIndexKeys.add(new Key(new Text("DATE_FIELD"), new Text("FAILED_NORMALIZATION_FIELD"), SHARD_ID_DT, VIS, stamp));
+ // reverse shardIndex keys
+ shardReverseIndexKeys.add(new Key(new Text("DLEIF_ETAD"), new Text("FAILED_NORMALIZATION_FIELD"), SHARD_ID_DT, VIS, stamp));
+
+ expectedKeys.get("shardKeys").addAll(shardKeys);
+ expectedKeys.get("shardIndexKeys").addAll(shardIndexKeys);
+ expectedKeys.get("shardReverseIndexKeys").addAll(shardReverseIndexKeys);
+
+ assertTrue("Can you read the second row of records?", reader.nextKeyValue());
+ RawRecordContainer event = reader.getEvent();
+ assertEquals("Do expected and actual timestamps equal?", SECOND_TS, event.getDate());
+
+ processEvent(abstractHandler, event, expectedKeys.get("shardKeys"), expectedKeys.get("shardIndexKeys"), expectedKeys.get("shardReverseIndexKeys"));
+
+ processThirdRow();
+
+ reader.close();
+ }
+
+ @Test
+ public void testFail() throws IOException {
+ conf.set(DATA_NAME + BaseIngestHelper.DEFAULT_FAILED_NORMALIZATION_POLICY, BaseIngestHelper.FailurePolicy.FAIL.name());
+
+ initializeTest();
+ processFirstRow();
+
+ HashMap> expectedKeys = getSecondRowExpectedKeys();
+
+ // Sharded Indexes added based upon LEAVE normalization policy
+ Set shardKeys = new HashSet<>();
+ Set shardIndexKeys = new HashSet<>();
+ Set shardReverseIndexKeys = new HashSet<>();
+
+ // for the fail policy, leave the exception and let the
+ // caller (EventMapper) fail the event
+
+ // event keys
+ shardKeys.add(new Key(ROW, C_FAM, new Text("DATE_FIELD\u000020240229 17:11:43"), VIS, SECOND_TS));
+ // index keys
+ shardKeys.add(new Key(ROW, new Text("fi\u0000DATE_FIELD"), new Text("20240229 17:11:43" + DT_UID), VIS, SECOND_TS));
+ // shardIndex keys
+ long stamp = SECOND_TS - (SECOND_TS % 86400000);
+ shardIndexKeys.add(new Key(new Text("20240229 17:11:43"), new Text("DATE_FIELD"), SHARD_ID_DT, VIS, stamp));
+ // reverse shardIndex keys
+ shardReverseIndexKeys.add(new Key(new Text("34:11:71 92204202"), new Text("DATE_FIELD"), SHARD_ID_DT, VIS, stamp));
+
+ expectedKeys.get("shardKeys").addAll(shardKeys);
+ expectedKeys.get("shardIndexKeys").addAll(shardIndexKeys);
+ expectedKeys.get("shardReverseIndexKeys").addAll(shardReverseIndexKeys);
+
+ assertTrue("Can you read the second row of records?", reader.nextKeyValue());
+ RawRecordContainer event = reader.getEvent();
+ assertEquals("Do expected and actual timestamps equal?", SECOND_TS, event.getDate());
+
+ processEvent(abstractHandler, event, expectedKeys.get("shardKeys"), expectedKeys.get("shardIndexKeys"), expectedKeys.get("shardReverseIndexKeys"));
+
+ processThirdRow();
+
+ reader.close();
+ }
+
+ private void initializeTest() throws IOException {
+ TaskAttemptContextImpl context = new TaskAttemptContextImpl(conf, new TaskAttemptID());
+ TypeRegistry registry = TypeRegistry.getInstance(context.getConfiguration());
+ Type type = registry.get(context.getConfiguration().get(DataTypeHelper.Properties.DATA_NAME));
+ type.clearIngestHelper();
+
+ abstractHandler = new AbstractColumnBasedHandler<>();
+ abstractHandler.setup(context);
+
+ reader = new CSVRecordReader();
+ reader.initialize(split, context);
+ }
+
+ private void processFirstRow() throws IOException {
+ HashMap> expectedKeys = getExpectedKeysFromFirstRow();
+
+ assertTrue("Can you read the first row of records?", reader.nextKeyValue());
+ RawRecordContainer event = reader.getEvent();
+ assertEquals("Do event timestamps equal?", FIRST_TS, event.getDate());
+
+ processEvent(abstractHandler, reader.getEvent(), expectedKeys.get("shardKeys"), expectedKeys.get("shardIndexKeys"),
+ expectedKeys.get("shardReverseIndexKeys"));
+ }
+
+ private void processThirdRow() throws IOException {
+ HashMap> expectedKeys = getExpectedKeysFromThirdRow();
+
+ assertTrue("Can you read the third row of records?", reader.nextKeyValue());
+ RawRecordContainer event = reader.getEvent();
+ assertEquals("Do event timestamps equal?", THIRD_TS, event.getDate());
+
+ processEvent(abstractHandler, reader.getEvent(), expectedKeys.get("shardKeys"), expectedKeys.get("shardIndexKeys"),
+ expectedKeys.get("shardReverseIndexKeys"));
+ }
+
+ private HashMap> getExpectedKeysFromFirstRow() {
+ HashMap> expected = new HashMap<>();
+
+ Set shardKeys = new HashSet<>();
+ Set shardIndexKeys = new HashSet<>();
+ Set shardReverseIndexKeys = new HashSet<>();
+
+ String uid = "7cabjd.k6nbu6.-go983l";
+ Text row = new Text("20240229_6");
+
+ String dtUid = NB + DATA_NAME + NB + uid;
+ Text cFam = new Text(DATA_NAME + NB + uid);
+ Text shardIdDt = new Text(row + NB + DATA_NAME);
+ // event keys
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_DATE\u00002024-02-29 12:01:47"), VIS, FIRST_TS));
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_ID\u0000header_one"), VIS, FIRST_TS));
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_NUMBER\u0000111"), VIS, FIRST_TS));
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_TEXT_1\u0000text one-one"), VIS, FIRST_TS));
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_TEXT_2\u0000text two-one"), VIS, FIRST_TS));
+ shardKeys.add(new Key(row, cFam, new Text("DATE_FIELD\u00002024-02-29 12:01:47"), VIS, FIRST_TS));
+ // index keys
+ shardKeys.add(new Key(row, new Text("fi\u0000HEADER_NUMBER"), new Text("+cE1.11" + dtUid), VIS, FIRST_TS));
+ shardKeys.add(new Key(row, new Text("fi\u0000HEADER_DATE"), new Text("2024-02-29T12:01:47.000Z" + dtUid), VIS, FIRST_TS));
+ shardKeys.add(new Key(row, new Text("fi\u0000HEADER_ID"), new Text("header_one" + dtUid), VIS, FIRST_TS));
+ shardKeys.add(new Key(row, new Text("fi\u0000DATE_FIELD"), new Text("2024-02-29T12:01:47.000Z" + dtUid), VIS, FIRST_TS));
+ // shardIndex keys
+ long stamp = FIRST_TS - (FIRST_TS % 86400000);
+ shardIndexKeys.add(new Key(new Text("+cE1.11"), new Text("HEADER_NUMBER"), shardIdDt, VIS, stamp));
+ shardIndexKeys.add(new Key(new Text("2024-02-29T12:01:47.000Z"), new Text("HEADER_DATE"), shardIdDt, VIS, stamp));
+ shardIndexKeys.add(new Key(new Text("header_one"), new Text("HEADER_ID"), shardIdDt, VIS, stamp));
+ shardIndexKeys.add(new Key(new Text("2024-02-29T12:01:47.000Z"), new Text("DATE_FIELD"), shardIdDt, VIS, stamp));
+ // reverse shardIndex keys
+ shardReverseIndexKeys.add(new Key(new Text("11.1Ec+"), new Text("HEADER_NUMBER"), shardIdDt, VIS, stamp));
+ shardReverseIndexKeys.add(new Key(new Text("Z000.74:10:21T92-20-4202"), new Text("HEADER_DATE"), shardIdDt, VIS, stamp));
+ shardReverseIndexKeys.add(new Key(new Text("eno_redaeh"), new Text("HEADER_ID"), shardIdDt, VIS, stamp));
+ shardReverseIndexKeys.add(new Key(new Text("Z000.74:10:21T92-20-4202"), new Text("DATE_FIELD"), shardIdDt, VIS, stamp));
+
+ expected.put("shardKeys", shardKeys);
+ expected.put("shardIndexKeys", shardIndexKeys);
+ expected.put("shardReverseIndexKeys", shardReverseIndexKeys);
+
+ return expected;
+ }
+
+ private HashMap> getSecondRowExpectedKeys() {
+ HashMap> expected = new HashMap<>();
+
+ Set shardKeys = new HashSet<>();
+ Set shardIndexKeys = new HashSet<>();
+ Set shardReverseIndexKeys = new HashSet<>();
+
+ // event keys
+ shardKeys.add(new Key(ROW, C_FAM, new Text("HEADER_DATE\u00002024-02-29 17:11:43"), VIS, SECOND_TS));
+ shardKeys.add(new Key(ROW, C_FAM, new Text("HEADER_ID\u0000header_two"), VIS, SECOND_TS));
+ shardKeys.add(new Key(ROW, C_FAM, new Text("HEADER_NUMBER\u0000222"), VIS, SECOND_TS));
+ shardKeys.add(new Key(ROW, C_FAM, new Text("HEADER_TEXT_1\u0000text one-two"), VIS, SECOND_TS));
+ shardKeys.add(new Key(ROW, C_FAM, new Text("HEADER_TEXT_2\u0000text two-two"), VIS, SECOND_TS));
+ // index keys
+ shardKeys.add(new Key(ROW, new Text("fi\u0000HEADER_NUMBER"), new Text("+cE2.22" + DT_UID), VIS, SECOND_TS));
+ shardKeys.add(new Key(ROW, new Text("fi\u0000HEADER_DATE"), new Text("2024-02-29T17:11:43.000Z" + DT_UID), VIS, SECOND_TS));
+ shardKeys.add(new Key(ROW, new Text("fi\u0000HEADER_ID"), new Text("header_two" + DT_UID), VIS, SECOND_TS));
+ // shardIndex keys
+ long stamp = SECOND_TS - (SECOND_TS % 86400000);
+ shardIndexKeys.add(new Key(new Text("+cE2.22"), new Text("HEADER_NUMBER"), SHARD_ID_DT, VIS, stamp));
+ shardIndexKeys.add(new Key(new Text("2024-02-29T17:11:43.000Z"), new Text("HEADER_DATE"), SHARD_ID_DT, VIS, stamp));
+ shardIndexKeys.add(new Key(new Text("header_two"), new Text("HEADER_ID"), SHARD_ID_DT, VIS, stamp));
+ // reverse shardIndex keys
+ shardReverseIndexKeys.add(new Key(new Text("22.2Ec+"), new Text("HEADER_NUMBER"), SHARD_ID_DT, VIS, stamp));
+ shardReverseIndexKeys.add(new Key(new Text("Z000.34:11:71T92-20-4202"), new Text("HEADER_DATE"), SHARD_ID_DT, VIS, stamp));
+ shardReverseIndexKeys.add(new Key(new Text("owt_redaeh"), new Text("HEADER_ID"), SHARD_ID_DT, VIS, stamp));
+
+ expected.put("shardKeys", shardKeys);
+ expected.put("shardIndexKeys", shardIndexKeys);
+ expected.put("shardReverseIndexKeys", shardReverseIndexKeys);
+
+ return expected;
+ }
+
+ private HashMap> getExpectedKeysFromThirdRow() {
+ HashMap> expected = new HashMap<>();
+
+ Set shardKeys = new HashSet<>();
+ Set shardIndexKeys = new HashSet<>();
+ Set shardReverseIndexKeys = new HashSet<>();
+
+ String uid = "2a6d40.b56k6m.pmeuml";
+ Text row = new Text("20240301_2");
+
+ String dtUid = NB + DATA_NAME + NB + uid;
+ Text cFam = new Text(DATA_NAME + NB + uid);
+ Text shardIdDt = new Text(row + NB + DATA_NAME);
+ // event keys
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_DATE\u00002024-03-01 12:01:24"), VIS, THIRD_TS));
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_ID\u0000header_three"), VIS, THIRD_TS));
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_NUMBER\u0000333"), VIS, THIRD_TS));
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_TEXT_1\u0000text one-three"), VIS, THIRD_TS));
+ shardKeys.add(new Key(row, cFam, new Text("HEADER_TEXT_2\u0000text two-three"), VIS, THIRD_TS));
+ shardKeys.add(new Key(row, cFam, new Text("DATE_FIELD\u00002024-03-01 12:01:24"), VIS, THIRD_TS));
+ // index keys
+ shardKeys.add(new Key(row, new Text("fi\u0000HEADER_NUMBER"), new Text("+cE3.33" + dtUid), VIS, THIRD_TS));
+ shardKeys.add(new Key(row, new Text("fi\u0000HEADER_DATE"), new Text("2024-03-01T12:01:24.000Z" + dtUid), VIS, THIRD_TS));
+ shardKeys.add(new Key(row, new Text("fi\u0000HEADER_ID"), new Text("header_three" + dtUid), VIS, THIRD_TS));
+ shardKeys.add(new Key(row, new Text("fi\u0000DATE_FIELD"), new Text("2024-03-01T12:01:24.000Z" + dtUid), VIS, THIRD_TS));
+ // shardIndex keys
+ long stamp = THIRD_TS - (THIRD_TS % 86400000);
+ shardIndexKeys.add(new Key(new Text("+cE3.33"), new Text("HEADER_NUMBER"), shardIdDt, VIS, stamp));
+ shardIndexKeys.add(new Key(new Text("2024-03-01T12:01:24.000Z"), new Text("HEADER_DATE"), shardIdDt, VIS, stamp));
+ shardIndexKeys.add(new Key(new Text("header_three"), new Text("HEADER_ID"), shardIdDt, VIS, stamp));
+ shardIndexKeys.add(new Key(new Text("2024-03-01T12:01:24.000Z"), new Text("DATE_FIELD"), shardIdDt, VIS, stamp));
+ // reverse shardIndex keys
+ shardReverseIndexKeys.add(new Key(new Text("33.3Ec+"), new Text("HEADER_NUMBER"), shardIdDt, VIS, stamp));
+ shardReverseIndexKeys.add(new Key(new Text("Z000.42:10:21T10-30-4202"), new Text("HEADER_DATE"), shardIdDt, VIS, stamp));
+ shardReverseIndexKeys.add(new Key(new Text("eerht_redaeh"), new Text("HEADER_ID"), shardIdDt, VIS, stamp));
+ shardReverseIndexKeys.add(new Key(new Text("Z000.42:10:21T10-30-4202"), new Text("DATE_FIELD"), shardIdDt, VIS, stamp));
+
+ expected.put("shardKeys", shardKeys);
+ expected.put("shardIndexKeys", shardIndexKeys);
+ expected.put("shardReverseIndexKeys", shardReverseIndexKeys);
+
+ return expected;
+ }
+
+ private void processEvent(DataTypeHandler handler, RawRecordContainer record, Set shardKeys, Set shardIndexKeys,
+ Set shardReverseIndexKeys) {
+ ColumnBasedHandlerTestUtil.processEvent(handler, record, shardKeys, shardIndexKeys, shardReverseIndexKeys);
+ }
+}
diff --git a/warehouse/ingest-csv/src/test/resources/config/ingest/norm-content-interface.xml b/warehouse/ingest-csv/src/test/resources/config/ingest/norm-content-interface.xml
new file mode 100644
index 0000000000..064fd78e9c
--- /dev/null
+++ b/warehouse/ingest-csv/src/test/resources/config/ingest/norm-content-interface.xml
@@ -0,0 +1,135 @@
+
+
+
+
+
+
+ data.name
+ mycsv
+
+
+
+ ingest.data.types
+ mycsv
+
+
+
+ mycsv.ingest.helper.class
+ datawave.ingest.data.config.ingest.CSVIngestHelper
+
+
+
+ mycsv.reader.class
+ datawave.ingest.csv.mr.input.CSVRecordReader
+
+
+
+ mycsv.handler.classes
+ datawave.ingest.csv.mr.handler.ContentCSVColumnBasedHandler
+
+
+
+ mycsv.ingest.policy.enforcer.class
+ datawave.policy.ExampleIngestPolicyEnforcer
+
+
+
+ mycsv.data.header
+ HEADER_DATE,HEADER_ID,HEADER_NUMBER,HEADER_TEXT_1,HEADER_TEXT_2
+
+
+
+ mycsv.data.separator
+ ,
+
+
+
+ mycsv.data.process.extra.fields
+ TRUE
+
+
+
+ mycsv.data.category.id.field
+ HEADER_ID
+
+
+
+ mycsv.data.category.uuid.fields
+ HEADER_ID
+
+
+
+ mycsv.data.category.index.disallowlist
+ HEADER_TEXT_1,HEADER_TEXT_2
+
+
+
+ mycsv.data.category.index.reverse.disallowlist
+ HEADER_TEXT_1,HEADER_TEXT_2
+
+
+
+ mycsv.data.category.marking.default
+ PRIVATE
+
+
+
+ mycsv.HEADER_DATE.data.field.type.class
+ datawave.data.type.DateType
+
+
+
+ mycsv.HEADER_ID.data.field.type.class
+ datawave.data.type.StringType
+
+
+
+ mycsv.HEADER_NUMBER.data.field.type.class
+ datawave.data.type.NumberType
+
+
+
+ mycsv.HEADER_TEXT_1.data.field.type.class
+ datawave.data.type.StringType
+
+
+
+ mycsv.HEADER_TEXT_2.data.field.type.class
+ datawave.data.type.StringType
+
+
+ mycsv.DATE_FIELD.data.field.type.class
+ datawave.data.type.DateType
+
+
+
+ num.shards
+ 20
+
+
+
+ shard.table.name
+ shard
+
+
+
+ shard.global.index.table.name
+ shardIndex
+
+
+
+ shard.global.rindex.table.name
+ shardReverseIndex
+
+
+
+ mycsv.data.category.date
+ HEADER_DATE
+
+
+
+ mycsv.data.category.date.formats
+ yyyy-MM-dd HH:mm:ss
+
+
+
\ No newline at end of file
diff --git a/warehouse/ingest-csv/src/test/resources/input/my-nci.csv b/warehouse/ingest-csv/src/test/resources/input/my-nci.csv
new file mode 100644
index 0000000000..10920e43cf
--- /dev/null
+++ b/warehouse/ingest-csv/src/test/resources/input/my-nci.csv
@@ -0,0 +1,3 @@
+2024-02-29 12:01:47,header_one,111,text one-one,text two-one,DATE_FIELD=2024-02-29 12:01:47
+2024-02-29 17:11:43,header_two,222,text one-two,text two-two,DATE_FIELD=20240229 17:11:43
+2024-03-01 12:01:24,header_three,333,text one-three,text two-three,DATE_FIELD=2024-03-01 12:01:24
\ No newline at end of file
From a32e258276728b07bc94ebbd928e83a3b205cacf Mon Sep 17 00:00:00 2001
From: hgklohr
Date: Thu, 25 Jul 2024 21:01:58 +0000
Subject: [PATCH 014/128] 7.2.1
---
common-test/pom.xml | 2 +-
contrib/datawave-quickstart/docker/pom.xml | 2 +-
core/cached-results/pom.xml | 2 +-
core/common-util/pom.xml | 2 +-
core/common/pom.xml | 2 +-
core/connection-pool/pom.xml | 2 +-
core/map-reduce/pom.xml | 2 +-
core/modification/pom.xml | 2 +-
core/pom.xml | 2 +-
core/query/pom.xml | 2 +-
core/utils/pom.xml | 2 +-
docs/pom.xml | 2 +-
microservices/pom.xml | 2 +-
microservices/services/pom.xml | 2 +-
microservices/starters/pom.xml | 2 +-
pom.xml | 2 +-
warehouse/accumulo-extensions/pom.xml | 2 +-
warehouse/age-off-utils/pom.xml | 2 +-
warehouse/age-off/pom.xml | 2 +-
warehouse/assemble/datawave/pom.xml | 2 +-
warehouse/assemble/pom.xml | 2 +-
warehouse/assemble/webservice/pom.xml | 2 +-
warehouse/common/pom.xml | 2 +-
warehouse/core/pom.xml | 2 +-
warehouse/data-dictionary-core/pom.xml | 2 +-
warehouse/edge-dictionary-core/pom.xml | 2 +-
warehouse/edge-model-configuration-core/pom.xml | 2 +-
warehouse/index-stats/pom.xml | 2 +-
warehouse/ingest-configuration/pom.xml | 2 +-
warehouse/ingest-core/pom.xml | 2 +-
warehouse/ingest-csv/pom.xml | 2 +-
warehouse/ingest-json/pom.xml | 2 +-
warehouse/ingest-nyctlc/pom.xml | 2 +-
warehouse/ingest-scripts/pom.xml | 2 +-
warehouse/ingest-ssdeep/pom.xml | 2 +-
warehouse/ingest-wikipedia/pom.xml | 2 +-
warehouse/metrics-core/pom.xml | 2 +-
warehouse/ops-tools/config-compare/pom.xml | 2 +-
warehouse/ops-tools/index-validation/pom.xml | 2 +-
warehouse/ops-tools/pom.xml | 2 +-
warehouse/pom.xml | 2 +-
warehouse/query-core/pom.xml | 2 +-
warehouse/regression-testing/pom.xml | 2 +-
warehouse/ssdeep-common/pom.xml | 2 +-
web-services/accumulo/pom.xml | 2 +-
web-services/atom/pom.xml | 2 +-
web-services/cached-results/pom.xml | 2 +-
web-services/client/pom.xml | 2 +-
web-services/common-util/pom.xml | 2 +-
web-services/common/pom.xml | 2 +-
web-services/deploy/application/pom.xml | 2 +-
web-services/deploy/configuration/pom.xml | 2 +-
web-services/deploy/docs/pom.xml | 2 +-
web-services/deploy/pom.xml | 2 +-
web-services/deploy/spring-framework-integration/pom.xml | 2 +-
web-services/dictionary/pom.xml | 2 +-
web-services/examples/client-login/pom.xml | 2 +-
web-services/examples/http-client/pom.xml | 2 +-
web-services/examples/jms-client/pom.xml | 2 +-
web-services/examples/pom.xml | 2 +-
web-services/examples/query-war/pom.xml | 2 +-
web-services/map-reduce-embedded/pom.xml | 2 +-
web-services/map-reduce-status/pom.xml | 2 +-
web-services/map-reduce/pom.xml | 2 +-
web-services/metrics/pom.xml | 2 +-
web-services/model/pom.xml | 2 +-
web-services/modification/pom.xml | 2 +-
web-services/pom.xml | 2 +-
web-services/query-websocket/pom.xml | 2 +-
web-services/query/pom.xml | 2 +-
web-services/rest-api/pom.xml | 2 +-
web-services/security/pom.xml | 2 +-
web-services/web-root/pom.xml | 2 +-
73 files changed, 73 insertions(+), 73 deletions(-)
diff --git a/common-test/pom.xml b/common-test/pom.xml
index daf1dcfa6a..4a88668ffc 100644
--- a/common-test/pom.xml
+++ b/common-test/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-common-test
${project.artifactId}
diff --git a/contrib/datawave-quickstart/docker/pom.xml b/contrib/datawave-quickstart/docker/pom.xml
index d5b845ae37..8e7ea5aaff 100644
--- a/contrib/datawave-quickstart/docker/pom.xml
+++ b/contrib/datawave-quickstart/docker/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
../../../pom.xml
quickstart
diff --git a/core/cached-results/pom.xml b/core/cached-results/pom.xml
index cf330a880e..18b1695057 100644
--- a/core/cached-results/pom.xml
+++ b/core/cached-results/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-core-cached-results
${project.artifactId}
diff --git a/core/common-util/pom.xml b/core/common-util/pom.xml
index 69b3a4af4f..3d62bdb434 100644
--- a/core/common-util/pom.xml
+++ b/core/common-util/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-core-common-util
${project.artifactId}
diff --git a/core/common/pom.xml b/core/common/pom.xml
index 9c28026696..a4caf24cde 100644
--- a/core/common/pom.xml
+++ b/core/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-core-common
${project.artifactId}
diff --git a/core/connection-pool/pom.xml b/core/connection-pool/pom.xml
index 69e58f532c..90555e2b6c 100644
--- a/core/connection-pool/pom.xml
+++ b/core/connection-pool/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-core-connection-pool
${project.artifactId}
diff --git a/core/map-reduce/pom.xml b/core/map-reduce/pom.xml
index 8c88d1856f..bed9c2cddd 100644
--- a/core/map-reduce/pom.xml
+++ b/core/map-reduce/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-core-map-reduce
${project.artifactId}
diff --git a/core/modification/pom.xml b/core/modification/pom.xml
index 396a4d7181..01490c94f7 100644
--- a/core/modification/pom.xml
+++ b/core/modification/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-core-modification
${project.artifactId}
diff --git a/core/pom.xml b/core/pom.xml
index b547f65996..15c63e19ee 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
gov.nsa.datawave.core
datawave-core-parent
diff --git a/core/query/pom.xml b/core/query/pom.xml
index 4d2b63986b..c442a35533 100644
--- a/core/query/pom.xml
+++ b/core/query/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-core-query
${project.artifactId}
diff --git a/core/utils/pom.xml b/core/utils/pom.xml
index 21adb9cc2e..54e18f060a 100644
--- a/core/utils/pom.xml
+++ b/core/utils/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
gov.nsa.datawave.core
datawave-utils-parent
diff --git a/docs/pom.xml b/docs/pom.xml
index e8eab01a0d..815b082dbe 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-docs
diff --git a/microservices/pom.xml b/microservices/pom.xml
index 86018808ed..c280104a8f 100644
--- a/microservices/pom.xml
+++ b/microservices/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
gov.nsa.datawave.microservice
datawave-microservice-build-parent
diff --git a/microservices/services/pom.xml b/microservices/services/pom.xml
index 8da49b8d1e..d0155e8332 100644
--- a/microservices/services/pom.xml
+++ b/microservices/services/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.microservice
datawave-microservice-build-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-microservice-service-build-parent
pom
diff --git a/microservices/starters/pom.xml b/microservices/starters/pom.xml
index e4bce0c87b..6948266b87 100644
--- a/microservices/starters/pom.xml
+++ b/microservices/starters/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.microservice
datawave-microservice-build-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-microservice-starter-build-parent
pom
diff --git a/pom.xml b/pom.xml
index e6f3fdd2f4..602ca079be 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
4.0.0
gov.nsa.datawave
datawave-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
pom
DataWave
DataWave is a Java-based ingest and query framework that leverages Apache Accumulo to provide fast, secure access to your data.
diff --git a/warehouse/accumulo-extensions/pom.xml b/warehouse/accumulo-extensions/pom.xml
index f4e3f6ea57..e404004d8f 100644
--- a/warehouse/accumulo-extensions/pom.xml
+++ b/warehouse/accumulo-extensions/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-accumulo-extensions
${project.artifactId}
diff --git a/warehouse/age-off-utils/pom.xml b/warehouse/age-off-utils/pom.xml
index 422e1f7c92..f0b01753e4 100644
--- a/warehouse/age-off-utils/pom.xml
+++ b/warehouse/age-off-utils/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-age-off-utils
${project.artifactId}
diff --git a/warehouse/age-off/pom.xml b/warehouse/age-off/pom.xml
index 24015f6c31..54614b2c70 100644
--- a/warehouse/age-off/pom.xml
+++ b/warehouse/age-off/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-age-off
${project.artifactId}
diff --git a/warehouse/assemble/datawave/pom.xml b/warehouse/assemble/datawave/pom.xml
index 7a017096f1..5b79c2c445 100644
--- a/warehouse/assemble/datawave/pom.xml
+++ b/warehouse/assemble/datawave/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
assemble-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
assemble-datawave
jar
diff --git a/warehouse/assemble/pom.xml b/warehouse/assemble/pom.xml
index 07f89c4b3d..436bf61519 100644
--- a/warehouse/assemble/pom.xml
+++ b/warehouse/assemble/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
assemble-parent
pom
diff --git a/warehouse/assemble/webservice/pom.xml b/warehouse/assemble/webservice/pom.xml
index 13a0f272c4..12df512174 100644
--- a/warehouse/assemble/webservice/pom.xml
+++ b/warehouse/assemble/webservice/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
assemble-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
assemble-webservice
${project.artifactId}
diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml
index 455f6e9b00..acbb9b2773 100644
--- a/warehouse/common/pom.xml
+++ b/warehouse/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-common
${project.artifactId}
diff --git a/warehouse/core/pom.xml b/warehouse/core/pom.xml
index 5c1c27393b..43aee07014 100644
--- a/warehouse/core/pom.xml
+++ b/warehouse/core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-core
jar
diff --git a/warehouse/data-dictionary-core/pom.xml b/warehouse/data-dictionary-core/pom.xml
index 18cebd0a23..36c8823c24 100644
--- a/warehouse/data-dictionary-core/pom.xml
+++ b/warehouse/data-dictionary-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-data-dictionary-core
jar
diff --git a/warehouse/edge-dictionary-core/pom.xml b/warehouse/edge-dictionary-core/pom.xml
index d0ff7e12cf..99c074c6b7 100644
--- a/warehouse/edge-dictionary-core/pom.xml
+++ b/warehouse/edge-dictionary-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-edge-dictionary-core
jar
diff --git a/warehouse/edge-model-configuration-core/pom.xml b/warehouse/edge-model-configuration-core/pom.xml
index 53ef6c61e0..d6c81402ff 100644
--- a/warehouse/edge-model-configuration-core/pom.xml
+++ b/warehouse/edge-model-configuration-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-edge-model-configuration-core
jar
diff --git a/warehouse/index-stats/pom.xml b/warehouse/index-stats/pom.xml
index 17483ab144..b8288e9abb 100644
--- a/warehouse/index-stats/pom.xml
+++ b/warehouse/index-stats/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-index-stats
jar
diff --git a/warehouse/ingest-configuration/pom.xml b/warehouse/ingest-configuration/pom.xml
index 1e6c9a11b6..6e3bc3e417 100644
--- a/warehouse/ingest-configuration/pom.xml
+++ b/warehouse/ingest-configuration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ingest-configuration
diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml
index 0e7c60c2ff..9a6e71fd11 100644
--- a/warehouse/ingest-core/pom.xml
+++ b/warehouse/ingest-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ingest-core
jar
diff --git a/warehouse/ingest-csv/pom.xml b/warehouse/ingest-csv/pom.xml
index 4b97eb7ee3..92515de6d8 100644
--- a/warehouse/ingest-csv/pom.xml
+++ b/warehouse/ingest-csv/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ingest-csv
jar
diff --git a/warehouse/ingest-json/pom.xml b/warehouse/ingest-json/pom.xml
index ac56d833cf..1fa33d78b7 100644
--- a/warehouse/ingest-json/pom.xml
+++ b/warehouse/ingest-json/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ingest-json
jar
diff --git a/warehouse/ingest-nyctlc/pom.xml b/warehouse/ingest-nyctlc/pom.xml
index cc6b814e92..2d50291614 100644
--- a/warehouse/ingest-nyctlc/pom.xml
+++ b/warehouse/ingest-nyctlc/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ingest-nyctlc
jar
diff --git a/warehouse/ingest-scripts/pom.xml b/warehouse/ingest-scripts/pom.xml
index 0e4050f0ed..d04f781a95 100644
--- a/warehouse/ingest-scripts/pom.xml
+++ b/warehouse/ingest-scripts/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ingest-scripts
${project.artifactId}
diff --git a/warehouse/ingest-ssdeep/pom.xml b/warehouse/ingest-ssdeep/pom.xml
index a3ed96b52e..3da9f25c37 100644
--- a/warehouse/ingest-ssdeep/pom.xml
+++ b/warehouse/ingest-ssdeep/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ingest-ssdeep
diff --git a/warehouse/ingest-wikipedia/pom.xml b/warehouse/ingest-wikipedia/pom.xml
index ffaf74cf1b..864d1ac922 100644
--- a/warehouse/ingest-wikipedia/pom.xml
+++ b/warehouse/ingest-wikipedia/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ingest-wikipedia
jar
diff --git a/warehouse/metrics-core/pom.xml b/warehouse/metrics-core/pom.xml
index 8ee6ce5486..beabb9469c 100644
--- a/warehouse/metrics-core/pom.xml
+++ b/warehouse/metrics-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-metrics-core
jar
diff --git a/warehouse/ops-tools/config-compare/pom.xml b/warehouse/ops-tools/config-compare/pom.xml
index 16349e90e0..90511aae44 100644
--- a/warehouse/ops-tools/config-compare/pom.xml
+++ b/warehouse/ops-tools/config-compare/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-ops-tools-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ops-tools-config-compare
diff --git a/warehouse/ops-tools/index-validation/pom.xml b/warehouse/ops-tools/index-validation/pom.xml
index 7bdd5f17f0..fe868f2dff 100644
--- a/warehouse/ops-tools/index-validation/pom.xml
+++ b/warehouse/ops-tools/index-validation/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-ops-tools-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ops-tools-index-validation
jar
diff --git a/warehouse/ops-tools/pom.xml b/warehouse/ops-tools/pom.xml
index 4c774f9c9b..54e1f9d780 100644
--- a/warehouse/ops-tools/pom.xml
+++ b/warehouse/ops-tools/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ops-tools-parent
pom
diff --git a/warehouse/pom.xml b/warehouse/pom.xml
index 47019a161b..cf41ca23ac 100644
--- a/warehouse/pom.xml
+++ b/warehouse/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-warehouse-parent
pom
diff --git a/warehouse/query-core/pom.xml b/warehouse/query-core/pom.xml
index f34ce1eac7..432715f05c 100644
--- a/warehouse/query-core/pom.xml
+++ b/warehouse/query-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-query-core
jar
diff --git a/warehouse/regression-testing/pom.xml b/warehouse/regression-testing/pom.xml
index b69ba2c74b..e276ac5f3b 100644
--- a/warehouse/regression-testing/pom.xml
+++ b/warehouse/regression-testing/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-regression-testing
${project.artifactId}
diff --git a/warehouse/ssdeep-common/pom.xml b/warehouse/ssdeep-common/pom.xml
index 5a54996db8..419d44535a 100644
--- a/warehouse/ssdeep-common/pom.xml
+++ b/warehouse/ssdeep-common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ssdeep-common
diff --git a/web-services/accumulo/pom.xml b/web-services/accumulo/pom.xml
index fd544184fb..5c8eee4372 100644
--- a/web-services/accumulo/pom.xml
+++ b/web-services/accumulo/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-accumulo
ejb
diff --git a/web-services/atom/pom.xml b/web-services/atom/pom.xml
index aca4f7774f..d3eab59e65 100644
--- a/web-services/atom/pom.xml
+++ b/web-services/atom/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-atom
ejb
diff --git a/web-services/cached-results/pom.xml b/web-services/cached-results/pom.xml
index c59a9076b0..5e01973fa5 100644
--- a/web-services/cached-results/pom.xml
+++ b/web-services/cached-results/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-cached-results
ejb
diff --git a/web-services/client/pom.xml b/web-services/client/pom.xml
index 4273e6bacc..f1dcc0e4df 100644
--- a/web-services/client/pom.xml
+++ b/web-services/client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-client
jar
diff --git a/web-services/common-util/pom.xml b/web-services/common-util/pom.xml
index 7d03d240bc..dacc512016 100644
--- a/web-services/common-util/pom.xml
+++ b/web-services/common-util/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-common-util
jar
diff --git a/web-services/common/pom.xml b/web-services/common/pom.xml
index c0726f4832..fb3c951bac 100644
--- a/web-services/common/pom.xml
+++ b/web-services/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-common
ejb
diff --git a/web-services/deploy/application/pom.xml b/web-services/deploy/application/pom.xml
index b4c4de6620..ed90c6f678 100644
--- a/web-services/deploy/application/pom.xml
+++ b/web-services/deploy/application/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-deploy-application
ear
diff --git a/web-services/deploy/configuration/pom.xml b/web-services/deploy/configuration/pom.xml
index 1c914de4cf..e9c95efd5b 100644
--- a/web-services/deploy/configuration/pom.xml
+++ b/web-services/deploy/configuration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-deploy-configuration
jar
diff --git a/web-services/deploy/docs/pom.xml b/web-services/deploy/docs/pom.xml
index e179b70533..02acd1c59f 100644
--- a/web-services/deploy/docs/pom.xml
+++ b/web-services/deploy/docs/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-deploy-docs
war
diff --git a/web-services/deploy/pom.xml b/web-services/deploy/pom.xml
index 32b5dca752..08f5eaa60b 100644
--- a/web-services/deploy/pom.xml
+++ b/web-services/deploy/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
diff --git a/web-services/deploy/spring-framework-integration/pom.xml b/web-services/deploy/spring-framework-integration/pom.xml
index d69ba7137b..8658ae454a 100644
--- a/web-services/deploy/spring-framework-integration/pom.xml
+++ b/web-services/deploy/spring-framework-integration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
spring-framework-integration
${project.artifactId}
diff --git a/web-services/dictionary/pom.xml b/web-services/dictionary/pom.xml
index 55b0dc11c5..815436b3ba 100644
--- a/web-services/dictionary/pom.xml
+++ b/web-services/dictionary/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-dictionary
ejb
diff --git a/web-services/examples/client-login/pom.xml b/web-services/examples/client-login/pom.xml
index 0c319397a6..49da894c58 100644
--- a/web-services/examples/client-login/pom.xml
+++ b/web-services/examples/client-login/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-examples-client-login
ejb
diff --git a/web-services/examples/http-client/pom.xml b/web-services/examples/http-client/pom.xml
index c0382445ea..dba8de91ae 100644
--- a/web-services/examples/http-client/pom.xml
+++ b/web-services/examples/http-client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-examples-http-client
jar
diff --git a/web-services/examples/jms-client/pom.xml b/web-services/examples/jms-client/pom.xml
index e504137e92..1c0384d6d0 100644
--- a/web-services/examples/jms-client/pom.xml
+++ b/web-services/examples/jms-client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-examples-jms-client
jar
diff --git a/web-services/examples/pom.xml b/web-services/examples/pom.xml
index ee28cfed2e..eb763dc123 100644
--- a/web-services/examples/pom.xml
+++ b/web-services/examples/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-examples-parent
pom
diff --git a/web-services/examples/query-war/pom.xml b/web-services/examples/query-war/pom.xml
index 7e858c744e..e0d9a97a3e 100644
--- a/web-services/examples/query-war/pom.xml
+++ b/web-services/examples/query-war/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-examples-query-war
war
diff --git a/web-services/map-reduce-embedded/pom.xml b/web-services/map-reduce-embedded/pom.xml
index a0337f4b3d..a5d7953153 100644
--- a/web-services/map-reduce-embedded/pom.xml
+++ b/web-services/map-reduce-embedded/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-map-reduce-embedded
jar
diff --git a/web-services/map-reduce-status/pom.xml b/web-services/map-reduce-status/pom.xml
index 35d7cd2753..3ceacb8877 100644
--- a/web-services/map-reduce-status/pom.xml
+++ b/web-services/map-reduce-status/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-map-reduce-status
ejb
diff --git a/web-services/map-reduce/pom.xml b/web-services/map-reduce/pom.xml
index ba338fdaaf..f8a0ba4e3d 100644
--- a/web-services/map-reduce/pom.xml
+++ b/web-services/map-reduce/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-map-reduce
ejb
diff --git a/web-services/metrics/pom.xml b/web-services/metrics/pom.xml
index af3fd381ae..43e5ce31f2 100644
--- a/web-services/metrics/pom.xml
+++ b/web-services/metrics/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-metrics
ejb
diff --git a/web-services/model/pom.xml b/web-services/model/pom.xml
index d8b5c17fa9..3aadf5363f 100644
--- a/web-services/model/pom.xml
+++ b/web-services/model/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-model
ejb
diff --git a/web-services/modification/pom.xml b/web-services/modification/pom.xml
index f2874d71ca..a7f29b1812 100644
--- a/web-services/modification/pom.xml
+++ b/web-services/modification/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-modification
ejb
diff --git a/web-services/pom.xml b/web-services/pom.xml
index 6de4ad5a69..57e56544d2 100644
--- a/web-services/pom.xml
+++ b/web-services/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
gov.nsa.datawave.webservices
datawave-ws-parent
diff --git a/web-services/query-websocket/pom.xml b/web-services/query-websocket/pom.xml
index 85b02d7c2b..bbdc55f94c 100644
--- a/web-services/query-websocket/pom.xml
+++ b/web-services/query-websocket/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-query-websocket
war
diff --git a/web-services/query/pom.xml b/web-services/query/pom.xml
index 5ea1aec85f..7528f946cb 100644
--- a/web-services/query/pom.xml
+++ b/web-services/query/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-query
ejb
diff --git a/web-services/rest-api/pom.xml b/web-services/rest-api/pom.xml
index b3f9db66fa..27ebaedb93 100644
--- a/web-services/rest-api/pom.xml
+++ b/web-services/rest-api/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-rest-api
war
diff --git a/web-services/security/pom.xml b/web-services/security/pom.xml
index e8c4f418b5..5868ecabc5 100644
--- a/web-services/security/pom.xml
+++ b/web-services/security/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-security
ejb
diff --git a/web-services/web-root/pom.xml b/web-services/web-root/pom.xml
index 62ca2642e3..ed3fca6635 100644
--- a/web-services/web-root/pom.xml
+++ b/web-services/web-root/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1-SNAPSHOT
+ 7.2.1
datawave-ws-web-root
war
From b8e29217b4520dab3205e9741467cd11c56a4ec8 Mon Sep 17 00:00:00 2001
From: hgklohr
Date: Thu, 25 Jul 2024 21:05:16 +0000
Subject: [PATCH 015/128] 7.2.2-SNAPSHOT
---
common-test/pom.xml | 2 +-
contrib/datawave-quickstart/docker/pom.xml | 2 +-
core/cached-results/pom.xml | 2 +-
core/common-util/pom.xml | 2 +-
core/common/pom.xml | 2 +-
core/connection-pool/pom.xml | 2 +-
core/map-reduce/pom.xml | 2 +-
core/modification/pom.xml | 2 +-
core/pom.xml | 2 +-
core/query/pom.xml | 2 +-
core/utils/pom.xml | 2 +-
docs/pom.xml | 2 +-
microservices/pom.xml | 2 +-
microservices/services/pom.xml | 2 +-
microservices/starters/pom.xml | 2 +-
pom.xml | 2 +-
warehouse/accumulo-extensions/pom.xml | 2 +-
warehouse/age-off-utils/pom.xml | 2 +-
warehouse/age-off/pom.xml | 2 +-
warehouse/assemble/datawave/pom.xml | 2 +-
warehouse/assemble/pom.xml | 2 +-
warehouse/assemble/webservice/pom.xml | 2 +-
warehouse/common/pom.xml | 2 +-
warehouse/core/pom.xml | 2 +-
warehouse/data-dictionary-core/pom.xml | 2 +-
warehouse/edge-dictionary-core/pom.xml | 2 +-
warehouse/edge-model-configuration-core/pom.xml | 2 +-
warehouse/index-stats/pom.xml | 2 +-
warehouse/ingest-configuration/pom.xml | 2 +-
warehouse/ingest-core/pom.xml | 2 +-
warehouse/ingest-csv/pom.xml | 2 +-
warehouse/ingest-json/pom.xml | 2 +-
warehouse/ingest-nyctlc/pom.xml | 2 +-
warehouse/ingest-scripts/pom.xml | 2 +-
warehouse/ingest-ssdeep/pom.xml | 2 +-
warehouse/ingest-wikipedia/pom.xml | 2 +-
warehouse/metrics-core/pom.xml | 2 +-
warehouse/ops-tools/config-compare/pom.xml | 2 +-
warehouse/ops-tools/index-validation/pom.xml | 2 +-
warehouse/ops-tools/pom.xml | 2 +-
warehouse/pom.xml | 2 +-
warehouse/query-core/pom.xml | 2 +-
warehouse/regression-testing/pom.xml | 2 +-
warehouse/ssdeep-common/pom.xml | 2 +-
web-services/accumulo/pom.xml | 2 +-
web-services/atom/pom.xml | 2 +-
web-services/cached-results/pom.xml | 2 +-
web-services/client/pom.xml | 2 +-
web-services/common-util/pom.xml | 2 +-
web-services/common/pom.xml | 2 +-
web-services/deploy/application/pom.xml | 2 +-
web-services/deploy/configuration/pom.xml | 2 +-
web-services/deploy/docs/pom.xml | 2 +-
web-services/deploy/pom.xml | 2 +-
web-services/deploy/spring-framework-integration/pom.xml | 2 +-
web-services/dictionary/pom.xml | 2 +-
web-services/examples/client-login/pom.xml | 2 +-
web-services/examples/http-client/pom.xml | 2 +-
web-services/examples/jms-client/pom.xml | 2 +-
web-services/examples/pom.xml | 2 +-
web-services/examples/query-war/pom.xml | 2 +-
web-services/map-reduce-embedded/pom.xml | 2 +-
web-services/map-reduce-status/pom.xml | 2 +-
web-services/map-reduce/pom.xml | 2 +-
web-services/metrics/pom.xml | 2 +-
web-services/model/pom.xml | 2 +-
web-services/modification/pom.xml | 2 +-
web-services/pom.xml | 2 +-
web-services/query-websocket/pom.xml | 2 +-
web-services/query/pom.xml | 2 +-
web-services/rest-api/pom.xml | 2 +-
web-services/security/pom.xml | 2 +-
web-services/web-root/pom.xml | 2 +-
73 files changed, 73 insertions(+), 73 deletions(-)
diff --git a/common-test/pom.xml b/common-test/pom.xml
index 4a88668ffc..5808bc29e8 100644
--- a/common-test/pom.xml
+++ b/common-test/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-common-test
${project.artifactId}
diff --git a/contrib/datawave-quickstart/docker/pom.xml b/contrib/datawave-quickstart/docker/pom.xml
index 8e7ea5aaff..e912ef9f64 100644
--- a/contrib/datawave-quickstart/docker/pom.xml
+++ b/contrib/datawave-quickstart/docker/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
../../../pom.xml
quickstart
diff --git a/core/cached-results/pom.xml b/core/cached-results/pom.xml
index 18b1695057..a3e4358e1f 100644
--- a/core/cached-results/pom.xml
+++ b/core/cached-results/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-core-cached-results
${project.artifactId}
diff --git a/core/common-util/pom.xml b/core/common-util/pom.xml
index 3d62bdb434..03fa097359 100644
--- a/core/common-util/pom.xml
+++ b/core/common-util/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-core-common-util
${project.artifactId}
diff --git a/core/common/pom.xml b/core/common/pom.xml
index a4caf24cde..722746b3a6 100644
--- a/core/common/pom.xml
+++ b/core/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-core-common
${project.artifactId}
diff --git a/core/connection-pool/pom.xml b/core/connection-pool/pom.xml
index 90555e2b6c..d638d0736d 100644
--- a/core/connection-pool/pom.xml
+++ b/core/connection-pool/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-core-connection-pool
${project.artifactId}
diff --git a/core/map-reduce/pom.xml b/core/map-reduce/pom.xml
index bed9c2cddd..6c52e19e63 100644
--- a/core/map-reduce/pom.xml
+++ b/core/map-reduce/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-core-map-reduce
${project.artifactId}
diff --git a/core/modification/pom.xml b/core/modification/pom.xml
index 01490c94f7..35e11d185e 100644
--- a/core/modification/pom.xml
+++ b/core/modification/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-core-modification
${project.artifactId}
diff --git a/core/pom.xml b/core/pom.xml
index 15c63e19ee..2cc7d61d92 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
gov.nsa.datawave.core
datawave-core-parent
diff --git a/core/query/pom.xml b/core/query/pom.xml
index c442a35533..b23f5ee1a1 100644
--- a/core/query/pom.xml
+++ b/core/query/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-core-query
${project.artifactId}
diff --git a/core/utils/pom.xml b/core/utils/pom.xml
index 54e18f060a..e73ee09f4e 100644
--- a/core/utils/pom.xml
+++ b/core/utils/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
gov.nsa.datawave.core
datawave-utils-parent
diff --git a/docs/pom.xml b/docs/pom.xml
index 815b082dbe..3e949f7ab1 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-docs
diff --git a/microservices/pom.xml b/microservices/pom.xml
index c280104a8f..e2cf524569 100644
--- a/microservices/pom.xml
+++ b/microservices/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
gov.nsa.datawave.microservice
datawave-microservice-build-parent
diff --git a/microservices/services/pom.xml b/microservices/services/pom.xml
index d0155e8332..9da1bdb930 100644
--- a/microservices/services/pom.xml
+++ b/microservices/services/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.microservice
datawave-microservice-build-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-microservice-service-build-parent
pom
diff --git a/microservices/starters/pom.xml b/microservices/starters/pom.xml
index 6948266b87..89df413679 100644
--- a/microservices/starters/pom.xml
+++ b/microservices/starters/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.microservice
datawave-microservice-build-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-microservice-starter-build-parent
pom
diff --git a/pom.xml b/pom.xml
index 602ca079be..5722397f60 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
4.0.0
gov.nsa.datawave
datawave-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
pom
DataWave
DataWave is a Java-based ingest and query framework that leverages Apache Accumulo to provide fast, secure access to your data.
diff --git a/warehouse/accumulo-extensions/pom.xml b/warehouse/accumulo-extensions/pom.xml
index e404004d8f..e4512a7edd 100644
--- a/warehouse/accumulo-extensions/pom.xml
+++ b/warehouse/accumulo-extensions/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-accumulo-extensions
${project.artifactId}
diff --git a/warehouse/age-off-utils/pom.xml b/warehouse/age-off-utils/pom.xml
index f0b01753e4..cb0cfeae7c 100644
--- a/warehouse/age-off-utils/pom.xml
+++ b/warehouse/age-off-utils/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-age-off-utils
${project.artifactId}
diff --git a/warehouse/age-off/pom.xml b/warehouse/age-off/pom.xml
index 54614b2c70..5f5bd035a9 100644
--- a/warehouse/age-off/pom.xml
+++ b/warehouse/age-off/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-age-off
${project.artifactId}
diff --git a/warehouse/assemble/datawave/pom.xml b/warehouse/assemble/datawave/pom.xml
index 5b79c2c445..2aa3ba8354 100644
--- a/warehouse/assemble/datawave/pom.xml
+++ b/warehouse/assemble/datawave/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
assemble-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
assemble-datawave
jar
diff --git a/warehouse/assemble/pom.xml b/warehouse/assemble/pom.xml
index 436bf61519..011b8ed37d 100644
--- a/warehouse/assemble/pom.xml
+++ b/warehouse/assemble/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
assemble-parent
pom
diff --git a/warehouse/assemble/webservice/pom.xml b/warehouse/assemble/webservice/pom.xml
index 12df512174..8f277d1a92 100644
--- a/warehouse/assemble/webservice/pom.xml
+++ b/warehouse/assemble/webservice/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
assemble-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
assemble-webservice
${project.artifactId}
diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml
index acbb9b2773..31c8b5489d 100644
--- a/warehouse/common/pom.xml
+++ b/warehouse/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-common
${project.artifactId}
diff --git a/warehouse/core/pom.xml b/warehouse/core/pom.xml
index 43aee07014..145d835736 100644
--- a/warehouse/core/pom.xml
+++ b/warehouse/core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-core
jar
diff --git a/warehouse/data-dictionary-core/pom.xml b/warehouse/data-dictionary-core/pom.xml
index 36c8823c24..e11166aaba 100644
--- a/warehouse/data-dictionary-core/pom.xml
+++ b/warehouse/data-dictionary-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-data-dictionary-core
jar
diff --git a/warehouse/edge-dictionary-core/pom.xml b/warehouse/edge-dictionary-core/pom.xml
index 99c074c6b7..32c281ba8b 100644
--- a/warehouse/edge-dictionary-core/pom.xml
+++ b/warehouse/edge-dictionary-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-edge-dictionary-core
jar
diff --git a/warehouse/edge-model-configuration-core/pom.xml b/warehouse/edge-model-configuration-core/pom.xml
index d6c81402ff..6283291a0d 100644
--- a/warehouse/edge-model-configuration-core/pom.xml
+++ b/warehouse/edge-model-configuration-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-edge-model-configuration-core
jar
diff --git a/warehouse/index-stats/pom.xml b/warehouse/index-stats/pom.xml
index b8288e9abb..49114241e2 100644
--- a/warehouse/index-stats/pom.xml
+++ b/warehouse/index-stats/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-index-stats
jar
diff --git a/warehouse/ingest-configuration/pom.xml b/warehouse/ingest-configuration/pom.xml
index 6e3bc3e417..87d60006ab 100644
--- a/warehouse/ingest-configuration/pom.xml
+++ b/warehouse/ingest-configuration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ingest-configuration
diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml
index 9a6e71fd11..3b718327aa 100644
--- a/warehouse/ingest-core/pom.xml
+++ b/warehouse/ingest-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ingest-core
jar
diff --git a/warehouse/ingest-csv/pom.xml b/warehouse/ingest-csv/pom.xml
index 92515de6d8..4794dc07b4 100644
--- a/warehouse/ingest-csv/pom.xml
+++ b/warehouse/ingest-csv/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ingest-csv
jar
diff --git a/warehouse/ingest-json/pom.xml b/warehouse/ingest-json/pom.xml
index 1fa33d78b7..897bf059af 100644
--- a/warehouse/ingest-json/pom.xml
+++ b/warehouse/ingest-json/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ingest-json
jar
diff --git a/warehouse/ingest-nyctlc/pom.xml b/warehouse/ingest-nyctlc/pom.xml
index 2d50291614..4e32dec0f1 100644
--- a/warehouse/ingest-nyctlc/pom.xml
+++ b/warehouse/ingest-nyctlc/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ingest-nyctlc
jar
diff --git a/warehouse/ingest-scripts/pom.xml b/warehouse/ingest-scripts/pom.xml
index d04f781a95..e497384c7b 100644
--- a/warehouse/ingest-scripts/pom.xml
+++ b/warehouse/ingest-scripts/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ingest-scripts
${project.artifactId}
diff --git a/warehouse/ingest-ssdeep/pom.xml b/warehouse/ingest-ssdeep/pom.xml
index 3da9f25c37..d97febe595 100644
--- a/warehouse/ingest-ssdeep/pom.xml
+++ b/warehouse/ingest-ssdeep/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ingest-ssdeep
diff --git a/warehouse/ingest-wikipedia/pom.xml b/warehouse/ingest-wikipedia/pom.xml
index 864d1ac922..9fd8c6c83a 100644
--- a/warehouse/ingest-wikipedia/pom.xml
+++ b/warehouse/ingest-wikipedia/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ingest-wikipedia
jar
diff --git a/warehouse/metrics-core/pom.xml b/warehouse/metrics-core/pom.xml
index beabb9469c..64663d270b 100644
--- a/warehouse/metrics-core/pom.xml
+++ b/warehouse/metrics-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-metrics-core
jar
diff --git a/warehouse/ops-tools/config-compare/pom.xml b/warehouse/ops-tools/config-compare/pom.xml
index 90511aae44..2afc774182 100644
--- a/warehouse/ops-tools/config-compare/pom.xml
+++ b/warehouse/ops-tools/config-compare/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-ops-tools-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ops-tools-config-compare
diff --git a/warehouse/ops-tools/index-validation/pom.xml b/warehouse/ops-tools/index-validation/pom.xml
index fe868f2dff..d9b21a01f5 100644
--- a/warehouse/ops-tools/index-validation/pom.xml
+++ b/warehouse/ops-tools/index-validation/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-ops-tools-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ops-tools-index-validation
jar
diff --git a/warehouse/ops-tools/pom.xml b/warehouse/ops-tools/pom.xml
index 54e1f9d780..0982684cb3 100644
--- a/warehouse/ops-tools/pom.xml
+++ b/warehouse/ops-tools/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ops-tools-parent
pom
diff --git a/warehouse/pom.xml b/warehouse/pom.xml
index cf41ca23ac..9b5227fafb 100644
--- a/warehouse/pom.xml
+++ b/warehouse/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-warehouse-parent
pom
diff --git a/warehouse/query-core/pom.xml b/warehouse/query-core/pom.xml
index 432715f05c..e3041805d0 100644
--- a/warehouse/query-core/pom.xml
+++ b/warehouse/query-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-query-core
jar
diff --git a/warehouse/regression-testing/pom.xml b/warehouse/regression-testing/pom.xml
index e276ac5f3b..c6437ec7c8 100644
--- a/warehouse/regression-testing/pom.xml
+++ b/warehouse/regression-testing/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-regression-testing
${project.artifactId}
diff --git a/warehouse/ssdeep-common/pom.xml b/warehouse/ssdeep-common/pom.xml
index 419d44535a..8d28628ae0 100644
--- a/warehouse/ssdeep-common/pom.xml
+++ b/warehouse/ssdeep-common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ssdeep-common
diff --git a/web-services/accumulo/pom.xml b/web-services/accumulo/pom.xml
index 5c8eee4372..c236c5d244 100644
--- a/web-services/accumulo/pom.xml
+++ b/web-services/accumulo/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-accumulo
ejb
diff --git a/web-services/atom/pom.xml b/web-services/atom/pom.xml
index d3eab59e65..c6bb6f3b78 100644
--- a/web-services/atom/pom.xml
+++ b/web-services/atom/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-atom
ejb
diff --git a/web-services/cached-results/pom.xml b/web-services/cached-results/pom.xml
index 5e01973fa5..3029f32d97 100644
--- a/web-services/cached-results/pom.xml
+++ b/web-services/cached-results/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-cached-results
ejb
diff --git a/web-services/client/pom.xml b/web-services/client/pom.xml
index f1dcc0e4df..5119b59039 100644
--- a/web-services/client/pom.xml
+++ b/web-services/client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-client
jar
diff --git a/web-services/common-util/pom.xml b/web-services/common-util/pom.xml
index dacc512016..3bfe3f5048 100644
--- a/web-services/common-util/pom.xml
+++ b/web-services/common-util/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-common-util
jar
diff --git a/web-services/common/pom.xml b/web-services/common/pom.xml
index fb3c951bac..df0907daa6 100644
--- a/web-services/common/pom.xml
+++ b/web-services/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-common
ejb
diff --git a/web-services/deploy/application/pom.xml b/web-services/deploy/application/pom.xml
index ed90c6f678..b72af15935 100644
--- a/web-services/deploy/application/pom.xml
+++ b/web-services/deploy/application/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-deploy-application
ear
diff --git a/web-services/deploy/configuration/pom.xml b/web-services/deploy/configuration/pom.xml
index e9c95efd5b..6abbdfafad 100644
--- a/web-services/deploy/configuration/pom.xml
+++ b/web-services/deploy/configuration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-deploy-configuration
jar
diff --git a/web-services/deploy/docs/pom.xml b/web-services/deploy/docs/pom.xml
index 02acd1c59f..7444f1fdc6 100644
--- a/web-services/deploy/docs/pom.xml
+++ b/web-services/deploy/docs/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-deploy-docs
war
diff --git a/web-services/deploy/pom.xml b/web-services/deploy/pom.xml
index 08f5eaa60b..3089b64538 100644
--- a/web-services/deploy/pom.xml
+++ b/web-services/deploy/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
diff --git a/web-services/deploy/spring-framework-integration/pom.xml b/web-services/deploy/spring-framework-integration/pom.xml
index 8658ae454a..1f9b3b0189 100644
--- a/web-services/deploy/spring-framework-integration/pom.xml
+++ b/web-services/deploy/spring-framework-integration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
spring-framework-integration
${project.artifactId}
diff --git a/web-services/dictionary/pom.xml b/web-services/dictionary/pom.xml
index 815436b3ba..16889aea52 100644
--- a/web-services/dictionary/pom.xml
+++ b/web-services/dictionary/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-dictionary
ejb
diff --git a/web-services/examples/client-login/pom.xml b/web-services/examples/client-login/pom.xml
index 49da894c58..30c6844591 100644
--- a/web-services/examples/client-login/pom.xml
+++ b/web-services/examples/client-login/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-examples-client-login
ejb
diff --git a/web-services/examples/http-client/pom.xml b/web-services/examples/http-client/pom.xml
index dba8de91ae..88b1b50709 100644
--- a/web-services/examples/http-client/pom.xml
+++ b/web-services/examples/http-client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-examples-http-client
jar
diff --git a/web-services/examples/jms-client/pom.xml b/web-services/examples/jms-client/pom.xml
index 1c0384d6d0..0b028c5b60 100644
--- a/web-services/examples/jms-client/pom.xml
+++ b/web-services/examples/jms-client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-examples-jms-client
jar
diff --git a/web-services/examples/pom.xml b/web-services/examples/pom.xml
index eb763dc123..62ed24727f 100644
--- a/web-services/examples/pom.xml
+++ b/web-services/examples/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-examples-parent
pom
diff --git a/web-services/examples/query-war/pom.xml b/web-services/examples/query-war/pom.xml
index e0d9a97a3e..454d262734 100644
--- a/web-services/examples/query-war/pom.xml
+++ b/web-services/examples/query-war/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-examples-query-war
war
diff --git a/web-services/map-reduce-embedded/pom.xml b/web-services/map-reduce-embedded/pom.xml
index a5d7953153..a51d6c94e9 100644
--- a/web-services/map-reduce-embedded/pom.xml
+++ b/web-services/map-reduce-embedded/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-map-reduce-embedded
jar
diff --git a/web-services/map-reduce-status/pom.xml b/web-services/map-reduce-status/pom.xml
index 3ceacb8877..39ec9bb55c 100644
--- a/web-services/map-reduce-status/pom.xml
+++ b/web-services/map-reduce-status/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-map-reduce-status
ejb
diff --git a/web-services/map-reduce/pom.xml b/web-services/map-reduce/pom.xml
index f8a0ba4e3d..489b76630d 100644
--- a/web-services/map-reduce/pom.xml
+++ b/web-services/map-reduce/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-map-reduce
ejb
diff --git a/web-services/metrics/pom.xml b/web-services/metrics/pom.xml
index 43e5ce31f2..afc78a7c45 100644
--- a/web-services/metrics/pom.xml
+++ b/web-services/metrics/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-metrics
ejb
diff --git a/web-services/model/pom.xml b/web-services/model/pom.xml
index 3aadf5363f..45962c7b65 100644
--- a/web-services/model/pom.xml
+++ b/web-services/model/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-model
ejb
diff --git a/web-services/modification/pom.xml b/web-services/modification/pom.xml
index a7f29b1812..9015d9f0f2 100644
--- a/web-services/modification/pom.xml
+++ b/web-services/modification/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-modification
ejb
diff --git a/web-services/pom.xml b/web-services/pom.xml
index 57e56544d2..a96d7f3dbe 100644
--- a/web-services/pom.xml
+++ b/web-services/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
gov.nsa.datawave.webservices
datawave-ws-parent
diff --git a/web-services/query-websocket/pom.xml b/web-services/query-websocket/pom.xml
index bbdc55f94c..f30e4f6bf2 100644
--- a/web-services/query-websocket/pom.xml
+++ b/web-services/query-websocket/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-query-websocket
war
diff --git a/web-services/query/pom.xml b/web-services/query/pom.xml
index 7528f946cb..7d1cc26c3f 100644
--- a/web-services/query/pom.xml
+++ b/web-services/query/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-query
ejb
diff --git a/web-services/rest-api/pom.xml b/web-services/rest-api/pom.xml
index 27ebaedb93..9a7258e30f 100644
--- a/web-services/rest-api/pom.xml
+++ b/web-services/rest-api/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-rest-api
war
diff --git a/web-services/security/pom.xml b/web-services/security/pom.xml
index 5868ecabc5..80509709ac 100644
--- a/web-services/security/pom.xml
+++ b/web-services/security/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-security
ejb
diff --git a/web-services/web-root/pom.xml b/web-services/web-root/pom.xml
index ed3fca6635..a6a031a09c 100644
--- a/web-services/web-root/pom.xml
+++ b/web-services/web-root/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.1
+ 7.2.2-SNAPSHOT
datawave-ws-web-root
war
From 77cf308bdf096c64df201e4c9cb83f56a61b48f8 Mon Sep 17 00:00:00 2001
From: Drew Farris
Date: Tue, 30 Jul 2024 19:00:54 +0100
Subject: [PATCH 016/128] Fixed off-by-one error that would prevent final
ssdeep ngram generation (#2478)
Co-authored-by: hgklohr
---
.../query/tables/ssdeep/SSDeepIndexQueryTest.java | 6 +++---
.../query/tables/ssdeep/SSDeepIngestQueryTest.java | 8 ++++----
.../query/tables/ssdeep/SSDeepSimilarityQueryTest.java | 6 +++---
.../ssdeep/SSDeepSimilarityQueryTransformerTest.java | 2 +-
.../datawave/query/tables/ssdeep/util/SSDeepTestUtil.java | 4 ++--
.../main/java/datawave/util/ssdeep/NGramGenerator.java | 2 +-
.../java/datawave/util/ssdeep/NGramGeneratorTest.java | 4 ++--
.../util/ssdeep/SSDeepNGramOverlapScorerTest.java | 2 +-
8 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIndexQueryTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIndexQueryTest.java
index d295bac7a3..04c8ab5432 100644
--- a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIndexQueryTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIndexQueryTest.java
@@ -176,7 +176,7 @@ private static void logSSDeepTestData(String tableName) throws TableNotFoundExce
public void runSingleQuery(boolean applyMinScoreThreshold) throws Exception {
String query = "CHECKSUM_SSDEEP:" + TEST_SSDEEPS[2];
- final int minScoreThreshold = applyMinScoreThreshold ? 65 : 0;
+ final int minScoreThreshold = applyMinScoreThreshold ? 67 : 0;
final int expectedEventCount = applyMinScoreThreshold ? 2 : 3;
EventQueryResponseBase response = runSSDeepQuery(query, minScoreThreshold);
@@ -187,10 +187,10 @@ public void runSingleQuery(boolean applyMinScoreThreshold) throws Exception {
Assert.assertEquals(expectedEventCount, eventCount);
// find the fields for the self match example.
- SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[2], "65", EXPECTED_2_2_OVERLAPS, "100", observedEvents);
+ SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[2], "67", EXPECTED_2_2_OVERLAPS, "100", observedEvents);
// find and validate the fields for the partial match example.
- SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[3], "51", EXPECTED_2_3_OVERLAPS, "96", observedEvents);
+ SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[3], "53", EXPECTED_2_3_OVERLAPS, "96", observedEvents);
if (applyMinScoreThreshold)
assertNoMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[3], observedEvents);
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java
index e79b8bc90e..a686d7ac0f 100644
--- a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java
@@ -144,14 +144,14 @@ public void testSSDeepSimilarity() throws Exception {
@SuppressWarnings("SpellCheckingInspection")
String testSSDeep = "384:nv/fP9FmWVMdRFj2aTgSO+u5QT4ZE1PIVS:nDmWOdRFNTTs504cQS";
String query = "CHECKSUM_SSDEEP:" + testSSDeep;
- String expectedOverlaps = "384:+u5QT4Z, 384:/fP9FmW, 384:2aTgSO+, 384:4ZE1PIV, 384:5QT4ZE1, 384:9FmWVMd, 384:Fj2aTgS, 384:FmWVMdR, 384:MdRFj2a, 384:O+u5QT4, 384:P9FmWVM, 384:QT4ZE1P, 384:RFj2aTg, 384:SO+u5QT, 384:T4ZE1PI, 384:TgSO+u5, 384:VMdRFj2, 384:WVMdRFj, 384:aTgSO+u, 384:dRFj2aT, 384:fP9FmWV, 384:gSO+u5Q, 384:j2aTgSO, 384:mWVMdRF, 384:nv/fP9F, 384:u5QT4ZE, 384:v/fP9Fm, 768:DmWOdRF, 768:FNTTs50, 768:NTTs504, 768:OdRFNTT, 768:RFNTTs5, 768:TTs504c, 768:Ts504cQ, 768:WOdRFNT, 768:dRFNTTs, 768:mWOdRFN, 768:nDmWOdR";
+ String expectedOverlaps = "384:+u5QT4Z, 384:/fP9FmW, 384:2aTgSO+, 384:4ZE1PIV, 384:5QT4ZE1, 384:9FmWVMd, 384:Fj2aTgS, 384:FmWVMdR, 384:MdRFj2a, 384:O+u5QT4, 384:P9FmWVM, 384:QT4ZE1P, 384:RFj2aTg, 384:SO+u5QT, 384:T4ZE1PI, 384:TgSO+u5, 384:VMdRFj2, 384:WVMdRFj, 384:ZE1PIVS, 384:aTgSO+u, 384:dRFj2aT, 384:fP9FmWV, 384:gSO+u5Q, 384:j2aTgSO, 384:mWVMdRF, 384:nv/fP9F, 384:u5QT4ZE, 384:v/fP9Fm, 768:DmWOdRF, 768:FNTTs50, 768:NTTs504, 768:OdRFNTT, 768:RFNTTs5, 768:TTs504c, 768:Ts504cQ, 768:WOdRFNT, 768:dRFNTTs, 768:mWOdRFN, 768:nDmWOdR, 768:s504cQS";
EventQueryResponseBase response = runSSDeepQuery(query, similarityQueryLogic, 0);
List events = response.getEvents();
Assert.assertEquals(1, events.size());
Map> observedEvents = extractObservedEvents(events);
- SSDeepTestUtil.assertSSDeepSimilarityMatch(testSSDeep, testSSDeep, "38", expectedOverlaps, "100", observedEvents);
+ SSDeepTestUtil.assertSSDeepSimilarityMatch(testSSDeep, testSSDeep, "40", expectedOverlaps, "100", observedEvents);
}
@Test
@@ -186,7 +186,7 @@ public void testChainedSSDeepDiscovery() throws Exception {
String testSSDeep = "384:nv/fP9FmWVMdRFj2aTgSO+u5QT4ZE1PIVS:nDmWOdRFNTTs504---";
String targetSSDeep = "384:nv/fP9FmWVMdRFj2aTgSO+u5QT4ZE1PIVS:nDmWOdRFNTTs504cQS";
String query = "CHECKSUM_SSDEEP:" + testSSDeep;
- String expectedOverlaps = "384:+u5QT4Z, 384:/fP9FmW, 384:2aTgSO+, 384:4ZE1PIV, 384:5QT4ZE1, 384:9FmWVMd, 384:Fj2aTgS, 384:FmWVMdR, 384:MdRFj2a, 384:O+u5QT4, 384:P9FmWVM, 384:QT4ZE1P, 384:RFj2aTg, 384:SO+u5QT, 384:T4ZE1PI, 384:TgSO+u5, 384:VMdRFj2, 384:WVMdRFj, 384:aTgSO+u, 384:dRFj2aT, 384:fP9FmWV, 384:gSO+u5Q, 384:j2aTgSO, 384:mWVMdRF, 384:nv/fP9F, 384:u5QT4ZE, 384:v/fP9Fm, 768:DmWOdRF, 768:FNTTs50, 768:NTTs504, 768:OdRFNTT, 768:RFNTTs5, 768:WOdRFNT, 768:dRFNTTs, 768:mWOdRFN, 768:nDmWOdR";
+ String expectedOverlaps = "384:+u5QT4Z, 384:/fP9FmW, 384:2aTgSO+, 384:4ZE1PIV, 384:5QT4ZE1, 384:9FmWVMd, 384:Fj2aTgS, 384:FmWVMdR, 384:MdRFj2a, 384:O+u5QT4, 384:P9FmWVM, 384:QT4ZE1P, 384:RFj2aTg, 384:SO+u5QT, 384:T4ZE1PI, 384:TgSO+u5, 384:VMdRFj2, 384:WVMdRFj, 384:ZE1PIVS, 384:aTgSO+u, 384:dRFj2aT, 384:fP9FmWV, 384:gSO+u5Q, 384:j2aTgSO, 384:mWVMdRF, 384:nv/fP9F, 384:u5QT4ZE, 384:v/fP9Fm, 768:DmWOdRF, 768:FNTTs50, 768:NTTs504, 768:OdRFNTT, 768:RFNTTs5, 768:WOdRFNT, 768:dRFNTTs, 768:mWOdRFN, 768:nDmWOdR";
EventQueryResponseBase response = runSSDeepQuery(query, similarityDiscoveryQueryLogic, 0);
@@ -206,7 +206,7 @@ public void testChainedSSDeepDiscovery() throws Exception {
// The results have been enriched with these fields at this point.
Assert.assertEquals(testSSDeep, resultFields.remove("QUERY"));
Assert.assertEquals("100", resultFields.remove("WEIGHTED_SCORE"));
- Assert.assertEquals("36", resultFields.remove("OVERLAP_SCORE"));
+ Assert.assertEquals("37", resultFields.remove("OVERLAP_SCORE"));
Assert.assertEquals(expectedOverlaps, resultFields.remove("OVERLAP_SSDEEP_NGRAMS"));
Assert.assertTrue("Results had unexpected fields: " + resultFields, resultFields.isEmpty());
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTest.java
index fdc85c32fc..916e99d4ea 100644
--- a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTest.java
@@ -121,7 +121,7 @@ private static void logSSDeepTestData() throws TableNotFoundException {
public void runSingleQuery(boolean applyMinScoreThreshold) throws Exception {
String query = "CHECKSUM_SSDEEP:" + TEST_SSDEEPS[2];
- final int minScoreThreshold = applyMinScoreThreshold ? 65 : 0;
+ final int minScoreThreshold = applyMinScoreThreshold ? 67 : 0;
final int expectedEventCount = applyMinScoreThreshold ? 2 : 3;
EventQueryResponseBase response = runSSDeepQuery(query, minScoreThreshold);
@@ -132,10 +132,10 @@ public void runSingleQuery(boolean applyMinScoreThreshold) throws Exception {
Assert.assertEquals(expectedEventCount, eventCount);
// find the fields for the self match example.
- SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[2], "65", EXPECTED_2_2_OVERLAPS, "100", observedEvents);
+ SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[2], "67", EXPECTED_2_2_OVERLAPS, "100", observedEvents);
// find and validate the fields for the partial match example.
- SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[3], "51", EXPECTED_2_3_OVERLAPS, "96", observedEvents);
+ SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[3], "53", EXPECTED_2_3_OVERLAPS, "96", observedEvents);
if (applyMinScoreThreshold)
SSDeepTestUtil.assertNoMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[3], observedEvents);
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformerTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformerTest.java
index fe52feb574..4699bdae6d 100644
--- a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformerTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformerTest.java
@@ -117,7 +117,7 @@ public void transformTest() {
assertEquals("96", field.getValueString());
break;
case "OVERLAP_SCORE":
- assertEquals("51", field.getValueString());
+ assertEquals("53", field.getValueString());
break;
case "OVERLAP_SSDEEP_NGRAMS":
assertEquals(EXPECTED_2_3_OVERLAPS, field.getValueString());
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/util/SSDeepTestUtil.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/util/SSDeepTestUtil.java
index d76bbfe680..c404c30d4c 100644
--- a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/util/SSDeepTestUtil.java
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/util/SSDeepTestUtil.java
@@ -45,8 +45,8 @@ public class SSDeepTestUtil {
public static final int BUCKET_ENCODING_BASE = BucketAccumuloKeyGenerator.DEFAULT_BUCKET_ENCODING_BASE;
public static final int BUCKET_ENCODING_LENGTH = BucketAccumuloKeyGenerator.DEFAULT_BUCKET_ENCODING_LENGTH;
- public static final String EXPECTED_2_2_OVERLAPS = "3072:/G8VYrs, 3072:02irbxz, 3072:123D6pI, 3072:23D6pIJ, 3072:2irbxzG, 3072:3D6pIJL, 3072:3dXVmbO, 3072:46Kjnz/, 3072:5u46Kjn, 3072:6Kjnz/G, 3072:6pIJLIO, 3072:7fKg3dX, 3072:8VYrs12, 3072:AFYDMxu, 3072:D6pIJLI, 3072:DMxud7f, 3072:FYDMxud, 3072:G8VYrs1, 3072:GAFYDMx, 3072:Kg3dXVm, 3072:Kjnz/G8, 3072:Mxud7fK, 3072:On5u46K, 3072:VYrs123, 3072:VmbOn5u, 3072:XVmbOn5, 3072:YDMxud7, 3072:Yrs123D, 3072:bOn5u46, 3072:bxzGAFY, 3072:d7fKg3d, 3072:dXVmbOn, 3072:fKg3dXV, 3072:g3dXVmb, 3072:irbxzGA, 3072:jnz/G8V, 3072:mbOn5u4, 3072:n5u46Kj, 3072:nz/G8VY, 3072:pIJLIOS, 3072:rbxzGAF, 3072:rs123D6, 3072:s123D6p, 3072:u46Kjnz, 3072:ud7fKg3, 3072:xud7fKg, 3072:xzGAFYD, 3072:z/G8VYr, 3072:zGAFYDM, 6144:02MKlWQ, 6144:2MKlWQ7, 6144:3d4bO96, 6144:4bO968r, 6144:7Sg3d4b, 6144:968rm7J, 6144:KlWQ7Sg, 6144:MKlWQ7S, 6144:O968rm7, 6144:Q7Sg3d4, 6144:Sg3d4bO, 6144:WQ7Sg3d, 6144:bO968rm, 6144:d4bO968, 6144:g3d4bO9, 6144:lWQ7Sg3";
- public static final String EXPECTED_2_3_OVERLAPS = "3072:/G8VYrs, 3072:123D6pI, 3072:23D6pIJ, 3072:3D6pIJL, 3072:3dXVmbO, 3072:46Kjnz/, 3072:5u46Kjn, 3072:6Kjnz/G, 3072:6pIJLIO, 3072:7fKg3dX, 3072:8VYrs12, 3072:AFYDMxu, 3072:D6pIJLI, 3072:DMxud7f, 3072:FYDMxud, 3072:G8VYrs1, 3072:GAFYDMx, 3072:Kg3dXVm, 3072:Kjnz/G8, 3072:Mxud7fK, 3072:On5u46K, 3072:VYrs123, 3072:VmbOn5u, 3072:XVmbOn5, 3072:YDMxud7, 3072:Yrs123D, 3072:bOn5u46, 3072:d7fKg3d, 3072:dXVmbOn, 3072:fKg3dXV, 3072:g3dXVmb, 3072:jnz/G8V, 3072:mbOn5u4, 3072:n5u46Kj, 3072:nz/G8VY, 3072:pIJLIOS, 3072:rs123D6, 3072:s123D6p, 3072:u46Kjnz, 3072:ud7fKg3, 3072:xud7fKg, 3072:z/G8VYr, 6144:3d4bO96, 6144:4bO968r, 6144:7Sg3d4b, 6144:968rm7J, 6144:O968rm7, 6144:Sg3d4bO, 6144:bO968rm, 6144:d4bO968, 6144:g3d4bO9";
+ public static final String EXPECTED_2_2_OVERLAPS = "3072:/G8VYrs, 3072:02irbxz, 3072:123D6pI, 3072:23D6pIJ, 3072:2irbxzG, 3072:3D6pIJL, 3072:3dXVmbO, 3072:46Kjnz/, 3072:5u46Kjn, 3072:6Kjnz/G, 3072:6pIJLIO, 3072:7fKg3dX, 3072:8VYrs12, 3072:AFYDMxu, 3072:D6pIJLI, 3072:DMxud7f, 3072:FYDMxud, 3072:G8VYrs1, 3072:GAFYDMx, 3072:IJLIOSP, 3072:Kg3dXVm, 3072:Kjnz/G8, 3072:Mxud7fK, 3072:On5u46K, 3072:VYrs123, 3072:VmbOn5u, 3072:XVmbOn5, 3072:YDMxud7, 3072:Yrs123D, 3072:bOn5u46, 3072:bxzGAFY, 3072:d7fKg3d, 3072:dXVmbOn, 3072:fKg3dXV, 3072:g3dXVmb, 3072:irbxzGA, 3072:jnz/G8V, 3072:mbOn5u4, 3072:n5u46Kj, 3072:nz/G8VY, 3072:pIJLIOS, 3072:rbxzGAF, 3072:rs123D6, 3072:s123D6p, 3072:u46Kjnz, 3072:ud7fKg3, 3072:xud7fKg, 3072:xzGAFYD, 3072:z/G8VYr, 3072:zGAFYDM, 6144:02MKlWQ, 6144:2MKlWQ7, 6144:3d4bO96, 6144:4bO968r, 6144:68rm7JO, 6144:7Sg3d4b, 6144:968rm7J, 6144:KlWQ7Sg, 6144:MKlWQ7S, 6144:O968rm7, 6144:Q7Sg3d4, 6144:Sg3d4bO, 6144:WQ7Sg3d, 6144:bO968rm, 6144:d4bO968, 6144:g3d4bO9, 6144:lWQ7Sg3";
+ public static final String EXPECTED_2_3_OVERLAPS = "3072:/G8VYrs, 3072:123D6pI, 3072:23D6pIJ, 3072:3D6pIJL, 3072:3dXVmbO, 3072:46Kjnz/, 3072:5u46Kjn, 3072:6Kjnz/G, 3072:6pIJLIO, 3072:7fKg3dX, 3072:8VYrs12, 3072:AFYDMxu, 3072:D6pIJLI, 3072:DMxud7f, 3072:FYDMxud, 3072:G8VYrs1, 3072:GAFYDMx, 3072:IJLIOSP, 3072:Kg3dXVm, 3072:Kjnz/G8, 3072:Mxud7fK, 3072:On5u46K, 3072:VYrs123, 3072:VmbOn5u, 3072:XVmbOn5, 3072:YDMxud7, 3072:Yrs123D, 3072:bOn5u46, 3072:d7fKg3d, 3072:dXVmbOn, 3072:fKg3dXV, 3072:g3dXVmb, 3072:jnz/G8V, 3072:mbOn5u4, 3072:n5u46Kj, 3072:nz/G8VY, 3072:pIJLIOS, 3072:rs123D6, 3072:s123D6p, 3072:u46Kjnz, 3072:ud7fKg3, 3072:xud7fKg, 3072:z/G8VYr, 6144:3d4bO96, 6144:4bO968r, 6144:68rm7JO, 6144:7Sg3d4b, 6144:968rm7J, 6144:O968rm7, 6144:Sg3d4bO, 6144:bO968rm, 6144:d4bO968, 6144:g3d4bO9";
public static final String EXPECTED_2_4_OVERLAPS = "3072:5u46Kjn, 3072:On5u46K, 3072:VmbOn5u, 3072:XVmbOn5, 3072:bOn5u46, 3072:dXVmbOn, 3072:mbOn5u4, 3072:n5u46Kj, 3072:u46Kjnz";
public static void loadSSDeepIndexTextData(AccumuloClient accumuloClient) throws Exception {
diff --git a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/NGramGenerator.java b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/NGramGenerator.java
index e8cb81f8b1..b90f5054c3 100644
--- a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/NGramGenerator.java
+++ b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/NGramGenerator.java
@@ -149,7 +149,7 @@ public Set generateNgrams(SSDeepHash ssDeepHash, int minHashSize) {
*/
public void generateNgrams(int chunkSize, String chunk, Set output) {
final int ngramCount = chunk.length() - ngramSize;
- for (int i = 0; i < ngramCount; i++) {
+ for (int i = 0; i <= ngramCount; i++) {
final String ngram = chunk.substring(i, i + ngramSize);
output.add(new NGramTuple(chunkSize, ngram));
}
diff --git a/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/NGramGeneratorTest.java b/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/NGramGeneratorTest.java
index ac522ac4b3..e6c048bc67 100644
--- a/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/NGramGeneratorTest.java
+++ b/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/NGramGeneratorTest.java
@@ -26,13 +26,13 @@ public class NGramGeneratorTest {
"12288:Hkwuliz", "12288:ikQ7lKH", "12288:5pGxlXT", "12288:ulizQ1l", "12288:lXTd8zb", "12288:beZHkwu", "12288:jeZHkwu", "12288:wulizQ1",
"12288:pGxlXTd", "12288:55pGxlX", "12288:GjeZHkw", "12288:1lK55pG", "12288:lKH5p5H", "12288:HkwuPik", "12288:kwulizQ", "12288:p5H9x1b",
"12288:eZHkwul", "12288:lizQ1lK", "12288:xlXTd8z", "12288:zQ1lK55", "12288:VG4GjeZ", "12288:7lKH5p5", "12288:kwuPikQ", "12288:2r/VG4G",
- "12288:GxlXTd8", "12288:KH5p5H9", "12288:9x1beZH"},
+ "12288:GxlXTd8", "12288:KH5p5H9", "12288:9x1beZH", "12288:XTd8zbW"},
{"6144:6e8o4to", "6144:6y7ppFj", "6144:2C3nq73", "6144:gC6e8o4", "6144:3NRW6/f", "6144:NRW6/ft", "6144:93NRW6/", "6144:TgC6e8o", "6144:3v1kHGh",
"6144:ZxoXb0n", "6144:6ZxoXb0", "6144:/ftZTgC", "6144:mk6ZxoX", "6144:73v1kHG", "6144:ppFj93N", "6144:nq73v1k", "6144:s6y7ppF",
"6144:Fj93NRW", "6144:tZTgC6e", "6144:toHZmk6", "6144:C6e8o4t", "6144:4toHZmk", "6144:pFj93NR", "6144:HZmk6Zx", "6144:k6ZxoXb",
"6144:HGhs6y7", "6144:ftZTgC6", "6144:j93NRW6", "6144:6/ftZTg", "6144:q73v1kH", "6144:1kHGhs6", "6144:y7ppFj9", "6144:Zmk6Zxo",
"6144:hs6y7pp", "6144:7ppFj93", "6144:C3nq73v", "6144:3nq73v1", "6144:v1kHGhs", "6144:ZTgC6e8", "6144:8o4toHZ", "6144:o4toHZm",
- "6144:oHZmk6Z", "6144:W6/ftZT", "6144:02C3nq7", "6144:RW6/ftZ", "6144:Ghs6y7p", "6144:kHGhs6y", "6144:e8o4toH"}};
+ "6144:oHZmk6Z", "6144:W6/ftZT", "6144:02C3nq7", "6144:RW6/ftZ", "6144:Ghs6y7p", "6144:kHGhs6y", "6144:e8o4toH", "6144:xoXb0ns"}};
@Test
public void testGenerateNGrams() {
diff --git a/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/SSDeepNGramOverlapScorerTest.java b/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/SSDeepNGramOverlapScorerTest.java
index f24030bf0c..a5e7cf9b3b 100644
--- a/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/SSDeepNGramOverlapScorerTest.java
+++ b/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/SSDeepNGramOverlapScorerTest.java
@@ -26,7 +26,7 @@ public class SSDeepNGramOverlapScorerTest {
};
- public static final int[] expectedScores = {65, 47, 16, 16, 0, 0, 0};
+ public static final int[] expectedScores = {67, 48, 17, 17, 0, 0, 0};
@Test
public void testCompare() {
From d55f9cd7975417e1da5e3035e576b54e6a5fe0b4 Mon Sep 17 00:00:00 2001
From: Whitney O'Meara
Date: Tue, 30 Jul 2024 09:56:36 -0400
Subject: [PATCH 017/128] Fixed a bug where the remote user operations cache
always used the same key. (#2483)
---
web-services/security/pom.xml | 10 +
.../remote/RemoteUserOperationsImpl.java | 2 +-
.../RemoteUserOperationsImplHttpTest.java | 174 +++++++++++++-----
3 files changed, 144 insertions(+), 42 deletions(-)
diff --git a/web-services/security/pom.xml b/web-services/security/pom.xml
index 80509709ac..de4c443431 100644
--- a/web-services/security/pom.xml
+++ b/web-services/security/pom.xml
@@ -253,11 +253,21 @@
weld-core-impl
test
+
+ org.mockito
+ mockito-core
+ test
+
org.springframework
spring-expression
test
+
+ org.springframework
+ spring-test
+ test
+
${project.artifactId}
diff --git a/web-services/security/src/main/java/datawave/security/authorization/remote/RemoteUserOperationsImpl.java b/web-services/security/src/main/java/datawave/security/authorization/remote/RemoteUserOperationsImpl.java
index 42772e14c2..faea5264f2 100644
--- a/web-services/security/src/main/java/datawave/security/authorization/remote/RemoteUserOperationsImpl.java
+++ b/web-services/security/src/main/java/datawave/security/authorization/remote/RemoteUserOperationsImpl.java
@@ -51,7 +51,7 @@ public void init() {
}
@Override
- @Cacheable(value = "getRemoteUser", key = "{#principal}", cacheManager = "remoteOperationsCacheManager")
+ @Cacheable(value = "getRemoteUser", key = "{#currentUser}", cacheManager = "remoteOperationsCacheManager")
public ProxiedUserDetails getRemoteUser(ProxiedUserDetails currentUser) throws AuthorizationException {
log.info("Cache fault: Retrieving user for " + currentUser.getPrimaryUser().getDn());
return UserOperations.super.getRemoteUser(currentUser);
diff --git a/web-services/security/src/test/java/datawave/security/authorization/remote/RemoteUserOperationsImplHttpTest.java b/web-services/security/src/test/java/datawave/security/authorization/remote/RemoteUserOperationsImplHttpTest.java
index a742f5a1c2..26650903d4 100644
--- a/web-services/security/src/test/java/datawave/security/authorization/remote/RemoteUserOperationsImplHttpTest.java
+++ b/web-services/security/src/test/java/datawave/security/authorization/remote/RemoteUserOperationsImplHttpTest.java
@@ -1,6 +1,7 @@
package datawave.security.authorization.remote;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
import java.io.IOException;
import java.math.BigInteger;
@@ -9,33 +10,58 @@
import java.nio.charset.Charset;
import java.security.KeyPair;
import java.security.KeyPairGenerator;
+import java.security.NoSuchAlgorithmException;
import java.security.PrivateKey;
import java.security.SecureRandom;
+import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.time.ZonedDateTime;
+import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashMap;
+import java.util.Collections;
+import java.util.List;
+import javax.enterprise.concurrent.ManagedExecutorService;
import javax.security.auth.x500.X500Principal;
import javax.ws.rs.core.MediaType;
+import org.apache.accumulo.core.security.Authorizations;
import org.apache.commons.io.IOUtils;
+import org.jboss.security.JSSESecurityDomain;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.cache.Cache;
+import org.springframework.cache.CacheManager;
+import org.springframework.cache.annotation.EnableCaching;
+import org.springframework.cache.concurrent.ConcurrentMapCache;
+import org.springframework.cache.support.SimpleCacheManager;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringRunner;
import org.wildfly.security.x500.cert.X509CertificateBuilder;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Sets;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;
import com.sun.net.httpserver.HttpServer;
import datawave.microservice.query.Query;
import datawave.security.authorization.DatawavePrincipal;
+import datawave.security.authorization.DatawaveUser;
+import datawave.security.authorization.ProxiedUserDetails;
+import datawave.security.authorization.SubjectIssuerDNPair;
+import datawave.security.authorization.UserOperations;
import datawave.security.util.DnUtils;
import datawave.user.AuthorizationsListBase;
import datawave.user.DefaultAuthorizationsList;
import datawave.webservice.common.json.DefaultMapperDecorator;
+import datawave.webservice.common.json.ObjectMapperDecorator;
import datawave.webservice.common.remote.TestJSSESecurityDomain;
import datawave.webservice.dictionary.data.DataDictionaryBase;
import datawave.webservice.dictionary.data.DescriptionBase;
@@ -54,46 +80,105 @@
import datawave.webservice.result.FacetQueryResponseBase;
import datawave.webservice.result.GenericResponse;
+@RunWith(SpringRunner.class)
+@ContextConfiguration
public class RemoteUserOperationsImplHttpTest {
- private static final int keysize = 2048;
+ @EnableCaching
+ @Configuration
+ static class Config {
+
+ @Bean
+ public CacheManager remoteOperationsCacheManager() {
+ SimpleCacheManager cacheManager = new SimpleCacheManager();
+ List caches = new ArrayList();
+ caches.add(new ConcurrentMapCache("listEffectiveAuthorizations"));
+ caches.add(new ConcurrentMapCache("getRemoteUser"));
+ cacheManager.setCaches(caches);
+ return cacheManager;
+ }
+
+ @Bean
+ public ObjectMapperDecorator objectMapperDecorator() {
+ return new DefaultMapperDecorator();
+ }
- private static final String commonName = "cn=www.test.us";
- private static final String alias = "tomcat";
- private static final char[] keyPass = "changeit".toCharArray();
+ @Bean
+ public ManagedExecutorService executorService() {
+ return Mockito.mock(ManagedExecutorService.class);
+ }
+
+ @Bean
+ public JSSESecurityDomain jsseSecurityDomain() throws CertificateException, NoSuchAlgorithmException {
+ String alias = "tomcat";
+ char[] keyPass = "changeit".toCharArray();
+ int keysize = 2048;
+ String commonName = "cn=www.test.us";
+
+ KeyPairGenerator generater = KeyPairGenerator.getInstance("RSA");
+ generater.initialize(keysize);
+ KeyPair keypair = generater.generateKeyPair();
+ PrivateKey privKey = keypair.getPrivate();
+ final X509Certificate[] chain = new X509Certificate[1];
+ X500Principal x500Principal = new X500Principal(commonName);
+ final ZonedDateTime start = ZonedDateTime.now().minusWeeks(1);
+ final ZonedDateTime until = start.plusYears(1);
+ X509CertificateBuilder builder = new X509CertificateBuilder().setIssuerDn(x500Principal).setSerialNumber(new BigInteger(10, new SecureRandom()))
+ .setNotValidBefore(start).setNotValidAfter(until).setSubjectDn(x500Principal).setPublicKey(keypair.getPublic())
+ .setSigningKey(keypair.getPrivate()).setSignatureAlgorithmName("SHA256withRSA");
+ chain[0] = builder.build();
+
+ return new TestJSSESecurityDomain(alias, privKey, keyPass, chain);
+ }
+
+ @Bean
+ public HttpServer server() throws IOException {
+ HttpServer server = HttpServer.create(new InetSocketAddress(PORT), 0);
+ server.setExecutor(null);
+ server.start();
+ return server;
+ }
+
+ @Bean
+ public RemoteUserOperationsImpl remote(HttpServer server) {
+ // create a remote event query logic that has our own server behind it
+ RemoteUserOperationsImpl remote = new RemoteUserOperationsImpl();
+ remote.setQueryServiceURI("/Security/User/");
+ remote.setQueryServiceScheme("http");
+ remote.setQueryServiceHost("localhost");
+ remote.setQueryServicePort(server.getAddress().getPort());
+ remote.setResponseObjectFactory(new MockResponseObjectFactory());
+ return remote;
+ }
+ }
- private X500Principal x500Principal;
+ private static final SubjectIssuerDNPair userDN = SubjectIssuerDNPair.of("userDn", "issuerDn");
+ private static final SubjectIssuerDNPair otherUserDN = SubjectIssuerDNPair.of("otherUserDn", "issuerDn");
+ private static Authorizations auths = new Authorizations("auth1", "auth2");
private static final int PORT = 0;
+ private final DatawaveUser user = new DatawaveUser(userDN, DatawaveUser.UserType.USER, Sets.newHashSet(auths.toString().split(",")), null, null, -1L);
+ private final DatawavePrincipal principal = new DatawavePrincipal((Collections.singleton(user)));
+
+ private final DatawaveUser otherUser = new DatawaveUser(otherUserDN, DatawaveUser.UserType.USER, Sets.newHashSet(auths.toString().split(",")), null, null,
+ -1L);
+ private final DatawavePrincipal otherPrincipal = new DatawavePrincipal((Collections.singleton(otherUser)));
+
+ @Autowired
private HttpServer server;
- private RemoteUserOperationsImpl remote;
+ @Autowired
+ private UserOperations remote;
+
+ private DefaultAuthorizationsList listEffectiveAuthResponse;
@Before
public void setup() throws Exception {
final ObjectMapper objectMapper = new DefaultMapperDecorator().decorate(new ObjectMapper());
System.setProperty(DnUtils.SUBJECT_DN_PATTERN_PROPERTY, ".*ou=server.*");
- KeyPairGenerator generater = KeyPairGenerator.getInstance("RSA");
- generater.initialize(keysize);
- KeyPair keypair = generater.generateKeyPair();
- PrivateKey privKey = keypair.getPrivate();
- final X509Certificate[] chain = new X509Certificate[1];
- x500Principal = new X500Principal(commonName);
- final ZonedDateTime start = ZonedDateTime.now().minusWeeks(1);
- final ZonedDateTime until = start.plusYears(1);
- X509CertificateBuilder builder = new X509CertificateBuilder().setIssuerDn(x500Principal).setSerialNumber(new BigInteger(10, new SecureRandom()))
- .setNotValidBefore(start).setNotValidAfter(until).setSubjectDn(x500Principal).setPublicKey(keypair.getPublic())
- .setSigningKey(keypair.getPrivate()).setSignatureAlgorithmName("SHA256withRSA");
- chain[0] = builder.build();
-
- server = HttpServer.create(new InetSocketAddress(PORT), 0);
- server.setExecutor(null);
- server.start();
-
- DefaultAuthorizationsList listEffectiveAuthResponse = new DefaultAuthorizationsList();
- listEffectiveAuthResponse.setUserAuths("testuserDn", "testissuerDn", Arrays.asList("auth1", "auth2"));
- listEffectiveAuthResponse.setAuthMapping(new HashMap<>());
+
+ setListEffectiveAuthResponse(userDN, auths);
HttpHandler listEffectiveAuthorizationsHandler = new HttpHandler() {
@Override
@@ -122,17 +207,6 @@ public void handle(HttpExchange exchange) throws IOException {
server.createContext("/Security/User/listEffectiveAuthorizations", listEffectiveAuthorizationsHandler);
server.createContext("/Security/User/flushCachedCredentials", flushHandler);
-
- // create a remote event query logic that has our own server behind it
- remote = new RemoteUserOperationsImpl();
- remote.setQueryServiceURI("/Security/User/");
- remote.setQueryServiceScheme("http");
- remote.setQueryServiceHost("localhost");
- remote.setQueryServicePort(server.getAddress().getPort());
- remote.setExecutorService(null);
- remote.setObjectMapperDecorator(new DefaultMapperDecorator());
- remote.setResponseObjectFactory(new MockResponseObjectFactory());
- remote.setJsseSecurityDomain(new TestJSSESecurityDomain(alias, privKey, keyPass, chain));
}
@After
@@ -142,15 +216,33 @@ public void after() {
}
}
+ private void setListEffectiveAuthResponse(SubjectIssuerDNPair userDN, Authorizations auths) {
+ listEffectiveAuthResponse = new DefaultAuthorizationsList();
+ listEffectiveAuthResponse.setUserAuths(userDN.subjectDN(), userDN.issuerDN(), Arrays.asList(auths.toString().split(",")));
+ listEffectiveAuthResponse.addAuths(userDN.subjectDN(), userDN.issuerDN(), Arrays.asList(auths.toString().split(",")));
+ }
+
@Test
public void testRemoteUserOperations() throws Exception {
- DatawavePrincipal principal = new DatawavePrincipal(commonName);
- AuthorizationsListBase auths = remote.listEffectiveAuthorizations(principal);
- assertEquals(2, auths.getAllAuths().size());
+ AuthorizationsListBase returnedAuths = remote.listEffectiveAuthorizations(principal);
+ assertEquals(2, returnedAuths.getAllAuths().size());
GenericResponse flush = remote.flushCachedCredentials(principal);
assertEquals("test flush result", flush.getResult());
+
+ ProxiedUserDetails returnedUser = remote.getRemoteUser(principal);
+
+ // ensure that we get the cached user details
+ ProxiedUserDetails dupeReturnedUser = remote.getRemoteUser(principal);
+ assertEquals(returnedUser, dupeReturnedUser);
+
+ // setup the list effective auth response for the other user
+ setListEffectiveAuthResponse(otherUserDN, auths);
+
+ // ensure that we get the other user details, not the cached user details
+ ProxiedUserDetails newReturnedUser = remote.getRemoteUser(otherPrincipal);
+ assertNotEquals(returnedUser, newReturnedUser);
}
public static class MockResponseObjectFactory extends ResponseObjectFactory {
From 46bae81c41f9248c450707f96b1acd68e055666f Mon Sep 17 00:00:00 2001
From: Drew Farris
Date: Tue, 30 Jul 2024 19:00:54 +0100
Subject: [PATCH 018/128] Fixed off-by-one error that would prevent final
ssdeep ngram generation (#2478)
Co-authored-by: hgklohr
---
.../query/tables/ssdeep/SSDeepIndexQueryTest.java | 6 +++---
.../query/tables/ssdeep/SSDeepIngestQueryTest.java | 8 ++++----
.../query/tables/ssdeep/SSDeepSimilarityQueryTest.java | 6 +++---
.../ssdeep/SSDeepSimilarityQueryTransformerTest.java | 2 +-
.../datawave/query/tables/ssdeep/util/SSDeepTestUtil.java | 4 ++--
.../main/java/datawave/util/ssdeep/NGramGenerator.java | 2 +-
.../java/datawave/util/ssdeep/NGramGeneratorTest.java | 4 ++--
.../util/ssdeep/SSDeepNGramOverlapScorerTest.java | 2 +-
8 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIndexQueryTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIndexQueryTest.java
index d295bac7a3..04c8ab5432 100644
--- a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIndexQueryTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIndexQueryTest.java
@@ -176,7 +176,7 @@ private static void logSSDeepTestData(String tableName) throws TableNotFoundExce
public void runSingleQuery(boolean applyMinScoreThreshold) throws Exception {
String query = "CHECKSUM_SSDEEP:" + TEST_SSDEEPS[2];
- final int minScoreThreshold = applyMinScoreThreshold ? 65 : 0;
+ final int minScoreThreshold = applyMinScoreThreshold ? 67 : 0;
final int expectedEventCount = applyMinScoreThreshold ? 2 : 3;
EventQueryResponseBase response = runSSDeepQuery(query, minScoreThreshold);
@@ -187,10 +187,10 @@ public void runSingleQuery(boolean applyMinScoreThreshold) throws Exception {
Assert.assertEquals(expectedEventCount, eventCount);
// find the fields for the self match example.
- SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[2], "65", EXPECTED_2_2_OVERLAPS, "100", observedEvents);
+ SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[2], "67", EXPECTED_2_2_OVERLAPS, "100", observedEvents);
// find and validate the fields for the partial match example.
- SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[3], "51", EXPECTED_2_3_OVERLAPS, "96", observedEvents);
+ SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[3], "53", EXPECTED_2_3_OVERLAPS, "96", observedEvents);
if (applyMinScoreThreshold)
assertNoMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[3], observedEvents);
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java
index e79b8bc90e..a686d7ac0f 100644
--- a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java
@@ -144,14 +144,14 @@ public void testSSDeepSimilarity() throws Exception {
@SuppressWarnings("SpellCheckingInspection")
String testSSDeep = "384:nv/fP9FmWVMdRFj2aTgSO+u5QT4ZE1PIVS:nDmWOdRFNTTs504cQS";
String query = "CHECKSUM_SSDEEP:" + testSSDeep;
- String expectedOverlaps = "384:+u5QT4Z, 384:/fP9FmW, 384:2aTgSO+, 384:4ZE1PIV, 384:5QT4ZE1, 384:9FmWVMd, 384:Fj2aTgS, 384:FmWVMdR, 384:MdRFj2a, 384:O+u5QT4, 384:P9FmWVM, 384:QT4ZE1P, 384:RFj2aTg, 384:SO+u5QT, 384:T4ZE1PI, 384:TgSO+u5, 384:VMdRFj2, 384:WVMdRFj, 384:aTgSO+u, 384:dRFj2aT, 384:fP9FmWV, 384:gSO+u5Q, 384:j2aTgSO, 384:mWVMdRF, 384:nv/fP9F, 384:u5QT4ZE, 384:v/fP9Fm, 768:DmWOdRF, 768:FNTTs50, 768:NTTs504, 768:OdRFNTT, 768:RFNTTs5, 768:TTs504c, 768:Ts504cQ, 768:WOdRFNT, 768:dRFNTTs, 768:mWOdRFN, 768:nDmWOdR";
+ String expectedOverlaps = "384:+u5QT4Z, 384:/fP9FmW, 384:2aTgSO+, 384:4ZE1PIV, 384:5QT4ZE1, 384:9FmWVMd, 384:Fj2aTgS, 384:FmWVMdR, 384:MdRFj2a, 384:O+u5QT4, 384:P9FmWVM, 384:QT4ZE1P, 384:RFj2aTg, 384:SO+u5QT, 384:T4ZE1PI, 384:TgSO+u5, 384:VMdRFj2, 384:WVMdRFj, 384:ZE1PIVS, 384:aTgSO+u, 384:dRFj2aT, 384:fP9FmWV, 384:gSO+u5Q, 384:j2aTgSO, 384:mWVMdRF, 384:nv/fP9F, 384:u5QT4ZE, 384:v/fP9Fm, 768:DmWOdRF, 768:FNTTs50, 768:NTTs504, 768:OdRFNTT, 768:RFNTTs5, 768:TTs504c, 768:Ts504cQ, 768:WOdRFNT, 768:dRFNTTs, 768:mWOdRFN, 768:nDmWOdR, 768:s504cQS";
EventQueryResponseBase response = runSSDeepQuery(query, similarityQueryLogic, 0);
List events = response.getEvents();
Assert.assertEquals(1, events.size());
Map> observedEvents = extractObservedEvents(events);
- SSDeepTestUtil.assertSSDeepSimilarityMatch(testSSDeep, testSSDeep, "38", expectedOverlaps, "100", observedEvents);
+ SSDeepTestUtil.assertSSDeepSimilarityMatch(testSSDeep, testSSDeep, "40", expectedOverlaps, "100", observedEvents);
}
@Test
@@ -186,7 +186,7 @@ public void testChainedSSDeepDiscovery() throws Exception {
String testSSDeep = "384:nv/fP9FmWVMdRFj2aTgSO+u5QT4ZE1PIVS:nDmWOdRFNTTs504---";
String targetSSDeep = "384:nv/fP9FmWVMdRFj2aTgSO+u5QT4ZE1PIVS:nDmWOdRFNTTs504cQS";
String query = "CHECKSUM_SSDEEP:" + testSSDeep;
- String expectedOverlaps = "384:+u5QT4Z, 384:/fP9FmW, 384:2aTgSO+, 384:4ZE1PIV, 384:5QT4ZE1, 384:9FmWVMd, 384:Fj2aTgS, 384:FmWVMdR, 384:MdRFj2a, 384:O+u5QT4, 384:P9FmWVM, 384:QT4ZE1P, 384:RFj2aTg, 384:SO+u5QT, 384:T4ZE1PI, 384:TgSO+u5, 384:VMdRFj2, 384:WVMdRFj, 384:aTgSO+u, 384:dRFj2aT, 384:fP9FmWV, 384:gSO+u5Q, 384:j2aTgSO, 384:mWVMdRF, 384:nv/fP9F, 384:u5QT4ZE, 384:v/fP9Fm, 768:DmWOdRF, 768:FNTTs50, 768:NTTs504, 768:OdRFNTT, 768:RFNTTs5, 768:WOdRFNT, 768:dRFNTTs, 768:mWOdRFN, 768:nDmWOdR";
+ String expectedOverlaps = "384:+u5QT4Z, 384:/fP9FmW, 384:2aTgSO+, 384:4ZE1PIV, 384:5QT4ZE1, 384:9FmWVMd, 384:Fj2aTgS, 384:FmWVMdR, 384:MdRFj2a, 384:O+u5QT4, 384:P9FmWVM, 384:QT4ZE1P, 384:RFj2aTg, 384:SO+u5QT, 384:T4ZE1PI, 384:TgSO+u5, 384:VMdRFj2, 384:WVMdRFj, 384:ZE1PIVS, 384:aTgSO+u, 384:dRFj2aT, 384:fP9FmWV, 384:gSO+u5Q, 384:j2aTgSO, 384:mWVMdRF, 384:nv/fP9F, 384:u5QT4ZE, 384:v/fP9Fm, 768:DmWOdRF, 768:FNTTs50, 768:NTTs504, 768:OdRFNTT, 768:RFNTTs5, 768:WOdRFNT, 768:dRFNTTs, 768:mWOdRFN, 768:nDmWOdR";
EventQueryResponseBase response = runSSDeepQuery(query, similarityDiscoveryQueryLogic, 0);
@@ -206,7 +206,7 @@ public void testChainedSSDeepDiscovery() throws Exception {
// The results have been enriched with these fields at this point.
Assert.assertEquals(testSSDeep, resultFields.remove("QUERY"));
Assert.assertEquals("100", resultFields.remove("WEIGHTED_SCORE"));
- Assert.assertEquals("36", resultFields.remove("OVERLAP_SCORE"));
+ Assert.assertEquals("37", resultFields.remove("OVERLAP_SCORE"));
Assert.assertEquals(expectedOverlaps, resultFields.remove("OVERLAP_SSDEEP_NGRAMS"));
Assert.assertTrue("Results had unexpected fields: " + resultFields, resultFields.isEmpty());
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTest.java
index fdc85c32fc..916e99d4ea 100644
--- a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTest.java
@@ -121,7 +121,7 @@ private static void logSSDeepTestData() throws TableNotFoundException {
public void runSingleQuery(boolean applyMinScoreThreshold) throws Exception {
String query = "CHECKSUM_SSDEEP:" + TEST_SSDEEPS[2];
- final int minScoreThreshold = applyMinScoreThreshold ? 65 : 0;
+ final int minScoreThreshold = applyMinScoreThreshold ? 67 : 0;
final int expectedEventCount = applyMinScoreThreshold ? 2 : 3;
EventQueryResponseBase response = runSSDeepQuery(query, minScoreThreshold);
@@ -132,10 +132,10 @@ public void runSingleQuery(boolean applyMinScoreThreshold) throws Exception {
Assert.assertEquals(expectedEventCount, eventCount);
// find the fields for the self match example.
- SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[2], "65", EXPECTED_2_2_OVERLAPS, "100", observedEvents);
+ SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[2], "67", EXPECTED_2_2_OVERLAPS, "100", observedEvents);
// find and validate the fields for the partial match example.
- SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[3], "51", EXPECTED_2_3_OVERLAPS, "96", observedEvents);
+ SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[3], "53", EXPECTED_2_3_OVERLAPS, "96", observedEvents);
if (applyMinScoreThreshold)
SSDeepTestUtil.assertNoMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[3], observedEvents);
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformerTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformerTest.java
index fe52feb574..4699bdae6d 100644
--- a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformerTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformerTest.java
@@ -117,7 +117,7 @@ public void transformTest() {
assertEquals("96", field.getValueString());
break;
case "OVERLAP_SCORE":
- assertEquals("51", field.getValueString());
+ assertEquals("53", field.getValueString());
break;
case "OVERLAP_SSDEEP_NGRAMS":
assertEquals(EXPECTED_2_3_OVERLAPS, field.getValueString());
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/util/SSDeepTestUtil.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/util/SSDeepTestUtil.java
index d76bbfe680..c404c30d4c 100644
--- a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/util/SSDeepTestUtil.java
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/util/SSDeepTestUtil.java
@@ -45,8 +45,8 @@ public class SSDeepTestUtil {
public static final int BUCKET_ENCODING_BASE = BucketAccumuloKeyGenerator.DEFAULT_BUCKET_ENCODING_BASE;
public static final int BUCKET_ENCODING_LENGTH = BucketAccumuloKeyGenerator.DEFAULT_BUCKET_ENCODING_LENGTH;
- public static final String EXPECTED_2_2_OVERLAPS = "3072:/G8VYrs, 3072:02irbxz, 3072:123D6pI, 3072:23D6pIJ, 3072:2irbxzG, 3072:3D6pIJL, 3072:3dXVmbO, 3072:46Kjnz/, 3072:5u46Kjn, 3072:6Kjnz/G, 3072:6pIJLIO, 3072:7fKg3dX, 3072:8VYrs12, 3072:AFYDMxu, 3072:D6pIJLI, 3072:DMxud7f, 3072:FYDMxud, 3072:G8VYrs1, 3072:GAFYDMx, 3072:Kg3dXVm, 3072:Kjnz/G8, 3072:Mxud7fK, 3072:On5u46K, 3072:VYrs123, 3072:VmbOn5u, 3072:XVmbOn5, 3072:YDMxud7, 3072:Yrs123D, 3072:bOn5u46, 3072:bxzGAFY, 3072:d7fKg3d, 3072:dXVmbOn, 3072:fKg3dXV, 3072:g3dXVmb, 3072:irbxzGA, 3072:jnz/G8V, 3072:mbOn5u4, 3072:n5u46Kj, 3072:nz/G8VY, 3072:pIJLIOS, 3072:rbxzGAF, 3072:rs123D6, 3072:s123D6p, 3072:u46Kjnz, 3072:ud7fKg3, 3072:xud7fKg, 3072:xzGAFYD, 3072:z/G8VYr, 3072:zGAFYDM, 6144:02MKlWQ, 6144:2MKlWQ7, 6144:3d4bO96, 6144:4bO968r, 6144:7Sg3d4b, 6144:968rm7J, 6144:KlWQ7Sg, 6144:MKlWQ7S, 6144:O968rm7, 6144:Q7Sg3d4, 6144:Sg3d4bO, 6144:WQ7Sg3d, 6144:bO968rm, 6144:d4bO968, 6144:g3d4bO9, 6144:lWQ7Sg3";
- public static final String EXPECTED_2_3_OVERLAPS = "3072:/G8VYrs, 3072:123D6pI, 3072:23D6pIJ, 3072:3D6pIJL, 3072:3dXVmbO, 3072:46Kjnz/, 3072:5u46Kjn, 3072:6Kjnz/G, 3072:6pIJLIO, 3072:7fKg3dX, 3072:8VYrs12, 3072:AFYDMxu, 3072:D6pIJLI, 3072:DMxud7f, 3072:FYDMxud, 3072:G8VYrs1, 3072:GAFYDMx, 3072:Kg3dXVm, 3072:Kjnz/G8, 3072:Mxud7fK, 3072:On5u46K, 3072:VYrs123, 3072:VmbOn5u, 3072:XVmbOn5, 3072:YDMxud7, 3072:Yrs123D, 3072:bOn5u46, 3072:d7fKg3d, 3072:dXVmbOn, 3072:fKg3dXV, 3072:g3dXVmb, 3072:jnz/G8V, 3072:mbOn5u4, 3072:n5u46Kj, 3072:nz/G8VY, 3072:pIJLIOS, 3072:rs123D6, 3072:s123D6p, 3072:u46Kjnz, 3072:ud7fKg3, 3072:xud7fKg, 3072:z/G8VYr, 6144:3d4bO96, 6144:4bO968r, 6144:7Sg3d4b, 6144:968rm7J, 6144:O968rm7, 6144:Sg3d4bO, 6144:bO968rm, 6144:d4bO968, 6144:g3d4bO9";
+ public static final String EXPECTED_2_2_OVERLAPS = "3072:/G8VYrs, 3072:02irbxz, 3072:123D6pI, 3072:23D6pIJ, 3072:2irbxzG, 3072:3D6pIJL, 3072:3dXVmbO, 3072:46Kjnz/, 3072:5u46Kjn, 3072:6Kjnz/G, 3072:6pIJLIO, 3072:7fKg3dX, 3072:8VYrs12, 3072:AFYDMxu, 3072:D6pIJLI, 3072:DMxud7f, 3072:FYDMxud, 3072:G8VYrs1, 3072:GAFYDMx, 3072:IJLIOSP, 3072:Kg3dXVm, 3072:Kjnz/G8, 3072:Mxud7fK, 3072:On5u46K, 3072:VYrs123, 3072:VmbOn5u, 3072:XVmbOn5, 3072:YDMxud7, 3072:Yrs123D, 3072:bOn5u46, 3072:bxzGAFY, 3072:d7fKg3d, 3072:dXVmbOn, 3072:fKg3dXV, 3072:g3dXVmb, 3072:irbxzGA, 3072:jnz/G8V, 3072:mbOn5u4, 3072:n5u46Kj, 3072:nz/G8VY, 3072:pIJLIOS, 3072:rbxzGAF, 3072:rs123D6, 3072:s123D6p, 3072:u46Kjnz, 3072:ud7fKg3, 3072:xud7fKg, 3072:xzGAFYD, 3072:z/G8VYr, 3072:zGAFYDM, 6144:02MKlWQ, 6144:2MKlWQ7, 6144:3d4bO96, 6144:4bO968r, 6144:68rm7JO, 6144:7Sg3d4b, 6144:968rm7J, 6144:KlWQ7Sg, 6144:MKlWQ7S, 6144:O968rm7, 6144:Q7Sg3d4, 6144:Sg3d4bO, 6144:WQ7Sg3d, 6144:bO968rm, 6144:d4bO968, 6144:g3d4bO9, 6144:lWQ7Sg3";
+ public static final String EXPECTED_2_3_OVERLAPS = "3072:/G8VYrs, 3072:123D6pI, 3072:23D6pIJ, 3072:3D6pIJL, 3072:3dXVmbO, 3072:46Kjnz/, 3072:5u46Kjn, 3072:6Kjnz/G, 3072:6pIJLIO, 3072:7fKg3dX, 3072:8VYrs12, 3072:AFYDMxu, 3072:D6pIJLI, 3072:DMxud7f, 3072:FYDMxud, 3072:G8VYrs1, 3072:GAFYDMx, 3072:IJLIOSP, 3072:Kg3dXVm, 3072:Kjnz/G8, 3072:Mxud7fK, 3072:On5u46K, 3072:VYrs123, 3072:VmbOn5u, 3072:XVmbOn5, 3072:YDMxud7, 3072:Yrs123D, 3072:bOn5u46, 3072:d7fKg3d, 3072:dXVmbOn, 3072:fKg3dXV, 3072:g3dXVmb, 3072:jnz/G8V, 3072:mbOn5u4, 3072:n5u46Kj, 3072:nz/G8VY, 3072:pIJLIOS, 3072:rs123D6, 3072:s123D6p, 3072:u46Kjnz, 3072:ud7fKg3, 3072:xud7fKg, 3072:z/G8VYr, 6144:3d4bO96, 6144:4bO968r, 6144:68rm7JO, 6144:7Sg3d4b, 6144:968rm7J, 6144:O968rm7, 6144:Sg3d4bO, 6144:bO968rm, 6144:d4bO968, 6144:g3d4bO9";
public static final String EXPECTED_2_4_OVERLAPS = "3072:5u46Kjn, 3072:On5u46K, 3072:VmbOn5u, 3072:XVmbOn5, 3072:bOn5u46, 3072:dXVmbOn, 3072:mbOn5u4, 3072:n5u46Kj, 3072:u46Kjnz";
public static void loadSSDeepIndexTextData(AccumuloClient accumuloClient) throws Exception {
diff --git a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/NGramGenerator.java b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/NGramGenerator.java
index e8cb81f8b1..b90f5054c3 100644
--- a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/NGramGenerator.java
+++ b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/NGramGenerator.java
@@ -149,7 +149,7 @@ public Set generateNgrams(SSDeepHash ssDeepHash, int minHashSize) {
*/
public void generateNgrams(int chunkSize, String chunk, Set output) {
final int ngramCount = chunk.length() - ngramSize;
- for (int i = 0; i < ngramCount; i++) {
+ for (int i = 0; i <= ngramCount; i++) {
final String ngram = chunk.substring(i, i + ngramSize);
output.add(new NGramTuple(chunkSize, ngram));
}
diff --git a/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/NGramGeneratorTest.java b/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/NGramGeneratorTest.java
index ac522ac4b3..e6c048bc67 100644
--- a/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/NGramGeneratorTest.java
+++ b/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/NGramGeneratorTest.java
@@ -26,13 +26,13 @@ public class NGramGeneratorTest {
"12288:Hkwuliz", "12288:ikQ7lKH", "12288:5pGxlXT", "12288:ulizQ1l", "12288:lXTd8zb", "12288:beZHkwu", "12288:jeZHkwu", "12288:wulizQ1",
"12288:pGxlXTd", "12288:55pGxlX", "12288:GjeZHkw", "12288:1lK55pG", "12288:lKH5p5H", "12288:HkwuPik", "12288:kwulizQ", "12288:p5H9x1b",
"12288:eZHkwul", "12288:lizQ1lK", "12288:xlXTd8z", "12288:zQ1lK55", "12288:VG4GjeZ", "12288:7lKH5p5", "12288:kwuPikQ", "12288:2r/VG4G",
- "12288:GxlXTd8", "12288:KH5p5H9", "12288:9x1beZH"},
+ "12288:GxlXTd8", "12288:KH5p5H9", "12288:9x1beZH", "12288:XTd8zbW"},
{"6144:6e8o4to", "6144:6y7ppFj", "6144:2C3nq73", "6144:gC6e8o4", "6144:3NRW6/f", "6144:NRW6/ft", "6144:93NRW6/", "6144:TgC6e8o", "6144:3v1kHGh",
"6144:ZxoXb0n", "6144:6ZxoXb0", "6144:/ftZTgC", "6144:mk6ZxoX", "6144:73v1kHG", "6144:ppFj93N", "6144:nq73v1k", "6144:s6y7ppF",
"6144:Fj93NRW", "6144:tZTgC6e", "6144:toHZmk6", "6144:C6e8o4t", "6144:4toHZmk", "6144:pFj93NR", "6144:HZmk6Zx", "6144:k6ZxoXb",
"6144:HGhs6y7", "6144:ftZTgC6", "6144:j93NRW6", "6144:6/ftZTg", "6144:q73v1kH", "6144:1kHGhs6", "6144:y7ppFj9", "6144:Zmk6Zxo",
"6144:hs6y7pp", "6144:7ppFj93", "6144:C3nq73v", "6144:3nq73v1", "6144:v1kHGhs", "6144:ZTgC6e8", "6144:8o4toHZ", "6144:o4toHZm",
- "6144:oHZmk6Z", "6144:W6/ftZT", "6144:02C3nq7", "6144:RW6/ftZ", "6144:Ghs6y7p", "6144:kHGhs6y", "6144:e8o4toH"}};
+ "6144:oHZmk6Z", "6144:W6/ftZT", "6144:02C3nq7", "6144:RW6/ftZ", "6144:Ghs6y7p", "6144:kHGhs6y", "6144:e8o4toH", "6144:xoXb0ns"}};
@Test
public void testGenerateNGrams() {
diff --git a/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/SSDeepNGramOverlapScorerTest.java b/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/SSDeepNGramOverlapScorerTest.java
index f24030bf0c..a5e7cf9b3b 100644
--- a/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/SSDeepNGramOverlapScorerTest.java
+++ b/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/SSDeepNGramOverlapScorerTest.java
@@ -26,7 +26,7 @@ public class SSDeepNGramOverlapScorerTest {
};
- public static final int[] expectedScores = {65, 47, 16, 16, 0, 0, 0};
+ public static final int[] expectedScores = {67, 48, 17, 17, 0, 0, 0};
@Test
public void testCompare() {
From ba1308f015046efe053afa4e37397f6b469551be Mon Sep 17 00:00:00 2001
From: hgklohr
Date: Wed, 31 Jul 2024 13:14:28 +0000
Subject: [PATCH 019/128] 7.2.2
---
common-test/pom.xml | 2 +-
contrib/datawave-quickstart/docker/pom.xml | 2 +-
core/cached-results/pom.xml | 2 +-
core/common-util/pom.xml | 2 +-
core/common/pom.xml | 2 +-
core/connection-pool/pom.xml | 2 +-
core/map-reduce/pom.xml | 2 +-
core/modification/pom.xml | 2 +-
core/pom.xml | 2 +-
core/query/pom.xml | 2 +-
core/utils/pom.xml | 2 +-
docs/pom.xml | 2 +-
microservices/pom.xml | 2 +-
microservices/services/pom.xml | 2 +-
microservices/starters/pom.xml | 2 +-
pom.xml | 2 +-
warehouse/accumulo-extensions/pom.xml | 2 +-
warehouse/age-off-utils/pom.xml | 2 +-
warehouse/age-off/pom.xml | 2 +-
warehouse/assemble/datawave/pom.xml | 2 +-
warehouse/assemble/pom.xml | 2 +-
warehouse/assemble/webservice/pom.xml | 2 +-
warehouse/common/pom.xml | 2 +-
warehouse/core/pom.xml | 2 +-
warehouse/data-dictionary-core/pom.xml | 2 +-
warehouse/edge-dictionary-core/pom.xml | 2 +-
warehouse/edge-model-configuration-core/pom.xml | 2 +-
warehouse/index-stats/pom.xml | 2 +-
warehouse/ingest-configuration/pom.xml | 2 +-
warehouse/ingest-core/pom.xml | 2 +-
warehouse/ingest-csv/pom.xml | 2 +-
warehouse/ingest-json/pom.xml | 2 +-
warehouse/ingest-nyctlc/pom.xml | 2 +-
warehouse/ingest-scripts/pom.xml | 2 +-
warehouse/ingest-ssdeep/pom.xml | 2 +-
warehouse/ingest-wikipedia/pom.xml | 2 +-
warehouse/metrics-core/pom.xml | 2 +-
warehouse/ops-tools/config-compare/pom.xml | 2 +-
warehouse/ops-tools/index-validation/pom.xml | 2 +-
warehouse/ops-tools/pom.xml | 2 +-
warehouse/pom.xml | 2 +-
warehouse/query-core/pom.xml | 2 +-
warehouse/regression-testing/pom.xml | 2 +-
warehouse/ssdeep-common/pom.xml | 2 +-
web-services/accumulo/pom.xml | 2 +-
web-services/atom/pom.xml | 2 +-
web-services/cached-results/pom.xml | 2 +-
web-services/client/pom.xml | 2 +-
web-services/common-util/pom.xml | 2 +-
web-services/common/pom.xml | 2 +-
web-services/deploy/application/pom.xml | 2 +-
web-services/deploy/configuration/pom.xml | 2 +-
web-services/deploy/docs/pom.xml | 2 +-
web-services/deploy/pom.xml | 2 +-
web-services/deploy/spring-framework-integration/pom.xml | 2 +-
web-services/dictionary/pom.xml | 2 +-
web-services/examples/client-login/pom.xml | 2 +-
web-services/examples/http-client/pom.xml | 2 +-
web-services/examples/jms-client/pom.xml | 2 +-
web-services/examples/pom.xml | 2 +-
web-services/examples/query-war/pom.xml | 2 +-
web-services/map-reduce-embedded/pom.xml | 2 +-
web-services/map-reduce-status/pom.xml | 2 +-
web-services/map-reduce/pom.xml | 2 +-
web-services/metrics/pom.xml | 2 +-
web-services/model/pom.xml | 2 +-
web-services/modification/pom.xml | 2 +-
web-services/pom.xml | 2 +-
web-services/query-websocket/pom.xml | 2 +-
web-services/query/pom.xml | 2 +-
web-services/rest-api/pom.xml | 2 +-
web-services/security/pom.xml | 2 +-
web-services/web-root/pom.xml | 2 +-
73 files changed, 73 insertions(+), 73 deletions(-)
diff --git a/common-test/pom.xml b/common-test/pom.xml
index 5808bc29e8..86b014a869 100644
--- a/common-test/pom.xml
+++ b/common-test/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-common-test
${project.artifactId}
diff --git a/contrib/datawave-quickstart/docker/pom.xml b/contrib/datawave-quickstart/docker/pom.xml
index e912ef9f64..1d12c2c8d6 100644
--- a/contrib/datawave-quickstart/docker/pom.xml
+++ b/contrib/datawave-quickstart/docker/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
../../../pom.xml
quickstart
diff --git a/core/cached-results/pom.xml b/core/cached-results/pom.xml
index a3e4358e1f..576ec1c3d5 100644
--- a/core/cached-results/pom.xml
+++ b/core/cached-results/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-core-cached-results
${project.artifactId}
diff --git a/core/common-util/pom.xml b/core/common-util/pom.xml
index 03fa097359..55f0f8e90d 100644
--- a/core/common-util/pom.xml
+++ b/core/common-util/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-core-common-util
${project.artifactId}
diff --git a/core/common/pom.xml b/core/common/pom.xml
index 722746b3a6..65c0ea068c 100644
--- a/core/common/pom.xml
+++ b/core/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-core-common
${project.artifactId}
diff --git a/core/connection-pool/pom.xml b/core/connection-pool/pom.xml
index d638d0736d..84c8eceef7 100644
--- a/core/connection-pool/pom.xml
+++ b/core/connection-pool/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-core-connection-pool
${project.artifactId}
diff --git a/core/map-reduce/pom.xml b/core/map-reduce/pom.xml
index 6c52e19e63..f19c5b091b 100644
--- a/core/map-reduce/pom.xml
+++ b/core/map-reduce/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-core-map-reduce
${project.artifactId}
diff --git a/core/modification/pom.xml b/core/modification/pom.xml
index 35e11d185e..5f2d7f75b2 100644
--- a/core/modification/pom.xml
+++ b/core/modification/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-core-modification
${project.artifactId}
diff --git a/core/pom.xml b/core/pom.xml
index 2cc7d61d92..a92fa3f27f 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
gov.nsa.datawave.core
datawave-core-parent
diff --git a/core/query/pom.xml b/core/query/pom.xml
index b23f5ee1a1..096d7caed2 100644
--- a/core/query/pom.xml
+++ b/core/query/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-core-query
${project.artifactId}
diff --git a/core/utils/pom.xml b/core/utils/pom.xml
index e73ee09f4e..8ada17698a 100644
--- a/core/utils/pom.xml
+++ b/core/utils/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
gov.nsa.datawave.core
datawave-utils-parent
diff --git a/docs/pom.xml b/docs/pom.xml
index 3e949f7ab1..962ac3bcae 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-docs
diff --git a/microservices/pom.xml b/microservices/pom.xml
index e2cf524569..0cfa3eb97a 100644
--- a/microservices/pom.xml
+++ b/microservices/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
gov.nsa.datawave.microservice
datawave-microservice-build-parent
diff --git a/microservices/services/pom.xml b/microservices/services/pom.xml
index 9da1bdb930..d6b4f6ceb5 100644
--- a/microservices/services/pom.xml
+++ b/microservices/services/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.microservice
datawave-microservice-build-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-microservice-service-build-parent
pom
diff --git a/microservices/starters/pom.xml b/microservices/starters/pom.xml
index 89df413679..7806988ceb 100644
--- a/microservices/starters/pom.xml
+++ b/microservices/starters/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.microservice
datawave-microservice-build-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-microservice-starter-build-parent
pom
diff --git a/pom.xml b/pom.xml
index 5722397f60..bfd9ccfcee 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
4.0.0
gov.nsa.datawave
datawave-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
pom
DataWave
DataWave is a Java-based ingest and query framework that leverages Apache Accumulo to provide fast, secure access to your data.
diff --git a/warehouse/accumulo-extensions/pom.xml b/warehouse/accumulo-extensions/pom.xml
index e4512a7edd..6bd7724cc7 100644
--- a/warehouse/accumulo-extensions/pom.xml
+++ b/warehouse/accumulo-extensions/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-accumulo-extensions
${project.artifactId}
diff --git a/warehouse/age-off-utils/pom.xml b/warehouse/age-off-utils/pom.xml
index cb0cfeae7c..69cf890612 100644
--- a/warehouse/age-off-utils/pom.xml
+++ b/warehouse/age-off-utils/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-age-off-utils
${project.artifactId}
diff --git a/warehouse/age-off/pom.xml b/warehouse/age-off/pom.xml
index 5f5bd035a9..9dae9c475c 100644
--- a/warehouse/age-off/pom.xml
+++ b/warehouse/age-off/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-age-off
${project.artifactId}
diff --git a/warehouse/assemble/datawave/pom.xml b/warehouse/assemble/datawave/pom.xml
index 2aa3ba8354..79d2d30a7b 100644
--- a/warehouse/assemble/datawave/pom.xml
+++ b/warehouse/assemble/datawave/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
assemble-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
assemble-datawave
jar
diff --git a/warehouse/assemble/pom.xml b/warehouse/assemble/pom.xml
index 011b8ed37d..fe12b10afb 100644
--- a/warehouse/assemble/pom.xml
+++ b/warehouse/assemble/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
assemble-parent
pom
diff --git a/warehouse/assemble/webservice/pom.xml b/warehouse/assemble/webservice/pom.xml
index 8f277d1a92..3caa493e01 100644
--- a/warehouse/assemble/webservice/pom.xml
+++ b/warehouse/assemble/webservice/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
assemble-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
assemble-webservice
${project.artifactId}
diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml
index 31c8b5489d..64948a734e 100644
--- a/warehouse/common/pom.xml
+++ b/warehouse/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-common
${project.artifactId}
diff --git a/warehouse/core/pom.xml b/warehouse/core/pom.xml
index 145d835736..7b34269273 100644
--- a/warehouse/core/pom.xml
+++ b/warehouse/core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-core
jar
diff --git a/warehouse/data-dictionary-core/pom.xml b/warehouse/data-dictionary-core/pom.xml
index e11166aaba..f6dae4b3cf 100644
--- a/warehouse/data-dictionary-core/pom.xml
+++ b/warehouse/data-dictionary-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-data-dictionary-core
jar
diff --git a/warehouse/edge-dictionary-core/pom.xml b/warehouse/edge-dictionary-core/pom.xml
index 32c281ba8b..764664cfb6 100644
--- a/warehouse/edge-dictionary-core/pom.xml
+++ b/warehouse/edge-dictionary-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-edge-dictionary-core
jar
diff --git a/warehouse/edge-model-configuration-core/pom.xml b/warehouse/edge-model-configuration-core/pom.xml
index 6283291a0d..fc4427fa1c 100644
--- a/warehouse/edge-model-configuration-core/pom.xml
+++ b/warehouse/edge-model-configuration-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-edge-model-configuration-core
jar
diff --git a/warehouse/index-stats/pom.xml b/warehouse/index-stats/pom.xml
index 49114241e2..72ab999740 100644
--- a/warehouse/index-stats/pom.xml
+++ b/warehouse/index-stats/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-index-stats
jar
diff --git a/warehouse/ingest-configuration/pom.xml b/warehouse/ingest-configuration/pom.xml
index 87d60006ab..ae6e2f5e0b 100644
--- a/warehouse/ingest-configuration/pom.xml
+++ b/warehouse/ingest-configuration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ingest-configuration
diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml
index 3b718327aa..cf2748ba6a 100644
--- a/warehouse/ingest-core/pom.xml
+++ b/warehouse/ingest-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ingest-core
jar
diff --git a/warehouse/ingest-csv/pom.xml b/warehouse/ingest-csv/pom.xml
index 4794dc07b4..67844d4318 100644
--- a/warehouse/ingest-csv/pom.xml
+++ b/warehouse/ingest-csv/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ingest-csv
jar
diff --git a/warehouse/ingest-json/pom.xml b/warehouse/ingest-json/pom.xml
index 897bf059af..5d509600c0 100644
--- a/warehouse/ingest-json/pom.xml
+++ b/warehouse/ingest-json/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ingest-json
jar
diff --git a/warehouse/ingest-nyctlc/pom.xml b/warehouse/ingest-nyctlc/pom.xml
index 4e32dec0f1..1c98fd7eef 100644
--- a/warehouse/ingest-nyctlc/pom.xml
+++ b/warehouse/ingest-nyctlc/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ingest-nyctlc
jar
diff --git a/warehouse/ingest-scripts/pom.xml b/warehouse/ingest-scripts/pom.xml
index e497384c7b..e85d717577 100644
--- a/warehouse/ingest-scripts/pom.xml
+++ b/warehouse/ingest-scripts/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ingest-scripts
${project.artifactId}
diff --git a/warehouse/ingest-ssdeep/pom.xml b/warehouse/ingest-ssdeep/pom.xml
index d97febe595..7a37d69a41 100644
--- a/warehouse/ingest-ssdeep/pom.xml
+++ b/warehouse/ingest-ssdeep/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ingest-ssdeep
diff --git a/warehouse/ingest-wikipedia/pom.xml b/warehouse/ingest-wikipedia/pom.xml
index 9fd8c6c83a..3d76a0872b 100644
--- a/warehouse/ingest-wikipedia/pom.xml
+++ b/warehouse/ingest-wikipedia/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ingest-wikipedia
jar
diff --git a/warehouse/metrics-core/pom.xml b/warehouse/metrics-core/pom.xml
index 64663d270b..e9d7f7d8c1 100644
--- a/warehouse/metrics-core/pom.xml
+++ b/warehouse/metrics-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-metrics-core
jar
diff --git a/warehouse/ops-tools/config-compare/pom.xml b/warehouse/ops-tools/config-compare/pom.xml
index 2afc774182..822ac7c6f3 100644
--- a/warehouse/ops-tools/config-compare/pom.xml
+++ b/warehouse/ops-tools/config-compare/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-ops-tools-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ops-tools-config-compare
diff --git a/warehouse/ops-tools/index-validation/pom.xml b/warehouse/ops-tools/index-validation/pom.xml
index d9b21a01f5..6a9a2ed1e0 100644
--- a/warehouse/ops-tools/index-validation/pom.xml
+++ b/warehouse/ops-tools/index-validation/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-ops-tools-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ops-tools-index-validation
jar
diff --git a/warehouse/ops-tools/pom.xml b/warehouse/ops-tools/pom.xml
index 0982684cb3..f25ff16190 100644
--- a/warehouse/ops-tools/pom.xml
+++ b/warehouse/ops-tools/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ops-tools-parent
pom
diff --git a/warehouse/pom.xml b/warehouse/pom.xml
index 9b5227fafb..f01fe0eeb4 100644
--- a/warehouse/pom.xml
+++ b/warehouse/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-warehouse-parent
pom
diff --git a/warehouse/query-core/pom.xml b/warehouse/query-core/pom.xml
index e3041805d0..d576e0fa56 100644
--- a/warehouse/query-core/pom.xml
+++ b/warehouse/query-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-query-core
jar
diff --git a/warehouse/regression-testing/pom.xml b/warehouse/regression-testing/pom.xml
index c6437ec7c8..88898239b6 100644
--- a/warehouse/regression-testing/pom.xml
+++ b/warehouse/regression-testing/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-regression-testing
${project.artifactId}
diff --git a/warehouse/ssdeep-common/pom.xml b/warehouse/ssdeep-common/pom.xml
index 8d28628ae0..14e0a53c93 100644
--- a/warehouse/ssdeep-common/pom.xml
+++ b/warehouse/ssdeep-common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ssdeep-common
diff --git a/web-services/accumulo/pom.xml b/web-services/accumulo/pom.xml
index c236c5d244..3d0c6f708f 100644
--- a/web-services/accumulo/pom.xml
+++ b/web-services/accumulo/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-accumulo
ejb
diff --git a/web-services/atom/pom.xml b/web-services/atom/pom.xml
index c6bb6f3b78..2e706b78b8 100644
--- a/web-services/atom/pom.xml
+++ b/web-services/atom/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-atom
ejb
diff --git a/web-services/cached-results/pom.xml b/web-services/cached-results/pom.xml
index 3029f32d97..7427af3b4d 100644
--- a/web-services/cached-results/pom.xml
+++ b/web-services/cached-results/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-cached-results
ejb
diff --git a/web-services/client/pom.xml b/web-services/client/pom.xml
index 5119b59039..7750c9e947 100644
--- a/web-services/client/pom.xml
+++ b/web-services/client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-client
jar
diff --git a/web-services/common-util/pom.xml b/web-services/common-util/pom.xml
index 3bfe3f5048..313a162386 100644
--- a/web-services/common-util/pom.xml
+++ b/web-services/common-util/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-common-util
jar
diff --git a/web-services/common/pom.xml b/web-services/common/pom.xml
index df0907daa6..2033354421 100644
--- a/web-services/common/pom.xml
+++ b/web-services/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-common
ejb
diff --git a/web-services/deploy/application/pom.xml b/web-services/deploy/application/pom.xml
index b72af15935..d8611c1f3a 100644
--- a/web-services/deploy/application/pom.xml
+++ b/web-services/deploy/application/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-deploy-application
ear
diff --git a/web-services/deploy/configuration/pom.xml b/web-services/deploy/configuration/pom.xml
index 6abbdfafad..17bc2c2737 100644
--- a/web-services/deploy/configuration/pom.xml
+++ b/web-services/deploy/configuration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-deploy-configuration
jar
diff --git a/web-services/deploy/docs/pom.xml b/web-services/deploy/docs/pom.xml
index 7444f1fdc6..7e2f02256d 100644
--- a/web-services/deploy/docs/pom.xml
+++ b/web-services/deploy/docs/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-deploy-docs
war
diff --git a/web-services/deploy/pom.xml b/web-services/deploy/pom.xml
index 3089b64538..1abbfcbf23 100644
--- a/web-services/deploy/pom.xml
+++ b/web-services/deploy/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
diff --git a/web-services/deploy/spring-framework-integration/pom.xml b/web-services/deploy/spring-framework-integration/pom.xml
index 1f9b3b0189..3251ac43fc 100644
--- a/web-services/deploy/spring-framework-integration/pom.xml
+++ b/web-services/deploy/spring-framework-integration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
spring-framework-integration
${project.artifactId}
diff --git a/web-services/dictionary/pom.xml b/web-services/dictionary/pom.xml
index 16889aea52..73c351fd5b 100644
--- a/web-services/dictionary/pom.xml
+++ b/web-services/dictionary/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-dictionary
ejb
diff --git a/web-services/examples/client-login/pom.xml b/web-services/examples/client-login/pom.xml
index 30c6844591..aa07548004 100644
--- a/web-services/examples/client-login/pom.xml
+++ b/web-services/examples/client-login/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-examples-client-login
ejb
diff --git a/web-services/examples/http-client/pom.xml b/web-services/examples/http-client/pom.xml
index 88b1b50709..fa977c4850 100644
--- a/web-services/examples/http-client/pom.xml
+++ b/web-services/examples/http-client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-examples-http-client
jar
diff --git a/web-services/examples/jms-client/pom.xml b/web-services/examples/jms-client/pom.xml
index 0b028c5b60..3df5ccf4d7 100644
--- a/web-services/examples/jms-client/pom.xml
+++ b/web-services/examples/jms-client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-examples-jms-client
jar
diff --git a/web-services/examples/pom.xml b/web-services/examples/pom.xml
index 62ed24727f..09a66b6a7b 100644
--- a/web-services/examples/pom.xml
+++ b/web-services/examples/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-examples-parent
pom
diff --git a/web-services/examples/query-war/pom.xml b/web-services/examples/query-war/pom.xml
index 454d262734..a1fe599574 100644
--- a/web-services/examples/query-war/pom.xml
+++ b/web-services/examples/query-war/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-examples-query-war
war
diff --git a/web-services/map-reduce-embedded/pom.xml b/web-services/map-reduce-embedded/pom.xml
index a51d6c94e9..98814949e1 100644
--- a/web-services/map-reduce-embedded/pom.xml
+++ b/web-services/map-reduce-embedded/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-map-reduce-embedded
jar
diff --git a/web-services/map-reduce-status/pom.xml b/web-services/map-reduce-status/pom.xml
index 39ec9bb55c..e0a1017b7f 100644
--- a/web-services/map-reduce-status/pom.xml
+++ b/web-services/map-reduce-status/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-map-reduce-status
ejb
diff --git a/web-services/map-reduce/pom.xml b/web-services/map-reduce/pom.xml
index 489b76630d..fe20ed1e73 100644
--- a/web-services/map-reduce/pom.xml
+++ b/web-services/map-reduce/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-map-reduce
ejb
diff --git a/web-services/metrics/pom.xml b/web-services/metrics/pom.xml
index afc78a7c45..a5f25ac0f5 100644
--- a/web-services/metrics/pom.xml
+++ b/web-services/metrics/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-metrics
ejb
diff --git a/web-services/model/pom.xml b/web-services/model/pom.xml
index 45962c7b65..fed9d161a5 100644
--- a/web-services/model/pom.xml
+++ b/web-services/model/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-model
ejb
diff --git a/web-services/modification/pom.xml b/web-services/modification/pom.xml
index 9015d9f0f2..95e8c2617e 100644
--- a/web-services/modification/pom.xml
+++ b/web-services/modification/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-modification
ejb
diff --git a/web-services/pom.xml b/web-services/pom.xml
index a96d7f3dbe..798c9bbcc0 100644
--- a/web-services/pom.xml
+++ b/web-services/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
gov.nsa.datawave.webservices
datawave-ws-parent
diff --git a/web-services/query-websocket/pom.xml b/web-services/query-websocket/pom.xml
index f30e4f6bf2..d554169908 100644
--- a/web-services/query-websocket/pom.xml
+++ b/web-services/query-websocket/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-query-websocket
war
diff --git a/web-services/query/pom.xml b/web-services/query/pom.xml
index 7d1cc26c3f..95666ec2bd 100644
--- a/web-services/query/pom.xml
+++ b/web-services/query/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-query
ejb
diff --git a/web-services/rest-api/pom.xml b/web-services/rest-api/pom.xml
index 9a7258e30f..29868991a0 100644
--- a/web-services/rest-api/pom.xml
+++ b/web-services/rest-api/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-rest-api
war
diff --git a/web-services/security/pom.xml b/web-services/security/pom.xml
index de4c443431..1fddfef5a9 100644
--- a/web-services/security/pom.xml
+++ b/web-services/security/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-security
ejb
diff --git a/web-services/web-root/pom.xml b/web-services/web-root/pom.xml
index a6a031a09c..3e00ce4e2b 100644
--- a/web-services/web-root/pom.xml
+++ b/web-services/web-root/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2-SNAPSHOT
+ 7.2.2
datawave-ws-web-root
war
From 01ebb7b9d7af3291e2f2319ebca214794d83bc7e Mon Sep 17 00:00:00 2001
From: hgklohr
Date: Wed, 31 Jul 2024 13:17:30 +0000
Subject: [PATCH 020/128] 7.2.3-SNAPSHOT
---
common-test/pom.xml | 2 +-
contrib/datawave-quickstart/docker/pom.xml | 2 +-
core/cached-results/pom.xml | 2 +-
core/common-util/pom.xml | 2 +-
core/common/pom.xml | 2 +-
core/connection-pool/pom.xml | 2 +-
core/map-reduce/pom.xml | 2 +-
core/modification/pom.xml | 2 +-
core/pom.xml | 2 +-
core/query/pom.xml | 2 +-
core/utils/pom.xml | 2 +-
docs/pom.xml | 2 +-
microservices/pom.xml | 2 +-
microservices/services/pom.xml | 2 +-
microservices/starters/pom.xml | 2 +-
pom.xml | 2 +-
warehouse/accumulo-extensions/pom.xml | 2 +-
warehouse/age-off-utils/pom.xml | 2 +-
warehouse/age-off/pom.xml | 2 +-
warehouse/assemble/datawave/pom.xml | 2 +-
warehouse/assemble/pom.xml | 2 +-
warehouse/assemble/webservice/pom.xml | 2 +-
warehouse/common/pom.xml | 2 +-
warehouse/core/pom.xml | 2 +-
warehouse/data-dictionary-core/pom.xml | 2 +-
warehouse/edge-dictionary-core/pom.xml | 2 +-
warehouse/edge-model-configuration-core/pom.xml | 2 +-
warehouse/index-stats/pom.xml | 2 +-
warehouse/ingest-configuration/pom.xml | 2 +-
warehouse/ingest-core/pom.xml | 2 +-
warehouse/ingest-csv/pom.xml | 2 +-
warehouse/ingest-json/pom.xml | 2 +-
warehouse/ingest-nyctlc/pom.xml | 2 +-
warehouse/ingest-scripts/pom.xml | 2 +-
warehouse/ingest-ssdeep/pom.xml | 2 +-
warehouse/ingest-wikipedia/pom.xml | 2 +-
warehouse/metrics-core/pom.xml | 2 +-
warehouse/ops-tools/config-compare/pom.xml | 2 +-
warehouse/ops-tools/index-validation/pom.xml | 2 +-
warehouse/ops-tools/pom.xml | 2 +-
warehouse/pom.xml | 2 +-
warehouse/query-core/pom.xml | 2 +-
warehouse/regression-testing/pom.xml | 2 +-
warehouse/ssdeep-common/pom.xml | 2 +-
web-services/accumulo/pom.xml | 2 +-
web-services/atom/pom.xml | 2 +-
web-services/cached-results/pom.xml | 2 +-
web-services/client/pom.xml | 2 +-
web-services/common-util/pom.xml | 2 +-
web-services/common/pom.xml | 2 +-
web-services/deploy/application/pom.xml | 2 +-
web-services/deploy/configuration/pom.xml | 2 +-
web-services/deploy/docs/pom.xml | 2 +-
web-services/deploy/pom.xml | 2 +-
web-services/deploy/spring-framework-integration/pom.xml | 2 +-
web-services/dictionary/pom.xml | 2 +-
web-services/examples/client-login/pom.xml | 2 +-
web-services/examples/http-client/pom.xml | 2 +-
web-services/examples/jms-client/pom.xml | 2 +-
web-services/examples/pom.xml | 2 +-
web-services/examples/query-war/pom.xml | 2 +-
web-services/map-reduce-embedded/pom.xml | 2 +-
web-services/map-reduce-status/pom.xml | 2 +-
web-services/map-reduce/pom.xml | 2 +-
web-services/metrics/pom.xml | 2 +-
web-services/model/pom.xml | 2 +-
web-services/modification/pom.xml | 2 +-
web-services/pom.xml | 2 +-
web-services/query-websocket/pom.xml | 2 +-
web-services/query/pom.xml | 2 +-
web-services/rest-api/pom.xml | 2 +-
web-services/security/pom.xml | 2 +-
web-services/web-root/pom.xml | 2 +-
73 files changed, 73 insertions(+), 73 deletions(-)
diff --git a/common-test/pom.xml b/common-test/pom.xml
index 86b014a869..dac088d32c 100644
--- a/common-test/pom.xml
+++ b/common-test/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-common-test
${project.artifactId}
diff --git a/contrib/datawave-quickstart/docker/pom.xml b/contrib/datawave-quickstart/docker/pom.xml
index 1d12c2c8d6..c2d06d4080 100644
--- a/contrib/datawave-quickstart/docker/pom.xml
+++ b/contrib/datawave-quickstart/docker/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
../../../pom.xml
quickstart
diff --git a/core/cached-results/pom.xml b/core/cached-results/pom.xml
index 576ec1c3d5..514cdfa6ca 100644
--- a/core/cached-results/pom.xml
+++ b/core/cached-results/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-core-cached-results
${project.artifactId}
diff --git a/core/common-util/pom.xml b/core/common-util/pom.xml
index 55f0f8e90d..53e61e7027 100644
--- a/core/common-util/pom.xml
+++ b/core/common-util/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-core-common-util
${project.artifactId}
diff --git a/core/common/pom.xml b/core/common/pom.xml
index 65c0ea068c..4cf5c839b6 100644
--- a/core/common/pom.xml
+++ b/core/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-core-common
${project.artifactId}
diff --git a/core/connection-pool/pom.xml b/core/connection-pool/pom.xml
index 84c8eceef7..9bc87fa28b 100644
--- a/core/connection-pool/pom.xml
+++ b/core/connection-pool/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-core-connection-pool
${project.artifactId}
diff --git a/core/map-reduce/pom.xml b/core/map-reduce/pom.xml
index f19c5b091b..719809096d 100644
--- a/core/map-reduce/pom.xml
+++ b/core/map-reduce/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-core-map-reduce
${project.artifactId}
diff --git a/core/modification/pom.xml b/core/modification/pom.xml
index 5f2d7f75b2..05eb50662a 100644
--- a/core/modification/pom.xml
+++ b/core/modification/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-core-modification
${project.artifactId}
diff --git a/core/pom.xml b/core/pom.xml
index a92fa3f27f..262dd7ac35 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
gov.nsa.datawave.core
datawave-core-parent
diff --git a/core/query/pom.xml b/core/query/pom.xml
index 096d7caed2..007d1c5bb3 100644
--- a/core/query/pom.xml
+++ b/core/query/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-core-query
${project.artifactId}
diff --git a/core/utils/pom.xml b/core/utils/pom.xml
index 8ada17698a..0158d90cf8 100644
--- a/core/utils/pom.xml
+++ b/core/utils/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.core
datawave-core-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
gov.nsa.datawave.core
datawave-utils-parent
diff --git a/docs/pom.xml b/docs/pom.xml
index 962ac3bcae..cf3b9eb9de 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-docs
diff --git a/microservices/pom.xml b/microservices/pom.xml
index 0cfa3eb97a..676227df63 100644
--- a/microservices/pom.xml
+++ b/microservices/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
gov.nsa.datawave.microservice
datawave-microservice-build-parent
diff --git a/microservices/services/pom.xml b/microservices/services/pom.xml
index d6b4f6ceb5..631743293a 100644
--- a/microservices/services/pom.xml
+++ b/microservices/services/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.microservice
datawave-microservice-build-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-microservice-service-build-parent
pom
diff --git a/microservices/starters/pom.xml b/microservices/starters/pom.xml
index 7806988ceb..84acded3bf 100644
--- a/microservices/starters/pom.xml
+++ b/microservices/starters/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.microservice
datawave-microservice-build-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-microservice-starter-build-parent
pom
diff --git a/pom.xml b/pom.xml
index bfd9ccfcee..98123d1a7d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
4.0.0
gov.nsa.datawave
datawave-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
pom
DataWave
DataWave is a Java-based ingest and query framework that leverages Apache Accumulo to provide fast, secure access to your data.
diff --git a/warehouse/accumulo-extensions/pom.xml b/warehouse/accumulo-extensions/pom.xml
index 6bd7724cc7..24eb4fb45a 100644
--- a/warehouse/accumulo-extensions/pom.xml
+++ b/warehouse/accumulo-extensions/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-accumulo-extensions
${project.artifactId}
diff --git a/warehouse/age-off-utils/pom.xml b/warehouse/age-off-utils/pom.xml
index 69cf890612..d30b3f52c6 100644
--- a/warehouse/age-off-utils/pom.xml
+++ b/warehouse/age-off-utils/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-age-off-utils
${project.artifactId}
diff --git a/warehouse/age-off/pom.xml b/warehouse/age-off/pom.xml
index 9dae9c475c..da5efda69d 100644
--- a/warehouse/age-off/pom.xml
+++ b/warehouse/age-off/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-age-off
${project.artifactId}
diff --git a/warehouse/assemble/datawave/pom.xml b/warehouse/assemble/datawave/pom.xml
index 79d2d30a7b..ca3d5c239f 100644
--- a/warehouse/assemble/datawave/pom.xml
+++ b/warehouse/assemble/datawave/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
assemble-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
assemble-datawave
jar
diff --git a/warehouse/assemble/pom.xml b/warehouse/assemble/pom.xml
index fe12b10afb..0a775eb011 100644
--- a/warehouse/assemble/pom.xml
+++ b/warehouse/assemble/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
assemble-parent
pom
diff --git a/warehouse/assemble/webservice/pom.xml b/warehouse/assemble/webservice/pom.xml
index 3caa493e01..b90d0a651f 100644
--- a/warehouse/assemble/webservice/pom.xml
+++ b/warehouse/assemble/webservice/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
assemble-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
assemble-webservice
${project.artifactId}
diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml
index 64948a734e..5d385032bd 100644
--- a/warehouse/common/pom.xml
+++ b/warehouse/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-common
${project.artifactId}
diff --git a/warehouse/core/pom.xml b/warehouse/core/pom.xml
index 7b34269273..e5f96be556 100644
--- a/warehouse/core/pom.xml
+++ b/warehouse/core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-core
jar
diff --git a/warehouse/data-dictionary-core/pom.xml b/warehouse/data-dictionary-core/pom.xml
index f6dae4b3cf..49617991a2 100644
--- a/warehouse/data-dictionary-core/pom.xml
+++ b/warehouse/data-dictionary-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-data-dictionary-core
jar
diff --git a/warehouse/edge-dictionary-core/pom.xml b/warehouse/edge-dictionary-core/pom.xml
index 764664cfb6..7f925f0a13 100644
--- a/warehouse/edge-dictionary-core/pom.xml
+++ b/warehouse/edge-dictionary-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-edge-dictionary-core
jar
diff --git a/warehouse/edge-model-configuration-core/pom.xml b/warehouse/edge-model-configuration-core/pom.xml
index fc4427fa1c..9f8cfa9cf8 100644
--- a/warehouse/edge-model-configuration-core/pom.xml
+++ b/warehouse/edge-model-configuration-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-edge-model-configuration-core
jar
diff --git a/warehouse/index-stats/pom.xml b/warehouse/index-stats/pom.xml
index 72ab999740..7e87516fb8 100644
--- a/warehouse/index-stats/pom.xml
+++ b/warehouse/index-stats/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-index-stats
jar
diff --git a/warehouse/ingest-configuration/pom.xml b/warehouse/ingest-configuration/pom.xml
index ae6e2f5e0b..f403bbd809 100644
--- a/warehouse/ingest-configuration/pom.xml
+++ b/warehouse/ingest-configuration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ingest-configuration
diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml
index cf2748ba6a..8d9b89576a 100644
--- a/warehouse/ingest-core/pom.xml
+++ b/warehouse/ingest-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ingest-core
jar
diff --git a/warehouse/ingest-csv/pom.xml b/warehouse/ingest-csv/pom.xml
index 67844d4318..71d151af95 100644
--- a/warehouse/ingest-csv/pom.xml
+++ b/warehouse/ingest-csv/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ingest-csv
jar
diff --git a/warehouse/ingest-json/pom.xml b/warehouse/ingest-json/pom.xml
index 5d509600c0..9d65db97bf 100644
--- a/warehouse/ingest-json/pom.xml
+++ b/warehouse/ingest-json/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ingest-json
jar
diff --git a/warehouse/ingest-nyctlc/pom.xml b/warehouse/ingest-nyctlc/pom.xml
index 1c98fd7eef..403cc48626 100644
--- a/warehouse/ingest-nyctlc/pom.xml
+++ b/warehouse/ingest-nyctlc/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ingest-nyctlc
jar
diff --git a/warehouse/ingest-scripts/pom.xml b/warehouse/ingest-scripts/pom.xml
index e85d717577..54f1e14e8d 100644
--- a/warehouse/ingest-scripts/pom.xml
+++ b/warehouse/ingest-scripts/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ingest-scripts
${project.artifactId}
diff --git a/warehouse/ingest-ssdeep/pom.xml b/warehouse/ingest-ssdeep/pom.xml
index 7a37d69a41..bbd419091c 100644
--- a/warehouse/ingest-ssdeep/pom.xml
+++ b/warehouse/ingest-ssdeep/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ingest-ssdeep
diff --git a/warehouse/ingest-wikipedia/pom.xml b/warehouse/ingest-wikipedia/pom.xml
index 3d76a0872b..76b64b9c63 100644
--- a/warehouse/ingest-wikipedia/pom.xml
+++ b/warehouse/ingest-wikipedia/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ingest-wikipedia
jar
diff --git a/warehouse/metrics-core/pom.xml b/warehouse/metrics-core/pom.xml
index e9d7f7d8c1..8343cb1d43 100644
--- a/warehouse/metrics-core/pom.xml
+++ b/warehouse/metrics-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-metrics-core
jar
diff --git a/warehouse/ops-tools/config-compare/pom.xml b/warehouse/ops-tools/config-compare/pom.xml
index 822ac7c6f3..7224dcbd67 100644
--- a/warehouse/ops-tools/config-compare/pom.xml
+++ b/warehouse/ops-tools/config-compare/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-ops-tools-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ops-tools-config-compare
diff --git a/warehouse/ops-tools/index-validation/pom.xml b/warehouse/ops-tools/index-validation/pom.xml
index 6a9a2ed1e0..6288208d08 100644
--- a/warehouse/ops-tools/index-validation/pom.xml
+++ b/warehouse/ops-tools/index-validation/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-ops-tools-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ops-tools-index-validation
jar
diff --git a/warehouse/ops-tools/pom.xml b/warehouse/ops-tools/pom.xml
index f25ff16190..fb87f017c9 100644
--- a/warehouse/ops-tools/pom.xml
+++ b/warehouse/ops-tools/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ops-tools-parent
pom
diff --git a/warehouse/pom.xml b/warehouse/pom.xml
index f01fe0eeb4..3282793696 100644
--- a/warehouse/pom.xml
+++ b/warehouse/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-warehouse-parent
pom
diff --git a/warehouse/query-core/pom.xml b/warehouse/query-core/pom.xml
index d576e0fa56..147354cb35 100644
--- a/warehouse/query-core/pom.xml
+++ b/warehouse/query-core/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-query-core
jar
diff --git a/warehouse/regression-testing/pom.xml b/warehouse/regression-testing/pom.xml
index 88898239b6..941d4f179d 100644
--- a/warehouse/regression-testing/pom.xml
+++ b/warehouse/regression-testing/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-regression-testing
${project.artifactId}
diff --git a/warehouse/ssdeep-common/pom.xml b/warehouse/ssdeep-common/pom.xml
index 14e0a53c93..f51e2cd4e0 100644
--- a/warehouse/ssdeep-common/pom.xml
+++ b/warehouse/ssdeep-common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-warehouse-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ssdeep-common
diff --git a/web-services/accumulo/pom.xml b/web-services/accumulo/pom.xml
index 3d0c6f708f..a86c16397d 100644
--- a/web-services/accumulo/pom.xml
+++ b/web-services/accumulo/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-accumulo
ejb
diff --git a/web-services/atom/pom.xml b/web-services/atom/pom.xml
index 2e706b78b8..ff1b5b365b 100644
--- a/web-services/atom/pom.xml
+++ b/web-services/atom/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-atom
ejb
diff --git a/web-services/cached-results/pom.xml b/web-services/cached-results/pom.xml
index 7427af3b4d..958abc7c46 100644
--- a/web-services/cached-results/pom.xml
+++ b/web-services/cached-results/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-cached-results
ejb
diff --git a/web-services/client/pom.xml b/web-services/client/pom.xml
index 7750c9e947..5bab90d2fe 100644
--- a/web-services/client/pom.xml
+++ b/web-services/client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-client
jar
diff --git a/web-services/common-util/pom.xml b/web-services/common-util/pom.xml
index 313a162386..3c626e899d 100644
--- a/web-services/common-util/pom.xml
+++ b/web-services/common-util/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-common-util
jar
diff --git a/web-services/common/pom.xml b/web-services/common/pom.xml
index 2033354421..580841f339 100644
--- a/web-services/common/pom.xml
+++ b/web-services/common/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-common
ejb
diff --git a/web-services/deploy/application/pom.xml b/web-services/deploy/application/pom.xml
index d8611c1f3a..3c2ca446dc 100644
--- a/web-services/deploy/application/pom.xml
+++ b/web-services/deploy/application/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-deploy-application
ear
diff --git a/web-services/deploy/configuration/pom.xml b/web-services/deploy/configuration/pom.xml
index 17bc2c2737..6fdc145228 100644
--- a/web-services/deploy/configuration/pom.xml
+++ b/web-services/deploy/configuration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-deploy-configuration
jar
diff --git a/web-services/deploy/docs/pom.xml b/web-services/deploy/docs/pom.xml
index 7e2f02256d..acbce9d30b 100644
--- a/web-services/deploy/docs/pom.xml
+++ b/web-services/deploy/docs/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-deploy-docs
war
diff --git a/web-services/deploy/pom.xml b/web-services/deploy/pom.xml
index 1abbfcbf23..4f18ef9f28 100644
--- a/web-services/deploy/pom.xml
+++ b/web-services/deploy/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
diff --git a/web-services/deploy/spring-framework-integration/pom.xml b/web-services/deploy/spring-framework-integration/pom.xml
index 3251ac43fc..8c542f893f 100644
--- a/web-services/deploy/spring-framework-integration/pom.xml
+++ b/web-services/deploy/spring-framework-integration/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-deploy-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
spring-framework-integration
${project.artifactId}
diff --git a/web-services/dictionary/pom.xml b/web-services/dictionary/pom.xml
index 73c351fd5b..80d36c4fba 100644
--- a/web-services/dictionary/pom.xml
+++ b/web-services/dictionary/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-dictionary
ejb
diff --git a/web-services/examples/client-login/pom.xml b/web-services/examples/client-login/pom.xml
index aa07548004..dbf5beccab 100644
--- a/web-services/examples/client-login/pom.xml
+++ b/web-services/examples/client-login/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-examples-client-login
ejb
diff --git a/web-services/examples/http-client/pom.xml b/web-services/examples/http-client/pom.xml
index fa977c4850..3d05c207ea 100644
--- a/web-services/examples/http-client/pom.xml
+++ b/web-services/examples/http-client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-examples-http-client
jar
diff --git a/web-services/examples/jms-client/pom.xml b/web-services/examples/jms-client/pom.xml
index 3df5ccf4d7..3a3a0af466 100644
--- a/web-services/examples/jms-client/pom.xml
+++ b/web-services/examples/jms-client/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-examples-jms-client
jar
diff --git a/web-services/examples/pom.xml b/web-services/examples/pom.xml
index 09a66b6a7b..cd76ff2492 100644
--- a/web-services/examples/pom.xml
+++ b/web-services/examples/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-examples-parent
pom
diff --git a/web-services/examples/query-war/pom.xml b/web-services/examples/query-war/pom.xml
index a1fe599574..935e75ff4e 100644
--- a/web-services/examples/query-war/pom.xml
+++ b/web-services/examples/query-war/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-examples-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-examples-query-war
war
diff --git a/web-services/map-reduce-embedded/pom.xml b/web-services/map-reduce-embedded/pom.xml
index 98814949e1..0e34f2ecd4 100644
--- a/web-services/map-reduce-embedded/pom.xml
+++ b/web-services/map-reduce-embedded/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-map-reduce-embedded
jar
diff --git a/web-services/map-reduce-status/pom.xml b/web-services/map-reduce-status/pom.xml
index e0a1017b7f..850dff4f48 100644
--- a/web-services/map-reduce-status/pom.xml
+++ b/web-services/map-reduce-status/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-map-reduce-status
ejb
diff --git a/web-services/map-reduce/pom.xml b/web-services/map-reduce/pom.xml
index fe20ed1e73..7b623e9913 100644
--- a/web-services/map-reduce/pom.xml
+++ b/web-services/map-reduce/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-map-reduce
ejb
diff --git a/web-services/metrics/pom.xml b/web-services/metrics/pom.xml
index a5f25ac0f5..eddcbfeee7 100644
--- a/web-services/metrics/pom.xml
+++ b/web-services/metrics/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-metrics
ejb
diff --git a/web-services/model/pom.xml b/web-services/model/pom.xml
index fed9d161a5..c44d563458 100644
--- a/web-services/model/pom.xml
+++ b/web-services/model/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-model
ejb
diff --git a/web-services/modification/pom.xml b/web-services/modification/pom.xml
index 95e8c2617e..5d7b8a5b9f 100644
--- a/web-services/modification/pom.xml
+++ b/web-services/modification/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-modification
ejb
diff --git a/web-services/pom.xml b/web-services/pom.xml
index 798c9bbcc0..5099c27a80 100644
--- a/web-services/pom.xml
+++ b/web-services/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave
datawave-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
gov.nsa.datawave.webservices
datawave-ws-parent
diff --git a/web-services/query-websocket/pom.xml b/web-services/query-websocket/pom.xml
index d554169908..fb591e510b 100644
--- a/web-services/query-websocket/pom.xml
+++ b/web-services/query-websocket/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-query-websocket
war
diff --git a/web-services/query/pom.xml b/web-services/query/pom.xml
index 95666ec2bd..317ed98542 100644
--- a/web-services/query/pom.xml
+++ b/web-services/query/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-query
ejb
diff --git a/web-services/rest-api/pom.xml b/web-services/rest-api/pom.xml
index 29868991a0..dd630619b7 100644
--- a/web-services/rest-api/pom.xml
+++ b/web-services/rest-api/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-rest-api
war
diff --git a/web-services/security/pom.xml b/web-services/security/pom.xml
index 1fddfef5a9..6469cd7a49 100644
--- a/web-services/security/pom.xml
+++ b/web-services/security/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-security
ejb
diff --git a/web-services/web-root/pom.xml b/web-services/web-root/pom.xml
index 3e00ce4e2b..dd923dea54 100644
--- a/web-services/web-root/pom.xml
+++ b/web-services/web-root/pom.xml
@@ -4,7 +4,7 @@
gov.nsa.datawave.webservices
datawave-ws-parent
- 7.2.2
+ 7.2.3-SNAPSHOT
datawave-ws-web-root
war
From 24ac799a293fe6c698a2c587a3f5dea87b010f8b Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Wed, 31 Jul 2024 10:13:51 -0400
Subject: [PATCH 021/128] Corrected logic in how negated, null, and not null
terms are handled by the IngestTypeVisitor and IngestTypePruningVisitor.
(#2476)
Added stochastic integration test for the IngestTypeVisitor
Updated QueryGenerator to support fine-grained query generation
Added concept of internal vs. external visits for the IngestTypeVisitor. Internal visits will return the effective types
for a node or subtree (the types for executable terms), while an external visit will return all types regardless of a
term's executability state.
Co-authored-by: Whitney O'Meara
---
.../visitors/IngestTypePruningVisitor.java | 188 ++++-
.../jexl/visitors/IngestTypeVisitor.java | 106 ++-
.../query/planner/DefaultQueryPlanner.java | 2 +-
.../tables/async/event/VisitorFunction.java | 2 +-
.../jexl/util/AbstractQueryGenerator.java | 97 ++-
.../query/jexl/util/JexlQueryGenerator.java | 195 +++--
.../jexl/util/JexlQueryGeneratorTest.java | 49 +-
.../query/jexl/util/QueryGenerator.java | 32 +
.../IngestTypePruningVisitorTest.java | 324 +++++++-
.../IngestTypeVisitorIntegrationTest.java | 551 ++++++++++++++
.../jexl/visitors/IngestTypeVisitorTest.java | 694 +++++++++++++++++-
11 files changed, 2079 insertions(+), 161 deletions(-)
create mode 100644 warehouse/query-core/src/test/java/datawave/query/jexl/visitors/IngestTypeVisitorIntegrationTest.java
diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypePruningVisitor.java b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypePruningVisitor.java
index a088ac3c2f..a5455343ad 100644
--- a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypePruningVisitor.java
+++ b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypePruningVisitor.java
@@ -1,5 +1,6 @@
package datawave.query.jexl.visitors;
+import static datawave.query.jexl.visitors.IngestTypeVisitor.IGNORED_TYPE;
import static datawave.query.jexl.visitors.IngestTypeVisitor.UNKNOWN_TYPE;
import java.util.HashSet;
@@ -33,17 +34,33 @@
/**
*
- * This visitor addresses the case when multiple ingest types share some but not all fields in a query
+ * This visitor prunes terms from a query based on exclusive ingest types. The decision to prune and the type of pruning depends on whether the term contributes
+ * positively or negatively to the overall evaluation, the type of term, and the parent junction.
*
*
* Consider the query (A AND B)
where term A maps to ingest type 1 and term B maps to ingest type 2. No document will ever satisfy this
- * intersection. Thus, this should prune to zero terms.
+ * intersection due to fully exclusive ingest types. Thus, this should prune to zero terms.
*
*
- * Consider the query (A AND (B OR C))
where term A and term B map to ingest type 1 and term C maps to ingest type 2. In this case term C should be
- * pruned from the nested union leaving the intersection (A AND B)
+ * Consider the query (A AND (B OR C))
where term A and term B map to ingest type 1 and term C maps to ingest type 2. A union requires outside
+ * context to determine inclusive vs. exclusive ingest types, in this case the A term provides context. The C term is fully exclusive and is pruned leaving the
+ * intersection (A AND B)
.
+ *
+ *
+ * Null terms for exclusive types evaluates to true by default and may be pruned from intersections, e.g. A AND C == null
. A null term should prune
+ * a whole union.
+ *
+ *
+ * Negated terms for exclusive types may be pruned, e.g. A AND !B
+ *
+ *
+ * Not Null terms should not be pruned in this visitor, e.g. A AND !(C == null)
. A separate visitor to handle not null terms of exclusive types
+ * should be used, similar to the {@link IsNotNullPruningVisitor}.
+ *
+ *
+ * A not null term for exclusive types evaluates to false and may be pruned from an union, e.g. A OR !(C == null)
. A not null term should prune a
+ * whole intersection.
*
- * This visitor can also accept a set of external ingest types and use those to prune the query tree
*/
public class IngestTypePruningVisitor extends BaseVisitor {
private static final Logger log = Logger.getLogger(IngestTypePruningVisitor.class);
@@ -53,6 +70,9 @@ public class IngestTypePruningVisitor extends BaseVisitor {
private final IngestTypeVisitor ingestTypeVisitor;
+ private boolean globalExternal = false;
+ private boolean localExternal = false;
+
public IngestTypePruningVisitor(TypeMetadata typeMetadata) {
this.ingestTypeVisitor = new IngestTypeVisitor(typeMetadata);
}
@@ -83,6 +103,7 @@ public static JexlNode prune(JexlNode node, TypeMetadata metadataHelper) {
*/
public static JexlNode prune(JexlNode node, TypeMetadata metadataHelper, Set ingestTypes) {
IngestTypePruningVisitor visitor = new IngestTypePruningVisitor(metadataHelper);
+ visitor.setGlobalExternal();
node.jjtAccept(visitor, ingestTypes);
if (visitor.getTermsPruned() > 0) {
log.info("pruned " + visitor.getTermsPruned() + " terms and " + visitor.getNodesPruned() + " nodes");
@@ -141,10 +162,17 @@ public Object visit(ASTJexlScript node, Object data) {
@Override
public Object visit(ASTNotNode node, Object data) {
- Object o = node.jjtGetChild(0).jjtAccept(this, data);
+ JexlNode child = node.jjtGetChild(0);
+ Object o = child.jjtAccept(this, data);
+
+ if (child.jjtGetNumChildren() == 0) {
+ pruneNodeFromParent(child);
+ }
+
if (node.jjtGetNumChildren() == 0) {
pruneNodeFromParent(node);
}
+
return o;
}
@@ -155,7 +183,13 @@ public Object visit(ASTFunctionNode node, Object data) {
@Override
public Object visit(ASTReference node, Object data) {
- Object o = node.jjtGetChild(0).jjtAccept(this, data);
+ JexlNode child = node.jjtGetChild(0);
+ Object o = child.jjtAccept(this, data);
+
+ if (child.jjtGetNumChildren() == 0) {
+ pruneNodeFromParent(child);
+ }
+
if (node.jjtGetNumChildren() == 0) {
pruneNodeFromParent(node);
}
@@ -164,7 +198,13 @@ public Object visit(ASTReference node, Object data) {
@Override
public Object visit(ASTReferenceExpression node, Object data) {
- Object o = node.jjtGetChild(0).jjtAccept(this, data);
+ JexlNode child = node.jjtGetChild(0);
+ Object o = child.jjtAccept(this, data);
+
+ if (child.jjtGetNumChildren() == 0) {
+ pruneNodeFromParent(child);
+ }
+
if (node.jjtGetNumChildren() == 0) {
pruneNodeFromParent(node);
}
@@ -178,6 +218,7 @@ public Object visit(ASTOrNode node, Object data) {
Set types;
if (data == null) {
// normal visit
+
types = new HashSet<>();
// must traverse the children in reverse order because of pruning
for (int i = node.jjtGetNumChildren() - 1; i >= 0; i--) {
@@ -187,9 +228,24 @@ public Object visit(ASTOrNode node, Object data) {
} else {
// pruning visit
Set pruningTypes = (Set) data;
+
// must traverse the children in reverse order because of pruning
for (int i = node.jjtGetNumChildren() - 1; i >= 0; i--) {
- node.jjtGetChild(i).jjtAccept(this, pruningTypes);
+ JexlNode child = node.jjtGetChild(i);
+
+ boolean prunableNegation = isNullEquality(child) || isNegatedLeaf(child);
+ if (prunableNegation && shouldPruneChild(pruningTypes, child)) {
+ // null equalities for exclusive types evaluate to true
+ // negated leaf nodes for exclusive types evaluate to true
+ // in either case the whole union evaluates to true and can be pruned
+ JexlNodes.setChildren(node);
+ break;
+ } else if (isNotNullEquality(child) && shouldPruneChild(pruningTypes, child)) {
+ // not null equality and exclusive type, term evaluates to false, drop from union
+ pruneNodeFromParent(child);
+ } else {
+ child.jjtAccept(this, pruningTypes);
+ }
}
types = pruningTypes;
}
@@ -217,8 +273,9 @@ public Object visit(ASTAndNode node, Object data) {
QueryPropertyMarker.Instance instance = QueryPropertyMarker.findInstance(node);
if (instance.isAnyType()) {
+ int numChildren = node.jjtGetNumChildren();
Object o = visitMarker(instance, node, data);
- if (node.jjtGetNumChildren() == 0) {
+ if (node.jjtGetNumChildren() < numChildren) {
pruneNodeFromParent(node);
}
return o;
@@ -242,11 +299,29 @@ public Object visit(ASTAndNode node, Object data) {
// prune using the aggregated ingest types
// this handles the case of a nested union
pruningTypes = ingestTypes;
+
+ if (isTopLevelIntersection(node)) {
+ localExternal = true;
+ }
}
// must traverse the children in reverse order because this visitor prunes as it visits
for (int i = node.jjtGetNumChildren() - 1; i >= 0; i--) {
- node.jjtGetChild(i).jjtAccept(this, pruningTypes);
+ JexlNode child = node.jjtGetChild(i);
+ if (isNullEquality(child) && shouldPruneChild(pruningTypes, child)) {
+ // null term for exclusive type causes the term to be dropped because it evaluates to true
+ pruneNodeFromParent(child);
+ } else if (isNotNullEquality(child) && shouldPruneChild(pruningTypes, child)) {
+ // not null exclusive terms cause the whole intersection to be dropped
+ JexlNodes.setChildren(node);
+ break;
+ } else {
+ child.jjtAccept(this, pruningTypes);
+ }
+ }
+
+ if (isTopLevelIntersection(node)) {
+ localExternal = false;
}
if (node.jjtGetNumChildren() == 0) {
@@ -297,8 +372,7 @@ private Set visitMarker(QueryPropertyMarker.Instance instance, JexlNode
// pruning methods
private Set visitOrPrune(JexlNode node, Object data) {
-
- Set ingestTypes = ingestTypeVisitor.getIngestTypes(node);
+ Set ingestTypes = ingestTypeVisitor.getIngestTypes(node, (localExternal || globalExternal));
// check for pruning
if (data instanceof Set>) {
@@ -348,10 +422,33 @@ private boolean shouldPrune(Set ingestTypes, Set includes) {
return false;
}
+ if (includes.contains(IGNORED_TYPE)) {
+ throw new IllegalStateException("Should not attempt to prune using an IGNORED_TYPE");
+ }
+
+ if (ingestTypes.contains(IGNORED_TYPE)) {
+ // technically shouldn't get here
+ return false;
+ }
+
// prune if there was no overlap
return Sets.intersection(ingestTypes, includes).isEmpty();
}
+ /**
+ * Should the provided node be pruned based on a set of ingest types
+ *
+ * @param ingestTypes
+ * the ingest types that drive the prune
+ * @param child
+ * an arbitrary jexl node
+ * @return true if the jexl node should be pruned
+ */
+ private boolean shouldPruneChild(Set ingestTypes, JexlNode child) {
+ Set nodeTypes = ingestTypeVisitor.getIngestTypes(child, globalExternal);
+ return shouldPrune(ingestTypes, nodeTypes);
+ }
+
private Set pruneJunction(JexlNode node, Object data) {
// must traverse the children in reverse order because this visitor prunes as it visits
for (int i = node.jjtGetNumChildren() - 1; i >= 0; i--) {
@@ -390,4 +487,69 @@ public int getTermsPruned() {
public int getNodesPruned() {
return nodesPruned;
}
+
+ private boolean isTopLevelIntersection(JexlNode node) {
+ if (!(node instanceof ASTAndNode)) {
+ return false;
+ }
+
+ while (node.jjtGetParent() != null) {
+ node = node.jjtGetParent();
+
+ if (node instanceof ASTAndNode) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ private boolean isNullEquality(JexlNode node) {
+ node = JexlASTHelper.dereference(node);
+
+ if (node instanceof ASTEQNode) {
+ return JexlASTHelper.getLiteralValueSafely(node) == null;
+ }
+
+ return false;
+ }
+
+ private boolean isNotNullEquality(JexlNode node) {
+ node = JexlASTHelper.dereference(node);
+
+ if (node instanceof ASTNotNode) {
+ node = node.jjtGetChild(0);
+ node = JexlASTHelper.dereference(node);
+ } else {
+ return false;
+ }
+
+ if (node instanceof ASTEQNode) {
+ return JexlASTHelper.getLiteralValueSafely(node) == null;
+ }
+
+ return false;
+ }
+
+ private boolean isNegatedLeaf(JexlNode node) {
+ node = JexlASTHelper.dereference(node);
+
+ if (!(node instanceof ASTNotNode)) {
+ return false;
+ }
+
+ node = node.jjtGetChild(0);
+ node = JexlASTHelper.dereference(node);
+
+ if (JexlASTHelper.getLiteralValueSafely(node) == null) {
+ // if the final node has a null literal ignore it. Another method will handle this case.
+ return false;
+ }
+
+ return !(node instanceof ASTAndNode || node instanceof ASTOrNode);
+ }
+
+ public void setGlobalExternal() {
+ globalExternal = true;
+ }
}
diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypeVisitor.java b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypeVisitor.java
index 2f54fe0cec..b609cce7ca 100644
--- a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypeVisitor.java
+++ b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypeVisitor.java
@@ -59,11 +59,19 @@ public class IngestTypeVisitor extends BaseVisitor {
private static final Logger log = Logger.getLogger(IngestTypeVisitor.class);
+ // in the case of arithmetic or a miss in the TypeMetadata
public static final String UNKNOWN_TYPE = "UNKNOWN_TYPE";
+
+ // in the case of a null literal or negated term (effectively non-executable terms)
+ public static final String IGNORED_TYPE = "IGNORED_TYPE";
+
// cache expensive calls to get ingest types per field
private final TypeMetadata typeMetadata;
private final Map> ingestTypeCache;
+ // flag that determines how 'non-executable' nodes are handled
+ private boolean external = false;
+
public IngestTypeVisitor(TypeMetadata typeMetadata) {
this.typeMetadata = typeMetadata;
this.ingestTypeCache = new HashMap<>();
@@ -109,6 +117,20 @@ public Set getIngestTypes(JexlNode node) {
}
}
+ /**
+ * Entrypoint for when this visitor is reused
+ *
+ * @param node
+ * a JexlNode
+ * @param external
+ * a flag that determines how 'non-executable' node are handled
+ * @return a set of ingest types for this node
+ */
+ public Set getIngestTypes(JexlNode node, boolean external) {
+ this.external = external;
+ return (Set) node.jjtAccept(this, null);
+ }
+
// leaf nodes
@Override
@@ -160,12 +182,17 @@ public Object visit(ASTJexlScript node, Object data) {
@Override
public Object visit(ASTNotNode node, Object data) {
- return getIngestTypes(node);
+ if (external) {
+ return node.jjtGetChild(0).jjtAccept(this, data);
+ }
+ Set types = new HashSet<>();
+ types.add(IGNORED_TYPE);
+ return types;
}
@Override
public Object visit(ASTFunctionNode node, Object data) {
- return getIngestTypes(node);
+ return getIngestTypesForFunction(node);
}
@Override
@@ -252,11 +279,20 @@ private boolean isJunction(JexlNode node) {
@SuppressWarnings("unchecked")
public Set getIngestTypesForJunction(JexlNode node) {
+ if (node instanceof ASTAndNode) {
+ return getIngestTypesForIntersection((ASTAndNode) node);
+ }
+
Set ingestTypes = new HashSet<>();
for (int i = 0; i < node.jjtGetNumChildren(); i++) {
Set found = (Set) node.jjtGetChild(i).jjtAccept(this, null);
ingestTypes.addAll(found);
}
+
+ if (ingestTypes.size() > 1) {
+ ingestTypes.remove(IGNORED_TYPE);
+ }
+
return ingestTypes;
}
@@ -270,8 +306,14 @@ public Set getIngestTypesForJunction(JexlNode node) {
*/
public Set getIngestTypesForLeaf(JexlNode node) {
node = JexlASTHelper.dereference(node);
-
Set ingestTypes = new HashSet<>();
+
+ Object literal = JexlASTHelper.getLiteralValueSafely(node);
+ if (literal == null && !external) {
+ ingestTypes.add(IGNORED_TYPE);
+ return ingestTypes;
+ }
+
Set fields = getFieldsForLeaf(node);
for (String field : fields) {
ingestTypes.addAll(getIngestTypesForField(field));
@@ -312,6 +354,37 @@ public Set getFieldsForLeaf(JexlNode node) {
// @formatter:on
}
+ /**
+ * Functions require a separate
+ *
+ * @param node
+ * an ASTFunctionNode
+ * @return the set of ingest types
+ */
+ public Set getIngestTypesForFunction(ASTFunctionNode node) {
+ Set fields = getFieldsForFunctionNode(node);
+
+ if (fields == null) {
+ throw new IllegalStateException("no fields should be an empty collection");
+ }
+
+ // trim any identifiers
+ fields = fields.stream().map(JexlASTHelper::deconstructIdentifier).collect(Collectors.toSet());
+
+ Set types = new HashSet<>();
+
+ // function fields are always treated as a union. there might be one exception to this.
+ for (String field : fields) {
+ types.addAll(getIngestTypesForField(field));
+ }
+
+ if (types.isEmpty()) {
+ types.add(UNKNOWN_TYPE);
+ }
+
+ return types;
+ }
+
/**
* Use the functions descriptor when getting fields for an {@link ASTFunctionNode}.
*
@@ -379,8 +452,8 @@ public Set getIngestTypesForField(String field) {
* A == '1' && B == null
*
*
- * The both queries appear to be non-executable due to exclusive datatypes. A normal intersection of the A and B terms should produce an empty set. However,
- * the A term is executable while in both cases the B term acts as a filter. The B term is always true by definition of being an exclusive datatype, so this
+ * Both queries appear to be non-executable due to exclusive datatypes. A normal intersection of the A and B terms should produce an empty set. However, the
+ * A term is executable while in both cases the B term acts as a filter. The B term is always true by definition of being an exclusive datatype, so this
* visitor will return ingest type 1 for this intersection. The IngestTypePruningVisitor will correctly detect that the B term is prunable and remove it
* from the query.
*
@@ -393,18 +466,13 @@ public Set getIngestTypesForIntersection(ASTAndNode node) {
Set ingestTypes = new HashSet<>();
for (int i = 0; i < node.jjtGetNumChildren(); i++) {
JexlNode child = JexlASTHelper.dereference(node.jjtGetChild(i));
-
- boolean negated = child instanceof ASTNotNode;
- boolean isNullLiteral = child instanceof ASTEQNode && JexlASTHelper.getLiteralValueSafely(child) == null;
- if (negated || isNullLiteral) {
- continue;
- }
-
Set childIngestTypes = (Set) child.jjtAccept(this, null);
if (childIngestTypes == null) {
// we could have a malformed query or a query with a _Drop_ marker
- continue;
+ log.warn("potentially malformed query");
+ childIngestTypes = new HashSet<>();
+ childIngestTypes.add(IGNORED_TYPE);
}
if (ingestTypes.isEmpty()) {
@@ -418,6 +486,11 @@ public Set getIngestTypesForIntersection(ASTAndNode node) {
break;
}
}
+
+ if (ingestTypes.size() > 1) {
+ ingestTypes.remove(IGNORED_TYPE);
+ }
+
return ingestTypes;
}
@@ -436,6 +509,13 @@ private Set intersectTypes(Set typesA, Set typesB) {
unknown.add(UNKNOWN_TYPE);
return unknown;
}
+
+ if ((typesA.contains(IGNORED_TYPE) && !typesB.contains(IGNORED_TYPE)) || (!typesA.contains(IGNORED_TYPE) && typesB.contains(IGNORED_TYPE))) {
+ typesA.addAll(typesB);
+ typesA.remove(IGNORED_TYPE);
+ return typesA;
+ }
+
typesA.retainAll(typesB);
return typesA;
}
diff --git a/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java b/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java
index e711d1b781..21924d9a2f 100644
--- a/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java
+++ b/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java
@@ -2643,7 +2643,7 @@ public Tuple2,Boolean> getQueryRanges(ScannerFactor
if (config.isRebuildDatatypeFilter()) {
Set ingestTypes = IngestTypeVisitor.getIngestTypes(config.getQueryTree(), getTypeMetadata());
- if (ingestTypes.contains(IngestTypeVisitor.UNKNOWN_TYPE)) {
+ if (ingestTypes.contains(IngestTypeVisitor.UNKNOWN_TYPE) || ingestTypes.contains(IngestTypeVisitor.IGNORED_TYPE)) {
// could not reduce ingest types based on the query structure, do nothing
} else if (config.getDatatypeFilter().isEmpty()) {
// if no filter specified, build and set filter from query fields
diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/async/event/VisitorFunction.java b/warehouse/query-core/src/main/java/datawave/query/tables/async/event/VisitorFunction.java
index 75eacf4688..c30a3dfb81 100644
--- a/warehouse/query-core/src/main/java/datawave/query/tables/async/event/VisitorFunction.java
+++ b/warehouse/query-core/src/main/java/datawave/query/tables/async/event/VisitorFunction.java
@@ -506,7 +506,7 @@ private void reduceIngestTypes(ASTJexlScript script, IteratorSetting newIterator
// get existing types from the query
Set datatypes = IngestTypeVisitor.getIngestTypes(script, cachedTypeMetadata);
- if (datatypes.contains(IngestTypeVisitor.UNKNOWN_TYPE)) {
+ if (datatypes.contains(IngestTypeVisitor.UNKNOWN_TYPE) || datatypes.contains(IngestTypeVisitor.IGNORED_TYPE)) {
return;
}
diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/util/AbstractQueryGenerator.java b/warehouse/query-core/src/test/java/datawave/query/jexl/util/AbstractQueryGenerator.java
index cdf5d0f37a..af4699afea 100644
--- a/warehouse/query-core/src/test/java/datawave/query/jexl/util/AbstractQueryGenerator.java
+++ b/warehouse/query-core/src/test/java/datawave/query/jexl/util/AbstractQueryGenerator.java
@@ -15,6 +15,9 @@ public abstract class AbstractQueryGenerator implements QueryGenerator {
protected boolean filterFunctionsEnabled = false;
protected boolean contentFunctionsEnabled = false;
protected boolean groupingFunctionsEnabled = false;
+ protected boolean noFieldedFunctionsEnabled = false;
+ protected boolean multiFieldedFunctionsEnabled = false;
+ protected boolean nullLiteralsEnabled = false;
protected final Random random = new Random();
protected final StringBuilder sb = new StringBuilder();
@@ -23,7 +26,7 @@ public abstract class AbstractQueryGenerator implements QueryGenerator {
protected final List values;
enum NodeType {
- EQ, NE, ER, FILTER_FUNCTION, CONTENT_FUNCTION, GROUPING_FUNCTION
+ EQ, NE, ER, FILTER_FUNCTION, CONTENT_FUNCTION, GROUPING_FUNCTION, EQ_NULL, NE_NULL
}
public AbstractQueryGenerator(Set fields, Set values) {
@@ -102,17 +105,105 @@ public QueryGenerator enableGroupingFunctions() {
return this;
}
+ public QueryGenerator disableNoFieldedFunctions() {
+ noFieldedFunctionsEnabled = false;
+ return this;
+ }
+
+ public QueryGenerator enableNoFieldedFunctions() {
+ noFieldedFunctionsEnabled = true;
+ return this;
+ }
+
public QueryGenerator disableGroupingFunctions() {
groupingFunctionsEnabled = false;
return this;
}
+ public QueryGenerator enableMultiFieldedFunctions() {
+ multiFieldedFunctionsEnabled = true;
+ return this;
+ }
+
+ public QueryGenerator disableMultiFieldedFunctions() {
+ multiFieldedFunctionsEnabled = false;
+ return this;
+ }
+
+ public QueryGenerator enableNullLiterals() {
+ nullLiteralsEnabled = true;
+ return this;
+ }
+
+ public QueryGenerator disableNullLiterals() {
+ nullLiteralsEnabled = false;
+ return this;
+ }
+
public QueryGenerator enableAllOptions() {
- return enableNegations().enableRegexes().enableFilterFunctions().enableContentFunctions().enableGroupingFunctions();
+ // @formatter:off
+ return enableNegations()
+ .enableRegexes()
+ .enableFilterFunctions()
+ .enableContentFunctions()
+ .enableGroupingFunctions()
+ .enableNoFieldedFunctions()
+ .enableMultiFieldedFunctions()
+ .enableNullLiterals();
+ // @formatter:on
}
public QueryGenerator disableAllOptions() {
- return disableNegations().disableRegexes().disableFilterFunctions().disableContentFunctions().disableGroupingFunctions();
+ // @formatter:off
+ return disableNegations()
+ .disableRegexes()
+ .disableFilterFunctions()
+ .disableContentFunctions()
+ .disableGroupingFunctions()
+ .disableNoFieldedFunctions()
+ .disableMultiFieldedFunctions()
+ .disableNullLiterals();
+ // @formatter:on
+ }
+
+ public boolean isIntersectionsEnabled() {
+ return intersectionsEnabled;
+ }
+
+ public boolean isUnionsEnabled() {
+ return unionsEnabled;
+ }
+
+ public boolean isNegationsEnabled() {
+ return negationsEnabled;
+ }
+
+ public boolean isRegexesEnabled() {
+ return regexEnabled;
+ }
+
+ public boolean isFilterFunctionsEnabled() {
+ return filterFunctionsEnabled;
+ }
+
+ public boolean isContentFunctionsEnabled() {
+ return contentFunctionsEnabled;
+ }
+
+ public boolean isGroupingFunctionsEnabled() {
+ return groupingFunctionsEnabled;
+ }
+
+ public boolean isNoFieldedFunctionsEnabled() {
+ return noFieldedFunctionsEnabled;
+ }
+
+ public boolean isMultiFieldedFunctionsEnabled() {
+ return multiFieldedFunctionsEnabled;
+ }
+
+ public boolean isNullLiteralsEnabled() {
+ return nullLiteralsEnabled;
}
protected abstract void buildNode();
diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/util/JexlQueryGenerator.java b/warehouse/query-core/src/test/java/datawave/query/jexl/util/JexlQueryGenerator.java
index fb4ff2aebe..5c0970da8a 100644
--- a/warehouse/query-core/src/test/java/datawave/query/jexl/util/JexlQueryGenerator.java
+++ b/warehouse/query-core/src/test/java/datawave/query/jexl/util/JexlQueryGenerator.java
@@ -15,6 +15,8 @@
* content functions
* grouping functions
*
+ *
+ * Optionally limit functions to those without fields, with single fields, and/or multi fields
*/
public class JexlQueryGenerator extends AbstractQueryGenerator {
@@ -173,6 +175,12 @@ protected void buildLeaf() {
case GROUPING_FUNCTION:
buildGroupingFunction();
break;
+ case EQ_NULL:
+ buildNullEquality();
+ break;
+ case NE_NULL:
+ buildNullNotEquals();
+ break;
default:
throw new IllegalStateException("Error building leaf of type: " + type);
}
@@ -181,7 +189,7 @@ protected void buildLeaf() {
private NodeType getNextNodeType() {
NodeType type = null;
while (type == null) {
- int index = random.nextInt(6);
+ int index = random.nextInt(8);
switch (index) {
case 0:
type = NodeType.EQ;
@@ -201,6 +209,12 @@ private NodeType getNextNodeType() {
case 5:
type = NodeType.GROUPING_FUNCTION;
break;
+ case 6:
+ type = NodeType.EQ_NULL;
+ break;
+ case 7:
+ type = NodeType.NE_NULL;
+ break;
default:
throw new IllegalStateException("Could not get next node type for index: " + index);
}
@@ -224,6 +238,14 @@ private NodeType getNextNodeType() {
if (!groupingFunctionsEnabled && type != null && type.equals(NodeType.GROUPING_FUNCTION)) {
type = null;
}
+
+ if (!nullLiteralsEnabled && type != null && type.equals(NodeType.EQ_NULL)) {
+ type = null;
+ }
+
+ if (!nullLiteralsEnabled && type != null && type.equals(NodeType.NE_NULL)) {
+ type = null;
+ }
}
return type;
}
@@ -273,7 +295,12 @@ private void buildFilterFunction() {
String value = getValue();
sb.append("filter:");
- int index = random.nextInt(14);
+ int bound = 8;
+ if (multiFieldedFunctionsEnabled) {
+ bound += 6;
+ }
+
+ int index = random.nextInt(bound);
String fields;
switch (index) {
case 0:
@@ -281,59 +308,62 @@ private void buildFilterFunction() {
sb.append("includeRegex(").append(field).append(",'").append(value).append(".*')");
break;
case 1:
- // include regex, multi-fielded
- fields = "(" + getField() + " || " + getField() + ")";
- sb.append("includeRegex(").append(fields).append(",'").append(value).append(".*')");
- break;
- case 2:
// exclude regex, fielded
sb.append("excludeRegex(").append(field).append(",'").append(value).append(".*')");
break;
- case 3:
- // exclude regex, multi-fielded
- fields = "(" + getField() + " || " + getField() + ")";
- sb.append("excludeRegex(").append(fields).append(",'").append(value).append(".*')");
- break;
- case 4:
+ case 2:
// isNull, single field
// the RewriteNullFunctionsVisitor should remove these functions, but test them anyway
sb.append("isNull(").append(field).append(")");
break;
+ case 3:
+ // isNotNull, single field
+ sb.append("isNotNull(").append(field).append(")");
+ break;
+ case 4:
+ sb.append("betweenDates(").append(field).append(", '2024-01-01', '2024-01-05')");
+ break;
case 5:
- // isNull, multi-field
- // the RewriteNullFunctionsVisitor should remove these functions, but test them anyway
- sb.append("isNull(").append(field).append(")");
+ sb.append("betweenLoadDates(").append(field).append(", '20240101', '20240105', 'yyyyMMdd')");
break;
case 6:
- // isNotNull, single field
- sb.append("isNotNull(").append(field).append(")");
+ sb.append("matchesAtLeastCountOf(1,").append(field).append(",").append(getValue()).append(")");
break;
case 7:
- // isNotNull, multi-field
- sb.append("isNotNull(").append(field).append(")");
+ // occurrence
+ sb.append("occurrence(").append(getField()).append(", '>', 1)");
break;
case 8:
- sb.append("betweenDates(").append(field).append(", '2024-01-01', '2024-01-05')");
+ // compare function is actually two fields. A user could compare the values of the same field
+ sb.append("compare(").append(field).append(", '>', 'ANY', ").append(getField()).append(")");
break;
case 9:
- sb.append("betweenLoadDates(").append(field).append(", '20240101', '20240105', 'yyyyMMdd')");
+ // filter:timeFunction(DEATH_DATE,BIRTH_DATE,'-','>',2522880000000L)
+ sb.append("timeFunction(").append(field).append(",").append(getField()).append(",'-','>',2522880000000L)");
break;
case 10:
- sb.append("matchesAtLeastCountOf(1,").append(field).append(",").append(getField()).append(")");
+ // include regex, multi-fielded
+ fields = "(" + getField() + " || " + getField() + ")";
+ sb.append("includeRegex(").append(fields).append(",'").append(value).append(".*')");
break;
case 11:
- // filter:timeFunction(DEATH_DATE,BIRTH_DATE,'-','>',2522880000000L)
- sb.append("timeFunction(").append(field).append(",").append(getField()).append(",'-','>',2522880000000L)");
+ // exclude regex, multi-fielded
+ fields = "(" + getField() + " || " + getField() + ")";
+ sb.append("excludeRegex(").append(fields).append(",'").append(value).append(".*')");
break;
case 12:
- // compare function is actually two fields. A user could compare the values of the same field
- sb.append("compare(").append(field).append(", '>', 'ANY', ").append(getField()).append(")");
+ // isNull, multi-field
+ // the RewriteNullFunctionsVisitor should remove these functions, but test them anyway
+ fields = getField() + " || " + getField();
+ sb.append("isNull(").append(fields).append(")");
break;
case 13:
- // occurrence
- sb.append("occurrence(").append(getField()).append(", '>', 1)");
+ // isNotNull, multi-field
+ fields = getField() + " || " + getField();
+ sb.append("isNotNull(").append(fields).append(")");
break;
default:
+ throw new IllegalStateException("should never get here. bound: " + bound + ", query: " + sb.toString());
}
}
@@ -341,56 +371,66 @@ private void buildContentFunction() {
incrementCounts();
sb.append("content:");
- int index = random.nextInt(12);
+ int minimum = 4;
+ if (noFieldedFunctionsEnabled) {
+ minimum = 0;
+ }
+
+ int bound = 4;
+ if (multiFieldedFunctionsEnabled) {
+ bound += 4;
+ }
+
+ int index = minimum + random.nextInt(bound);
String fields;
switch (index) {
case 0:
- // adjacent, no field
- sb.append("adjacent(termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
+ // phrase, no field
+ sb.append("phrase(termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
break;
case 1:
- // adjacent, fielded
- sb.append("adjacent(").append(getField()).append(", termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
+ // adjacent, no field
+ sb.append("adjacent(termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
break;
case 2:
- // adjacent, multi-fielded
- fields = "(" + getField() + " || " + getField() + ")";
- sb.append("adjacent(").append(fields).append(", termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
+ // within, no field
+ sb.append("within(2, termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
break;
case 3:
- // phrase, no field
- sb.append("phrase(termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
+ // scored phrase, no field
+ sb.append("scoredPhrase(-1.5, termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
break;
case 4:
- // phrase, fielded
- sb.append("phrase(").append(getField()).append(", termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
+ // adjacent, fielded
+ sb.append("adjacent(").append(getField()).append(", termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
break;
case 5:
- // phrase, multi-fielded
- fields = "(" + getField() + " || " + getField() + ")";
- sb.append("phrase(").append(fields).append(", termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
+ // phrase, fielded
+ sb.append("phrase(").append(getField()).append(", termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
break;
case 6:
- // within, no field
- sb.append("within(2, termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
- break;
- case 7:
// within, fielded
sb.append("within(").append(getField()).append(", 2, termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
break;
+ case 7:
+ // scored phrase, fielded
+ sb.append("scoredPhrase(").append(getField()).append(", -1.5, termOffsetMap, '").append(getValue()).append("', '").append(getValue())
+ .append("')");
+ break;
case 8:
- // within, multi-fielded
+ // adjacent, multi-fielded
fields = "(" + getField() + " || " + getField() + ")";
- sb.append("within(").append(fields).append(", 2, termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
+ sb.append("adjacent(").append(fields).append(", termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
break;
case 9:
- // scored phrase, no field
- sb.append("scoredPhrase(-1.5, termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
+ // phrase, multi-fielded
+ fields = "(" + getField() + " || " + getField() + ")";
+ sb.append("phrase(").append(fields).append(", termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
break;
case 10:
- // scored phrase, fielded
- sb.append("scoredPhrase(").append(getField()).append(", -1.5, termOffsetMap, '").append(getValue()).append("', '").append(getValue())
- .append("')");
+ // within, multi-fielded
+ fields = "(" + getField() + " || " + getField() + ")";
+ sb.append("within(").append(fields).append(", 2, termOffsetMap, '").append(getValue()).append("', '").append(getValue()).append("')");
break;
case 11:
// scored phrase, multi-fielded
@@ -406,7 +446,12 @@ private void buildGroupingFunction() {
incrementCounts();
sb.append("f:");
- int index = random.nextInt(6);
+ int bound = 5;
+ if (multiFieldedFunctionsEnabled) {
+ bound += 1;
+ }
+
+ int index = random.nextInt(bound);
switch (index) {
case 0:
// sum
@@ -417,9 +462,21 @@ private void buildGroupingFunction() {
sb.append("count(").append(getField()).append(")");
break;
case 2:
+ // average
+ sb.append("average(").append(getField()).append(")");
+ break;
+ case 3:
+ // min
+ sb.append("min(").append(getField()).append(")");
+ break;
+ case 4:
+ // max
+ sb.append("max(").append(getField()).append(")");
+ break;
+ case 5:
// count, multi field
sb.append("count(");
- int extraFields = random.nextInt(3);
+ int extraFields = 1 + random.nextInt(3);
for (int i = 0; i < extraFields; i++) {
sb.append(getField());
if (i < extraFields - 1) {
@@ -428,23 +485,23 @@ private void buildGroupingFunction() {
}
sb.append(")");
break;
- case 3:
- // average
- sb.append("average(").append(getField()).append(")");
- break;
- case 4:
- // min
- sb.append("min(").append(getField()).append(")");
- break;
- case 5:
- // max
- sb.append("max(").append(getField()).append(")");
- break;
default:
throw new IllegalStateException("unknown index: " + index);
}
}
+ private void buildNullEquality() {
+ incrementCounts();
+ String field = getField();
+ sb.append(field).append(" == null");
+ }
+
+ private void buildNullNotEquals() {
+ incrementCounts();
+ String field = getField();
+ sb.append("!(").append(field).append(" == null)");
+ }
+
private void incrementCounts() {
currentTermCount++;
if (reservedCapacity > 0) {
diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/util/JexlQueryGeneratorTest.java b/warehouse/query-core/src/test/java/datawave/query/jexl/util/JexlQueryGeneratorTest.java
index 330a01526a..099a8066c2 100644
--- a/warehouse/query-core/src/test/java/datawave/query/jexl/util/JexlQueryGeneratorTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/jexl/util/JexlQueryGeneratorTest.java
@@ -10,6 +10,7 @@
import org.apache.log4j.Logger;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Sets;
@@ -19,6 +20,12 @@
class JexlQueryGeneratorTest {
+ private static final org.slf4j.Logger log = LoggerFactory.getLogger(JexlQueryGeneratorTest.class);
+
+ private final int maxIterations = 1_000;
+ private final int size = 1;
+ private final int minSize = 2;
+ private final int maxSize = 15;
private final ASTValidator validator = new ASTValidator();
@BeforeEach
@@ -31,7 +38,7 @@ void testSimpleQuery() {
Set fields = Collections.singleton("FIELD");
Set values = Collections.singleton("value");
QueryGenerator generator = new JexlQueryGenerator(fields, values).enableAllOptions();
- generateFixedSize(generator, 1, 1);
+ generateQueries(generator);
}
@Test
@@ -39,7 +46,7 @@ void testNegations() {
Set fields = Collections.singleton("FIELD");
Set values = Collections.singleton("value");
QueryGenerator generator = new JexlQueryGenerator(fields, values).enableNegations();
- generateFixedSize(generator, 10, 1);
+ generateQueries(generator);
}
@Test
@@ -47,7 +54,7 @@ void testRegexes() {
Set fields = Collections.singleton("FIELD");
Set values = Collections.singleton("value");
QueryGenerator generator = new JexlQueryGenerator(fields, values).enableRegexes();
- generateFixedSize(generator, 10, 1);
+ generateQueries(generator);
}
@Test
@@ -55,7 +62,7 @@ void testFilterFunctions() {
Set fields = Collections.singleton("FIELD");
Set values = Collections.singleton("value");
QueryGenerator generator = new JexlQueryGenerator(fields, values).enableFilterFunctions();
- generateFixedSize(generator, 10, 1);
+ testFunctions(generator);
}
@Test
@@ -63,7 +70,7 @@ void testContentFunctions() {
Set fields = Collections.singleton("FIELD");
Set values = Collections.singleton("value");
QueryGenerator generator = new JexlQueryGenerator(fields, values).enableContentFunctions();
- generateFixedSize(generator, 10, 1);
+ testFunctions(generator);
}
@Test
@@ -71,7 +78,7 @@ void testGroupingFunctions() {
Set fields = Collections.singleton("FIELD");
Set values = Collections.singleton("value");
QueryGenerator generator = new JexlQueryGenerator(fields, values).enableGroupingFunctions();
- generateFixedSize(generator, 10, 1);
+ testFunctions(generator);
}
@Test
@@ -79,7 +86,7 @@ void testLargeJunction() {
Set fields = Collections.singleton("FIELD");
Set values = Collections.singleton("value");
QueryGenerator generator = new JexlQueryGenerator(fields, values).enableAllOptions();
- generateFixedSize(generator, 10, 5);
+ generateFixedSize(generator, maxIterations, maxSize);
}
@Test
@@ -87,7 +94,7 @@ void testHighCount() {
Set fields = Sets.newHashSet("F1", "F2", "F3");
Set values = Sets.newHashSet("v", "bar", "foo");
QueryGenerator generator = new JexlQueryGenerator(fields, values).enableAllOptions();
- generateFixedSize(generator, 15, 20);
+ generateFixedSize(generator, 100, 100);
}
@Test
@@ -95,7 +102,7 @@ void testVariableQuerySize() {
Set fields = Sets.newHashSet("F1", "F2", "F3", "$11", "$22");
Set values = Sets.newHashSet("v", "bar", "foo");
QueryGenerator generator = new JexlQueryGenerator(fields, values).enableAllOptions();
- generateVariableSize(generator, 1500, 1, 15);
+ generateVariableSize(generator, 1500, minSize, maxSize);
}
@Test
@@ -103,7 +110,7 @@ void testGenerateUnions() {
Set fields = Collections.singleton("F");
Set values = Sets.newHashSet("1", "2", "3", "4");
QueryGenerator generator = new JexlQueryGenerator(fields, values).disableIntersections();
- generateVariableSize(generator, 10, 3, 7);
+ generateVariableSize(generator, maxIterations, minSize, maxSize);
}
@Test
@@ -111,7 +118,26 @@ void testGenerateIntersections() {
Set fields = Collections.singleton("F");
Set values = Sets.newHashSet("1", "2", "3", "4");
QueryGenerator generator = new JexlQueryGenerator(fields, values).disableUnions();
- generateVariableSize(generator, 10, 3, 7);
+ generateQueries(generator);
+ }
+
+ private void testFunctions(QueryGenerator generator) {
+ generateFixedSize(generator, maxIterations, size);
+ generateVariableSize(generator, maxIterations, minSize, maxSize);
+
+ generator.enableNoFieldedFunctions();
+ generateFixedSize(generator, maxIterations, size);
+ generateVariableSize(generator, maxIterations, minSize, maxSize);
+
+ generator.disableNoFieldedFunctions();
+ generator.enableMultiFieldedFunctions();
+ generateFixedSize(generator, maxIterations, size);
+ generateVariableSize(generator, maxIterations, minSize, maxSize);
+ }
+
+ private void generateQueries(QueryGenerator generator) {
+ generateFixedSize(generator, maxIterations, size);
+ generateVariableSize(generator, maxIterations, minSize, maxSize);
}
private void generateFixedSize(QueryGenerator generator, int maxIterations, int size) {
@@ -132,6 +158,7 @@ private void validateQuery(String query) {
try {
assertTrue(validator.isValid(JexlASTHelper.parseAndFlattenJexlQuery(query)));
} catch (InvalidQueryTreeException | ParseException e) {
+ log.info("Failed to parse query: {}", query);
throw new RuntimeException(e);
}
}
diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/util/QueryGenerator.java b/warehouse/query-core/src/test/java/datawave/query/jexl/util/QueryGenerator.java
index b6ce73d719..616ee0342c 100644
--- a/warehouse/query-core/src/test/java/datawave/query/jexl/util/QueryGenerator.java
+++ b/warehouse/query-core/src/test/java/datawave/query/jexl/util/QueryGenerator.java
@@ -38,7 +38,39 @@ public interface QueryGenerator {
QueryGenerator disableGroupingFunctions();
+ QueryGenerator enableNoFieldedFunctions();
+
+ QueryGenerator disableNoFieldedFunctions();
+
+ QueryGenerator enableMultiFieldedFunctions();
+
+ QueryGenerator disableMultiFieldedFunctions();
+
+ QueryGenerator enableNullLiterals();
+
+ QueryGenerator disableNullLiterals();
+
QueryGenerator enableAllOptions();
QueryGenerator disableAllOptions();
+
+ boolean isIntersectionsEnabled();
+
+ boolean isUnionsEnabled();
+
+ boolean isNegationsEnabled();
+
+ boolean isRegexesEnabled();
+
+ boolean isFilterFunctionsEnabled();
+
+ boolean isContentFunctionsEnabled();
+
+ boolean isGroupingFunctionsEnabled();
+
+ boolean isNoFieldedFunctionsEnabled();
+
+ boolean isMultiFieldedFunctionsEnabled();
+
+ boolean isNullLiteralsEnabled();
}
diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/IngestTypePruningVisitorTest.java b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/IngestTypePruningVisitorTest.java
index 7cd110f043..14a5adc08f 100644
--- a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/IngestTypePruningVisitorTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/IngestTypePruningVisitorTest.java
@@ -1,5 +1,6 @@
package datawave.query.jexl.visitors;
+import static datawave.query.jexl.visitors.IngestTypeVisitor.IGNORED_TYPE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -8,8 +9,10 @@
import java.util.Set;
import org.apache.commons.jexl3.parser.ASTJexlScript;
+import org.apache.commons.jexl3.parser.ParseException;
import org.apache.log4j.Logger;
import org.junit.BeforeClass;
+import org.junit.Ignore;
import org.junit.Test;
import com.google.common.collect.Sets;
@@ -451,7 +454,7 @@ public void testPruneNegation() {
@Test
public void testFullyPrunedTree() {
String query = "(false)";
- test(query, query);
+ test(query, "");
}
@Test
@@ -497,8 +500,8 @@ public void testExternalPrune() {
metadata.put("B", "ingestType1", LcType.class.getTypeName());
metadata.put("C", "ingestType1", LcType.class.getTypeName());
- test("A == '1' && (B == '2' || C == '3')", null, metadata, Collections.singleton("ingestType2"));
- test("A == '1' || (B == '2' && C == '3')", null, metadata, Collections.singleton("ingestType2"));
+ testExternalPrune("A == '1' && (B == '2' || C == '3')", null, metadata, Collections.singleton("ingestType2"));
+ testExternalPrune("A == '1' || (B == '2' && C == '3')", null, metadata, Collections.singleton("ingestType2"));
}
@Test
@@ -515,7 +518,7 @@ public void testExternalPruneWithSelfPrune() {
Set externalTypes = Sets.newHashSet("ingestType2", "ingestType3", "ingestType4");
// A term pruned by external types
// C and D terms should self prune
- test(query, expected, metadata, externalTypes);
+ testExternalPrune(query, expected, metadata, externalTypes);
}
@Test
@@ -541,12 +544,12 @@ public void testAndNotNull() {
metadata.put("B", "ingestType2", LcType.class.getTypeName());
metadata.put("9", "ingestType2", LcType.class.getTypeName());
- // visitor is smart enough to prune out a negation for an exclusive ingest type
+ // not null exclusive type evaluates to false, causing whole intersection to be dropped
String query = "A == '1' && !(B == null)";
- test(query, "A == '1'", metadata);
+ test(query, "", metadata);
query = "A == '1' && !($B == null)"; // same form but with an identifier
- test(query, "A == '1'", metadata);
+ test(query, "", metadata);
}
@Test
@@ -610,6 +613,7 @@ public void testNotNullAndNestedUnion() {
test(query, query, metadata);
}
+ @Ignore
@Test
public void testFilterFunctionExcludeExpandedIntoMutuallyExclusiveFields() {
// there might be an exclude like #EXCLUDE(MODEL_FIELD, '.*.*')
@@ -621,6 +625,8 @@ public void testFilterFunctionExcludeExpandedIntoMutuallyExclusiveFields() {
metadata.put("C", "type2", LcType.class.getTypeName());
metadata.put("D", "type3", LcType.class.getTypeName());
+ // pushdown negations visitor would rewrite this
+ // into A == '1' && !(B == null) && !...
String query = "A == '1' && !((B == null || C == null || D == null))";
String expected = "A == '1' && !((B == null))";
test(query, expected, metadata);
@@ -638,21 +644,289 @@ public void testUnionOfNotNullTerms() {
test(query, query);
}
+ @Test
+ public void testIntersectionsWithNullTerms() {
+ TypeMetadata metadata = new TypeMetadata();
+ metadata.put("A", "type-1", LcType.class.getTypeName());
+ metadata.put("B", "type-1", LcType.class.getTypeName());
+ metadata.put("C", "type-2", LcType.class.getTypeName());
+ metadata.put("123", "type-1", LcType.class.getTypeName());
+ metadata.put("234", "type-2", LcType.class.getTypeName());
+
+ // same field, no change
+ String query = "A == '1' && A == null";
+ test(query, query, metadata);
+
+ // different field, same type, no change
+ query = "A == '1' && B == null";
+ test(query, query, metadata);
+
+ // field with exclusive type, pruned
+ query = "A == '1' && C == null";
+ test(query, "A == '1'", metadata);
+
+ // identifier with same type, no change
+ query = "A == '1' && $123 == null";
+ test(query, query, metadata);
+
+ // identifier with exclusive type, prune
+ query = "A == '1' && $234 == null";
+ test(query, "A == '1'", metadata);
+ }
+
+ // is not null terms with exclusive datatypes should NOT be pruned
+ // the query is *technically* non-executable
+ @Test
+ public void testIntersectionsWithNotNullTerms() {
+ TypeMetadata metadata = new TypeMetadata();
+ metadata.put("A", "type-1", LcType.class.getTypeName());
+ metadata.put("B", "type-1", LcType.class.getTypeName());
+ metadata.put("C", "type-2", LcType.class.getTypeName());
+ metadata.put("123", "type-1", LcType.class.getTypeName());
+ metadata.put("234", "type-2", LcType.class.getTypeName());
+
+ // same field, no change
+ String query = "A == '1' && !(A == null)";
+ test(query, query, metadata);
+
+ // different field, same type, no change
+ query = "A == '1' && !(B == null)";
+ test(query, query, metadata);
+
+ // not null term with exclusive type is false, prune whole intersection
+ query = "A == '1' && !(C == null)";
+ test(query, "", metadata);
+
+ // identifier with same type, no change
+ query = "A == '1' && !($123 == null)";
+ test(query, query, metadata);
+
+ // identifier with exclusive type, prune
+ query = "A == '1' && !($234 == null)";
+ test(query, "", metadata);
+ }
+
+ @Test
+ public void testNestedUnionsWithNullTerms() {
+ TypeMetadata metadata = new TypeMetadata();
+ metadata.put("A", "type-1", LcType.class.getTypeName());
+ metadata.put("B", "type-1", LcType.class.getTypeName());
+ metadata.put("C", "type-2", LcType.class.getTypeName());
+ metadata.put("123", "type-1", LcType.class.getTypeName());
+ metadata.put("234", "type-2", LcType.class.getTypeName());
+
+ // same field, same types, no change
+ String query = "A == '1' && (A == '2' || A == null)";
+ test(query, query, metadata);
+
+ // different fields, subset types, no change
+ query = "A == '1' && (A == '2' || B == null)";
+ test(query, query, metadata);
+
+ // exclusive null term evaluates to true, should prune the whole union
+ query = "A == '1' && (B == '2' || C == null)";
+ String expected = "A == '1'";
+ test(query, expected, metadata);
+ }
+
+ @Test
+ public void testNestedUnionsWithNotNullTerms() {
+ TypeMetadata metadata = new TypeMetadata();
+ metadata.put("A", "type-1", LcType.class.getTypeName());
+ metadata.put("B", "type-1", LcType.class.getTypeName());
+ metadata.put("C", "type-2", LcType.class.getTypeName());
+ metadata.put("123", "type-1", LcType.class.getTypeName());
+ metadata.put("234", "type-2", LcType.class.getTypeName());
+
+ String query = "A == '1' && (A == '2' || !(A == null))";
+ test(query, query, metadata);
+
+ query = "A == '1' && (A == '2' || !(B == null))";
+ test(query, query, metadata);
+
+ // not null exclusive term evaluates to false, can safely drop from a union
+ query = "A == '1' && (B == '2' || !(C == null))";
+ test(query, "A == '1' && (B == '2')", metadata);
+ }
+
+ @Test
+ public void testNestedUnionsWithNegatedTerms() {
+ TypeMetadata metadata = new TypeMetadata();
+ metadata.put("A", "type-1", LcType.class.getTypeName());
+ metadata.put("B", "type-1", LcType.class.getTypeName());
+ metadata.put("C", "type-2", LcType.class.getTypeName());
+ metadata.put("123", "type-1", LcType.class.getTypeName());
+ metadata.put("234", "type-2", LcType.class.getTypeName());
+
+ // same field, same types, no change
+ String query = "A == '1' && (A == '2' || !(A == '3'))";
+ test(query, query, metadata);
+
+ // different fields, subset types, no change
+ query = "A == '1' && (A == '2' || !(B == '3'))";
+ test(query, query, metadata);
+
+ // exclusive negated term evaluates to true, drop the whole union
+ query = "A == '1' && (B == '2' || !(C == '3'))";
+ String expected = "A == '1'";
+ test(query, expected, metadata);
+ }
+
+ @Test
+ public void testNestedIntersectionsWithNullTerms() {
+ TypeMetadata metadata = new TypeMetadata();
+ metadata.put("A", "type-1", LcType.class.getTypeName());
+ metadata.put("B", "type-1", LcType.class.getTypeName());
+ metadata.put("C", "type-2", LcType.class.getTypeName());
+ metadata.put("123", "type-1", LcType.class.getTypeName());
+ metadata.put("234", "type-2", LcType.class.getTypeName());
+
+ // same field, same types, no change
+ String query = "A == '1' || (A == '2' && A == null)";
+ test(query, query, metadata);
+
+ // different fields, subset types, no change
+ query = "A == '1' || (A == '2' && B == null)";
+ test(query, query, metadata);
+
+ // exclusive type evaluates to true, may be safely pruned
+ query = "A == '1' || (A == '2' && C == null)";
+ test(query, "A == '1' || (A == '2')", metadata);
+ }
+
+ @Test
+ public void testNestedIntersectionsWithNotNullTerms() {
+ TypeMetadata metadata = new TypeMetadata();
+ metadata.put("A", "type-1", LcType.class.getTypeName());
+ metadata.put("B", "type-1", LcType.class.getTypeName());
+ metadata.put("C", "type-2", LcType.class.getTypeName());
+ metadata.put("123", "type-1", LcType.class.getTypeName());
+ metadata.put("234", "type-2", LcType.class.getTypeName());
+
+ String query = "A == '1' || (A == '2' && !(A == null))";
+ test(query, query, metadata);
+
+ query = "A == '1' || (A == '2' && !(B == null))";
+ test(query, query, metadata);
+
+ // not null term for exclusive type evaluates to false, whole intersection must be pruned
+ query = "A == '1' || (A == '2' && !(C == null))";
+ test(query, "A == '1'", metadata);
+ }
+
+ @Test
+ public void testNestedIntersectionsWithNegatedTerms() {
+ TypeMetadata metadata = new TypeMetadata();
+ metadata.put("A", "type-1", LcType.class.getTypeName());
+ metadata.put("B", "type-1", LcType.class.getTypeName());
+ metadata.put("C", "type-2", LcType.class.getTypeName());
+ metadata.put("123", "type-1", LcType.class.getTypeName());
+ metadata.put("234", "type-2", LcType.class.getTypeName());
+
+ // same field, same types, no change
+ String query = "A == '1' || (A == '2' && !(A == '3'))";
+ test(query, query, metadata);
+
+ // different fields, subset types, no change
+ query = "A == '1' || (A == '2' && !(B == '3'))";
+ test(query, query, metadata);
+
+ // negated term for exclusive type evaluates to false, safely dropped from intersection
+ query = "A == '1' || (A == '2' && !(C == '3'))";
+ test(query, "A == '1' || (A == '2')", metadata);
+ }
+
+ /**
+ * This query is technically a top level union, but the nested intersections are both top level intersections and can thus drive pruning in their
+ * nested unions.
+ */
+ @Test
+ public void testContrivedCaseWithTwoTopLevelIntersectionsUnderUnion() {
+
+ TypeMetadata metadata = new TypeMetadata();
+ metadata.put("A", "type-1", LcType.class.getTypeName());
+ metadata.put("B", "type-1", LcType.class.getTypeName());
+ metadata.put("C", "type-3", LcType.class.getTypeName());
+ metadata.put("X", "type-7", LcType.class.getTypeName());
+ metadata.put("Y", "type-7", LcType.class.getTypeName());
+ metadata.put("Z", "type-9", LcType.class.getTypeName());
+
+ String query = "(A == '1' && (B == '2' || C == '3')) || (X == '7' && (Y == '8' || Z == '9'))";
+ String expected = "(A == '1' && (B == '2')) || (X == '7' && (Y == '8'))";
+ test(query, expected, metadata);
+ }
+
private void test(String query, String expected) {
- test(query, expected, typeMetadata, null);
+ test(query, expected, typeMetadata);
+ }
+
+ private void test(String query, String expected, TypeMetadata metadata) {
+ ASTJexlScript internal = testInternalPrune(query, expected, metadata);
+ ASTJexlScript external = testExternalPrune(query, expected, metadata);
+
+ // validate and compare internal vs. external pruning
+ verifyEquality(internal, external);
+ }
+
+ private void testInternalPrune(String query, String expected) {
+ testInternalPrune(query, expected, typeMetadata);
+ }
+
+ private ASTJexlScript testInternalPrune(String query, String expected, TypeMetadata metadata) {
+ try {
+ ASTJexlScript script = parseQuery(query);
+ ASTJexlScript pruned = (ASTJexlScript) IngestTypePruningVisitor.prune(script, metadata);
+
+ log.info("input : " + query);
+ log.info("output : " + JexlStringBuildingVisitor.buildQuery(pruned));
+ log.info("expected: " + expected);
+
+ // all pruned scripts must be valid
+ assertTrue(validator.isValid(pruned));
+
+ // we might be expecting nothing as a result
+ if (expected == null) {
+ log.trace("expected null! " + JexlStringBuildingVisitor.buildQuery(pruned));
+ assertEquals("failed for query: " + query, 0, pruned.jjtGetNumChildren());
+ return null;
+ }
+
+ ASTJexlScript expectedScript = parseQuery(expected);
+ verifyEquality(pruned, expectedScript);
+ return pruned;
+ } catch (Exception e) {
+ e.printStackTrace();
+ fail("test failed: " + e.getMessage());
+ }
+ return null;
+ }
+
+ private void testExternalPrune(String query, String expected) {
+ testExternalPrune(query, expected, typeMetadata);
}
private void testExternalPrune(String query, String expected, Set ingestTypes) {
- test(query, expected, typeMetadata, ingestTypes);
+ testExternalPrune(query, expected, typeMetadata, ingestTypes);
}
- private void test(String query, String expected, TypeMetadata metadata) {
- test(query, expected, metadata, null);
+ private ASTJexlScript testExternalPrune(String query, String expected, TypeMetadata metadata) {
+ ASTJexlScript script = parseQuery(query);
+ Set ingestTypes = IngestTypeVisitor.getIngestTypes(script, metadata);
+ return testExternalPrune(query, expected, metadata, ingestTypes);
}
- private void test(String query, String expected, TypeMetadata metadata, Set ingestTypes) {
+ private ASTJexlScript testExternalPrune(String query, String expected, TypeMetadata metadata, Set ingestTypes) {
try {
- ASTJexlScript script = JexlASTHelper.parseAndFlattenJexlQuery(query);
+ ASTJexlScript script = parseQuery(query);
+
+ if (ingestTypes == null) {
+ ingestTypes = IngestTypeVisitor.getIngestTypes(script, metadata);
+ }
+
+ if (ingestTypes.contains(IGNORED_TYPE)) {
+ return parseQuery(expected);
+ }
+
ASTJexlScript pruned = (ASTJexlScript) IngestTypePruningVisitor.prune(script, metadata, ingestTypes);
log.info("input : " + query);
@@ -666,16 +940,30 @@ private void test(String query, String expected, TypeMetadata metadata, Set
+ * The following permutations are tested
+ *
+ * - unions, intersections, mix of both
+ * - equality nodes, complex nodes, marker nodes, etc
+ * - TypeMetadata that is fully inclusive, partially inclusive, partially exclusive, fully exclusive
+ *
+ *
+ * Negations and null literals are tested separately due to being exceptional cases.
+ */
+class IngestTypeVisitorIntegrationTest {
+
+ private final static Logger log = LoggerFactory.getLogger(IngestTypeVisitorIntegrationTest.class);
+
+ private JexlQueryGenerator generator;
+ private final Set fields = Set.of("A", "B", "C", "D", "E");
+ private final Set values = Set.of("v1", "v2", "v3", "v4", "v5", "v6");
+
+ private final int maxIterations = 1_000;
+ private final int minTerms = 1;
+ private final int maxTerms = 10;
+ private boolean assertionEnabled = true;
+
+ private final UnionTypes unionOperand = new UnionTypes();
+ private final UnionTypes unionNullOperand = new UnionNullTypes();
+ private final IntersectTypes intersectOperand = new IntersectTypes();
+ private final IntersectNullTypes intersectNullOperand = new IntersectNullTypes();
+
+ @BeforeEach
+ public void setup() {
+ generator = new JexlQueryGenerator(fields, values);
+ generator.disableAllOptions();
+ generator.disableIntersections();
+ generator.disableUnions();
+
+ assertionEnabled = true;
+ }
+
+ @Test
+ void testUnions() {
+ generator.enableUnions();
+ log.info("full inclusive");
+ drive(getFullyInclusiveTypeMetadata(), unionOperand);
+
+ log.info("partial inclusive");
+ drive(getPartiallyInclusiveTypeMetadata(), unionOperand);
+
+ log.info("partial exclusive");
+ drive(getPartiallyExclusiveTypeMetadata(), unionOperand);
+
+ log.info("full exclusive");
+ drive(getFullyExclusiveTypeMetadata(), unionOperand);
+ }
+
+ @Test
+ void testIntersections() {
+ generator.enableIntersections();
+ log.info("full inclusive");
+ drive(getFullyInclusiveTypeMetadata(), intersectOperand);
+
+ log.info("partial inclusive");
+ drive(getPartiallyInclusiveTypeMetadata(), intersectOperand);
+
+ log.info("partial exclusive");
+ drive(getPartiallyExclusiveTypeMetadata(), intersectOperand);
+
+ log.info("full exclusive");
+ drive(getFullyExclusiveTypeMetadata(), intersectOperand);
+ }
+
+ @Test
+ void testEdgeCase_01() {
+ String query = "(A == 'v5' && content:scoredPhrase(B, -1.5, termOffsetMap, 'v5', 'v1'))";
+ assertTypes(query, getFullyInclusiveTypeMetadata(), intersectOperand);
+ }
+
+ @Test
+ void testEdgeCase_02() {
+ String query = "(filter:matchesAtLeastCountOf(1,B,v1) || E == 'v4')";
+ assertTypes(query, getPartiallyInclusiveTypeMetadata(), unionOperand);
+ }
+
+ @Test
+ void testEdgeCase_03() {
+ String query = "(filter:matchesAtLeastCountOf(1,C,v1) || f:count(D,E,A))";
+ assertTypes(query, getPartiallyInclusiveTypeMetadata(), unionOperand);
+ }
+
+ /**
+ * Enable several generator options drive a bunch of queries through
+ *
+ * @param metadata
+ * a TypeMetadata instance
+ * @param operand
+ * a TypeOperand
+ */
+ private void drive(TypeMetadata metadata, TypeOperand operand) {
+ driveInner(metadata, operand);
+
+ log.info("enabling regexes");
+ generator.enableRegexes();
+ driveInner(metadata, operand);
+
+ log.info("enabling content functions");
+ generator.enableContentFunctions();
+ driveInner(metadata, operand);
+
+ log.info("enabling filter functions");
+ generator.enableFilterFunctions();
+ driveInner(metadata, operand);
+
+ log.info("enable grouping functions");
+ generator.enableGroupingFunctions();
+ driveInner(metadata, operand);
+
+ if (operand instanceof UnionTypes) {
+ log.info("enable multi fielded functions for unions");
+ generator.enableMultiFieldedFunctions();
+ driveInner(metadata, operand);
+ generator.disableMultiFieldedFunctions();
+ }
+
+ log.info("enable null literals");
+ generator.enableNullLiterals();
+ if (operand instanceof IntersectTypes) {
+ driveInner(metadata, intersectNullOperand);
+ } else if (operand instanceof UnionTypes) {
+ driveInner(metadata, unionNullOperand);
+ }
+ generator.disableNullLiterals();
+
+ log.info("enable negations");
+ generator.enableNegations();
+ if (operand instanceof IntersectTypes) {
+ driveInner(metadata, intersectNullOperand);
+ } else if (operand instanceof UnionType) {
+ driveInner(metadata, unionNullOperand);
+ }
+ generator.disableNegations();
+
+ generator.disableAllOptions();
+ }
+
+ private void driveInner(TypeMetadata metadata, TypeOperand operand) {
+ for (int i = 0; i < maxIterations; i++) {
+ String query = generator.getQuery(minTerms, maxTerms);
+ assertTypes(query, metadata, operand);
+ }
+ }
+
+ private void assertTypes(String query, TypeMetadata metadata, TypeOperand operand) {
+ JexlNode node = parseQuery(query);
+ Set types = IngestTypeVisitor.getIngestTypes(node, metadata);
+
+ Set queryTypes = operand.apply(node, metadata, this.fields);
+ assertEquals(queryTypes, types, "Bad assertion for query: " + query);
+ }
+
+ @Test
+ void testIntersectionOfUnions() {
+ generator.enableUnions();
+ driveIntersectionOfUnions(getFullyInclusiveTypeMetadata());
+ driveIntersectionOfUnions(getPartiallyInclusiveTypeMetadata());
+ driveIntersectionOfUnions(getPartiallyExclusiveTypeMetadata());
+ driveIntersectionOfUnions(getFullyExclusiveTypeMetadata());
+ }
+
+ @Test
+ void testFullRandom() {
+ generator.enableAllOptions();
+ generator.disableIntersections();
+ generator.enableUnions();
+ assertionEnabled = false;
+
+ driveIntersectionOfUnions(getFullyInclusiveTypeMetadata());
+ driveIntersectionOfUnions(getPartiallyInclusiveTypeMetadata());
+ driveIntersectionOfUnions(getPartiallyExclusiveTypeMetadata());
+ driveIntersectionOfUnions(getFullyExclusiveTypeMetadata());
+ }
+
+ private void driveIntersectionOfUnions(TypeMetadata metadata) {
+ driveIntersectionsOfUnionsInner(metadata);
+
+ log.info("enabling regexes");
+ generator.enableRegexes();
+ driveIntersectionsOfUnionsInner(metadata);
+
+ log.info("enabling content functions");
+ generator.enableContentFunctions();
+ driveIntersectionsOfUnionsInner(metadata);
+
+ log.info("enabling filter functions");
+ generator.enableFilterFunctions();
+ driveIntersectionsOfUnionsInner(metadata);
+
+ log.info("enabling grouping functions");
+ generator.enableGroupingFunctions();
+ driveIntersectionsOfUnionsInner(metadata);
+
+ log.info("enabling multi fielded functions");
+ generator.enableMultiFieldedFunctions();
+ driveIntersectionsOfUnionsInner(metadata);
+ }
+
+ private void driveIntersectionsOfUnionsInner(TypeMetadata metadata) {
+ for (int i = 0; i < maxIterations; i++) {
+ String leftQuery = generator.getQuery(minTerms, maxTerms);
+ String rightQuery = generator.getQuery(minTerms, maxTerms);
+ assertComplexQuery(leftQuery, rightQuery, metadata);
+ }
+ }
+
+ private void assertComplexQuery(String leftQuery, String rightQuery, TypeMetadata metadata) {
+ String fullQuery = leftQuery + " && " + rightQuery;
+
+ JexlNode node = parseQuery(fullQuery);
+ Set queryTypes = IngestTypeVisitor.getIngestTypes(node, metadata);
+
+ if (!assertionEnabled) {
+ // verify no odd cases when the visitor operates on the query
+ return;
+ }
+
+ JexlNode leftNode = parseQuery(leftQuery);
+ JexlNode rightNode = parseQuery(rightQuery);
+ Set expectedTypes = new HashSet<>();
+
+ if (generator.isNegationsEnabled() && (allNegated(leftNode) || allNegated(rightNode))) {
+ // only add types for non-negated branches
+ if (!allNegated((leftNode))) {
+ expectedTypes.addAll(getFieldsAndTypes(leftNode, metadata).values());
+ }
+
+ if (!allNegated(rightNode)) {
+ expectedTypes.addAll(getFieldsAndTypes(rightNode, metadata).values());
+ }
+ } else {
+ expectedTypes.addAll(getFieldsAndTypes(leftNode, metadata).values());
+ expectedTypes.retainAll(getFieldsAndTypes(rightNode, metadata).values());
+ }
+
+ assertEquals(expectedTypes, queryTypes, "Query: " + fullQuery);
+ }
+
+ private JexlNode parseQuery(String query) {
+ try {
+ return JexlASTHelper.parseAndFlattenJexlQuery(query);
+ } catch (Exception e) {
+ fail("Failed to parse query: " + query);
+ throw new IllegalArgumentException("Bad query: " + query);
+ }
+ }
+
+ private Multimap getFieldsAndTypes(JexlNode node, TypeMetadata metadata) {
+ Multimap map = HashMultimap.create();
+ Set queryFields = JexlASTHelper.getIdentifierNames(node);
+ queryFields = Sets.intersection(queryFields, fields); // don't pick up any extras by mistake
+
+ for (String queryField : queryFields) {
+ Set typesForField = metadata.getDataTypesForField(queryField);
+ map.putAll(queryField, typesForField);
+ }
+ return map;
+ }
+
+ private boolean allNegated(JexlNode node) {
+ for (int i = 0; i < node.jjtGetNumChildren(); i++) {
+ JexlNode child = node.jjtGetChild(i);
+ if (!(child instanceof ASTNotNode || child instanceof ASTNENode || child instanceof ASTNRNode)) {
+ return false;
+ }
+
+ // check for FIELD == null
+ if (child instanceof ASTEQNode && JexlASTHelper.getLiteralValueSafely(child) == null) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ // all fields map to the same type
+ private TypeMetadata getFullyInclusiveTypeMetadata() {
+ TypeMetadata metadata = new TypeMetadata();
+ metadata.put("A", "type1", LcType.class.getTypeName());
+ metadata.put("B", "type1", LcType.class.getTypeName());
+ metadata.put("C", "type1", LcType.class.getTypeName());
+ metadata.put("D", "type1", LcType.class.getTypeName());
+ metadata.put("E", "type1", LcType.class.getTypeName());
+ return metadata;
+ }
+
+ // sliding start of ingest types, all overlap
+ private TypeMetadata getPartiallyInclusiveTypeMetadata() {
+ TypeMetadata metadata = new TypeMetadata();
+ metadata.put("A", "type1", LcType.class.getTypeName());
+ metadata.put("A", "type2", LcType.class.getTypeName());
+ metadata.put("A", "type3", LcType.class.getTypeName());
+ metadata.put("A", "type4", LcType.class.getTypeName());
+ metadata.put("A", "type5", LcType.class.getTypeName());
+ metadata.put("B", "type2", LcType.class.getTypeName());
+ metadata.put("B", "type3", LcType.class.getTypeName());
+ metadata.put("B", "type4", LcType.class.getTypeName());
+ metadata.put("B", "type5", LcType.class.getTypeName());
+ metadata.put("C", "type3", LcType.class.getTypeName());
+ metadata.put("C", "type4", LcType.class.getTypeName());
+ metadata.put("C", "type5", LcType.class.getTypeName());
+ metadata.put("D", "type4", LcType.class.getTypeName());
+ metadata.put("D", "type5", LcType.class.getTypeName());
+ metadata.put("E", "type5", LcType.class.getTypeName());
+ return metadata;
+ }
+
+ // sliding window of ingest types
+ private TypeMetadata getPartiallyExclusiveTypeMetadata() {
+ TypeMetadata metadata = new TypeMetadata();
+ metadata.put("A", "type1", LcType.class.getTypeName());
+ metadata.put("A", "type2", LcType.class.getTypeName());
+ metadata.put("A", "type3", LcType.class.getTypeName());
+ metadata.put("B", "type2", LcType.class.getTypeName());
+ metadata.put("B", "type3", LcType.class.getTypeName());
+ metadata.put("B", "type4", LcType.class.getTypeName());
+ metadata.put("C", "type3", LcType.class.getTypeName());
+ metadata.put("C", "type4", LcType.class.getTypeName());
+ metadata.put("C", "type5", LcType.class.getTypeName());
+ metadata.put("D", "type4", LcType.class.getTypeName());
+ metadata.put("D", "type5", LcType.class.getTypeName());
+ metadata.put("D", "type6", LcType.class.getTypeName());
+ metadata.put("E", "type5", LcType.class.getTypeName());
+ metadata.put("E", "type6", LcType.class.getTypeName());
+ metadata.put("E", "type1", LcType.class.getTypeName());
+ return metadata;
+ }
+
+ // no field maps to the same type
+ private TypeMetadata getFullyExclusiveTypeMetadata() {
+ TypeMetadata metadata = new TypeMetadata();
+ metadata.put("A", "type1", LcType.class.getTypeName());
+ metadata.put("B", "type2", LcType.class.getTypeName());
+ metadata.put("C", "type3", LcType.class.getTypeName());
+ metadata.put("D", "type4", LcType.class.getTypeName());
+ metadata.put("E", "type5", LcType.class.getTypeName());
+ return metadata;
+ }
+
+ private interface TypeOperand {
+ Set apply(JexlNode node, TypeMetadata metadata, Set fields);
+ }
+
+ private static class UnionTypes implements TypeOperand {
+
+ @Override
+ public Set apply(JexlNode node, TypeMetadata metadata, Set fields) {
+ Set types = new HashSet<>();
+ Set queryFields = JexlASTHelper.getIdentifierNames(node);
+ queryFields = Sets.intersection(queryFields, fields);
+ for (String queryField : queryFields) {
+ types.addAll(metadata.getDataTypesForField(queryField));
+ }
+ return types;
+ }
+ }
+
+ private static class IntersectTypes implements TypeOperand {
+
+ @Override
+ public Set apply(JexlNode node, TypeMetadata metadata, Set fields) {
+ Set types = new HashSet<>();
+ Set queryFields = JexlASTHelper.getIdentifierNames(node);
+ queryFields = Sets.intersection(queryFields, fields);
+ for (String queryField : queryFields) {
+ if (types.isEmpty()) {
+ types.addAll(metadata.getDataTypesForField(queryField));
+ } else {
+ types.retainAll(metadata.getDataTypesForField(queryField));
+ if (types.isEmpty()) {
+ return new HashSet<>();
+ }
+ }
+ }
+ return types;
+ }
+ }
+
+ /**
+ * Handles null, not null, and negated terms
+ */
+ private static class IntersectNullTypes extends IntersectTypes {
+
+ @Override
+ public Set apply(JexlNode node, TypeMetadata metadata, Set fields) {
+ JexlNode flattened = TreeFlatteningRebuildingVisitor.flatten(node);
+ assertInstanceOf(ASTJexlScript.class, flattened);
+
+ JexlNode and = JexlASTHelper.dereference(flattened.jjtGetChild(0));
+ List children = new LinkedList<>();
+
+ if (!(and instanceof ASTAndNode)) {
+ // add single leaf
+ children.add(JexlASTHelper.dereference(and));
+ } else {
+ for (int i = 0; i < and.jjtGetNumChildren(); i++) {
+ children.add(JexlASTHelper.dereference(and.jjtGetChild(i)));
+ }
+ }
+
+ // extract types
+ boolean intersectedToZero = false;
+ Set types = new HashSet<>();
+ for (JexlNode child : children) {
+ Set nodeFields = JexlASTHelper.getIdentifierNames(child);
+ Set nodeTypes = new HashSet<>();
+ for (String nodeField : nodeFields) {
+ nodeTypes.addAll(metadata.getDataTypesForField(nodeField));
+ }
+
+ if (isNullEquality(child) || isNotNullEquality(child) || isNegated(child)) {
+ continue;
+ }
+
+ if (types.isEmpty()) {
+ types.addAll(nodeTypes);
+ } else {
+ types.retainAll(nodeTypes);
+ if (types.isEmpty()) {
+ intersectedToZero = true;
+ break;
+ }
+ }
+ }
+
+ if (!intersectedToZero && types.isEmpty()) {
+ types.add(IngestTypeVisitor.IGNORED_TYPE);
+ }
+
+ return types;
+ }
+ }
+
+ /**
+ * Handles null, not null, and negated terms
+ */
+ private static class UnionNullTypes extends UnionTypes {
+
+ @Override
+ public Set apply(JexlNode node, TypeMetadata metadata, Set fields) {
+ JexlNode flattened = TreeFlatteningRebuildingVisitor.flatten(node);
+ assertInstanceOf(ASTJexlScript.class, flattened);
+
+ JexlNode or = JexlASTHelper.dereference(flattened.jjtGetChild(0));
+ List children = new LinkedList<>();
+
+ if (!(or instanceof ASTOrNode)) {
+ // dealing with a leaf, add it
+ children.add(JexlASTHelper.dereference(or));
+ } else {
+ for (int i = 0; i < or.jjtGetNumChildren(); i++) {
+ JexlNode child = or.jjtGetChild(i);
+ child = JexlASTHelper.dereference(child);
+ children.add(child);
+ }
+ }
+
+ // extract types
+ Set types = new HashSet<>();
+ for (JexlNode child : children) {
+ // when performing an internal visit of a union, skip null and not null equality nodes
+ if (!isNullEquality(child) && !isNotNullEquality(child) && !isNegated(child)) {
+ Set nodeFields = JexlASTHelper.getIdentifierNames(child);
+ for (String nodeField : nodeFields) {
+ types.addAll(metadata.getDataTypesForField(nodeField));
+ }
+ }
+ }
+
+ if (types.isEmpty()) {
+ // if no node returned a type, then this whole union can be ignored
+ types.add(IngestTypeVisitor.IGNORED_TYPE);
+ }
+ return types;
+ }
+ }
+
+ private static boolean isNullEquality(JexlNode node) {
+ node = JexlASTHelper.dereference(node);
+
+ if (node instanceof ASTEQNode) {
+ return JexlASTHelper.getLiteralValueSafely(node) == null;
+ }
+
+ return false;
+ }
+
+ private static boolean isNotNullEquality(JexlNode node) {
+ node = JexlASTHelper.dereference(node);
+
+ if (node instanceof ASTNotNode) {
+ node = node.jjtGetChild(0);
+ node = JexlASTHelper.dereference(node);
+ } else {
+ return false;
+ }
+
+ if (node instanceof ASTEQNode) {
+ return JexlASTHelper.getLiteralValueSafely(node) == null;
+ }
+
+ return false;
+ }
+
+ private static boolean isNegated(JexlNode node) {
+ node = JexlASTHelper.dereference(node);
+ return node instanceof ASTNotNode;
+ }
+}
diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/IngestTypeVisitorTest.java b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/IngestTypeVisitorTest.java
index 3a0c8819a4..9d2037d32e 100644
--- a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/IngestTypeVisitorTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/IngestTypeVisitorTest.java
@@ -1,9 +1,12 @@
package datawave.query.jexl.visitors;
+import static datawave.query.jexl.visitors.IngestTypeVisitor.IGNORED_TYPE;
+import static datawave.query.jexl.visitors.IngestTypeVisitor.UNKNOWN_TYPE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
import java.util.Collections;
+import java.util.HashSet;
import java.util.Set;
import java.util.TreeSet;
@@ -17,6 +20,7 @@
import datawave.data.type.LcType;
import datawave.query.jexl.JexlASTHelper;
+import datawave.query.jexl.util.JexlQueryGenerator;
import datawave.query.util.TypeMetadata;
class IngestTypeVisitorTest {
@@ -29,6 +33,11 @@ class IngestTypeVisitorTest {
private final Set xType = Collections.singleton("ingestType1");
private final Set yType = Collections.singleton("ingestType2");
private final Set zType = Collections.singleton("ingestType3");
+ private final Set identifierType = Collections.singleton("ingestType1");
+
+ // special types
+ private final Set unknownType = Set.of(UNKNOWN_TYPE);
+ private final Set ignoredType = Set.of(IGNORED_TYPE);
@BeforeAll
public static void setup() {
@@ -55,27 +64,390 @@ public static void setup() {
typeMetadata.put("Y", "ingestType2", LcType.class.getName());
// Z is 3
typeMetadata.put("Z", "ingestType3", LcType.class.getName());
+ // $123 and $234 as identifiers
+ typeMetadata.put("123", "ingestType1", LcType.class.getName());
+ typeMetadata.put("234", "ingestType1", LcType.class.getName());
}
@Test
- void testSingleTerms() {
- String query = "A == '1'";
- assertSingleNode(query, aTypes);
+ void testLeafEq() {
+ // EQ node with known field
+ assertStandard("A == '1'", aTypes);
+ assertInternal("A == '1'", aTypes);
+ assertExternal("A == '1'", aTypes);
+
+ // EQ node with known identifier
+ assertStandard("$123 == '1'", identifierType);
+ assertInternal("$123 == '1'", identifierType);
+ assertExternal("$123 == '1'", identifierType);
+
+ // EQ node with unknown field
+ assertStandard("D == '1'", Set.of(UNKNOWN_TYPE));
+ assertInternal("D == '1'", Set.of(UNKNOWN_TYPE));
+ assertExternal("D == '1'", Set.of(UNKNOWN_TYPE));
+
+ // EQ node with unknown identifier
+ assertStandard("$987 == '1'", Set.of(UNKNOWN_TYPE));
+ assertInternal("$987 == '1'", Set.of(UNKNOWN_TYPE));
+ assertExternal("$987 == '1'", Set.of(UNKNOWN_TYPE));
+ }
+
+ @Test
+ void testLeafNe() {
+ // NE node with known field
+ assertStandard("A != '1'", aTypes);
+ assertInternal("A != '1'", aTypes);
+ assertExternal("A != '1'", aTypes);
+
+ // NE node with known identifier
+ assertStandard("$123 != '1'", identifierType);
+ assertInternal("$123 != '1'", identifierType);
+ assertExternal("$123 != '1'", identifierType);
+
+ // NE node with unknown field
+ assertStandard("D != '1'", Set.of(UNKNOWN_TYPE));
+ assertInternal("D != '1'", Set.of(UNKNOWN_TYPE));
+ assertExternal("D != '1'", Set.of(UNKNOWN_TYPE));
+
+ // NE node with unknown identifier
+ assertStandard("$987 != '1'", Set.of(UNKNOWN_TYPE));
+ assertInternal("$987 != '1'", Set.of(UNKNOWN_TYPE));
+ assertExternal("$987 != '1'", Set.of(UNKNOWN_TYPE));
+ }
+
+ @Test
+ void testLeafEr() {
+ // ER node with known field
+ assertStandard("A =~ 'ba.*'", aTypes);
+ assertInternal("A =~ 'ba.*'", aTypes);
+ assertExternal("A =~ 'ba.*'", aTypes);
+
+ // ER node with known identifier
+ assertStandard("$123 =~ 'ba.*'", identifierType);
+ assertInternal("$123 =~ 'ba.*'", identifierType);
+ assertExternal("$123 =~ 'ba.*'", identifierType);
+
+ // ER node with unknown field
+ assertStandard("D =~ 'ba.*'", Set.of(UNKNOWN_TYPE));
+ assertInternal("D =~ 'ba.*'", Set.of(UNKNOWN_TYPE));
+ assertExternal("D =~ 'ba.*'", Set.of(UNKNOWN_TYPE));
+
+ // ER node with unknown identifier
+ assertStandard("$987 =~ 'ba.*'", Set.of(UNKNOWN_TYPE));
+ assertInternal("$987 =~ 'ba.*'", Set.of(UNKNOWN_TYPE));
+ assertExternal("$987 =~ 'ba.*'", Set.of(UNKNOWN_TYPE));
+ }
+
+ @Test
+ void testLeafNr() {
+ // NR node with known field
+ assertStandard("A !~ 'ba.*'", aTypes);
+ assertInternal("A !~ 'ba.*'", aTypes);
+ assertExternal("A !~ 'ba.*'", aTypes);
+
+ // NR node with known identifier
+ assertStandard("$123 !~ 'ba.*'", identifierType);
+ assertInternal("$123 !~ 'ba.*'", identifierType);
+ assertExternal("$123 !~ 'ba.*'", identifierType);
+
+ // NR node with unknown field
+ assertStandard("D !~ 'ba.*'", Set.of(UNKNOWN_TYPE));
+ assertInternal("D !~ 'ba.*'", Set.of(UNKNOWN_TYPE));
+ assertExternal("D !~ 'ba.*'", Set.of(UNKNOWN_TYPE));
+
+ // NR node with unknown identifier
+ assertStandard("$987 !~ 'ba.*'", Set.of(UNKNOWN_TYPE));
+ assertInternal("$987 !~ 'ba.*'", Set.of(UNKNOWN_TYPE));
+ assertExternal("$987 !~ 'ba.*'", Set.of(UNKNOWN_TYPE));
+ }
+
+ @Test
+ void testLeafGT() {
+ // GT node with known field
+ assertStandard("A > '1'", aTypes);
+ assertInternal("A > '1'", aTypes);
+ assertExternal("A > '1'", aTypes);
+
+ // GT node with known identifier
+ assertStandard("$123 > '1'", identifierType);
+ assertInternal("$123 > '1'", identifierType);
+ assertExternal("$123 > '1'", identifierType);
+
+ // GT node with unknown field
+ assertStandard("D > '1'", unknownType);
+ assertInternal("D > '1'", unknownType);
+ assertExternal("D > '1'", unknownType);
+
+ // GT node with missing identifier
+ assertStandard("$987 > '1'", unknownType);
+ assertInternal("$987 > '1'", unknownType);
+ assertExternal("$987 > '1'", unknownType);
+ }
+
+ @Test
+ void testLeafLT() {
+ // LT node with known field
+ assertStandard("A < '1'", aTypes);
+ assertInternal("A < '1'", aTypes);
+ assertExternal("A < '1'", aTypes);
+
+ // LT node with known identifier
+ assertStandard("$123 < '1'", identifierType);
+ assertInternal("$123 < '1'", identifierType);
+ assertExternal("$123 < '1'", identifierType);
+
+ // LT node with unknown field
+ assertStandard("D < '1'", unknownType);
+ assertInternal("D < '1'", unknownType);
+ assertExternal("D < '1'", unknownType);
+
+ // LT node with unknown identifier
+ assertStandard("$987 < '1'", unknownType);
+ assertInternal("$987 < '1'", unknownType);
+ assertExternal("$987 < '1'", unknownType);
+ }
+
+ @Test
+ void testLeafGE() {
+ // GE node with known field
+ assertStandard("A >= '1'", aTypes);
+ assertInternal("A >= '1'", aTypes);
+ assertExternal("A >= '1'", aTypes);
+
+ // GE node with known identifier
+ assertStandard("$123 >= '1'", identifierType);
+ assertInternal("$123 >= '1'", identifierType);
+ assertExternal("$123 >= '1'", identifierType);
+
+ // GE node with unknown field
+ assertStandard("D >= '1'", unknownType);
+ assertInternal("D >= '1'", unknownType);
+ assertExternal("D >= '1'", unknownType);
+
+ // GE node with unknown identifier
+ assertStandard("$987 >= '1'", unknownType);
+ assertInternal("$987 >= '1'", unknownType);
+ assertExternal("$987 >= '1'", unknownType);
+ }
+
+ @Test
+ void testLeafLE() {
+ // LE node with known field
+ assertStandard("A <= '1'", aTypes);
+ assertInternal("A <= '1'", aTypes);
+ assertExternal("A <= '1'", aTypes);
+
+ // LE node with known identifier
+ assertStandard("$123 <= '1'", identifierType);
+ assertInternal("$123 <= '1'", identifierType);
+ assertExternal("$123 <= '1'", identifierType);
+
+ // LE node with unknown field
+ assertStandard("D <= '1'", unknownType);
+ assertInternal("D <= '1'", unknownType);
+ assertExternal("D <= '1'", unknownType);
+
+ // LE node with unknown identifier
+ assertStandard("$987 <= '1'", unknownType);
+ assertInternal("$987 <= '1'", unknownType);
+ assertExternal("$987 <= '1'", unknownType);
+ }
+
+ @Test
+ void testNot() {
+ // Not node with known field
+ assertStandard("!(A == '1')", Set.of(IGNORED_TYPE));
+ assertInternal("!(A == '1')", Set.of(IGNORED_TYPE));
+ assertExternal("!(A >= '1')", aTypes);
+
+ // Not node with known identifier
+ assertStandard("!($123 == '1')", Set.of(IGNORED_TYPE));
+ assertInternal("!($123 == '1')", Set.of(IGNORED_TYPE));
+ assertExternal("!($123 >= '1')", identifierType);
+
+ // Not node with unknown field
+ assertStandard("!(D == '1')", Set.of(IGNORED_TYPE));
+ assertInternal("!(D == '1')", Set.of(IGNORED_TYPE));
+ assertExternal("!(D >= '1')", unknownType);
+
+ // Not node with unknown identifier
+ assertStandard("!($987 == '1')", Set.of(IGNORED_TYPE));
+ assertInternal("!($987 == '1')", Set.of(IGNORED_TYPE));
+ assertExternal("!($987 >= '1')", unknownType);
+ }
+
+ @Test
+ void testNullLiteral() {
+ // null literal with known field
+ assertStandard("A == null", Set.of(IGNORED_TYPE));
+ assertInternal("A == null", Set.of(IGNORED_TYPE));
+ assertExternal("A == null", aTypes);
+
+ // null literal with known identifier
+ assertStandard("$123 == null", Set.of(IGNORED_TYPE));
+ assertInternal("$123 == null", Set.of(IGNORED_TYPE));
+ assertExternal("$123 == null", identifierType);
+
+ // null literal with unknown field
+ assertStandard("D == null", Set.of(IGNORED_TYPE));
+ assertInternal("D == null", Set.of(IGNORED_TYPE));
+ assertExternal("D == null", unknownType);
+
+ // null literal with unknown identifier
+ assertStandard("$987 == null", Set.of(IGNORED_TYPE));
+ assertInternal("$987 == null", Set.of(IGNORED_TYPE));
+ assertExternal("$987 == null", unknownType);
+ }
+
+ @Test
+ void testNotNullLiteral() {
+ // not null literal with known field
+ assertStandard("!(A == null)", Set.of(IGNORED_TYPE));
+ assertInternal("!(A == null)", Set.of(IGNORED_TYPE));
+ assertExternal("!(A == null)", aTypes);
+
+ // not null literal with known literal
+ assertStandard("!($123 == null)", Set.of(IGNORED_TYPE));
+ assertInternal("!($123 == null)", Set.of(IGNORED_TYPE));
+ assertExternal("!($123 == null)", identifierType);
+
+ // not null literal with unknown field
+ assertStandard("!(D == null)", Set.of(IGNORED_TYPE));
+ assertInternal("!(D == null)", Set.of(IGNORED_TYPE));
+ assertExternal("!(D == null)", Set.of(UNKNOWN_TYPE));
+
+ // not null literal with unknown identifier
+ assertStandard("!($987 == null)", Set.of(IGNORED_TYPE));
+ assertInternal("!($987 == null)", Set.of(IGNORED_TYPE));
+ assertExternal("!($987 == null)", Set.of(UNKNOWN_TYPE));
+ }
+
+ @Test
+ void testContentFunctionNoField() {
+ assertStandard("content:phrase(termOffsetMap, 'foo', 'bar')", unknownType);
+ assertInternal("content:phrase(termOffsetMap, 'foo', 'bar')", unknownType);
+ assertExternal("content:phrase(termOffsetMap, 'foo', 'bar')", unknownType);
+ }
+
+ @Test
+ void testContentFunctionSingleField() {
+ // function known field
+ assertStandard("content:phrase(A, termOffsetMap, 'foo', 'bar')", aTypes);
+ assertInternal("content:phrase(A, termOffsetMap, 'foo', 'bar')", aTypes);
+ assertExternal("content:phrase(A, termOffsetMap, 'foo', 'bar')", aTypes);
+
+ // function known identifier
+ assertStandard("content:phrase($123, termOffsetMap, 'foo', 'bar')", identifierType);
+ assertInternal("content:phrase($123, termOffsetMap, 'foo', 'bar')", identifierType);
+ assertExternal("content:phrase($123, termOffsetMap, 'foo', 'bar')", identifierType);
+
+ // function unknown field
+ assertStandard("content:phrase(D, termOffsetMap, 'foo', 'bar')", unknownType);
+ assertInternal("content:phrase(D, termOffsetMap, 'foo', 'bar')", unknownType);
+ assertExternal("content:phrase(D, termOffsetMap, 'foo', 'bar')", unknownType);
+
+ // function unknown identifier
+ assertStandard("content:phrase($987, termOffsetMap, 'foo', 'bar')", unknownType);
+ assertInternal("content:phrase($987, termOffsetMap, 'foo', 'bar')", unknownType);
+ assertExternal("content:phrase($987, termOffsetMap, 'foo', 'bar')", unknownType);
+ }
+
+ @Test
+ void testContentFunctionMultiField() {
+ // function known fields
+ assertStandard("content:phrase((A || B), termOffsetMap, 'foo', 'bar')", aTypes);
+ assertInternal("content:phrase((A || B), termOffsetMap, 'foo', 'bar')", aTypes);
+ assertExternal("content:phrase((A || B), termOffsetMap, 'foo', 'bar')", aTypes);
+
+ // function known identifiers
+ assertStandard("content:phrase(($123 || $234), termOffsetMap, 'foo', 'bar')", identifierType);
+ assertInternal("content:phrase(($123 || $234), termOffsetMap, 'foo', 'bar')", identifierType);
+ assertExternal("content:phrase(($123 || $234), termOffsetMap, 'foo', 'bar')", identifierType);
- query = "B == '2'";
- assertSingleNode(query, bTypes);
+ // function unknown fields
+ assertStandard("content:phrase((D || E), termOffsetMap, 'foo', 'bar')", unknownType);
+ assertInternal("content:phrase((D || E), termOffsetMap, 'foo', 'bar')", unknownType);
+ assertExternal("content:phrase((D || E), termOffsetMap, 'foo', 'bar')", unknownType);
- query = "C == '3'";
- assertSingleNode(query, cTypes);
+ // function unknown identifiers
+ assertStandard("content:phrase(($987 || $876), termOffsetMap, 'foo', 'bar')", unknownType);
+ assertInternal("content:phrase(($987 || $876), termOffsetMap, 'foo', 'bar')", unknownType);
+ assertExternal("content:phrase(($987 || $876), termOffsetMap, 'foo', 'bar')", unknownType);
- query = "X == '7'";
- assertSingleNode(query, xType);
+ // TODO -- run through the different combinations
- query = "Y == '8'";
- assertSingleNode(query, yType);
+ // known field and known identifier
- query = "Z == '9'";
- assertSingleNode(query, zType);
+ // known field and unknown field
+
+ // known identifier and unknown field
+
+ // known identifier and unknown identifier
+ }
+
+ // verify fields the correct types
+ @Test
+ void testSingleTerms() {
+ // A field
+ assertStandard("A == '1'", aTypes);
+ assertInternal("A == '1'", aTypes);
+ assertExternal("A == '1'", aTypes);
+
+ // B field
+ assertStandard("B == '2'", bTypes);
+ assertInternal("B == '2'", bTypes);
+ assertExternal("B == '2'", bTypes);
+
+ // C field
+ assertStandard("C == '3'", cTypes);
+ assertInternal("C == '3'", cTypes);
+ assertExternal("C == '3'", cTypes);
+
+ // X field
+ assertStandard("X == '7'", xType);
+ assertInternal("X == '7'", xType);
+ assertExternal("X == '7'", xType);
+
+ // Y field
+ assertStandard("Y == '8'", yType);
+ assertInternal("Y == '8'", yType);
+ assertExternal("Y == '8'", yType);
+
+ // Z field
+ assertStandard("Z == '9'", zType);
+ assertInternal("Z == '9'", zType);
+ assertExternal("Z == '9'", zType);
+
+ // $123 identifier
+ assertStandard("$123 == 'a'", identifierType);
+ assertInternal("$123 == 'a'", identifierType);
+ assertExternal("$123 == 'a'", identifierType);
+
+ // $234 identifier
+ assertStandard("$234 == 'b'", identifierType);
+ assertInternal("$234 == 'b'", identifierType);
+ assertExternal("$234 == 'b'", identifierType);
+ }
+
+ @Test
+ public void testIdentifierNotNull() {
+ TypeMetadata metadata = new TypeMetadata();
+ metadata.put("123", "type-a", LcType.class.getName());
+
+ // identifier
+ assertStandard("!($123 == null)", Set.of(IGNORED_TYPE), metadata);
+ assertInternal("!($123 == null)", Set.of(IGNORED_TYPE), metadata);
+ assertExternal("!($123 >= null)", Set.of("type-a"), metadata);
+
+ // identifier not found, should flip from IGNORED to UNKNOWN
+ assertStandard("!($987 == null)", Set.of(IGNORED_TYPE), metadata);
+ assertInternal("!($987 == null)", Set.of(IGNORED_TYPE), metadata);
+ assertExternal("!($987 >= null)", Set.of(UNKNOWN_TYPE), metadata);
+ }
+
+ @Test
+ void testSingleton() {
+ assertInternal("!(A == '1')", Set.of(IGNORED_TYPE));
}
@Test
@@ -129,10 +501,33 @@ void testNestedJunctionLogic() {
}
@Test
- void testNegatedLogic() {
+ void testIntersectionsWithNegatedTerm() {
// negated terms do not contribute to an intersection
assertSingleNode("A == '1' && !(B == '2')", aTypes);
assertSingleNode("B == '2' && !(C == '3')", bTypes);
+
+ // internal visit does not consider negated terms
+ assertInternal("A == '1' && !(B == '2')", aTypes);
+ assertInternal("B == '2' && !(C == '3')", bTypes);
+
+ // external visit considers negated terms
+ assertExternal("A == '1' && !(B == '2')", Sets.intersection(aTypes, bTypes));
+ assertExternal("B == '2' && !(C == '3')", Set.of());
+ }
+
+ @Test
+ void testIntersectionsWithNullTerm() {
+ // negated terms do not contribute to an intersection
+ assertSingleNode("A == '1' && !(B == null)", aTypes);
+ assertSingleNode("B == '2' && !(C == null)", bTypes);
+
+ // internal visit does not consider negated terms
+ assertInternal("A == '1' && !(B == null)", aTypes);
+ assertInternal("B == '2' && !(C == null)", bTypes);
+
+ // external visit considers negated terms
+ assertExternal("A == '1' && !(B == null)", Sets.intersection(aTypes, bTypes));
+ assertExternal("B == '2' && !(C == null)", Set.of());
}
@Test
@@ -145,8 +540,8 @@ void testUnknownType() {
@Test
void testEqNull() {
- assertSingleNode("A == null", aTypes);
- assertSingleNode("!(A == null)", aTypes);
+ assertSingleNode("A == null", Set.of(IGNORED_TYPE));
+ assertSingleNode("!(A == null)", Set.of(IGNORED_TYPE));
}
@Test
@@ -293,13 +688,159 @@ void testFilterFunctionExcludeExpandedIntoMutuallyExclusiveFields() {
@Test
void testUnionOfNegatedTerms() {
String query = "!(A == '1') || !(B == '2') || !(C == '3')";
- test(query, Sets.union(aTypes, bTypes));
+ test(query, Set.of(IGNORED_TYPE));
}
@Test
void testUnionOfNotNullTerms() {
String query = "!(A == null) || !(B == null) || !(C == null)";
- test(query, Sets.union(aTypes, bTypes));
+ test(query, Set.of(IGNORED_TYPE));
+ }
+
+ @Test
+ void testIntersectionEdgeCases() {
+ // all null literals
+ test("A == null && B == null", Set.of(IGNORED_TYPE));
+ // all negated
+ test("!(A == '1') && !(B == '2')", Set.of(IGNORED_TYPE));
+ // mix of negated terms and null literals
+ test("!(A == '1') && B == null", Set.of(IGNORED_TYPE));
+
+ // anchor term and negated term
+ test("A == '1' && !(B == '2')", aTypes);
+ test("!(B == '2') && A == '1'", aTypes);
+ // anchor term and null literal
+ test("A == '1' && B == null", aTypes);
+ test("B == null && A == '1' ", aTypes);
+ // anchor term and negated null literal
+ test("A == '1' && !(B == null)", aTypes);
+ test("!(B == null) && A == '1'", aTypes);
+ }
+
+ // @formatter:off
+ // proposed rules
+ // 1. leaf nodes can return NO_TYPE or UNKNOWN_TYPE
+ // 2. intersection/union logic knows how to return this
+ // 3. pruning logic knows how to ignore it
+ // 4. will only prune exclusive, single node negations [A == '1' && !(B == '2')]
+
+ // pruning needs to know if it's internal or external pruning.
+ // 1. internal pruning rules are different vs. external when it comes to intersecting negated/null terms
+ // for example; A == '1' && !(B == null)
+ // in isolation this is not executable and should prune. but if the context is
+ // this: (B == '0' && (Z == '9' || (A == '1' && !(B == null))...then who knows
+ // @formatter:on
+
+ @Test
+ void testUnionEdgeCases() {
+ // all null literals
+ test("A == null || B == null", Set.of(IGNORED_TYPE));
+ // all negated
+ test("!(A == '1') || !(B == '2')", Set.of(IGNORED_TYPE));
+ // mix of negated terms and null literals
+ test("!(A == '1') || B == null", Set.of(IGNORED_TYPE));
+
+ // anchor term and negated term
+ test("A == '1' || !(B == '2')", aTypes);
+ test("!(B == '2') || A == '1'", aTypes);
+ // anchor term and null literal
+ test("A == '1' || B == null", aTypes);
+ test("B == null || A == '1'", aTypes);
+ // anchor term and negated null literal
+ test("A == '1' || !(B == null)", aTypes);
+ test("!(B == null) || A == '1'", aTypes);
+ }
+
+ @Test
+ void testRandomUnionOfSimpleTerms() throws Exception {
+ TypeMetadata metadata = new TypeMetadata();
+ metadata.put("A", "type1", LcType.class.getTypeName());
+ metadata.put("B", "type2", LcType.class.getTypeName());
+ metadata.put("C", "type3", LcType.class.getTypeName());
+ metadata.put("D", "type4", LcType.class.getTypeName());
+ metadata.put("E", "type5", LcType.class.getTypeName());
+ metadata.put("F", "type6", LcType.class.getTypeName());
+
+ Set fields = Set.of("A", "B", "C", "D", "E", "F");
+ Set values = Set.of("v1", "v2", "v3", "v4", "v5");
+
+ JexlQueryGenerator generator = new JexlQueryGenerator(fields, values);
+ generator.disableAllOptions();
+ generator.disableIntersections();
+ generator.enableUnions();
+
+ int minTerms = 2;
+ int maxTerms = 10;
+ int iterations = 1000;
+ for (int i = 0; i < iterations; i++) {
+ String query = generator.getQuery(minTerms, maxTerms);
+ JexlNode node = JexlASTHelper.parseAndFlattenJexlQuery(query);
+ Set types = IngestTypeVisitor.getIngestTypes(node, metadata);
+
+ Set queryFields = JexlASTHelper.getIdentifierNames(node);
+ Set queryTypes = new HashSet<>();
+ for (String queryField : queryFields) {
+ Set typesForField = metadata.getDataTypesForField(queryField);
+ if (typesForField != null) {
+ queryTypes.addAll(typesForField);
+ }
+ }
+
+ if (types.isEmpty() || types.size() != queryTypes.size()) {
+ Set typesForTesting = IngestTypeVisitor.getIngestTypes(node, metadata);
+ int k = 0;
+ }
+ assertEquals(queryTypes, types);
+ }
+ }
+
+ @Test
+ void testRandomIntersectionOfSimpleTerms() throws Exception {
+ TypeMetadata metadata = new TypeMetadata();
+ metadata.put("A", "type1", LcType.class.getTypeName());
+ metadata.put("A", "type2", LcType.class.getTypeName());
+ metadata.put("B", "type2", LcType.class.getTypeName());
+ metadata.put("B", "type3", LcType.class.getTypeName());
+ metadata.put("C", "type3", LcType.class.getTypeName());
+ metadata.put("C", "type4", LcType.class.getTypeName());
+
+ Set fields = Set.of("A", "B", "C");
+ Set values = Set.of("v1", "v2", "v3", "v4", "v5");
+
+ JexlQueryGenerator generator = new JexlQueryGenerator(fields, values);
+ generator.disableAllOptions();
+ generator.disableUnions();
+ generator.enableIntersections();
+
+ int minTerms = 2;
+ int maxTerms = 10;
+ int iterations = 1000;
+ for (int i = 0; i < iterations; i++) {
+ String query = generator.getQuery(minTerms, maxTerms);
+ JexlNode node = JexlASTHelper.parseAndFlattenJexlQuery(query);
+ Set types = IngestTypeVisitor.getIngestTypes(node, metadata);
+
+ Set queryFields = JexlASTHelper.getIdentifierNames(node);
+ Set queryTypes = new HashSet<>();
+ for (String queryField : queryFields) {
+ Set typesForField = metadata.getDataTypesForField(queryField);
+
+ if (queryTypes.isEmpty()) {
+ queryTypes.addAll(typesForField);
+ } else {
+ queryTypes.retainAll(typesForField);
+ if (queryTypes.isEmpty()) {
+ break;
+ }
+ }
+ }
+
+ if (types.size() != queryTypes.size()) {
+ Set typesForTesting = IngestTypeVisitor.getIngestTypes(node, metadata);
+ int k = 0;
+ }
+ assertEquals(queryTypes, types);
+ }
}
private void assertSingleNode(String query, Set expectedIngestTypes) {
@@ -312,31 +853,120 @@ private void assertSingleNode(String query, Set expectedIngestTypes, Typ
assertEquals(expectedIngestTypes, ingestTypes);
}
- private void assertJunction(String query, Set[] expectedIngestTypes) {
- assertJunction(query, expectedIngestTypes, typeMetadata);
+ private void test(String query, Set expected) {
+ assertStandard(query, expected);
}
- private void assertJunction(String query, Set[] expectedIngestTypes, TypeMetadata typeMetadata) {
- JexlNode node = parseQuery(query);
+ private void test(String query, Set expected, TypeMetadata typeMetadata) {
+ assertStandard(query, expected, typeMetadata);
+ }
- assertEquals(expectedIngestTypes.length, node.jjtGetNumChildren(), "Child array and expected type array had differing lengths!");
- for (int i = 0; i < node.jjtGetNumChildren(); i++) {
- JexlNode child = node.jjtGetChild(i);
- Set ingestTypes = IngestTypeVisitor.getIngestTypes(child, typeMetadata);
- assertEquals(expectedIngestTypes[i], ingestTypes);
- }
+ /**
+ * Standard entry via {@link IngestTypeVisitor#getIngestTypes(JexlNode, TypeMetadata)}
+ *
+ * @param query
+ * the query
+ * @param expected
+ * the expected types
+ */
+ private void assertStandard(String query, Set expected) {
+ assertStandard(query, expected, typeMetadata);
}
- private void test(String query, Set expected) {
- test(query, expected, typeMetadata);
+ /**
+ * Standard entry via {@link IngestTypeVisitor#getIngestTypes(JexlNode, TypeMetadata)}
+ *
+ * @param query
+ * the query
+ * @param expected
+ * the expected types
+ * @param metadata
+ * the {@link TypeMetadata}
+ */
+ private void assertStandard(String query, Set expected, TypeMetadata metadata) {
+ ASTJexlScript script = parseQuery(query);
+ Set types = IngestTypeVisitor.getIngestTypes(script, metadata);
+ assertEquals(new TreeSet<>(expected), new TreeSet<>(types)); // sorted sets make differences easy to spot
}
- private void test(String query, Set expected, TypeMetadata typeMetadata) {
+ /**
+ * External entry via {@link IngestTypeVisitor#getIngestTypes(JexlNode, boolean)} with boolean set to true
+ *
+ * @param query
+ * the query
+ * @param expected
+ * the expected types
+ */
+ private void assertExternal(String query, Set expected) {
+ assertExternal(query, expected, typeMetadata);
+ }
+
+ /**
+ * External entry via {@link IngestTypeVisitor#getIngestTypes(JexlNode, boolean)} with boolean set to true
+ *
+ * @param query
+ * the query
+ * @param expected
+ * the expected types
+ * @param metadata
+ * the {@link TypeMetadata}
+ */
+ private void assertExternal(String query, Set expected, TypeMetadata metadata) {
+ assertTypes(query, expected, metadata, true);
+ }
+
+ /**
+ * Internal entry via {@link IngestTypeVisitor#getIngestTypes(JexlNode, boolean)} with boolean set to false
+ *
+ * @param query
+ * the query
+ * @param expected
+ * the expected types
+ */
+ private void assertInternal(String query, Set expected) {
+ assertInternal(query, expected, typeMetadata);
+ }
+
+ /**
+ * Internal entry via {@link IngestTypeVisitor#getIngestTypes(JexlNode, boolean)} with boolean set to false
+ *
+ * @param query
+ * the query
+ * @param expected
+ * the expected types
+ * @param metadata
+ * the {@link TypeMetadata}
+ */
+ private void assertInternal(String query, Set expected, TypeMetadata metadata) {
+ assertTypes(query, expected, metadata, false);
+ }
+
+ /**
+ * Assert types for the reusable entry point into the {@link IngestTypeVisitor} using the external argument.
+ *
+ * @param query
+ * the query
+ * @param expected
+ * the expected types
+ * @param metadata
+ * the {@link TypeMetadata}
+ * @param external
+ * boolean flag indicating what type of visit this is
+ */
+ private void assertTypes(String query, Set expected, TypeMetadata metadata, boolean external) {
ASTJexlScript script = parseQuery(query);
- Set types = IngestTypeVisitor.getIngestTypes(script, typeMetadata);
+ IngestTypeVisitor visitor = new IngestTypeVisitor(metadata);
+ Set types = visitor.getIngestTypes(script.jjtGetChild(0), external);
assertEquals(new TreeSet<>(expected), new TreeSet<>(types)); // sorted sets make differences easy to spot
}
+ /**
+ * Helper routine to parse a query, or fail the test with a helpful message
+ *
+ * @param query
+ * the query
+ * @return an ASTJexlScript
+ */
private ASTJexlScript parseQuery(String query) {
try {
return JexlASTHelper.parseAndFlattenJexlQuery(query);
From 142df3acbffff0a6d19b2cbd4e576f55eda25e3b Mon Sep 17 00:00:00 2001
From: palindrome <31748527+hlgp@users.noreply.github.com>
Date: Wed, 31 Jul 2024 12:21:43 -0400
Subject: [PATCH 022/128] Feature/one splits file to rule them all (#1409)
fixes #1309: splits file refactoring
* Refactored the splits file to include locations for partitioners that require them
* Created a Trie implemntation to hold the location map in memory
* Updated partitioners to use the new splits file
* Removed the ShardedTableMapFile and NonShardedSplitsFile and use the SplitsFile instead
---
.../ingest/config/BaseHdfsFileCacheUtil.java | 27 +-
.../mapreduce/job/DelegatingPartitioner.java | 1 +
.../ingest/mapreduce/job/IngestJob.java | 18 +-
.../job/MultiRFileOutputFormatter.java | 5 +-
.../mapreduce/job/NonShardedSplitsFile.java | 206 -
.../mapreduce/job/PartitionerCache.java | 2 +-
.../ingest/mapreduce/job/ShardReindexJob.java | 4 +-
.../mapreduce/job/ShardedTableMapFile.java | 420 -
.../ingest/mapreduce/job/SplitsFile.java | 201 +
.../mapreduce/job/TableSplitsCache.java | 288 +-
.../partition/BalancedShardPartitioner.java | 60 +-
.../partition/DelegatePartitioner.java | 4 +
.../partition/LimitedKeyPartitioner.java | 10 +
.../partition/LocalityGroupPartitioner.java | 10 +
.../MultiTableRRRangePartitioner.java | 6 +-
.../partition/MultiTableRangePartitioner.java | 87 +-
.../partition/RowHashingPartitioner.java | 10 +
.../partition/ShardIdPartitioner.java | 10 +
.../partition/SplitBasedHashPartitioner.java | 4 -
.../TabletLocationHashPartitioner.java | 15 +-
.../TabletLocationNamePartitioner.java | 14 +-
.../ingest/util/GenerateSplitsFile.java | 2 +-
.../ingest/util/ShardLocationTrieMap.java | 510 +
.../job/DelegatingPartitionerTest.java | 10 +
.../job/MultiRFileOutputFormatterTest.java | 7 +-
.../mapreduce/job/PartitionerCacheTest.java | 10 +
.../job/ShardedTableMapFileTest.java | 400 -
.../ingest/mapreduce/job/SplitsFileTest.java | 261 +
.../mapreduce/job/TableSplitsCacheTest.java | 87 +-
.../BalancedShardPartitionerTest.java | 25 +-
.../MultiTableRRRangePartitionerTest.java | 65 +-
.../MultiTableRangePartitionerTest.java | 19 +-
.../SplitBasedHashPartitionerTest.java | 16 +-
.../TabletLocationHashPartitionerTest.java | 5 +
.../TabletLocationNamePartitionerTest.java | 7 +-
.../partition/TestShardGenerator.java | 58 +-
.../ingest/mapreduce/job/all-splits.rf | Bin 0 -> 331 bytes
.../ingest/mapreduce/job/all-splits.txt | 12 +-
.../mapreduce/partition/shards_n_locs.list | 187739 +++++++++++++++
39 files changed, 189369 insertions(+), 1266 deletions(-)
delete mode 100644 warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/NonShardedSplitsFile.java
delete mode 100644 warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/ShardedTableMapFile.java
create mode 100644 warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/SplitsFile.java
create mode 100644 warehouse/ingest-core/src/main/java/datawave/ingest/util/ShardLocationTrieMap.java
delete mode 100644 warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/job/ShardedTableMapFileTest.java
create mode 100644 warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/job/SplitsFileTest.java
create mode 100644 warehouse/ingest-core/src/test/resources/datawave/ingest/mapreduce/job/all-splits.rf
create mode 100644 warehouse/ingest-core/src/test/resources/datawave/ingest/mapreduce/partition/shards_n_locs.list
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/config/BaseHdfsFileCacheUtil.java b/warehouse/ingest-core/src/main/java/datawave/ingest/config/BaseHdfsFileCacheUtil.java
index 781542ad05..57f7d5f899 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/config/BaseHdfsFileCacheUtil.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/config/BaseHdfsFileCacheUtil.java
@@ -18,7 +18,9 @@ public abstract class BaseHdfsFileCacheUtil {
protected Path cacheFilePath;
protected final Configuration conf;
protected AccumuloHelper accumuloHelper;
+
protected String delimiter = "\t";
+ private static final int MAX_RETRIES = 3;
protected short cacheReplicas = 3;
private static final Logger log = Logger.getLogger(BaseHdfsFileCacheUtil.class);
@@ -40,14 +42,23 @@ public void setDelimiter(String delimiter) {
}
public void read() throws IOException {
- log.info("Reading cache at " + this.cacheFilePath);
- try (BufferedReader in = new BufferedReader(new InputStreamReader(FileSystem.get(this.cacheFilePath.toUri(), conf).open(this.cacheFilePath)))) {
- readCache(in);
- } catch (IOException ex) {
- if (shouldRefreshCache(this.conf)) {
- update();
- } else {
- throw new IOException("Unable to read cache file at " + this.cacheFilePath, ex);
+
+ int attempts = 0;
+ boolean retry = true;
+ while (retry && attempts <= MAX_RETRIES) {
+ attempts++;
+
+ log.info("Reading cache at " + this.cacheFilePath);
+ try (BufferedReader in = new BufferedReader(new InputStreamReader(FileSystem.get(this.cacheFilePath.toUri(), conf).open(this.cacheFilePath)))) {
+ readCache(in);
+ retry = false;
+ } catch (IOException ex) {
+ if (shouldRefreshCache(this.conf)) {
+ update();
+ } else if (attempts == MAX_RETRIES) {
+ throw new IOException("Unable to read cache file at " + this.cacheFilePath, ex);
+ }
+
}
}
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/DelegatingPartitioner.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/DelegatingPartitioner.java
index 09d2da79b4..6c4f52af35 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/DelegatingPartitioner.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/DelegatingPartitioner.java
@@ -132,4 +132,5 @@ private ArrayList getTableNames(String commaSeparatedTableNames) {
}
return result;
}
+
}
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/IngestJob.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/IngestJob.java
index 7196ec43e1..1fa69a1b01 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/IngestJob.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/IngestJob.java
@@ -357,7 +357,8 @@ public int run(String[] args) throws Exception {
log.info("InputFormat: " + job.getInputFormatClass().getName());
log.info("Mapper: " + job.getMapperClass().getName());
log.info("Reduce tasks: " + (useMapOnly ? 0 : reduceTasks));
- log.info("Split File: " + workDirPath + "/splits.txt");
+ log.info("Split File: " + conf.get(TableSplitsCache.SPLITS_CACHE_DIR) + "/"
+ + conf.get(TableSplitsCache.SPLITS_CACHE_FILE, TableSplitsCache.DEFAULT_SPLITS_CACHE_FILE));
// Note that if we run any other jobs in the same vm (such as a sampler), then we may
// need to catch and throw away an exception here
@@ -419,6 +420,8 @@ public int run(String[] args) throws Exception {
}
}
}
+ long setupStop = System.currentTimeMillis();
+ log.info("JOB SETUP TIME: " + (setupStop - setupStart) + "ms");
job.waitForCompletion(true);
long stop = System.currentTimeMillis();
@@ -714,9 +717,6 @@ protected Configuration parseArguments(String[] args, Configuration conf) throws
maxRFileEntries = Integer.parseInt(args[++i]);
} else if (args[i].equals("-maxRFileUncompressedSize")) {
maxRFileSize = Long.parseLong(args[++i]);
- } else if (args[i].equals("-shardedMapFiles")) {
- conf.set(ShardedTableMapFile.SHARDED_MAP_FILE_PATHS_RAW, args[++i]);
- ShardedTableMapFile.extractShardedTableMapFilePaths(conf);
} else if (args[i].equals("-createTables")) {
createTables = true;
} else if (args[i].startsWith(REDUCE_TASKS_ARG_PREFIX)) {
@@ -826,12 +826,16 @@ protected Configuration parseArguments(String[] args, Configuration conf) throws
*/
protected void configureBulkPartitionerAndOutputFormatter(Job job, AccumuloHelper cbHelper, Configuration conf, FileSystem outputFs)
throws AccumuloSecurityException, AccumuloException, IOException, URISyntaxException, TableExistsException, TableNotFoundException {
- if (null == conf.get("split.work.dir")) {
- conf.set("split.work.dir", conf.get("ingest.work.dir.qualified"));
+ if (null == conf.get(SplitsFile.SPLIT_WORK_DIR)) {
+ conf.set(SplitsFile.SPLIT_WORK_DIR, conf.get("ingest.work.dir.qualified"));
}
conf.setInt("splits.num.reduce", this.reduceTasks);
// used by the output formatter and the sharded partitioner
- ShardedTableMapFile.setupFile(conf);
+ long before = System.currentTimeMillis();
+ SplitsFile.setupFile(job, conf);
+ long after = System.currentTimeMillis();
+
+ log.info("Sharded splits files setup time: " + (after - before) + "ms");
conf.setInt(MultiRFileOutputFormatter.EVENT_PARTITION_COUNT, this.reduceTasks * 2);
configureMultiRFileOutputFormatter(conf, compressionType, compressionTableDisallowList, maxRFileEntries, maxRFileSize, generateMapFileRowKeys);
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/MultiRFileOutputFormatter.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/MultiRFileOutputFormatter.java
index daef981a8f..1495bf78b8 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/MultiRFileOutputFormatter.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/MultiRFileOutputFormatter.java
@@ -467,7 +467,8 @@ public RecordWriter getRecordWriter(final TaskAttemptContex
conf.setInt("io.seqfile.compress.blocksize", getSeqFileBlockSize());
// Get the list of tables
- String[] tableNames = conf.getStrings(ShardedTableMapFile.CONFIGURED_SHARDED_TABLE_NAMES);
+
+ String[] tableNames = conf.getStrings(SplitsFile.CONFIGURED_SHARDED_TABLE_NAMES);
if (null == tableNames) {
log.warn("Could not find the list of sharded table names");
@@ -669,7 +670,7 @@ protected Map getShardLocations(String tableName) throws IOExceptio
}
if (null == this.tableShardLocations.get(tableName)) {
- this.tableShardLocations.put(tableName, ShardedTableMapFile.getShardIdToLocations(conf, tableName));
+ this.tableShardLocations.put(tableName, SplitsFile.getSplitsAndLocations(conf, tableName));
}
return tableShardLocations.get(tableName);
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/NonShardedSplitsFile.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/NonShardedSplitsFile.java
deleted file mode 100644
index 52975abd6c..0000000000
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/NonShardedSplitsFile.java
+++ /dev/null
@@ -1,206 +0,0 @@
-package datawave.ingest.mapreduce.job;
-
-import java.io.BufferedOutputStream;
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.accumulo.core.client.TableExistsException;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.Text;
-import org.apache.log4j.Logger;
-
-import datawave.ingest.data.config.ConfigurationHelper;
-import datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler;
-
-/**
- * Extracted from IngestJob Creates a splits file with all the requested tables, omitting the sharded ones Adds it to the specified work dir and sets the confi
- */
-public class NonShardedSplitsFile {
- protected static final Logger log = Logger.getLogger(NonShardedSplitsFile.class);
- public static final String SPLITS_FILE_NAME_PROPERTY_KEY = "datawave.ingest.bulk.NonShardedSplitsFile.cutFile";
- private static final String SPLITS_FILE_NAME_PROPERTY_VALUE = "splits.txt";
-
- public static class Writer {
- private URI uri;
- private final int reduceTasks;
- private final Path workDirPath;
- private final String[] tableNames;
- private final List shardedTableNames;
- private final boolean isTrimmed;
- private final Configuration conf;
- private final FileSystem fs;
-
- /**
- * Writes out the split points for the tables to splits.txt in the work directory.
- *
- * @param conf
- * hadoop job configuration
- * @param reduceTasks
- * number of reduce tasks
- * @param workDirPath
- * base dir in HDFS where the file is written
- * @param outputFs
- * the filesystem to which the splits file should be written
- * @param tableNames
- * names of the table to retrieve splits for
- * @throws IOException
- * if there is an issue with read or write
- * @throws TableExistsException
- * if the table already exists
- * @throws TableNotFoundException
- * if the table could not be found
- * @throws URISyntaxException
- * if there is an issue with the URI syntax
- */
- public Writer(Configuration conf, int reduceTasks, Path workDirPath, FileSystem outputFs, String[] tableNames)
- throws TableNotFoundException, IOException, TableExistsException, URISyntaxException {
- this(conf, reduceTasks, workDirPath, outputFs, tableNames, true);
- }
-
- public Writer(Configuration conf, int reduceTasks, Path workDirPath, FileSystem fs, String[] tableNames, boolean isTrimmed)
- throws TableNotFoundException, IOException, TableExistsException, URISyntaxException {
- this.conf = conf;
- this.reduceTasks = reduceTasks;
- this.workDirPath = workDirPath;
- this.fs = fs;
- this.tableNames = tableNames;
- this.isTrimmed = isTrimmed;
- this.shardedTableNames = Arrays.asList(ConfigurationHelper.isNull(conf, ShardedDataTypeHandler.SHARDED_TNAMES, String[].class));
- }
-
- public void createFile(boolean isTrimmed) {
- try {
- TableSplitsCache splits = new TableSplitsCache(conf);
- boolean isCacheValid = TableSplitsCacheStatus.isCacheValid(conf);
- boolean shouldRefreshSplits = TableSplitsCache.shouldRefreshSplits(conf);
- if (shouldRefreshSplits && !isCacheValid) {
- log.info("Recreating splits");
- splits.update();
- } else if (!shouldRefreshSplits && !isCacheValid) {
- throw new Exception("Splits cache is invalid");
- }
- writeSplitsToFile(splits);
- uri = new URI(workDirPath + "/" + createFileName(isTrimmed));
- } catch (Exception e) {
- throw new RuntimeException("Could not create splits file for the job. See documentation for using generateSplitsFile.sh", e);
- }
- }
-
- private void writeSplitsToFile(TableSplitsCache splits) throws IOException {
- PrintStream out = new PrintStream(new BufferedOutputStream(fs.create(new Path(workDirPath, createFileName(isTrimmed)))));
- outputSplitsForNonShardTables(splits, out);
- out.close();
- }
-
- private void outputSplitsForNonShardTables(TableSplitsCache splits, PrintStream out) throws IOException {
- for (String table : tableNames) {
- if (null != shardedTableNames && shardedTableNames.contains(table)) {
- continue;
- }
- outputSplitsForTable(splits, out, table);
- }
- }
-
- private void outputSplitsForTable(TableSplitsCache splits, PrintStream out, String table) throws IOException {
- Collection tableSplits;
- if (isTrimmed) {
- tableSplits = splits.getSplits(table, reduceTasks - 1);
- } else {
- tableSplits = splits.getSplits(table);
- }
- for (Text split : tableSplits) {
- out.println(table + "\t" + new String(Base64.encodeBase64(split.getBytes())));
- if (log.isTraceEnabled()) {
- log.trace(table + " split: " + split);
- }
- }
- }
-
- public URI getUri() {
- return uri;
- }
- }
-
- private static String createFileName(boolean isTrimmed) {
- return (isTrimmed ? "trimmed_" : "full_") + SPLITS_FILE_NAME_PROPERTY_VALUE;
- }
-
- public static Path findSplitsFile(Configuration conf, Path[] filesToCheck, boolean isTrimmed) {
- String fileName = createFileName(isTrimmed);
- if (filesToCheck != null) {
- for (Path cacheFile : filesToCheck) {
- if (matchesFileName(fileName, cacheFile)) {
- return cacheFile;
- }
- }
- }
- return null;
- }
-
- private static boolean matchesFileName(String cutFileName, Path cacheFile) {
- return cacheFile.getName().endsWith(cutFileName);
- }
-
- public static class Reader {
- private Map splits;
-
- public Reader(Configuration conf, Path[] filesToCheck, boolean isTrimmed) throws IOException {
- Path cacheFile = findSplitsFile(conf, filesToCheck, isTrimmed);
- if (null == cacheFile) {
- throw new RuntimeException("Could not find cut point file");
- }
-
- splits = new HashMap<>();
- ArrayList cutPoints = new ArrayList<>();
- String previousTableName = null;
- try (BufferedReader in = new BufferedReader(new FileReader(cacheFile.toString()))) {
- String line;
- while ((line = in.readLine()) != null) {
- String[] parts = line.split("\\t");
- if (parts[0].equals(previousTableName)) {
- if (parts.length > 1)
- cutPoints.add(new Text(Base64.decodeBase64(parts[1].getBytes())));
- } else if (previousTableName == null) {
- previousTableName = parts[0];
- if (parts.length > 1)
- cutPoints.add(new Text(Base64.decodeBase64(parts[1].getBytes())));
- } else {
- Collections.sort(cutPoints);
- splits.put(previousTableName, cutPoints.toArray(new Text[cutPoints.size()]));
- log.info("Adding cut points for table: " + previousTableName);
- previousTableName = parts[0];
- cutPoints.clear();
- if (parts.length > 1)
- cutPoints.add(new Text(Base64.decodeBase64(parts[1].getBytes())));
- }
- }
- } finally {
- if (null != previousTableName) {
- // Add the last batch.
- Collections.sort(cutPoints);
- splits.put(previousTableName, cutPoints.toArray(new Text[cutPoints.size()]));
- log.info("Adding cut points for table: " + previousTableName);
- }
- }
- }
-
- public Map getSplitsByTable() {
- return splits;
- }
- }
-}
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/PartitionerCache.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/PartitionerCache.java
index ff352acc2a..77dc701756 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/PartitionerCache.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/PartitionerCache.java
@@ -146,7 +146,7 @@ private Partitioner getDefaultPartitioner() {
return defaultDelegatePartitioner;
}
- private boolean hasPartitionerOverride(Text tableName) {
+ public boolean hasPartitionerOverride(Text tableName) {
return isMemberOfACategory(tableName) || hasDedicatedPartitioner(tableName);
}
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/ShardReindexJob.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/ShardReindexJob.java
index 179e4871d0..102f5fad34 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/ShardReindexJob.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/ShardReindexJob.java
@@ -1,6 +1,6 @@
package datawave.ingest.mapreduce.job;
-import static datawave.ingest.mapreduce.job.ShardedTableMapFile.SPLIT_WORK_DIR;
+import static datawave.ingest.mapreduce.job.SplitsFile.SPLIT_WORK_DIR;
import java.io.File;
import java.io.IOException;
@@ -138,7 +138,7 @@ private Job setupJob() throws IOException, ParseException, AccumuloException, Ta
throw new IllegalStateException("split.work.dir and job.output.table.names must be configured");
}
- ShardedTableMapFile.setupFile(configuration);
+ SplitsFile.setupFile(Job.getInstance(getConf()), configuration);
// setup the output format
IngestJob.configureMultiRFileOutputFormatter(configuration, null, null, 0, 0, false);
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/ShardedTableMapFile.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/ShardedTableMapFile.java
deleted file mode 100644
index 25a002b577..0000000000
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/ShardedTableMapFile.java
+++ /dev/null
@@ -1,420 +0,0 @@
-package datawave.ingest.mapreduce.job;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.accumulo.core.client.AccumuloClient;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.admin.Locations;
-import org.apache.accumulo.core.client.admin.TableOperations;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.TabletId;
-import org.apache.commons.lang.time.DateUtils;
-import org.apache.commons.lang3.mutable.MutableInt;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.log4j.Logger;
-
-import datawave.ingest.data.config.ConfigurationHelper;
-import datawave.ingest.data.config.ingest.AccumuloHelper;
-import datawave.ingest.mapreduce.handler.shard.ShardIdFactory;
-import datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler;
-import datawave.util.StringUtils;
-import datawave.util.time.DateHelper;
-
-/**
- * Extracted from IngestJob
- */
-public class ShardedTableMapFile {
- public static final String SHARDS_BALANCED_DAYS_TO_VERIFY = "shards.balanced.days.to.verify";
- private static final String PREFIX = ShardedTableMapFile.class.getName();
- private static final int MAX_RETRY_ATTEMPTS = 10;
-
- private static final Logger log = Logger.getLogger(ShardedTableMapFile.class);
-
- public static final String SHARD_TSERVER_MAP_FILE = PREFIX + ".shardTServerMapFile";
- public static final String SPLIT_WORK_DIR = "split.work.dir";
-
- public static final String CONFIGURED_SHARDED_TABLE_NAMES = ShardedDataTypeHandler.SHARDED_TNAMES + ".configured";
- public static final String SHARDED_MAP_FILE_PATHS_RAW = "shardedMap.file.paths.raw";
- public static final String SHARD_VALIDATION_ENABLED = "shardedMap.validation.enabled";
- public static final String MAX_SHARDS_PER_TSERVER = "shardedMap.max.shards.per.tserver";
-
- public static void setupFile(Configuration conf) throws IOException, URISyntaxException, AccumuloSecurityException, AccumuloException {
- // want validation turned off by default
- boolean doValidation = conf.getBoolean(ShardedTableMapFile.SHARD_VALIDATION_ENABLED, false);
-
- Map map = loadMap(conf, doValidation);
- if (null == map) {
- log.fatal("Receieved a null mapping of sharded tables to split files, exiting...");
- throw new RuntimeException("Receieved a null mapping of sharded tables to split files, exiting...");
- }
- addToConf(conf, map);
- }
-
- private static SequenceFile.Reader getReader(Configuration conf, String tableName) throws IOException {
- String shardMapFileName = conf.get(SHARD_TSERVER_MAP_FILE + "." + tableName);
- try {
- return new SequenceFile.Reader(conf, SequenceFile.Reader.file(new Path(shardMapFileName)));
- } catch (Exception e) {
- throw new IOException("Failed to create sequence file reader for " + shardMapFileName + " for " + tableName, e);
- }
- }
-
- public static TreeMap getShardIdToLocations(Configuration conf, String tableName) throws IOException {
- TreeMap locations = new TreeMap<>();
-
- try (SequenceFile.Reader reader = ShardedTableMapFile.getReader(conf, tableName)) {
- Text shardID = new Text();
- Text location = new Text();
- while (reader.next(shardID, location)) {
- locations.put(new Text(shardID), location.toString());
- }
- }
- return locations;
- }
-
- public static void validateShardIdLocations(Configuration conf, String tableName, int daysToVerify, Map shardIdToLocation) {
- ShardIdFactory shardIdFactory = new ShardIdFactory(conf);
- // assume true unless proven otherwise
- boolean isValid = true;
- int maxShardsPerTserver = conf.getInt(MAX_SHARDS_PER_TSERVER, 1);
-
- for (int daysAgo = 0; daysAgo <= daysToVerify; daysAgo++) {
- long inMillis = System.currentTimeMillis() - (daysAgo * DateUtils.MILLIS_PER_DAY);
- String datePrefix = DateHelper.format(inMillis);
- int expectedNumberOfShards = shardIdFactory.getNumShards(datePrefix);
- boolean shardsExist = shardsExistForDate(shardIdToLocation, datePrefix, expectedNumberOfShards);
- if (!shardsExist) {
- log.error("Shards for " + datePrefix + " for table " + tableName + " do not exist!");
- isValid = false;
- continue;
- }
- boolean shardsAreBalanced = shardsAreBalanced(shardIdToLocation, datePrefix, maxShardsPerTserver);
- if (!shardsAreBalanced) {
- log.error("Shards for " + datePrefix + " for table " + tableName + " are not balanced!");
- isValid = false;
- }
- }
- if (!isValid) {
- throw new IllegalStateException("Shard validation failed for " + tableName + ". Please ensure that "
- + "shards have been generated. Check log for details about the dates in question");
- }
- }
-
- /**
- * Existence check for the shard splits for the specified date
- *
- * @param locations
- * mapping of shard to tablet
- * @param datePrefix
- * to check
- * @param expectedNumberOfShards
- * that should exist
- * @return if the number of shards for the given date are as expected
- */
- private static boolean shardsExistForDate(Map locations, String datePrefix, int expectedNumberOfShards) {
- int count = 0;
- byte[] prefixBytes = datePrefix.getBytes();
- for (Text key : locations.keySet()) {
- if (prefixMatches(prefixBytes, key.getBytes(), key.getLength())) {
- count++;
- }
- }
- return count == expectedNumberOfShards;
- }
-
- /**
- * Checks that the shard splits for the given date have been assigned to unique tablets.
- *
- * @param locations
- * mapping of shard to tablet
- * @param datePrefix
- * to check
- * @param maxShardsPerTserver
- * the max shards per server
- * @return if the shards are distributed in a balanced fashion
- */
- private static boolean shardsAreBalanced(Map locations, String datePrefix, int maxShardsPerTserver) {
- // assume true unless proven wrong
- boolean dateIsBalanced = true;
-
- Map tabletsSeenForDate = new HashMap<>();
- byte[] prefixBytes = datePrefix.getBytes();
-
- for (Entry entry : locations.entrySet()) {
- Text key = entry.getKey();
- // only check entries for specified date
- if (prefixMatches(prefixBytes, key.getBytes(), key.getLength())) {
- String value = entry.getValue();
- MutableInt cnt = tabletsSeenForDate.get(value);
- if (null == cnt) {
- cnt = new MutableInt(0);
- }
- // increment here before checking
- cnt.increment();
-
- // if shard is assigned to more tservers than allowed, then the shards are not balanced
- if (cnt.intValue() > maxShardsPerTserver) {
- log.warn(cnt.toInteger() + " Shards for " + datePrefix + " assigned to tablet " + value);
- dateIsBalanced = false;
- }
-
- tabletsSeenForDate.put(value, cnt);
- }
- }
-
- return dateIsBalanced;
- }
-
- private static boolean prefixMatches(byte[] prefixBytes, byte[] keyBytes, int keyLen) {
- // if key length is less than prefix size, no use comparing
- if (prefixBytes.length > keyLen) {
- return false;
- }
- for (int i = 0; i < prefixBytes.length; i++) {
- if (prefixBytes[i] != keyBytes[i]) {
- return false;
- }
- }
- // at this point didn't fail match, so should be good
- return true;
- }
-
- public static void addToConf(Configuration conf, Map map) {
- for (Map.Entry entry : map.entrySet()) {
- log.info("Loading sharded partitioner for table '" + entry.getKey() + "' with shardedMapFile '" + entry.getValue() + "'");
- conf.set(SHARD_TSERVER_MAP_FILE + "." + entry.getKey(), entry.getValue().toString());
- }
- Set var = map.keySet();
- conf.setStrings(CONFIGURED_SHARDED_TABLE_NAMES, var.toArray(new String[0]));
- }
-
- private static Map loadMap(Configuration conf, boolean doValidation)
- throws IOException, URISyntaxException, AccumuloSecurityException, AccumuloException {
- AccumuloHelper accumuloHelper = null;
- Path workDir = new Path(conf.get(SPLIT_WORK_DIR));// todo make sure this is set in ingest job
- String[] tableNames = StringUtils.split(conf.get(TableConfigurationUtil.JOB_OUTPUT_TABLE_NAMES), ",");// todo make sure this is set in ingest job
- Map shardedTableMapFilePaths = extractShardedTableMapFilePaths(conf);
- // Get a list of "sharded" tables
- String[] shardedTableNames = ConfigurationHelper.isNull(conf, ShardedDataTypeHandler.SHARDED_TNAMES, String[].class);
-
- Set configuredShardedTableNames = new HashSet<>(Arrays.asList(shardedTableNames));
-
- // Remove all "sharded" tables that we aren't actually outputting to
- configuredShardedTableNames.retainAll(Arrays.asList(tableNames));
-
- Map shardedTableMapFiles = new HashMap<>();
-
- // Pull the list of table that we "shard":
- // Use the sequence file of splits for the current table, or pull them off of the configured Instance
- for (String shardedTableName : configuredShardedTableNames) {
- Path shardedMapFile;
-
- // If an existing splits file was provided on the command line, use it.
- // Otherwise, calculate one from the Accumulo instance
- if (shardedTableMapFilePaths.containsKey(shardedTableName) && null != shardedTableMapFilePaths.get(shardedTableName)) {
- shardedMapFile = new Path(shardedTableMapFilePaths.get(shardedTableName));
- } else {
- if (null == accumuloHelper) {
- accumuloHelper = new AccumuloHelper();
- accumuloHelper.setup(conf);
- }
- shardedMapFile = createShardedMapFile(log, conf, workDir, accumuloHelper, shardedTableName, doValidation);
- }
-
- // Ensure that we either computed, or were given, a valid path to the shard mappings
- if (null == shardedMapFile || !shardedMapFile.getFileSystem(conf).exists(shardedMapFile)) {
- log.fatal("Could not find the supplied shard map file: " + shardedMapFile);
- return null;
- } else {
- shardedTableMapFiles.put(shardedTableName, shardedMapFile);
- }
- }
-
- return shardedTableMapFiles;
- }
-
- static Map extractShardedTableMapFilePaths(Configuration conf) {
- Map shardedTableMapFilePaths = new HashMap<>();
- String commaSeparatedFileNamesByTable = conf.get(SHARDED_MAP_FILE_PATHS_RAW);
- if (null == commaSeparatedFileNamesByTable) {
- return shardedTableMapFilePaths;
- }
-
- String[] pairs = StringUtils.split(commaSeparatedFileNamesByTable, ',');
-
- for (String pair : pairs) {
- int index = pair.indexOf('=');
- if (index < 0) {
- log.warn("WARN: Skipping bad tableName=/path/to/tableNameSplits.seq property: " + pair);
- } else {
- String tableName = pair.substring(0, index), splitsFile = pair.substring(index + 1);
-
- log.info("Using splits file '" + splitsFile + "' for table '" + tableName + "'");
-
- shardedTableMapFilePaths.put(tableName, splitsFile);
- }
- }
- return shardedTableMapFilePaths;
- }
-
- /**
- * Build a file that maps shard IDs (row keys in the sharded table) to the tablet server where they are currently stored.
- *
- * @param log
- * logger for reporting errors
- * @param conf
- * hadoop configuration
- * @param workDir
- * base dir in HDFS where the file is written
- * @param accumuloHelper
- * Accumulo helper to query shard locations
- * @param shardedTableName
- * name of the shard table--the table whose locations we are querying
- * @param validateShardLocations
- * if validation of shards mappings should be performed
- * @return the path to the sharded table map file
- * @throws IOException
- * if there is an issue with read or write
- */
- public static Path createShardedMapFile(Logger log, Configuration conf, Path workDir, AccumuloHelper accumuloHelper, String shardedTableName,
- boolean validateShardLocations) throws IOException {
- Path shardedMapFile = null;
- // minus one to make zero based indexed
- int daysToVerify = conf.getInt(SHARDS_BALANCED_DAYS_TO_VERIFY, 2) - 1;
-
- if (null != shardedTableName) {
- // Read all the metadata entries for the sharded table so that we can
- // get the mapping of shard IDs to tablet locations.
- log.info("Reading metadata entries for " + shardedTableName);
-
- Map splitToLocations = getLocations(log, accumuloHelper, shardedTableName);
- if (validateShardLocations) {
- validateShardIdLocations(conf, shardedTableName, daysToVerify, splitToLocations);
- }
-
- // Now write all of the assignments out to a file stored in HDFS
- // we're ok with putting the sharded table file in the hdfs workdir. why is that not good enough for the non sharded splits?
- shardedMapFile = new Path(workDir, shardedTableName + "_shards.lst");
- log.info("Writing shard assignments to " + shardedMapFile);
- long count = writeSplitsFile(splitToLocations, shardedMapFile, conf);
- log.info("Wrote " + count + " shard assignments to " + shardedMapFile);
- }
-
- return shardedMapFile;
- }
-
- /**
- * Continually scans the metdata table attempting to get the split locations for the shard table.
- *
- * @param log
- * logger for reporting errors
- * @param accumuloHelper
- * Accumulo helper to query shard locations
- * @param shardedTableName
- * name of the shard table--the table whose locations we are querying
- * @return a map of split (endRow) to the location of those tablets in accumulo
- */
- public static Map getLocations(Logger log, AccumuloHelper accumuloHelper, String shardedTableName) {
- // split (endRow) -> String location mapping
- Map splitToLocation = new TreeMap<>();
-
- boolean keepRetrying = true;
- int attempts = 0;
- while (keepRetrying && attempts < MAX_RETRY_ATTEMPTS) {
- try (AccumuloClient client = accumuloHelper.newClient()) {
- TableOperations tableOps = client.tableOperations();
- attempts++;
- // if table does not exist don't want to catch the errors and end up in infinite loop
- if (!tableOps.exists(shardedTableName)) {
- log.error("Table " + shardedTableName + " not found, skipping split locations for missing table");
- } else {
- Range range = new Range();
- Locations locations = tableOps.locate(shardedTableName, Collections.singletonList(range));
- List tabletIds = locations.groupByRange().get(range);
-
- tabletIds.stream().filter(tId -> tId.getEndRow() != null)
- .forEach(tId -> splitToLocation.put(tId.getEndRow(), locations.getTabletLocation(tId)));
- }
- // made it here, no errors so break out
- keepRetrying = false;
- } catch (Exception e) {
- log.warn(e.getClass().getName() + ":" + e.getMessage() + " ... retrying ...", e);
- try {
- TimeUnit.MILLISECONDS.sleep(3000);
- } catch (InterruptedException ex) {
- log.error(e.getMessage(), ex);
- }
-
- splitToLocation.clear();
- }
- }
-
- return splitToLocation;
- }
-
- /**
- * Writes the contents of splits out to a sequence file on the given FileSystem.
- *
- * @param splits
- * map of split points for a table
- * @param file
- * the file to which the splits should be written
- * @param conf
- * hadoop configuration
- * @return the number of entries written to the splits file
- * @throws IOException
- * if file system interaction fails
- */
- public static long writeSplitsFile(Map splits, Path file, Configuration conf) throws IOException {
- FileSystem fs = file.getFileSystem(conf);
- if (fs.exists(file))
- fs.delete(file, false);
-
- long count = 0;
- // reusable value for writing
- Text value = new Text();
- try (SequenceFile.Writer writer = SequenceFile.createWriter(conf, SequenceFile.Writer.file(file), SequenceFile.Writer.keyClass(Text.class),
- SequenceFile.Writer.valueClass(Text.class))) {
- for (Entry entry : splits.entrySet()) {
- count++;
- value.set(entry.getValue());
- writer.append(entry.getKey(), value);
- }
- }
- return count;
- }
-
- /**
- * Writes the contents of splits out to a sequence file on the given FileSystem.
- *
- * @param splits
- * map of split points for a table
- * @param file
- * the file to which the splits should be written
- * @param conf
- * hadoop configuration
- * @return the number of entries written to the splits file
- * @throws IOException
- * if file system interaction fails
- */
- public static long writeSplitsFileLegacy(Map splits, Path file, Configuration conf) throws IOException {
- return writeSplitsFile(splits, file, conf);
- }
-
-}
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/SplitsFile.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/SplitsFile.java
new file mode 100644
index 0000000000..28283f940b
--- /dev/null
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/SplitsFile.java
@@ -0,0 +1,201 @@
+package datawave.ingest.mapreduce.job;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.commons.lang.time.DateUtils;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.log4j.Logger;
+
+import com.google.common.collect.Table;
+
+import datawave.ingest.mapreduce.handler.shard.ShardIdFactory;
+import datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler;
+import datawave.util.time.DateHelper;
+
+public class SplitsFile {
+ private static final Logger log = Logger.getLogger(SplitsFile.class);
+
+ public static final String SPLIT_WORK_DIR = "split.work.dir";
+ public static final String MAX_SHARDS_PER_TSERVER = "shardedMap.max.shards.per.tserver";
+ public static final String SHARD_VALIDATION_ENABLED = "shardedMap.validation.enabled";
+ public static final String SHARDS_BALANCED_DAYS_TO_VERIFY = "shards.balanced.days.to.verify";
+ public static final String CONFIGURED_SHARDED_TABLE_NAMES = ShardedDataTypeHandler.SHARDED_TNAMES + ".configured";
+ public static final String DIST_CACHE_LABEL = "splitsFile";
+
+ public static void setupFile(Job job, Configuration conf) throws IOException, URISyntaxException, AccumuloSecurityException, AccumuloException {
+
+ Path baseSplitsPath = TableSplitsCache.getSplitsPath(conf);
+ FileSystem sourceFs = baseSplitsPath.getFileSystem(conf);
+ FileSystem destFs = new Path(conf.get(SPLIT_WORK_DIR)).getFileSystem(conf);
+
+ // want validation turned off by default
+ boolean doValidation = conf.getBoolean(SHARD_VALIDATION_ENABLED, false);
+
+ try {
+ log.info("Base splits: " + baseSplitsPath);
+
+ Path destSplits = new Path(
+ conf.get(SPLIT_WORK_DIR) + "/" + conf.get(TableSplitsCache.SPLITS_CACHE_FILE, TableSplitsCache.DEFAULT_SPLITS_CACHE_FILE));
+ log.info("Dest splits: " + destSplits);
+
+ FileUtil.copy(sourceFs, baseSplitsPath, destFs, destSplits, false, conf);
+ conf.set(TableSplitsCache.SPLITS_CACHE_DIR, conf.get(SPLIT_WORK_DIR));
+
+ // // if we want the freshest splits, go ahead and update them in the work dir
+ // if (TableSplitsCache.shouldRefreshSplits(conf)) {
+ // TableSplitsCache.getCurrentCache(conf).update();
+ // }
+
+ job.addCacheFile(new URI(destSplits.toString() + "#" + DIST_CACHE_LABEL));
+
+ if (doValidation) {
+ validate(conf);
+ }
+
+ } catch (Exception e) {
+ log.error("Unable to use splits file because " + e.getMessage());
+ throw e;
+ }
+ }
+
+ private static void validate(Configuration conf) throws IOException {
+ TableSplitsCache cache = TableSplitsCache.getCurrentCache(conf);
+ int daysToVerify = conf.getInt(SHARDS_BALANCED_DAYS_TO_VERIFY, 2) - 1;
+
+ for (String table : conf.getStrings(ShardedDataTypeHandler.SHARDED_TNAMES)) {
+ validateShardIdLocations(conf, table, daysToVerify, cache.getSplitsAndLocationByTable(table));
+ }
+
+ }
+
+ public static void validateShardIdLocations(Configuration conf, String tableName, int daysToVerify, Map shardIdToLocation) {
+ ShardIdFactory shardIdFactory = new ShardIdFactory(conf);
+ // assume true unless proven otherwise
+ boolean isValid = true;
+ int maxShardsPerTserver = conf.getInt(MAX_SHARDS_PER_TSERVER, 1);
+
+ for (int daysAgo = 0; daysAgo <= daysToVerify; daysAgo++) {
+ long inMillis = System.currentTimeMillis() - (daysAgo * DateUtils.MILLIS_PER_DAY);
+ String datePrefix = DateHelper.format(inMillis);
+ int expectedNumberOfShards = shardIdFactory.getNumShards(datePrefix);
+ boolean shardsExist = shardsExistForDate(shardIdToLocation, datePrefix, expectedNumberOfShards);
+ if (!shardsExist) {
+ log.error("Shards for " + datePrefix + " for table " + tableName + " do not exist!");
+ isValid = false;
+ continue;
+ }
+ boolean shardsAreBalanced = shardsAreBalanced(shardIdToLocation, datePrefix, maxShardsPerTserver);
+ if (!shardsAreBalanced) {
+ log.error("Shards for " + datePrefix + " for table " + tableName + " are not balanced!");
+ isValid = false;
+ }
+ }
+ if (!isValid) {
+ throw new IllegalStateException("Shard validation failed for " + tableName + ". Please ensure that "
+ + "shards have been generated. Check log for details about the dates in question");
+ }
+ }
+
+ private static boolean prefixMatches(byte[] prefixBytes, byte[] keyBytes, int keyLen) {
+ // if key length is less than prefix size, no use comparing
+ if (prefixBytes.length > keyLen) {
+ return false;
+ }
+ for (int i = 0; i < prefixBytes.length; i++) {
+ if (prefixBytes[i] != keyBytes[i]) {
+ return false;
+ }
+ }
+ // at this point didn't fail match, so should be good
+ return true;
+ }
+
+ /**
+ * Existence check for the shard splits for the specified date
+ *
+ * @param locations
+ * mapping of shard to tablet
+ * @param datePrefix
+ * to check
+ * @param expectedNumberOfShards
+ * that should exist
+ * @return if the number of shards for the given date are as expected
+ */
+ private static boolean shardsExistForDate(Map locations, String datePrefix, int expectedNumberOfShards) {
+ int count = 0;
+ byte[] prefixBytes = datePrefix.getBytes();
+ for (Text key : locations.keySet()) {
+ if (prefixMatches(prefixBytes, key.getBytes(), key.getLength())) {
+ count++;
+ }
+ }
+ return count == expectedNumberOfShards;
+ }
+
+ /**
+ * Checks that the shard splits for the given date have been assigned to unique tablets.
+ *
+ * @param locations
+ * mapping of shard to tablet
+ * @param datePrefix
+ * to check
+ * @return if the shards are distributed in a balanced fashion
+ */
+ private static boolean shardsAreBalanced(Map locations, String datePrefix, int maxShardsPerTserver) {
+ // assume true unless proven wrong
+ boolean dateIsBalanced = true;
+
+ Map tabletsSeenForDate = new HashMap<>();
+ byte[] prefixBytes = datePrefix.getBytes();
+
+ for (Map.Entry entry : locations.entrySet()) {
+ Text key = entry.getKey();
+ // only check entries for specified date
+ if (prefixMatches(prefixBytes, key.getBytes(), key.getLength())) {
+ String value = entry.getValue();
+ MutableInt cnt = tabletsSeenForDate.get(value);
+ if (null == cnt) {
+ cnt = new MutableInt(0);
+ }
+ // increment here before checking
+ cnt.increment();
+
+ // if shard is assigned to more tservers than allowed, then the shards are not balanced
+ if (cnt.intValue() > maxShardsPerTserver) {
+ log.warn(cnt.toInteger() + " Shards for " + datePrefix + " assigned to tablet " + value);
+ dateIsBalanced = false;
+ }
+
+ tabletsSeenForDate.put(value, cnt);
+ }
+ }
+
+ return dateIsBalanced;
+ }
+
+ public static Map> getSplits(Configuration conf) throws IOException {
+ return TableSplitsCache.getCurrentCache(conf).getSplits();
+
+ }
+
+ public static Map getSplitsAndLocations(Configuration conf, String tableName) throws IOException {
+ return TableSplitsCache.getCurrentCache(conf).getSplitsAndLocationByTable(tableName);
+ }
+
+ public static List getSplits(Configuration conf, String tableName) throws IOException {
+ return TableSplitsCache.getCurrentCache(conf).getSplits(tableName);
+ }
+}
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/TableSplitsCache.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/TableSplitsCache.java
index ea67036a9a..ab90184567 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/TableSplitsCache.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/job/TableSplitsCache.java
@@ -2,8 +2,11 @@
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Collections;
@@ -11,21 +14,33 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.clientImpl.ClientContext;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.MetadataServicer;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.log4j.Logger;
+import com.google.common.collect.Maps;
+
import datawave.ingest.config.BaseHdfsFileCacheUtil;
+import datawave.ingest.mapreduce.partition.BalancedShardPartitioner;
+import datawave.ingest.mapreduce.partition.DelegatePartitioner;
+import datawave.ingest.util.ShardLocationTrieMap;
import datawave.util.StringUtils;
/**
@@ -37,26 +52,61 @@ public class TableSplitsCache extends BaseHdfsFileCacheUtil {
public static final String REFRESH_SPLITS = "datawave.ingest.refresh.splits";
public static final String SPLITS_CACHE_DIR = "datawave.ingest.splits.cache.dir";
+ public static final String SPLITS_CACHE_FILE = "datawave.ingest.splits.cache.fileName";
+
public static final String MAX_SPLIT_DECREASE = "datawave.ingest.splits.max.decrease.number";
public static final String MAX_SPLIT_PERCENTAGE_DECREASE = "datawave.ingest.splits.max.decrease.percentage";
private static final Logger log = Logger.getLogger(TableSplitsCache.class);
private static final String DEFAULT_SPLITS_CACHE_DIR = "/data/splitsCache";
- private static final String SPLITS_CACHE_FILE = "all-splits.txt";
+ public static final String DEFAULT_SPLITS_CACHE_FILE = "all-splits.txt";
private static final short DEFAULT_MAX_SPLIT_DECREASE = 42;
private static final double DEFAULT_MAX_SPLIT_PERCENTAGE_DECREASE = .5;
private static final boolean DEFAULT_REFRESH_SPLITS = true;
+ private static final String NO_LOCATION = "noloc";
+ private static TableSplitsCache cache;
+ private volatile boolean cacheFileRead = false;
+ private Object semaphore = new Object();
+
private Path splitsPath = null;
- private Map> splits = null;
+ private Map> splitLocations = new HashMap<>();
+ private Map> splits = new HashMap<>();
+
+ private PartitionerCache partitionerCache;
+
+ private Map getSplitsWithLocation(String table) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ SortedMap tabletLocations = new TreeMap<>();
+
+ AccumuloClient client = accumuloHelper.newClient();
+ MetadataServicer.forTableName((ClientContext) client, table).getTabletLocations(tabletLocations);
+
+ return tabletLocations.entrySet().stream().filter(k -> k.getKey().endRow() != null).collect(
+ Collectors.toMap(e -> e.getKey().endRow(), e -> e.getValue() == null ? NO_LOCATION : e.getValue(), (o1, o2) -> o1, TreeMap::new));
+ }
/**
- *
* @param conf
* the configuration
*/
public TableSplitsCache(Configuration conf) {
+
super(conf);
this.splitsPath = getSplitsPath(conf);
+ this.partitionerCache = new PartitionerCache(conf);
+ this.cacheFileRead = false;
+
+ }
+
+ public static TableSplitsCache getCurrentCache(Configuration conf) {
+ if (null == cache) {
+ cache = new TableSplitsCache(conf);
+ }
+
+ return cache;
+ }
+
+ public static void clear() {
+ cache = null;
}
/**
@@ -73,7 +123,7 @@ public static List trimSplits(List tableSplits, int maxSplits) {
double r = (maxSplits + 1) / (double) (tableSplits.size());
double pos = 0;
- ArrayList subset = new ArrayList<>(maxSplits);
+ List subset = new ArrayList<>();
int numberSplitsUsed = 0;
for (int i = 0; i < tableSplits.size() && numberSplitsUsed < maxSplits; i++) {
@@ -90,11 +140,15 @@ public static List trimSplits(List tableSplits, int maxSplits) {
@Override
public void setCacheFilePath(Configuration conf) {
- this.cacheFilePath = new Path(conf.get(SPLITS_CACHE_DIR, DEFAULT_SPLITS_CACHE_DIR), SPLITS_CACHE_FILE);
+ this.cacheFilePath = new Path(conf.get(SPLITS_CACHE_DIR, DEFAULT_SPLITS_CACHE_DIR), conf.get(SPLITS_CACHE_FILE, DEFAULT_SPLITS_CACHE_FILE));
+ }
+
+ public void setCacheFilePath(Path path) {
+ this.cacheFilePath = path;
}
public static Path getSplitsPath(Configuration conf) {
- return new Path(conf.get(SPLITS_CACHE_DIR, DEFAULT_SPLITS_CACHE_DIR), SPLITS_CACHE_FILE);
+ return new Path(conf.get(SPLITS_CACHE_DIR, DEFAULT_SPLITS_CACHE_DIR), conf.get(SPLITS_CACHE_FILE, DEFAULT_SPLITS_CACHE_FILE));
}
@Override
@@ -141,34 +195,89 @@ private Set getIngestTableNames() {
*/
@Override
public void writeCacheFile(FileSystem fs, Path tmpSplitsFile) throws IOException {
- TableOperations tops = null;
initAccumuloHelper();
- try (AccumuloClient client = this.accumuloHelper.newClient()) {
- tops = client.tableOperations();
- Set tableNames = getIngestTableNames();
- Map splitsPerTable = new HashMap<>();
- try (PrintStream out = new PrintStream(new BufferedOutputStream(fs.create(tmpSplitsFile)))) {
- this.splits = new HashMap<>();
- // gather the splits and write to PrintStream
- for (String table : tableNames) {
- log.info("Retrieving splits for " + table);
- List splits = new ArrayList<>(tops.listSplits(table));
- this.splits.put(table, splits);
- splitsPerTable.put(table, splits.size());
- log.info("Writing " + splits.size() + " splits.");
- for (Text split : splits) {
- out.println(table + this.delimiter + new String(Base64.encodeBase64(split.getBytes())));
+ Map splitsPerTable = new HashMap<>();
+ Set tableNames = getIngestTableNames();
+
+ try {
+ updatePartitionerCache(tableNames);
+ } catch (ClassNotFoundException cnf) {
+ throw new IOException("Unable to generate splits. Invalid partitioner configuration ", cnf);
+ }
+
+ try (PrintStream out = new PrintStream(new BufferedOutputStream(fs.create(tmpSplitsFile)))) {
+ // gather the splits and write to PrintStream
+ for (String table : tableNames) {
+ Partitioner partitioner = partitionerCache.getPartitioner(new Text(table));
+
+ log.info("Retrieving splits for " + table);
+ Map splitLocations = getSplitsWithLocation(table);
+ List splits = new ArrayList<>(splitLocations.keySet());
+ splitsPerTable.put(table, splits.size());
+
+ log.info("Writing " + splits.size() + " splits.");
+
+ // write splits according to what each table's configured partitioner will need
+ if (partitioner instanceof DelegatePartitioner) {
+ if (!((DelegatePartitioner) partitioner).needSplits()) {
+ log.info("Splits not required for " + partitioner.getClass().getName() + ", " + table);
+ continue;
+ } else {
+ if (((DelegatePartitioner) partitioner).needSplitLocations()) {
+ // only write locations if we need them for partitioner logic
+ if (partitioner instanceof BalancedShardPartitioner) {
+ splitLocations = reverseSortByShardIds(splitLocations);
+ }
+ writeLocations(out, table, splits, splitLocations);
+
+ } else {
+ writeSplits(out, table, splits);
+ }
}
+
+ } else {
+ writeSplits(out, table, splits);
}
- if (null != getFileStatus() && exceedsMaxSplitsDeviation(splitsPerTable)) {
- // if the file exists and the new file would exceed the deviation threshold, don't replace it
- throw new IOException("Splits file will not be replaced");
- }
- } catch (IOException | AccumuloSecurityException | AccumuloException | TableNotFoundException ex) {
- log.error("Unable to write new splits file", ex);
- throw new IOException(ex);
+
+ }
+ if (null != getFileStatus() && exceedsMaxSplitsDeviation(splitsPerTable)) {
+ // if the file exists and the new file would exceed the deviation threshold, don't replace it
+ throw new IOException("Splits file will not be replaced");
+ }
+ } catch (IOException | AccumuloSecurityException | AccumuloException | TableNotFoundException ex) {
+ log.error("Unable to write new splits file", ex);
+ throw new IOException(ex);
+ }
+
+ }
+
+ private void writeSplits(PrintStream out, String table, List splits) {
+ for (Text split : splits) {
+ out.println(table + this.delimiter + new String(Base64.encodeBase64(split.getBytes())));
+ }
+ }
+
+ private void writeLocations(PrintStream out, String table, List splits, Map splitLocations) {
+ for (Text split : splits) {
+ out.println(table + this.delimiter + new String(Base64.encodeBase64(split.getBytes())) + "\t" + splitLocations.get(split));
+ }
+ }
+
+ public TreeMap reverseSortByShardIds(Map shardIdToLocations) {
+
+ TreeMap shardIdsToTservers = Maps.newTreeMap((o1, o2) -> o2.compareTo(o1));
+ shardIdsToTservers.putAll(shardIdToLocations);
+ return shardIdsToTservers;
+ }
+
+ private void updatePartitionerCache(Set tableNames) throws ClassNotFoundException {
+ ArrayList tables = new ArrayList<>();
+ for (String t : tableNames) {
+ if (partitionerCache.hasPartitionerOverride(new Text(t))) {
+ tables.add(new Text(t));
}
}
+ partitionerCache.createAndCachePartitioners(tables);
}
@@ -177,10 +286,8 @@ private boolean exceedsMaxSplitsDeviation(Map tmpSplitsPerTable)
if (!currentSplitsPerTable.isEmpty()) {
Set currentTables = currentSplitsPerTable.keySet();
for (String tableName : currentTables) {
- if (currentSplitsPerTable.get(tableName)
- * (1 - conf.getDouble(MAX_SPLIT_PERCENTAGE_DECREASE, DEFAULT_MAX_SPLIT_PERCENTAGE_DECREASE)) > tmpSplitsPerTable.get(tableName)
- && currentSplitsPerTable.get(tableName) - tmpSplitsPerTable.get(tableName) > conf.getInt(MAX_SPLIT_DECREASE,
- DEFAULT_MAX_SPLIT_DECREASE)) {
+ if (tableExceedsMaxSplitDeviation(tmpSplitsPerTable.get(tableName), currentSplitsPerTable.get(tableName), tableName)) {
+
log.warn(tableName
+ "Splits have decreased by greater than MAX_SPLIT_DECREASE or MAX_SPLIT_PERCENTAGE_DECREASE. Splits file will not be replaced. To force replacement, delete the current file and run generateSplitsFile.sh");
return true;
@@ -191,20 +298,54 @@ private boolean exceedsMaxSplitsDeviation(Map tmpSplitsPerTable)
}
+ boolean tableExceedsMaxSplitDeviation(Integer newCount, Integer currentCount, String tableName) {
+ double maxSplitPercentDecrease = conf.getDouble(MAX_SPLIT_PERCENTAGE_DECREASE, DEFAULT_MAX_SPLIT_PERCENTAGE_DECREASE);
+ int maxSplitDecrease = conf.getInt(MAX_SPLIT_DECREASE, DEFAULT_MAX_SPLIT_DECREASE);
+ return currentCount * (1 - maxSplitPercentDecrease) > newCount && currentCount - newCount > maxSplitDecrease;
+ }
+
private Map getCurrentSplitsPerTable() {
Map currentSplitsPerTable = new HashMap<>();
- if (null == this.splits)
+ if (splitLocations.isEmpty())
try {
read();
} catch (IOException ex) {
- log.warn("No splits file exists");
+ log.warn("Failed to read splits file", ex);
}
- for (String tableName : this.splits.keySet()) {
- currentSplitsPerTable.put(tableName, this.splits.get(tableName).size());
+ for (String tableName : this.splitLocations.keySet()) {
+ currentSplitsPerTable.put(tableName, this.splitLocations.get(tableName).size());
}
return currentSplitsPerTable;
}
+ @Override
+ public void read() throws IOException {
+ if (cacheFileRead) {
+ return;
+ }
+
+ synchronized (semaphore) {
+ if (cacheFileRead) {
+ return;
+ }
+
+ try {
+ log.info("Attempting to read splits from distributed cache");
+ File distCacheSplitsFile = new File("./" + SplitsFile.DIST_CACHE_LABEL);
+ FileInputStream fis = new FileInputStream(distCacheSplitsFile);
+ BufferedReader bis = new BufferedReader(new InputStreamReader(fis));
+ readCache(bis);
+ cacheFileRead = true;
+ log.info("Great success! High five!");
+ } catch (IOException e) {
+ log.info("Unable to read splits file from Distributed Cache. Proceeding to HDFS");
+ super.read();
+ cacheFileRead = true;
+ }
+
+ }
+ }
+
/**
* Read a stream into a map of table name to split points
*
@@ -215,24 +356,49 @@ private Map getCurrentSplitsPerTable() {
*/
@Override
protected void readCache(BufferedReader in) throws IOException {
+ this.splitLocations = new HashMap<>();
this.splits = new HashMap<>();
+ // since we do not know whether -XX:+UseStringDeduplication is being used, lets do it ourselves
+ Map locationDedup = new HashMap<>();
String line;
String tableName = null;
- List splits = null;
+ Map tmpSplitLocations = new ShardLocationTrieMap();
+ List tmpSplits = null;
+
while ((line = in.readLine()) != null) {
String[] parts = StringUtils.split(line, this.delimiter);
if (tableName == null || !tableName.equals(parts[0])) {
+ if (!tmpSplitLocations.isEmpty()) {
+ this.splitLocations.put(tableName, tmpSplitLocations);
+ }
tableName = parts[0];
- splits = new ArrayList<>();
- this.splits.put(tableName, splits);
+ tmpSplitLocations = new ShardLocationTrieMap();
+ tmpSplits = new ArrayList<>();
+ this.splits.put(tableName, Collections.unmodifiableList(tmpSplits));
}
- if (parts.length > 1) {
- splits.add(new Text(Base64.decodeBase64(parts[1].getBytes())));
+ if (parts.length >= 2) {
+ Text split = new Text(Base64.decodeBase64(parts[1]));
+ tmpSplits.add(split);
+ if (parts.length == 3) {
+ tmpSplitLocations.put(split, dedup(locationDedup, parts[2]));
+ }
}
}
+ if (!tmpSplitLocations.isEmpty()) {
+ this.splitLocations.put(tableName, tmpSplitLocations);
+ }
in.close();
}
+ private String dedup(Map dedupMap, String value) {
+ String dedup = dedupMap.get(value);
+ if (dedup == null) {
+ dedupMap.put(value, value);
+ return value;
+ }
+ return dedup;
+ }
+
/**
*
* @param table
@@ -242,13 +408,11 @@ protected void readCache(BufferedReader in) throws IOException {
* for issues with read or write
*/
public List getSplits(String table) throws IOException {
- if (null == this.splits)
+ if (this.splits.isEmpty()) {
read();
- List tableSplits = this.splits.get(table);
- if (tableSplits == null) {
- return Collections.emptyList();
}
- return tableSplits;
+ List splitList = this.splits.get(table);
+ return (splitList == null ? Collections.emptyList() : splitList);
}
/**
@@ -271,9 +435,33 @@ public List getSplits(String table, int maxSplits) throws IOException {
* for issues with read or write
*/
public Map> getSplits() throws IOException {
- if (null == this.splits)
+ if (this.splits.isEmpty())
read();
- return new HashMap<>(splits);
+ return Collections.unmodifiableMap(splits);
}
+ /**
+ * @param table
+ * @return map of splits to tablet locations for the table
+ * @throws IOException
+ */
+ public Map getSplitsAndLocationByTable(String table) throws IOException {
+ if (this.splitLocations.isEmpty())
+ read();
+ if (this.splitLocations.containsKey(table)) {
+ return this.splitLocations.get(table);
+ } else {
+ return Collections.emptyMap();
+ }
+ }
+
+ /**
+ * @return map of splits to table name to map of splits to table locations for the table
+ * @throws java.io.IOException
+ */
+ public Map> getSplitsAndLocation() throws IOException {
+ if (this.splitLocations.isEmpty())
+ read();
+ return Collections.unmodifiableMap(splitLocations);
+ }
}
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/BalancedShardPartitioner.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/BalancedShardPartitioner.java
index cdb55307b4..4eb8b2b43f 100755
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/BalancedShardPartitioner.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/BalancedShardPartitioner.java
@@ -5,6 +5,7 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.TreeMap;
@@ -20,8 +21,9 @@
import com.google.common.collect.Maps;
import datawave.ingest.mapreduce.handler.shard.ShardIdFactory;
+import datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler;
import datawave.ingest.mapreduce.job.BulkIngestKey;
-import datawave.ingest.mapreduce.job.ShardedTableMapFile;
+import datawave.ingest.mapreduce.job.SplitsFile;
import datawave.util.time.DateHelper;
/**
@@ -39,7 +41,6 @@ public class BalancedShardPartitioner extends Partitioner i
private static final String today = formatDay(0);
private Configuration conf;
private Map> shardPartitionsByTable;
- private Map> shardIdToLocations = Maps.newHashMap();
private Map offsetsFactorByTable;
int missingShardIdCount = 0;
@@ -54,6 +55,9 @@ public synchronized int getPartition(BulkIngestKey key, Value value, int numRedu
int partition = getAssignedPartition(key.getTableName().toString(), key.getKey().getRow());
// the offsets should help send today's shard data to a different set of reducers than today's error shard data
+ if (null == offsetsFactorByTable.get(key.getTableName())) {
+ log.error("We have received a key for a table we are not configured for. Please verify your sharded table configurations.");
+ }
int offsetForTable = shardIdFactory.getNumShards(key.getKey().getTimestamp()) * offsetsFactorByTable.get(key.getTableName());
return (partition + offsetForTable) % numReduceTasks;
@@ -130,14 +134,12 @@ private HashMap getPartitionsByShardId(String tableName) throws IO
if (log.isDebugEnabled())
log.debug("Loading splits data for " + tableName);
- TreeMap shardIdToLocation = shardIdToLocations.get(tableName);
- if (null == shardIdToLocation) {
- shardIdToLocation = ShardedTableMapFile.getShardIdToLocations(conf, tableName);
- shardIdToLocations.put(tableName, shardIdToLocation);
- }
+ List sortedSplits = SplitsFile.getSplits(conf, tableName);
+ Map shardIdToLocation = SplitsFile.getSplitsAndLocations(conf, tableName);
+
if (log.isDebugEnabled())
log.debug("Assigning partitioners for each shard in " + tableName);
- return assignPartitionsForEachShard(shardIdToLocation);
+ return assignPartitionsForEachShard(sortedSplits, shardIdToLocation);
}
/**
@@ -156,12 +158,11 @@ private HashMap getPartitionsByShardId(String tableName) throws IO
* the map of shard ids and their location
* @return shardId to
*/
- private HashMap assignPartitionsForEachShard(TreeMap shardIdToLocations) {
+ private HashMap assignPartitionsForEachShard(List sortedShardIds, Map shardIdToLocations) {
int totalNumUniqueTServers = calculateNumberOfUniqueTservers(shardIdToLocations);
- TreeMap sortedShardIdsToTservers = reverseSortByShardIds(shardIdToLocations);
- HashMap partitionsByTServer = getTServerAssignments(totalNumUniqueTServers, sortedShardIdsToTservers);
- HashMap partitionsByShardId = getShardIdAssignments(sortedShardIdsToTservers, partitionsByTServer);
+ HashMap partitionsByTServer = getTServerAssignments(totalNumUniqueTServers, sortedShardIds, shardIdToLocations);
+ HashMap partitionsByShardId = getShardIdAssignments(shardIdToLocations, partitionsByTServer);
if (log.isDebugEnabled())
log.debug("Number of shardIds assigned: " + partitionsByShardId.size());
@@ -169,31 +170,26 @@ private HashMap assignPartitionsForEachShard(TreeMap
return partitionsByShardId;
}
- private int calculateNumberOfUniqueTservers(TreeMap shardIdToLocations) {
+ private int calculateNumberOfUniqueTservers(Map shardIdToLocations) {
+
int totalNumUniqueTServers = new HashSet(shardIdToLocations.values()).size();
if (log.isDebugEnabled())
log.debug("Total TServers involved: " + totalNumUniqueTServers);
return totalNumUniqueTServers;
}
- private TreeMap reverseSortByShardIds(TreeMap shardIdToLocations) {
- // drop the dates after today's date
- TreeMap shardIdsToTservers = Maps.newTreeMap((o1, o2) -> o2.compareTo(o1));
- shardIdsToTservers.putAll(shardIdToLocations);
- return shardIdsToTservers;
- }
-
- private HashMap getTServerAssignments(int totalNumTServers, TreeMap shardIdsToTservers) {
+ private HashMap getTServerAssignments(int totalNumTServers, List sortedShardIds, Map shardIdsToTservers) {
HashMap partitionsByTServer = new HashMap<>(totalNumTServers);
int nextAvailableSlot = 0;
boolean alreadySkippedFutureShards = false;
- for (Map.Entry entry : shardIdsToTservers.entrySet()) {
- if (alreadySkippedFutureShards || !isFutureShard(entry.getKey())) { // short circuiting for performance
+ for (Text shard : sortedShardIds) {
+ if (alreadySkippedFutureShards || !isFutureShard(shard)) { // short circuiting for performance
alreadySkippedFutureShards = true;
- Integer assignedPartition = partitionsByTServer.get(entry.getValue());
+ String location = shardIdsToTservers.get(shard);
+ Integer assignedPartition = partitionsByTServer.get(location);
if (null == assignedPartition) {
assignedPartition = nextAvailableSlot;
- partitionsByTServer.put(entry.getValue(), assignedPartition);
+ partitionsByTServer.put(location, assignedPartition);
nextAvailableSlot++;
}
if (partitionsByTServer.size() == totalNumTServers) {
@@ -217,7 +213,7 @@ private static String formatDay(int numDaysBack) {
return DateHelper.format(now - (DateUtils.MILLIS_PER_DAY * numDaysBack));
}
- private HashMap getShardIdAssignments(TreeMap shardIdsToTservers, HashMap partitionsByTServer) {
+ private HashMap getShardIdAssignments(Map shardIdsToTservers, HashMap partitionsByTServer) {
HashMap partitionsByShardId = new HashMap<>();
for (Map.Entry entry : shardIdsToTservers.entrySet()) {
partitionsByShardId.put(entry.getKey(), partitionsByTServer.get(entry.getValue()));
@@ -236,6 +232,16 @@ public int getNumPartitions() {
@Override
public void initializeJob(Job job) {}
+ @Override
+ public boolean needSplits() {
+ return true;
+ }
+
+ @Override
+ public boolean needSplitLocations() {
+ return true;
+ }
+
@Override
public Configuration getConf() {
return conf;
@@ -251,7 +257,7 @@ public void setConf(Configuration conf) {
private void defineOffsetsForTables(Configuration conf) {
offsetsFactorByTable = new HashMap<>();
int offsetFactor = 0;
- for (String tableName : conf.getStrings(ShardedTableMapFile.CONFIGURED_SHARDED_TABLE_NAMES)) {
+ for (String tableName : conf.getStrings(ShardedDataTypeHandler.SHARDED_TNAMES)) {
offsetsFactorByTable.put(new Text(tableName), offsetFactor++);
}
}
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/DelegatePartitioner.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/DelegatePartitioner.java
index b531af2982..78f369b9f2 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/DelegatePartitioner.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/DelegatePartitioner.java
@@ -33,4 +33,8 @@ public interface DelegatePartitioner extends Configurable {
* job to be configured
*/
void initializeJob(Job job);
+
+ boolean needSplits();
+
+ boolean needSplitLocations();
}
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/LimitedKeyPartitioner.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/LimitedKeyPartitioner.java
index c5a6c037ce..5b19cbea8b 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/LimitedKeyPartitioner.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/LimitedKeyPartitioner.java
@@ -62,4 +62,14 @@ public int getNumPartitions() {
public void initializeJob(Job job) {
// no op
}
+
+ @Override
+ public boolean needSplits() {
+ return false;
+ }
+
+ @Override
+ public boolean needSplitLocations() {
+ return false;
+ }
}
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/LocalityGroupPartitioner.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/LocalityGroupPartitioner.java
index 9d5c186605..09ed19d15d 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/LocalityGroupPartitioner.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/LocalityGroupPartitioner.java
@@ -77,4 +77,14 @@ public int getNumPartitions() {
public void initializeJob(Job job) {
// no op
}
+
+ @Override
+ public boolean needSplits() {
+ return false;
+ }
+
+ @Override
+ public boolean needSplitLocations() {
+ return false;
+ }
}
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/MultiTableRRRangePartitioner.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/MultiTableRRRangePartitioner.java
index 597d7c0502..1b5f59747c 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/MultiTableRRRangePartitioner.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/MultiTableRRRangePartitioner.java
@@ -19,7 +19,7 @@ protected int calculateIndex(int index, int numPartitions, String tableName, int
int i = cutPointArrayLength;
int reducer = numPartitions - 1;
Map tempSplitReducerMap = new HashMap<>();
- // start with the indice that represents a value greater than all values in the cutPointArray, start filling in the mapping of indices to reducers
+ // start with the index that represents a value greater than all values in the cutPointArray, start filling in the mapping of indices to reducers
tempSplitReducerMap.put(i, reducer);
tempSplitReducerMap.put(-i - 1, reducer);
i--;
@@ -48,8 +48,4 @@ protected int calculateIndex(int index, int numPartitions, String tableName, int
}
}
- @Override
- protected boolean isTrimmed() {
- return false;
- }
}
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/MultiTableRangePartitioner.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/MultiTableRangePartitioner.java
index 3a287a97bb..dba0e0547f 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/MultiTableRangePartitioner.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/partition/MultiTableRangePartitioner.java
@@ -1,17 +1,15 @@
package datawave.ingest.mapreduce.partition;
import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
import java.text.DecimalFormat;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
import java.util.Map;
-import org.apache.accumulo.core.client.TableExistsException;
-import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.Value;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
@@ -20,11 +18,10 @@
import org.apache.log4j.Logger;
import datawave.ingest.mapreduce.job.BulkIngestKey;
-import datawave.ingest.mapreduce.job.NonShardedSplitsFile;
-import datawave.ingest.mapreduce.job.TableConfigurationUtil;
+import datawave.ingest.mapreduce.job.SplitsFile;
/**
- * Range partitioner that uses a split file with the format: {@code tableNamesplitPoint}
+ * Range partitioner that uses a split file with the format: {@code tableNamesplitPointtabletLocation}
*
*/
public class MultiTableRangePartitioner extends Partitioner implements DelegatePartitioner {
@@ -35,21 +32,21 @@ public class MultiTableRangePartitioner extends Partitioner
static TaskInputOutputContext,?,?,?> context = null;
private static boolean collectStats = false;
- private volatile boolean cacheFilesRead = false;
+ protected volatile boolean cacheFilesRead = false;
+
private Text holder = new Text();
- private ThreadLocal