Add FederatedQueryPlanner

core/query/src/main/java/datawave/core/query/configuration/GenericQueryConfiguration.java

@@ -96,25 +96,36 @@ public GenericQueryConfiguration(BaseQueryLogic<?> configuredLogic) {
         this(configuredLogic.getConfig());
     }
 
-    public GenericQueryConfiguration(GenericQueryConfiguration genericConfig) {
-        this.setQuery(genericConfig.getQuery());
-        this.setCheckpointable(genericConfig.isCheckpointable());
-        this.setBaseIteratorPriority(genericConfig.getBaseIteratorPriority());
-        this.setBypassAccumulo(genericConfig.getBypassAccumulo());
-        this.setAccumuloPassword(genericConfig.getAccumuloPassword());
-        this.setConnPoolName(genericConfig.getConnPoolName());
-        this.setAuthorizations(genericConfig.getAuthorizations());
-        this.setBeginDate(genericConfig.getBeginDate());
-        this.setClient(genericConfig.getClient());
-        this.setEndDate(genericConfig.getEndDate());
-        this.setMaxWork(genericConfig.getMaxWork());
-        this.setQueries(genericConfig.getQueries());
-        this.setQueriesIter(genericConfig.getQueriesIter());
-        this.setQueryString(genericConfig.getQueryString());
-        this.setTableName(genericConfig.getTableName());
-        this.setReduceResults(genericConfig.isReduceResults());
-        this.setTableConsistencyLevels(genericConfig.getTableConsistencyLevels());
-        this.setTableHints(genericConfig.getTableHints());
+    @SuppressWarnings("CopyConstructorMissesField")
+    public GenericQueryConfiguration(GenericQueryConfiguration other) {
+        copyFrom(other);
+    }
+
+    /**
+     * Deeply copies over all fields from the given {@link GenericQueryConfiguration} to this {@link GenericQueryConfiguration}.
+     *
+     * @param other
+     *            the {@link GenericQueryConfiguration} to copy values from
+     */
+    public void copyFrom(GenericQueryConfiguration other) {
+        this.setQuery(other.getQuery());
+        this.setCheckpointable(other.isCheckpointable());
+        this.setBaseIteratorPriority(other.getBaseIteratorPriority());
+        this.setBypassAccumulo(other.getBypassAccumulo());
+        this.setAccumuloPassword(other.getAccumuloPassword());
+        this.setConnPoolName(other.getConnPoolName());
+        this.setAuthorizations(other.getAuthorizations());
+        this.setBeginDate(other.getBeginDate());
+        this.setClient(other.getClient());
+        this.setEndDate(other.getEndDate());
+        this.setMaxWork(other.getMaxWork());
+        this.setQueries(other.getQueries());
+        this.setQueriesIter(other.getQueriesIter());
+        this.setQueryString(other.getQueryString());
+        this.setTableName(other.getTableName());
+        this.setReduceResults(other.isReduceResults());
+        this.setTableConsistencyLevels(other.getTableConsistencyLevels());
+        this.setTableHints(other.getTableHints());
     }
 
     public Collection<QueryData> getQueries() {

warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java

@@ -492,6 +492,12 @@ public class ShardQueryConfiguration extends GenericQueryConfiguration implement
      */
     private boolean sortQueryByCounts = false;
 
+    /**
+     * The minimum percentage threshold that the count for an index row must meet compared to the count for the corresponding frequency row in the metadata
+     * table in order to NOT be considered a field index hole. The value must be between 0.0-1.0, where 1.0 is equivalent to 100%.
+     */
+    private double fieldIndexHoleMinThreshold = 1.0d;
+
     /**
      * Default constructor
      */
@@ -506,10 +512,20 @@ public ShardQueryConfiguration() {
      * @param other
      *            - another ShardQueryConfiguration instance
      */
+    @SuppressWarnings("CopyConstructorMissesField")
     public ShardQueryConfiguration(ShardQueryConfiguration other) {
+        copyFrom(other);
+    }
 
+    /**
+     * Deeply copies over all fields from the given {@link ShardQueryConfiguration} to this {@link ShardQueryConfiguration}.
+     *
+     * @param other
+     *            the {@link ShardQueryConfiguration} to copy values from
+     */
+    public void copyFrom(ShardQueryConfiguration other) {
         // GenericQueryConfiguration copy first
-        super(other);
+        super.copyFrom(other);
 
         // ShardQueryConfiguration copy
         this.setCheckpointable(other.isCheckpointable());
@@ -716,6 +732,7 @@ public ShardQueryConfiguration(ShardQueryConfiguration other) {
         this.setUseTermCounts(other.getUseTermCounts());
         this.setSortQueryBeforeGlobalIndex(other.isSortQueryBeforeGlobalIndex());
         this.setSortQueryByCounts(other.isSortQueryByCounts());
+        this.setFieldIndexHoleMinThreshold(other.getFieldIndexHoleMinThreshold());
     }
 
     /**
@@ -2055,6 +2072,14 @@ public QueryStopwatch getTimers() {
         return timers;
     }
 
+    public void setTimers(QueryStopwatch timers) {
+        this.timers = timers;
+    }
+
+    public void appendTimers(QueryStopwatch timers) {
+        this.timers.appendTimers(timers);
+    }
+
     public ASTJexlScript getQueryTree() {
         return queryTree;
     }
@@ -2688,6 +2713,14 @@ public void setRebuildDatatypeFilterPerShard(boolean rebuildDatatypeFilterPerSha
         this.rebuildDatatypeFilterPerShard = rebuildDatatypeFilterPerShard;
     }
 
+    public double getFieldIndexHoleMinThreshold() {
+        return fieldIndexHoleMinThreshold;
+    }
+
+    public void setFieldIndexHoleMinThreshold(double fieldIndexHoleMinThreshold) {
+        this.fieldIndexHoleMinThreshold = fieldIndexHoleMinThreshold;
+    }
+
     public boolean getReduceIngestTypes() {
         return reduceIngestTypes;
     }

warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java

@@ -340,6 +340,7 @@ protected DefaultQueryPlanner(DefaultQueryPlanner other) {
         setSourceLimit(other.sourceLimit);
         setPushdownThreshold(other.getPushdownThreshold());
         setVisitorManager(other.getVisitorManager());
+        setTransformRules(other.getTransformRules() == null ? null : new ArrayList<>(other.transformRules));
     }
 
     public void setMetadataHelper(final MetadataHelper metadataHelper) {
@@ -1923,17 +1924,23 @@ protected ASTJexlScript parseQueryAndValidatePattern(String query, TraceStopwatc
             if (log.isTraceEnabled()) {
                 log.trace("Stack trace for overflow " + soe);
             }
-            stopwatch.stop();
+            if (stopwatch != null) {
+                stopwatch.stop();
+            }
             PreConditionFailedQueryException qe = new PreConditionFailedQueryException(DatawaveErrorCode.QUERY_DEPTH_OR_TERM_THRESHOLD_EXCEEDED, soe);
             log.warn(qe);
             throw new DatawaveFatalQueryException(qe);
         } catch (ParseException e) {
-            stopwatch.stop();
+            if (stopwatch != null) {
+                stopwatch.stop();
+            }
             BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.UNPARSEABLE_JEXL_QUERY, e, MessageFormat.format("Query: {0}", query));
             log.warn(qe);
             throw new DatawaveFatalQueryException(qe);
         } catch (PatternSyntaxException e) {
-            stopwatch.stop();
+            if (stopwatch != null) {
+                stopwatch.stop();
+            }
             BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.INVALID_REGEX, e, MessageFormat.format("Query: {0}", query));
             log.warn(qe);
             throw new DatawaveFatalQueryException(qe);
@@ -2977,6 +2984,10 @@ public String getPlannedScript() {
         return plannedScript;
     }
 
+    public void setPlannedScript(String plannedScript) {
+        this.plannedScript = plannedScript;
+    }
+
     protected Multimap<String,Type<?>> configureIndexedAndNormalizedFields(MetadataHelper metadataHelper, ShardQueryConfiguration config,
                     ASTJexlScript queryTree) throws DatawaveQueryException {
         // Fetch the mapping of fields to Types from the DatawaveMetadata table

warehouse/query-core/src/main/java/datawave/query/planner/FederatedQueryIterable.java

@@ -0,0 +1,100 @@
+package datawave.query.planner;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import datawave.core.query.configuration.QueryData;
+import datawave.query.CloseableIterable;
+
+/**
+ * Implementation of {@link CloseableIterable} intended to be used by {@link FederatedQueryPlanner}. This iterable
+ */
+public class FederatedQueryIterable implements CloseableIterable<QueryData> {
+
+    private final List<CloseableIterable<QueryData>> iterables = new ArrayList<>();
+
+    /**
+     * Add an iterable to this {@link FederatedQueryIterable}.
+     *
+     * @param iterable
+     *            the iterable to add
+     */
+    public void addIterable(CloseableIterable<QueryData> iterable) {
+        if (iterable != null) {
+            iterables.add(iterable);
+        }
+    }
+
+    /**
+     * Closes and clears each iterable in this {@link FederatedQueryIterable}.
+     *
+     * @throws IOException
+     *             if an error occurred when closing an iterable
+     */
+    @Override
+    public void close() throws IOException {
+        for (CloseableIterable<QueryData> iterable : iterables) {
+            iterable.close();
+        }
+        iterables.clear();
+    }
+
+    /**
+     * Returns an iterator that will iterate over the {@link QueryData} returned by each iterable in this {@link FederatedQueryIterable}.
+     *
+     * @return the iterator
+     */
+    @Override
+    public Iterator<QueryData> iterator() {
+        return new Iter();
+    }
+
+    /**
+     * Iterator implementation that provides the ability to iterate over each {@link QueryData} of the iterables in {@link #iterables}.
+     */
+    private class Iter implements Iterator<QueryData> {
+
+        // Iterator that traverses over the iterables.
+        private final Iterator<CloseableIterable<QueryData>> iterableIterator = iterables.iterator();
+
+        // The current sub iterator.
+        private Iterator<QueryData> currentSubIterator = null;
+
+        @Override
+        public boolean hasNext() {
+            seekToNextAvailableQueryData();
+            return currentSubIterator != null && currentSubIterator.hasNext();
+        }
+
+        @Override
+        public QueryData next() {
+            return currentSubIterator.next();
+        }
+
+        /**
+         * Seek to the next sub-iterator that has a {@link QueryData} remaining in it.
+         */
+        private void seekToNextAvailableQueryData() {
+            // If the current sub iterator is null, attempt to get the next available iterator, or return early if there are no more iterators.
+            if (currentSubIterator == null) {
+                if (iterableIterator.hasNext()) {
+                    currentSubIterator = iterableIterator.next().iterator();
+                } else {
+                    return;
+                }
+            }
+            // If the current sub iterator does not have any more elements remaining, move to the next sub iterator that does have elements.
+            if (!currentSubIterator.hasNext()) {
+                while (iterableIterator.hasNext()) {
+                    // We must ensure we only ever call iterator() once on each sub-iterator.
+                    currentSubIterator = iterableIterator.next().iterator();
+                    if (currentSubIterator.hasNext()) {
+                        return;
+                    }
+                }
+            }
+        }
+    }
+}