From 1290186d0051e3f9c0ea00951a9232f802c44821 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 8 Mar 2024 17:45:50 +0000 Subject: [PATCH] Updated jexl parsing to disable some features which we aren't using to improve parsing performance. --- .../datawave/query/jexl/JexlASTHelper.java | 50 ++++++++++++++++++- .../visitors/NodeTypeCountVisitorTest.java | 10 ---- .../visitors/QueryPruningVisitorTest.java | 21 -------- .../visitors/TreeEqualityVisitorTest.java | 18 ------- 4 files changed, 49 insertions(+), 50 deletions(-) diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/JexlASTHelper.java b/warehouse/query-core/src/main/java/datawave/query/jexl/JexlASTHelper.java index a5f3dc3300..a099053b8d 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/JexlASTHelper.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/JexlASTHelper.java @@ -134,6 +134,54 @@ public static ASTJexlScript parseAndFlattenJexlQuery(String query) throws ParseE return TreeFlatteningRebuildingVisitor.flatten(script); } + protected static JexlFeatures jexlFeatures() { + // @formatter:off + return new JexlFeatures() + // mostly used internally by Jexl + .register(false) // default false + // allow usage of let, const, and var variables + .localVar(false) // default true + // allow side-effect operators (e.g. +=, -=, |=, <<=, etc) for global vars. needed to assign values (e.g. _Value_ = true) + .sideEffectGlobal(true) // default true + // allow side-effect operators (e.g. +=, -=, |=, <<=, etc). needed to assign values (e.g. _Value_ = true) + .sideEffect(true) // default true + // allow array indexing via reference expression (e.g. array[some expression]) + .arrayReferenceExpr(false) // default true + // allow method calls on expressions + .methodCall(true) // default true + // allow array/map/set literal expressions (e.g. [1, 2, "three"], {"one": 1, "two": 2}, {"one", 2, "more"} + .structuredLiteral(false) // default true + // allow creation of new instances + .newInstance(false) // default true + // allow loop constructs + .loops(false) // default true + // allow lambda/function constructs (not the same as namespaced functions) + .lambda(false) // default true + // allow thin-arrow lambda syntax (e.g. ->) + .thinArrow(false) // default true + // allow fat-arrow lambda syntax (e.g. =>) + .fatArrow(false) // default false + // allow comparator names (e.g. eq, ne, lt, gt, etc) + .comparatorNames(false) // default true + // allow pragma constructs (e.g. #pragma some.option some-value) + .pragma(false) // default true + // allow pragma constructs anywhere in the code + .pragmaAnywhere(false) // default true + // allow namespace pragmas (e.g. '#pragma jexl.namespace.str java.lang.String' to use 'str' in place of 'java.lang.String') + .namespacePragma(false) // default true + // allow import pragmas (e.g. '#pragma jexl.import java.net' to use new URL() instead of new java.net.URL()) + .importPragma(false) // default true + // allow annotations + .annotation(false) // default true + // allow multiple semicolon-terminated expressions per jexl string + .script(false) // default true + // whether redefining local variables is an error + .lexical(false) // default false + // whether local variables shade global variables + .lexicalShade(false); // default false + // @formatter: on + } + /** * Parse a query string using a JEXL parser and transform it into a parse tree of our RefactoredDatawaveTreeNodes. This also sets all convenience maps that * the analyzer provides. @@ -161,7 +209,7 @@ public static ASTJexlScript parseJexlQuery(String query) throws ParseException { } else { // Parse the original query try { - return parser.parse(null, new JexlFeatures(), caseFixQuery, null); + return parser.parse(null, jexlFeatures(), caseFixQuery, null); } catch (TokenMgrException | JexlException e) { throw new ParseException(e.getMessage()); } diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/NodeTypeCountVisitorTest.java b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/NodeTypeCountVisitorTest.java index 080cd1ed85..ed86311ec9 100644 --- a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/NodeTypeCountVisitorTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/NodeTypeCountVisitorTest.java @@ -246,16 +246,6 @@ public void testASTStringLiteral() { assertEquals(1, count(JexlNodes.makeStringLiteral()).getTotal(ASTStringLiteral.class)); } - @Test - public void testASTArrayLiteral() throws ParseException { - assertEquals(1, count("[1, 2, 3]").getTotal(ASTArrayLiteral.class)); - } - - @Test - public void testASTMapLiteral() throws ParseException { - assertEquals(1, count("{'one':1, 'two':2, 'three':3}").getTotal(ASTMapLiteral.class)); - } - @Test public void testASTMapEntry() { assertEquals(1, count(new ASTMapEntry(ParserTreeConstants.JJTMAPENTRY)).getTotal(ASTMapEntry.class)); diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/QueryPruningVisitorTest.java b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/QueryPruningVisitorTest.java index 84fde2178e..31e358b236 100644 --- a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/QueryPruningVisitorTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/QueryPruningVisitorTest.java @@ -502,27 +502,6 @@ public void nestedMarkerBoundedRangeTest() throws ParseException { assertEquals(query, JexlStringBuildingVisitor.buildQuery(QueryPruningVisitor.reduce(script, false))); } - @Test - public void dualStatementQueryTest() throws ParseException { - String query = "(Expression = 'somevalue'); FIELD == 'x'"; - ASTJexlScript script = JexlASTHelper.parseJexlQuery(query); - - assertEquals(QueryPruningVisitor.TruthState.UNKNOWN, QueryPruningVisitor.getState(script)); - assertEquals(query, JexlStringBuildingVisitor.buildQuery(QueryPruningVisitor.reduce(script, false))); - - query = "(Expression = 'somevalue'); FIELD == 'x' || true"; - script = JexlASTHelper.parseJexlQuery(query); - - assertEquals(QueryPruningVisitor.TruthState.TRUE, QueryPruningVisitor.getState(script)); - assertEquals("(Expression = 'somevalue'); true", JexlStringBuildingVisitor.buildQuery(QueryPruningVisitor.reduce(script, false))); - - query = "(Expression = 'somevalue'); FIELD == 'x' && false"; - script = JexlASTHelper.parseJexlQuery(query); - - assertEquals(QueryPruningVisitor.TruthState.UNKNOWN, QueryPruningVisitor.getState(script)); - assertEquals("(Expression = 'somevalue'); false", JexlStringBuildingVisitor.buildQuery(QueryPruningVisitor.reduce(script, false))); - } - @Test public void propertyMarkerTest() throws ParseException { String query = "((_Value_ = true) && (FIELD = 'x'))"; diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/TreeEqualityVisitorTest.java b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/TreeEqualityVisitorTest.java index 6434c57b92..c13b67ddcc 100644 --- a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/TreeEqualityVisitorTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/TreeEqualityVisitorTest.java @@ -45,24 +45,6 @@ public void testNonEqualImage() throws ParseException { "Did not find a matching child for EQNode in [EQNode]: Did not find a matching child for bar in [bat]: Node images differ: bar vs bat"); } - /** - * Verify that two nodes with differing number of children are considered non-equal. - */ - @Test - public void testNonEqualChildrenSize() throws ParseException { - assertNotEquivalent("[1, 2, 3, 4]", "[1, 2]", - "Did not find a matching child for [ 1, 2, 3, 4 ] in [[ 1, 2 ]]: Num children differ: [1, 2, 3, 4] vs [1, 2]"); - } - - /** - * Verify that two nodes with different children are considered non-equal. - */ - @Test - public void testNonEqualChildren() throws ParseException { - assertNotEquivalent("[1, 2]", "[1, 3]", - "Did not find a matching child for [ 1, 2 ] in [[ 1, 3 ]]: Did not find a matching child for 2 in [3]: Node images differ: 2 vs 3"); - } - /** * Verify that two identical trees are considered equal. */