Skip to content

Commit

Permalink
Avoid useless object creation where possible by delegating to TypeFac…
Browse files Browse the repository at this point in the history 
…tory (#2487)

* Avoid useless object creation where possible by delegating to TypeFactory

* Update submodules for metadata-utils and type-utils

* Expose TypeFactory cache size and ttl options in LimitFields and EventFieldAggregator, migrate to 'get or create' paradigm.

* Revert submodule updates in favor of formal tags
Increment type-utils to 3.0.2
Increment metadata-utils to 4.0.6
  • Loading branch information
@apmoriarty
apmoriarty committed Aug 21, 2024
1 parent d6e5db0 commit ed6a3da
Show file tree
Hide file tree
Showing 4 changed files with 109 additions and 28 deletions.
4 changes: 2 additions & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,11 +107,11 @@
<version.microservice.common-utils>3.0.0</version.microservice.common-utils>
<version.microservice.dictionary-api>4.0.0</version.microservice.dictionary-api>
<version.microservice.mapreduce-query-api>1.0.0</version.microservice.mapreduce-query-api>
<version.microservice.metadata-utils>4.0.5</version.microservice.metadata-utils>
<version.microservice.metadata-utils>4.0.6</version.microservice.metadata-utils>
<version.microservice.metrics-reporter>3.0.0</version.microservice.metrics-reporter>
<version.microservice.query-api>1.0.0</version.microservice.query-api>
<version.microservice.query-metric-api>4.0.0</version.microservice.query-metric-api>
<version.microservice.type-utils>3.0.1</version.microservice.type-utils>
<version.microservice.type-utils>3.0.2</version.microservice.type-utils>
<version.minlog>1.2</version.minlog>
<version.mockito>2.23.0</version.mockito>
<version.mysql-connector>8.0.28</version.mysql-connector>
Expand Down
42 changes: 41 additions & 1 deletion warehouse/query-core/src/main/java/datawave/query/function/LimitFields.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import com.google.common.collect.Multimap;

import datawave.data.type.Type;
import datawave.data.type.TypeFactory;
import datawave.query.attributes.Attribute;
import datawave.query.attributes.Attributes;
import datawave.query.attributes.Content;
Expand Down Expand Up @@ -64,6 +65,10 @@ public class LimitFields implements Function<Entry<Key,Document>,Entry<Key,Docum
// should not be dropped
private final Set<Set<String>> matchingFieldSets;

private int typeCacheSize = -1;
private int typeCacheTimeoutMinutes = -1;
private TypeFactory typeFactory;

public LimitFields(Map<String,Integer> limitFieldsMap, Set<Set<String>> matchingFieldSets) {
this.limitFieldsMap = limitFieldsMap;
this.matchingFieldSets = matchingFieldSets;
Expand Down Expand Up @@ -295,7 +300,7 @@ private boolean isHit(String keyWithGrouping, Attribute<?> attr, Multimap<String
for (Object hitValue : hitTermMap.get(keyWithGrouping)) {
try {
if (Type.class.isAssignableFrom(clazz)) {
Type<?> thing = (Type<?>) clazz.newInstance();
Type<?> thing = getTypeFactory().createType(clazz.getName());
thing.setDelegateFromString(String.valueOf(hitValue));
hitValue = thing;
} else { // otherwise, s is not a Type, just compare as string values
Expand Down Expand Up @@ -396,4 +401,39 @@ public Integer put(String key, Integer value) {
}
}

/**
* Get the TypeFactory. If no TypeFactory exists one will be created. Configs for cache size and timeout may be configured.
*
* @return the TypeFactory
*/
private TypeFactory getTypeFactory() {
if (typeFactory == null) {
if (typeCacheSize != -1 && typeCacheTimeoutMinutes != -1) {
typeFactory = new TypeFactory(typeCacheSize, typeCacheTimeoutMinutes);
} else {
typeFactory = new TypeFactory();
}
}
return typeFactory;
}

/**
* Set the cache size for the TypeFactory
*
* @param typeCacheSize
* the cache size
*/
public void setTypeCacheSize(int typeCacheSize) {
this.typeCacheSize = typeCacheSize;
}

/**
* Set the timeout for the TypeFactory
*
* @param typeCacheTimeoutMinutes
* the timeout
*/
public void setTypeCacheTimeoutMinutes(int typeCacheTimeoutMinutes) {
this.typeCacheTimeoutMinutes = typeCacheTimeoutMinutes;
}
}
72 changes: 47 additions & 25 deletions warehouse/query-core/src/main/java/datawave/query/jexl/functions/EventFieldAggregator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,14 @@
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;

import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;

import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;

import datawave.data.type.Type;
import datawave.data.type.TypeFactory;
import datawave.query.attributes.Attribute;
import datawave.query.attributes.AttributeFactory;
import datawave.query.attributes.Document;
Expand All @@ -28,18 +23,13 @@
import datawave.query.util.TypeMetadata;

public class EventFieldAggregator extends IdentityAggregator {
// speedy cache loading for types, duplicated from AttributeFactory with caching of types rather than classes
protected static final LoadingCache<String,Type<?>> typeCache = CacheBuilder.newBuilder().maximumSize(128).expireAfterAccess(1, TimeUnit.HOURS)
.build(new CacheLoader<String,Type<?>>() {
@Override
public Type<?> load(String clazz) throws Exception {
Class<?> c = Class.forName(clazz);
return (Type<?>) c.newInstance();
}
});

private TypeMetadata typeMetadata;
private String defaultTypeClass;

private final TypeMetadata typeMetadata;
private final String defaultTypeClass;

private int typeCacheSize = -1;
private int typeCacheTimeoutMinutes = -1;
private TypeFactory typeFactory;

public EventFieldAggregator(String field, EventDataQueryFilter filter, int maxNextCount, TypeMetadata typeMetadata, String defaultTypeClass) {
super(Collections.singleton(field), filter, maxNextCount);
Expand Down Expand Up @@ -122,16 +112,48 @@ private Set<String> getNormalizedValues(String dataType, String fieldName, Strin

// transform the key for each type and add it to the normalized set
for (String typeClass : typeClasses) {
try {
Type<?> type = typeCache.get(typeClass);
String normalizedValue = type.normalize(fieldValue);
Type<?> type = getTypeFactory().createType(typeClass);
String normalizedValue = type.normalize(fieldValue);

normalizedValues.add(normalizedValue);
} catch (ExecutionException e) {
throw new RuntimeException("cannot instantiate class '" + typeClass + "'", e);
}
normalizedValues.add(normalizedValue);
}

return normalizedValues;
}

/**
* Get the TypeFactory. If no TypeFactory exists one will be created. Configs for cache size and timeout may be configured.
*
* @return the TypeFactory
*/
private TypeFactory getTypeFactory() {
if (typeFactory == null) {
if (typeCacheSize != -1 && typeCacheTimeoutMinutes != -1) {
typeFactory = new TypeFactory(typeCacheSize, typeCacheTimeoutMinutes);
} else {
typeFactory = new TypeFactory();
}
}
return typeFactory;
}

/**
* Set the cache size for the TypeFactory
*
* @param typeCacheSize
* the cache size
*/
public void setTypeCacheSize(int typeCacheSize) {
this.typeCacheSize = typeCacheSize;
}

/**
* Set the timeout for the TypeFactory
*
* @param typeCacheTimeoutMinutes
* the timeout
*/
public void setTypeCacheTimeoutMinutes(int typeCacheTimeoutMinutes) {
this.typeCacheTimeoutMinutes = typeCacheTimeoutMinutes;
}
}
19 changes: 19 additions & 0 deletions warehouse/query-core/src/test/java/datawave/query/attributes/AttributeFactoryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import java.util.Collection;

import org.apache.accumulo.core.data.Key;
import org.apache.log4j.Logger;
import org.junit.Assert;
import org.junit.Test;
Expand All @@ -11,6 +12,7 @@
import datawave.data.type.LcNoDiacriticsType;
import datawave.data.type.NoOpType;
import datawave.data.type.NumberType;
import datawave.query.util.TypeMetadata;

/**
*
Expand Down Expand Up @@ -44,7 +46,24 @@ public void testFindersKeepers() {
Assert.assertEquals(AttributeFactory.getKeepers(three), expectedThree);
Assert.assertEquals(AttributeFactory.getKeepers(four), expectedFour);
Assert.assertEquals(AttributeFactory.getKeepers(five), expectedFive);
}

@Test
public void testTypeFactoryCache() {
TypeMetadata metadata = new TypeMetadata();
metadata.put("FIELD", "ingest-type", LcNoDiacriticsType.class.getTypeName());

AttributeFactory factory = new AttributeFactory(metadata);

Key key = new Key("row", "ingest-type\0uid");
Attribute<?> red = factory.create("FIELD", "red", key, true);
Attribute<?> blue = factory.create("FIELD", "blue", key, true);

TypeAttribute<?> redType = (TypeAttribute<?>) red;
TypeAttribute<?> blueType = (TypeAttribute<?>) blue;

Assert.assertEquals("red", redType.getType().getNormalizedValue());
Assert.assertEquals("blue", blueType.getType().getNormalizedValue());
}

}

0 comments on commit ed6a3da

Please sign in to comment.