diff --git a/core/src/main/java/com/orientechnologies/orient/core/index/OIndexManagerShared.java b/core/src/main/java/com/orientechnologies/orient/core/index/OIndexManagerShared.java index b1de280641c..fd313fc966a 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/index/OIndexManagerShared.java +++ b/core/src/main/java/com/orientechnologies/orient/core/index/OIndexManagerShared.java @@ -129,7 +129,7 @@ public OIndex createIndex( || indexDefinition.getClassName() == null || indexDefinition.getFields() == null || indexDefinition.getFields().isEmpty(); - if (manualIndexesAreUsed) { + if (manualIndexesAreUsed && !"LUCENE_CROSS_CLASS".equals(algorithm)) { OIndexAbstract.manualIndexesWarning(); } else { checkSecurityConstraintsForIndexCreate(database, indexDefinition); diff --git a/lucene/src/main/java/com/orientechnologies/lucene/OLuceneCrossClassIndexFactory.java b/lucene/src/main/java/com/orientechnologies/lucene/OLuceneCrossClassIndexFactory.java new file mode 100755 index 00000000000..87fc4dcb9c9 --- /dev/null +++ b/lucene/src/main/java/com/orientechnologies/lucene/OLuceneCrossClassIndexFactory.java @@ -0,0 +1,186 @@ +/* + * Copyright 2010-2016 OrientDB LTD (http://orientdb.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.orientechnologies.lucene; + +import static com.orientechnologies.orient.core.metadata.schema.OClass.INDEX_TYPE.FULLTEXT; + +import com.orientechnologies.common.log.OLogManager; +import com.orientechnologies.lucene.engine.OLuceneCrossClassIndexEngine; +import com.orientechnologies.lucene.index.OLuceneFullTextIndex; +import com.orientechnologies.orient.core.Orient; +import com.orientechnologies.orient.core.db.ODatabaseDocumentInternal; +import com.orientechnologies.orient.core.db.ODatabaseInternal; +import com.orientechnologies.orient.core.db.ODatabaseLifecycleListener; +import com.orientechnologies.orient.core.exception.OConfigurationException; +import com.orientechnologies.orient.core.index.*; +import com.orientechnologies.orient.core.index.engine.OBaseIndexEngine; +import com.orientechnologies.orient.core.record.impl.ODocument; +import com.orientechnologies.orient.core.storage.OStorage; +import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import org.apache.lucene.analysis.standard.StandardAnalyzer; + +public class OLuceneCrossClassIndexFactory implements OIndexFactory, ODatabaseLifecycleListener { + + public static final String LUCENE_CROSS_CLASS = "LUCENE_CROSS_CLASS"; + + private static final Set TYPES; + private static final Set ALGORITHMS; + + static { + final Set types = new HashSet(); + types.add(FULLTEXT.toString()); + TYPES = Collections.unmodifiableSet(types); + } + + static { + final Set algorithms = new HashSet(); + algorithms.add(LUCENE_CROSS_CLASS); + ALGORITHMS = Collections.unmodifiableSet(algorithms); + } + + public OLuceneCrossClassIndexFactory() { + this(false); + } + + public OLuceneCrossClassIndexFactory(boolean manual) { + if (!manual) Orient.instance().addDbLifecycleListener(this); + } + + @Override + public int getLastVersion(final String algorithm) { + return 0; + } + + @Override + public Set getTypes() { + return TYPES; + } + + @Override + public Set getAlgorithms() { + return ALGORITHMS; + } + + @Override + public OIndexInternal createIndex( + String name, + OStorage storage, + String indexType, + String algorithm, + String valueContainerAlgorithm, + ODocument metadata, + int version) + throws OConfigurationException { + + OAbstractPaginatedStorage paginated = (OAbstractPaginatedStorage) storage.getUnderlying(); + + if (metadata == null) { + metadata = new ODocument().field("analyzer", StandardAnalyzer.class.getName()); + } + + if (FULLTEXT.toString().equalsIgnoreCase(indexType)) { + final int binaryFormatVersion = paginated.getConfiguration().getBinaryFormatVersion(); + OLuceneFullTextIndex index = + new OLuceneFullTextIndex( + name, + indexType, + algorithm, + version, + paginated, + valueContainerAlgorithm, + metadata, + binaryFormatVersion); + + return index; + } + + throw new OConfigurationException("Unsupported type : " + algorithm); + } + + public OBaseIndexEngine createIndexEngine( + int indexId, + String algorithm, + String indexName, + Boolean durableInNonTxMode, + OStorage storage, + int version, + int apiVersion, + boolean multiValue, + Map engineProperties) { + + if (LUCENE_CROSS_CLASS.equalsIgnoreCase(algorithm)) { + return new OLuceneCrossClassIndexEngine(indexId, storage, indexName); + } + throw new OConfigurationException("Unsupported type : " + algorithm); + } + + @Override + public PRIORITY getPriority() { + return PRIORITY.REGULAR; + } + + @Override + public void onCreate(ODatabaseInternal db) { + createCrossClassSearchIndex(db); + } + + @Override + public void onOpen(ODatabaseInternal db) { + createCrossClassSearchIndex(db); + } + + @Override + public void onClose(ODatabaseInternal db) { + OLogManager.instance().debug(this, "onClose"); + } + + @Override + public void onDrop(final ODatabaseInternal db) { + try { + if (db.isClosed()) return; + + OLogManager.instance().debug(this, "Dropping Lucene indexes..."); + + final ODatabaseDocumentInternal internal = (ODatabaseDocumentInternal) db; + internal.getMetadata().getIndexManagerInternal().getIndexes(internal).stream() + .filter(idx -> idx.getInternal() instanceof OLuceneCrossClassIndexEngine) + .peek(idx -> OLogManager.instance().debug(this, "deleting index " + idx.getName())) + .forEach(idx -> idx.delete()); + + } catch (Exception e) { + OLogManager.instance().warn(this, "Error on dropping Lucene indexes", e); + } + } + + @Override + public void onLocalNodeConfigurationRequest(ODocument iConfiguration) {} + + private void createCrossClassSearchIndex(ODatabaseInternal db) { + final ODatabaseDocumentInternal internal = (ODatabaseDocumentInternal) db; + final OIndexManagerAbstract indexManager = internal.getMetadata().getIndexManagerInternal(); + + if (!indexManager.existsIndex("CrossClassSearchIndex")) { + OLogManager.instance().info(this, "creating cross class Lucene index"); + + db.command("CREATE INDEX CrossClassSearchIndex FULLTEXT ENGINE LUCENE_CROSS_CLASS").close(); + } + } +} diff --git a/lucene/src/main/java/com/orientechnologies/lucene/engine/OLuceneCrossClassIndexEngine.java b/lucene/src/main/java/com/orientechnologies/lucene/engine/OLuceneCrossClassIndexEngine.java new file mode 100644 index 00000000000..cd7fd3c32bb --- /dev/null +++ b/lucene/src/main/java/com/orientechnologies/lucene/engine/OLuceneCrossClassIndexEngine.java @@ -0,0 +1,397 @@ +/* + * Copyright 2010-2016 OrientDB LTD (http://orientdb.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.orientechnologies.lucene.engine; + +import static com.orientechnologies.lucene.OLuceneIndexFactory.LUCENE_ALGORITHM; + +import com.orientechnologies.common.log.OLogManager; +import com.orientechnologies.common.serialization.types.OBinarySerializer; +import com.orientechnologies.common.util.ORawPair; +import com.orientechnologies.lucene.analyzer.OLucenePerFieldAnalyzerWrapper; +import com.orientechnologies.lucene.collections.OLuceneResultSet; +import com.orientechnologies.lucene.index.OLuceneFullTextIndex; +import com.orientechnologies.lucene.parser.OLuceneMultiFieldQueryParser; +import com.orientechnologies.lucene.query.OLuceneKeyAndMetadata; +import com.orientechnologies.lucene.query.OLuceneQueryContext; +import com.orientechnologies.lucene.tx.OLuceneTxChanges; +import com.orientechnologies.orient.core.db.ODatabaseDocumentInternal; +import com.orientechnologies.orient.core.db.ODatabaseRecordThreadLocal; +import com.orientechnologies.orient.core.db.record.OIdentifiable; +import com.orientechnologies.orient.core.encryption.OEncryption; +import com.orientechnologies.orient.core.id.OContextualRecordId; +import com.orientechnologies.orient.core.id.ORID; +import com.orientechnologies.orient.core.index.*; +import com.orientechnologies.orient.core.metadata.schema.OClass; +import com.orientechnologies.orient.core.metadata.schema.OType; +import com.orientechnologies.orient.core.record.impl.ODocument; +import com.orientechnologies.orient.core.storage.OStorage; +import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage; +import com.orientechnologies.orient.core.storage.impl.local.paginated.atomicoperations.OAtomicOperation; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiReader; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.highlight.TextFragment; + +/** Created by frank on 03/11/2016. */ +public class OLuceneCrossClassIndexEngine implements OLuceneIndexEngine { + + private final OStorage storage; + private final String indexName; + private final AtomicLong bonsayFileId = new AtomicLong(0); + private final int indexId; + + public OLuceneCrossClassIndexEngine(int indexId, OStorage storage, String indexName) { + this.indexId = indexId; + + this.storage = storage; + this.indexName = indexName; + + OAbstractPaginatedStorage s = (OAbstractPaginatedStorage) storage; + } + + @Override + public void init( + String indexName, + String indexType, + OIndexDefinition indexDefinition, + boolean isAutomatic, + ODocument metadata) {} + + @Override + public void flush() {} + + @Override + public int getId() { + return indexId; + } + + @Override + public void create( + OAtomicOperation atomicOperation, + OBinarySerializer valueSerializer, + boolean isAutomatic, + OType[] keyTypes, + boolean nullPointerSupport, + OBinarySerializer keySerializer, + int keySize, + Map engineProperties, + OEncryption encryption) {} + + @Override + public void delete(OAtomicOperation atomicOperation) {} + + @Override + public void load( + String indexName, + OBinarySerializer valueSerializer, + boolean isAutomatic, + OBinarySerializer keySerializer, + OType[] keyTypes, + boolean nullPointerSupport, + int keySize, + Map engineProperties, + OEncryption encryption) {} + + @Override + public boolean remove(OAtomicOperation atomicOperation, Object key) { + return false; + } + + @Override + public void clear(OAtomicOperation atomicOperation) {} + + @Override + public void close() {} + + @Override + public Object get(Object key) { + + final OLuceneKeyAndMetadata keyAndMeta = (OLuceneKeyAndMetadata) key; + final ODocument metadata = keyAndMeta.metadata; + final List excludes = + Optional.ofNullable(metadata.>getProperty("excludes")) + .orElse(Collections.emptyList()); + final List includes = + Optional.ofNullable(metadata.>getProperty("includes")) + .orElse(Collections.emptyList()); + + final ODatabaseDocumentInternal db = ODatabaseRecordThreadLocal.instance().get(); + final Collection indexes = + db.getMetadata().getIndexManagerInternal().getIndexes(db).stream() + .filter(i -> !excludes.contains(i.getName())) + .filter(i -> includes.isEmpty() || includes.contains(i.getName())) + .collect(Collectors.toList()); + + final OLucenePerFieldAnalyzerWrapper globalAnalyzer = + new OLucenePerFieldAnalyzerWrapper(new StandardAnalyzer()); + + final List globalFields = new ArrayList(); + + final List globalReaders = new ArrayList(); + final Map types = new HashMap<>(); + + try { + for (OIndex index : indexes) { + + if (index.getAlgorithm().equalsIgnoreCase(LUCENE_ALGORITHM) + && index.getType().equalsIgnoreCase(OClass.INDEX_TYPE.FULLTEXT.toString())) { + + final OIndexDefinition definition = index.getDefinition(); + final String className = definition.getClassName(); + + String[] indexFields = + definition.getFields().toArray(new String[definition.getFields().size()]); + + for (int i = 0; i < indexFields.length; i++) { + String field = indexFields[i]; + + types.put(className + "." + field, definition.getTypes()[i]); + globalFields.add(className + "." + field); + } + + OLuceneFullTextIndex fullTextIndex = (OLuceneFullTextIndex) index.getInternal(); + + globalAnalyzer.add((OLucenePerFieldAnalyzerWrapper) fullTextIndex.queryAnalyzer()); + + globalReaders.add(fullTextIndex.searcher().getIndexReader()); + } + } + + IndexReader indexReader = new MultiReader(globalReaders.toArray(new IndexReader[] {})); + + IndexSearcher searcher = new IndexSearcher(indexReader); + + Map boost = + Optional.ofNullable(metadata.>getProperty("boost")) + .orElse(new HashMap<>()); + + OLuceneMultiFieldQueryParser p = + new OLuceneMultiFieldQueryParser( + types, globalFields.toArray(new String[] {}), globalAnalyzer, boost); + + p.setAllowLeadingWildcard( + Optional.ofNullable(metadata.getProperty("allowLeadingWildcard")).orElse(false)); + + p.setSplitOnWhitespace( + Optional.ofNullable(metadata.getProperty("splitOnWhitespace")).orElse(true)); + + Object params = keyAndMeta.key.getKeys().get(0); + + Query query = p.parse(params.toString()); + + final List fields = OLuceneIndexEngineUtils.buildSortFields(metadata); + + OLuceneQueryContext ctx = new OLuceneQueryContext(null, searcher, query, fields); + return new OLuceneResultSet(this, ctx, metadata); + } catch (IOException e) { + OLogManager.instance().error(this, "unable to create multi-reader", e); + } catch (ParseException e) { + OLogManager.instance().error(this, "unable to parse query", e); + } + + return null; + } + + @Override + public void put(OAtomicOperation atomicOperation, Object key, Object value) {} + + @Override + public void update( + OAtomicOperation atomicOperation, Object key, OIndexKeyUpdater updater) { + put(atomicOperation, key, updater.update(null, bonsayFileId).getValue()); + } + + @Override + public boolean validatedPut( + OAtomicOperation atomicOperation, Object key, ORID value, Validator validator) { + return false; + } + + @Override + public Stream> iterateEntriesBetween( + Object rangeFrom, + boolean fromInclusive, + Object rangeTo, + boolean toInclusive, + boolean ascSortOrder, + ValuesTransformer transformer) { + return null; + } + + @Override + public Stream> iterateEntriesMajor( + Object fromKey, boolean isInclusive, boolean ascSortOrder, ValuesTransformer transformer) { + return null; + } + + @Override + public Stream> iterateEntriesMinor( + Object toKey, boolean isInclusive, boolean ascSortOrder, ValuesTransformer transformer) { + return null; + } + + @Override + public Stream> stream(ValuesTransformer valuesTransformer) { + return null; + } + + @Override + public Stream> descStream(ValuesTransformer valuesTransformer) { + return null; + } + + @Override + public Stream keyStream() { + return null; + } + + @Override + public long size(ValuesTransformer transformer) { + return 0; + } + + @Override + public boolean hasRangeQuerySupport() { + return false; + } + + @Override + public String getName() { + return indexName; + } + + @Override + public boolean acquireAtomicExclusiveLock(Object key) { + return false; + } + + @Override + public String getIndexNameByKey(Object key) { + return null; + } + + @Override + public String indexName() { + return null; + } + + @Override + public void onRecordAddedToResultSet( + OLuceneQueryContext queryContext, + OContextualRecordId recordId, + Document ret, + final ScoreDoc score) { + + recordId.setContext( + new HashMap() { + { + Map frag = queryContext.getFragments(); + + frag.entrySet().stream() + .forEach( + f -> { + TextFragment[] fragments = f.getValue(); + StringBuilder hlField = new StringBuilder(); + for (int j = 0; j < fragments.length; j++) { + if ((fragments[j] != null) && (fragments[j].getScore() > 0)) { + hlField.append(fragments[j].toString()); + } + } + put("$" + f.getKey() + "_hl", hlField.toString()); + }); + + put("$score", score.score); + } + }); + } + + @Override + public Document buildDocument(Object key, OIdentifiable value) { + return null; + } + + @Override + public Query buildQuery(Object query) { + return null; + } + + @Override + public Analyzer indexAnalyzer() { + return null; + } + + @Override + public Analyzer queryAnalyzer() { + return null; + } + + @Override + public boolean remove(Object key, OIdentifiable value) { + return false; + } + + @Override + public IndexSearcher searcher() { + return null; + } + + @Override + public void release(IndexSearcher searcher) {} + + @Override + public Set getInTx(Object key, OLuceneTxChanges changes) { + return null; + } + + @Override + public long sizeInTx(OLuceneTxChanges changes) { + return 0; + } + + @Override + public OLuceneTxChanges buildTxChanges() throws IOException { + return null; + } + + @Override + public Query deleteQuery(Object key, OIdentifiable value) { + return null; + } + + @Override + public boolean isCollectionIndex() { + return false; + } + + @Override + public void freeze(boolean throwException) {} + + @Override + public void release() {} +} diff --git a/lucene/src/main/java/com/orientechnologies/lucene/functions/OLuceneFunctionsFactory.java b/lucene/src/main/java/com/orientechnologies/lucene/functions/OLuceneFunctionsFactory.java index b50146a4bbc..fd83c65b8b7 100755 --- a/lucene/src/main/java/com/orientechnologies/lucene/functions/OLuceneFunctionsFactory.java +++ b/lucene/src/main/java/com/orientechnologies/lucene/functions/OLuceneFunctionsFactory.java @@ -23,5 +23,6 @@ public OLuceneFunctionsFactory() { register(new OLuceneSearchOnFieldsFunction()); register(new OLuceneSearchOnClassFunction()); register(new OLuceneSearchMoreLikeThisFunction()); + register(new OLuceneSearchOnCrossClassFunction()); } } diff --git a/lucene/src/main/java/com/orientechnologies/lucene/functions/OLuceneSearchOnCrossClassFunction.java b/lucene/src/main/java/com/orientechnologies/lucene/functions/OLuceneSearchOnCrossClassFunction.java new file mode 100644 index 00000000000..e1e9a223a16 --- /dev/null +++ b/lucene/src/main/java/com/orientechnologies/lucene/functions/OLuceneSearchOnCrossClassFunction.java @@ -0,0 +1,173 @@ +package com.orientechnologies.lucene.functions; + +import static com.orientechnologies.lucene.OLuceneCrossClassIndexFactory.LUCENE_CROSS_CLASS; + +import com.orientechnologies.common.log.OLogManager; +import com.orientechnologies.lucene.builder.OLuceneQueryBuilder; +import com.orientechnologies.lucene.collections.OLuceneCompositeKey; +import com.orientechnologies.lucene.index.OLuceneFullTextIndex; +import com.orientechnologies.lucene.query.OLuceneKeyAndMetadata; +import com.orientechnologies.orient.core.command.OCommandContext; +import com.orientechnologies.orient.core.db.ODatabaseDocumentInternal; +import com.orientechnologies.orient.core.db.ODatabaseRecordThreadLocal; +import com.orientechnologies.orient.core.db.record.OIdentifiable; +import com.orientechnologies.orient.core.id.ORID; +import com.orientechnologies.orient.core.index.OIndex; +import com.orientechnologies.orient.core.record.impl.ODocument; +import com.orientechnologies.orient.core.sql.functions.OIndexableSQLFunction; +import com.orientechnologies.orient.core.sql.functions.OSQLFunctionAbstract; +import com.orientechnologies.orient.core.sql.parser.OBinaryCompareOperator; +import com.orientechnologies.orient.core.sql.parser.OExpression; +import com.orientechnologies.orient.core.sql.parser.OFromClause; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * This function uses the CrossClassIndex to search documents across all the Lucene indexes defined + * in a database + */ +public class OLuceneSearchOnCrossClassFunction extends OSQLFunctionAbstract + implements OIndexableSQLFunction { + + public static final String NAME = "SEARCH_CROSS"; + + public OLuceneSearchOnCrossClassFunction() { + super(NAME, 1, 2); + } + + @Override + public Iterable searchFromTarget( + OFromClause target, + OBinaryCompareOperator operator, + Object rightValue, + OCommandContext ctx, + OExpression... args) { + + OLuceneFullTextIndex fullTextIndex = searchForIndex(); + + OExpression expression = args[0]; + String query = (String) expression.execute((OIdentifiable) null, ctx); + + if (fullTextIndex != null) { + + ODocument metadata = getMetadata(args); + final List luceneResultSet; + try (Stream rids = + fullTextIndex.getRids( + new OLuceneKeyAndMetadata( + new OLuceneCompositeKey(Arrays.asList(query)).setContext(ctx), metadata))) { + luceneResultSet = rids.collect(Collectors.toList()); + } + return luceneResultSet; + } + return Collections.emptySet(); + } + + @Override + public long estimate( + OFromClause target, + OBinaryCompareOperator operator, + Object rightValue, + OCommandContext ctx, + OExpression... args) { + return 1L; + } + + @Override + public boolean canExecuteInline( + OFromClause target, + OBinaryCompareOperator operator, + Object rightValue, + OCommandContext ctx, + OExpression... args) { + return false; + } + + @Override + public boolean allowsIndexedExecution( + OFromClause target, + OBinaryCompareOperator operator, + Object rightValue, + OCommandContext ctx, + OExpression... args) { + return true; + } + + @Override + public boolean shouldExecuteAfterSearch( + OFromClause target, + OBinaryCompareOperator operator, + Object rightValue, + OCommandContext ctx, + OExpression... args) { + return false; + } + + protected OLuceneFullTextIndex searchForIndex() { + Collection indexes = + getDb().getMetadata().getIndexManagerInternal().getIndexes(getDb()); + for (OIndex index : indexes) { + if (index.getInternal() instanceof OLuceneFullTextIndex) { + if (index.getAlgorithm().equalsIgnoreCase(LUCENE_CROSS_CLASS)) { + return (OLuceneFullTextIndex) index; + } + } + } + return null; + } + + protected ODatabaseDocumentInternal getDb() { + return ODatabaseRecordThreadLocal.instance().get(); + } + + private ODocument getMetadata(OExpression[] args) { + if (args.length == 2) { + return new ODocument().fromJSON(args[1].toString()); + } + return OLuceneQueryBuilder.EMPTY_METADATA; + } + + @Override + public Object execute( + Object iThis, + OIdentifiable currentRecord, + Object currentResult, + Object[] params, + OCommandContext ctx) { + + OLuceneFullTextIndex fullTextIndex = searchForIndex(); + + String query = (String) params[0]; + + if (fullTextIndex != null) { + + ODocument metadata = getMetadata(params); + + final List luceneResultSet; + try (Stream rids = + fullTextIndex.getRids( + new OLuceneKeyAndMetadata( + new OLuceneCompositeKey(Arrays.asList(query)).setContext(ctx), metadata))) { + luceneResultSet = rids.collect(Collectors.toList()); + } + return luceneResultSet; + } + return Collections.emptySet(); + } + + private ODocument getMetadata(Object[] params) { + + if (params.length == 2) { + return new ODocument().fromMap((Map) params[1]); + } + + return OLuceneQueryBuilder.EMPTY_METADATA; + } + + @Override + public String getSyntax() { + OLogManager.instance().info(this, "syntax"); + return "SEARCH_CROSS('', {metadata})"; + } +} diff --git a/lucene/src/main/resources/META-INF/services/com.orientechnologies.orient.core.index.OIndexFactory b/lucene/src/main/resources/META-INF/services/com.orientechnologies.orient.core.index.OIndexFactory index 3e0b986cbad..b54d93fec27 100755 --- a/lucene/src/main/resources/META-INF/services/com.orientechnologies.orient.core.index.OIndexFactory +++ b/lucene/src/main/resources/META-INF/services/com.orientechnologies.orient.core.index.OIndexFactory @@ -18,3 +18,4 @@ com.orientechnologies.lucene.OLuceneIndexFactory com.orientechnologies.spatial.OLuceneSpatialIndexFactory +com.orientechnologies.lucene.OLuceneCrossClassIndexFactory diff --git a/lucene/src/test/java/com/orientechnologies/lucene/functions/OLuceneCrossClassIndexTest.java b/lucene/src/test/java/com/orientechnologies/lucene/functions/OLuceneCrossClassIndexTest.java new file mode 100644 index 00000000000..3aad69e3a33 --- /dev/null +++ b/lucene/src/test/java/com/orientechnologies/lucene/functions/OLuceneCrossClassIndexTest.java @@ -0,0 +1,267 @@ +/* + * + * * Copyright 2010-2016 OrientDB LTD (http://orientdb.com) + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.orientechnologies.lucene.functions; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.orientechnologies.lucene.tests.OLuceneBaseTest; +import com.orientechnologies.orient.core.record.OElement; +import com.orientechnologies.orient.core.sql.executor.OResult; +import com.orientechnologies.orient.core.sql.executor.OResultSet; +import java.io.InputStream; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.junit.Before; +import org.junit.Test; + +public class OLuceneCrossClassIndexTest extends OLuceneBaseTest { + + @Before + public void setUp() throws Exception { + InputStream stream = ClassLoader.getSystemResourceAsStream("testLuceneIndex.sql"); + + db.execute("sql", getScriptFromStream(stream)); + + db.command( + "create index Song.title on Song (title,author) FULLTEXT ENGINE LUCENE METADATA {\"analyzer\":\"" + + StandardAnalyzer.class.getName() + + "\"}"); + db.command( + "create index Author.name on Author(name,score) FULLTEXT ENGINE LUCENE METADATA {\"analyzer\":\"" + + StandardAnalyzer.class.getName() + + "\"}"); + } + + @Test + public void shouldSearchTermAcrossAllSubIndexes() throws Exception { + + String query = "select expand(search_CROSS('mountain'))"; + + OResultSet resultSet = db.query(query); + List elements = fetchElements(resultSet).collect(Collectors.toList()); + + assertThat(elements).isNotEmpty(); + + elements.forEach( + el -> { + String className = el.getSchemaType().get().getName(); + if (className.equals("Song")) + assertThat(el.getProperty("title")).containsIgnoringCase("mountain"); + + if (className.equals("Author")) + assertThat(el.getProperty("name")).containsIgnoringCase("mountain"); + }); + } + + private Stream fetchElements(OResultSet resultSet) { + return resultSet.stream() + .map(OResult::getElement) + .filter(Optional::isPresent) + .map(Optional::get); + } + + @Test + public void shouldSearchAcrossAllSubIndexesWithStrictQuery() { + + String query = "select expand(SEARCH_CROSS('Song.title:mountain Author.name:Chuck') )"; + OResultSet resultSet = db.query(query); + + List elements = fetchElements(resultSet).collect(Collectors.toList()); + + assertThat(elements).isNotEmpty(); + + elements.forEach( + el -> { + String className = el.getSchemaType().get().getName(); + if (className.equals("Song")) + assertThat(el.getProperty("title")).containsIgnoringCase("mountain"); + + if (className.equals("Author")) + assertThat(el.getProperty("name")).containsIgnoringCase("chuck"); + }); + } + + @Test + public void shouldSearchAcrossAllSubIndexesUsingMetadataFields() { + + String query = + "SELECT expand(SEARCH_CROSS('(+_CLASS:Song +title:mountain) (+_CLASS:Author +name:Chuck)') )"; + OResultSet resultSet = db.query(query); + + List elements = fetchElements(resultSet).collect(Collectors.toList()); + + assertThat(elements).isNotEmpty(); + + elements.forEach( + el -> { + String className = el.getSchemaType().get().getName(); + if (className.equals("Song")) + assertThat(el.getProperty("title")).containsIgnoringCase("mountain"); + + if (className.equals("Author")) + assertThat(el.getProperty("name")).containsIgnoringCase("chuck"); + }); + } + + @Test + public void shouldSearchAcrossAllClassesWithRangeQuery() { + + String query = + "select expand(SEARCH_CROSS('Song.title:mountain Author.score:[4 TO 7]', {'allowLeadingWildcard' : true})) "; + OResultSet resultSet = db.query(query); + List elements = fetchElements(resultSet).collect(Collectors.toList()); + + assertThat(elements).isNotEmpty(); + + elements.forEach( + el -> { + String className = el.getSchemaType().get().getName(); + assertThat(className).isIn("Song", "Author"); + if (className.equals("Song")) + assertThat(el.getProperty("title")).containsIgnoringCase("mountain"); + + if (className.equals("Author")) + assertThat(el.getProperty("score")) + .isGreaterThanOrEqualTo(4) + .isLessThanOrEqualTo(7); + }); + } + + @Test + public void shouldSearchAcrossExcludingSongIndex() { + + String query = + "select expand(SEARCH_CROSS('Song.title:mountain Author.score:[4 TO 7]', " + + "{'excludes' : ['Song.title']})) "; + OResultSet resultSet = db.query(query); + List elements = fetchElements(resultSet).collect(Collectors.toList()); + + assertThat(elements).isNotEmpty(); + + final List song = + elements.stream() + .filter(el -> el.getSchemaType().get().getName().equals("Song")) + .collect(Collectors.toList()); + + assertThat(song).hasSize(0); + + final List authors = + elements.stream() + .filter(el -> el.getSchemaType().get().getName().equals("Author")) + .collect(Collectors.toList()); + + assertThat(authors.size()).isGreaterThan(0); + } + + @Test + public void shouldSearchAcrossIncludingAuthorIndex() { + + String query = + "select expand(SEARCH_CROSS('Song.title:mountain Author.score:[4 TO 7]', " + + "{'excludes' : ['Song.title']," + + "'includes' : ['Author.name']" + + "})) "; + OResultSet resultSet = db.query(query); + List elements = fetchElements(resultSet).collect(Collectors.toList()); + + assertThat(elements).isNotEmpty(); + + final List song = + elements.stream() + .filter(el -> el.getSchemaType().get().getName().equals("Song")) + .collect(Collectors.toList()); + + assertThat(song).hasSize(0); + + final List authors = + elements.stream() + .filter(el -> el.getSchemaType().get().getName().equals("Author")) + .collect(Collectors.toList()); + + assertThat(authors.size()).isGreaterThan(0); + } + + @Test + public void shouldSearchAcrossAllClassesWithMetedata() { + + String query = + "select expand(SEARCH_CROSS('Author.name:bob Song.title:*tain', {" + + "'allowLeadingWildcard' : true," + + "'boost': {'Author.name':2.0}" + + "})) "; + OResultSet resultSet = db.query(query); + List elements = fetchElements(resultSet).collect(Collectors.toList()); + + assertThat(elements).isNotEmpty(); + + elements.forEach( + el -> { + String className = el.getSchemaType().get().getName(); + assertThat(className).isIn("Song", "Author"); + if (className.equals("Song")) + assertThat(el.getProperty("title")).containsIgnoringCase("mountain"); + + if (className.equals("Author")) + assertThat(el.getProperty("name")).containsIgnoringCase("bob"); + }); + } + + @Test + public void shouldSearchAcrossAllClassesWithCustomSort() { + + String query = + "select expand(SEARCH_CROSS('*:* ', {" + + "sort: [{field:'Author.score', reverse:true, type:'INT' }," + + "{field:'Song.author', type:'STRING'}]" + + "})) limit 11"; + OResultSet resultSet = db.query(query); + + List elements = fetchElements(resultSet).collect(Collectors.toList()); + + assertThat(elements).isNotEmpty(); + + // Authors sorted in reverse mode on score + List scores = + elements.stream() + .filter(e -> e.getSchemaType().get().getName().equals("Author")) + .map(o -> o.getProperty("score")) + .collect(Collectors.toList()); + + assertThat(scores).containsExactly(10, 10, 7, 5, 4); + + // song author sorted on name + List songAuthoer = + elements.stream() + .filter(e -> e.getSchemaType().get().getName().equals("Song")) + .map(o -> o.getProperty("author")) + .collect(Collectors.toList()); + + assertThat(songAuthoer) + .startsWith( + "A.P.Carter", + "Al Green", + "Arthur Cruddup", + "Arthur Cruddup", + "Arthur Cruddup", + "B and S Womack"); + } +}