Fixed issue with indexing buckets/types with duplicated keys

engine/src/main/java/com/arcadedb/database/TransactionContext.java

@@ -30,7 +30,7 @@
 import com.arcadedb.exception.RecordNotFoundException;
 import com.arcadedb.exception.SchemaException;
 import com.arcadedb.exception.TransactionException;
-import com.arcadedb.index.Index;
+import com.arcadedb.index.IndexInternal;
 import com.arcadedb.index.lsm.LSMTreeIndexAbstract;
 import com.arcadedb.log.LogManager;
 
@@ -367,7 +367,8 @@ public void kill() {
    * Executes 1st phase from a replica.
    */
   public void commitFromReplica(final WALFile.WALTransaction buffer,
-      final Map<String, TreeMap<TransactionIndexContext.ComparableKey, Set<TransactionIndexContext.IndexKey>>> keysTx) throws TransactionException {
+      final Map<String, TreeMap<TransactionIndexContext.ComparableKey, Map<TransactionIndexContext.IndexKey, TransactionIndexContext.IndexKey>>> keysTx)
+      throws TransactionException {
 
     final int totalImpactedPages = buffer.pages.length;
     if (totalImpactedPages == 0 && keysTx.isEmpty()) {
@@ -568,8 +569,8 @@ public void commit2ndPhase(final TransactionContext.TransactionPhase1 changes) {
     }
   }
 
-  public void addIndexOperation(final Index index, final boolean addOperation, final Object[] keys, final RID rid) {
-    indexChanges.addIndexKeyLock(index.getName(), addOperation, keys, rid);
+  public void addIndexOperation(final IndexInternal index, final boolean addOperation, final Object[] keys, final RID rid) {
+    indexChanges.addIndexKeyLock(index, addOperation, keys, rid);
   }
 
   @Override

engine/src/main/java/com/arcadedb/database/TransactionIndexContext.java

@@ -32,8 +32,8 @@
 import java.util.logging.*;
 
 public class TransactionIndexContext {
-  private final DatabaseInternal                                   database;
-  private       Map<String, TreeMap<ComparableKey, Set<IndexKey>>> indexEntries = new LinkedHashMap<>(); // MOST COMMON USE CASE INSERTION IS ORDERED, USE AN ORDERED MAP TO OPTIMIZE THE INDEX
+  private final DatabaseInternal                                             database;
+  private       Map<String, TreeMap<ComparableKey, Map<IndexKey, IndexKey>>> indexEntries = new LinkedHashMap<>(); // MOST COMMON USE CASE INSERTION IS ORDERED, USE AN ORDERED MAP TO OPTIMIZE THE INDEX
 
   public static class IndexKey {
     public final boolean  addOperation;
@@ -141,14 +141,14 @@ public void removeIndex(final String indexName) {
 
   public int getTotalEntries() {
     int total = 0;
-    for (Map<ComparableKey, Set<IndexKey>> entry : indexEntries.values()) {
+    for (Map<ComparableKey, Map<IndexKey, IndexKey>> entry : indexEntries.values()) {
       total += entry.values().size();
     }
     return total;
   }
 
   public int getTotalEntriesByIndex(final String indexName) {
-    final Map<ComparableKey, Set<IndexKey>> entries = indexEntries.get(indexName);
+    final Map<ComparableKey, Map<IndexKey, IndexKey>> entries = indexEntries.get(indexName);
     if (entries == null)
       return 0;
     return entries.size();
@@ -157,12 +157,12 @@ public int getTotalEntriesByIndex(final String indexName) {
   public void commit() {
     checkUniqueIndexKeys();
 
-    for (Map.Entry<String, TreeMap<ComparableKey, Set<IndexKey>>> entry : indexEntries.entrySet()) {
+    for (Map.Entry<String, TreeMap<ComparableKey, Map<IndexKey, IndexKey>>> entry : indexEntries.entrySet()) {
       final Index index = database.getSchema().getIndexByName(entry.getKey());
-      final Map<ComparableKey, Set<IndexKey>> keys = entry.getValue();
+      final Map<ComparableKey, Map<IndexKey, IndexKey>> keys = entry.getValue();
 
-      for (Map.Entry<ComparableKey, Set<IndexKey>> keyValueEntries : keys.entrySet()) {
-        final Set<IndexKey> values = keyValueEntries.getValue();
+      for (Map.Entry<ComparableKey, Map<IndexKey, IndexKey>> keyValueEntries : keys.entrySet()) {
+        final Collection<IndexKey> values = keyValueEntries.getValue().values();
 
         if (values.size() > 1) {
           // BATCH MODE. USE SET TO SKIP DUPLICATES
@@ -225,50 +225,81 @@ public void addFilesToLock(final Set<Integer> modifiedFiles) {
     }
   }
 
-  public Map<String, TreeMap<ComparableKey, Set<IndexKey>>> toMap() {
+  public Map<String, TreeMap<ComparableKey, Map<IndexKey, IndexKey>>> toMap() {
     return indexEntries;
   }
 
-  public void setKeys(final Map<String, TreeMap<ComparableKey, Set<IndexKey>>> keysTx) {
+  public void setKeys(final Map<String, TreeMap<ComparableKey, Map<IndexKey, IndexKey>>> keysTx) {
     indexEntries = keysTx;
   }
 
   public boolean isEmpty() {
     return indexEntries.isEmpty();
   }
 
-  public void addIndexKeyLock(final String indexName, final boolean addOperation, final Object[] keysValues, final RID rid) {
-    TreeMap<ComparableKey, Set<IndexKey>> keys = indexEntries.get(indexName);
+  public void addIndexKeyLock(final IndexInternal index, final boolean addOperation, final Object[] keysValues, final RID rid) {
+    final String indexName = index.getName();
+
+    TreeMap<ComparableKey, Map<IndexKey, IndexKey>> keys = indexEntries.get(indexName);
 
     final ComparableKey k = new ComparableKey(keysValues);
     final IndexKey v = new IndexKey(addOperation, keysValues, rid);
 
-    Set<IndexKey> values;
+    Map<IndexKey, IndexKey> values;
     if (keys == null) {
       keys = new TreeMap<>(); // ORDERED TO KEEP INSERTION ORDER
       indexEntries.put(indexName, keys);
 
-      values = new HashSet<>();
+      values = new HashMap<>();
       keys.put(k, values);
     } else {
       values = keys.get(k);
       if (values == null) {
-        values = new HashSet<>();
+        values = new HashMap<>();
         keys.put(k, values);
       } else {
-        // CHECK FOR REMOVING ENTRIES WITH THE SAME KEY IN TX CONTEXT
+        if (addOperation && index.isUnique()) {
+          // CHECK IMMEDIATELY (INSTEAD OF AT COMMIT TIME) FOR DUPLICATED KEY IN CASE 2 ENTRIES WITH THE SAME KEY ARE SAVED IN TX.
+          final IndexKey entry = values.get(v);
+          if (entry != null && entry.addOperation && !entry.rid.equals(rid))
+            throw new DuplicatedKeyException(indexName, Arrays.toString(keysValues), null);
+        }
+
+        // REPLACE EXISTENT WITH THIS
         values.remove(v);
       }
     }
 
-    values.add(v);
+    if (addOperation && index.isUnique()) {
+      // CHECK FOR UNIQUE ON OTHER SUB-INDEXES
+      final TypeIndex typeIndex = index.getTypeIndex();
+      if (typeIndex != null) {
+        for (IndexInternal idx : typeIndex.getIndexesOnBuckets()) {
+          if (index.equals(idx))
+            // ALREADY CHECKED ABOVE
+            continue;
+
+          final TreeMap<ComparableKey, Map<IndexKey, IndexKey>> entries = indexEntries.get(idx.getName());
+          if (entries != null) {
+            final Map<IndexKey, IndexKey> otherIndexValues = entries.get(k);
+            if (otherIndexValues != null)
+              for (IndexKey e : otherIndexValues.values()) {
+                if (e.addOperation)
+                  throw new DuplicatedKeyException(indexName, Arrays.toString(keysValues), null);
+              }
+          }
+        }
+      }
+    }
+
+    values.put(v, v);
   }
 
   public void reset() {
     indexEntries.clear();
   }
 
-  public TreeMap<ComparableKey, Set<IndexKey>> getIndexKeys(final String indexName) {
+  public TreeMap<ComparableKey, Map<IndexKey, IndexKey>> getIndexKeys(final String indexName) {
     return indexEntries.get(indexName);
   }
 
@@ -308,13 +339,16 @@ private void checkUniqueIndexKeys(final Index index, final IndexKey key) {
   }
 
   private void checkUniqueIndexKeys() {
-    for (Map.Entry<String, TreeMap<ComparableKey, Set<IndexKey>>> indexEntry : indexEntries.entrySet()) {
-      final Index index = database.getSchema().getIndexByName(indexEntry.getKey());
+    for (Map.Entry<String, TreeMap<ComparableKey, Map<IndexKey, IndexKey>>> indexEntries : indexEntries.entrySet()) {
+      final Index index = database.getSchema().getIndexByName(indexEntries.getKey());
       if (index.isUnique()) {
-        final Map<ComparableKey, Set<IndexKey>> entries = indexEntry.getValue();
-        for (Set<IndexKey> keys : entries.values())
-          for (IndexKey entry : keys)
+        final Map<ComparableKey, Map<IndexKey, IndexKey>> txEntriesPerIndex = indexEntries.getValue();
+        for (Map.Entry<ComparableKey, Map<IndexKey, IndexKey>> txEntriesPerKey : txEntriesPerIndex.entrySet()) {
+          final Map<IndexKey, IndexKey> valuesPerKey = txEntriesPerKey.getValue();
+
+          for (IndexKey entry : valuesPerKey.values())
             checkUniqueIndexKeys(index, entry);
+        }
       }
     }
   }

engine/src/main/java/com/arcadedb/engine/Bucket.java

@@ -20,6 +20,7 @@
 import com.arcadedb.database.RID;
 import com.arcadedb.database.Record;
 import com.arcadedb.database.RecordInternal;
+import com.arcadedb.exception.ArcadeDBException;
 import com.arcadedb.exception.DatabaseOperationException;
 import com.arcadedb.exception.RecordNotFoundException;
 import com.arcadedb.log.LogManager;
@@ -172,6 +173,8 @@ public void scan(final RawRecordCallback callback) {
                 if (view != null && !callback.onRecord(rid, view))
                   return;
               }
+            } catch (ArcadeDBException e) {
+              throw e;
             } catch (Exception e) {
               final String msg = String.format("Error on loading record #%d:%d (error: %s)", file.getFileId(), (pageId * maxRecordsInPage) + recordIdInPage,
                   e.getMessage());

engine/src/main/java/com/arcadedb/engine/FileManager.java

@@ -43,6 +43,21 @@ public FileChange(final boolean create, final int fileId, final String fileName)
       this.fileId = fileId;
       this.fileName = fileName;
     }
+
+    @Override
+    public boolean equals(final Object o) {
+      if (this == o)
+        return true;
+      if (!(o instanceof FileChange))
+        return false;
+      final FileChange that = (FileChange) o;
+      return fileId == that.fileId;
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(fileId);
+    }
   }
 
   public static class FileManagerStats {
@@ -111,8 +126,14 @@ public void dropFile(final int fileId) throws IOException {
       files.set(fileId, null);
       file.drop();
 
-      if (recordedChanges != null)
-        recordedChanges.add(new FileChange(false, fileId, file.getFileName()));
+      final FileChange entry = new FileChange(false, fileId, file.getFileName());
+      if (recordedChanges != null) {
+        if (recordedChanges.remove(entry))
+          // JUST ADDED: REMOVE THE ENTRY
+          return;
+
+        recordedChanges.add(entry);
+      }
     }
   }
 

engine/src/main/java/com/arcadedb/index/IndexException.java

@@ -17,14 +17,12 @@
 
 import com.arcadedb.exception.ArcadeDBException;
 
-import java.io.IOException;
-
 public class IndexException extends ArcadeDBException {
   public IndexException(final String s) {
     super(s);
   }
 
-  public IndexException(String s, IOException e) {
+  public IndexException(final String s, final Throwable e) {
     super(s, e);
   }
 }

engine/src/main/java/com/arcadedb/index/IndexInternal.java

@@ -44,4 +44,8 @@ public interface IndexInternal extends Index {
   byte[] getBinaryKeyTypes();
 
   List<Integer> getFileIds();
+
+  void setTypeIndex(TypeIndex typeIndex);
+
+  TypeIndex getTypeIndex();
 }

engine/src/main/java/com/arcadedb/index/TypeIndex.java

@@ -187,11 +187,7 @@ public EmbeddedSchema.INDEX_TYPE getType() {
 
   @Override
   public String getTypeName() {
-    checkIsValid();
-    if (indexesOnBuckets.isEmpty())
-      return null;
-
-    return indexesOnBuckets.get(0).getTypeName();
+    return type.getName();
   }
 
   @Override
@@ -212,7 +208,6 @@ public void drop() {
     checkIsValid();
 
     final DocumentType t = type.getSchema().getType(getTypeName());
-    t.removeIndexInternal(this);
 
     for (Index index : new ArrayList<>(indexesOnBuckets))
       type.getSchema().dropIndex(index.getName());
@@ -367,6 +362,16 @@ public List<Integer> getFileIds() {
     return ids;
   }
 
+  @Override
+  public void setTypeIndex(final TypeIndex typeIndex) {
+    throw new UnsupportedOperationException("setTypeIndex");
+  }
+
+  @Override
+  public TypeIndex getTypeIndex() {
+    return null;
+  }
+
   @Override
   public int getAssociatedBucketId() {
     return -1;
@@ -378,6 +383,7 @@ public void addIndexOnBucket(final IndexInternal index) {
       throw new IllegalArgumentException("Invalid subIndex " + index);
 
     indexesOnBuckets.add(index);
+    index.setTypeIndex(this);
   }
 
   public void removeIndexOnBucket(final IndexInternal index) {
@@ -386,10 +392,10 @@ public void removeIndexOnBucket(final IndexInternal index) {
       throw new IllegalArgumentException("Invalid subIndex " + index);
 
     indexesOnBuckets.remove(index);
+    index.setTypeIndex(null);
   }
 
   public IndexInternal[] getIndexesOnBuckets() {
-    checkIsValid();
     return indexesOnBuckets.toArray(new IndexInternal[indexesOnBuckets.size()]);
   }
 

engine/src/main/java/com/arcadedb/index/lsm/LSMTreeFullTextIndex.java

@@ -27,6 +27,7 @@
 import com.arcadedb.index.IndexException;
 import com.arcadedb.index.IndexInternal;
 import com.arcadedb.index.TempIndexCursor;
+import com.arcadedb.index.TypeIndex;
 import com.arcadedb.schema.EmbeddedSchema;
 import com.arcadedb.schema.Schema;
 import com.arcadedb.schema.Type;
@@ -61,6 +62,7 @@
 public class LSMTreeFullTextIndex implements Index, IndexInternal {
   private final LSMTreeIndex underlyingIndex;
   private final Analyzer     analyzer;
+  private       TypeIndex    typeIndex;
 
   public static class IndexFactoryHandler implements com.arcadedb.index.IndexFactoryHandler {
     @Override
@@ -282,6 +284,16 @@ public List<Integer> getFileIds() {
     return underlyingIndex.getFileIds();
   }
 
+  @Override
+  public void setTypeIndex(final TypeIndex typeIndex) {
+    this.typeIndex = typeIndex;
+  }
+
+  @Override
+  public TypeIndex getTypeIndex() {
+    return typeIndex;
+  }
+
   @Override
   public long build(BuildIndexCallback callback) {
     return underlyingIndex.build(callback);