2289 enforce data classification tagging (#8)

* Conditional data validation WIP

* initial data filtering by classification marking status

* Data stewardship role sourced from keycloak

* Added additional plumbing to make ACCM properties configurable. Cleanup and comments

* cleanup

* updated data steward role keyword

* Fixed data steward role not being passed to perm checkers correctly

* remove obe debug

engine/pom.xml

@@ -144,6 +144,19 @@
             <version>${graalvm.version}</version>
             <scope>runtime</scope>
         </dependency>
+
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>1.18.30</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>2.0.9</version>
+        </dependency>
+
     </dependencies>
 
 </project>

engine/src/main/java/com/arcadedb/database/DocumentValidator.java

@@ -31,6 +31,15 @@
  * @author Luca Garulli ([email protected])
  */
 public class DocumentValidator {
+
+  // TODO move ACCM validation here?
+
+  public static void validateSpecificProperties(final MutableDocument document, final List<Property> properties) throws ValidationException {
+    document.checkForLazyLoadingProperties();
+    for (Property entry : properties)
+      validateField(document, entry);
+  }
+
   public static void validate(final MutableDocument document) throws ValidationException {
     document.checkForLazyLoadingProperties();
     for (Property entry : document.getType().getProperties())

engine/src/main/java/com/arcadedb/database/EmbeddedDatabase.java

@@ -784,7 +784,7 @@ public void createRecordNoLock(final Record record, final String bucketName, fin
     setDefaultValues(record);
 
     if (record instanceof MutableDocument)
-      ((MutableDocument) record).validate();
+      ((MutableDocument) record).validateAndAccmCheck();
 
     // INVOKE EVENT CALLBACKS
     if (!events.onBeforeCreate(record))
@@ -840,7 +840,7 @@ public void updateRecord(final Record record) {
       throw new DatabaseIsReadOnlyException("Cannot update a record");
 
     if (record instanceof MutableDocument)
-      ((MutableDocument) record).validate();
+      ((MutableDocument) record).validateAndAccmCheck();
 
     // INVOKE EVENT CALLBACKS
     if (!events.onBeforeUpdate(record))

engine/src/main/java/com/arcadedb/engine/BucketIterator.java

@@ -21,6 +21,8 @@
 import com.arcadedb.database.Binary;
 import com.arcadedb.database.Database;
 import com.arcadedb.database.DatabaseInternal;
+import com.arcadedb.database.Document;
+import com.arcadedb.database.MutableDocument;
 import com.arcadedb.database.RID;
 import com.arcadedb.database.Record;
 import com.arcadedb.exception.DatabaseOperationException;
@@ -96,6 +98,10 @@ private void fetchNext() {
               if (!bucket.existsRecord(rid))
                 continue;
 
+              if (!checkPermissionsOnDocument(rid.asDocument(true))) {
+                continue;
+              }
+
               next = rid.getRecord(false);
               return null;
 
@@ -109,8 +115,14 @@ private void fetchNext() {
               if (view == null)
                 continue;
 
-              next = database.getRecordFactory()
+              var record = database.getRecordFactory()
                   .newImmutableRecord(database, database.getSchema().getType(database.getSchema().getTypeNameByBucketId(rid.getBucketId())), rid, view, null);
+
+              if (!checkPermissionsOnDocument(record.asDocument(true))) {
+                continue;
+              }
+
+              next = record;
               return null;
             }
           } catch (final Exception e) {
@@ -132,6 +144,19 @@ private void fetchNext() {
     });
   }
 
+  private boolean checkPermissionsOnDocument(final Document document) {
+    var currentUser = database.getContext().getCurrentUser();
+
+    if (currentUser.isServiceAccount() || currentUser.isDataSteward(document.getTypeName())) {
+      return true;
+    }
+
+    if ((!document.has(MutableDocument.CLASSIFICATION_MARKED) || !document.getBoolean(MutableDocument.CLASSIFICATION_MARKED))) {
+      return false;
+    }
+    return true;
+  }
+
   @Override
   public boolean hasNext() {
     if (limit > -1 && browsed >= limit)

engine/src/main/java/com/arcadedb/exception/ValidationException.java

@@ -24,7 +24,8 @@
  * @author Luca Garulli ([email protected])
  */
 public class ValidationException extends ArcadeDBException {
-  public ValidationException(final String message) {
+
+public ValidationException(final String message) {
     super(message);
   }
 

engine/src/main/java/com/arcadedb/schema/Property.java

@@ -27,12 +27,18 @@
 import com.arcadedb.query.sql.parser.SqlParser;
 import com.arcadedb.serializer.json.JSONObject;
 
+import lombok.Setter;
+import lombok.ToString;
+
 import java.io.*;
 import java.util.*;
 
+@ToString
 public class Property {
   private final        DocumentType        owner;
-  private final        String              name;
+
+  @Setter
+  private              String              name;
   private final        Type                type;
   private final        int                 id;
   protected final      Map<String, Object> custom          = new HashMap<>();
@@ -76,6 +82,10 @@ public Index getOrCreateIndex(final EmbeddedSchema.INDEX_TYPE type, final boolea
     return owner.getSchema().buildTypeIndex(owner.getName(), new String[] { name }).withType(type).withUnique(unique).withIgnoreIfExists(true).create();
   }
 
+  public DocumentType getOwner() {
+    return owner;
+  }
+
   public String getName() {
     return name;
   }

engine/src/main/java/com/arcadedb/security/ACCM/AccmProperty.java

@@ -0,0 +1,29 @@
+package com.arcadedb.security.ACCM;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.arcadedb.schema.Type;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.experimental.Accessors;
+
+@Data
+@NoArgsConstructor
+@Accessors(chain = true, fluent = true)
+public class AccmProperty {
+
+    // Split name by periods into property path
+    private String name;
+    private String parentType;
+    private Type dataType;
+    private boolean required;
+    private boolean notNull;
+    private boolean readOnly;
+    private List<String> options = new ArrayList<>();
+    private String validationRegex;
+
+   // private String keycloakUserInfoPropertyName;
+   // might also be coming from roles
+}

engine/src/main/java/com/arcadedb/security/SecurityDatabaseUser.java

@@ -64,6 +64,10 @@ public static DATABASE_ACCESS getByName(final String name) {
 
   boolean requestAccessOnFile(int fileId, ACCESS access);
 
+  boolean isDataSteward(String type);
+
+  boolean isServiceAccount();
+
   String getName();
 
   long getResultSetLimit();

server/src/main/java/com/arcadedb/server/security/ServerSecurity.java

@@ -374,8 +374,7 @@ private ServerSecurityUser getOrCreateUser(final String username) {
     userJson.put("databases", databases);
 
     log.debug("getOrCreateUser userJson {}", userJson.toString());
-
-    final ServerSecurityUser serverSecurityUser = new ServerSecurityUser(server, userJson);
+    ServerSecurityUser serverSecurityUser = new ServerSecurityUser(server, userJson, arcadeRoles);
     users.put(username, serverSecurityUser);
 
     return serverSecurityUser;

server/src/main/java/com/arcadedb/server/security/ServerSecurityDatabaseUser.java

@@ -25,6 +25,8 @@
 import com.arcadedb.security.SecurityManager;
 import com.arcadedb.serializer.json.JSONArray;
 import com.arcadedb.serializer.json.JSONObject;
+import com.arcadedb.server.security.oidc.ArcadeRole;
+import com.arcadedb.server.security.oidc.role.RoleType;
 
 import lombok.extern.slf4j.Slf4j;
 
@@ -41,11 +43,13 @@ public class ServerSecurityDatabaseUser implements SecurityDatabaseUser {
   private long resultSetLimit = -1;
   private long readTimeout = -1;
   private final boolean[] databaseAccessMap = new boolean[DATABASE_ACCESS.values().length];
+  private List<ArcadeRole> arcadeRoles = new ArrayList<>();
 
-  public ServerSecurityDatabaseUser(final String databaseName, final String userName, final String[] groups) {
+  public ServerSecurityDatabaseUser(final String databaseName, final String userName, final String[] groups, final List<ArcadeRole> arcadeRoles) {
     this.databaseName = databaseName;
     this.userName = userName;
     this.groups = groups;
+    this.arcadeRoles = arcadeRoles;
   }
 
   public String[] getGroups() {
@@ -299,4 +303,24 @@ public static boolean[] updateAccessArray(final boolean[] array, final JSONArray
   //  log.debug("updateAccessArray: accessObj: {}; array: {}", access, array);
     return array;
   }
+
+  /**
+   * Checks if the user is a data steward role type for type of object requested.
+   */
+  @Override
+  public boolean isDataSteward(String type) {
+    for (ArcadeRole role : arcadeRoles) {
+      if (role.getRoleType().equals(RoleType.DATA_STEWARD) && role.isTypeMatch(type)) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  @Override
+  public boolean isServiceAccount() {
+    // Keycloak client service account naming convention follows this. Update as needed.
+    return this.userName.startsWith("service-account-");
+  }
 }

server/src/main/java/com/arcadedb/server/security/ServerSecurityUser.java

@@ -20,10 +20,10 @@
 
 import com.arcadedb.database.Database;
 import com.arcadedb.database.DatabaseInternal;
-import com.arcadedb.log.LogManager;
 import com.arcadedb.security.SecurityManager;
 import com.arcadedb.security.SecurityUser;
 import com.arcadedb.server.ArcadeDBServer;
+import com.arcadedb.server.security.oidc.ArcadeRole;
 
 import lombok.extern.slf4j.Slf4j;
 
@@ -33,6 +33,7 @@
 import java.util.*;
 import java.util.concurrent.*;
 import java.util.logging.Level;
+import java.util.stream.Collectors;
 
 @Slf4j
 public class ServerSecurityUser implements SecurityUser {
@@ -42,8 +43,13 @@ public class ServerSecurityUser implements SecurityUser {
   private       Set<String>                                           databasesNames;
   private       String                                                password;
   private final ConcurrentHashMap<String, ServerSecurityDatabaseUser> databaseCache = new ConcurrentHashMap();
+  private List<ArcadeRole> arcadeRoles = new ArrayList<>();
 
   public ServerSecurityUser(final ArcadeDBServer server, final JSONObject userConfiguration) {
+    this(server, userConfiguration, new ArrayList<>());
+  }
+
+  public ServerSecurityUser(final ArcadeDBServer server, final JSONObject userConfiguration, List<ArcadeRole> arcadeRoles) {
     this.server = server;
     this.userConfiguration = userConfiguration;
 
@@ -57,6 +63,8 @@ public ServerSecurityUser(final ArcadeDBServer server, final JSONObject userConf
     } else {
       databasesNames = Collections.emptySet();
     }
+
+    this.arcadeRoles = arcadeRoles;
   }
 
   @Override
@@ -98,7 +106,7 @@ else if (userDatabases.has(SecurityManager.ANY))
 
     if (dbu == null)
       // USER HAS NO ACCESS TO THE DATABASE, RETURN A USER WITH NO AX
-      dbu = new ServerSecurityDatabaseUser(databaseName, name, new String[0]);
+      dbu = new ServerSecurityDatabaseUser(databaseName, name, new String[0], getRelevantRoles(arcadeRoles, databaseName));
 
     final ServerSecurityDatabaseUser prev = databaseCache.putIfAbsent(databaseName, dbu);
     if (prev != null)
@@ -108,6 +116,18 @@ else if (userDatabases.has(SecurityManager.ANY))
     return dbu;
   }
 
+  /**
+   * Return roles that are applicable to the user conducting operations against the database.
+   * @param arcadeRoles
+   * @param databaseName
+   * @return
+   */
+  private List<ArcadeRole> getRelevantRoles(List<ArcadeRole> arcadeRoles, String databaseName) {
+    return arcadeRoles.stream()
+                .filter(role -> role.isDatabaseMatch(databaseName))
+                .collect(Collectors.toList());
+  }
+
   public JSONObject toJSON() {
     return userConfiguration;
   }
@@ -158,7 +178,9 @@ private ServerSecurityDatabaseUser registerDatabaseUser(final ArcadeDBServer ser
     final JSONObject userDatabases = userConfiguration.getJSONObject("databases");
     final List<Object> groupList = userDatabases.getJSONArray(databaseName).toList();
     log.debug("XX registerDatabaseUser: name: {}; database: {}; groupList: {}", name, databaseName, groupList.toString());
-    ServerSecurityDatabaseUser dbu = new ServerSecurityDatabaseUser(databaseName, name, groupList.toArray(new String[groupList.size()]));
+
+    ServerSecurityDatabaseUser dbu = new ServerSecurityDatabaseUser(databaseName, name, groupList.toArray(new String[groupList.size()]), 
+            getRelevantRoles(arcadeRoles, databaseName));
 
     final ServerSecurityDatabaseUser prev = databaseCache.putIfAbsent(databaseName, dbu);
     if (prev != null)