Df 269 add df classification validation

engine/pom.xml

@@ -163,6 +163,5 @@
             <artifactId>slf4j-api</artifactId>
             <version>2.0.9</version>
         </dependency>
-
     </dependencies>
 </project>

engine/src/main/java/com/arcadedb/GlobalConfiguration.java

@@ -179,6 +179,7 @@ public Object call(final Object value) {
 
   OIDC_AUTH("arcadedb.oidcAuth", SCOPE.SERVER, "Use OIDC Auth instead of basic", Boolean.class, true),
   KEYCLOAK_ROOT_URL("arcadedb.keycloakRootUrl", SCOPE.SERVER, "Keycloak root URL", String.class, "http://df-keycloak:8080"),
+  DF_CLASSIFICATION_URL("arcadedb.classificationRootUrl", SCOPE.SERVER, "DF-classification root URL", String.class, "http://localhost:8000"),
   KEYCLOAK_ADMIN_USERNAME("arcadedb.keycloakAdminUsername", SCOPE.SERVER, "Keycloak admin username", String.class, "admin"),
   KEYCLOAK_ADMIN_PASSWORD("arcadedb.keycloakAdminPassword", SCOPE.SERVER, "Keycloak admin password", String.class, ""),
   KEYCLOAK_CLIENT_ID("arcadedb.keycloakClientId", SCOPE.SERVER, "Keycloak client ID", String.class, "df-backend"),

engine/src/main/java/com/arcadedb/database/DocumentValidator.java

@@ -25,6 +25,7 @@
 import com.arcadedb.schema.Property;
 import com.arcadedb.schema.Type;
 import com.arcadedb.security.AuthorizationUtils;
+import com.arcadedb.security.DataFabricClassificationClient;
 import com.arcadedb.security.SecurityDatabaseUser;
 import com.arcadedb.serializer.json.JSONObject;
 
@@ -52,6 +53,7 @@ public static void verifyDocumentClassificationValidForDeployment(String toCheck
     if (!classificationOptions.containsKey(toCheck))
       throw new ValidationException("Classification must be one of " + classificationOptions);
 
+    // TODO: Do not default to S. App should not come up if no deployment classification is set.
     var deploymentClassification = System.getProperty("deploymentClassification", "S");
     if (classificationOptions.get(deploymentClassification) < classificationOptions.get(toCheck))
       throw new ValidationException("Classification " + toCheck + " is not allowed in this deployment");
@@ -61,78 +63,59 @@ public static void verifyDocumentClassificationValidForDeployment(String toCheck
   }
 
   public static void validateClassificationMarkings(final MutableDocument document, 
-          SecurityDatabaseUser securityDatabaseUser, RecordAction action) {
+          SecurityDatabaseUser securityDatabaseUser, RecordAction action) throws ValidationException {
 
     if (document == null) {
-      throw new ValidationException("Document is null");
+      throw new ValidationException("Document is null!");
     }
 
+    String message = "Validating classification markings on " + document.toJSON(true)
+            + ". CRUD op '" + action + "'...";
+    LogManager.instance().log(DocumentValidator.class, Level.INFO, message);
+
     if (document instanceof MutableEmbeddedDocument) {
+      LogManager.instance().log(DocumentValidator.class, Level.INFO,
+              "Document is a MutableEmbeddedDocument. Skipping validation.");
       return;
     }
 
     if (document.getRecordType() == EmbeddedDocument.RECORD_TYPE) {
+      LogManager.instance().log(DocumentValidator.class, Level.INFO,
+              "Document is a RECORD_TYPE. Skipping validation.");
       return;
     }
 
     // Skip validation checks if classification validation is disabled for the database
     if (!document.getDatabase().getSchema().getEmbedded().isClassificationValidationEnabled()) {
+      LogManager.instance().log(DocumentValidator.class, Level.INFO,
+              "Classification validation is disabled on database. Skipping validation.");
       return;
     }
 
-    boolean validSources = false;
-    // validate sources, if present
-    if (document.has(MutableDocument.SOURCES_ARRAY_ATTRIBUTE) && !document.toJSON().getJSONArray(MutableDocument.SOURCES_ARRAY_ATTRIBUTE).isEmpty()) {
-      validateSources(document, securityDatabaseUser, action);
-      validSources = true;
-    }
-
-    if (document.has(MutableDocument.CLASSIFICATION_PROPERTY) 
-        && document.toJSON().getJSONObject(MutableDocument.CLASSIFICATION_PROPERTY).has("components") && document.toJSON().getJSONObject(MutableDocument.CLASSIFICATION_PROPERTY).getJSONObject("components").has(MutableDocument.CLASSIFICATION_PROPERTY)) {
-
-      var classificationMarkings = document.toJSON().getJSONObject(MutableDocument.CLASSIFICATION_PROPERTY).getJSONObject("components")
-          .getString(MutableDocument.CLASSIFICATION_GENERAL_PROPERTY);
-
-      if (classificationMarkings.trim().isEmpty()) {
-        throw new ValidationException("Classification " + classificationMarkings + " is not valid");
-      }
-
-      // TODO handle SBU, LES, etc.
-      String classification = classificationMarkings;
-      if (classificationMarkings.contains("//")) {
-        classification = classificationMarkings.substring(0, classificationMarkings.indexOf("//"));
-      }
+    // TODO: Add sources validation back in.
 
-      // Validate the user can set the classification of the document. Can't create higher than what you can access.
-      if (!AuthorizationUtils.checkPermissionsOnDocument(document, securityDatabaseUser, action)) {
-        throw new ValidationException("User cannot set classification markings on documents higher than or outside their current access.");
-      }
-
-      try {
-        var databaseClassification = document.getDatabase().getSchema().getEmbedded().getClassification();
-        verifyDocumentClassificationValidForDeployment(classification, databaseClassification);
-      } catch (IllegalArgumentException e) {
-        throw new ValidationException("Invalid classification: " + classification);
-      }
-
-      var classificationObj = new JSONObject(document.get(MutableDocument.CLASSIFICATION_PROPERTY).toString());
-
-      var nonSystemPropsCount = document.getPropertyNames().stream()
-          .filter(prop -> !prop.startsWith("@"))
-          .filter(prop -> !MutableDocument.CUSTOM_SYSTEM_PROPERTIES.contains(prop))
-          .count();
+    if (!document.has(MutableDocument.CLASSIFICATION_PROPERTY)) {
+      throw new ValidationException("Document has no classification property!");
+    }
 
-      // TODO this check only matters if there are non system attributes on the object.
+    if (!document.toJSON().getJSONObject(MutableDocument.CLASSIFICATION_PROPERTY).has("components")) {
+      throw new ValidationException("Document has no classification.components property!");
+    }
 
-      // TODO reenable attribute classification checks.s
-      // if (nonSystemPropsCount > 0 && !classificationObj.has(MutableDocument.CLASSIFICATION_ATTRIBUTES_PROPERTY)) {
-      //   throw new ValidationException("Missing classification attributes on document");
-      // }
+    var component = document.toJSON().getJSONObject(MutableDocument.CLASSIFICATION_PROPERTY).getJSONObject("components").toString();
+    boolean valid = DataFabricClassificationClient.validateDocumentClassification(component);
+    if (!valid) {
+      throw new ValidationException("Document has no valid classification defined!");
+    }
+    // TODO handle SBU, LES, etc.
 
-      validateAttributeClassificationTagging(document, classificationObj.getJSONObject(MutableDocument.CLASSIFICATION_ATTRIBUTES_PROPERTY), securityDatabaseUser, action);
-    } else if (!validSources){
-      throw new ValidationException("Missing overall classification data on document");
+    // Validate the user can set the classification of the document. Can't create higher than what you can access.
+    if (!AuthorizationUtils.checkPermissionsOnDocument(document, securityDatabaseUser, action)) {
+      throw new ValidationException("User cannot set classification markings on documents higher than or outside their current access.");
     }
+
+    var classificationObj = new JSONObject(document.get(MutableDocument.CLASSIFICATION_PROPERTY).toString());
+    validateAttributeClassificationTagging(document, classificationObj.getJSONObject(MutableDocument.CLASSIFICATION_ATTRIBUTES_PROPERTY), securityDatabaseUser, action);
   }
 
   private static void validateAttributeClassificationTagging(final MutableDocument document, final JSONObject attributes, SecurityDatabaseUser securityDatabaseUser, RecordAction action) {
@@ -160,15 +143,29 @@ private static void validateAttributeClassificationTagging(final MutableDocument
         var attributeClassification = value.toString();
 
         if (attributeClassification != null && !attributeClassification.trim().isEmpty()) {
-            // This call might not be necessary after integrating with df-classification.
+            // TODO: This call might not be necessary after integrating with df-classification.
             verifyDocumentClassificationValidForDeployment(
                     attributeClassification,
                     document.getDatabase().getSchema().getEmbedded().getClassification());
 
             // Taking the attribute's classification value and fetching its numerical representation.
             var inputIndex = AuthorizationUtils.classificationOptions.get(attributeClassification);
             var userClearance = securityDatabaseUser.getClearanceForCountryOrTetragraphCode("USA");
-            var userClearanceIndex = AuthorizationUtils.classificationOptions.get(userClearance);
+
+            // TODO temp fix. refactor this to depend on the same deployment config specifying classification/clearance options.
+            int userClearanceIndex = 0;
+
+            switch(userClearance.toUpperCase()) {
+              case "UNCLASSIFIED": userClearanceIndex = 0;
+              case "CUI": userClearanceIndex = 1;
+              case "CONFIDENTIAL": userClearanceIndex = 2;
+              case "SECRET": userClearanceIndex = 3;
+              case "TOP SECRET": userClearanceIndex = 4;
+            }
+
+           // var userClearanceIndex = AuthorizationUtils.classificationOptions.get(userClearance);
+
+            LogManager.instance().log(DocumentValidator.class, Level.INFO, inputIndex + "_" + userClearanceIndex);
 
             if (inputIndex > userClearanceIndex) {
                 throw new ValidationException(
@@ -192,6 +189,8 @@ private static void validateAttributeClassificationTagging(final MutableDocument
    */
   private static void validateSources(final MutableDocument document, SecurityDatabaseUser securityDatabaseUser, RecordAction action) {
     var sources = document.toJSON().getJSONArray(MutableDocument.SOURCES_ARRAY_ATTRIBUTE);
+
+    LogManager.instance().log(DocumentValidator.class, Level.INFO, "Validating source classifications...");
     sources.forEach(obj -> {
 
       var jo = (JSONObject) obj;

engine/src/main/java/com/arcadedb/database/MutableDocument.java

@@ -24,7 +24,6 @@
 import com.arcadedb.schema.Property;
 import com.arcadedb.schema.Type;
 import com.arcadedb.security.SecurityDatabaseUser;
-import com.arcadedb.security.ACCM.AccmProperty;
 import com.arcadedb.serializer.json.JSONObject;
 
 import java.lang.reflect.*;
@@ -76,8 +75,6 @@ public synchronized boolean isDirty() {
   public synchronized void setBuffer(final Binary buffer) {
     super.setBuffer(buffer);
     dirty = false;
-    // map = null; // AVOID RESETTING HERE FOR INDEXES THAT CAN LOOKUP UP FOR FIELDS
-    // CAUSING AN UNMARSHALLING
   }
 
   @Override
@@ -137,39 +134,6 @@ public synchronized JSONObject toJSON(final boolean includeMetadata) {
     return result;
   }
 
-  /**
-   * Convert externally configurable required proeprty defintiions into arcade internal required properties.
-   * Not used at the moment, leaving short term for possible use.
-   * @return
-   */
-  public List<Property> getAccmProperties() {
-
-    List<AccmProperty> accmProperties = new ArrayList<>();
-    List<Property> properties = new ArrayList<>();
-
-    for (AccmProperty accmProperty : accmProperties) {
-      DocumentType parentType = database.getSchema().getType(accmProperty.parentType());
-
-      if (parentType == null) {
-        continue;
-      }
-
-      Property p = new Property(parentType, 
-          accmProperty.name(), accmProperty.dataType());
-      p.setMandatory(accmProperty.required());
-      p.setNotNull(accmProperty.notNull());
-
-      if (!accmProperty.options().isEmpty()) {
-        p.setRegexp(String.join("|", accmProperty.options()));
-      } else if (accmProperty.validationRegex() != null) {
-        p.setRegexp(accmProperty.validationRegex());
-      }
-      properties.add(p);
-    }
-
-    return properties;
-  }
-
   /**
    * Triggers the native required property valiation of arcade, as well as the one time ACCM validation.
    * ACCM validation follows a different recursive type checking pattern than arcade, so it is done separately.
@@ -227,29 +191,6 @@ public void validate() throws ValidationException {
     DocumentValidator.validate(this);
   }
 
-  private MutableEmbeddedDocument getEmbeddedDocumentToValidate(MutableDocument mutableDocument, final String name) {
-    // form of embeddedDoc.property
-    // could be embeddedDoc1.embeddedDoc2.property
-
-    if (name == null || name == "" || !name.contains(".")) {
-      return (MutableEmbeddedDocument) mutableDocument;
-    }
-
-    String embeddedDocName = name.substring(0, name.indexOf("."));
-    String remainingPropertyName = name.substring(name.indexOf(".") + 1);
-    final Object fieldValue = mutableDocument.get(embeddedDocName);
-
-    if (fieldValue != null && fieldValue instanceof MutableEmbeddedDocument) {
-      if (remainingPropertyName.split(".").length > 1) {
-        return getEmbeddedDocumentToValidate((MutableDocument) fieldValue, remainingPropertyName);
-      } else {
-        return (MutableEmbeddedDocument) fieldValue;
-      }
-    } else {
-       throw new ValidationException("Document classification incomplete for property: " + name);
-    }
-  }
-
   @Override
   public synchronized boolean has(final String propertyName) {
     checkForLazyLoadingProperties();

engine/src/main/java/com/arcadedb/security/ACCM/Argument.java

@@ -181,6 +181,7 @@ private boolean evaluateInternal(JSONObject json) {
                 // check if this.value is a list
                 if (this.value instanceof List) {
                     for (Object val : (List<Object>) this.value) {
+                        LogManager.instance().log(this, Level.INFO, "val type: " + val.getClass().getName());
                         if (val.equals(docFieldValue)) {
                             return true;
                         }