Merge branch 'develop' of github.com:intelowlproject/IntelOwl into do…

…cument_urls_extraction

api_app/admin.py

@@ -5,6 +5,7 @@
 
 from django.contrib import admin, messages
 from django.contrib.admin import widgets
+from django.contrib.admin.models import LogEntry
 from django.db.models import JSONField, ManyToManyField
 from django.http import HttpRequest
 from prettyjson.widgets import PrettyJSONWidget
@@ -254,3 +255,27 @@ class OrganizationPluginConfigurationAdminView(CustomAdminView):
     exclude = ["content_type", "object_id"]
     list_filter = ["organization", "content_type"]
     form = OrganizationPluginConfigurationForm
+
+
+@admin.register(LogEntry)
+class LogEntryAdmin(admin.ModelAdmin):
+    ordering = ["-action_time"]
+    list_display = [
+        "pk",
+        "user",
+        "object_repr",
+        "action_flag",
+        "change_message",
+        "action_time",
+    ]
+    list_filter = ["user", "action_flag", "action_time", "content_type"]
+    search_fields = ["user__username", "object_repr", "change_message"]
+
+    def has_delete_permission(self, request, obj=None):
+        return False
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False

api_app/signals.py

@@ -5,6 +5,7 @@
 
 from django import dispatch
 from django.conf import settings
+from django.contrib.admin.models import LogEntry
 from django.db import models
 from django.dispatch import receiver
 
@@ -272,7 +273,7 @@ def post_save_python_config_cache(sender, instance, *args, **kwargs):
 
 
 @receiver(models.signals.post_delete)
-def post_delete_python_config_cache(sender, instance, using, origin, *args, **kwargs):
+def post_delete_python_config_cache(sender, instance, *args, **kwargs):
     """
     Signal receiver for the post_delete signal.
     Deletes class cache keys for instances of ListCachable models after deletion.
@@ -287,3 +288,18 @@ def post_delete_python_config_cache(sender, instance, using, origin, *args, **kw
     """
     if issubclass(sender, ListCachable):
         instance.delete_class_cache_keys()
+
+
+@receiver(models.signals.post_save, sender=LogEntry)
+def post_save_log_entry(sender, instance: LogEntry, using, origin, *args, **kwargs):
+    """
+    Signal receiver for the post_save signal.
+    Add a line of log
+
+    Args:
+        sender (Model): The model class sending the signal.
+        instance: The instance of the model being saved.
+        *args: Additional positional arguments.
+        **kwargs: Additional keyword arguments.
+    """
+    logger.info(str(instance))

docker/entrypoints/celery_default.sh

@@ -14,7 +14,16 @@ else
    worker_number=8
 fi
 
-ARGUMENTS="-A intel_owl.celery worker -n worker_default --uid www-data --gid www-data --time-limit=10000 --pidfile= -c $worker_number -Ofair -Q default,broadcast,config -E --without-gossip"
+
+if [ "$AWS_SQS" = "True" ]
+then
+  queues="default.fifo,config.fifo"
+else
+  queues="default,broadcast,config"
+fi
+
+
+ARGUMENTS="-A intel_owl.celery worker -n worker_default --uid www-data --gid www-data --time-limit=10000 --pidfile= -c $worker_number -Ofair -Q ${queues} -E --without-gossip"
 if [[ $DEBUG == "True" ]] && [[ $DJANGO_TEST_SERVER == "True" ]];
 then
     echo "Running celery with autoreload"

docker/entrypoints/celery_ingestor.sh

@@ -4,7 +4,15 @@ until cd /opt/deploy/intel_owl
 do
     echo "Waiting for server volume..."
 done
-ARGUMENTS="-A intel_owl.celery worker -n worker_ingestor --uid www-data --gid www-data --time-limit=40000 --pidfile= -Ofair -Q ingestor,broadcast,config -E --autoscale=1,15 --without-gossip"
+
+if [ "$AWS_SQS" = "True" ]
+then
+  queues="ingestor.fifo,config.fifo"
+else
+  queues="ingestor,broadcast,config"
+fi
+
+ARGUMENTS="-A intel_owl.celery worker -n worker_ingestor --uid www-data --gid www-data --time-limit=40000 --pidfile= -Ofair -Q ${queues} -E --autoscale=1,15 --without-gossip"
 if [[ $DEBUG == "True" ]] && [[ $DJANGO_TEST_SERVER == "True" ]];
 then
     echo "Running celery with autoreload"

docker/entrypoints/celery_local.sh

@@ -4,8 +4,14 @@ until cd /opt/deploy/intel_owl
 do
     echo "Waiting for server volume..."
 done
+if [ "$AWS_SQS" = "True" ]
+then
+  queues="local.fifo,config.fifo"
+else
+  queues="local,broadcast,config"
+fi
 
-ARGUMENTS="-A intel_owl.celery worker -n worker_local --uid www-data --time-limit=10000 --gid www-data --pidfile= -Ofair -Q local,broadcast,config -E --without-gossip"
+ARGUMENTS="-A intel_owl.celery worker -n worker_local --uid www-data --time-limit=10000 --gid www-data --pidfile= -Ofair -Q ${queues} -E --without-gossip"
 if [[ $DEBUG == "True" ]] && [[ $DJANGO_TEST_SERVER == "True" ]];
 then
     echo "Running celery with autoreload"

docker/entrypoints/celery_long.sh

@@ -4,7 +4,14 @@ until cd /opt/deploy/intel_owl
 do
     echo "Waiting for server volume..."
 done
-ARGUMENTS="-A intel_owl.celery worker -n worker_long --uid www-data --gid www-data --time-limit=40000 --pidfile= -Ofair -Q long,broadcast,config -E --without-gossip"
+if [ "$AWS_SQS" = "True" ]
+then
+  queues="long.fifo,config.fifo"
+else
+  queues="long,broadcast,config"
+fi
+
+ARGUMENTS="-A intel_owl.celery worker -n worker_long --uid www-data --gid www-data --time-limit=40000 --pidfile= -Ofair -Q ${queues} -E --without-gossip"
 if [[ $DEBUG == "True" ]] && [[ $DJANGO_TEST_SERVER == "True" ]];
 then
     echo "Running celery with autoreload"

frontend/src/utils/observables.js

@@ -28,10 +28,10 @@ export function sanitizeObservable(observable) {
       .replaceAll("[", "")
       .replaceAll("]", "")
       .trim()
-      .replace(/\W*$/, "")
+      .replace(/(?!\\(|\\))\W*$/, "")
       /* ignore + at the start of the string to support phone number:
     this could lead to a match problem in the loading observable feature */
-      .replace(/^(?!\+)\W/, "")
+      .replace(/^(?!\+|\\(|\\))\W/, "")
   );
 }
 

frontend/tests/utils/observables.test.js

@@ -61,14 +61,15 @@ describe("Observable validators tests", () => {
     "test.com ",
     "test.com;",
     " test.com ",
+    "(test.com)",
   ])("test valid domains (%s)", (valueToValidate) => {
     expect(observableValidators(valueToValidate)).toStrictEqual({
       classification: "domain",
       observable: "test.com",
     });
   });
 
-  test.each(["test.exe", "test.pdf,", ",test.js,", "256.256.256.256"])(
+  test.each(["test.exe", "test.pdf,", ",test.js,", "256.256.256.256", "google(.)com"])(
     "test invalid domains (%s)",
     (valueToValidate) => {
       expect(observableValidators(valueToValidate)).toBeNull();