Merge pull request #197 from intelowlproject/develop

v1.7.1

.env

@@ -5,7 +5,7 @@
 ### the COMPOSE_FILE variable each separated with ':'. If you are on windows, replace all ':' with ';'.
 ### Reference to Docker's official Docs: https://docs.docker.com/compose/reference/envvars/#compose_file#compose_file
 
-INTELOWL_TAG_VERSION=v1.7.0
+INTELOWL_TAG_VERSION=v1.7.1
 
 ###### Default (Production) ######
 

README.md

@@ -74,7 +74,7 @@ license terms.
 [APKiD](https://github.com/rednaga/APKiD/blob/master/LICENSE.COMMERCIAL),
 [Box-JS](https://github.com/CapacitorSet/box-js/blob/master/LICENSE),
 [Capa](https://github.com/fireeye/capa/blob/master/LICENSE.txt),
-[Quark-Engine](https://github.com/quark-engine/quark-engine)
+[Quark-Engine](https://github.com/quark-engine/quark-engine),
 [IntelX](https://intelx.io/terms-of-service)
 
 ## Acknowledgments

api_app/admin.py

@@ -51,7 +51,10 @@ class CustomOutstandingTokenAdmin(OutstandingTokenAdmin):
         (
             "Create API Token For",
             {
-                "fields": ("user", "token",),
+                "fields": (
+                    "user",
+                    "token",
+                ),
                 "description": f"""
                     <h3>Token will be auto-generated on save.</h3>
                     <h5>Please note that this token,</h5>

api_app/models.py

@@ -30,7 +30,12 @@ def __str__(self):
 class Job(models.Model):
     class Meta:
         indexes = [
-            models.Index(fields=["md5", "status",]),
+            models.Index(
+                fields=[
+                    "md5",
+                    "status",
+                ]
+            ),
         ]
 
     source = models.CharField(max_length=50, blank=False, default="none")

api_app/script_analyzers/file_analyzers/vt3_scan.py

@@ -40,7 +40,12 @@ def run(self):
 
 
 def vt_scan_file(
-    api_key, md5, job_id, rescan_instead=False, max_tries=100, poll_distance=5,
+    api_key,
+    md5,
+    job_id,
+    rescan_instead=False,
+    max_tries=100,
+    poll_distance=5,
 ):
     try:
         binary = get_binary(job_id)

api_app/script_analyzers/file_analyzers/xlm_macro_deobfuscator.py

@@ -0,0 +1,50 @@
+import logging
+from XLMMacroDeobfuscator.deobfuscator import process_file
+from api_app.script_analyzers.classes import FileAnalyzer
+from celery.exceptions import SoftTimeLimitExceeded
+
+logger = logging.getLogger(__name__)
+
+
+class XlmMacroDeobfuscator(FileAnalyzer):
+    def set_config(self, additional_config_params):
+        self.passwords_to_check = [""]
+        additional_passwords_to_check = additional_config_params.get(
+            "passwords_to_check", []
+        )
+        if isinstance(additional_passwords_to_check, list):
+            self.passwords_to_check.extend(additional_passwords_to_check)
+        elif isinstance(additional_passwords_to_check, str):
+            self.passwords_to_check.append(additional_passwords_to_check)
+
+    def run(self):
+        results = {}
+        try:
+            for password in self.passwords_to_check:
+                results = self.decrypt(password)
+                if results:
+                    break
+            if not results:
+                results["error"] = "Can't decrypt with current passwords"
+        except SoftTimeLimitExceeded:
+            self._handle_base_exception("Soft Time Limit Exceeded")
+        return results
+
+    def decrypt(self, xlmpassword=""):
+        args = {
+            "file": self.filepath,
+            "noindent": True,
+            "nointeractive": True,
+            "return_deobfuscated": True,
+            "output_level": 3,
+        }
+        if xlmpassword:
+            args["password"] = xlmpassword
+        try:
+            results = {"output": process_file(**args), "correct_password": xlmpassword}
+
+            return results
+        except Exception as e:
+            if "Failed to decrypt" in str(e):
+                return {}
+            return {"errors": str(e)}

api_app/script_analyzers/observable_analyzers/vt3_get.py

@@ -39,7 +39,11 @@ def run(self):
 
 
 def vt_get_report(
-    api_key, observable_name, obs_clfn, additional_config_params, job_id,
+    api_key,
+    observable_name,
+    obs_clfn,
+    additional_config_params,
+    job_id,
 ):
     headers = {"x-apikey": api_key}
 

configuration/analyzer_config.json

@@ -165,7 +165,8 @@
       "text/x-ms-iqy",
       "application/excel",
       "text/xml",
-      "application/xml"
+      "application/xml",
+      "application/zip"
     ],
     "decription": "static generic document analysis",
     "python_module": "docinfo_run"
@@ -938,6 +939,23 @@
       "directories_with_rules": ["/opt/deploy/yara/rules"]
     }
   },
+  "Xlm_Macro_Deobfuscator": {
+    "type": "file",
+    "supported_filetypes": [
+      "application/vnd.ms-excel.addin.macroEnabled",
+      "application/x-mspublisher",
+      "application/vnd.ms-excel",
+      "application/vnd.ms-excel.sheet.macroEnabled.12",
+      "application/excel",
+      "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+      "application/zip"
+    ],
+    "decription": "Xlm macro deobfuscator",
+    "python_module": "xlm_deobfuscator_run",
+    "additional_config_params": {
+        "passwords_to_check": ["agenzia", "inps", "coronavirus"]
+    }
+  },
   "Yara_Scan_Florian": {
     "type": "file",
     "description": "scan a file with Neo23x0 yara rules",

configuration/ldap_config.py

@@ -11,7 +11,9 @@
 AUTH_LDAP_BIND_DN = "cn=django-agent,dc=example,dc=com"
 AUTH_LDAP_BIND_PASSWORD = ""
 AUTH_LDAP_USER_SEARCH = LDAPSearch(
-    "ou=users,dc=example,dc=com", ldap.SCOPE_SUBTREE, "(uid=%(user)s)",
+    "ou=users,dc=example,dc=com",
+    ldap.SCOPE_SUBTREE,
+    "(uid=%(user)s)",
 )
 # Or:
 # AUTH_LDAP_USER_DN_TEMPLATE = 'uid=%(user)s,ou=users,dc=example,dc=com'

docs/source/Contribute.md

@@ -83,7 +83,7 @@ $ flake8 . --show-source --statistics
 
 ```bash
 $ docker-compose -f docker-compose-for-tests.yml build
-$ docker-compose -f docker-compose-for-tests.yml up`
+$ docker-compose -f docker-compose-for-tests.yml up
 ```
 
 3. Here, we simulate the travis CI tests locally by running the following 3 tests,

docs/source/Usage.md

@@ -37,6 +37,7 @@ The following is the list of the available analyzers you can run out-of-the-box:
 * `Rtf_Info`: static RTF analysis
 * `Doc_Info`: static generic document analysis
 * `Doc_Info_Experimental`: static document analysis with new features to analyze XLM macros, encrypted macros and more
+* `Xlm_Macro_Deobfuscator`:  [XlmMacroDeobfuscator](https://github.com/DissectMalware/XLMMacroDeobfuscator) deobfuscate xlm macros
 * `PE_Info`: static PE analysis
 * `Signature_Info`: PE signature extractor
 * `Speakeasy`: Speakeasy binary emulation

intel_owl/settings.py

@@ -79,7 +79,9 @@
 WSGI_APPLICATION = "intel_owl.wsgi.application"
 
 REST_FRAMEWORK = {
-    "DEFAULT_RENDERER_CLASSES": ["rest_framework.renderers.JSONRenderer",],
+    "DEFAULT_RENDERER_CLASSES": [
+        "rest_framework.renderers.JSONRenderer",
+    ],
     "DEFAULT_AUTHENTICATION_CLASSES": [
         "rest_framework_simplejwt.authentication.JWTAuthentication",
     ],
@@ -182,9 +184,15 @@
     {
         "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
     },
-    {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",},
-    {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",},
-    {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",},
+    {
+        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
+    },
 ]
 
 

intel_owl/tasks.py

@@ -22,6 +22,7 @@
     apkid,
     quark_engine,
     unpac_me,
+    xlm_macro_deobfuscator,
 )
 from api_app.script_analyzers.observable_analyzers import (
     abuseipdb,
@@ -612,6 +613,15 @@ def docinfo_run(
     ).start()
 
 
+@shared_task(soft_time_limit=30)
+def xlm_deobfuscator_run(
+    analyzer_name, job_id, filepath, filename, md5, additional_config_params
+):
+    xlm_macro_deobfuscator.XlmMacroDeobfuscator(
+        analyzer_name, job_id, filepath, filename, md5, additional_config_params
+    ).start()
+
+
 @shared_task(soft_time_limit=30)
 def rtfinfo_run(
     analyzer_name, job_id, filepath, filename, md5, additional_config_params

requirements.txt

@@ -38,7 +38,7 @@ more-itertools==7.2.0
 msoffcrypto-tool==4.10.1
 numpy==1.17.1
 olefile==0.46
-git+git://github.com/mlodic/oletools.git@aef91b95cc9e870da2dd4205f9fc8a933e2ea2dd
+git+git://github.com/mlodic/oletools.git@4274e151bf9697b7f001e6e22007c0641ef2cb19
 OTXv2==1.5.9
 peepdf==0.4.2
 https://github.com/mlodic/pefile/archive/master.tar.gz
@@ -78,7 +78,7 @@ zipp==0.6.0
 djangorestframework-simplejwt==4.4.0
 djangorestframework-guardian==0.3.0
 flake8==3.8.2 
-black==19.10b0
+black==20.8b1
 quark-engine==20.8
-git+git://github.com/DissectMalware/XLMMacroDeobfuscator.git@c78b9e443e2667399470462fd863ebe4e2b8c978
+git+git://github.com/DissectMalware/XLMMacroDeobfuscator.git@89fbce0c87014a4b5a22c1aef09c8b3ea9bf16c0
 speakeasy-emulator==1.4.4

tests/test_files.py

@@ -27,6 +27,7 @@
     apkid,
     quark_engine,
     unpac_me,
+    xlm_macro_deobfuscator,
 )
 from api_app.script_analyzers.observable_analyzers import vt3_get
 
@@ -256,7 +257,12 @@ def test_peframe_scan_file(self, mock_get=None, mock_post=None):
     @mock_connections(patch("requests.post", side_effect=mocked_docker_analyzer_post))
     def test_capa_scan_file(self, mock_get=None, mock_post=None):
         report = capa_info.CapaInfo(
-            "Capa_Info", self.job_id, self.filepath, self.filename, self.md5, {},
+            "Capa_Info",
+            self.job_id,
+            self.filepath,
+            self.filename,
+            self.md5,
+            {},
         ).start()
         self.assertEqual(report.get("success", False), True)
 
@@ -295,6 +301,34 @@ def test_speakeasy_dll(self):
         self.assertEqual(report.get("success", False), True)
 
 
+class FileAnalyzersExcelTests(TestCase):
+    def setUp(self):
+        params = {
+            "source": "test",
+            "is_sample": True,
+            "file_mimetype": "application/vnd.ms-excel",
+            "force_privacy": False,
+            "analyzers_requested": ["test"],
+        }
+        filename = "document.xls"
+        test_job = _generate_test_job_with_file(params, filename)
+        self.job_id = test_job.id
+        self.filepath, self.filename = utils.get_filepath_filename(self.job_id)
+        self.runtime_configuration = test_job.runtime_configuration
+        self.md5 = test_job.md5
+
+    def test_xlm_macro_deobfuscator_excel(self):
+        report = xlm_macro_deobfuscator.XlmMacroDeobfuscator(
+            "Xlm_Macro_Deobfuscator",
+            self.job_id,
+            self.filepath,
+            self.filename,
+            self.md5,
+            {},
+        ).start()
+        self.assertEqual(report.get("success", False), True)
+
+
 class FileAnalyzersDocTests(TestCase):
     def setUp(self):
         params = {
@@ -416,7 +450,12 @@ def setUp(self):
     @mock_connections(patch("requests.post", side_effect=mocked_docker_analyzer_post))
     def test_thug_html(self, mock_get=None, mock_post=None):
         report = thug_file.ThugFile(
-            "Thug_HTML_Info", self.job_id, self.filepath, self.filename, self.md5, {},
+            "Thug_HTML_Info",
+            self.job_id,
+            self.filepath,
+            self.filename,
+            self.md5,
+            {},
         ).start()
         self.assertEqual(report.get("success", False), True)
 
@@ -480,7 +519,12 @@ def test_apkid(self, mock_get=None, mock_post=None):
 
     def test_quark_engine(self, mock_get=None, mock_post=None):
         report = quark_engine.QuarkEngine(
-            "Quark_Engine_APK", self.job_id, self.filepath, self.filename, self.md5, {},
+            "Quark_Engine_APK",
+            self.job_id,
+            self.filepath,
+            self.filename,
+            self.md5,
+            {},
         ).start()
         self.assertEqual(report.get("success", False), True)