From 49c7ba51d01a912f4a63f28f2f8944cf142d3dbd Mon Sep 17 00:00:00 2001 From: Federico Gibertoni Date: Tue, 16 Apr 2024 16:48:29 +0200 Subject: [PATCH 1/6] Added migrations to remove analyzers and playbook --- .../0079_remove_dns0_rrsets_analyzer.py | 26 ++++++++++++++ .../0080_remove_dns0_names_analyzer.py | 26 ++++++++++++++ ...ete_dns0_playbook_free_to_use_analyzers.py | 35 +++++++++++++++++++ 3 files changed, 87 insertions(+) create mode 100644 api_app/analyzers_manager/migrations/0079_remove_dns0_rrsets_analyzer.py create mode 100644 api_app/analyzers_manager/migrations/0080_remove_dns0_names_analyzer.py create mode 100644 api_app/playbooks_manager/migrations/0032_delete_dns0_playbook_free_to_use_analyzers.py diff --git a/api_app/analyzers_manager/migrations/0079_remove_dns0_rrsets_analyzer.py b/api_app/analyzers_manager/migrations/0079_remove_dns0_rrsets_analyzer.py new file mode 100644 index 0000000000..62e49a52a9 --- /dev/null +++ b/api_app/analyzers_manager/migrations/0079_remove_dns0_rrsets_analyzer.py @@ -0,0 +1,26 @@ +from django.db import migrations + + +def migrate(apps, schema_editor): + PythonModule = apps.get_model("api_app", "PythonModule") + pm = PythonModule.objects.get( + module="dns0.dns0_rrsets.DNS0Rrsets", + base_path="api_app.analyzers_manager.observable_analyzers", + ) + pm.analyzerconfigs.all().delete() + pm.delete() + + +def reverse_migrate(apps, schema_editor): + pass + + +class Migration(migrations.Migration): + dependencies = [ + ("api_app", "0062_alter_parameter_python_module"), + ("playbooks_manager", "0032_delete_dns0_playbook_free_to_use_analyzers"), + ("analyzers_manager", "0078_analyzer_config_hfinger"), + ] + operations = [ + migrations.RunPython(migrate, reverse_migrate), + ] diff --git a/api_app/analyzers_manager/migrations/0080_remove_dns0_names_analyzer.py b/api_app/analyzers_manager/migrations/0080_remove_dns0_names_analyzer.py new file mode 100644 index 0000000000..7704d99445 --- /dev/null +++ b/api_app/analyzers_manager/migrations/0080_remove_dns0_names_analyzer.py @@ -0,0 +1,26 @@ +from django.db import migrations + + +def migrate(apps, schema_editor): + PythonModule = apps.get_model("api_app", "PythonModule") + pm = PythonModule.objects.get( + module="dns0.dns0_names.DNS0Names", + base_path="api_app.analyzers_manager.observable_analyzers", + ) + pm.analyzerconfigs.all().delete() + pm.delete() + + +def reverse_migrate(apps, schema_editor): + pass + + +class Migration(migrations.Migration): + dependencies = [ + ("api_app", "0062_alter_parameter_python_module"), + ("playbooks_manager", "0032_delete_dns0_playbook_free_to_use_analyzers"), + ("analyzers_manager", "0079_remove_dns0_rrsets_analyzer"), + ] + operations = [ + migrations.RunPython(migrate, reverse_migrate), + ] diff --git a/api_app/playbooks_manager/migrations/0032_delete_dns0_playbook_free_to_use_analyzers.py b/api_app/playbooks_manager/migrations/0032_delete_dns0_playbook_free_to_use_analyzers.py new file mode 100644 index 0000000000..cd8d340de7 --- /dev/null +++ b/api_app/playbooks_manager/migrations/0032_delete_dns0_playbook_free_to_use_analyzers.py @@ -0,0 +1,35 @@ +# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl +# See the file 'LICENSE' for copying permission. + + +from django.db import migrations + + +def migrate(apps, schema_editor): + playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig") + AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig") + pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS") + pc.analyzers.remove(AnalyzerConfig.objects.get(name="DNS0_rrsets_name").id) + pc.analyzers.remove(AnalyzerConfig.objects.get(name="DNS0_names").id) + pc.full_clean() + pc.save() + + +def reverse_migrate(apps, schema_editor): + playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig") + AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig") + pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS") + pc.analyzers.add(AnalyzerConfig.objects.get(name="DNS0_rrsets_name").id) + pc.analyzers.add(AnalyzerConfig.objects.get(name="DNS0_names").id) + pc.full_clean() + pc.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("playbooks_manager", "0031_add_hfinger_analyzer_free_to_use"), + ] + + operations = [ + migrations.RunPython(migrate, reverse_migrate), + ] From d710e81a80c3c10b0b77548f3b18404d80396e26 Mon Sep 17 00:00:00 2001 From: Federico Gibertoni Date: Tue, 16 Apr 2024 16:52:32 +0200 Subject: [PATCH 2/6] Removed analyzers sources --- .../observable_analyzers/dns0/__init__.py | 0 .../observable_analyzers/dns0/dns0_base.py | 133 ----------- .../observable_analyzers/dns0/dns0_names.py | 209 ---------------- .../observable_analyzers/dns0/dns0_rrsets.py | 225 ------------------ 4 files changed, 567 deletions(-) delete mode 100644 api_app/analyzers_manager/observable_analyzers/dns0/__init__.py delete mode 100644 api_app/analyzers_manager/observable_analyzers/dns0/dns0_base.py delete mode 100644 api_app/analyzers_manager/observable_analyzers/dns0/dns0_names.py delete mode 100644 api_app/analyzers_manager/observable_analyzers/dns0/dns0_rrsets.py diff --git a/api_app/analyzers_manager/observable_analyzers/dns0/__init__.py b/api_app/analyzers_manager/observable_analyzers/dns0/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/api_app/analyzers_manager/observable_analyzers/dns0/dns0_base.py b/api_app/analyzers_manager/observable_analyzers/dns0/dns0_base.py deleted file mode 100644 index e80dd98983..0000000000 --- a/api_app/analyzers_manager/observable_analyzers/dns0/dns0_base.py +++ /dev/null @@ -1,133 +0,0 @@ -import re -import typing -from abc import ABCMeta -from logging import getLogger - -import dateparser - -from api_app.analyzers_manager.classes import BaseAnalyzerMixin -from api_app.analyzers_manager.exceptions import ( - AnalyzerConfigurationException, - AnalyzerRunException, -) -from certego_saas.apps.user.models import User - -_supported_sort_types = [ - "first_seen", - "last_seen", -] - -_min_limit_value = 0 -_max_limit_value = 50000 - -_min_offset_value = 0 - -logger = getLogger(__name__) - - -class DNS0Mixin(BaseAnalyzerMixin, metaclass=ABCMeta): - base_url: str = "https://api.dns0.eu/" - - _api_key: str - from_date: str = "-1M" - sort: str - format: str - limit: int = 100 - offset: int - - def config(self, runtime_configuration: typing.Dict): - super().config(runtime_configuration) - # workaround to not being able to use "from" as variable name - if not hasattr(self, "from"): - setattr(self, "from", self.from_date) - - def _create_headers(self): - headers = {"Accept": "application/json", "User-Agent": "IntelOwl"} - if hasattr(self, "_api_key") and self._api_key: - headers["Authorization"] = f"Bearer {self._api_key}" - return headers - - @staticmethod - def convert_date_type(date_string): - if not date_string: - return False - - date_parsed = ( - DNS0Mixin.convert_unix_timestamp(date_string) - or DNS0Mixin.convert_relative_date(date_string) - or DNS0Mixin.convert_date(date_string) - ) - if not date_parsed: - raise AnalyzerRunException("Error in date format!") - return date_parsed - - @staticmethod - def convert_relative_date(date): - # accepts string matching the format: - # - at the beginning - # a number - # a character indicating Year, Month or Day - pattern = re.compile(r"-\d+[YMD]") - if match := pattern.match(date): - return match.group() - return False - - @staticmethod - def convert_date(date): - pattern = re.compile(r"^(\d{4}-\d{2}-\d{2})$") - if match := pattern.match(date): - return dateparser.parse(match.group()) - return False - - @staticmethod - def convert_unix_timestamp(timestamp): - try: - return str(int(timestamp)) - except Exception: - return False - - def _validate_params(self): - if ( - hasattr(self, "sort") - and self.sort - and self.sort not in _supported_sort_types - ): - raise AnalyzerConfigurationException( - f"Sort type {self.sort} not supported! " - f"Available sort types are: {_supported_sort_types}" - ) - - if ( - hasattr(self, "limit") - and self.limit - and not _min_limit_value < self.limit <= _max_limit_value - ): - raise AnalyzerConfigurationException( - f"{self.limit} is out of bound! " - f"Max value is {_max_limit_value}, min value is {_min_limit_value}" - ) - - if hasattr(self, "offset") and self.offset and self.offset < _min_offset_value: - raise AnalyzerConfigurationException( - f"{self.offset} can't be below {_min_offset_value}" - ) - - def _create_params(self): - params = {} - # convert dates to correct format - dates = ["from", "to", "not_before"] - parameters = ["sort", "format", "limit", "offset"] - - for date in dates: - if getattr(self, date, None): - if result := self.convert_date_type(getattr(self, date)): - params[date] = result - - for p in parameters: - if getattr(self, p, None): - params[p] = getattr(self, p) - - return params - - def _get_health_check_url(self, user: User = None) -> typing.Optional[str]: - return self.base_url diff --git a/api_app/analyzers_manager/observable_analyzers/dns0/dns0_names.py b/api_app/analyzers_manager/observable_analyzers/dns0/dns0_names.py deleted file mode 100644 index b62681ad74..0000000000 --- a/api_app/analyzers_manager/observable_analyzers/dns0/dns0_names.py +++ /dev/null @@ -1,209 +0,0 @@ -from logging import getLogger -from typing import Dict -from urllib.parse import urlparse - -import requests - -from api_app.analyzers_manager import classes -from api_app.analyzers_manager.exceptions import AnalyzerConfigurationException -from api_app.analyzers_manager.models import AnalyzerConfig -from api_app.analyzers_manager.observable_analyzers.dns0.dns0_base import DNS0Mixin -from api_app.models import Parameter, PluginConfig -from tests.mock_utils import MockUpResponse, if_mock_connections, patch - -logger = getLogger(__name__) - -_supported_fuzzy_params = [ - "swap", - "omit", - "repeat", - "add", - "typo", - "bitflip", - "hyphen", - "fatfinger", - "subdomain", - "vowels", - "homoglyph", - "all", -] - -_supported_format_types = [ - "json", - "dig", -] - - -class DNS0Names(classes.ObservableAnalyzer, DNS0Mixin): - endpoint: str = "names" - - root: bool - fuzzy: list[str] - - def config(self, runtime_configuration: Dict): - super().config(runtime_configuration) - self._validate_params() - - def run(self): - params = self._create_params() - headers = self._create_headers() - - response = requests.get( - self.base_url + self.endpoint, params=params, headers=headers - ) - response.raise_for_status() - - return response.json() - - def update(self) -> bool: - pass - - def _validate_params(self): - super()._validate_params() - if ( - hasattr(self, "fuzzy") - and self.fuzzy - and any( - fuzzy_params not in _supported_fuzzy_params - for fuzzy_params in self.fuzzy - ) - ): - raise AnalyzerConfigurationException( - "Fuzzy type not supported! " - "The list of supported fuzzy is at: " - "https://docs.dns0.eu/dns-api/names#fuzziness" - ) - - if ( - hasattr(self, "format") - and self.format - and self.format not in _supported_format_types - ): - raise AnalyzerConfigurationException( - f"Format type {self.format} not supported! " - f"Available format types are: {_supported_format_types}" - ) - - def _create_params(self): - params = super()._create_params() - target_observable = self.observable_name - if self.observable_classification == self.ObservableTypes.URL: - target_observable = urlparse(self.observable_name).hostname - params["q"] = target_observable - - # convert root parameter into 1 or 0 - if hasattr(self, "root") and self.root: - params["root"] = int(self.root) - - # pass list of fuzzy parameter - if hasattr(self, "fuzzy") and self.fuzzy: - params["fuzzy"] = self.fuzzy - - return params - - @classmethod - def _monkeypatch(cls): - ac = AnalyzerConfig.objects.get(name="DNS0_names") - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="from", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value="-1M", - ) - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="to", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value="", - ) - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="not_before", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value="", - ) - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="sort", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value="first_seen", - ) - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="format", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value="json", - ) - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="limit", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value=100, - ) - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="offset", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value=0, - ) - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="fuzzy", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value=[], - ) - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="root", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value=True, - ) - - patches = [ - if_mock_connections( - patch( - "requests.get", - return_value=MockUpResponse( - { - "data": [ - { - "first_seen": "2023-12-14T16:37:44.000Z", - "last_seen": "2023-12-14T16:37:44.000Z", - "name": "gcfr2.example.opentlc.com.", - } - ], - "meta": {"results": 834824}, - }, - 200, - ), - ), - ) - ] - return super()._monkeypatch(patches=patches) diff --git a/api_app/analyzers_manager/observable_analyzers/dns0/dns0_rrsets.py b/api_app/analyzers_manager/observable_analyzers/dns0/dns0_rrsets.py deleted file mode 100644 index e2b12f185e..0000000000 --- a/api_app/analyzers_manager/observable_analyzers/dns0/dns0_rrsets.py +++ /dev/null @@ -1,225 +0,0 @@ -from logging import getLogger -from typing import Dict - -import requests - -from api_app.analyzers_manager import classes -from api_app.analyzers_manager.exceptions import AnalyzerConfigurationException -from api_app.analyzers_manager.models import AnalyzerConfig -from api_app.analyzers_manager.observable_analyzers.dns0.dns0_base import DNS0Mixin -from api_app.models import Parameter, PluginConfig -from tests.mock_utils import MockUpResponse, if_mock_connections, patch - -logger = getLogger(__name__) - -_supported_format_types = [ - "json", - "cof", - "dig", -] - -_supported_directions = [ - "right", - "left", -] - - -class DNS0Rrsets(classes.ObservableAnalyzer, DNS0Mixin): - endpoint: str = "rrsets" - - direction: str - name: str - data: str - type: list[str] - include_subdomain: bool - - def config(self, runtime_configuration: Dict): - super().config(runtime_configuration) - self._validate_params() - - def run(self): - params = self._create_params() - headers = self._create_headers() - - response = requests.get( - self.base_url + self.endpoint, params=params, headers=headers - ) - response.raise_for_status() - - return response.json() - - def update(self) -> bool: - pass - - def _validate_params(self): - super()._validate_params() - if ( - hasattr(self, "direction") - and self.direction - and self.direction not in _supported_directions - ): - raise AnalyzerConfigurationException("Matching direction not specified!") - - if ( - hasattr(self, "format") - and self.format - and self.format not in _supported_format_types - ): - raise AnalyzerConfigurationException( - f"Format type {self.format} not supported! " - f"Available format types are: {_supported_format_types}" - ) - - def _create_params(self): - params = super()._create_params() - query_type = None - if hasattr(self, "direction") and self.direction: - if self.direction == "left": - query_type = "name" - elif self.direction == "right": - query_type = "data" - - query = self.observable_name - if hasattr(self, "include_subdomain") and self.include_subdomain: - query = "." + query - params[query_type] = query - - # pass list of dns types parameter - if hasattr(self, "type") and self.type: - # convert the element that are int - res = [int(elem) if elem.isdigit() else elem for elem in self.type] - params["type"] = res - - return params - - @classmethod - def _monkeypatch(cls): - for config in ["DNS0_rrsets_data", "DNS0_rrsets_name"]: - ac = AnalyzerConfig.objects.get(name=config) - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="from", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value="-1M", - ) - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="to", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value="", - ) - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="not_before", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value="", - ) - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="sort", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value="first_seen", - ) - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="format", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value="json", - ) - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="limit", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value=100, - ) - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="offset", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value=0, - ) - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="type", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value=[], - ) - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="include_subdomain", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value=False, - ) - - ac = AnalyzerConfig.objects.get(name="DNS0_rrsets_name") - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="direction", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value="left", - ) - - ac = AnalyzerConfig.objects.get(name="DNS0_rrsets_data") - PluginConfig.objects.get_or_create( - analyzer_config=ac, - parameter=Parameter.objects.get( - name="direction", python_module__pk=ac.python_module_id - ), - for_organization=False, - owner=None, - value="right", - ) - - patches = [ - if_mock_connections( - patch( - "requests.get", - return_value=MockUpResponse( - { - "data": [ - { - "first_seen": "2023-04-15T16:50:52.000Z", - "last_seen": "2023-12-14T00:23:52.000Z", - "name": "example.com.", - "type": "A", - "data": ["93.184.216.34"], - } - ], - "meta": {"results": 6}, - }, - 200, - ), - ), - ) - ] - return super()._monkeypatch(patches=patches) From f7cb71ec36e80837fd4311e3feeb61aa63737445 Mon Sep 17 00:00:00 2001 From: Federico Gibertoni Date: Tue, 16 Apr 2024 17:32:44 +0200 Subject: [PATCH 3/6] Removed dns0 analyzer from docs --- docs/source/Advanced-Usage.md | 5 ----- docs/source/Usage.md | 3 --- 2 files changed, 8 deletions(-) diff --git a/docs/source/Advanced-Usage.md b/docs/source/Advanced-Usage.md index 4d66212c38..f0200bb9eb 100644 --- a/docs/source/Advanced-Usage.md +++ b/docs/source/Advanced-Usage.md @@ -234,11 +234,6 @@ Some analyzers could require a special configuration: - The `repositories` values is what will be used to actually run the analysis: if you have added private repositories, remember to add the url in `repositories` too! - You can add local rules inside the directory at `/opt/deploy/files_required/yara/YOUR_USERNAME/custom_rules/`. Please remember that these rules are not synced in a cluster deploy: for this reason is advised to upload them on GitHub and use the `repositories` or `private_repositories` attributes. -- `DNS0_rrsets_name` and `DNS0_rrsets_data` ([DNS0 API](https://docs.dns0.eu/dns-api/rrsets)): - - Both these analyzers have a default parameter named `direction` that is used to dispatch the type of query to run. - - The value `right` for this parameter runs the query using `data` API parameter. Otherwise, if the parameter value is `left` it runs the query using the `name` API parameter. - - This parameter should not be changed from default value. - ## Notifications Since v4, IntelOwl integrated the notification system from the `certego_saas` package, allowing the admins to create notification that every user will be able to see. diff --git a/docs/source/Usage.md b/docs/source/Usage.md index f77ec8a9d1..9913d408a7 100644 --- a/docs/source/Usage.md +++ b/docs/source/Usage.md @@ -178,9 +178,6 @@ The following is the list of the available analyzers you can run out-of-the-box. * `DNSDB`: scan an observable against the [Passive DNS Farsight Database](https://www.farsightsecurity.com/solutions/dnsdb/) (support both v1 and v2 versions) * `DNS0_EU`: Retrieve current domain resolution with DNS0.eu DoH (DNS over HTTPS) * `DNS0_EU_Malicious_Detector`: Check if a domain or an url is marked as malicious in DNS0.eu database ([Zero](https://www.dns0.eu/zero) service) -* `DNS0_names`: Run advanced searches on billions of current and historical domain names. ([DNS0 /names](https://docs.dns0.eu/dns-api/names)) -* `DNS0_rrsets_data`: Query billions of current and historical DNS resource records sets. Performs right-hand side matching. ([DNS0 /rrsets](https://docs.dns0.eu/dns-api/rrsets)) -* `DNS0_rrsets_name`: Query billions of current and historical DNS resource records sets. Performs left-hand side matching. ([DNS0 /rrsets](https://docs.dns0.eu/dns-api/rrsets)) * `DocGuard_Get`: check if an hash was analyzed on DocGuard. [DocGuard](https://www.docguard.io) * `Feodo_Tracker`: [Feodo Tracker](https://feodotracker.abuse.ch/) offers various blocklists, helping network owners to protect their users from Dridex and Emotet/Heodo. * `FileScan_Search`: Finds reports and uploaded files by various tokens, like hash, filename, verdict, IOCs etc via [FileScan.io API](https://www.filescan.io/api/docs). From 2f1e879416c361ee60e3023410f7cc24dd7c4e2d Mon Sep 17 00:00:00 2001 From: 0ssigeno Date: Wed, 17 Apr 2024 09:24:20 +0200 Subject: [PATCH 4/6] Fix Signed-off-by: 0ssigeno --- authentication/migrations/0005_create_profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/authentication/migrations/0005_create_profiles.py b/authentication/migrations/0005_create_profiles.py index 6e669c210e..83b3585e49 100644 --- a/authentication/migrations/0005_create_profiles.py +++ b/authentication/migrations/0005_create_profiles.py @@ -9,7 +9,7 @@ def migrate(apps, schema_editor): Profile = apps.get_model("authentication", "UserProfile") for user in User.objects.all(): is_robot = user.username.endswith("Ingestor") - if not hasattr(user, "profile") and not user.profile: + if not hasattr(user, "profile") or not user.profile: profile = Profile( user=user, task_priority=7 if is_robot else 10, is_robot=is_robot ) From 9b2f45783d2d0ad4a3a3a66b7f76dc3d3f3cb062 Mon Sep 17 00:00:00 2001 From: Federico Gibertoni Date: Wed, 17 Apr 2024 09:30:06 +0200 Subject: [PATCH 5/6] Fix black --- intel_owl/tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/intel_owl/tasks.py b/intel_owl/tasks.py index 3e630f38ef..9e86fd66d7 100644 --- a/intel_owl/tasks.py +++ b/intel_owl/tasks.py @@ -95,11 +95,13 @@ def remove_old_jobs(): @shared_task(base=FailureLoggedTask) def refresh_cache(python_class_str: str): from django.utils.module_loading import import_string + logger.info(f"Refreshing cache for {python_class_str}") python_class = import_string(python_class_str) python_class.delete_class_cache_keys() from api_app.models import PythonConfig + if issubclass(python_class, PythonConfig): for config in python_class.objects.all(): config.refresh_cache_keys() From eeeb4f8a085f5490be24b9e9624f581a8788398c Mon Sep 17 00:00:00 2001 From: Federico Gibertoni Date: Thu, 18 Apr 2024 09:54:42 +0200 Subject: [PATCH 6/6] Added if conditions for saver migrations --- .../0079_remove_dns0_rrsets_analyzer.py | 9 +++--- .../0080_remove_dns0_names_analyzer.py | 9 +++--- ...ete_dns0_playbook_free_to_use_analyzers.py | 30 ++++++++++++------- 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/api_app/analyzers_manager/migrations/0079_remove_dns0_rrsets_analyzer.py b/api_app/analyzers_manager/migrations/0079_remove_dns0_rrsets_analyzer.py index 62e49a52a9..25df9bbc4c 100644 --- a/api_app/analyzers_manager/migrations/0079_remove_dns0_rrsets_analyzer.py +++ b/api_app/analyzers_manager/migrations/0079_remove_dns0_rrsets_analyzer.py @@ -3,12 +3,13 @@ def migrate(apps, schema_editor): PythonModule = apps.get_model("api_app", "PythonModule") - pm = PythonModule.objects.get( + pm = PythonModule.objects.filter( module="dns0.dns0_rrsets.DNS0Rrsets", base_path="api_app.analyzers_manager.observable_analyzers", - ) - pm.analyzerconfigs.all().delete() - pm.delete() + ).first() + if pm: + pm.analyzerconfigs.all().delete() + pm.delete() def reverse_migrate(apps, schema_editor): diff --git a/api_app/analyzers_manager/migrations/0080_remove_dns0_names_analyzer.py b/api_app/analyzers_manager/migrations/0080_remove_dns0_names_analyzer.py index 7704d99445..8868a52580 100644 --- a/api_app/analyzers_manager/migrations/0080_remove_dns0_names_analyzer.py +++ b/api_app/analyzers_manager/migrations/0080_remove_dns0_names_analyzer.py @@ -3,12 +3,13 @@ def migrate(apps, schema_editor): PythonModule = apps.get_model("api_app", "PythonModule") - pm = PythonModule.objects.get( + pm = PythonModule.objects.filter( module="dns0.dns0_names.DNS0Names", base_path="api_app.analyzers_manager.observable_analyzers", - ) - pm.analyzerconfigs.all().delete() - pm.delete() + ).first() + if pm: + pm.analyzerconfigs.all().delete() + pm.delete() def reverse_migrate(apps, schema_editor): diff --git a/api_app/playbooks_manager/migrations/0032_delete_dns0_playbook_free_to_use_analyzers.py b/api_app/playbooks_manager/migrations/0032_delete_dns0_playbook_free_to_use_analyzers.py index cd8d340de7..7745a42b94 100644 --- a/api_app/playbooks_manager/migrations/0032_delete_dns0_playbook_free_to_use_analyzers.py +++ b/api_app/playbooks_manager/migrations/0032_delete_dns0_playbook_free_to_use_analyzers.py @@ -8,21 +8,31 @@ def migrate(apps, schema_editor): playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig") AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig") - pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS") - pc.analyzers.remove(AnalyzerConfig.objects.get(name="DNS0_rrsets_name").id) - pc.analyzers.remove(AnalyzerConfig.objects.get(name="DNS0_names").id) - pc.full_clean() - pc.save() + pc = playbook_config.objects.filter(name="FREE_TO_USE_ANALYZERS").first() + if pc: + for analyzer_config_name in ["DNS0_rrsets_name", "DNS0_names"]: + analyzer_config = AnalyzerConfig.objects.filter( + name=analyzer_config_name + ).first() + if analyzer_config: + pc.analyzers.remove(analyzer_config.id) + pc.full_clean() + pc.save() def reverse_migrate(apps, schema_editor): playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig") AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig") - pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS") - pc.analyzers.add(AnalyzerConfig.objects.get(name="DNS0_rrsets_name").id) - pc.analyzers.add(AnalyzerConfig.objects.get(name="DNS0_names").id) - pc.full_clean() - pc.save() + pc = playbook_config.objects.filter(name="FREE_TO_USE_ANALYZERS").first() + if pc: + for analyzer_config_name in ["DNS0_rrsets_name", "DNS0_names"]: + analyzer_config = AnalyzerConfig.objects.filter( + name=analyzer_config_name + ).first() + if analyzer_config: + pc.analyzers.add(analyzer_config.id) + pc.full_clean() + pc.save() class Migration(migrations.Migration):