diff --git a/api_app/playbooks_manager/views.py b/api_app/playbooks_manager/views.py index 4285296ec5..686fb26a5f 100644 --- a/api_app/playbooks_manager/views.py +++ b/api_app/playbooks_manager/views.py @@ -54,7 +54,8 @@ def analyze_multiple_observables(self, request): data=request.data, many=True, context={"request": request} ) oas.is_valid(raise_exception=True) - jobs = oas.save(send_task=True) + parent_job = oas.validated_data[0].get("parent_job", None) + jobs = oas.save(send_task=True, parent=parent_job) return Response( JobResponseSerializer(jobs, many=True).data, status=status.HTTP_200_OK, diff --git a/api_app/serializers/job.py b/api_app/serializers/job.py index f1e006abef..dae9e6ab4f 100644 --- a/api_app/serializers/job.py +++ b/api_app/serializers/job.py @@ -25,6 +25,7 @@ from api_app.helpers import calculate_md5, gen_random_colorhex from api_app.investigations_manager.models import Investigation from api_app.models import Comment, Job, Tag +from api_app.pivots_manager.models import PivotMap from api_app.playbooks_manager.models import PlaybookConfig from api_app.serializers import AbstractBIInterface from api_app.serializers.report import AbstractReportSerializerInterface @@ -97,6 +98,7 @@ class Meta: "scan_mode", "scan_check_time", "investigation", + "parent_job", ) md5 = rfs.HiddenField(default=None) @@ -116,6 +118,7 @@ class Meta: investigation = rfs.PrimaryKeyRelatedField( queryset=Investigation.objects.all(), many=False, required=False, default=None ) + parent_job = rfs.PrimaryKeyRelatedField(queryset=Job.objects.all(), required=False) connectors_requested = rfs.SlugRelatedField( slug_field="name", queryset=ConnectorConfig.objects.all(), @@ -340,6 +343,7 @@ def check_previous_jobs(self, validated_data: Dict) -> Job: def create(self, validated_data: Dict) -> Job: warnings = validated_data.pop("warnings") send_task = validated_data.pop("send_task", False) + parent_job = validated_data.pop("parent_job", None) if validated_data["scan_mode"] == ScanMode.CHECK_PREVIOUS_ANALYSIS.value: try: return self.check_previous_jobs(validated_data) @@ -350,6 +354,10 @@ def create(self, validated_data: Dict) -> Job: job.warnings = warnings job.save() logger.info(f"Job {job.pk} created") + if parent_job: + PivotMap.objects.create( + starting_job=validated_data["parent"], ending_job=job, pivot_config=None + ) if send_task: from intel_owl.tasks import job_pipeline @@ -505,12 +513,18 @@ class Meta: ) playbook_requested = rfs.SlugRelatedField(read_only=True, slug_field="name") playbook_to_execute = rfs.SlugRelatedField(read_only=True, slug_field="name") + investigation = rfs.SerializerMethodField(read_only=True, default=None) permissions = rfs.SerializerMethodField() def get_pivots_to_execute(self, obj: Job): # skipcq: PYL-R0201 # this cast is required or serializer doesn't work with websocket return list(obj.pivots_to_execute.all().values_list("name", flat=True)) + def get_investigation(self, instance: Job): # skipcq: PYL-R0201 + if root_investigation := instance.get_root().investigation: + return root_investigation.pk + return instance.investigation + def get_fields(self): # this method override is required for a cyclic import from api_app.analyzers_manager.serializers import AnalyzerReportSerializer @@ -566,6 +580,8 @@ def save(self, parent: Job = None, **kwargs): # the parent has already an investigation # so we don't need to do anything because everything is already connected if parent.investigation: + parent.investigation.status = parent.investigation.Status.RUNNING.value + parent.investigation.save() return jobs # if we have a parent, it means we are pivoting from one job to another else: @@ -924,10 +940,7 @@ class JobResponseSerializer(rfs.ModelSerializer): source="playbook_to_execute", slug_field="name", ) - investigation = rfs.SlugRelatedField( - read_only=True, - slug_field="pk", - ) + investigation = rfs.SerializerMethodField(read_only=True, default=None) class Meta: model = Job @@ -942,6 +955,11 @@ class Meta: extra_kwargs = {"warnings": {"read_only": True, "required": False}} list_serializer_class = JobEnvelopeSerializer + def get_investigation(self, instance: Job): # skipcq: PYL-R0201 + if root_investigation := instance.get_root().investigation: + return root_investigation.pk + return instance.investigation + def to_representation(self, instance: Job): result = super().to_representation(instance) result["status"] = self.STATUS_ACCEPTED diff --git a/api_app/views.py b/api_app/views.py index 5239630f69..01f0e6c735 100644 --- a/api_app/views.py +++ b/api_app/views.py @@ -232,7 +232,8 @@ def analyze_multiple_observables(request): data=request.data, many=True, context={"request": request} ) oas.is_valid(raise_exception=True) - jobs = oas.save(send_task=True) + parent_job = oas.validated_data[0].get("parent_job", None) + jobs = oas.save(send_task=True, parent=parent_job) jrs = JobResponseSerializer(jobs, many=True).data logger.info(f"finished analyze_multiple_observables from user {request.user}") return Response( diff --git a/docs/source/Contribute.md b/docs/source/Contribute.md index 61b101d6b1..ea270e5863 100644 --- a/docs/source/Contribute.md +++ b/docs/source/Contribute.md @@ -259,7 +259,7 @@ After having written the new python module, you have to remember to: 6. *Maximum tlp: maximum tlp to allow the run on the connector 7. *Run on failure: if the connector should be run even if the job fails -### Hot to add a new Ingestor +### How to add a new Ingestor 1. Put the module in the `ingestors` directory 2. Remember to use `_monkeypatch()` in its class to create automated tests for the new ingestor. This is a trick to have tests in the same class of its ingestor. 3. Create the configuration inside django admin in `Ingestors_manager/IngestorConfigs` (* = mandatory, ~ = mandatory on conditions) @@ -271,6 +271,17 @@ After having written the new python module, you have to remember to: 6. *Playbook to Execute: Playbook that **will** be executed on every IOC retrieved 7. *Schedule: Crontab object that describes the schedule of the ingestor. You are able to create a new clicking the `plus` symbol. +### How to add a new Pivot +1. Put the module in the `pivots` directory +2. Remember to use `_monkeypatch()` in its class to create automated tests for the new pivot. This is a trick to have tests in the same class of its pivot. +3. Create the configuration inside django admin in `Pivots_manager/PivotConfigs` (* = mandatory, ~ = mandatory on conditions) + 1. *Name: specific name of the configuration + 2. *Python module: . + 3. *Description: description of the configuration + 4. *Routing key: celery queue that will be used + 5. *Soft_time_limit: maximum time for the task execution + 6. *Playbook to Execute: Playbook that **will** be executed in the Job generated by the Pivot + ### How to add a new Visualizer #### Configuration diff --git a/docs/source/Usage.md b/docs/source/Usage.md index 523bdef699..dc74f15c04 100644 --- a/docs/source/Usage.md +++ b/docs/source/Usage.md @@ -309,11 +309,31 @@ Pivots are designed to create a job from another job. This plugin allows the use This is a "SOAR" feature that allows the users to connect multiple analysis together. -Right now the support for this kind of plugin in the GUI is very limited, while the backend is fully operative. We are working on the frontend. - #### List of pre-built Pivots None +You can build your own custom Pivot with your custom logic with just few lines of code. See the [Contribute](https://intelowl.readthedocs.io/en/latest/Contribute.html#how-to-add-a-new-pivot) section for more info. + +#### Creating Pivots from the GUI + +From the GUI, the users can pivot in two ways: +- If a Job executed a [Visualizer](#visualizers), it is possible to select a field extracted and analyze its value by clicking the "Pivot" button (see following image). In this way, the user is able to "jump" from one indicator to another. +![img.png](../static/pivot_job_report.png) + +- Starting from an already existing [Investigation](#investigations-framework), it is possible to select a Job block and click the "Pivot" button to analyze the same observable again, usually choosing another [Playbook](#playbooks) (see following image) +![img.png](../static/pivot_investigation_report.png) + +In both cases, the user is redirected to the Scan Page that is precompiled with the observable selected. Then the user would be able to select the [Playbook](#playbooks) to execute in the new job. +![img.png](../static/pivot_scan_page.png) + +After the new Job is started, a new [Investigation](#investigations-framework) will be created (if it does not already exist) and both the jobs will be added to the same Investigation. + +In the following image you can find an example of an [Investigation](#investigations-framework) composed by 3 pivots generated manually: +* leveraging the first way to create a Pivot, the 2 Jobs that analyzed IP addresses have been generated from the first `test\.com` Job +* leveraging the second way to create a Pivot, the second `test\.com` analysis had been created with a different Playbook. + +![img.png](../static/pivot_investigation.png) + ### Visualizers With IntelOwl v5 we introduced a new plugin type called **Visualizers**. @@ -570,7 +590,7 @@ Things to know about the framework: *Investigations* are created in 2 ways: * automatically: * if you scan multiple observables at the same time, a new investigation will be created by default and all the observables they will be automatically connected to the same investigation. - * if you run a Job with a Playbook which contains a Pivot that triggers another Job, a new investigation will be created and both the Jobs will be added to the same investigation. + * if you run a Job with a Playbook which contains a [Pivot](#pivots) that triggers another Job, a new investigation will be created and both the Jobs will be added to the same investigation. See how you can create a new [Pivot manually from the GUI](#creating-pivots-from-the-gui). * manually: by clicking on the button in the "History" section you can create an Investigation from scratch without any job attached (see following image) ![img.png](../static/create_investigation.png) diff --git a/docs/static/pivot_investigation.png b/docs/static/pivot_investigation.png new file mode 100644 index 0000000000..9ab41a3d70 Binary files /dev/null and b/docs/static/pivot_investigation.png differ diff --git a/docs/static/pivot_investigation_report.png b/docs/static/pivot_investigation_report.png new file mode 100644 index 0000000000..9753a5b508 Binary files /dev/null and b/docs/static/pivot_investigation_report.png differ diff --git a/docs/static/pivot_job_report.png b/docs/static/pivot_job_report.png new file mode 100644 index 0000000000..e7409a1c69 Binary files /dev/null and b/docs/static/pivot_job_report.png differ diff --git a/docs/static/pivot_scan_page.png b/docs/static/pivot_scan_page.png new file mode 100644 index 0000000000..dd516d91b0 Binary files /dev/null and b/docs/static/pivot_scan_page.png differ diff --git a/frontend/src/components/investigations/flow/CustomJobNode.jsx b/frontend/src/components/investigations/flow/CustomJobNode.jsx index 8b8fa8fb01..efebd6ddb0 100644 --- a/frontend/src/components/investigations/flow/CustomJobNode.jsx +++ b/frontend/src/components/investigations/flow/CustomJobNode.jsx @@ -4,6 +4,7 @@ import { NodeToolbar, Handle, Position } from "reactflow"; import "reactflow/dist/style.css"; import { Button } from "reactstrap"; import { AiOutlineLink } from "react-icons/ai"; +import { LuGitBranchPlus } from "react-icons/lu"; import { RemoveJob } from "./investigationActions"; @@ -32,6 +33,15 @@ function CustomJobNode({ data }) { > Link + {data.isFirstLevel && }
@@ -32,6 +41,17 @@ export function VisualizerTooltip({ > Link + {textClassification !== ObservableClassifications.GENERIC && ( + + )}
{description && (
{ expect(removeJobButton).toBeInTheDocument(); const linkFirstJobButton = screen.getByRole("link", { name: "Link" }); expect(linkFirstJobButton).toBeInTheDocument(); + const firstJobPivotButton = screen.getByRole("link", { name: "Pivot" }); + expect(firstJobPivotButton).toBeInTheDocument(); // link to job page expect(linkFirstJobButton.href).toContain("/jobs/10/visualizer"); + // link pivot + expect(firstJobPivotButton.href).toContain( + "/scan?parent=10&observable=test1.com", + ); // job info const jobInfo = container.querySelector("#job10-info"); expect(jobInfo).toBeInTheDocument(); @@ -317,8 +323,14 @@ describe("test InvestigationFlow", () => { expect(removeSecondJobButton).toBeNull(); // no remove button in pivot const linkSecondJobButton = screen.getByRole("link", { name: "Link" }); expect(linkSecondJobButton).toBeInTheDocument(); + const secondJobPivotButton = screen.getByRole("link", { name: "Pivot" }); + expect(secondJobPivotButton).toBeInTheDocument(); // link to job page expect(linkSecondJobButton.href).toContain("/jobs/11/visualizer"); + // link pivot + expect(secondJobPivotButton.href).toContain( + "/scan?parent=11&observable=test11.com", + ); // job info const secondJobInfo = container.querySelector("#job11-info"); expect(secondJobInfo).toBeInTheDocument(); diff --git a/frontend/tests/components/jobs/result/visualizer/elements/base.test.jsx b/frontend/tests/components/jobs/result/visualizer/elements/base.test.jsx index 8e078b3b2e..21a257681d 100644 --- a/frontend/tests/components/jobs/result/visualizer/elements/base.test.jsx +++ b/frontend/tests/components/jobs/result/visualizer/elements/base.test.jsx @@ -5,6 +5,14 @@ import userEvent from "@testing-library/user-event"; import { BaseVisualizer } from "../../../../../../src/components/jobs/result/visualizer/elements/base"; import { getIcon } from "../../../../../../src/components/jobs/result/visualizer/icons"; +// mock useLocation +jest.mock("react-router-dom", () => ({ + ...jest.requireActual("react-router-dom"), + useLocation: () => ({ + pathname: "localhost/jobs/123/visualizer", + }), +})); + describe("BaseVisualizer component", () => { test("required-only params", async () => { const { container } = render( diff --git a/frontend/tests/components/jobs/result/visualizer/elements/bool.test.jsx b/frontend/tests/components/jobs/result/visualizer/elements/bool.test.jsx index 7dfa0167b3..ba15057646 100644 --- a/frontend/tests/components/jobs/result/visualizer/elements/bool.test.jsx +++ b/frontend/tests/components/jobs/result/visualizer/elements/bool.test.jsx @@ -5,6 +5,14 @@ import userEvent from "@testing-library/user-event"; import { getIcon } from "../../../../../../src/components/jobs/result/visualizer/icons"; import { BooleanVisualizer } from "../../../../../../src/components/jobs/result/visualizer/elements/bool"; +// mock useLocation +jest.mock("react-router-dom", () => ({ + ...jest.requireActual("react-router-dom"), + useLocation: () => ({ + pathname: "localhost/jobs/123/visualizer", + }), +})); + describe("BooleanVisualizer component", () => { test("required-only params", async () => { const { container } = render( diff --git a/frontend/tests/components/jobs/result/visualizer/elements/horizontalList.test.jsx b/frontend/tests/components/jobs/result/visualizer/elements/horizontalList.test.jsx index d56d15e707..0255ad2d72 100644 --- a/frontend/tests/components/jobs/result/visualizer/elements/horizontalList.test.jsx +++ b/frontend/tests/components/jobs/result/visualizer/elements/horizontalList.test.jsx @@ -7,6 +7,14 @@ import { BooleanVisualizer } from "../../../../../../src/components/jobs/result/ import { TitleVisualizer } from "../../../../../../src/components/jobs/result/visualizer/elements/title"; import { VerticalListVisualizer } from "../../../../../../src/components/jobs/result/visualizer/elements/verticalList"; +// mock useLocation +jest.mock("react-router-dom", () => ({ + ...jest.requireActual("react-router-dom"), + useLocation: () => ({ + pathname: "localhost/jobs/123/visualizer", + }), +})); + describe("HorizontalListVisualizer component", () => { test("required-only params", () => { const { container } = render( diff --git a/frontend/tests/components/jobs/result/visualizer/elements/title.test.jsx b/frontend/tests/components/jobs/result/visualizer/elements/title.test.jsx index f49d48ff0f..87e7bf3f08 100644 --- a/frontend/tests/components/jobs/result/visualizer/elements/title.test.jsx +++ b/frontend/tests/components/jobs/result/visualizer/elements/title.test.jsx @@ -4,6 +4,14 @@ import { render, screen } from "@testing-library/react"; import { TitleVisualizer } from "../../../../../../src/components/jobs/result/visualizer/elements/title"; import { BaseVisualizer } from "../../../../../../src/components/jobs/result/visualizer/elements/base"; +// mock useLocation +jest.mock("react-router-dom", () => ({ + ...jest.requireActual("react-router-dom"), + useLocation: () => ({ + pathname: "localhost/jobs/123/visualizer", + }), +})); + describe("TitleVisualizer component", () => { test("required-only params", () => { const { container } = render( diff --git a/frontend/tests/components/jobs/result/visualizer/elements/verticalList.test.jsx b/frontend/tests/components/jobs/result/visualizer/elements/verticalList.test.jsx index dfc78b1644..4acc30420d 100644 --- a/frontend/tests/components/jobs/result/visualizer/elements/verticalList.test.jsx +++ b/frontend/tests/components/jobs/result/visualizer/elements/verticalList.test.jsx @@ -5,6 +5,14 @@ import { BaseVisualizer } from "../../../../../../src/components/jobs/result/vis import { VerticalListVisualizer } from "../../../../../../src/components/jobs/result/visualizer/elements/verticalList"; import { HorizontalListVisualizer } from "../../../../../../src/components/jobs/result/visualizer/elements/horizontalList"; +// mock useLocation +jest.mock("react-router-dom", () => ({ + ...jest.requireActual("react-router-dom"), + useLocation: () => ({ + pathname: "localhost/jobs/123/visualizer", + }), +})); + describe("VerticalListVisualizer component", () => { test("required-only params", () => { const { container } = render( diff --git a/frontend/tests/components/jobs/result/visualizer/visualizer.test.jsx b/frontend/tests/components/jobs/result/visualizer/visualizer.test.jsx index db8b2b5930..60b39da95c 100644 --- a/frontend/tests/components/jobs/result/visualizer/visualizer.test.jsx +++ b/frontend/tests/components/jobs/result/visualizer/visualizer.test.jsx @@ -3,6 +3,14 @@ import "@testing-library/jest-dom"; import { render, screen, within } from "@testing-library/react"; import VisualizerReport from "../../../../../src/components/jobs/result/visualizer/visualizer"; +// mock useLocation +jest.mock("react-router-dom", () => ({ + ...jest.requireActual("react-router-dom"), + useLocation: () => ({ + pathname: "localhost/jobs/123/visualizer", + }), +})); + describe("test VisualizerReport (conversion from backend data to frontend components)", () => { test("visualizer failed with error", () => { render( diff --git a/frontend/tests/components/jobs/result/visualizer/visualizerTooltip.test.jsx b/frontend/tests/components/jobs/result/visualizer/visualizerTooltip.test.jsx index b180fc8cc2..1c0d19cef9 100644 --- a/frontend/tests/components/jobs/result/visualizer/visualizerTooltip.test.jsx +++ b/frontend/tests/components/jobs/result/visualizer/visualizerTooltip.test.jsx @@ -1,21 +1,24 @@ import React from "react"; import "@testing-library/jest-dom"; import { render, screen, waitFor } from "@testing-library/react"; +import { MemoryRouter } from "react-router-dom"; import userEvent from "@testing-library/user-event"; import { VisualizerTooltip } from "../../../../../src/components/jobs/result/visualizer/VisualizerTooltip"; describe("VisualizerTooltip component", () => { test("copy button - link button disabled", async () => { render( - <> -
Test
- - , + + <> +
Test
+ + +
, ); const user = userEvent.setup(); await user.hover(screen.getByText("Test")); @@ -36,15 +39,17 @@ describe("VisualizerTooltip component", () => { test("copy button - link button enabled", async () => { render( - <> -
Test
- - , + + <> +
Test
+ + +
, ); const user = userEvent.setup(); await user.hover(screen.getByText("Test")); @@ -66,15 +71,17 @@ describe("VisualizerTooltip component", () => { test("buttons and description", async () => { render( - <> -
Test
- - , + + <> +
Test
+ + +
, ); const user = userEvent.setup(); await user.hover(screen.getByText("Test")); @@ -90,7 +97,13 @@ describe("VisualizerTooltip component", () => { const linkButton = screen.getByText("Link"); expect(linkButton).toBeInTheDocument(); expect(linkButton.className).not.toContain("disabled"); - expect(screen.getByRole("link").href).toBe("https://google.com/"); + expect(linkButton.href).toBe("https://google.com/"); + // check pivot button + const pivotButton = screen.getByText("Pivot"); + expect(pivotButton).toBeInTheDocument(); + expect(pivotButton.href).toContain( + "/scan?parent=123&observable=google.com", + ); // check description expect(screen.getByText("description tooltip")).toBeInTheDocument(); }); diff --git a/frontend/tests/components/scan/ScanForm/ScanForm.advanced.test.jsx b/frontend/tests/components/scan/ScanForm/ScanForm.advanced.test.jsx index 5f38d4706e..da5b4fb209 100644 --- a/frontend/tests/components/scan/ScanForm/ScanForm.advanced.test.jsx +++ b/frontend/tests/components/scan/ScanForm/ScanForm.advanced.test.jsx @@ -133,6 +133,60 @@ describe("ScanForm adavanced use", () => { }); }); + test("test scan page with a parent and an observable in the GET parameters", async () => { + const user = userEvent.setup(); + render( + + + } /> + + , + ); + + // check value has been loaded + expect(screen.getAllByRole("textbox")[0].value).toBe( + "thisIsTheParamObservable.com", + ); + // check playbooks has been loaded + expect(screen.getByText("TEST_PLAYBOOK_DOMAIN")).toBeInTheDocument(); + + // start scan button + const startScanButton = screen.getByRole("button", { + name: "Start Scan", + }); + expect(startScanButton).toBeInTheDocument(); + expect(startScanButton.className).not.toContain("disabled"); + + await user.click(startScanButton); + + await waitFor(() => { + expect(axios.post.mock.calls[0]).toEqual( + // axios call + [ + PLAYBOOKS_ANALYZE_MULTIPLE_OBSERVABLE_URI, + { + observables: [["domain", "thisIsTheParamObservable.com"]], + playbook_requested: "TEST_PLAYBOOK_DOMAIN", + tlp: "CLEAR", + scan_mode: 2, + scan_check_time: "48:00:00", + runtime_configuration: { + analyzers: {}, + connectors: {}, + visualizers: {}, + }, + parent_job: "1", + }, + { headers: { "Content-Type": "application/json" } }, + ], + ); + }); + }); + test("test playbooks advanced change time", async () => { const user = userEvent.setup();