Added pivot buttons

api_app/playbooks_manager/views.py

@@ -54,7 +54,8 @@ def analyze_multiple_observables(self, request):
             data=request.data, many=True, context={"request": request}
         )
         oas.is_valid(raise_exception=True)
-        jobs = oas.save(send_task=True)
+        parent_job = oas.validated_data[0].get("parent_job", None)
+        jobs = oas.save(send_task=True, parent=parent_job)
         return Response(
             JobResponseSerializer(jobs, many=True).data,
             status=status.HTTP_200_OK,

api_app/serializers/job.py

@@ -25,6 +25,7 @@
 from api_app.helpers import calculate_md5, gen_random_colorhex
 from api_app.investigations_manager.models import Investigation
 from api_app.models import Comment, Job, Tag
+from api_app.pivots_manager.models import PivotMap
 from api_app.playbooks_manager.models import PlaybookConfig
 from api_app.serializers import AbstractBIInterface
 from api_app.serializers.report import AbstractReportSerializerInterface
@@ -97,6 +98,7 @@ class Meta:
             "scan_mode",
             "scan_check_time",
             "investigation",
+            "parent_job",
         )
 
     md5 = rfs.HiddenField(default=None)
@@ -116,6 +118,7 @@ class Meta:
     investigation = rfs.PrimaryKeyRelatedField(
         queryset=Investigation.objects.all(), many=False, required=False, default=None
     )
+    parent_job = rfs.PrimaryKeyRelatedField(queryset=Job.objects.all(), required=False)
     connectors_requested = rfs.SlugRelatedField(
         slug_field="name",
         queryset=ConnectorConfig.objects.all(),
@@ -340,6 +343,7 @@ def check_previous_jobs(self, validated_data: Dict) -> Job:
     def create(self, validated_data: Dict) -> Job:
         warnings = validated_data.pop("warnings")
         send_task = validated_data.pop("send_task", False)
+        parent_job = validated_data.pop("parent_job", None)
         if validated_data["scan_mode"] == ScanMode.CHECK_PREVIOUS_ANALYSIS.value:
             try:
                 return self.check_previous_jobs(validated_data)
@@ -350,6 +354,10 @@ def create(self, validated_data: Dict) -> Job:
         job.warnings = warnings
         job.save()
         logger.info(f"Job {job.pk} created")
+        if parent_job:
+            PivotMap.objects.create(
+                starting_job=validated_data["parent"], ending_job=job, pivot_config=None
+            )
         if send_task:
             from intel_owl.tasks import job_pipeline
 
@@ -505,12 +513,18 @@ class Meta:
     )
     playbook_requested = rfs.SlugRelatedField(read_only=True, slug_field="name")
     playbook_to_execute = rfs.SlugRelatedField(read_only=True, slug_field="name")
+    investigation = rfs.SerializerMethodField(read_only=True, default=None)
     permissions = rfs.SerializerMethodField()
 
     def get_pivots_to_execute(self, obj: Job):  # skipcq: PYL-R0201
         # this cast is required or serializer doesn't work with websocket
         return list(obj.pivots_to_execute.all().values_list("name", flat=True))
 
+    def get_investigation(self, instance: Job):  # skipcq: PYL-R0201
+        if root_investigation := instance.get_root().investigation:
+            return root_investigation.pk
+        return instance.investigation
+
     def get_fields(self):
         # this method override is required for a cyclic import
         from api_app.analyzers_manager.serializers import AnalyzerReportSerializer
@@ -566,6 +580,8 @@ def save(self, parent: Job = None, **kwargs):
             # the parent has already an investigation
             # so we don't need to do anything because everything is already connected
             if parent.investigation:
+                parent.investigation.status = parent.investigation.Status.RUNNING.value
+                parent.investigation.save()
                 return jobs
             # if we have a parent, it means we are pivoting from one job to another
             else:
@@ -924,10 +940,7 @@ class JobResponseSerializer(rfs.ModelSerializer):
         source="playbook_to_execute",
         slug_field="name",
     )
-    investigation = rfs.SlugRelatedField(
-        read_only=True,
-        slug_field="pk",
-    )
+    investigation = rfs.SerializerMethodField(read_only=True, default=None)
 
     class Meta:
         model = Job
@@ -942,6 +955,11 @@ class Meta:
         extra_kwargs = {"warnings": {"read_only": True, "required": False}}
         list_serializer_class = JobEnvelopeSerializer
 
+    def get_investigation(self, instance: Job):  # skipcq: PYL-R0201
+        if root_investigation := instance.get_root().investigation:
+            return root_investigation.pk
+        return instance.investigation
+
     def to_representation(self, instance: Job):
         result = super().to_representation(instance)
         result["status"] = self.STATUS_ACCEPTED

api_app/views.py

@@ -232,7 +232,8 @@ def analyze_multiple_observables(request):
         data=request.data, many=True, context={"request": request}
     )
     oas.is_valid(raise_exception=True)
-    jobs = oas.save(send_task=True)
+    parent_job = oas.validated_data[0].get("parent_job", None)
+    jobs = oas.save(send_task=True, parent=parent_job)
     jrs = JobResponseSerializer(jobs, many=True).data
     logger.info(f"finished analyze_multiple_observables from user {request.user}")
     return Response(

docs/source/Contribute.md

@@ -259,7 +259,7 @@ After having written the new python module, you have to remember to:
    6. *Maximum tlp: maximum tlp to allow the run on the connector
    7. *Run on failure: if the connector should be run even if the job fails
 
-### Hot to add a new Ingestor
+### How to add a new Ingestor
 1. Put the module in the `ingestors` directory
 2. Remember to use `_monkeypatch()` in its class to create automated tests for the new ingestor. This is a trick to have tests in the same class of its ingestor.
 3. Create the configuration inside django admin in `Ingestors_manager/IngestorConfigs` (* = mandatory, ~ = mandatory on conditions)
@@ -271,6 +271,17 @@ After having written the new python module, you have to remember to:
    6. *Playbook to Execute: Playbook that **will** be executed on every IOC retrieved
    7. *Schedule: Crontab object that describes the schedule of the ingestor. You are able to create a new clicking the `plus` symbol.
 
+### How to add a new Pivot
+1. Put the module in the `pivots` directory
+2. Remember to use `_monkeypatch()` in its class to create automated tests for the new pivot. This is a trick to have tests in the same class of its pivot.
+3. Create the configuration inside django admin in `Pivots_manager/PivotConfigs` (* = mandatory, ~ = mandatory on conditions)
+   1. *Name: specific name of the configuration
+   2. *Python module: <module_name>.<class_name>
+   3. *Description: description of the configuration
+   4. *Routing key: celery queue that will be used
+   5. *Soft_time_limit: maximum time for the task execution
+   6. *Playbook to Execute: Playbook that **will** be executed in the Job generated by the Pivot
+
 ### How to add a new Visualizer
 
 #### Configuration

docs/source/Usage.md

@@ -309,11 +309,29 @@ Pivots are designed to create a job from another job. This plugin allows the use
 
 This is a "SOAR" feature that allows the users to connect multiple analysis together.
 
-Right now the support for this kind of plugin in the GUI is very limited, while the backend is fully operative. We are working on the frontend.
-
 #### List of pre-built Pivots
 None
 
+You can build your own custom Pivot with your custom logic with just few lines of code. See the [Contribute](https://intelowl.readthedocs.io/en/latest/Contribute.html#how-to-add-a-new-pivot) section for more info.
+
+#### Creating Pivots from the GUI
+
+From the GUI, the users can pivot in two ways:
+- If a Job executed a [Visualizer](#visualizers), it is possible to select a field extracted and analyze its value by clicking the "Pivot" button (see following image). In this way, the user is able to "jump" from one indicator to another.
+![img.png](../static/pivot_job_report.png)
+
+- Starting from an already existing [Investigation](#investigations-framework), it is possible to select a Job block and click the "Pivot" button to analyze the same observable again, usually choosing another [Playbook](#playbooks) (see following image)
+![img.png](../static/pivot_investigation_report.png)
+
+In both cases, the user is redirected to the Scan Page that is precompiled with the observable selected. Then the user would be able to select the [Playbook](#playbooks) to execute in the new job.
+After the new Job is started, a new [Investigation](#investigations-framework) will be created (if it does not already exist) and both the jobs will be added to the same Investigation.
+
+In the following image you can find an example of an [Investigation](#investigations-framework) composed by 3 pivots generated manually:
+* leveraging the first way to create a Pivot, the 2 Jobs that analyzed IP addresses have been generated from the first `test\.com` Job
+* leveraging the second way to create a Pivot, the second `test\.com` analysis had been created with a different Playbook.
+
+![img.png](../static/pivot_investigation.png)
+
 ### Visualizers
 
 With IntelOwl v5 we introduced a new plugin type called **Visualizers**.
@@ -570,7 +588,7 @@ Things to know about the framework:
 *Investigations* are created in 2 ways:
   * automatically:
     * if you scan multiple observables at the same time, a new investigation will be created by default and all the observables they will be automatically connected to the same investigation.
-    * if you run a Job with a Playbook which contains a Pivot that triggers another Job, a new investigation will be created and both the Jobs will be added to the same investigation.
+    * if you run a Job with a Playbook which contains a [Pivot](#pivots) that triggers another Job, a new investigation will be created and both the Jobs will be added to the same investigation. See how you can create a new [Pivot manually from the GUI](#creating-pivots-from-the-gui).
   * manually: by clicking on the button in the "History" section you can create an Investigation from scratch without any job attached (see following image)
 
 ![img.png](../static/create_investigation.png)

frontend/src/components/investigations/flow/CustomJobNode.jsx

@@ -4,6 +4,7 @@ import { NodeToolbar, Handle, Position } from "reactflow";
 import "reactflow/dist/style.css";
 import { Button } from "reactstrap";
 import { AiOutlineLink } from "react-icons/ai";
+import { LuGitBranchPlus } from "react-icons/lu";
 
 import { RemoveJob } from "./investigationActions";
 
@@ -32,6 +33,15 @@ function CustomJobNode({ data }) {
           >
             <AiOutlineLink /> Link
           </Button>
+          <Button
+            className="mx-1 p-2"
+            size="sm"
+            href={`/scan?parent=${data.id}&observable=${data.name}`}
+            target="_blank"
+            rel="noreferrer"
+          >
+            <LuGitBranchPlus /> Pivot
+          </Button>
           {data.isFirstLevel && <RemoveJob data={data} />}
         </div>
         <div

frontend/src/components/jobs/result/visualizer/VisualizerTooltip.jsx

@@ -4,6 +4,11 @@ import { CopyToClipboardButton } from "@certego/certego-ui";
 import { Button, UncontrolledTooltip } from "reactstrap";
 import { MdContentCopy } from "react-icons/md";
 import { AiOutlineLink } from "react-icons/ai";
+import { LuGitBranchPlus } from "react-icons/lu";
+import { useLocation } from "react-router-dom";
+
+import { ObservableClassifications } from "../../../../constants/jobConst";
+import { getObservableClassification } from "../../../../utils/observables";
 
 export function VisualizerTooltip({
   idElement,
@@ -12,6 +17,10 @@ export function VisualizerTooltip({
   description,
   disable,
 }) {
+  const location = useLocation();
+  const jobId = location?.pathname.split("/")[2];
+  const textClassification = getObservableClassification(copyText);
+
   return (
     <UncontrolledTooltip target={idElement} placement="right" autohide={false}>
       <div className="p-0 my-2 d-flex justify-content-start">
@@ -32,6 +41,17 @@ export function VisualizerTooltip({
         >
           <AiOutlineLink /> Link
         </Button>
+        {textClassification !== ObservableClassifications.GENERIC && (
+          <Button
+            className="mx-1 p-2"
+            size="sm"
+            href={`/scan?parent=${jobId}&observable=${copyText}`}
+            target="_blank"
+            rel="noreferrer"
+          >
+            <LuGitBranchPlus /> Pivot
+          </Button>
+        )}
       </div>
       {description && (
         <div

frontend/src/components/scan/ScanForm.jsx

@@ -82,6 +82,7 @@ export default function ScanForm() {
   const [searchParams, _] = useSearchParams();
   const observableParam = searchParams.get(JobTypes.OBSERVABLE);
   const investigationIdParam = searchParams.get("investigation") || null;
+  const parentIdParam = searchParams.get("parent");
   const { guideState, setGuideState } = useGuideContext();
 
   const { pluginsState: organizationPluginsState } = useOrganizationStore(
@@ -202,6 +203,7 @@ export default function ScanForm() {
         values.scan_mode,
         values.scan_check_time,
         investigationIdParam,
+        parentIdParam,
       );
 
       // multiple job or investigation id in GET param

frontend/src/components/scan/scanApi.jsx

@@ -33,6 +33,7 @@ function createJobPayload(
   _scanMode,
   scanCheckTime,
   investigationIdParam,
+  parentIdParam,
 ) {
   let payload = {};
   /* we add a custom function to the object to reuse the code:
@@ -105,6 +106,10 @@ function createJobPayload(
   if (investigationIdParam) {
     payload.append("investigation", investigationIdParam);
   }
+  // parent id in param
+  if (parentIdParam) {
+    payload.append("parent_job", parentIdParam);
+  }
   // remove custom method in order to avoid to send it to the backend
   if (!isSample) delete payload.append;
   console.debug("job request params:");
@@ -162,6 +167,7 @@ export async function createJob(
   _scanMode,
   scanCheckTime,
   investigationIdParam,
+  parentIdParam,
 ) {
   try {
     console.debug(
@@ -202,6 +208,7 @@ export async function createJob(
       _scanMode,
       scanCheckTime,
       investigationIdParam,
+      parentIdParam,
     );
     const resp = await axios.post(apiUrl, payload, {
       headers: {

frontend/tests/components/investigations/flow/InvestigationFlow.test.jsx

@@ -298,8 +298,14 @@ describe("test InvestigationFlow", () => {
     expect(removeJobButton).toBeInTheDocument();
     const linkFirstJobButton = screen.getByRole("link", { name: "Link" });
     expect(linkFirstJobButton).toBeInTheDocument();
+    const firstJobPivotButton = screen.getByRole("link", { name: "Pivot" });
+    expect(firstJobPivotButton).toBeInTheDocument();
     // link to job page
     expect(linkFirstJobButton.href).toContain("/jobs/10/visualizer");
+    // link pivot
+    expect(firstJobPivotButton.href).toContain(
+      "/scan?parent=10&observable=test1.com",
+    );
     // job info
     const jobInfo = container.querySelector("#job10-info");
     expect(jobInfo).toBeInTheDocument();
@@ -317,8 +323,14 @@ describe("test InvestigationFlow", () => {
     expect(removeSecondJobButton).toBeNull(); // no remove button in pivot
     const linkSecondJobButton = screen.getByRole("link", { name: "Link" });
     expect(linkSecondJobButton).toBeInTheDocument();
+    const secondJobPivotButton = screen.getByRole("link", { name: "Pivot" });
+    expect(secondJobPivotButton).toBeInTheDocument();
     // link to job page
     expect(linkSecondJobButton.href).toContain("/jobs/11/visualizer");
+    // link pivot
+    expect(secondJobPivotButton.href).toContain(
+      "/scan?parent=11&observable=test11.com",
+    );
     // job info
     const secondJobInfo = container.querySelector("#job11-info");
     expect(secondJobInfo).toBeInTheDocument();

frontend/tests/components/jobs/result/visualizer/elements/base.test.jsx

@@ -5,6 +5,14 @@ import userEvent from "@testing-library/user-event";
 import { BaseVisualizer } from "../../../../../../src/components/jobs/result/visualizer/elements/base";
 import { getIcon } from "../../../../../../src/components/jobs/result/visualizer/icons";
 
+// mock useLocation
+jest.mock("react-router-dom", () => ({
+  ...jest.requireActual("react-router-dom"),
+  useLocation: () => ({
+    pathname: "localhost/jobs/123/visualizer",
+  }),
+}));
+
 describe("BaseVisualizer component", () => {
   test("required-only params", async () => {
     const { container } = render(

frontend/tests/components/jobs/result/visualizer/elements/bool.test.jsx

@@ -5,6 +5,14 @@ import userEvent from "@testing-library/user-event";
 import { getIcon } from "../../../../../../src/components/jobs/result/visualizer/icons";
 import { BooleanVisualizer } from "../../../../../../src/components/jobs/result/visualizer/elements/bool";
 
+// mock useLocation
+jest.mock("react-router-dom", () => ({
+  ...jest.requireActual("react-router-dom"),
+  useLocation: () => ({
+    pathname: "localhost/jobs/123/visualizer",
+  }),
+}));
+
 describe("BooleanVisualizer component", () => {
   test("required-only params", async () => {
     const { container } = render(

frontend/tests/components/jobs/result/visualizer/elements/horizontalList.test.jsx

@@ -7,6 +7,14 @@ import { BooleanVisualizer } from "../../../../../../src/components/jobs/result/
 import { TitleVisualizer } from "../../../../../../src/components/jobs/result/visualizer/elements/title";
 import { VerticalListVisualizer } from "../../../../../../src/components/jobs/result/visualizer/elements/verticalList";
 
+// mock useLocation
+jest.mock("react-router-dom", () => ({
+  ...jest.requireActual("react-router-dom"),
+  useLocation: () => ({
+    pathname: "localhost/jobs/123/visualizer",
+  }),
+}));
+
 describe("HorizontalListVisualizer component", () => {
   test("required-only params", () => {
     const { container } = render(

frontend/tests/components/jobs/result/visualizer/elements/title.test.jsx

@@ -4,6 +4,14 @@ import { render, screen } from "@testing-library/react";
 import { TitleVisualizer } from "../../../../../../src/components/jobs/result/visualizer/elements/title";
 import { BaseVisualizer } from "../../../../../../src/components/jobs/result/visualizer/elements/base";
 
+// mock useLocation
+jest.mock("react-router-dom", () => ({
+  ...jest.requireActual("react-router-dom"),
+  useLocation: () => ({
+    pathname: "localhost/jobs/123/visualizer",
+  }),
+}));
+
 describe("TitleVisualizer component", () => {
   test("required-only params", () => {
     const { container } = render(