Fix issue with JobMetadata not longer supporting __setitem__ method (#…

…404)

spidermon/contrib/actions/jobs/tags.py

@@ -23,8 +23,9 @@ def run_action(self):
             if not self.data.job:
                 raise NotConfigured("Job not available!")
             job_metadata = self.data.job.metadata
-            self.process_tags(job_metadata)
-            job_metadata.save()
+            job_tags = job_metadata.get("tags")
+            self.process_tags(job_tags)
+            job_metadata.set("tags", job_tags)
 
     def process_tags(self, job_metadata):
         raise NotImplementedError
@@ -33,16 +34,13 @@ def process_tags(self, job_metadata):
 class AddJobTags(JobTagsAction):
     tag_settings = "SPIDERMON_JOB_TAGS_TO_ADD"
 
-    def process_tags(self, job_metadata):
-        for tag in self.tags:
-            if tag not in job_metadata["tags"]:
-                job_metadata["tags"].append(tag)
+    def process_tags(self, job_tags):
+        tags_to_add = [tag for tag in self.tags if tag not in job_tags]
+        job_tags += tags_to_add
 
 
 class RemoveJobTags(JobTagsAction):
     tag_settings = "SPIDERMON_JOB_TAGS_TO_REMOVE"
 
-    def process_tags(self, job_metadata):
-        for tag in self.tags:
-            if tag in job_metadata["tags"]:
-                job_metadata["tags"].remove(tag)
+    def process_tags(self, job_tags):
+        job_tags[:] = [tag for tag in job_tags if tag not in self.tags]

tests/contrib/actions/jobs/test_tags.py

@@ -0,0 +1,61 @@
+from unittest.mock import MagicMock, patch
+
+import pytest
+from scrapy.utils.test import get_crawler
+
+from spidermon.contrib.actions.jobs.tags import AddJobTags, JobTagsAction, RemoveJobTags
+from spidermon.exceptions import NotConfigured
+
+
+@pytest.fixture
+def test_settings():
+    return {
+        "SPIDERMON_JOB_TAGS_TO_ADD": ["add_foo", "add_bar"],
+        "SPIDERMON_JOB_TAGS_TO_REMOVE": ["remove_foo", "remove_bar"],
+    }
+
+
+class SettableDict(dict):
+    def set(self, key, value):
+        self[key] = value
+
+
+def test_run_action(test_settings):
+    crawler = get_crawler(settings_dict=test_settings)
+    job_tags_action = JobTagsAction.from_crawler(crawler)
+
+    job_tags_action.tags = MagicMock(return_value=["foo", "bar"])
+    job_tags_action.data = MagicMock()
+    job_tags_action.data.job = None
+    with pytest.raises(NotConfigured):
+        job_tags_action.run_action()
+
+    job_tags_action.data.job = MagicMock()
+    with pytest.raises(NotImplementedError):
+        job_tags_action.run_action()
+
+
+def test_add_job_tags(test_settings):
+    crawler = get_crawler(settings_dict=test_settings)
+    add_job_tags = AddJobTags.from_crawler(crawler)
+
+    assert add_job_tags.tags == ["add_foo", "add_bar"]
+
+    add_job_tags.data = MagicMock()
+    add_job_tags.data.job.metadata = SettableDict({"tags": []})
+    add_job_tags.run_action()
+    assert add_job_tags.data.job.metadata["tags"] == ["add_foo", "add_bar"]
+
+
+def test_remove_job_tags(test_settings):
+    crawler = get_crawler(settings_dict=test_settings)
+    remove_job_tags = RemoveJobTags.from_crawler(crawler)
+
+    assert remove_job_tags.tags == ["remove_foo", "remove_bar"]
+
+    remove_job_tags.data = MagicMock()
+    remove_job_tags.data.job.metadata = SettableDict(
+        {"tags": ["remove_foo", "remove_bar"]}
+    )
+    remove_job_tags.run_action()
+    assert remove_job_tags.data.job.metadata["tags"] == []