Merge pull request #45 from chaws/find-test-attachment-from-tuxsuite

tradefed: support fetching tradefed file from tuxsuite

test/test_tradefed.py

@@ -493,7 +493,7 @@ def tearDown(self):
     @patch("tradefed.Tradefed._get_from_artifactorial")
     @patch("tradefed.update_testjob_status.delay")
     @patch("tradefed.Tradefed.tradefed_results_url", new_callable=PropertyMock)
-    def test_postprocess_testjob(
+    def test_postprocess_testjob_lava(
         self,
         results_url_mock,
         update_testjob_status_mock,
@@ -528,6 +528,48 @@ def test_postprocess_testjob(
         create_testrun_attachment_mock.assert_not_called()
         update_testjob_status_mock.assert_called()
 
+    @patch("tradefed.Tradefed._get_tradefed_url_from_tuxsuite")
+    @patch("tradefed.Tradefed._download_results")
+    @patch("tradefed.Tradefed._extract_cts_results")
+    @patch("tradefed.Tradefed._create_testrun_attachment")
+    @patch("tradefed.Tradefed._assign_test_log")
+    @patch("tradefed.update_testjob_status.delay")
+    @patch("tradefed.Tradefed.tradefed_results_url", new_callable=PropertyMock)
+    def test_postprocess_testjob_tuxsuite(
+        self,
+        results_url_mock,
+        update_testjob_status_mock,
+        assign_test_log_mock,
+        create_testrun_attachment_mock,
+        extract_cts_results_mock,
+        download_results_mock,
+        get_tradefed_url_from_tuxsuite_mock,
+    ):
+        results_url_mock.return_value = "http://foo.com"
+        download_results_mock.return_value = ResultFiles()
+        testjob_mock = MagicMock()
+        id_mock = PropertyMock(return_value="999111")
+        type(testjob_mock).pk = id_mock
+        job_id_mock = PropertyMock(return_value="1234")
+        type(testjob_mock).job_id = job_id_mock
+        testjob_mock.backend = MagicMock()
+        implementation_type_mock = PropertyMock(return_value="tuxsuite")
+        type(testjob_mock.backend).implementation_type = implementation_type_mock
+        testjob_target = MagicMock()
+        project_settings_mock = PropertyMock(return_value='{}')
+        type(testjob_target).project_settings = project_settings_mock
+        type(testjob_mock).target = testjob_target
+        self.plugin.postprocess_testjob(testjob_mock)
+        implementation_type_mock.assert_called_once_with()
+        results_url_mock.assert_called_with()
+        testjob_mock.testrun.metadata.__setitem__.assert_called_with('tradefed_results_url_1234', 'http://foo.com')
+        testjob_mock.testrun.save.assert_called_with()
+        assign_test_log_mock.assert_not_called()
+        create_testrun_attachment_mock.assert_not_called()
+        update_testjob_status_mock.assert_called()
+        get_tradefed_url_from_tuxsuite_mock.assert_called()
+        download_results_mock.assert_called()
+
     @patch("tradefed.Tradefed._create_testrun_attachment")
     @patch("tradefed.Tradefed._assign_test_log")
     @patch("tradefed.Tradefed._get_from_artifactorial")
@@ -1169,3 +1211,87 @@ def test_extract_tarball_filename_from_url(self):
 
         filename = self.plugin._extract_tarball_filename_from_url("http://some.url/?param1=val1&filename=tradefed.tar.xz")
         self.assertEqual("tradefed.tar.xz", filename)
+
+    def test_get_tradefed_url_from_tuxsuite_bad_request(self):
+        job_url = "http://tuxsuite.com/job#123"
+
+        testjob = MagicMock()
+        testjob.url = job_url
+
+        with requests_mock.Mocker() as fake_request:
+            fake_request.get(
+                f"{job_url}/results",
+                status_code=404,
+            )
+
+            url = self.plugin._get_tradefed_url_from_tuxsuite(testjob)
+            self.assertIsNone(url)
+
+    def test_get_tradefed_url_from_tuxsuite_errored_json(self):
+        job_url = "http://tuxsuite.com/job#123"
+
+        testjob = MagicMock()
+        testjob.url = job_url
+
+        with requests_mock.Mocker() as fake_request:
+            fake_request.get(
+                f"{job_url}/results",
+                status_code=200,
+                content=b'{"error": "some error"}',
+                headers={"Content-Type": "application/json"},
+            )
+
+            url = self.plugin._get_tradefed_url_from_tuxsuite(testjob)
+            self.assertIsNone(url)
+
+    def test_get_tradefed_url_from_tuxsuite_no_test_attachment(self):
+        job_url = "http://tuxsuite.com/job#123"
+
+        testjob = MagicMock()
+        testjob.url = job_url
+
+        with requests_mock.Mocker() as fake_request:
+            fake_request.get(
+                f"{job_url}/results",
+                status_code=200,
+                content=b'{"suite": {"test1": {"result": "pass"}}}',
+                headers={"Content-Type": "application/json"},
+            )
+
+            url = self.plugin._get_tradefed_url_from_tuxsuite(testjob)
+            self.assertIsNone(url)
+
+    def test_get_tradefed_url_from_tuxsuite_no_reference(self):
+        job_url = "http://tuxsuite.com/job#123"
+
+        testjob = MagicMock()
+        testjob.url = job_url
+
+        with requests_mock.Mocker() as fake_request:
+            fake_request.get(
+                f"{job_url}/results",
+                status_code=200,
+                content=b'{"suite": {"test-attachment": {"result": "pass"}}}',
+                headers={"Content-Type": "application/json"},
+            )
+
+            url = self.plugin._get_tradefed_url_from_tuxsuite(testjob)
+            self.assertIsNone(url)
+
+    def test_get_tradefed_url_from_tuxsuite(self):
+        job_url = "http://tuxsuite.com/job#123"
+        tradefed_url = "http://tradefed.url"
+
+        testjob = MagicMock()
+        testjob.url = job_url
+
+        with requests_mock.Mocker() as fake_request:
+            fake_request.get(
+                f"{job_url}/results",
+                status_code=200,
+                content=b'{"suite": {"test-attachment": {"result": "pass", "reference": "http://tradefed.url"}}}',
+                headers={"Content-Type": "application/json"},
+            )
+
+            url = self.plugin._get_tradefed_url_from_tuxsuite(testjob)
+            self.assertEqual(tradefed_url, url)

tradefed/__init__.py

@@ -550,34 +550,91 @@ def _extract_tradefed_from_job_definition(self, testjob):
 
         return tradefed_files
 
+    def _get_tradefed_url_from_tuxsuite(self, testjob):
+        """
+        Android jobs coming from Tuxsuite should have a test names "test-attachment"
+        and it should have a reference attached to it, same as regular LAVA.
+
+        The test looks like this:
+
+        {
+            'test-attachment': {
+                'reference': 'https://qa-reports.linaro.org/api/testruns/23505462/attachments/?filename=tradefed-output-20240417085054.tar.xz',
+                'result': 'pass'
+            }
+        }
+
+        The tricky part is that SQUAD only reads "result" and disregard "reference".
+        So Tradefed plugin needs to request results once more from Tuxsuite just to
+        get the "reference" value.
+        """
+
+        session = get_session()
+        results_url = f"{testjob.url}/results"
+        response = session.get(results_url)
+        if response.status_code != 200:
+            logger.error(f"Failed to retrieve results from Tuxsuite: {response.content}")
+            return None
+
+        results = response.json()
+        error = results.get("error", None)
+        if error is not None:
+            logger.error(f"Failed to retrieve results from Tuxsuite: {error}")
+            return None
+
+        for suite, suite_tests in results.items():
+            test = suite_tests.get("test-attachment")
+            if test:
+
+                reference = test.get("reference")
+                if reference is None:
+                    logger.error("Failed to retrieve results from Tuxsuite: there is no 'reference' in 'test-attachment'")
+                    return None
+
+                return reference
+
+        logger.info("No 'test-attachment' found for this Tuxsuite job")
+        return None
+
     def postprocess_testjob(self, testjob):
         self.extra_args["job_id"] = testjob.id
 
         logger.info("Starting CTS/VTS plugin for test job: %s" % testjob)
-        if not testjob.backend.implementation_type == 'lava':
-            logger.error(f"Test job {testjob.id} doesn't come from LAVA")
+        backend_type = testjob.backend.implementation_type
+        if backend_type not in ['lava', 'tuxsuite']:
+            logger.error(f"Test job {testjob.id} does not come from LAVA nor Tuxsuite")
             update_testjob_status.delay(testjob.id, self.extra_args.get("job_status"))
             return
 
-        tradefed_files = self._extract_tradefed_from_job_definition(testjob)
-        if len(tradefed_files) != 1:
-            logger.info(f"Job {testjob.id} has {len(tradefed_files)} tradefed files in the definition, it should have 1, aborting")
-            update_testjob_status.delay(testjob.id, self.extra_args.get("job_status"))
-            logger.info("Finishing CTS/VTS plugin for test run: %s" % testjob)
-            return
-
-        tradefed_name, results_format = tradefed_files[0]
-
-        results_extracted = False
         results = None
-        try:
-            results = self._get_from_artifactorial(testjob, tradefed_name)
-        except xmlrpc.client.ProtocolError as err:
-            error_cleaned = 'Failed to process CTS/VTS tests: %s - %s' % (err.errcode, testjob.backend.get_implementation().url_remove_token(str(err.errmsg)))
-            logger.error(error_cleaned)
+        if backend_type == 'lava':
+            tradefed_files = self._extract_tradefed_from_job_definition(testjob)
+            if len(tradefed_files) != 1:
+                logger.error(f"Job {testjob.id} has {len(tradefed_files)} tradefed files in the definition, it should have 1")
+                update_testjob_status.delay(testjob.id, self.extra_args.get("job_status"))
+                logger.info("Finishing CTS/VTS plugin for test run: %s" % testjob)
+                return
+
+            try:
+                tradefed_name, results_format = tradefed_files[0]
+                results = self._get_from_artifactorial(testjob, tradefed_name)
+            except xmlrpc.client.ProtocolError as err:
+                error_cleaned = 'Failed to process CTS/VTS tests: %s - %s' % (err.errcode, testjob.backend.get_implementation().url_remove_token(str(err.errmsg)))
+                logger.error(error_cleaned)
+
+                testjob.failure += error_cleaned
+                testjob.save()
+        else:
+            # Get it from Tuxsuite
+            url = self._get_tradefed_url_from_tuxsuite(testjob)
+            if url is None:
+                logger.info("Aborting CTS/VTS, no tradefed URL found within Tuxsuite results")
+                update_testjob_status.delay(testjob.id, self.extra_args.get("job_status"))
+                return
 
-            testjob.failure += error_cleaned
-            testjob.save()
+            results = self._download_results(url)
+            tradefed_name = "vts-lkft"
+            results_format = "aggregated"
 
         if results is None:
             logger.info("Aborting CTS/VTS, no tradefed file found")
@@ -588,6 +645,7 @@ def postprocess_testjob(self, testjob):
         testjob.testrun.metadata["tradefed_results_url_%s" % testjob.job_id] = self.tradefed_results_url
         testjob.testrun.save()
 
+        results_extracted = False
         if results.test_results is not None:
             if testjob.target.get_setting("PLUGINS_TRADEFED_EXTRACT_AGGREGATED", False) and results_format == "aggregated":
                 self._extract_cts_results(results.test_results.contents, testjob.testrun, tradefed_name)