chore(ci_visibility): add unique tests request to test vis API client

ddtrace/internal/ci_visibility/constants.py

@@ -46,6 +46,7 @@
 GIT_API_BASE_PATH = "/api/v2/git"
 SETTING_ENDPOINT = "/api/v2/libraries/tests/services/setting"
 SKIPPABLE_ENDPOINT = "/api/v2/ci/tests/skippable"
+UNIQUE_TESTS_ENDPOINT = "/api/v2/ci/libraries/tests"
 
 # Intelligent Test Runner constants
 ITR_UNSKIPPABLE_REASON = "datadog_itr_unskippable"

ddtrace/internal/ci_visibility/recorder.py

@@ -195,6 +195,7 @@ def __init__(self, tracer=None, config=None, service=None):
         self._should_upload_git_metadata = True
         self._itr_meta = {}  # type: Dict[str, Any]
         self._itr_data: Optional[ITRData] = None
+        self._unique_tests: Set[InternalTestId] = set()
 
         self._session: Optional[TestVisibilitySession] = None
 
@@ -270,6 +271,10 @@ def __init__(self, tracer=None, config=None, service=None):
             self._api_settings.itr_enabled,
             self._api_settings.skipping_enabled,
         )
+        log.info(
+            "API-provided settings: early flake detection enabled: %s",
+            self._api_settings.early_flake_detection.enabled,
+        )
         log.info("Detected configurations: %s", str(self._configurations))
 
         try:

ddtrace/internal/ci_visibility/telemetry/api_request.py

@@ -0,0 +1,46 @@
+import dataclasses
+from typing import Optional
+
+from ddtrace.internal.ci_visibility.telemetry.constants import CIVISIBILITY_TELEMETRY_NAMESPACE as _NAMESPACE
+from ddtrace.internal.ci_visibility.telemetry.constants import ERROR_TYPES
+from ddtrace.internal.logger import get_logger
+from ddtrace.internal.telemetry import telemetry_writer
+
+
+log = get_logger(__name__)
+
+
+@dataclasses.dataclass(frozen=True)
+class APIRequestMetricNames:
+    count: str
+    duration: str
+    response_bytes: str
+    error: str
+
+
+def record_api_request(
+    metric_names: APIRequestMetricNames,
+    duration: float,
+    response_bytes: Optional[int] = None,
+    error: Optional[ERROR_TYPES] = None,
+):
+    log.debug(
+        "Recording early flake detection telemetry for %s: %s, %s, %s",
+        metric_names.count,
+        duration,
+        response_bytes,
+        error,
+    )
+
+    telemetry_writer.add_count_metric(_NAMESPACE, f"{metric_names.count}", 1)
+    telemetry_writer.add_distribution_metric(_NAMESPACE, f"{metric_names.duration}", duration)
+    if response_bytes is not None:
+        telemetry_writer.add_distribution_metric(_NAMESPACE, f"{metric_names.response_bytes}", response_bytes)
+
+    if error is not None:
+        record_api_request_error(metric_names.error, error)
+
+
+def record_api_request_error(error_metric_name: str, error: ERROR_TYPES):
+    log.debug("Recording early flake detection request error telemetry: %s", error)
+    telemetry_writer.add_count_metric(_NAMESPACE, error_metric_name, 1, (("error_type", error),))

ddtrace/internal/ci_visibility/telemetry/early_flake_detection.py

@@ -0,0 +1,24 @@
+from enum import Enum
+
+from ddtrace.internal.ci_visibility.telemetry.constants import CIVISIBILITY_TELEMETRY_NAMESPACE as _NAMESPACE
+from ddtrace.internal.logger import get_logger
+from ddtrace.internal.telemetry import telemetry_writer
+
+
+log = get_logger(__name__)
+
+EARLY_FLAKE_DETECTION_TELEMETRY_PREFIX = "early_flake_detection."
+RESPONSE_TESTS = f"{EARLY_FLAKE_DETECTION_TELEMETRY_PREFIX}response_tests"
+
+
+class EARLY_FLAKE_DETECTION_TELEMETRY(str, Enum):
+    REQUEST = "early_flake_detection.request"
+    REQUEST_MS = "early_flake_detection.request_ms"
+    REQUEST_ERRORS = "early_flake_detection.request_errors"
+    RESPONSE_BYTES = "early_flake_detection.response_bytes"
+    RESPONSE_TESTS = "early_flake_detection.response_tests"
+
+
+def record_early_flake_detection_tests_count(early_flake_detection_count: int):
+    log.debug("Recording early flake detection tests count telemetry: %s", early_flake_detection_count)
+    telemetry_writer.add_count_metric(_NAMESPACE, RESPONSE_TESTS, early_flake_detection_count)

ddtrace/internal/ci_visibility/telemetry/git.py

@@ -45,22 +45,20 @@ def record_objects_pack_data(num_files: int, num_bytes: int) -> None:
     telemetry_writer.add_distribution_metric(_NAMESPACE, GIT_TELEMETRY.OBJECTS_PACK_FILES, num_files)
 
 
-def record_settings(
-    duration: float,
+def record_settings_response(
     coverage_enabled: Optional[bool] = False,
     skipping_enabled: Optional[bool] = False,
     require_git: Optional[bool] = False,
     itr_enabled: Optional[bool] = False,
-    error: Optional[ERROR_TYPES] = None,
+    early_flake_detection_enabled: Optional[bool] = False,
 ) -> None:
     log.debug(
-        "Recording settings telemetry: %s, %s, %s, %s, %s, %s",
-        duration,
+        "Recording settings telemetry: %s, %s, %s, %s, %s",
         coverage_enabled,
         skipping_enabled,
         require_git,
         itr_enabled,
-        error,
+        early_flake_detection_enabled,
     )
     # Telemetry "booleans" are true if they exist, otherwise false
     response_tags = []
@@ -72,13 +70,8 @@ def record_settings(
         response_tags.append(("require_git", "1"))
     if itr_enabled:
         response_tags.append(("itrskip_enabled", "1"))
+    if early_flake_detection_enabled:
+        response_tags.append(("early_flake_detection_enabled", "1"))
 
-    telemetry_writer.add_count_metric(_NAMESPACE, GIT_TELEMETRY.SETTINGS_COUNT, 1)
-    telemetry_writer.add_distribution_metric(_NAMESPACE, GIT_TELEMETRY.SETTINGS_MS, duration)
-
-    telemetry_writer.add_count_metric(
-        _NAMESPACE, GIT_TELEMETRY.SETTINGS_RESPONSE, 1, tuple(response_tags) if response_tags else None
-    )
-    if error is not None:
-        error_tags = (("error", error),)
-        telemetry_writer.add_count_metric(_NAMESPACE, GIT_TELEMETRY.SETTINGS_ERRORS, 1, error_tags)
+    if response_tags:
+        telemetry_writer.add_count_metric(_NAMESPACE, GIT_TELEMETRY.SETTINGS_RESPONSE, 1, tuple(response_tags))

ddtrace/internal/ci_visibility/telemetry/itr.py

@@ -1,10 +1,8 @@
 from enum import Enum
 import functools
-from typing import Optional
 
 from ddtrace.internal.ci_visibility.constants import SUITE
 from ddtrace.internal.ci_visibility.telemetry.constants import CIVISIBILITY_TELEMETRY_NAMESPACE as _NAMESPACE
-from ddtrace.internal.ci_visibility.telemetry.constants import ERROR_TYPES
 from ddtrace.internal.ci_visibility.telemetry.constants import EVENT_TYPES
 from ddtrace.internal.logger import get_logger
 from ddtrace.internal.telemetry import telemetry_writer
@@ -63,37 +61,3 @@ def record_skippable_count(skippable_count: int, skipping_level: str):
         else SKIPPABLE_TESTS_TELEMETRY.RESPONSE_TESTS
     )
     telemetry_writer.add_count_metric(_NAMESPACE, skippable_count_metric, skippable_count)
-
-
-def record_itr_skippable_request_error(error: ERROR_TYPES):
-    log.debug("Recording itr skippable request error telemetry")
-    telemetry_writer.add_count_metric(_NAMESPACE, SKIPPABLE_TESTS_TELEMETRY.REQUEST_ERRORS, 1, (("error_type", error),))
-
-
-def record_itr_skippable_request(
-    duration: float,
-    response_bytes: int,
-    skipping_level: str,
-    skippable_count: Optional[int] = None,
-    error: Optional[ERROR_TYPES] = None,
-):
-    log.debug(
-        "Recording itr skippable request telemetry: %s, %s, %s, %s, %s",
-        duration,
-        response_bytes,
-        skippable_count,
-        skipping_level,
-        error,
-    )
-
-    telemetry_writer.add_count_metric(_NAMESPACE, SKIPPABLE_TESTS_TELEMETRY.REQUEST, 1)
-    telemetry_writer.add_distribution_metric(_NAMESPACE, SKIPPABLE_TESTS_TELEMETRY.REQUEST_MS, duration)
-    telemetry_writer.add_distribution_metric(_NAMESPACE, SKIPPABLE_TESTS_TELEMETRY.RESPONSE_BYTES, response_bytes)
-
-    if error is not None:
-        record_itr_skippable_request_error(error)
-        # If there was an error, assume no skippable items can be counted
-        return
-
-    if skippable_count is not None:
-        record_skippable_count(skippable_count, skipping_level)

tests/ci_visibility/api_client/_util.py

@@ -45,38 +45,66 @@ def _get_setting_api_response(
     require_git=False,
     itr_enabled=False,
     flaky_test_retries_enabled=False,
+    efd_present=False,  # This controls whether a default EFD response is present (instead of only {"enabled": false}
     efd_detection_enabled=False,
     efd_5s=10,
     efd_10s=5,
     efd_30s=3,
     efd_5m=2,
     efd_session_threshold=30.0,
 ):
+    body = {
+        "data": {
+            "id": "1234",
+            "type": "ci_app_tracers_test_service_settings",
+            "attributes": {
+                "code_coverage": code_coverage,
+                "early_flake_detection": {
+                    "enabled": False,
+                },
+                "flaky_test_retries_enabled": flaky_test_retries_enabled,
+                "itr_enabled": itr_enabled,
+                "require_git": require_git,
+                "tests_skipping": tests_skipping,
+            },
+        }
+    }
+
+    if efd_present or efd_detection_enabled:
+        body["data"]["attributes"]["early_flake_detection"].update(
+            {
+                "enabled": efd_detection_enabled,
+                "slow_test_retries": {"10s": efd_10s, "30s": efd_30s, "5m": efd_5m, "5s": efd_5s},
+                "faulty_session_threshold": efd_session_threshold,
+            }
+        )
+
+    return Response(status=status_code, body=json.dumps(body))
+
+
+def _get_skippable_api_response():
     return Response(
-        status=status_code,
-        body=json.dumps(
+        200,
+        json.dumps(
             {
-                "data": {
-                    "id": "1234",
-                    "type": "ci_app_tracers_test_service_settings",
-                    "attributes": {
-                        "code_coverage": code_coverage,
-                        "early_flake_detection": {
-                            "enabled": efd_detection_enabled,
-                            "slow_test_retries": {"10s": efd_10s, "30s": efd_30s, "5m": efd_5m, "5s": efd_5s},
-                            "faulty_session_threshold": efd_session_threshold,
-                        },
-                        "flaky_test_retries_enabled": flaky_test_retries_enabled,
-                        "itr_enabled": itr_enabled,
-                        "require_git": require_git,
-                        "tests_skipping": tests_skipping,
-                    },
-                }
+                "data": [],
+                "meta": {
+                    "correlation_id": "1234ideclareacorrelationid",
+                },
             }
         ),
     )
 
 
+def _get_tests_api_response(tests_body: t.Optional[t.Dict] = None):
+    response = {"data": {"id": "J0ucvcSApX8", "type": "ci_app_libraries_tests", "attributes": {"tests": {}}}}
+
+    if tests_body is not None:
+        response["data"]["attributes"]["tests"].update(tests_body)
+
+    return Response(200, json.dumps(response))
+
+
 def _make_fqdn_internal_test_id(module_name: str, suite_name: str, test_name: str, parameters: t.Optional[str] = None):
     """An easy way to create a test id "from the bottom up"
 
@@ -164,7 +192,7 @@ def _get_test_client(
                 client_timeout,
             )
 
-    def _get_expected_do_request_payload(
+    def _get_expected_do_request_setting_payload(
         self,
         itr_skipping_level: ITR_SKIPPING_LEVEL = ITR_SKIPPING_LEVEL.TEST,
         git_data: GitData = None,
@@ -195,6 +223,64 @@ def _get_expected_do_request_payload(
             },
         }
 
+    def _get_expected_do_request_skippable_payload(
+        self,
+        itr_skipping_level: ITR_SKIPPING_LEVEL = ITR_SKIPPING_LEVEL.TEST,
+        git_data: GitData = None,
+        dd_service: t.Optional[str] = None,
+        dd_env: t.Optional[str] = None,
+    ):
+        git_data = self.default_git_data if git_data is None else git_data
+
+        return {
+            "data": {
+                "id": "checkoutmyuuid4",
+                "type": "test_params",
+                "attributes": {
+                    "test_level": "test" if itr_skipping_level == ITR_SKIPPING_LEVEL.TEST else "suite",
+                    "service": dd_service,
+                    "env": dd_env,
+                    "repository_url": git_data.repository_url,
+                    "sha": git_data.commit_sha,
+                    "configurations": {
+                        "os.architecture": "arm64",
+                        "os.platform": "PlatForm",
+                        "os.version": "9.8.a.b",
+                        "runtime.name": "RPython",
+                        "runtime.version": "11.5.2",
+                    },
+                },
+            },
+        }
+
+    def _get_expected_do_request_tests_payload(
+        self,
+        repository_url: str = None,
+        dd_service: t.Optional[str] = None,
+        dd_env: t.Optional[str] = None,
+    ):
+        if repository_url is None:
+            repository_url = self.default_git_data.repository_url
+
+        return {
+            "data": {
+                "id": "checkoutmyuuid4",
+                "type": "ci_app_libraries_tests_request",
+                "attributes": {
+                    "service": dd_service,
+                    "env": dd_env,
+                    "repository_url": repository_url,
+                    "configurations": {
+                        "os.architecture": "arm64",
+                        "os.platform": "PlatForm",
+                        "os.version": "9.8.a.b",
+                        "runtime.name": "RPython",
+                        "runtime.version": "11.5.2",
+                    },
+                },
+            },
+        }
+
     @staticmethod
     def _get_mock_civisibility(requests_mode, suite_skipping_mode):
         with mock.patch.object(CIVisibility, "__init__", return_value=None):
@@ -234,5 +320,5 @@ def _get_mock_civisibility(requests_mode, suite_skipping_mode):
     def _test_context_manager(self):
         with mock.patch("ddtrace.internal.ci_visibility._api_client.uuid4", return_value="checkoutmyuuid4"), mock.patch(
             "ddtrace.internal.ci_visibility._api_client.DEFAULT_TIMEOUT", 12.34
-        ):
+        ), mock.patch("ddtrace.internal.ci_visibility._api_client.DEFAULT_ITR_SKIPPABLE_TIMEOUT", 43.21):
             yield