Merge pull request #2980 from DataDog/zach.montoya/config-error-statu…

…ses/apmapi-349

[Tracing] Add feature/scenarios for tracing configuration consistency and implement tests for DD_TRACE_HTTP_SERVER_ERROR_STATUSES

.github/workflows/run-end-to-end.yml

@@ -145,6 +145,11 @@ jobs:
       run: ./run.sh LIBRARY_CONF_CUSTOM_HEADER_TAGS_INVALID
       env:
         DD_API_KEY: ${{ secrets.DD_API_KEY }}
+    - name: Run TRACING_CONFIG_NONDEFAULT scenario
+      if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"TRACING_CONFIG_NONDEFAULT"')
+      run: ./run.sh TRACING_CONFIG_NONDEFAULT
+      env:
+        DD_API_KEY: ${{ secrets.DD_API_KEY }}
     - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES scenario
       if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES"')
       run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES

.vscode/launch.json

@@ -54,6 +54,16 @@
             "justMyCode": true,
             "python": "${workspaceFolder}/venv/bin/python"
         },
+        {
+            "name": "Run TRACING_CONFIG_NONDEFAULT scenario",
+            "type": "python",
+            "request": "launch",
+            "module": "pytest",
+            "args": ["-S", "TRACING_CONFIG_NONDEFAULT", "-p", "no:warnings"],
+            "console": "integratedTerminal",
+            "justMyCode": true,
+            "python": "${workspaceFolder}/venv/bin/python"
+        },
         {
             "name": "Run PROFILING scenario",
             "type": "python",

manifests/cpp.yml

@@ -161,6 +161,9 @@ tests/:
   stats/:
     test_miscs.py:
       Test_Miscs: missing_feature
+  test_config_consistency.py:
+    Test_Config_HttpServerErrorStatuses_Default: missing_feature
+    Test_Config_HttpServerErrorStatuses_FeatureFlagCustom: missing_feature
   test_distributed.py:
     Test_DistributedHttp: missing_feature
   test_identify.py: irrelevant

manifests/dotnet.yml

@@ -349,6 +349,9 @@ tests/:
       Test_RemoteConfigurationUpdateSequenceFeaturesNoCache: irrelevant (cache is implemented)
       Test_RemoteConfigurationUpdateSequenceLiveDebugging: v2.15.0
       Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: irrelevant (cache is implemented)
+  test_config_consistency.py:
+    Test_Config_HttpServerErrorStatuses_Default: missing_feature
+    Test_Config_HttpServerErrorStatuses_FeatureFlagCustom: missing_feature
   test_data_integrity.py:
     Test_LibraryHeaders: v2.46.0
   test_distributed.py:

manifests/golang.yml

@@ -471,6 +471,9 @@ tests/:
       Test_RemoteConfigurationUpdateSequenceFeaturesNoCache: irrelevant (cache is implemented)
       Test_RemoteConfigurationUpdateSequenceLiveDebugging: missing_feature
       Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: irrelevant (cache is implemented)
+  test_config_consistency.py:
+    Test_Config_HttpServerErrorStatuses_Default: missing_feature
+    Test_Config_HttpServerErrorStatuses_FeatureFlagCustom: missing_feature
   test_data_integrity.py:
     Test_LibraryHeaders: v1.60.0.dev0
   test_distributed.py:

manifests/java.yml

@@ -1218,6 +1218,9 @@ tests/:
     test_json_report.py:
       Test_Mock: v0.0.99
       Test_NotReleased: missing_feature
+  test_config_consistency.py:
+    Test_Config_HttpServerErrorStatuses_Default: missing_feature
+    Test_Config_HttpServerErrorStatuses_FeatureFlagCustom: missing_feature
   test_data_integrity.py:
     Test_LibraryHeaders: v1.29.0
   test_distributed.py:

manifests/nodejs.yml

@@ -522,6 +522,9 @@ tests/:
       Test_RemoteConfigurationUpdateSequenceFeaturesNoCache: irrelevant (cache is implemented)
       Test_RemoteConfigurationUpdateSequenceLiveDebugging: *ref_5_16_0 #actual version unknown
       Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: irrelevant (cache is implemented)
+  test_config_consistency.py:
+    Test_Config_HttpServerErrorStatuses_Default: missing_feature
+    Test_Config_HttpServerErrorStatuses_FeatureFlagCustom: missing_feature
   test_distributed.py:
     Test_DistributedHttp: missing_feature
   test_identify.py:

manifests/php.yml

@@ -308,6 +308,9 @@ tests/:
   stats/:
     test_miscs.py:
       Test_Miscs: missing_feature
+  test_config_consistency.py:
+    Test_Config_HttpServerErrorStatuses_Default: missing_feature
+    Test_Config_HttpServerErrorStatuses_FeatureFlagCustom: missing_feature
   test_distributed.py:
     Test_DistributedHttp: missing_feature
   test_identify.py:

manifests/python.yml

@@ -728,6 +728,9 @@ tests/:
       Test_RemoteConfigurationUpdateSequenceFeaturesNoCache: irrelevant (cache is implemented)
       Test_RemoteConfigurationUpdateSequenceLiveDebugging: v2.8.0.dev
       Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: missing_feature
+  test_config_consistency.py:
+    Test_Config_HttpServerErrorStatuses_Default: missing_feature
+    Test_Config_HttpServerErrorStatuses_FeatureFlagCustom: missing_feature
   test_data_integrity.py:
     Test_LibraryHeaders: v2.7.0
   test_distributed.py:

manifests/ruby.yml

@@ -371,6 +371,9 @@ tests/:
   stats/:
     test_miscs.py:
       Test_Miscs: missing_feature
+  test_config_consistency.py:
+    Test_Config_HttpServerErrorStatuses_Default: missing_feature
+    Test_Config_HttpServerErrorStatuses_FeatureFlagCustom: missing_feature
   test_distributed.py:
     Test_DistributedHttp: missing_feature
   test_identify.py:

scenario_groups.yml

@@ -38,6 +38,10 @@ TELEMETRY_SCENARIOS: &telemetry_scenarios
   - TELEMETRY_METRIC_GENERATION_DISABLED
   - TELEMETRY_METRIC_GENERATION_ENABLED
 
+# Scenarios covering tracing configurations
+TRACING_CONFIG_SCENARIOS: &tracing_config_scenarios
+  - TRACING_CONFIG_NONDEFAULT
+
 # Scenarios to run before a tracer release, basically, all stable scenarios
 TRACER_RELEASE_SCENARIOS:
   - DEFAULT
@@ -48,6 +52,7 @@ TRACER_RELEASE_SCENARIOS:
   - *appsec_scenarios
   - *remote_config_scenarios
   - *telemetry_scenarios
+  - *tracing_config_scenarios
 
 # Scenarios to run on tracers PR.
 # Those scenarios are the one that offer the best probability-to-catch-bug/time-to-run ratio
@@ -57,6 +62,7 @@ TRACER_ESSENTIAL_SCENARIOS:
   - APPSEC_API_SECURITY_RC
   - REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
   - INTEGRATIONS
+  - *tracing_config_scenarios
 
 # Scenarios that rely on backend (and thus, may be a little bit hard to avoid flakyness)
 APM_TRACING_E2E_SCENARIOS:

tests/test_config_consistency.py

@@ -0,0 +1,73 @@
+# Unless explicitly stated otherwise all files in this repository are licensed under the the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2022 Datadog, Inc.
+
+from utils import weblog, interfaces, scenarios, features
+
+
+@scenarios.default
+@features.tracing_configuration_consistency
+class Test_Config_HttpServerErrorStatuses_Default:
+    """ Verify behavior of http clients and distributed traces """
+
+    def setup_status_code_400(self):
+        self.r = weblog.get("/status?code=400")
+
+    def test_status_code_400(self):
+
+        assert self.r.status_code == 400
+
+        interfaces.library.assert_trace_exists(self.r)
+        spans = interfaces.agent.get_spans_list(self.r)
+        assert len(spans) == 1, "Agent received the incorrect amount of spans"
+
+        assert spans[0]["type"] == "web"
+        assert spans[0]["meta"]["http.status_code"] == "400"
+        assert "error" not in spans[0] or spans[0]["error"] == 0
+
+    def setup_status_code_500(self):
+        self.r = weblog.get("/status?code=500")
+
+    def test_status_code_500(self):
+        assert self.r.status_code == 500
+
+        interfaces.library.assert_trace_exists(self.r)
+        spans = interfaces.agent.get_spans_list(self.r)
+        assert len(spans) == 1, "Agent received the incorrect amount of spans"
+
+        assert spans[0]["meta"]["http.status_code"] == "500"
+        assert spans[0]["error"] == 1
+
+
+@scenarios.tracing_config_nondefault
+@features.tracing_configuration_consistency
+class Test_Config_HttpServerErrorStatuses_FeatureFlagCustom:
+    """ Verify behavior of http clients and distributed traces """
+
+    def setup_status_code_200(self):
+        self.r = weblog.get("/status?code=200")
+
+    def test_status_code_200(self):
+        assert self.r.status_code == 200
+
+        interfaces.library.assert_trace_exists(self.r)
+        spans = interfaces.agent.get_spans_list(self.r)
+        assert len(spans) == 1, "Agent received the incorrect amount of spans"
+
+        assert spans[0]["type"] == "web"
+        assert spans[0]["meta"]["http.status_code"] == "200"
+        assert spans[0]["error"] == 1
+
+    def setup_status_code_202(self):
+        self.r = weblog.get("/status?code=202")
+
+    def test_status_code_202(self):
+        assert self.r.status_code == 202
+
+        interfaces.library.assert_trace_exists(self.r)
+        spans = interfaces.agent.get_spans_list(self.r)
+        assert len(spans) == 1, "Agent received the incorrect amount of spans"
+
+        assert spans[0]["type"] == "web"
+        assert spans[0]["meta"]["http.status_code"] == "202"
+        assert spans[0]["error"] == 1

utils/_context/_scenarios/__init__.py

@@ -470,6 +470,10 @@ def all_endtoend_scenarios(test_object):
         doc="Scenario with custom headers for DD_TRACE_HEADER_TAGS that libraries should reject",
     )
 
+    tracing_config_nondefault = EndToEndScenario(
+        "TRACING_CONFIG_NONDEFAULT", weblog_env={"DD_TRACE_HTTP_SERVER_ERROR_STATUSES": "200-201,202"}, doc="",
+    )
+
     parametric = ParametricScenario("PARAMETRIC", doc="WIP")
 
     debugger_probes_status = EndToEndScenario(

utils/_features.py

@@ -2356,3 +2356,13 @@ def iast_source_path_parameter(test_object):
         """
         pytest.mark.features(feature_id=324)(test_object)
         return test_object
+
+    @staticmethod
+    def tracing_configuration_consistency(test_object):
+        """
+        Enforces standardized behaviors for configurations across the tracing libraries.
+
+        https://feature-parity.us1.prod.dog/#/?feature=325
+        """
+        pytest.mark.features(feature_id=325)(test_object)
+        return test_object