diff --git a/.github/workflows/run-end-to-end.yml b/.github/workflows/run-end-to-end.yml index 0a19286330..3173b28792 100644 --- a/.github/workflows/run-end-to-end.yml +++ b/.github/workflows/run-end-to-end.yml @@ -145,6 +145,11 @@ jobs: run: ./run.sh LIBRARY_CONF_CUSTOM_HEADER_TAGS_INVALID env: DD_API_KEY: ${{ secrets.DD_API_KEY }} + - name: Run TRACING_CONFIG_NONDEFAULT scenario + if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"TRACING_CONFIG_NONDEFAULT"') + run: ./run.sh TRACING_CONFIG_NONDEFAULT + env: + DD_API_KEY: ${{ secrets.DD_API_KEY }} - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES scenario if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES"') run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES diff --git a/.vscode/launch.json b/.vscode/launch.json index b294a1f24a..a4c9970892 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -54,6 +54,16 @@ "justMyCode": true, "python": "${workspaceFolder}/venv/bin/python" }, + { + "name": "Run TRACING_CONFIG_NONDEFAULT scenario", + "type": "python", + "request": "launch", + "module": "pytest", + "args": ["-S", "TRACING_CONFIG_NONDEFAULT", "-p", "no:warnings"], + "console": "integratedTerminal", + "justMyCode": true, + "python": "${workspaceFolder}/venv/bin/python" + }, { "name": "Run PROFILING scenario", "type": "python", diff --git a/manifests/cpp.yml b/manifests/cpp.yml index 0b1f0a7caa..5bfd049fb2 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -161,6 +161,9 @@ tests/: stats/: test_miscs.py: Test_Miscs: missing_feature + test_config_consistency.py: + Test_Config_HttpServerErrorStatuses_Default: missing_feature + Test_Config_HttpServerErrorStatuses_FeatureFlagCustom: missing_feature test_distributed.py: Test_DistributedHttp: missing_feature test_identify.py: irrelevant diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index e7832d640a..27c0e6fb02 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -349,6 +349,9 @@ tests/: Test_RemoteConfigurationUpdateSequenceFeaturesNoCache: irrelevant (cache is implemented) Test_RemoteConfigurationUpdateSequenceLiveDebugging: v2.15.0 Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: irrelevant (cache is implemented) + test_config_consistency.py: + Test_Config_HttpServerErrorStatuses_Default: missing_feature + Test_Config_HttpServerErrorStatuses_FeatureFlagCustom: missing_feature test_data_integrity.py: Test_LibraryHeaders: v2.46.0 test_distributed.py: diff --git a/manifests/golang.yml b/manifests/golang.yml index d245b51b30..5322cd7ece 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -471,6 +471,9 @@ tests/: Test_RemoteConfigurationUpdateSequenceFeaturesNoCache: irrelevant (cache is implemented) Test_RemoteConfigurationUpdateSequenceLiveDebugging: missing_feature Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: irrelevant (cache is implemented) + test_config_consistency.py: + Test_Config_HttpServerErrorStatuses_Default: missing_feature + Test_Config_HttpServerErrorStatuses_FeatureFlagCustom: missing_feature test_data_integrity.py: Test_LibraryHeaders: v1.60.0.dev0 test_distributed.py: diff --git a/manifests/java.yml b/manifests/java.yml index 7b7921f88d..51eae9c1f7 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -1218,6 +1218,9 @@ tests/: test_json_report.py: Test_Mock: v0.0.99 Test_NotReleased: missing_feature + test_config_consistency.py: + Test_Config_HttpServerErrorStatuses_Default: missing_feature + Test_Config_HttpServerErrorStatuses_FeatureFlagCustom: missing_feature test_data_integrity.py: Test_LibraryHeaders: v1.29.0 test_distributed.py: diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 1c8304b2f1..610f6e8db6 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -522,6 +522,9 @@ tests/: Test_RemoteConfigurationUpdateSequenceFeaturesNoCache: irrelevant (cache is implemented) Test_RemoteConfigurationUpdateSequenceLiveDebugging: *ref_5_16_0 #actual version unknown Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: irrelevant (cache is implemented) + test_config_consistency.py: + Test_Config_HttpServerErrorStatuses_Default: missing_feature + Test_Config_HttpServerErrorStatuses_FeatureFlagCustom: missing_feature test_distributed.py: Test_DistributedHttp: missing_feature test_identify.py: diff --git a/manifests/php.yml b/manifests/php.yml index 97f9d7dc5c..409594f385 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -308,6 +308,9 @@ tests/: stats/: test_miscs.py: Test_Miscs: missing_feature + test_config_consistency.py: + Test_Config_HttpServerErrorStatuses_Default: missing_feature + Test_Config_HttpServerErrorStatuses_FeatureFlagCustom: missing_feature test_distributed.py: Test_DistributedHttp: missing_feature test_identify.py: diff --git a/manifests/python.yml b/manifests/python.yml index 9e26df4f5f..af777ac7a1 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -728,6 +728,9 @@ tests/: Test_RemoteConfigurationUpdateSequenceFeaturesNoCache: irrelevant (cache is implemented) Test_RemoteConfigurationUpdateSequenceLiveDebugging: v2.8.0.dev Test_RemoteConfigurationUpdateSequenceLiveDebuggingNoCache: missing_feature + test_config_consistency.py: + Test_Config_HttpServerErrorStatuses_Default: missing_feature + Test_Config_HttpServerErrorStatuses_FeatureFlagCustom: missing_feature test_data_integrity.py: Test_LibraryHeaders: v2.7.0 test_distributed.py: diff --git a/manifests/ruby.yml b/manifests/ruby.yml index fdeb1348ce..4638bf09b6 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -371,6 +371,9 @@ tests/: stats/: test_miscs.py: Test_Miscs: missing_feature + test_config_consistency.py: + Test_Config_HttpServerErrorStatuses_Default: missing_feature + Test_Config_HttpServerErrorStatuses_FeatureFlagCustom: missing_feature test_distributed.py: Test_DistributedHttp: missing_feature test_identify.py: diff --git a/scenario_groups.yml b/scenario_groups.yml index 33f1146ef8..db191ea2a8 100644 --- a/scenario_groups.yml +++ b/scenario_groups.yml @@ -38,6 +38,10 @@ TELEMETRY_SCENARIOS: &telemetry_scenarios - TELEMETRY_METRIC_GENERATION_DISABLED - TELEMETRY_METRIC_GENERATION_ENABLED +# Scenarios covering tracing configurations +TRACING_CONFIG_SCENARIOS: &tracing_config_scenarios + - TRACING_CONFIG_NONDEFAULT + # Scenarios to run before a tracer release, basically, all stable scenarios TRACER_RELEASE_SCENARIOS: - DEFAULT @@ -48,6 +52,7 @@ TRACER_RELEASE_SCENARIOS: - *appsec_scenarios - *remote_config_scenarios - *telemetry_scenarios + - *tracing_config_scenarios # Scenarios to run on tracers PR. # Those scenarios are the one that offer the best probability-to-catch-bug/time-to-run ratio @@ -57,6 +62,7 @@ TRACER_ESSENTIAL_SCENARIOS: - APPSEC_API_SECURITY_RC - REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES - INTEGRATIONS + - *tracing_config_scenarios # Scenarios that rely on backend (and thus, may be a little bit hard to avoid flakyness) APM_TRACING_E2E_SCENARIOS: diff --git a/tests/test_config_consistency.py b/tests/test_config_consistency.py new file mode 100644 index 0000000000..3f41d13471 --- /dev/null +++ b/tests/test_config_consistency.py @@ -0,0 +1,73 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2022 Datadog, Inc. + +from utils import weblog, interfaces, scenarios, features + + +@scenarios.default +@features.tracing_configuration_consistency +class Test_Config_HttpServerErrorStatuses_Default: + """ Verify behavior of http clients and distributed traces """ + + def setup_status_code_400(self): + self.r = weblog.get("/status?code=400") + + def test_status_code_400(self): + + assert self.r.status_code == 400 + + interfaces.library.assert_trace_exists(self.r) + spans = interfaces.agent.get_spans_list(self.r) + assert len(spans) == 1, "Agent received the incorrect amount of spans" + + assert spans[0]["type"] == "web" + assert spans[0]["meta"]["http.status_code"] == "400" + assert "error" not in spans[0] or spans[0]["error"] == 0 + + def setup_status_code_500(self): + self.r = weblog.get("/status?code=500") + + def test_status_code_500(self): + assert self.r.status_code == 500 + + interfaces.library.assert_trace_exists(self.r) + spans = interfaces.agent.get_spans_list(self.r) + assert len(spans) == 1, "Agent received the incorrect amount of spans" + + assert spans[0]["meta"]["http.status_code"] == "500" + assert spans[0]["error"] == 1 + + +@scenarios.tracing_config_nondefault +@features.tracing_configuration_consistency +class Test_Config_HttpServerErrorStatuses_FeatureFlagCustom: + """ Verify behavior of http clients and distributed traces """ + + def setup_status_code_200(self): + self.r = weblog.get("/status?code=200") + + def test_status_code_200(self): + assert self.r.status_code == 200 + + interfaces.library.assert_trace_exists(self.r) + spans = interfaces.agent.get_spans_list(self.r) + assert len(spans) == 1, "Agent received the incorrect amount of spans" + + assert spans[0]["type"] == "web" + assert spans[0]["meta"]["http.status_code"] == "200" + assert spans[0]["error"] == 1 + + def setup_status_code_202(self): + self.r = weblog.get("/status?code=202") + + def test_status_code_202(self): + assert self.r.status_code == 202 + + interfaces.library.assert_trace_exists(self.r) + spans = interfaces.agent.get_spans_list(self.r) + assert len(spans) == 1, "Agent received the incorrect amount of spans" + + assert spans[0]["type"] == "web" + assert spans[0]["meta"]["http.status_code"] == "202" + assert spans[0]["error"] == 1 diff --git a/utils/_context/_scenarios/__init__.py b/utils/_context/_scenarios/__init__.py index 3523e5dfac..d6637ed9d5 100644 --- a/utils/_context/_scenarios/__init__.py +++ b/utils/_context/_scenarios/__init__.py @@ -470,6 +470,10 @@ def all_endtoend_scenarios(test_object): doc="Scenario with custom headers for DD_TRACE_HEADER_TAGS that libraries should reject", ) + tracing_config_nondefault = EndToEndScenario( + "TRACING_CONFIG_NONDEFAULT", weblog_env={"DD_TRACE_HTTP_SERVER_ERROR_STATUSES": "200-201,202"}, doc="", + ) + parametric = ParametricScenario("PARAMETRIC", doc="WIP") debugger_probes_status = EndToEndScenario( diff --git a/utils/_features.py b/utils/_features.py index c945129686..cca93b3c23 100644 --- a/utils/_features.py +++ b/utils/_features.py @@ -2356,3 +2356,13 @@ def iast_source_path_parameter(test_object): """ pytest.mark.features(feature_id=324)(test_object) return test_object + + @staticmethod + def tracing_configuration_consistency(test_object): + """ + Enforces standardized behaviors for configurations across the tracing libraries. + + https://feature-parity.us1.prod.dog/#/?feature=325 + """ + pytest.mark.features(feature_id=325)(test_object) + return test_object