Merge branch 'main' into sanchda/make_contextvars_indirect

.github/CODEOWNERS

@@ -51,26 +51,31 @@ tests/runtime                       @DataDog/apm-core-python
 tests/tracer                        @DataDog/apm-core-python
 
 # CI App and related
-ddtrace/contrib/coverage            @DataDog/ci-app-libraries
-ddtrace/contrib/pytest              @DataDog/ci-app-libraries
-ddtrace/contrib/pytest_bdd          @DataDog/ci-app-libraries
-ddtrace/contrib/unittest            @DataDog/ci-app-libraries
-tests/contrib/pytest                @DataDog/ci-app-libraries
-tests/contrib/pytest_bdd            @DataDog/ci-app-libraries
-tests/contrib/unittest_plugin       @DataDog/ci-app-libraries
-ddtrace/ext/ci.py                   @DataDog/ci-app-libraries
-ddtrace/ext/test_visibility         @DataDog/ci-app-libraries
-ddtrace/ext/test.py                 @DataDog/ci-app-libraries
-ddtrace/internal/ci_visibility      @DataDog/ci-app-libraries
-ddtrace/internal/test_visibility    @DataDog/ci-app-libraries
-ddtrace/internal/codeowners.py      @DataDog/apm-core-python @datadog/ci-app-libraries
-ddtrace/internal/coverage           @DataDog/apm-core-python @datadog/ci-app-libraries
-tests/internal/test_codeowners.py   @datadog/ci-app-libraries
-tests/ci_visibility                 @DataDog/ci-app-libraries
-tests/coverage                      @DataDog/apm-core-python @DataDog/ci-app-libraries
-tests/tracer/test_ci.py             @DataDog/ci-app-libraries
-ddtrace/ext/git.py                  @DataDog/ci-app-libraries @DataDog/apm-core-python
-scripts/ci_visibility/*             @DataDog/ci-app-libraries
+ddtrace/contrib/asynctest                             @DataDog/ci-app-libraries
+ddtrace/contrib/coverage                              @DataDog/ci-app-libraries
+ddtrace/contrib/pytest                                @DataDog/ci-app-libraries
+ddtrace/contrib/pytest_bdd                            @DataDog/ci-app-libraries
+ddtrace/contrib/pytest_benchmark                      @DataDog/ci-app-libraries
+ddtrace/contrib/unittest                              @DataDog/ci-app-libraries
+tests/contrib/asynctest                               @DataDog/ci-app-libraries
+tests/contrib/pytest                                  @DataDog/ci-app-libraries
+tests/contrib/pytest_bdd                              @DataDog/ci-app-libraries
+tests/contrib/pytest_benchmark                        @DataDog/ci-app-libraries
+tests/contrib/unittest                                @DataDog/ci-app-libraries
+tests/integration/test_integration_civisibility.py    @DataDog/ci-app-libraries
+ddtrace/ext/ci.py                                     @DataDog/ci-app-libraries
+ddtrace/ext/test_visibility                           @DataDog/ci-app-libraries
+ddtrace/ext/test.py                                   @DataDog/ci-app-libraries
+ddtrace/internal/ci_visibility                        @DataDog/ci-app-libraries
+ddtrace/internal/test_visibility                      @DataDog/ci-app-libraries
+ddtrace/internal/codeowners.py                        @DataDog/apm-core-python @datadog/ci-app-libraries
+ddtrace/internal/coverage                             @DataDog/apm-core-python @datadog/ci-app-libraries
+tests/internal/test_codeowners.py                     @datadog/ci-app-libraries
+tests/ci_visibility                                   @DataDog/ci-app-libraries
+tests/coverage                                        @DataDog/apm-core-python @DataDog/ci-app-libraries
+tests/tracer/test_ci.py                               @DataDog/ci-app-libraries
+ddtrace/ext/git.py                                    @DataDog/ci-app-libraries @DataDog/apm-core-python
+scripts/ci_visibility/*                               @DataDog/ci-app-libraries
 
 # Debugger
 ddtrace/debugging/                  @DataDog/debugger-python

.gitlab-ci.yml

@@ -44,7 +44,14 @@ onboarding_tests_installer:
 onboarding_tests_k8s_injection:
   parallel:
     matrix:
-      - WEBLOG_VARIANT: [dd-lib-python-init-test-django, dd-lib-python-init-test-django-gunicorn, dd-lib-python-init-test-django-uvicorn, dd-lib-python-init-test-protobuf-old]
+      - WEBLOG_VARIANT:
+          - dd-lib-python-init-test-django
+          - dd-lib-python-init-test-django-gunicorn
+          - dd-lib-python-init-test-django-gunicorn-alpine
+          - dd-lib-python-init-test-django-preinstalled
+          - dd-lib-python-init-test-django-unsupported-package-force
+          - dd-lib-python-init-test-django-uvicorn
+          - dd-lib-python-init-test-protobuf-old
 
 deploy_to_di_backend:manual:
   stage: shared-pipeline

.gitlab/tests/appsec.yml

@@ -15,6 +15,7 @@ appsec iast:
     SUITE_NAME: "appsec_iast$"
     TEST_POSTGRES_HOST: "postgres"
   retry: 2
+  timeout: 25m
 
 appsec iast tdd_propagation:
   extends: .test_base_riot_snapshot

ddtrace/_trace/span.py

@@ -113,7 +113,7 @@ class Span(object):
         "duration_ns",
         # Internal attributes
         "_context",
-        "_local_root",
+        "_local_root_value",
         "_parent",
         "_ignored_exceptions",
         "_on_finish_callbacks",
@@ -206,7 +206,7 @@ def __init__(
         self._events: List[SpanEvent] = []
         self._parent: Optional["Span"] = None
         self._ignored_exceptions: Optional[List[Type[Exception]]] = None
-        self._local_root: Optional["Span"] = None
+        self._local_root_value: Optional["Span"] = None  # None means this is the root span.
         self._store: Optional[Dict[str, Any]] = None
 
     def _ignore_exception(self, exc: Type[Exception]) -> None:
@@ -595,6 +595,23 @@ def context(self) -> Context:
             self._context = Context(trace_id=self.trace_id, span_id=self.span_id, is_remote=False)
         return self._context
 
+    @property
+    def _local_root(self) -> "Span":
+        if self._local_root_value is None:
+            return self
+        return self._local_root_value
+
+    @_local_root.setter
+    def _local_root(self, value: "Span") -> None:
+        if value is not self:
+            self._local_root_value = value
+        else:
+            self._local_root_value = None
+
+    @_local_root.deleter
+    def _local_root(self) -> None:
+        del self._local_root_value
+
     def link_span(self, context: Context, attributes: Optional[Dict[str, Any]] = None) -> None:
         """Defines a causal relationship between two spans"""
         if not context.trace_id or not context.span_id:

ddtrace/_trace/trace_handlers.py

@@ -704,6 +704,36 @@ def _on_botocore_patched_bedrock_api_call_success(ctx, reqid, latency, input_tok
     span.set_tag_str("bedrock.usage.completion_tokens", output_token_count)
 
 
+def _propagate_context(ctx, headers):
+    distributed_tracing_enabled = ctx["integration_config"].distributed_tracing_enabled
+    call_key = ctx.get_item("call_key")
+    if call_key is None:
+        log.warning("call_key not found in ctx")
+    if distributed_tracing_enabled and call_key:
+        span = ctx[ctx["call_key"]]
+        HTTPPropagator.inject(span.context, headers)
+
+
+def _after_job_execution(ctx, job_failed, span_tags):
+    """sets job.status and job.origin span tags after job is performed"""
+    # get_status() returns None when ttl=0
+    call_key = ctx.get_item("call_key")
+    if call_key:
+        span = ctx[ctx["call_key"]]
+    if span:
+        if job_failed:
+            span.error = 1
+        for k in span_tags.keys():
+            span.set_tag_str(k, span_tags[k])
+
+
+def _on_end_of_traced_method_in_fork(ctx):
+    """Force flush to agent since the process `os.exit()`s
+    immediately after this method returns
+    """
+    ctx["pin"].tracer.flush()
+
+
 def _on_botocore_bedrock_process_response(
     ctx: core.ExecutionContext,
     formatted_response: Dict[str, Any],
@@ -851,6 +881,9 @@ def listen():
     core.on("test_visibility.enable", _on_test_visibility_enable)
     core.on("test_visibility.disable", _on_test_visibility_disable)
     core.on("test_visibility.is_enabled", _on_test_visibility_is_enabled, "is_enabled")
+    core.on("rq.worker.perform_job", _after_job_execution)
+    core.on("rq.worker.after.perform.job", _on_end_of_traced_method_in_fork)
+    core.on("rq.queue.enqueue_job", _propagate_context)
 
     for context_name in (
         "flask.call",
@@ -869,6 +902,11 @@ def listen():
         "botocore.patched_stepfunctions_api_call",
         "botocore.patched_bedrock_api_call",
         "redis.command",
+        "rq.queue.enqueue_job",
+        "rq.traced_queue_fetch_job",
+        "rq.worker.perform_job",
+        "rq.job.perform",
+        "rq.job.fetch_many",
     ):
         core.on(f"context.started.start_span.{context_name}", _start_span)
 

ddtrace/_trace/tracer.py

@@ -797,8 +797,6 @@ def _start_span(
                 span._parent = parent
                 span._local_root = parent._local_root
 
-            if span._local_root is None:
-                span._local_root = span
             for k, v in _get_metas_to_propagate(context):
                 # We do not want to propagate AppSec propagation headers
                 # to children spans, only across distributed spans
@@ -815,7 +813,6 @@ def _start_span(
                 span_api=span_api,
                 on_finish=[self._on_span_finish],
             )
-            span._local_root = span
             if config.report_hostname:
                 span.set_tag_str(HOSTNAME_KEY, hostname.get_hostname())
 

ddtrace/appsec/_handlers.py

@@ -301,13 +301,34 @@ def _on_django_func_wrapped(fn_args, fn_kwargs, first_arg_expected_type, *_):
         http_req.COOKIES = taint_structure(http_req.COOKIES, OriginType.COOKIE_NAME, OriginType.COOKIE)
         http_req.GET = taint_structure(http_req.GET, OriginType.PARAMETER_NAME, OriginType.PARAMETER)
         http_req.POST = taint_structure(http_req.POST, OriginType.BODY, OriginType.BODY)
-        if not is_pyobject_tainted(getattr(http_req, "_body", None)):
-            http_req._body = taint_pyobject(
-                http_req.body,
-                source_name=origin_to_str(OriginType.BODY),
-                source_value=http_req.body,
-                source_origin=OriginType.BODY,
-            )
+        if (
+            getattr(http_req, "_body", None) is not None
+            and len(getattr(http_req, "_body", None)) > 0
+            and not is_pyobject_tainted(getattr(http_req, "_body", None))
+        ):
+            try:
+                http_req._body = taint_pyobject(
+                    http_req._body,
+                    source_name=origin_to_str(OriginType.BODY),
+                    source_value=http_req._body,
+                    source_origin=OriginType.BODY,
+                )
+            except AttributeError:
+                log.debug("IAST can't set attribute http_req._body", exc_info=True)
+        elif (
+            getattr(http_req, "body", None) is not None
+            and len(getattr(http_req, "body", None)) > 0
+            and not is_pyobject_tainted(getattr(http_req, "body", None))
+        ):
+            try:
+                http_req.body = taint_pyobject(
+                    http_req.body,
+                    source_name=origin_to_str(OriginType.BODY),
+                    source_value=http_req.body,
+                    source_origin=OriginType.BODY,
+                )
+            except AttributeError:
+                log.debug("IAST can't set attribute http_req.body", exc_info=True)
 
         http_req.headers = taint_structure(http_req.headers, OriginType.HEADER_NAME, OriginType.HEADER)
         http_req.path = taint_pyobject(

ddtrace/appsec/_iast/_metrics.py

@@ -67,7 +67,7 @@ def _set_iast_error_metric(msg: Text) -> None:
             exception_type, exception_instance, _traceback_list = sys.exc_info()
             res = []
             # first 10 frames are this function, the exception in aspects and the error line
-            res.extend(traceback.format_stack(limit=10))
+            res.extend(traceback.format_stack(limit=20))
 
             # get the frame with the error and the error message
             result = traceback.format_exception(exception_type, exception_instance, _traceback_list)

ddtrace/appsec/_iast/_taint_tracking/__init__.py

@@ -161,7 +161,7 @@ def taint_pyobject(pyobject: Any, source_name: Any, source_value: Any, source_or
         _set_metric_iast_executed_source(source_origin)
         return pyobject_newid
     except ValueError as e:
-        iast_taint_log_error("Tainting object error (pyobject type %s): %s" % (type(pyobject), e))
+        log.debug("Tainting object error (pyobject type %s): %s", type(pyobject), e)
     return pyobject
 
 
@@ -203,16 +203,18 @@ def trace_calls_and_returns(frame, event, arg):
             return
         if event == "call":
             f_locals = frame.f_locals
-            if any([is_pyobject_tainted(f_locals[arg]) for arg in f_locals]):
-                TAINTED_FRAMES.append(frame)
-                log.debug("Call to %s on line %s of %s, args: %s", func_name, line_no, filename, frame.f_locals)
-                log.debug("Tainted arguments:")
-                for arg in f_locals:
-                    if is_pyobject_tainted(f_locals[arg]):
-                        log.debug("\t%s: %s", arg, f_locals[arg])
-                log.debug("-----")
-
-            return trace_calls_and_returns
+            try:
+                if any([is_pyobject_tainted(f_locals[arg]) for arg in f_locals]):
+                    TAINTED_FRAMES.append(frame)
+                    log.debug("Call to %s on line %s of %s, args: %s", func_name, line_no, filename, frame.f_locals)
+                    log.debug("Tainted arguments:")
+                    for arg in f_locals:
+                        if is_pyobject_tainted(f_locals[arg]):
+                            log.debug("\t%s: %s", arg, f_locals[arg])
+                    log.debug("-----")
+                return trace_calls_and_returns
+            except AttributeError:
+                pass
         elif event == "return":
             if frame in TAINTED_FRAMES:
                 TAINTED_FRAMES.remove(frame)

ddtrace/appsec/_iast/_taint_tracking/tests/CMakeLists.txt

@@ -28,3 +28,8 @@ target_link_libraries(native_tests ${NATIVE_TEST_LIBRARIES})
 # Discover tests
 include(GoogleTest)
 gtest_discover_tests(native_tests)
+
+add_custom_target(test_valgrind
+        COMMAND valgrind --leak-check=full --suppressions=../../../../../scripts/iast/valgrind-python.supp --show-reachable=yes ${CMAKE_BINARY_DIR}/tests/native_tests --gtest_filter=-TestTimer.*
+        WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests)
+add_dependencies(test_valgrind native_tests)

ddtrace/contrib/internal/django/utils.py

@@ -289,7 +289,7 @@ def _extract_body(request):
 def _remake_body(request):
     # some libs that utilize django (Spyne) require the body stream to be unread or else will throw errors
     # see: https://github.com/arskom/spyne/blob/f105ec2f41495485fef1211fe73394231b3f76e5/spyne/server/wsgi.py#L538
-    if request.method in _BODY_METHODS:
+    if request.method in _BODY_METHODS and getattr(request, "_body", None):
         try:
             unread_body = io.BytesIO(request._body)
             if unread_body.seekable():