Merge with master

.circleci/config.continue.yml.j2

@@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
-default_system_tests_commit: &default_system_tests_commit 39bf65e0d94278c60fdd6ff85ba668cb436467f0
+default_system_tests_commit: &default_system_tests_commit c87bd359aad64a29f280fc5c70a879f7c7f4846e
 
 parameters:
   nightly:
@@ -54,7 +54,7 @@ parameters:
   global_pattern:
     # Pattern for files that should always trigger a test jobs
     type: string
-    default: "^build.gradle$|^settings.gradle$|^gradle.properties$|^buildSrc/|^gradle/|.circleci"
+    default: "^build.gradle$|^settings.gradle$|^gradle.properties$|^buildSrc/|^gradle/|.circleci|^gradlew|^mvnw|^.mvn/"
 
   docker_image:
     type: string
@@ -273,13 +273,6 @@ commands:
             git fetch origin << parameters.systemTestsCommit >>
             git reset --hard FETCH_HEAD
 
-      - run:
-          name: Install python 3.12
-          command: |
-            sudo apt-get update
-            sudo apt-get install -y python3.12-full python3.12-dev python3.12-venv
-            echo 'export PATH="$HOME/.local/bin:$PATH"' >>"$BASH_ENV"
-
 jobs:
   build:
     <<: *defaults
@@ -667,7 +660,8 @@ jobs:
           - DD_POOL_TRACE_CHECK_FAILURES=true
           - DD_DISABLE_ERROR_RESPONSES=true
           - ENABLED_CHECKS=trace_content_length,trace_stall,meta_tracer_version_header,trace_count_header,trace_peer_service,trace_dd_service
-    resource_class: xlarge
+    # TODO: merge xlarge_tests and tests? or rename this?
+    resource_class: large
 
 
   # The only way to do fan-in in CircleCI seems to have a proper job, so let's have one that
@@ -1106,8 +1100,8 @@ build_test_jobs: &build_test_jobs
       triggeredBy: *instrumentation_modules
       stage: instrumentation
       cacheType: inst
-      parallelism: 4
-      maxWorkers: 4
+      parallelism: 12
+      maxWorkers: 3
       matrix:
         <<: *test_matrix
 
@@ -1120,8 +1114,8 @@ build_test_jobs: &build_test_jobs
       triggeredBy: *instrumentation_modules
       stage: instrumentation
       cacheType: inst
-      parallelism: 4
-      maxWorkers: 4
+      parallelism: 12
+      maxWorkers: 3
       testJvm: "8"
 
   - xlarge_tests:
@@ -1134,8 +1128,8 @@ build_test_jobs: &build_test_jobs
       triggeredBy: *instrumentation_modules
       stage: instrumentation
       cacheType: latestdep
-      parallelism: 4
-      maxWorkers: 4
+      parallelism: 12
+      maxWorkers: 3
       testJvm: "8"
 
   - xlarge_tests:
@@ -1148,8 +1142,8 @@ build_test_jobs: &build_test_jobs
       triggeredBy: *instrumentation_modules
       stage: instrumentation
       cacheType: latestdep
-      parallelism: 4
-      maxWorkers: 4
+      parallelism: 12
+      maxWorkers: 3
       testJvm: "17"
 
   - xlarge_tests:
@@ -1162,8 +1156,8 @@ build_test_jobs: &build_test_jobs
       triggeredBy: *instrumentation_modules
       stage: instrumentation
       cacheType: latestdep
-      parallelism: 4
-      maxWorkers: 4
+      parallelism: 12
+      maxWorkers: 3
       testJvm: "21"
 
 {% if flaky %}
@@ -1191,7 +1185,7 @@ build_test_jobs: &build_test_jobs
       triggeredBy: *instrumentation_modules
       stage: instrumentation
       cacheType: inst
-      parallelism: 2
+      parallelism: 12
       maxWorkers: 4
       testJvm: "8"
 

.github/CODEOWNERS

@@ -45,6 +45,8 @@ dd-java-agent/instrumentation/*iast*   @DataDog/asm-java
 dd-java-agent/instrumentation/*appsec* @DataDog/asm-java
 dd-java-agent/instrumentation/json/    @DataDog/asm-java
 dd-java-agent/instrumentation/snakeyaml/    @DataDog/asm-java
+dd-java-agent/instrumentation/velocity/    @DataDog/asm-java
+dd-java-agent/instrumentation/freemarker/    @DataDog/asm-java
 dd-smoke-tests/iast-util/              @DataDog/asm-java
 dd-smoke-tests/spring-security/        @DataDog/asm-java
 dd-java-agent/instrumentation/commons-fileupload/    @DataDog/asm-java
@@ -53,6 +55,9 @@ dd-java-agent/instrumentation/spring-security-5/ @DataDog/asm-java
 **/iast/                               @DataDog/asm-java
 **/Iast*.java                          @DataDog/asm-java
 **/Iast*.groovy                        @DataDog/asm-java
+**/rasp/                               @Datadog/asm-java
+**/*Rasp*.java                          @DataDog/asm-java
+**/*Rasp*.groovy                        @DataDog/asm-java
 
 # @DataDog/data-jobs-monitoring
 dd-java-agent/instrumentation/spark/            @DataDog/data-jobs-monitoring

.github/workflows/README.md

@@ -1,6 +1,6 @@
 # GitHub Actions Documentation
 
-This lists and describes the repository GitHub actions.
+This lists and describes the repository GitHub actions, how to maintain and test them.
 
 ## Release Management
 
@@ -18,15 +18,16 @@ _Trigger:_ When a release is published.
 
 _Action:_ Append the new release to the Cloud Foundry repository.
 
-_Recovery:_ Manually edit and push the `index.yml` file from [the cloudfoundry branch](https://github.com/DataDog/dd-trace-java/tree/cloudfoundry).
+_Recovery:_ Manually edit and push the `index.yml` file from [the cloudfoundry branch](https://github.com/DataDog/dd-trace-java/tree/cloudfoundry).
 
 ### create-next-milestone [🔗](create-next-milestone.yaml)
 
 _Trigger:_ When closing a milestone.
 
 _Action:_ Create a new milestone by incrementing minor version.
 
-_Comment:_ Already done when closing a tag. To delete?
+_Comment:_ Disabled as also covered by increment-milestone-on-tag.
+This will be removed after some testing.  
 
 ### draft-release-notes-on-tag [🔗](draft-release-notes-on-tag.yaml)
 
@@ -40,18 +41,17 @@ _Actions:_
 
 _Recovery:_ Manually trigger the action again on the relevant tag.
 
-### increment-milestones-on-tag [🔗](increment-milestones-on-tag.yaml)
+### increment-milestone-on-tag [🔗](increment-milestone-on-tag.yaml)
 
-_Trigger:_ When creating a tag. Release Candidate tags containing "-RC" or "-rc" will skip this.
+_Trigger:_ When creating a minor or major version tag.
 
 _Actions:_
 * Close the milestone related to the tag,
 * Create a new milestone by incrementing minor version.
 
-_Recovery:_ Manually close the related milestone and create a new one.
+_Recovery:_ Manually [close the related milestone and create a new one](https://github.com/DataDog/dd-trace-java/milestones).
 
-_Notes:_ This actions will handle _minor_ releases only.
-As there is no milestone for _patch_ releases, it won't close and create _patch_ releated milestone.
+_Notes:_ This action will not apply to release candidate versions using `-RC` tags.
 
 ### update-download-releases [🔗](update-download-releases.yaml)
 
@@ -117,3 +117,12 @@ Run the following script to get the list of actions to declare according the sta
 ```bash
 find .github/workflows -name "*.yaml" -exec  awk '/uses:/{print $2 ","}' {} \; | grep -vE '^(actions|github)/' | sort | uniq
 ```
+
+## Testing
+
+Workflows can be locally tested using the [`act` CLI](https://github.com/nektos/act/).
+The [.github/workflows/tests/](./tests) folder contains test scripts and event payloads to locally trigger workflows.
+
+> [!WARNING]
+> Locally running workflows will still query GitHub backend and will update the GitHub project accordingly.
+> Pay extra attention to the workflow jobs you trigger to not create development disruption.

.github/workflows/analyze-changes.yaml

@@ -44,7 +44,8 @@ jobs:
         with:
           submodules: 'recursive'
 
-      - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+      - name: Cache Gradle dependencies
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
         with:
           path: |
             ~/.gradle/caches
@@ -97,7 +98,8 @@ jobs:
         with:
           submodules: 'recursive'
 
-      - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+      - name: Cache Gradle dependencies
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
         with:
           path: |
             ~/.gradle/caches

.github/workflows/comment-on-submodule-update.yaml

@@ -12,6 +12,7 @@ jobs:
   comment_on_submodule_update:
     permissions:
       issues: write # Required to create a comment on the pull request
+      pull-requests: write # Required to create a comment on the pull request
     runs-on: ubuntu-latest
 
     steps:

.github/workflows/update-gradle-dependencies.yaml

@@ -59,6 +59,10 @@ jobs:
               COUNT=0
             fi
           done
+          # Check at uncommited files
+          echo "Checking uncommited files"
+          git status
+          # Create a PR from the created branch
           if [ $COUNT -gt 0 ]; then
             echo "Creating a commit to $BRANCH_NAME@$BRANCH_HEAD with $GH_ADD_ARGS"
             ghcommit --repository ${{ github.repository }} --branch $BRANCH_NAME --sha $BRANCH_HEAD $GH_ADD_ARGS --message "chore: Update Gradle dependencies"

.github/workflows/update-issues-on-release.yaml

@@ -15,54 +15,59 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Get milestone for release
-        id: milestone
-        uses: actions/github-script@47f7cf65b5ced0830a325f705cad64f2f58dddf7 # 3.1.0
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # 7.0.1
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}
           script: |
-            const milestone_name = ("${{github.event_name}}" == "workflow_dispatch") ?
-              "${{github.event.inputs.milestone}}" : "${{github.event.release.name}}"
-
-            const milestones = await github.paginate(github.issues.listMilestones, {
+            // Get release milestone name
+            const milestoneName = "${{github.event_name}}" == "workflow_dispatch" ?
+                "${{github.event.inputs.milestone}}" :
+                "${{github.event.release.name}}"
+            // Get release URL
+            const releaseURL = "${{github.event_name}}" == "workflow_dispatch" ?
+                "${{github.event.repository.html_url}}/releases/tag/v${{github.event.release.name}}" :
+                "${{github.event.release.html_url}}"
+            core.info(`Notify of release availability: v${milestoneName}`)
+            // Get release milestone number
+            const milestoneNumber = await github.paginate(github.rest.issues.listMilestones, {
               owner: context.repo.owner,
               repo: context.repo.repo,
               state: 'all'
-            })
-
-            return milestones.find( milestone => milestone.title == milestone_name ).number
-      - name: Get issues for milestone
-        id: issues
-        uses: actions/github-script@47f7cf65b5ced0830a325f705cad64f2f58dddf7 # 3.1.0
-        with:
-          github-token: ${{secrets.GITHUB_TOKEN}}
-          script: |
-            const issues = await github.paginate(github.issues.listForRepo, {
+            }, (response, done) => {
+              const releaseMilestone = response.data.find(milestone => milestone.title == milestoneName)
+              if (releaseMilestone) {
+                done()
+                return releaseMilestone.number
+              }
+            }).then(milestones => milestones.pop())
+            if (!milestoneNumber) {
+              core.setFailed(`Failed to find milestone: ${milestoneName}`)
+              return
+            }
+            // Get all release milestone issues
+            const issues = await github.paginate(github.rest.issues.listForRepo, {
               owner: context.repo.owner,
               repo: context.repo.repo,
               state: 'all',
-              milestone: ${{steps.milestone.outputs.result}}
-            })
-
-            // Pull requests are issues so filter them out
-            return issues.filter( issue => !issue["pull_request"] ).map( issue => issue.number )
-      - name: Comment and close issues
-        uses: actions/github-script@47f7cf65b5ced0830a325f705cad64f2f58dddf7 # 3.1.0
-        with:
-          github-token: ${{secrets.GITHUB_TOKEN}}
-          script: |
-            for (let issue_number of ${{ steps.issues.outputs.result }}) {
-              // This can be parallelized better by moving the await but it might trip rate limits
-              await github.issues.createComment({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                issue_number: issue_number,
-                body: ':robot: This issue has been addressed in the latest release.  See full details in the [Release Notes]( ${{ github.event.release.html_url }}).'
-              })
-
-              await github.issues.update({
+              milestone: milestoneNumber
+            }, (response) => response.data.filter(issue => !issue["pull_request"]))
+            // Comment and close issues
+            for (const issue of issues) {
+              core.info(issue.state == 'open' ?
+                `Commenting and closing issue ${issue.number}` :
+                `Commenting on closed issue ${issue.number}`)
+              await github.rest.issues.createComment({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
-                issue_number: issue_number,
-                state: 'closed'
+                issue_number: issue.number,
+                body: `:robot: This issue has been addressed in the latest release. Check full details in the [Release Notes](${releaseURL}).`
               })
+              if (issue.state == 'open') {
+                await github.rest.issues.update({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: issue.number,
+                  state: 'closed'
+                })
+              }
             }

.gitlab-ci.yml

@@ -216,7 +216,10 @@ package-oci:
 onboarding_tests_installer:
   parallel:
     matrix:
-      - ONBOARDING_FILTER_WEBLOG: [test-app-java, test-app-java-container, test-app-java-container-jdk15, test-app-java-alpine-libgcc, test-app-java-buildpack]
+      - ONBOARDING_FILTER_WEBLOG: [test-app-java, test-app-java-container, test-app-java-container-jdk15, test-app-java-alpine-libgcc]
+        SCENARIO: [ SIMPLE_INSTALLER_AUTO_INJECTION, SIMPLE_AUTO_INJECTION_PROFILING ]
+      - ONBOARDING_FILTER_WEBLOG: [test-app-java-buildpack]
+        SCENARIO: [ SIMPLE_INSTALLER_AUTO_INJECTION ]
 
 onboarding_tests_k8s_injection:
   variables: