Merge branch 'master' into alejandro.gonzalez/Send-secure-marks-to-ba…

…ckend

BUILDING.md

@@ -10,7 +10,7 @@ This documentation is dedicated to developers to setup their environment, and bu
 
 ## Development environment quick check
 
-As a preliminary step, you can quickly check that your development environment is properly set up to build the project running `./setup.sh` from the project root:
+To check that your development environment is properly set up to build the project, you will eventually run `./setup.sh` from the project root and should have its output look something like this:
 
 ```bash
 $ ./setup.sh

communication/src/main/java/datadog/communication/BackendApiFactory.java

@@ -34,8 +34,13 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni
             "Agentless mode is enabled and api key is not set. Please set application key");
       }
       String traceId = config.getIdGenerationStrategy().generateTraceId().toString();
-      long timeoutMillis = config.getCiVisibilityBackendApiTimeoutMillis();
-      return new IntakeApi(agentlessUrl, apiKey, traceId, timeoutMillis, retryPolicyFactory, true);
+      return new IntakeApi(
+          agentlessUrl,
+          apiKey,
+          traceId,
+          retryPolicyFactory,
+          sharedCommunicationObjects.okHttpClient,
+          true);
     }
 
     DDAgentFeaturesDiscovery featuresDiscovery =

communication/src/main/java/datadog/communication/IntakeApi.java

@@ -37,16 +37,15 @@ public IntakeApi(
       HttpUrl hostUrl,
       String apiKey,
       String traceId,
-      long timeoutMillis,
       HttpRetryPolicy.Factory retryPolicyFactory,
+      OkHttpClient httpClient,
       boolean responseCompression) {
     this.hostUrl = hostUrl;
     this.apiKey = apiKey;
     this.traceId = traceId;
     this.retryPolicyFactory = retryPolicyFactory;
     this.responseCompression = responseCompression;
-
-    httpClient = OkHttpUtils.buildHttpClient(hostUrl, timeoutMillis);
+    this.httpClient = httpClient;
   }
 
   @Override

communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java

@@ -36,11 +36,21 @@ public void createRemaining(Config config) {
       }
     }
     if (okHttpClient == null) {
-      String unixDomainSocket = SocketUtils.discoverApmSocket(config);
-      String namedPipe = config.getAgentNamedPipe();
+      String unixDomainSocket;
+      String namedPipe;
+      HttpUrl url;
+      if (!config.isCiVisibilityAgentlessEnabled()) {
+        unixDomainSocket = SocketUtils.discoverApmSocket(config);
+        namedPipe = config.getAgentNamedPipe();
+        url = agentUrl;
+      } else {
+        unixDomainSocket = null;
+        namedPipe = null;
+        url = null;
+      }
       okHttpClient =
           OkHttpUtils.buildHttpClient(
-              agentUrl, unixDomainSocket, namedPipe, getHttpClientTimeout(config));
+              url, unixDomainSocket, namedPipe, getHttpClientTimeout(config));
     }
   }
 

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityServices.java

@@ -1,12 +1,20 @@
 package datadog.trace.civisibility;
 
+import com.squareup.moshi.JsonAdapter;
+import com.squareup.moshi.Moshi;
+import com.squareup.moshi.Types;
 import datadog.communication.BackendApi;
 import datadog.communication.BackendApiFactory;
 import datadog.communication.ddagent.SharedCommunicationObjects;
+import datadog.communication.http.HttpRetryPolicy;
+import datadog.communication.http.OkHttpUtils;
 import datadog.trace.api.Config;
 import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
 import datadog.trace.api.git.GitInfoProvider;
 import datadog.trace.civisibility.ci.CIProviderInfoFactory;
+import datadog.trace.civisibility.ci.env.CiEnvironment;
+import datadog.trace.civisibility.ci.env.CiEnvironmentImpl;
+import datadog.trace.civisibility.ci.env.CompositeCiEnvironment;
 import datadog.trace.civisibility.config.CachingJvmInfoFactory;
 import datadog.trace.civisibility.config.JvmInfoFactory;
 import datadog.trace.civisibility.config.JvmInfoFactoryImpl;
@@ -19,28 +27,35 @@
 import datadog.trace.civisibility.source.ByteCodeMethodLinesResolver;
 import datadog.trace.civisibility.source.CompilerAidedMethodLinesResolver;
 import datadog.trace.civisibility.source.MethodLinesResolver;
-import datadog.trace.civisibility.source.index.CachingRepoIndexBuilderFactory;
-import datadog.trace.civisibility.source.index.ConventionBasedResourceResolver;
-import datadog.trace.civisibility.source.index.PackageResolver;
-import datadog.trace.civisibility.source.index.PackageResolverImpl;
-import datadog.trace.civisibility.source.index.RepoIndexFetcher;
-import datadog.trace.civisibility.source.index.RepoIndexProvider;
-import datadog.trace.civisibility.source.index.ResourceResolver;
+import datadog.trace.civisibility.source.index.*;
 import datadog.trace.civisibility.utils.ProcessHierarchyUtils;
+import java.lang.reflect.Type;
 import java.net.InetSocketAddress;
 import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
 import java.nio.file.Path;
+import java.util.Collections;
+import java.util.Map;
 import javax.annotation.Nullable;
+import okhttp3.HttpUrl;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import org.jetbrains.annotations.NotNull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Services that do not need repository root location to be instantiated. Can be shared between
  * multiple sessions.
  */
 public class CiVisibilityServices {
 
+  private static final Logger logger = LoggerFactory.getLogger(CiVisibilityServices.class);
+
   private static final String GIT_FOLDER_NAME = ".git";
 
+  static final String DD_ENV_VARS_PROVIDER_KEY_HEADER = "DD-Env-Vars-Provider-Key";
+
   final Config config;
   final CiVisibilityMetricCollector metricCollector;
   final BackendApi backendApi;
@@ -63,13 +78,15 @@ public class CiVisibilityServices {
         new BackendApiFactory(config, sco).createBackendApi(BackendApiFactory.Intake.API);
     this.jvmInfoFactory = new CachingJvmInfoFactory(config, new JvmInfoFactoryImpl());
     this.gitClientFactory = new GitClient.Factory(config, metricCollector);
-    this.ciProviderInfoFactory = new CIProviderInfoFactory(config);
+
+    CiEnvironment environment = buildCiEnvironment(config, sco);
+    this.ciProviderInfoFactory = new CIProviderInfoFactory(config, environment);
     this.methodLinesResolver =
         new BestEffortMethodLinesResolver(
             new CompilerAidedMethodLinesResolver(), new ByteCodeMethodLinesResolver());
 
     this.gitInfoProvider = gitInfoProvider;
-    gitInfoProvider.registerGitInfoBuilder(new CIProviderGitInfoBuilder());
+    gitInfoProvider.registerGitInfoBuilder(new CIProviderGitInfoBuilder(config, environment));
     gitInfoProvider.registerGitInfoBuilder(
         new CILocalGitInfoBuilder(gitClientFactory, GIT_FOLDER_NAME));
     gitInfoProvider.registerGitInfoBuilder(new GitClientGitInfoBuilder(config, gitClientFactory));
@@ -94,6 +111,55 @@ public class CiVisibilityServices {
     }
   }
 
+  @NotNull
+  private static CiEnvironment buildCiEnvironment(Config config, SharedCommunicationObjects sco) {
+    String remoteEnvVarsProviderUrl = config.getCiVisibilityRemoteEnvVarsProviderUrl();
+    if (remoteEnvVarsProviderUrl != null) {
+      String remoteEnvVarsProviderKey = config.getCiVisibilityRemoteEnvVarsProviderKey();
+      CiEnvironment remoteEnvironment =
+          new CiEnvironmentImpl(
+              getRemoteEnvironment(
+                  remoteEnvVarsProviderUrl, remoteEnvVarsProviderKey, sco.okHttpClient));
+      CiEnvironment localEnvironment = new CiEnvironmentImpl(System.getenv());
+      return new CompositeCiEnvironment(remoteEnvironment, localEnvironment);
+    } else {
+      return new CiEnvironmentImpl(System.getenv());
+    }
+  }
+
+  static Map<String, String> getRemoteEnvironment(String url, String key, OkHttpClient httpClient) {
+    HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0, true);
+
+    HttpUrl httpUrl = HttpUrl.get(url);
+    Request request =
+        new Request.Builder()
+            .url(httpUrl)
+            .header(DD_ENV_VARS_PROVIDER_KEY_HEADER, key)
+            .get()
+            .build();
+    try (okhttp3.Response response =
+        OkHttpUtils.sendWithRetries(httpClient, retryPolicyFactory, request)) {
+
+      if (response.isSuccessful()) {
+        Moshi moshi = new Moshi.Builder().build();
+        Type type = Types.newParameterizedType(Map.class, String.class, String.class);
+        JsonAdapter<Map<String, String>> adapter = moshi.adapter(type);
+        return adapter.fromJson(response.body().source());
+      } else {
+        logger.warn(
+            "Could not get remote CI environment (HTTP code "
+                + response.code()
+                + ")"
+                + (response.body() != null ? ": " + response.body().string() : ""));
+        return Collections.emptyMap();
+      }
+
+    } catch (Exception e) {
+      logger.warn("Could not get remote CI environment", e);
+      return Collections.emptyMap();
+    }
+  }
+
   CiVisibilityRepoServices repoServices(Path path) {
     return new CiVisibilityRepoServices(this, path);
   }

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ci/AppVeyorInfo.java

@@ -8,6 +8,7 @@
 import datadog.trace.api.git.CommitInfo;
 import datadog.trace.api.git.GitInfo;
 import datadog.trace.api.git.PersonInfo;
+import datadog.trace.civisibility.ci.env.CiEnvironment;
 
 class AppVeyorInfo implements CIProviderInfo {
 
@@ -31,13 +32,19 @@ class AppVeyorInfo implements CIProviderInfo {
   public static final String APPVEYOR_REPO_COMMIT_AUTHOR_EMAIL =
       "APPVEYOR_REPO_COMMIT_AUTHOR_EMAIL";
 
+  private final CiEnvironment environment;
+
+  AppVeyorInfo(CiEnvironment environment) {
+    this.environment = environment;
+  }
+
   @Override
   public GitInfo buildCIGitInfo() {
-    final String repoProvider = System.getenv(APPVEYOR_REPO_PROVIDER);
-    final String messageSubject = System.getenv(APPVEYOR_REPO_COMMIT_MESSAGE_SUBJECT);
-    final String messageBody = System.getenv(APPVEYOR_REPO_COMMIT_MESSAGE_BODY);
+    final String repoProvider = environment.get(APPVEYOR_REPO_PROVIDER);
+    final String messageSubject = environment.get(APPVEYOR_REPO_COMMIT_MESSAGE_SUBJECT);
+    final String messageBody = environment.get(APPVEYOR_REPO_COMMIT_MESSAGE_BODY);
     return new GitInfo(
-        buildGitRepositoryUrl(repoProvider, System.getenv(APPVEYOR_REPO_NAME)),
+        buildGitRepositoryUrl(repoProvider, environment.get(APPVEYOR_REPO_NAME)),
         buildGitBranch(repoProvider),
         buildGitTag(repoProvider),
         new CommitInfo(
@@ -60,23 +67,23 @@ private static String getCommitMessage(String messageSubject, String messageBody
   @Override
   public CIInfo buildCIInfo() {
     final String url =
-        buildPipelineUrl(System.getenv(APPVEYOR_REPO_NAME), System.getenv(APPVEYOR_BUILD_ID));
-    return CIInfo.builder()
+        buildPipelineUrl(environment.get(APPVEYOR_REPO_NAME), environment.get(APPVEYOR_BUILD_ID));
+    return CIInfo.builder(environment)
         .ciProviderName(APPVEYOR_PROVIDER_NAME)
-        .ciPipelineId(System.getenv(APPVEYOR_BUILD_ID))
-        .ciPipelineName(System.getenv(APPVEYOR_REPO_NAME))
-        .ciPipelineNumber(System.getenv(APPVEYOR_PIPELINE_NUMBER))
+        .ciPipelineId(environment.get(APPVEYOR_BUILD_ID))
+        .ciPipelineName(environment.get(APPVEYOR_REPO_NAME))
+        .ciPipelineNumber(environment.get(APPVEYOR_PIPELINE_NUMBER))
         .ciPipelineUrl(url)
         .ciJobUrl(url)
-        .ciWorkspace(expandTilde(System.getenv(APPVEYOR_WORKSPACE_PATH)))
+        .ciWorkspace(expandTilde(environment.get(APPVEYOR_WORKSPACE_PATH)))
         .build();
   }
 
   private String buildGitBranch(final String repoProvider) {
     if ("github".equals(repoProvider)) {
-      String branch = System.getenv(APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH);
+      String branch = environment.get(APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH);
       if (branch == null || branch.isEmpty()) {
-        branch = System.getenv(APPVEYOR_REPO_BRANCH);
+        branch = environment.get(APPVEYOR_REPO_BRANCH);
       }
       return normalizeBranch(branch);
     }
@@ -85,14 +92,14 @@ private String buildGitBranch(final String repoProvider) {
 
   private String buildGitTag(final String repoProvider) {
     if ("github".equals(repoProvider)) {
-      return normalizeTag(System.getenv(APPVEYOR_REPO_TAG_NAME));
+      return normalizeTag(environment.get(APPVEYOR_REPO_TAG_NAME));
     }
     return null;
   }
 
   private String buildGitCommit() {
-    if ("github".equals(System.getenv(APPVEYOR_REPO_PROVIDER))) {
-      return System.getenv(APPVEYOR_REPO_COMMIT);
+    if ("github".equals(environment.get(APPVEYOR_REPO_PROVIDER))) {
+      return environment.get(APPVEYOR_REPO_COMMIT);
     }
     return null;
   }
@@ -110,8 +117,8 @@ private String buildPipelineUrl(final String repoName, final String buildId) {
 
   private PersonInfo buildGitCommitAuthor() {
     return new PersonInfo(
-        System.getenv(APPVEYOR_REPO_COMMIT_AUTHOR_NAME),
-        System.getenv(APPVEYOR_REPO_COMMIT_AUTHOR_EMAIL));
+        environment.get(APPVEYOR_REPO_COMMIT_AUTHOR_NAME),
+        environment.get(APPVEYOR_REPO_COMMIT_AUTHOR_EMAIL));
   }
 
   @Override

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/ci/AwsCodePipelineInfo.java

@@ -2,6 +2,7 @@
 
 import datadog.trace.api.civisibility.telemetry.tag.Provider;
 import datadog.trace.api.git.GitInfo;
+import datadog.trace.civisibility.ci.env.CiEnvironment;
 
 class AwsCodePipelineInfo implements CIProviderInfo {
 
@@ -11,16 +12,22 @@ class AwsCodePipelineInfo implements CIProviderInfo {
   public static final String AWS_CODEPIPELINE_ACTION_EXECUTION_ID = "DD_ACTION_EXECUTION_ID";
   public static final String AWS_CODEPIPELINE_ARN = "CODEBUILD_BUILD_ARN";
 
+  private final CiEnvironment environment;
+
+  AwsCodePipelineInfo(CiEnvironment environment) {
+    this.environment = environment;
+  }
+
   @Override
   public GitInfo buildCIGitInfo() {
     return GitInfo.NOOP;
   }
 
   @Override
   public CIInfo buildCIInfo() {
-    return CIInfo.builder()
+    return CIInfo.builder(environment)
         .ciProviderName(AWS_CODEPIPELINE_PROVIDER_NAME)
-        .ciPipelineId(System.getenv(AWS_CODEPIPELINE_EXECUTION_ID))
+        .ciPipelineId(environment.get(AWS_CODEPIPELINE_EXECUTION_ID))
         .ciEnvVars(
             AWS_CODEPIPELINE_EXECUTION_ID,
             AWS_CODEPIPELINE_ACTION_EXECUTION_ID,