metal,vsphere-cluster-resource: retrieve EKS-A binary at runtime

Adds a new field for specifying the EKS-A release manifest URL to both
the metal and vSphere cluster resource agent configurations. At runtime,
the agents will download the EKS-A binary archive tagged as the latest
release in the EKS-A release manifest. If a manifest is not specified,
then the agents defaults to using the upstream official EKS-A release
manifest.

bottlerocket/agents/Cargo.toml

@@ -22,6 +22,7 @@ aws-sdk-cloudformation = "0.24"
 aws-smithy-types = "0.54"
 base64 = "0.20"
 env_logger = "0.10"
+flate2 = "1.0"
 hex ="0.4"
 k8s-openapi = { version = "0.18", default-features = false, features = ["v1_24"] }
 kube = { version = "0.82", default-features = false, features = ["config", "derive", "client"] }
@@ -37,6 +38,7 @@ serde_plain = "1"
 serde_yaml = "0.8"
 sha2 = "0.10"
 snafu = "0.7"
+tar = "0.4"
 tempfile = "3"
 test-agent = { version = "0.0.8", path = "../../agent/test-agent" }
 tokio = { version = "1", default-features = false, features = ["macros", "rt-multi-thread", "time"] }

bottlerocket/agents/src/bin/metal-k8s-cluster-resource-agent/metal_k8s_cluster_provider.rs

@@ -27,8 +27,10 @@ use agent_utils::aws::aws_config;
 use agent_utils::base64_decode_write_file;
 use agent_utils::ssm::{create_ssm_activation, ensure_ssm_service_role, wait_for_ssm_ready};
 use bottlerocket_agents::clusters::{
-    retrieve_workload_cluster_kubeconfig, write_validate_mgmt_kubeconfig,
+    fetch_eksa_binary, get_eksa_archive_url, retrieve_workload_cluster_kubeconfig,
+    write_validate_mgmt_kubeconfig,
 };
+use bottlerocket_agents::constants::{DEFAULT_EKS_A_RELEASE_MANIFEST, USR_LOCAL_BIN};
 use bottlerocket_types::agent_config::{
     CustomUserData, MetalK8sClusterConfig, AWS_CREDENTIALS_SECRET_NAME,
 };
@@ -47,6 +49,7 @@ use std::collections::HashSet;
 use std::convert::TryFrom;
 use std::fmt::Debug;
 use std::path::Path;
+use std::path::PathBuf;
 use std::process::{Command, Stdio};
 use std::time::Duration;
 use std::{env, fs};
@@ -233,6 +236,15 @@ impl Create for MetalK8sClusterCreator {
             ),
         )?;
 
+        let eks_a_release_manifest_url = &spec
+            .configuration
+            .eks_a_release_manifest_url
+            .unwrap_or(DEFAULT_EKS_A_RELEASE_MANIFEST.to_string());
+        let eks_a_archive_url =
+            get_eksa_archive_url(eks_a_release_manifest_url, &resources).await?;
+        info!("Fetching EKS-A binary archive from '{}'", eks_a_archive_url);
+        fetch_eksa_binary(eks_a_archive_url, PathBuf::from(USR_LOCAL_BIN), &resources).await?;
+
         info!("Creating cluster");
         memo.current_status = "Creating cluster".to_string();
         memo.cluster_name = Some(cluster_name.clone());
@@ -536,6 +548,15 @@ impl Destroy for MetalK8sClusterDestroyer {
         let configuration = spec
             .context(resources, "The spec was not provided for destruction")?
             .configuration;
+
+        let eks_a_release_manifest_url = &configuration
+            .eks_a_release_manifest_url
+            .unwrap_or(DEFAULT_EKS_A_RELEASE_MANIFEST.to_string());
+        let eks_a_archive_url =
+            get_eksa_archive_url(eks_a_release_manifest_url, &resources).await?;
+        info!("Fetching EKS-A binary archive from '{}'", eks_a_archive_url);
+        fetch_eksa_binary(eks_a_archive_url, PathBuf::from(USR_LOCAL_BIN), &resources).await?;
+
         base64_decode_write_file(
             &configuration.mgmt_cluster_kubeconfig_base64,
             &mgmt_kubeconfig_path,

bottlerocket/agents/src/bin/vsphere-k8s-cluster-resource-agent/vsphere_k8s_cluster_provider.rs

@@ -1,8 +1,11 @@
 use agent_utils::base64_decode_write_file;
 use bottlerocket_agents::clusters::{
-    retrieve_workload_cluster_kubeconfig, write_validate_mgmt_kubeconfig,
+    fetch_eksa_binary, get_eksa_archive_url, retrieve_workload_cluster_kubeconfig,
+    write_validate_mgmt_kubeconfig,
+};
+use bottlerocket_agents::constants::{
+    DEFAULT_EKS_A_RELEASE_MANIFEST, TEST_CLUSTER_KUBECONFIG_PATH, USR_LOCAL_BIN,
 };
-use bottlerocket_agents::constants::TEST_CLUSTER_KUBECONFIG_PATH;
 use bottlerocket_agents::is_cluster_creation_required;
 use bottlerocket_agents::tuf::{download_target, tuf_repo_urls};
 use bottlerocket_agents::vsphere::vsphere_credentials;
@@ -24,6 +27,7 @@ use std::convert::TryFrom;
 use std::fmt::Debug;
 use std::fs::File;
 use std::path::Path;
+use std::path::PathBuf;
 use std::process::{Command, Stdio};
 use std::{env, fs};
 use testsys_model::{Configuration, SecretName};
@@ -148,6 +152,16 @@ impl Create for VSphereK8sClusterCreator {
 
         let mgmt_kubeconfig_path = format!("{}/mgmt.kubeconfig", WORKING_DIR);
         let encoded_kubeconfig = if do_create {
+            let eks_a_release_manifest_url = &spec
+                .configuration
+                .eks_a_release_manifest_url
+                .to_owned()
+                .unwrap_or(DEFAULT_EKS_A_RELEASE_MANIFEST.to_string());
+            let eks_a_archive_url =
+                get_eksa_archive_url(eks_a_release_manifest_url, &resources).await?;
+            info!("Fetching EKS-A binary archive from '{}'", eks_a_archive_url);
+            fetch_eksa_binary(eks_a_archive_url, PathBuf::from(USR_LOCAL_BIN), &resources).await?;
+
             info!("Creating cluster");
             memo.current_status = "Creating cluster".to_string();
             client
@@ -673,6 +687,15 @@ impl Destroy for VSphereK8sClusterDestroyer {
             "Failed to write out kubeconfig for the CAPI management cluster",
         )?;
 
+        let eks_a_release_manifest_url = &spec
+            .configuration
+            .eks_a_release_manifest_url
+            .unwrap_or(DEFAULT_EKS_A_RELEASE_MANIFEST.to_string());
+        let eks_a_archive_url =
+            get_eksa_archive_url(eks_a_release_manifest_url, &resources).await?;
+        info!("Fetching EKS-A binary archive from '{}'", eks_a_archive_url);
+        fetch_eksa_binary(eks_a_archive_url, PathBuf::from(USR_LOCAL_BIN), &resources).await?;
+
         // For cluster deletion, EKS-A needs the workload cluster's kubeconfig at
         // './${CLUSTER_NAME}/${CLUSTER_NAME}-eks-a-cluster.kubeconfig'
         let cluster_dir = format!("{}/{}/", WORKING_DIR, &spec.configuration.name);

bottlerocket/agents/src/clusters.rs

@@ -1,10 +1,16 @@
 use agent_utils::base64_decode_write_file;
+use flate2::read::GzDecoder;
 use k8s_openapi::api::core::v1::Secret;
 use kube::config::Kubeconfig;
 use kube::{Api, Config};
 use log::debug;
-use resource_agent::provider::{IntoProviderError, ProviderResult, Resources};
+use reqwest::IntoUrl;
+use resource_agent::provider::{IntoProviderError, ProviderError, ProviderResult, Resources};
+use serde::Deserialize;
 use std::convert::TryFrom;
+use std::env;
+use std::fmt::Display;
+use std::path::PathBuf;
 
 /// Write out and check CAPI management cluster is accessible and valid
 pub async fn write_validate_mgmt_kubeconfig(
@@ -60,3 +66,113 @@ pub async fn retrieve_workload_cluster_kubeconfig(
         .trim_matches('"')
         .to_string())
 }
+
+pub async fn get_eksa_archive_url<S>(
+    eks_a_release_manifest_url: S,
+    resources: &Resources,
+) -> ProviderResult<String>
+where
+    S: AsRef<str> + IntoUrl + Display,
+{
+    let manifest = reqwest::get(eks_a_release_manifest_url.to_string())
+        .await
+        .context(
+            resources,
+            format!(
+                "Unable to request EKS-A release manifest '{}'",
+                eks_a_release_manifest_url
+            ),
+        )?
+        .text()
+        .await
+        .context(
+            resources,
+            format!(
+                "Unable to retrieve EKS-A release manifest at '{}'",
+                eks_a_release_manifest_url
+            ),
+        )?;
+    let deserialized_manifest = serde_yaml::Deserializer::from_str(&manifest)
+        .map(|config| {
+            serde_yaml::Value::deserialize(config)
+                .context(resources, "Unable to deserialize eksa config file")
+        })
+        .collect::<ProviderResult<Vec<_>>>()?;
+
+    let latest_release_version = deserialized_manifest
+        .iter()
+        .find(|config| {
+            config.get("kind") == Some(&serde_yaml::Value::String("Release".to_string()))
+        })
+        .and_then(|release| release.get("spec"))
+        .and_then(|spec| spec.get("latestVersion"))
+        .and_then(|ver| ver.as_str())
+        .context(resources, "Unable to get latest version for EKS-A")?;
+
+    let arch = match env::consts::ARCH {
+        "x86_64" => "amd64",
+        "aarch64" => "arm64",
+        rest => rest,
+    };
+    deserialized_manifest
+        .iter()
+        .find(|manifests| {
+            manifests.get("kind") == Some(&serde_yaml::Value::String("Release".to_string()))
+        })
+        .and_then(|manifest| manifest.get("spec"))
+        .and_then(|spec| spec.get("releases"))
+        .and_then(|list| list.as_sequence())
+        .and_then(|releases| {
+            releases.iter().find(|release| {
+                release.get("version")
+                    == Some(&serde_yaml::Value::String(
+                        latest_release_version.to_string(),
+                    ))
+            })
+        })
+        .and_then(|release| release.get("eksACLI"))
+        .and_then(|list| list.get(env::consts::OS.to_ascii_lowercase()))
+        .and_then(|binaries| binaries.get(arch.to_ascii_lowercase()))
+        .and_then(|binaries| binaries.get("uri"))
+        .and_then(|uri| uri.as_str())
+        .context(
+            resources,
+            format!(
+                "Unable to get the URL for the latest EKS-A version ({})",
+                latest_release_version
+            ),
+        )
+        .map(|s| s.to_string())
+}
+
+pub async fn fetch_eksa_binary(
+    archive_url: String,
+    dest: PathBuf,
+    resources: &Resources,
+) -> ProviderResult<()> {
+    if !archive_url.ends_with("tar.gz") {
+        return Err(ProviderError::new_with_context(
+            resources,
+            format!(
+                "EKS-A binary archive at '{}' is not tar.gz compressed",
+                archive_url
+            ),
+        ));
+    }
+    let tar_bytes = reqwest::get(&archive_url)
+        .await
+        .context(
+            resources,
+            format!("Unable to request binary at '{}'", archive_url),
+        )?
+        .bytes()
+        .await
+        .context(resources, "Unable to retrieve binary archive bytes.")?;
+
+    // Decompress tar.gz archive
+    let decoder = GzDecoder::new(&tar_bytes[..]);
+    let mut archive = tar::Archive::new(decoder);
+    archive
+        .unpack(dest)
+        .context(resources, "Failed to unpack EKS-A binary archive.")
+}

bottlerocket/agents/src/constants.rs

@@ -1,3 +1,6 @@
 pub const DEFAULT_TASK_DEFINITION: &str = "testsys-bottlerocket-aws-default-ecs-smoke-test-v1";
 pub const TEST_CLUSTER_KUBECONFIG_PATH: &str = "/local/test-cluster.kubeconfig";
 pub const E2E_REPO_CONFIG_PATH: &str = "/local/e2e-repo-config.yaml";
+pub const USR_LOCAL_BIN: &str = "/usr/local/bin/";
+pub const DEFAULT_EKS_A_RELEASE_MANIFEST: &str =
+    "https://anywhere-assets.eks.amazonaws.com/releases/eks-a/manifest.yaml";

bottlerocket/types/src/agent_config.rs

@@ -184,6 +184,10 @@ pub struct VSphereK8sClusterConfig {
     /// Workloads folder to create the K8s cluster control plane in
     pub vcenter_workload_folder: String,
 
+    /// URL for an EKS-A release manifest that contains URLs for EKS-A binary archives.
+    /// Defaults to upstream EKS-A release channels.
+    pub eks_a_release_manifest_url: Option<String>,
+
     /// Base64-encoded Kubeconfig for the CAPI management cluster
     pub mgmt_cluster_kubeconfig_base64: String,
 }
@@ -193,7 +197,11 @@ pub struct VSphereK8sClusterConfig {
 #[serde(rename_all = "camelCase")]
 #[crd("Resource")]
 pub struct MetalK8sClusterConfig {
-    // Base64-encoded Kubeconfig for the CAPI management cluster
+    /// URL for an EKS-A release manifest that contains URLs for EKS-A binary archives.
+    /// Defaults to upstream EKS-A release channels.
+    pub eks_a_release_manifest_url: Option<String>,
+
+    /// Base64-encoded Kubeconfig for the CAPI management cluster
     pub mgmt_cluster_kubeconfig_base64: String,
 
     /// The role that should be assumed when activating SSM for the machines.