AWS-STS-S3 backingstore creation

Signed-off-by: Mahesh Shetty <[email protected]>

ocs_ci/framework/pytest_customization/marks.py

@@ -196,6 +196,11 @@
     reason="Test runs only on Stretch cluster with arbiter deployments",
 )
 
+sts_deployment_required = pytest.mark.skipif(
+    config.DEPLOYMENT.get("sts_enabled") is False,
+    reason="Test runs only on the AWS STS enabled cluster deployments",
+)
+
 google_api_required = pytest.mark.skipif(
     not os.path.exists(os.path.expanduser(config.RUN["google_api_secret"])),
     reason="Google API credentials don't exist",
@@ -386,6 +391,7 @@
     reason="Test will not run on Baremetal PSI",
 )
 
+
 skipif_managed_service = pytest.mark.skipif(
     config.ENV_DATA["platform"].lower() in MANAGED_SERVICE_PLATFORMS,
     reason="Test will not run on Managed service cluster",

ocs_ci/ocs/bucket_utils.py

@@ -676,6 +676,28 @@ def cli_create_aws_backingstore(mcg_obj, cld_mgr, backingstore_name, uls_name, r
     )
 
 
+def cli_create_aws_sts_backingstore(
+    mcg_obj, cld_mgr, backingstore_name, uls_name, region
+):
+    """
+    Create a new backingstore of type aws-sts-s3 with aws underlying storage and the role-ARN
+
+    Args:
+        mcg_obj (MCG): Used for execution for the NooBaa CLI command
+        cld_mgr (CloudManager): holds roleARN for backingstore creation
+        backingstore_name (str): backingstore name
+        uls_name (str): underlying storage name
+        region (str): which region to create backingstore (should be the same as uls)
+
+    """
+    mcg_obj.exec_mcg_cmd(
+        f"backingstore create aws-sts-s3 {backingstore_name} "
+        f"--aws-sts-arn {cld_mgr.aws_sts_client.role_arn} "
+        f"--target-bucket {uls_name} --region {region}",
+        use_yes=True,
+    )
+
+
 def oc_create_google_backingstore(cld_mgr, backingstore_name, uls_name, region):
     """
     Create a new backingstore with GCP underlying storage using oc create command

ocs_ci/ocs/constants.py

@@ -1981,7 +1981,7 @@
 PRODUCTION_JOBS_PREFIX = ["jnk"]
 
 # Cloud Manager available platforms
-CLOUD_MNGR_PLATFORMS = ["AWS", "GCP", "AZURE", "AZURE_WITH_LOGS", "IBMCOS"]
+CLOUD_MNGR_PLATFORMS = ["AWS", "GCP", "AZURE", "AZURE_WITH_LOGS", "IBMCOS", "AWS_STS"]
 
 # Vault related configurations
 VAULT_VERSION_INFO_URL = "https://github.com/hashicorp/vault/releases/latest"

ocs_ci/ocs/exceptions.py

@@ -690,3 +690,7 @@ class NoRunningCephToolBoxException(Exception):
 
 class UsernameNotFoundException(Exception):
     pass
+
+
+class ClusterNotInSTSModeException(Exception):
+    pass

ocs_ci/ocs/resources/backingstore.py

@@ -15,6 +15,7 @@
     cli_create_ibmcos_backingstore,
     cli_create_aws_backingstore,
     cli_create_rgw_backingstore,
+    cli_create_aws_sts_backingstore,
 )
 from ocs_ci.ocs.exceptions import (
     TimeoutExpiredError,
@@ -277,6 +278,7 @@ def backingstore_factory(request, cld_mgr, mcg_obj, cloud_uls_factory):
             "ibmcos": cli_create_ibmcos_backingstore,
             "rgw": cli_create_rgw_backingstore,
             "pv": cli_create_pv_backingstore,
+            "aws-sts": cli_create_aws_sts_backingstore,
         },
     }
 

ocs_ci/ocs/resources/cloud_manager.py

@@ -24,7 +24,11 @@
 from ocs_ci.ocs.resources.rgw import RGW
 from ocs_ci.utility import templating
 from ocs_ci.utility.aws import update_config_from_s3
-from ocs_ci.utility.utils import TimeoutSampler, load_auth_config
+from ocs_ci.utility.utils import (
+    TimeoutSampler,
+    load_auth_config,
+    get_role_arn_from_sub,
+)
 
 logger = logging.getLogger(name=__file__)
 
@@ -37,6 +41,7 @@ class CloudManager(ABC):
 
     def __init__(self):
         cloud_map = {
+            "AWS_STS": AwsSTSClient,
             "AWS": S3Client,
             "GCP": GoogleClient,
             "AZURE": AzureClient,
@@ -97,6 +102,13 @@ def __init__(self):
         except CommandFailed:
             setattr(self, "rgw_client", None)
 
+        # set the client for STS enabled cluster
+        role_arn = get_role_arn_from_sub()
+        cred_dict["AWS"]["ROLE_ARN"] = role_arn
+        setattr(
+            self, "aws_sts_client", cloud_map["AWS_STS"](auth_dict=cred_dict["AWS"])
+        )
+
 
 class CloudClient(ABC):
     """
@@ -630,3 +642,18 @@ def create_azure_secret(self):
         ).decode("ascii")
 
         return create_resource(**bs_secret_data)
+
+
+class AwsSTSClient(S3Client):
+    def __init__(
+        self,
+        auth_dict,
+        verify=True,
+        endpoint="https://s3.amazonaws.com",
+        *args,
+        **kwargs,
+    ):
+        super().__init__(
+            auth_dict=auth_dict, verify=verify, endpoint=endpoint, *args, **kwargs
+        )
+        self.role_arn = auth_dict["ROLE_ARN"]

ocs_ci/ocs/resources/cloud_uls.py

@@ -22,6 +22,7 @@ def cloud_uls_factory(request, cld_mgr):
 
     """
     all_created_uls = {
+        "aws-sts": set(),
         "aws": set(),
         "gcp": set(),
         "azure": set(),
@@ -31,6 +32,7 @@ def cloud_uls_factory(request, cld_mgr):
     }
     try:
         ulsMap = {
+            "aws-sts": cld_mgr.aws_sts_client,
             "aws": cld_mgr.aws_client,
             "gcp": cld_mgr.gcp_client,
             "azure": cld_mgr.azure_client,
@@ -48,6 +50,11 @@ def cloud_uls_factory(request, cld_mgr):
     except AttributeError:
         log.info("RGW is not available and was not initialized")
 
+    try:
+        ulsMap["aws-sts"] = cld_mgr.aws_sts_client
+    except Exception:
+        log.info("Cluster is not deployed STS mode")
+
     def _create_uls(uls_dict):
         """
         Creates and deletes all underlying storage that were created as part of the test
@@ -65,6 +72,7 @@ def _create_uls(uls_dict):
 
         """
         current_call_created_uls = {
+            "aws-sts": set(),
             "aws": set(),
             "gcp": set(),
             "azure": set(),

ocs_ci/utility/utils.py

@@ -37,7 +37,6 @@
 from semantic_version import Version
 from tempfile import NamedTemporaryFile, mkdtemp, TemporaryDirectory
 from jinja2 import FileSystemLoader, Environment
-
 from ocs_ci.framework import config
 from ocs_ci.framework import GlobalVariables as GV
 from ocs_ci.ocs import constants, defaults
@@ -57,6 +56,7 @@
     NotFoundError,
     CephToolBoxNotFoundException,
     NoRunningCephToolBoxException,
+    ClusterNotInSTSModeException,
 )
 from ocs_ci.utility import version as version_module
 from ocs_ci.utility.flexy import load_cluster_info
@@ -4689,3 +4689,26 @@ def exec_nb_db_query(query):
         output = output[2:-1]
 
     return output
+
+
+def get_role_arn_from_sub():
+    """
+    Get the RoleARN from the OCS subscription
+
+    """
+    from ocs_ci.ocs.ocp import OCP
+
+    if config.DEPLOYMENT.get("sts_enabled"):
+        role_arn = None
+        odf_sub = OCP(
+            kind=constants.SUBSCRIPTION,
+            resource_name=constants.ODF_SUBSCRIPTION,
+            namespace=constants.OPENSHIFT_STORAGE_NAMESPACE,
+        )
+        for item in odf_sub.get()["spec"]["config"]["env"]:
+            if item["name"] == "ROLEARN":
+                role_arn = item["value"]
+                break
+        return role_arn
+    else:
+        raise ClusterNotInSTSModeException

tests/functional/object/mcg/test_bucket_deletion.py

@@ -14,6 +14,7 @@
     runs_on_provider,
     red_squad,
     mcg,
+    sts_deployment_required,
 )
 from ocs_ci.framework.testlib import MCGTest
 from ocs_ci.helpers.helpers import create_unique_resource_name
@@ -87,6 +88,16 @@ class TestBucketDeletion(MCGTest):
                 ],
                 marks=[tier1],
             ),
+            pytest.param(
+                *[
+                    "CLI",
+                    {
+                        "interface": "CLI",
+                        "backingstore_dict": {"aws-sts": [(1, "eu-central-1")]},
+                    },
+                ],
+                marks=[tier1, sts_deployment_required],
+            ),
         ],
         ids=[
             "S3",
@@ -97,6 +108,7 @@ class TestBucketDeletion(MCGTest):
             "OC-GCP",
             "OC-IBMCOS",
             "CLI-IBMCOS",
+            "CLI-AWS-STS",
         ],
     )
     @flaky

tests/functional/object/mcg/test_bucket_replication.py

@@ -9,6 +9,7 @@
     runs_on_provider,
     red_squad,
     mcg,
+    sts_deployment_required,
 )
 from ocs_ci.ocs.exceptions import CommandFailed
 from ocs_ci.framework.testlib import MCGTest
@@ -59,6 +60,14 @@ class TestReplication(MCGTest):
                 {"interface": "OC", "backingstore_dict": {"azure": [(1, None)]}},
                 marks=[tier1, pytest.mark.polarion_id("OCS-2678")],
             ),
+            pytest.param(
+                {
+                    "interface": "CLI",
+                    "backingstore_dict": {"aws-sts": [(1, "eu-central-1")]},
+                },
+                {"interface": "OC", "backingstore_dict": {"azure": [(1, None)]}},
+                marks=[tier2, sts_deployment_required],
+            ),
             pytest.param(
                 {
                     "interface": "OC",
@@ -120,6 +129,7 @@ class TestReplication(MCGTest):
         ],
         ids=[
             "AWStoAZURE-BS-OC",
+            "AWS-STStoAZURE-BS-Hybrid",
             "GCPtoAWS-BS-OC",
             "AZUREtoCGP-BS-CLI",
             "AWStoAZURE-BS-CLI",
@@ -256,10 +266,16 @@ def test_unidirectional_namespace_bucket_replication(
                 {"interface": "OC", "backingstore_dict": {"azure": [(1, None)]}},
                 marks=[tier1, pytest.mark.polarion_id("OCS-2683")],
             ),
+            pytest.param(
+                {
+                    "interface": "CLI",
+                    "backingstore_dict": {"aws-sts": [(1, "eu-central-1")]},
+                },
+                {"interface": "OC", "backingstore_dict": {"azure": [(1, None)]}},
+                marks=[tier2, sts_deployment_required],
+            ),
         ],
-        ids=[
-            "AWStoAZURE-BS-OC",
-        ],
+        ids=["AWStoAZURE-BS-OC", "AWS-STStoAZURE-BS-Hybrid"],
     )
     def test_bidirectional_bucket_replication(
         self,