Change the bucket cleanup script (red-hat-storage#9353)

* Change the bucket cleanup script

Signed-off-by: oviner <[email protected]>

ocs_ci/cleanup/aws/cleanup.py

@@ -312,19 +312,20 @@ def cluster_cleanup():
         p.join()
 
 
-def delete_buckets(bucket_prefix, days):
+def delete_buckets(bucket_prefix, hours):
     """
     Delete the S3 buckets with given prefix
 
     Args:
-        bucket_prefix (str): Bucket prefix to delete.
-        days (int): Days older than this will be considered to delete
+        bucket_prefix (dict): Bucket prefix as key and maximum hours to run/exist as value
+        hours (int): hours older than this will be considered to delete
 
     """
     aws = AWS()
-    buckets_to_delete = aws.get_buckets_with_prefix_(bucket_prefix, days)
+    buckets_to_delete = aws.get_buckets_to_delete(bucket_prefix, hours)
     logger.info(f"buckets to delete: {buckets_to_delete}")
     for bucket in buckets_to_delete:
+        logger.info(f"Delete bucket {bucket}")
         aws.delete_bucket(bucket)
 
 
@@ -341,7 +342,10 @@ def aws_cleanup():
         help="""
             Maximum running time of the cluster (in hours).
             Clusters older than this will be deleted.
-            The minimum is 10 hours
+            The minimum is 10 hours.
+            If sweep-buckets flag enabled:
+            Running time for the buckets in hours.
+            Buckets older than to this will be deleted.
             """,
     )
     parser.add_argument(
@@ -385,29 +389,15 @@ def aws_cleanup():
     bucket_group.add_argument(
         "--sweep-buckets", action="store_true", help="Deleting S3 buckets."
     )
-    bucket_group.add_argument(
-        "--bucket-prefix",
-        help="Prefix for S3 buckets. Only buckets with this prefix will be considered to delete.",
-    )
-    bucket_group.add_argument(
-        "--days",
-        type=int,
-        default=5,
-        help="""
-                Maximum running time for the buckets in days.
-                Buckets older than to this will be deleted.
-                The minimum is 5 days
-                """,
-    )
-
     args = parser.parse_args()
 
     if args.sweep_buckets:
-        if not args.bucket_prefix:
-            parser.error(
-                "--bucket-prefix is required when --sweep-buckets is specified."
-            )
-        delete_buckets(args.bucket_prefix, args.days)
+        bucket_hours = (
+            args.hours
+            if args.hours is not None
+            else defaults.DEFAULT_BUCKET_RUNNING_TIME
+        )
+        delete_buckets(defaults.BUCKET_PREFIXES_SPECIAL_RULES, bucket_hours)
         return
 
     if not args.force:

ocs_ci/cleanup/aws/defaults.py

@@ -1,6 +1,8 @@
 """
 Defaults module for AWS cleanup
 """
+import sys
+
 AWS_REGION = "us-east-2"
 CLUSTER_PREFIXES_SPECIAL_RULES = {
     "jnk-pr": 16,  # keep it as first item before jnk prefix for fist match
@@ -16,3 +18,22 @@
 }
 MINIMUM_CLUSTER_RUNNING_TIME = 10
 CONFIRMATION_ANSWER = "yes-i-am-sure-i-want-to-proceed"
+
+BUCKET_PREFIXES_SPECIAL_RULES = {
+    "dnd": sys.maxsize,
+    "acmobservability": sys.maxsize,
+    "ocs-ci-data": sys.maxsize,
+    "ocs-ci-public": sys.maxsize,
+    "ocs-qe-upi": sys.maxsize,
+    "ocs-qe-upi-1": sys.maxsize,
+    "ocs-qe-upi-us-east-2": sys.maxsize,
+    "ocsci-test-files": sys.maxsize,
+    "openshift-qe-upi": sys.maxsize,
+    "j-": 500,
+    "lr1-": 100,
+    "lr2-": 150,
+    "lr3-": 200,
+    "lr4-": 250,
+    "lr5-": 300,
+}
+DEFAULT_BUCKET_RUNNING_TIME = 100

ocs_ci/utility/aws.py

@@ -1935,37 +1935,56 @@ def list_buckets(self):
         """
         return self.s3_client.list_buckets()["Buckets"]
 
-    def get_buckets_with_prefix_(self, bucket_prefix, days):
+    def get_buckets_to_delete(self, bucket_prefix, hours):
         """
-        Get the bucket with prefix which are older than given days
+        Get the bucket with prefix which are older than given hours
 
         Args:
             bucket_prefix (str): prefix for the buckets to fetch
-            days (int): fetch buckets that are older than to the specified number of days
+            hours (int): fetch buckets that are older than to the specified number of hours
 
         """
-        buckets_with_prefix = []
+        buckets_to_delete = []
         # Get the current date in UTC
         current_date = datetime.now(timezone.utc)
         all_buckets = self.list_buckets()
         for bucket in all_buckets:
             bucket_name = bucket["Name"]
-            if bucket_name.startswith(bucket_prefix):
-                # Get the creation date of the bucket in UTC
-                bucket_creation_date = bucket["CreationDate"].replace(
-                    tzinfo=timezone.utc
+
+            bucket_delete_time = self.get_bucket_time_based_rules(
+                bucket_prefix, bucket_name, hours
+            )
+            # Get the creation date of the bucket in UTC
+            bucket_creation_date = bucket["CreationDate"].replace(tzinfo=timezone.utc)
+
+            # Calculate the age of the bucket
+            age_of_bucket = current_date - bucket_creation_date
+
+            # Check if the bucket is older than given hours
+            if (age_of_bucket.days) * 24 >= bucket_delete_time:
+                logger.info(
+                    f"{bucket_name} (Created on {bucket_creation_date} and age is {age_of_bucket}) can be deleted"
                 )
+                buckets_to_delete.append(bucket_name)
+        return buckets_to_delete
+
+    def get_bucket_time_based_rules(self, bucket_prefixes, bucket_name, hours):
+        """
+        Get the time bucket based prefix and hours
 
-                # Calculate the age of the bucket
-                age_of_bucket = current_date - bucket_creation_date
+        Args:
+            bucket_prefixes (dict): The rules according to them determine the number of hours the bucket can exist
+            bucket_name (str): bucket name
+            hours (int): The number of hours bucket can exist if there is no compliance with one of the rules
 
-                # Check if the bucket is older than given days
-                if age_of_bucket.days >= days:
-                    logger.info(
-                        f"{bucket_name} (Created on {bucket_creation_date} and age is {age_of_bucket}) can be deleted"
-                    )
-                    buckets_with_prefix.append(bucket_name)
-        return buckets_with_prefix
+        Returns:
+            int: The number of hours bucket can exist
+
+        """
+        for bucket_prefix in bucket_prefixes:
+            if bool(re.match(bucket_prefix, bucket_name, re.I)):
+                return bucket_prefixes[bucket_prefix]
+        return hours
 
     def delete_objects_in_bucket(self, bucket):
         """
@@ -2484,7 +2503,7 @@ def create_and_attach_ebs_volumes(
                     instance_id=worker["id"],
                     name=f"{worker['name']}_extra_volume_{number}",
                     size=size,
-                    device=f"/dev/{device_names[number-1]}",
+                    device=f"/dev/{device_names[number - 1]}",
                 )