cmd/prune-containers: add a GC script for containers images

This script calls skopeo delete to prune image from a remote
directory. Currently only supports the FCOS tag structure.

This consumes the same policy.yaml defined in
#3798

See coreos/fedora-coreos-tracker#1367
See coreos/fedora-coreos-pipeline#995

cmd/coreos-assembler.go

@@ -16,7 +16,7 @@ var buildCommands = []string{"init", "fetch", "build", "run", "prune", "clean",
 var advancedBuildCommands = []string{"buildfetch", "buildupload", "oc-adm-release", "push-container"}
 var buildextendCommands = []string{"aliyun", "applehv", "aws", "azure", "digitalocean", "exoscale", "extensions-container", "gcp", "hashlist-experimental", "hyperv", "ibmcloud", "kubevirt", "live", "metal", "metal4k", "nutanix", "openstack", "qemu", "secex", "virtualbox", "vmware", "vultr"}
 
-var utilityCommands = []string{"aws-replicate", "compress", "copy-container", "koji-upload", "kola", "push-container-manifest", "remote-build-container", "remote-prune", "remote-session", "sign", "tag", "update-variant"}
+var utilityCommands = []string{"aws-replicate", "compress", "copy-container", "koji-upload", "kola", "push-container-manifest", "prune-containers", "remote-build-container", "remote-prune", "remote-session", "sign", "tag", "update-variant"}
 var otherCommands = []string{"shell", "meta"}
 
 func init() {

src/cmd-prune-containers

@@ -0,0 +1,171 @@
+#!/usr/bin/python3 -u
+
+# Prune containers from a remote registry
+# according to the images age
+
+import argparse
+import datetime
+import json
+import re as regexp
+import os
+import subprocess
+from dateutil.relativedelta import relativedelta
+import requests
+import yaml
+
+# Dict of known streams
+STREAMS = {"next": 1, "testing": 2, "stable": 3,
+           "next-devel": 10, "testing-devel": 20,
+           "rawhide": 91, "branched": 92}
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(prog="coreos-assembler prune-containers")
+    parser.add_argument("--policy", required=True, type=str, help="Path to policy YAML file")
+    parser.add_argument("--dry-run", help="Don't actually delete anything", action='store_true')
+    parser.add_argument("-v", help="Increase verbosity", action='store_true')
+    parser.add_argument("--registry-auth-file", default=os.environ.get("REGISTRY_AUTHFILE"),
+                        help="Path to docker registry auth file. Directly passed to skopeo.")
+    parser.add_argument("--stream", type=str, help="CoreOS stream", required=True, choices=STREAMS.keys())
+    parser.add_argument("repository_url", help="container images URL")
+    return parser.parse_args()
+
+
+def convert_to_days(duration_arg):
+
+    days = 0
+    match = regexp.match(r'^([0-9]+)([dDmMyYwW])$', duration_arg)
+
+    if match is None:
+        raise ValueError(f"Incorrect duration '{duration_arg}'. Valid values are in the form of 1d, 2w, 3m, 4y")
+
+    unit = match.group(2)
+    value = int(match.group(1))
+    match unit:
+        case "y" | "Y":
+            days = value * 365
+        case "m" | "M":
+            days = value * 30
+        case "w" | "W":
+            days = value * 7
+        case "d" | "D":
+            days = value
+        case _:
+            raise ValueError(f"Invalid unit '{match.group(2)}'. Please use y (years), m (months), w (weeks), or d (days).")
+
+    return days
+
+
+def skopeo_delete(repo, image, auth):
+
+    skopeo_args = ["skopeo", "delete", f"docker://{repo}:{image}"]
+    if auth is not None:
+        skopeo_args.append(f"--authfile {auth}")
+
+    subprocess.run(skopeo_args)
+
+
+# FIXME : move to cosa_lib
+# https://github.com/coreos/coreos-assembler/pull/3798/files#r1673481990
+
+def parse_fcos_version(version):
+    m = regexp.match(r'^([0-9]{2})\.([0-9]{8})\.([0-9]+)\.([0-9]+)$', version)
+    if m is None:
+        raise ValueError(f"Incorrect versioning for FCOS build {version}")
+    try:
+        timestamp = datetime.datetime.strptime(m.group(2), '%Y%m%d')
+    except ValueError:
+        raise Exception(f"FCOS build {version} has incorrect date format. It should be in (%Y%m%d)")
+    return (timestamp, int(m.group(3)))
+
+
+def get_update_graph(stream):
+
+    url = f"https://builds.coreos.fedoraproject.org/updates/{stream}.json"
+    r = requests.get(url)
+    if r.status_code != 200:
+        raise Exception(f"Could not download update graph for {stream}. HTTP {r.status_code}")
+    return r.json()
+
+
+class BarrierRelease(Exception):
+    pass
+
+
+def main():
+
+    args = parse_args()
+
+    # Load the policy file
+    with open(args.policy, "r") as f:
+        policy = yaml.safe_load(f)
+    if args.stream not in policy:
+        print(f"Stream {args.stream} is not defined in policy file.")
+        exit(1)
+    if 'containers' not in policy[args.stream]:
+        print(f"No containers section for {args.stream} stream in policy.")
+        exit(1)
+    policy = policy[args.stream]["containers"]
+
+    print(f"Pulling tags from {args.repository_url}")
+    # This is a JSON object:
+    # {"Repository": "quay.io/jbtrystramtestimages/fcos",
+    #    "Tags": [
+    #        "40.20"40.20240301.1.0",.....]}
+    tags_data = subprocess.check_output(["skopeo", "list-tags",
+                                        f"docker://{args.repository_url}"])
+
+    tags_json = json.loads(tags_data)
+    tags = tags_json['Tags']
+    # Compute the date before we should prune images
+    # today - prune-policy
+    today = datetime.datetime.now()
+    date_limit = today - relativedelta(days=convert_to_days(policy))
+    print(f"This will delete any images older than {date_limit} from the stream {args.stream}")
+
+    stream_id = STREAMS[args.stream]
+    barrier_releases = []
+    # Get the update graph for stable streams
+    if args.stream in ['stable', 'testing', 'next']:
+        update_graph = get_update_graph(args.stream)['releases']
+        # Keep only the barrier releases
+        # filter(lambda release: "barrier" in release["metadata"], update_graph)
+        for release in update_graph:
+            if "barrier" in release["metadata"]:
+                barrier_releases.append(release["version"])
+
+    for tag in tags:
+        # silently skip known moving tags (next, stable...)
+        if tag in STREAMS:
+            continue
+
+        # Process the build id and stream number
+        # TODO reuse this from  https://github.com/coreos/coreos-assembler/pull/3798/files#r1673481990
+        try:
+            (build_date, tag_stream) = parse_fcos_version(tag)
+        # ignore the named moving tags ("stable", "next" etc..)
+        except ValueError:
+            print(f"Ignoring unexpected tag: {tag}")
+            continue
+        if stream_id != tag_stream:
+            if args.v:
+                print(f"Skipping tag {tag} not in {args.stream} stream")
+            continue
+
+        # Make sure this is not a barrier release (for stable streams)
+        # For non-production streams barrier_releases will be empty so
+        # this will be no-op
+        if tag in barrier_releases:
+            print(f"Release {tag} is a barrier release, keeping.")
+            continue
+
+        if build_date < date_limit:
+            if args.dry_run:
+                print(f"Dry-run: would prune image {args.repository_url}:{tag}")
+            else:
+                print(f"Production tag {tag} is older than {date_limit.strftime("%Y%m%d")}, pruning.")
+                skopeo_delete(args.repository_url, tag, args.registry_auth_file)
+
+
+if __name__ == "__main__":
+    main()