diff --git a/cmd/coreos-assembler.go b/cmd/coreos-assembler.go index 2824864fa5..e7e7e05e3f 100644 --- a/cmd/coreos-assembler.go +++ b/cmd/coreos-assembler.go @@ -16,7 +16,7 @@ var buildCommands = []string{"init", "fetch", "build", "run", "prune", "clean", var advancedBuildCommands = []string{"buildfetch", "buildupload", "oc-adm-release", "push-container"} var buildextendCommands = []string{"aliyun", "applehv", "aws", "azure", "digitalocean", "exoscale", "extensions-container", "gcp", "hashlist-experimental", "hyperv", "ibmcloud", "kubevirt", "live", "metal", "metal4k", "nutanix", "openstack", "qemu", "secex", "virtualbox", "vmware", "vultr"} -var utilityCommands = []string{"aws-replicate", "compress", "copy-container", "koji-upload", "kola", "push-container-manifest", "remote-build-container", "remote-prune", "remote-session", "sign", "tag", "update-variant"} +var utilityCommands = []string{"aws-replicate", "compress", "copy-container", "koji-upload", "kola", "push-container-manifest", "prune-containers", "remote-build-container", "remote-prune", "remote-session", "sign", "tag", "update-variant"} var otherCommands = []string{"shell", "meta"} func init() { diff --git a/src/cmd-prune-containers b/src/cmd-prune-containers new file mode 100755 index 0000000000..646fdc8716 --- /dev/null +++ b/src/cmd-prune-containers @@ -0,0 +1,171 @@ +#!/usr/bin/python3 -u + +# Prune containers from a remote registry +# according to the images age + +import argparse +import datetime +import json +import re as regexp +import os +import subprocess +from dateutil.relativedelta import relativedelta +import requests +import yaml + +# Dict of known streams +STREAMS = {"next": 1, "testing": 2, "stable": 3, + "next-devel": 10, "testing-devel": 20, + "rawhide": 91, "branched": 92} + + +def parse_args(): + parser = argparse.ArgumentParser(prog="coreos-assembler prune-containers") + parser.add_argument("--policy", required=True, type=str, help="Path to policy YAML file") + parser.add_argument("--dry-run", help="Don't actually delete anything", action='store_true') + parser.add_argument("-v", help="Increase verbosity", action='store_true') + parser.add_argument("--registry-auth-file", default=os.environ.get("REGISTRY_AUTHFILE"), + help="Path to docker registry auth file. Directly passed to skopeo.") + parser.add_argument("--stream", type=str, help="CoreOS stream", required=True, choices=STREAMS.keys()) + parser.add_argument("repository_url", help="container images URL") + return parser.parse_args() + + +def convert_to_days(duration_arg): + + days = 0 + match = regexp.match(r'^([0-9]+)([dDmMyYwW])$', duration_arg) + + if match is None: + raise ValueError(f"Incorrect duration '{duration_arg}'. Valid values are in the form of 1d, 2w, 3m, 4y") + + unit = match.group(2) + value = int(match.group(1)) + match unit: + case "y" | "Y": + days = value * 365 + case "m" | "M": + days = value * 30 + case "w" | "W": + days = value * 7 + case "d" | "D": + days = value + case _: + raise ValueError(f"Invalid unit '{match.group(2)}'. Please use y (years), m (months), w (weeks), or d (days).") + + return days + + +def skopeo_delete(repo, image, auth): + + skopeo_args = ["skopeo", "delete", f"docker://{repo}:{image}"] + if auth is not None: + skopeo_args.append(f"--authfile {auth}") + + subprocess.run(skopeo_args) + + +# FIXME : move to cosa_lib +# https://github.com/coreos/coreos-assembler/pull/3798/files#r1673481990 + +def parse_fcos_version(version): + m = regexp.match(r'^([0-9]{2})\.([0-9]{8})\.([0-9]+)\.([0-9]+)$', version) + if m is None: + raise ValueError(f"Incorrect versioning for FCOS build {version}") + try: + timestamp = datetime.datetime.strptime(m.group(2), '%Y%m%d') + except ValueError: + raise Exception(f"FCOS build {version} has incorrect date format. It should be in (%Y%m%d)") + return (timestamp, int(m.group(3))) + + +def get_update_graph(stream): + + url = f"https://builds.coreos.fedoraproject.org/updates/{stream}.json" + r = requests.get(url) + if r.status_code != 200: + raise Exception(f"Could not download update graph for {stream}. HTTP {r.status_code}") + return r.json() + + +class BarrierRelease(Exception): + pass + + +def main(): + + args = parse_args() + + # Load the policy file + with open(args.policy, "r") as f: + policy = yaml.safe_load(f) + if args.stream not in policy: + print(f"Stream {args.stream} is not defined in policy file.") + exit(1) + if 'containers' not in policy[args.stream]: + print(f"No containers section for {args.stream} stream in policy.") + exit(1) + policy = policy[args.stream]["containers"] + + print(f"Pulling tags from {args.repository_url}") + # This is a JSON object: + # {"Repository": "quay.io/jbtrystramtestimages/fcos", + # "Tags": [ + # "40.20"40.20240301.1.0",.....]} + tags_data = subprocess.check_output(["skopeo", "list-tags", + f"docker://{args.repository_url}"]) + + tags_json = json.loads(tags_data) + tags = tags_json['Tags'] + # Compute the date before we should prune images + # today - prune-policy + today = datetime.datetime.now() + date_limit = today - relativedelta(days=convert_to_days(policy)) + print(f"This will delete any images older than {date_limit} from the stream {args.stream}") + + stream_id = STREAMS[args.stream] + barrier_releases = [] + # Get the update graph for stable streams + if args.stream in ['stable', 'testing', 'next']: + update_graph = get_update_graph(args.stream)['releases'] + # Keep only the barrier releases + # filter(lambda release: "barrier" in release["metadata"], update_graph) + for release in update_graph: + if "barrier" in release["metadata"]: + barrier_releases.append(release["version"]) + + for tag in tags: + # silently skip known moving tags (next, stable...) + if tag in STREAMS: + continue + + # Process the build id and stream number + # TODO reuse this from https://github.com/coreos/coreos-assembler/pull/3798/files#r1673481990 + try: + (build_date, tag_stream) = parse_fcos_version(tag) + # ignore the named moving tags ("stable", "next" etc..) + except ValueError: + print(f"Ignoring unexpected tag: {tag}") + continue + if stream_id != tag_stream: + if args.v: + print(f"Skipping tag {tag} not in {args.stream} stream") + continue + + # Make sure this is not a barrier release (for stable streams) + # For non-production streams barrier_releases will be empty so + # this will be no-op + if tag in barrier_releases: + print(f"Release {tag} is a barrier release, keeping.") + continue + + if build_date < date_limit: + if args.dry_run: + print(f"Dry-run: would prune image {args.repository_url}:{tag}") + else: + print(f"Production tag {tag} is older than {date_limit.strftime("%Y%m%d")}, pruning.") + skopeo_delete(args.repository_url, tag, args.registry_auth_file) + + +if __name__ == "__main__": + main()