From ede15d1efe3fe20115039383bfcd5bddd7780385 Mon Sep 17 00:00:00 2001 From: Mark Street Date: Fri, 15 Sep 2023 11:56:18 +0100 Subject: [PATCH] 40 --- .github/workflows/ci.yaml | 3 +- TODO.md | 3 + download.py | 168 ++++++++++++++++++++++++++------------ 3 files changed, 121 insertions(+), 53 deletions(-) create mode 100644 TODO.md diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0038947..ef402dd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -38,7 +38,8 @@ jobs: - name: Create matrix of images to generate id: create-matrix run: | - echo "matrix=$(python3 matrix.py --dockerfiles changed_files.txt)" >> $GITHUB_OUTPUT + # echo "matrix=$(python3 matrix.py --dockerfiles changed_files.txt)" >> $GITHUB_OUTPUT + echo "matrix=$(python3 matrix.py --plaforms saturn)" >> $GITHUB_OUTPUT run_matrix: name: Run Matrix diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..0f78404 --- /dev/null +++ b/TODO.md @@ -0,0 +1,3 @@ +# TODO + +- update matrix.py to use values.yaml rather than glob diff --git a/download.py b/download.py index f25029e..01f6a69 100644 --- a/download.py +++ b/download.py @@ -1,33 +1,28 @@ #!/usr/bin/env python3 import argparse +import datetime import functools import logging import os import platform import shutil -import sys import tempfile from pathlib import Path from multiprocessing import Pool -try: - import docker -except ModuleNotFoundError: - print("Please 'pip install docker' package to use this script") - sys.exit(1) - +import docker +import podman logger = logging.getLogger(__name__) - COMPILERS_DIR: Path = Path(os.path.dirname(os.path.realpath(__file__))) DOWNLOAD_CACHE = COMPILERS_DIR / "download_cache" DOWNLOAD_CACHE.mkdir(exist_ok=True) -# TODO: can we do this better? +# TODO: can we pull this out into json/yaml and load based on HOST_ARCH? HOST_ARCH = platform.system().lower() if HOST_ARCH == "darwin": @@ -195,65 +190,121 @@ } -CLIENT = docker.from_env() +class ContainerManager(): + def pull(self, docker_image): + return self.client.images.pull(docker_image) + + def create_container(self, docker_image, commands=None): + if commands is None: + commands = [""] + return self.client.containers.create(docker_image, command=commands) + + +class PodmanManager(ContainerManager): + def __init__(self, uri="unix:///tmp/podman.sock"): + self.client = podman.PodmanClient(base_url=uri) + # sanity check that service is up and running + try: + self.client.images.list() + except FileNotFoundError: + raise Exception("%s not found, is the podman service running?") + + def get_remote_image_digest(self, docker_image, os="linux"): + # this is the arch-specific sha256 + try: + manifest = self.client.manifests.get(docker_image) + except podman.errors.exceptions.NotFound: + return None + os_digest = list(filter(lambda x: x["platform"]["os"] == os, manifest.attrs["manifests"])) + digest = os_digest[0]["digest"] + return digest + + def get_local_image_digest(self, docker_image): + local_tags = [] + for image in self.client.images.list(): + local_tags += image.tags + if docker_image in local_tags: + image = self.client.images.get(docker_image) + # NOTE: image.attrs["Digest"] is the overall sha256 of a multi-arch image + # but we cannot get the equivalent from the registry using podman's API. + rd = image.manager.get_registry_data(docker_image) + digest = rd.attrs["RepoDigests"][-1] + return digest.split("@")[-1] + + return None + + +class DockerManager(ContainerManager): + def __init__(self): + self.client = docker.from_env() + + def get_remote_image_digest(self, docker_image): + try: + # this is the overall sha256 of a multi-arch image + image = self.client.api.inspect_distribution(docker_image) + except docker.errors.APIError: + return None + digest = image["Descriptor"]["digest"] + return digest + + def get_local_image_digest(self, docker_image): + try: + image = self.client.api.inspect_image(docker_image) + except docker.errors.ImageNotFound: + return None + # TODO: confirm assumption that last one is the right one + digest = image["RepoDigests"][-1] + return digest.split("@")[-1] + +def get_compiler(platform_id, compiler_id, host_arch="linux", podman=False, force=False, github_repo="mkst/compilers"): + # TODO: seems to be issues trying to share a single instance? + client_manager = PodmanManager() if podman else DockerManager() -def get_compiler(platform_id, compiler_id, force=False, github_repo="mkst/compilers"): logger.info("Processing %s (%s)", compiler_id, platform_id) compiler_dir = COMPILERS_DIR / platform_id / compiler_id + image_digest = compiler_dir / ".image_digest" clean_compiler_id = compiler_id.lower().replace("+", "plus") # assume arch-less image to begin with docker_image = f"ghcr.io/{github_repo}/{platform_id}/{clean_compiler_id}:latest" - try: - logger.debug(f"Checking for %s in registry", docker_image) - remote_image_digest = CLIENT.api.inspect_distribution(docker_image)[ - "Descriptor" - ]["digest"] - except docker.errors.APIError: + logger.debug(f"Checking for %s in registry", docker_image) + remote_image_digest = client_manager.get_remote_image_digest(docker_image) + if remote_image_digest is None: logger.debug( f"%s not found in registry, checking for '%s' specific version", docker_image, - HOST_ARCH, + host_arch, ) - docker_image = f"ghcr.io/{github_repo}/{platform_id}/{clean_compiler_id}/{HOST_ARCH}:latest" - try: - remote_image_digest = CLIENT.api.inspect_distribution(docker_image)[ - "Descriptor" - ]["digest"] - except docker.errors.APIError: + docker_image = f"ghcr.io/{github_repo}/{platform_id}/{clean_compiler_id}/{host_arch}:latest" + remote_image_digest = client_manager.get_remote_image_digest(docker_image) + if remote_image_digest is None: logger.error(f"%s not found in registry!", docker_image) return - try: - local_image = CLIENT.api.inspect_image(docker_image) - logger.debug(f"%s exists locally", docker_image) - local_image_digest = local_image["RepoDigests"][-1].split("@")[-1] - except docker.errors.ImageNotFound: - logger.info(f"%s not found locally; pulling ...", docker_image) - local_image = CLIENT.images.pull(docker_image) - local_image_digest = local_image.attrs["RepoDigests"][-1].split("@")[-1] - - if remote_image_digest == local_image_digest: - if compiler_dir.exists(): - if force: - logger.warning( - f"%s is present and at latest version, continuing!", compiler_id - ) - else: - logger.info( - f"%s is present and at latest version, skipping!", compiler_id - ) - return + if not compiler_dir.exists() or force is True: + # we need to extract something, check if we need to pull the image + if client_manager.get_local_image_digest(docker_image) != remote_image_digest: + logger.info("%s has newer image available; pulling ...", docker_image) + client_manager.pull(docker_image) + else: + logger.info(f"%s is present and at latest version, continuing!", compiler_id) + else: + # compiler_dir exists, is it up to date with remote? + if image_digest.exists() and image_digest.read_text() == remote_image_digest: + logger.info(f"%s is present and at latest version, skipping!", compiler_id) + return + # image_digest missing or out of date, so pull logger.info("%s has newer image available; pulling ...", docker_image) - CLIENT.images.pull(docker_image) + client_manager.pull(docker_image) + try: - container = CLIENT.containers.create(docker_image, "ls") + container = client_manager.create_container(docker_image) except Exception as err: logger.error("Unable to create container for %s: %s", docker_image, err) return @@ -287,6 +338,8 @@ def get_compiler(platform_id, compiler_id, force=False, github_repo="mkst/compil ) shutil.move(str(DOWNLOAD_CACHE / compiler_id), str(compiler_dir)) + image_digest.write_text(remote_image_digest) + logger.info( "%s was successfully downloaded into %s", compiler_id, compiler_dir ) @@ -295,9 +348,14 @@ def get_compiler(platform_id, compiler_id, force=False, github_repo="mkst/compil container.remove() + return True + def main(): parser = argparse.ArgumentParser() + parser.add_argument( + "--podman", action="store_true", help="Use podman instead of docker" + ) parser.add_argument( "--force", help="(re)download compiler when latest image already present", @@ -316,25 +374,31 @@ def main(): to_download = [] for platform_id, compilers in COMPILERS.items(): + # platforms are considered enabled unless explicitly disabled platform_enabled = ( - os.environ.get(f"ENABLE_{platform_id.upper()}_SUPPORT", "YES").upper() - != "NO" + os.environ.get(f"ENABLE_{platform_id.upper()}_SUPPORT", "NO").upper() + == "YES" ) if platform_enabled: to_download += [(platform_id, compiler) for compiler in compilers] if len(to_download) == 0: - logger.warning("No compilers configured to be downloaded!") + logger.warning("No platforms are configured to be downloaded for host architecture (%s)", HOST_ARCH) return + start = datetime.datetime.now() with Pool(processes=args.threads) as pool: - pool.starmap( + results = pool.starmap( functools.partial( - get_compiler, force=args.force, github_repo=args.github_repo + get_compiler, host_arch=HOST_ARCH, podman=args.podman, force=args.force, github_repo=args.github_repo ), to_download, ) - logger.info("Finished processing %i compiler(s)", len(to_download)) + end = datetime.datetime.now() + + compilers_downloaded = len(list(filter(lambda x: x, results))) + logger.info("Updated %i / %i compiler(s) in %.2f second(s)", compilers_downloaded, len(to_download), (end - start).total_seconds()) + if __name__ == "__main__":