40

.github/workflows/ci.yaml

@@ -38,7 +38,8 @@ jobs:
       - name: Create matrix of images to generate
         id: create-matrix
         run: |
-          echo "matrix=$(python3 matrix.py --dockerfiles changed_files.txt)" >> $GITHUB_OUTPUT
+          # echo "matrix=$(python3 matrix.py --dockerfiles changed_files.txt)" >> $GITHUB_OUTPUT
+          echo "matrix=$(python3 matrix.py --plaforms saturn)" >> $GITHUB_OUTPUT
 
   run_matrix:
     name: Run Matrix

TODO.md

@@ -0,0 +1,3 @@
+# TODO
+
+- update matrix.py to use values.yaml rather than glob

download.py

@@ -1,33 +1,28 @@
 #!/usr/bin/env python3
 
 import argparse
+import datetime
 import functools
 import logging
 import os
 import platform
 import shutil
-import sys
 import tempfile
 
 from pathlib import Path
 
 from multiprocessing import Pool
 
-try:
-    import docker
-except ModuleNotFoundError:
-    print("Please 'pip install docker' package to use this script")
-    sys.exit(1)
-
+import docker
+import podman
 
 logger = logging.getLogger(__name__)
 
-
 COMPILERS_DIR: Path = Path(os.path.dirname(os.path.realpath(__file__)))
 DOWNLOAD_CACHE = COMPILERS_DIR / "download_cache"
 DOWNLOAD_CACHE.mkdir(exist_ok=True)
 
-# TODO: can we do this better?
+# TODO: can we pull this out into json/yaml and load based on HOST_ARCH?
 HOST_ARCH = platform.system().lower()
 
 if HOST_ARCH == "darwin":
@@ -195,65 +190,121 @@
     }
 
 
-CLIENT = docker.from_env()
+class ContainerManager():
+    def pull(self, docker_image):
+        return self.client.images.pull(docker_image)
+
+    def create_container(self, docker_image, commands=None):
+        if commands is None:
+            commands = [""]
+        return self.client.containers.create(docker_image, command=commands)
+
+
+class PodmanManager(ContainerManager):
+    def __init__(self, uri="unix:///tmp/podman.sock"):
+        self.client = podman.PodmanClient(base_url=uri)
+        # sanity check that service is up and running
+        try:
+            self.client.images.list()
+        except FileNotFoundError:
+            raise Exception("%s not found, is the podman service running?")
+
+    def get_remote_image_digest(self, docker_image, os="linux"):
+        # this is the arch-specific sha256
+        try:
+            manifest = self.client.manifests.get(docker_image)
+        except podman.errors.exceptions.NotFound:
+            return None
+        os_digest = list(filter(lambda x: x["platform"]["os"] == os, manifest.attrs["manifests"]))
+        digest = os_digest[0]["digest"]
+        return digest
+
+    def get_local_image_digest(self, docker_image):
+        local_tags = []
+        for image in self.client.images.list():
+            local_tags += image.tags
+        if docker_image in local_tags:
+            image = self.client.images.get(docker_image)
+            # NOTE: image.attrs["Digest"] is the overall sha256 of a multi-arch image
+            #       but we cannot get the equivalent from the registry using podman's API.
+            rd = image.manager.get_registry_data(docker_image)
+            digest = rd.attrs["RepoDigests"][-1]
+            return digest.split("@")[-1]
+
+        return None
+
+
+class DockerManager(ContainerManager):
+    def __init__(self):
+        self.client = docker.from_env()
+
+    def get_remote_image_digest(self, docker_image):
+        try:
+            # this is the overall sha256 of a multi-arch image
+            image = self.client.api.inspect_distribution(docker_image)
+        except docker.errors.APIError:
+            return None
+        digest = image["Descriptor"]["digest"]
+        return digest
+
+    def get_local_image_digest(self, docker_image):
+        try:
+            image = self.client.api.inspect_image(docker_image)
+        except docker.errors.ImageNotFound:
+            return None
+        # TODO: confirm assumption that last one is the right one
+        digest = image["RepoDigests"][-1]
+        return digest.split("@")[-1]
+
 
+def get_compiler(platform_id, compiler_id, host_arch="linux", podman=False, force=False, github_repo="mkst/compilers"):
+    # TODO: seems to be issues trying to share a single instance?
+    client_manager = PodmanManager() if podman else DockerManager()
 
-def get_compiler(platform_id, compiler_id, force=False, github_repo="mkst/compilers"):
     logger.info("Processing %s (%s)", compiler_id, platform_id)
 
     compiler_dir = COMPILERS_DIR / platform_id / compiler_id
+    image_digest = compiler_dir / ".image_digest"
 
     clean_compiler_id = compiler_id.lower().replace("+", "plus")
 
     # assume arch-less image to begin with
     docker_image = f"ghcr.io/{github_repo}/{platform_id}/{clean_compiler_id}:latest"
 
-    try:
-        logger.debug(f"Checking for %s in registry", docker_image)
-        remote_image_digest = CLIENT.api.inspect_distribution(docker_image)[
-            "Descriptor"
-        ]["digest"]
-    except docker.errors.APIError:
+    logger.debug(f"Checking for %s in registry", docker_image)
+    remote_image_digest = client_manager.get_remote_image_digest(docker_image)
+    if remote_image_digest is None:
         logger.debug(
             f"%s not found in registry, checking for '%s' specific version",
             docker_image,
-            HOST_ARCH,
+            host_arch,
         )
-        docker_image = f"ghcr.io/{github_repo}/{platform_id}/{clean_compiler_id}/{HOST_ARCH}:latest"
-        try:
-            remote_image_digest = CLIENT.api.inspect_distribution(docker_image)[
-                "Descriptor"
-            ]["digest"]
-        except docker.errors.APIError:
+        docker_image = f"ghcr.io/{github_repo}/{platform_id}/{clean_compiler_id}/{host_arch}:latest"
+        remote_image_digest = client_manager.get_remote_image_digest(docker_image)
+        if remote_image_digest is None:
             logger.error(f"%s not found in registry!", docker_image)
             return
 
-    try:
-        local_image = CLIENT.api.inspect_image(docker_image)
-        logger.debug(f"%s exists locally", docker_image)
-        local_image_digest = local_image["RepoDigests"][-1].split("@")[-1]
-    except docker.errors.ImageNotFound:
-        logger.info(f"%s not found locally; pulling ...", docker_image)
-        local_image = CLIENT.images.pull(docker_image)
-        local_image_digest = local_image.attrs["RepoDigests"][-1].split("@")[-1]
-
-    if remote_image_digest == local_image_digest:
-        if compiler_dir.exists():
-            if force:
-                logger.warning(
-                    f"%s is present and at latest version, continuing!", compiler_id
-                )
-            else:
-                logger.info(
-                    f"%s is present and at latest version, skipping!", compiler_id
-                )
-                return
+    if not compiler_dir.exists() or force is True:
+        # we need to extract something, check if we need to pull the image
+        if client_manager.get_local_image_digest(docker_image) != remote_image_digest:
+            logger.info("%s has newer image available; pulling ...", docker_image)
+            client_manager.pull(docker_image)
+        else:
+            logger.info(f"%s is present and at latest version, continuing!", compiler_id)
+
     else:
+        # compiler_dir exists, is it up to date with remote?
+        if image_digest.exists() and image_digest.read_text() == remote_image_digest:
+            logger.info(f"%s is present and at latest version, skipping!", compiler_id)
+            return
+        # image_digest missing or out of date, so pull
         logger.info("%s has newer image available; pulling ...", docker_image)
-        CLIENT.images.pull(docker_image)
+        client_manager.pull(docker_image)
+
 
     try:
-        container = CLIENT.containers.create(docker_image, "ls")
+        container = client_manager.create_container(docker_image)
     except Exception as err:
         logger.error("Unable to create container for %s: %s", docker_image, err)
         return
@@ -287,6 +338,8 @@ def get_compiler(platform_id, compiler_id, force=False, github_repo="mkst/compil
             )
             shutil.move(str(DOWNLOAD_CACHE / compiler_id), str(compiler_dir))
 
+            image_digest.write_text(remote_image_digest)
+
             logger.info(
                 "%s was successfully downloaded into %s", compiler_id, compiler_dir
             )
@@ -295,9 +348,14 @@ def get_compiler(platform_id, compiler_id, force=False, github_repo="mkst/compil
 
     container.remove()
 
+    return True
+
 
 def main():
     parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--podman", action="store_true", help="Use podman instead of docker"
+    )
     parser.add_argument(
         "--force",
         help="(re)download compiler when latest image already present",
@@ -316,25 +374,31 @@ def main():
 
     to_download = []
     for platform_id, compilers in COMPILERS.items():
+        # platforms are considered enabled unless explicitly disabled
         platform_enabled = (
-            os.environ.get(f"ENABLE_{platform_id.upper()}_SUPPORT", "YES").upper()
-            != "NO"
+            os.environ.get(f"ENABLE_{platform_id.upper()}_SUPPORT", "NO").upper()
+            == "YES"
         )
         if platform_enabled:
             to_download += [(platform_id, compiler) for compiler in compilers]
 
     if len(to_download) == 0:
-        logger.warning("No compilers configured to be downloaded!")
+        logger.warning("No platforms are configured to be downloaded for host architecture (%s)", HOST_ARCH)
         return
 
+    start = datetime.datetime.now()
     with Pool(processes=args.threads) as pool:
-        pool.starmap(
+        results = pool.starmap(
             functools.partial(
-                get_compiler, force=args.force, github_repo=args.github_repo
+                get_compiler, host_arch=HOST_ARCH, podman=args.podman, force=args.force, github_repo=args.github_repo
             ),
             to_download,
         )
-    logger.info("Finished processing %i compiler(s)", len(to_download))
+    end = datetime.datetime.now()
+
+    compilers_downloaded = len(list(filter(lambda x: x, results)))
+    logger.info("Updated %i / %i compiler(s) in %.2f second(s)", compilers_downloaded, len(to_download), (end - start).total_seconds())
+
 
 
 if __name__ == "__main__":