Init

.dockerignore

@@ -0,0 +1,6 @@
+.git/
+.github/
+templates/
+*.py
+*.yaml
+*.md

.github/workflows/ci.yaml

@@ -0,0 +1,84 @@
+name: Publish
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  check_templated_files:
+    name: Ensure templated Dockerfiles are up to date
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+
+      - name: Generate templates
+        run: python3 template.py values.yaml
+
+      - name: Fail if anything has changed
+        run: git diff --exit-code
+
+  create_matrix:
+    name: Create Matrix
+    runs-on: ubuntu-22.04
+    outputs:
+      matrix: ${{ steps.create-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+
+      - name: Create list of changed files
+        run: |
+          # See https://github.community/t/check-pushed-file-changes-with-git-diff-tree-in-github-actions/17220/10
+          if [ $GITHUB_BASE_REF ]; then
+            # Pull Request
+            git fetch origin $GITHUB_BASE_REF --depth=1
+            export DIFF=$( git diff --name-only origin/$GITHUB_BASE_REF $GITHUB_SHA )
+            echo "Diff between origin/$GITHUB_BASE_REF and $GITHUB_SHA"
+          else
+            # Push
+            git fetch origin ${{ github.event.before }} --depth=1
+            export DIFF=$( git diff --name-only ${{ github.event.before }} $GITHUB_SHA )
+            echo "Diff between ${{ github.event.before }} and $GITHUB_SHA"
+          fi
+          echo "$DIFF" | tee changed_files.txt
+
+      - name: Create matrix of images to generate
+        id: create-matrix
+        run: |
+          # e.g. if we wanted to run the matrix for a set of platforms:
+          # echo "matrix=$(python3 matrix.py values.yaml --platforms n64 ps1 saturn)" >> $GITHUB_OUTPUT
+          echo "matrix=$(python3 matrix.py values.yaml --dockerfiles changed_files.txt)" >> $GITHUB_OUTPUT
+
+  run_matrix:
+    name: Run Matrix
+    permissions: write-all
+    runs-on: ubuntu-22.04
+    needs: [check_templated_files, create_matrix]
+    if: ${{ needs.create_matrix.outputs.matrix != '' && needs.create_matrix.outputs.matrix != '{"include":[]}' }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.create_matrix.outputs.matrix) }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push to Github registry
+        uses: docker/build-push-action@v4
+        with:
+          # only publish from main branch and don't publish on forks
+          push: ${{ github.ref == 'refs/heads/main' && ! github.event.pull_request.head.repo.fork }}
+          tags: ghcr.io/${{ github.repository }}/${{ matrix.image }}:latest
+          file: ${{ matrix.dockerfile }}

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+.mypy_cache/

README.md

@@ -0,0 +1,24 @@
+# decomp.me compilers
+
+This repository contains the build instructions for all the compilers that are used on decomp.me.
+
+There is a GitHub Action which will "build" each compiler and upload it to the GitHub Container Registry (ghcr.io).
+
+When decomp.me is deployed it will fetch each compiler image from the registry.
+
+## (re-)Generating Dockerfiles from jinja templates
+
+Due to the large amount of shared steps, the Dockerfiles are driven from jinja templates.
+The values that are used to populate the templates are found in values.yaml.
+
+Rather than modifying the Dockerfiles directly, the `.j2` template should be changed and the Dockerfiles regenerated.
+
+```sh
+python3 -m pip install Jinja2
+python3 template.py
+```
+
+## Notes:
+
+- Docker image names must be lowercase therefore compiler ids will be lowercase'd before being uploaded.
+- `+` is not a valid character for Docker image names, therefore `+` will be replaced with `_` in image names before being uploaded.

matrix.py

@@ -0,0 +1,100 @@
+import argparse
+import json
+import yaml
+
+from pathlib import Path
+
+
+def parse_dockerfile_path(file_path):
+    if not file_path.name == "Dockerfile":
+        return None
+
+    file_path_parts = file_path.parts
+    if len(file_path_parts) == 4:
+        # e.g. platforms/n64/ido6.0/Dockerfile
+        _, platform, compiler_id, _ = file_path_parts
+        return [platform, compiler_id, None]
+
+    if len(file_path_parts) == 5:
+        # e.g. platforms/n64/gcc2.7.2kmc/darwin/Dockerfile
+        _, platform, compiler_id, arch, _ = file_path_parts
+        return [platform, compiler_id, arch]
+
+    raise Exception(f"We do not know how to handle this path: {file_path}")
+
+
+def process_dockerfile(platform, compiler_id, arch, base_dir="platforms"):
+    # image names must be lowercase and "+" is not a valid character
+    clean_compiler_id = compiler_id.lower().replace("+", "plus")
+
+    return dict(
+        platform=platform,
+        compiler_id=compiler_id,
+        clean_compiler_id=clean_compiler_id,
+        dockerfile=f"platforms/{platform}/{compiler_id}{'/' + arch if arch else ''}/Dockerfile",
+        image=f"{platform}/{clean_compiler_id}{'/' + arch if arch else ''}",
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "filename",
+        type=argparse.FileType("r"),
+        help="Configuration file, e.g. `values.yaml`",
+    )
+    parser.add_argument(
+        "--dockerfiles",
+        "--filepaths",
+        type=argparse.FileType("r"),
+        help="Path to file containing list of Dockerfiles to filter on, e.g. `--dockerfiles files.txt`",
+    )
+    parser.add_argument(
+        "--platforms",
+        type=str,
+        nargs="+",
+        help="List of platforms to filter on, e.g. `--platforms n64 ps2`",
+    )
+    args = parser.parse_args()
+
+    values = yaml.safe_load(args.filename)
+
+    local_dockerfiles = []
+    for compiler in values.get("compilers", []):
+        local_dockerfiles.append(
+            [
+                compiler["platform"],
+                compiler["id"],
+                compiler.get("arch"),
+            ]
+        )
+
+    if args.dockerfiles:
+        allowed_dockerfiles = []
+        lines = [x.strip() for x in args.dockerfiles.readlines()]
+        for line in lines:
+            path = Path(line)
+            res = parse_dockerfile_path(path)
+            if res is not None:
+                allowed_dockerfiles.append(res)
+
+        if len(allowed_dockerfiles) > 0:
+            local_dockerfiles = filter(
+                lambda x: x in allowed_dockerfiles, local_dockerfiles
+            )
+        else:
+            # no Dockerfiles were modified
+            local_dockerfiles = []
+
+    if args.platforms:
+        local_dockerfiles = filter(
+            lambda x: x[0] in args.platforms, local_dockerfiles
+        )
+
+    includes = [process_dockerfile(p, c, a) for (p, c, a) in local_dockerfiles]
+    matrix = dict(include=includes)
+    print(json.dumps(matrix, separators=(",", ":")))
+
+
+if __name__ == "__main__":
+    main()

template.py

@@ -0,0 +1,70 @@
+import argparse
+from pathlib import Path
+
+import yaml
+
+from jinja2 import Environment, FileSystemLoader, select_autoescape
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "filename",
+        type=argparse.FileType("r"),
+        help="Configuration file, e.g. `values.yaml`",
+    )
+    parser.add_argument(
+        "--template-dir",
+        "--templates-dir",
+        type=Path,
+        default=Path("templates"),
+        help="Directory to find templates",
+    )
+    parser.add_argument(
+        "--base-path",
+        "--dockerfile-path",
+        type=Path,
+        default=Path("platforms"),
+        help="Directory to save templated Dockerfiles",
+    )
+    args = parser.parse_args()
+
+    env = Environment(
+        loader=FileSystemLoader(args.template_dir), autoescape=select_autoescape()
+    )
+
+    values = yaml.safe_load(args.filename)
+    for compiler in values.get("compilers", []):
+        if "arch" in compiler:
+            target_path = (
+                args.base_path
+                / compiler["platform"]
+                / compiler["id"]
+                / compiler["arch"]
+                / "Dockerfile"
+            )
+        else:
+            target_path = (
+                args.base_path / compiler["platform"] / compiler["id"] / "Dockerfile"
+            )
+
+        print(
+            f"{compiler['id']}: Creating {target_path} from {compiler['template']}.j2"
+        )
+
+        template_path = compiler["template"]
+        template = env.get_template(f"{template_path}.j2")
+        rendered = template.render(compiler)
+
+        target_path.parent.mkdir(parents=True, exist_ok=True)
+        with target_path.open("w") as f:
+            f.write(
+                "# NOTE: This file is generated automatically via template.py. Do not edit manually!\n\n"
+            )
+            f.write("\n")
+            f.write(rendered)
+            f.write("\n")  # enforce trailing newline
+
+
+if __name__ == "__main__":
+    main()

templates/common/default.j2

@@ -0,0 +1,33 @@
+FROM alpine:3.18 as base
+
+RUN mkdir -p /compilers/{{ platform }}/{{ id }}
+{% for url in files | default([file]) | default([]) %}
+{% set filename = url.split("/")[-1].split("?")[0] %}
+{%- if filename.endswith('.tar.gz') -%}
+RUN wget -O {{ filename }} "{{ url }}"
+RUN tar xvzf {{ filename }} -C /compilers/{{ platform }}/{{ id }}
+{%- elif filename.endswith('.tar.xz') -%}
+RUN wget -O {{ filename }} "{{ url }}"
+RUN tar xvJf {{ filename }} -C /compilers/{{ platform }}/{{ id }}
+{%- elif filename.endswith('.tar.bz2') -%}
+RUN wget -O {{ filename }} "{{ url }}"
+RUN tar xvjf {{ filename }} -C /compilers/{{ platform }}/{{ id }}
+{%- elif filename.endswith('.zip') -%}
+RUN wget -O {{ filename }} "{{ url }}"
+RUN unzip {{ filename }} -d /compilers/{{ platform }}/{{ id }}
+{%- else -%}
+RUN wget -O /compilers/{{ platform }}/{{ id }}/{{ filename }} "{{ url }}"
+{%- endif -%}
+{% endfor %}
+
+{%- for dir in rm_dirs | default([]) %}
+RUN rm -rf /compilers/{{ platform }}/{{ id }}/{{ dir }}
+{%- endfor %}
+
+RUN chown -R root:root /compilers/{{ platform }}/{{ id }}/
+RUN chmod +x /compilers/{{ platform }}/{{ id }}/*
+
+
+FROM scratch as release
+
+COPY --from=base /compilers /compilers

templates/common/xz.j2

@@ -0,0 +1,22 @@
+FROM alpine:3.18 as base
+
+# download xz first to allow for Docker caching
+
+WORKDIR /root
+
+{%- set filename = file.split("/")[-1].split("?")[0] %}
+
+RUN wget -O {{ filename }} "{{ file }}"
+RUN tar xvJf {{ filename }}
+
+RUN mkdir -p /compilers/{{ platform }}/{{ id }}
+
+RUN cp -r {{ package_dir | default(id) }}/* /compilers/{{ platform }}/{{ id }}
+
+RUN chown -R root:root /compilers/{{ platform }}/{{ id }}/
+RUN chmod +x /compilers/{{ platform }}/{{ id }}/*
+
+
+FROM scratch as release
+
+COPY --from=base /compilers /compilers

templates/common/zip.j2

@@ -0,0 +1,22 @@
+FROM alpine:3.18 as base
+
+# download zip first to allow for Docker caching
+
+WORKDIR /root
+
+{%- set filename = file.split("/")[-1].split("?")[0] %}
+
+RUN wget -O {{ filename }} "{{ file }}"
+RUN unzip {{ filename }}
+
+RUN mkdir -p /compilers/{{ platform }}/{{ id }}
+
+RUN cp -r {{ package_dir | default(id) }}/* /compilers/{{ platform }}/{{ id }}
+
+RUN chown -R root:root /compilers/{{ platform }}/{{ id }}/
+RUN chmod +x /compilers/{{ platform }}/{{ id }}/*
+
+
+FROM scratch as release
+
+COPY --from=base /compilers /compilers