Init

.dockerignore

@@ -0,0 +1,6 @@
+.git/
+.github/
+templates/
+*.py
+*.yaml
+*.md

.github/workflows/ci.yaml

@@ -0,0 +1,72 @@
+name: Publish
+on:
+  push:
+    paths:
+      - .github/workflows/*
+      - platforms/*/*/Dockerfile
+      - platforms/*/*/*/Dockerfile
+      - matrix.py
+      - values.yaml
+      - template.py
+
+jobs:
+  create_matrix:
+    name: Create Matrix
+    runs-on: ubuntu-22.04
+    outputs:
+      matrix: ${{ steps.create-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+
+      - name: Create list of changed files
+        run: |
+          # See https://github.community/t/check-pushed-file-changes-with-git-diff-tree-in-github-actions/17220/10
+          if [ $GITHUB_BASE_REF ]; then
+            # Pull Request
+            git fetch origin $GITHUB_BASE_REF --depth=1
+            export DIFF=$( git diff --name-only origin/$GITHUB_BASE_REF $GITHUB_SHA )
+            echo "Diff between origin/$GITHUB_BASE_REF and $GITHUB_SHA"
+          else
+            # Push
+            git fetch origin ${{ github.event.before }} --depth=1
+            export DIFF=$( git diff --name-only ${{ github.event.before }} $GITHUB_SHA )
+            echo "Diff between ${{ github.event.before }} and $GITHUB_SHA"
+          fi
+          echo "$DIFF" | tee changed_files.txt
+
+      - name: Create matrix of images to generate
+        id: create-matrix
+        run: |
+          # e.g. if we wanted to run the matrix for a set of platforms:
+          # echo "matrix=$(python3 matrix.py --platforms n64,ps1,saturn)" >> $GITHUB_OUTPUT
+          echo "matrix=$(python3 matrix.py --dockerfiles changed_files.txt)" >> $GITHUB_OUTPUT
+
+  run_matrix:
+    name: Run Matrix
+    permissions: write-all
+    runs-on: ubuntu-22.04
+    needs: create_matrix
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.create_matrix.outputs.matrix) }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push to Github registry
+        uses: docker/build-push-action@v4
+        with:
+          push: true
+          tags: ghcr.io/mkst/compilers/${{ matrix.image }}:latest
+          file: ${{ matrix.dockerfile }}

.gitignore

@@ -0,0 +1 @@
+__pycache__/

README.md

@@ -0,0 +1,24 @@
+# decomp.me compilers
+
+This repository contains the build instructions for all the compilers that are used on decomp.me.
+
+There is a GitHub Action which will "build" each compiler and upload it to the GitHub Container Registry (ghcr.io).
+
+When decomp.me is deployed it will fetch each compiler image from the registry.
+
+## (re-)Generating Dockerfiles from jinja templates
+
+Due to the large amount of shared steps, the Dockerfiles are driven from jinja templates.
+The values that are used to populate the templates are found in values.yaml.
+
+Rather than modifying the Dockerfiles directly, the `.j2` template should be changed and the Dockerfiles regenerated.
+
+```sh
+python3 -m pip install Jinja2
+python3 template.py
+```
+
+## Notes:
+
+- Docker image names must be lowercase therefore compiler ids will be lowercase'd before being uploaded.
+- `+` is not a valid character for Docker image names, therefore `+` will be replaced with `_` in image names before being uploaded.

matrix.py

@@ -0,0 +1,88 @@
+import argparse
+import json
+import yaml
+
+
+def parse_dockerfile_path(filepath):
+    if not filepath.endswith("Dockerfile"):
+        return None
+
+    if filepath.count("/") == 3:
+        # e.g. platforms/n64/ido6.0/Dockerfile
+        _, platform, compiler_id, _ = filepath.split("/")
+        arch = None
+    elif filepath.count("/") == 4:
+        # e.g. platforms/n64/gcc2.7.2kmc/darwin/Dockerfile
+        _, platform, compiler_id, arch, _ = filepath.split("/")
+    else:
+        raise Exception(f"We do not know how to handle this path: {filepath}")
+    return [platform, compiler_id, arch]
+
+
+def process_dockerfile(platform, compiler_id, arch, base_dir="platforms"):
+    # image names must be lowercase and "+" is not a valid character
+    clean_compiler_id = compiler_id.lower().replace("+", "plus")
+
+    return dict(
+        platform=platform,
+        compiler_id=compiler_id,
+        clean_compiler_id=clean_compiler_id,
+        dockerfile=f"platforms/{platform}/{compiler_id}{'/' + arch if arch else ''}/Dockerfile",
+        image=f"{platform}/{clean_compiler_id}{'/' + arch if arch else ''}",
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--dockerfiles",
+        help="Path to file containing list of Dockerfiles to filter on, e.g. 'files.txt'",
+        type=str,
+    )
+    parser.add_argument(
+        "--platforms",
+        help="Comma-separated list of platforms to filter on, e.g. 'n64,ps2'",
+        type=str,
+    )
+    args = parser.parse_args()
+
+    with open("values.yaml", "r") as f:
+        values = yaml.safe_load(f)
+
+    local_dockerfiles = []
+    for compiler in values.get("compilers", []):
+        local_dockerfiles.append(
+            [
+                compiler["platform"],
+                compiler["id"],
+                compiler.get("arch"),
+            ]
+        )
+
+    changed_dockerfiles = []
+    if args.dockerfiles:
+        with open(args.dockerfiles, "r") as f:
+            lines = [x.strip() for x in f.readlines()]
+            for line in lines:
+                res = parse_dockerfile_path(line)
+                if res is not None:
+                    changed_dockerfiles.append(res)
+
+    if args.platforms:
+        filtered_platforms = args.platforms.split(",")
+        local_dockerfiles = filter(
+            lambda x: x[0] in filtered_platforms, local_dockerfiles
+        )
+
+    if len(changed_dockerfiles) > 0:
+        local_dockerfiles = filter(
+            lambda x: x in changed_dockerfiles, local_dockerfiles
+        )
+
+    includes = [process_dockerfile(p, c, a) for (p, c, a) in local_dockerfiles]
+    matrix = dict(include=includes)
+    print(json.dumps(matrix, separators=(",", ":")))
+
+
+if __name__ == "__main__":
+    main()

platforms/gba/agbcc/Dockerfile

@@ -0,0 +1,17 @@
+# NOTE: This file is generated automatically via template.py. Do not edit manually!
+
+
+FROM alpine:3.18 as base
+
+RUN mkdir -p /compilers/gba/agbcc
+
+RUN wget -O agbcc.tar.gz "https://github.com/pret/agbcc/releases/download/release/agbcc.tar.gz"
+RUN tar xvzf agbcc.tar.gz -C /compilers/gba/agbcc
+
+RUN chown -R root:root /compilers/gba/agbcc/
+RUN chmod +x /compilers/gba/agbcc/*
+
+
+FROM scratch as release
+
+COPY --from=base /compilers /compilers

platforms/gba/agbccpp/Dockerfile

@@ -0,0 +1,17 @@
+# NOTE: This file is generated automatically via template.py. Do not edit manually!
+
+
+FROM alpine:3.18 as base
+
+RUN mkdir -p /compilers/gba/agbccpp
+
+RUN wget -O agbcc.tar.gz "https://github.com/notyourav/agbcc/releases/download/cp/agbcc.tar.gz"
+RUN tar xvzf agbcc.tar.gz -C /compilers/gba/agbccpp
+
+RUN chown -R root:root /compilers/gba/agbccpp/
+RUN chmod +x /compilers/gba/agbccpp/*
+
+
+FROM scratch as release
+
+COPY --from=base /compilers /compilers

platforms/gc_wii/mwcc_233_144/Dockerfile

@@ -0,0 +1,30 @@
+# NOTE: This file is generated automatically via template.py. Do not edit manually!
+
+
+FROM alpine:3.18 as base
+
+# download zip first to allow for Docker caching
+
+WORKDIR /root
+RUN wget -O GC_WII_COMPILERS.zip "https://cdn.discordapp.com/attachments/727918646525165659/1129759991696457728/GC_WII_COMPILERS.zip"
+RUN unzip GC_WII_COMPILERS.zip
+
+RUN mkdir -p /compilers/gc_wii/mwcc_233_144
+
+RUN cp -r GC/1.0/* /compilers/gc_wii/mwcc_233_144
+
+RUN chmod +x /compilers/gc_wii/mwcc_233_144/mwcceppc.exe
+RUN touch /compilers/gc_wii/mwcc_233_144/license.dat
+RUN if [[ -f /compilers/gc_wii/mwcc_233_144/lmgr326b.dll ]]; then \
+    mv /compilers/gc_wii/mwcc_233_144/lmgr326b.dll /compilers/gc_wii/mwcc_233_144/LMGR326B.dll; \
+  fi
+RUN if [[ -f /compilers/gc_wii/mwcc_233_144/lmgr8c.dll ]]; then \
+    mv /compilers/gc_wii/mwcc_233_144/lmgr8c.dll /compilers/gc_wii/mwcc_233_144/LMGR8C.dll; \
+  fi
+
+RUN chown -R root:root /compilers/gc_wii/mwcc_233_144/
+
+
+FROM scratch as release
+
+COPY --from=base /compilers /compilers

platforms/gc_wii/mwcc_233_159/Dockerfile

@@ -0,0 +1,30 @@
+# NOTE: This file is generated automatically via template.py. Do not edit manually!
+
+
+FROM alpine:3.18 as base
+
+# download zip first to allow for Docker caching
+
+WORKDIR /root
+RUN wget -O GC_WII_COMPILERS.zip "https://cdn.discordapp.com/attachments/727918646525165659/1129759991696457728/GC_WII_COMPILERS.zip"
+RUN unzip GC_WII_COMPILERS.zip
+
+RUN mkdir -p /compilers/gc_wii/mwcc_233_159
+
+RUN cp -r GC/1.1/* /compilers/gc_wii/mwcc_233_159
+
+RUN chmod +x /compilers/gc_wii/mwcc_233_159/mwcceppc.exe
+RUN touch /compilers/gc_wii/mwcc_233_159/license.dat
+RUN if [[ -f /compilers/gc_wii/mwcc_233_159/lmgr326b.dll ]]; then \
+    mv /compilers/gc_wii/mwcc_233_159/lmgr326b.dll /compilers/gc_wii/mwcc_233_159/LMGR326B.dll; \
+  fi
+RUN if [[ -f /compilers/gc_wii/mwcc_233_159/lmgr8c.dll ]]; then \
+    mv /compilers/gc_wii/mwcc_233_159/lmgr8c.dll /compilers/gc_wii/mwcc_233_159/LMGR8C.dll; \
+  fi
+
+RUN chown -R root:root /compilers/gc_wii/mwcc_233_159/
+
+
+FROM scratch as release
+
+COPY --from=base /compilers /compilers

platforms/gc_wii/mwcc_233_163/Dockerfile

@@ -0,0 +1,30 @@
+# NOTE: This file is generated automatically via template.py. Do not edit manually!
+
+
+FROM alpine:3.18 as base
+
+# download zip first to allow for Docker caching
+
+WORKDIR /root
+RUN wget -O GC_WII_COMPILERS.zip "https://cdn.discordapp.com/attachments/727918646525165659/1129759991696457728/GC_WII_COMPILERS.zip"
+RUN unzip GC_WII_COMPILERS.zip
+
+RUN mkdir -p /compilers/gc_wii/mwcc_233_163
+
+RUN cp -r GC/1.2.5/* /compilers/gc_wii/mwcc_233_163
+
+RUN chmod +x /compilers/gc_wii/mwcc_233_163/mwcceppc.exe
+RUN touch /compilers/gc_wii/mwcc_233_163/license.dat
+RUN if [[ -f /compilers/gc_wii/mwcc_233_163/lmgr326b.dll ]]; then \
+    mv /compilers/gc_wii/mwcc_233_163/lmgr326b.dll /compilers/gc_wii/mwcc_233_163/LMGR326B.dll; \
+  fi
+RUN if [[ -f /compilers/gc_wii/mwcc_233_163/lmgr8c.dll ]]; then \
+    mv /compilers/gc_wii/mwcc_233_163/lmgr8c.dll /compilers/gc_wii/mwcc_233_163/LMGR8C.dll; \
+  fi
+
+RUN chown -R root:root /compilers/gc_wii/mwcc_233_163/
+
+
+FROM scratch as release
+
+COPY --from=base /compilers /compilers

platforms/gc_wii/mwcc_233_163e/Dockerfile

@@ -0,0 +1,33 @@
+# NOTE: This file is generated automatically via template.py. Do not edit manually!
+
+
+FROM alpine:3.18 as base
+
+# download zip first to allow for Docker caching
+
+WORKDIR /root
+RUN wget -O GC_WII_COMPILERS.zip "https://cdn.discordapp.com/attachments/727918646525165659/1129759991696457728/GC_WII_COMPILERS.zip"
+RUN unzip GC_WII_COMPILERS.zip
+
+RUN mkdir -p /compilers/gc_wii/mwcc_233_163e
+
+RUN cp -r GC/1.2.5e/* /compilers/gc_wii/mwcc_233_163e
+
+RUN chmod +x /compilers/gc_wii/mwcc_233_163e/mwcceppc.exe
+RUN touch /compilers/gc_wii/mwcc_233_163e/license.dat
+RUN if [[ -f /compilers/gc_wii/mwcc_233_163e/lmgr326b.dll ]]; then \
+    mv /compilers/gc_wii/mwcc_233_163e/lmgr326b.dll /compilers/gc_wii/mwcc_233_163e/LMGR326B.dll; \
+  fi
+RUN if [[ -f /compilers/gc_wii/mwcc_233_163e/lmgr8c.dll ]]; then \
+    mv /compilers/gc_wii/mwcc_233_163e/lmgr8c.dll /compilers/gc_wii/mwcc_233_163e/LMGR8C.dll; \
+  fi
+RUN wget -O /compilers/gc_wii/mwcc_233_163e/frank.py https://raw.githubusercontent.com/doldecomp/melee/master/tools/frank.py
+RUN cp -r GC/1.2.5/mwcceppc.exe /compilers/gc_wii/mwcc_233_163e/mwcceppc.125.exe
+RUN chmod +x /compilers/gc_wii/mwcc_233_163e/mwcceppc.125.exe
+
+RUN chown -R root:root /compilers/gc_wii/mwcc_233_163e/
+
+
+FROM scratch as release
+
+COPY --from=base /compilers /compilers

platforms/gc_wii/mwcc_233_163n/Dockerfile

@@ -0,0 +1,30 @@
+# NOTE: This file is generated automatically via template.py. Do not edit manually!
+
+
+FROM alpine:3.18 as base
+
+# download zip first to allow for Docker caching
+
+WORKDIR /root
+RUN wget -O GC_WII_COMPILERS.zip "https://cdn.discordapp.com/attachments/727918646525165659/1129759991696457728/GC_WII_COMPILERS.zip"
+RUN unzip GC_WII_COMPILERS.zip
+
+RUN mkdir -p /compilers/gc_wii/mwcc_233_163n
+
+RUN cp -r GC/1.2.5n/* /compilers/gc_wii/mwcc_233_163n
+
+RUN chmod +x /compilers/gc_wii/mwcc_233_163n/mwcceppc.exe
+RUN touch /compilers/gc_wii/mwcc_233_163n/license.dat
+RUN if [[ -f /compilers/gc_wii/mwcc_233_163n/lmgr326b.dll ]]; then \
+    mv /compilers/gc_wii/mwcc_233_163n/lmgr326b.dll /compilers/gc_wii/mwcc_233_163n/LMGR326B.dll; \
+  fi
+RUN if [[ -f /compilers/gc_wii/mwcc_233_163n/lmgr8c.dll ]]; then \
+    mv /compilers/gc_wii/mwcc_233_163n/lmgr8c.dll /compilers/gc_wii/mwcc_233_163n/LMGR8C.dll; \
+  fi
+
+RUN chown -R root:root /compilers/gc_wii/mwcc_233_163n/
+
+
+FROM scratch as release
+
+COPY --from=base /compilers /compilers