From 77371919918b7b8d210037a3032be2875d9cc77e Mon Sep 17 00:00:00 2001 From: Antheas Kapenekakis Date: Sat, 3 Aug 2024 21:11:28 +0300 Subject: [PATCH] add changelog support --- .github/workflows/online_test_deck.yml | 23 +++- 3_chunk.sh | 23 +++- action.yml | 35 +++++- src/rechunk/__main__.py | 20 ++- src/rechunk/alg.py | 57 ++++++--- src/rechunk/model.py | 11 +- src/rechunk/utils.py | 161 +++++++++++++++++++------ 7 files changed, 268 insertions(+), 62 deletions(-) diff --git a/.github/workflows/online_test_deck.yml b/.github/workflows/online_test_deck.yml index 907e169..a720da8 100644 --- a/.github/workflows/online_test_deck.yml +++ b/.github/workflows/online_test_deck.yml @@ -41,13 +41,14 @@ jobs: prev-ref: ${{ github.event.inputs.prev == 'true' && 'ghcr.io/hhd-dev/bazzite-automated-deck:stable' || '' }} rechunk: 'ghcr.io/hhd-dev/rechunk:latest' version: 'rc${{ github.event.inputs.ref }}' + revision: ${{ github.sha }} pretty: 'Rechunked (from ${{ github.event.inputs.ref }})' + git: "${{ github.workspace }}" labels: | io.artifacthub.package.logo-url=https://raw.githubusercontent.com/ublue-os/bazzite/main/repo_content/logo.png io.artifacthub.package.readme-url=https://bazzite.gg/ org.opencontainers.image.created= org.opencontainers.image.licenses=Apache-2.0 - org.opencontainers.image.revision=${{ github.sha }} org.opencontainers.image.source=https://github.com/ublue-os/bazzite org.opencontainers.image.title=bazzite org.opencontainers.image.url=https://github.com/ublue-os/bazzite @@ -66,6 +67,26 @@ jobs: HHD: , Adjustor: , HHD-UI: ] + + changelog: | + Bazzite Deck + Version: + + Major Components: + - Kernel: + - Gamescope: + - KDE: + + Handheld Daemon: + - HHD: + - Adjustor: + - HHD-UI: + + Changes since version : + + + Package Changes: + - name: Upload Image id: upload diff --git a/3_chunk.sh b/3_chunk.sh index 382d72d..b514279 100755 --- a/3_chunk.sh +++ b/3_chunk.sh @@ -72,6 +72,15 @@ fi if [ -n "$PRETTY" ]; then PREV_ARG+=("--pretty" "$PRETTY") fi +if [ -n "$CHANGELOG" ]; then + PREV_ARG+=("--changelog" "$CHANGELOG") +fi +if [ -n "$GIT_DIR" ]; then + PREV_ARG+=("--git-dir" "$GIT_DIR") +fi +if [ -n "$REFISION" ]; then + PREV_ARG+=("--revision" "$REVISION") +fi LABEL_ARR=() if [ -n "$LABELS" ]; then @@ -89,8 +98,12 @@ if [ -n "$DESCRIPTION" ]; then LABEL_ARR+=("--label" "org.opencontainers.image.description=$DESCRIPTION") fi -echo $RECHUNK -r "$REPO" -b "$OUT_TAG" -c "$CONTENT_META" "${PREV_ARG[@]}" "${LABEL_ARR[@]}" -$RECHUNK -r "$REPO" -b "$OUT_TAG" -c "$CONTENT_META" "${PREV_ARG[@]}" "${LABEL_ARR[@]}" +cmd=$RECHUNK -r "$REPO" -b "$OUT_TAG" -c "$CONTENT_META" \ + --changelog-fn "${OUT_NAME}.changelog.txt" \ + "${PREV_ARG[@]}" "${LABEL_ARR[@]}" +echo Running "$cmd" +$cmd + PREV_ARG="" if [ -n "$SKIP_COMPRESSION" ]; then @@ -107,10 +120,8 @@ ostree-ext-cli \ echo Created archive with ref ${OUT_REF} -# TODO: Temporarily remove to try newlines -# echo Writing manifests to ./$OUT_NAME.manifest.json, ./$OUT_NAME.manifest.raw.json -# skopeo inspect ${OUT_REF} > ${OUT_NAME}.manifest.json -# skopeo inspect --raw ${OUT_REF} > ${OUT_NAME}.manifest.raw.json +echo Writing manifest to ./$OUT_NAME.manifest.json +skopeo inspect ${OUT_REF} > ${OUT_NAME}.manifest.json # Reset perms to make the files usable chmod 666 -R ${OUT_NAME}* \ No newline at end of file diff --git a/action.yml b/action.yml index 1169f45..63067db 100644 --- a/action.yml +++ b/action.yml @@ -61,6 +61,21 @@ inputs: By default, this action will remove the ref image provided in `ref`. This variable will disable discarding it after the OSTree commit is created. May cause the storage overflow. + changelog: + description: | + The changelog of the image. Can be substituted with the variable + and will be placed + git: + description: | + The git repository to use for the action. Used for versioning the output. + Defaults to the current repository. + revision: + description: | + The revision that will be recorded in the image metadata and in + "org.opencontainers.image.revision". Used for the + tag along with the git path. + If is not used, providing it as part of a "org.opencontainers.image.revision" + is the same. outputs: ref: @@ -76,6 +91,14 @@ outputs: description: | The filesystem location of the rechunked image, so that it can be removed. value: ${{ steps.rechunk.outputs.location }} + changelog: + description: | + The changelog of the image, with the variable substituted. + value: ${{ steps.rechunk.outputs.changelog }} + manifest: + description: | + The skopeo manifest of the rechunked image. + value: ${{ steps.rechunk.outputs.manifest }} runs: using: 'composite' @@ -124,9 +147,14 @@ runs: shell: bash run: | OUT_NAME=$(echo ${{ inputs.ref }} | rev | cut -d'/' -f1 | rev | sed 's/:/_/') - + GIT_PATH="${{ inputs.git }}" + if [ -n "$GIT_PATH" ]; then + GIT_PATH="${{ github.workspace }}" + fi + sudo podman run --rm \ -v "${{ github.workspace }}:/workspace" \ + -v "$GIT_PATH:/var/git" \ -v "cache_ostree:/var/ostree" \ -e REPO=/var/ostree/repo \ -e MAX_LAYERS="${{ inputs.max-layers }}" \ @@ -138,7 +166,10 @@ runs: -e VERSION_FN="/workspace/version.txt" \ -e PRETTY="${{ inputs.pretty }}" \ -e DESCRIPTION="${{ inputs.description }}" \ + -e CHANGELOG="${{ inputs.changelog }}" \ -e OUT_REF="oci:$OUT_NAME" \ + -e GIT_DIR="/var/git" \ + -e REVISION="${{ inputs.revision }}" \ -e PREV_REF_FAIL="${{ inputs.prev-ref-fail }}" \ -u 0:0 \ ${{ inputs.rechunk }} \ @@ -147,6 +178,8 @@ runs: echo "version=$(sudo cat ${{ github.workspace }}/version.txt)" >> $GITHUB_OUTPUT echo "ref=oci:${{ github.workspace }}/$OUT_NAME" >> $GITHUB_OUTPUT echo "location=${{ github.workspace }}/$OUT_NAME" >> $GITHUB_OUTPUT + echo "changelog=${{ github.workspace }}/$OUT_NAME.changelog.txt" >> $GITHUB_OUTPUT + echo "manifest=${{ github.workspace }}/$OUT_NAME.manifest.json" >> $GITHUB_OUTPUT # Remove root permissions sudo chown $(id -u):$(id -g) -R "${{ github.workspace }}/$OUT_NAME" diff --git a/src/rechunk/__main__.py b/src/rechunk/__main__.py index 961ac7d..4ee881f 100644 --- a/src/rechunk/__main__.py +++ b/src/rechunk/__main__.py @@ -64,11 +64,23 @@ def argparse_func(): parser.add_argument( "-l", "--label", help="Add labels to the output image.", action="append" ) + parser.add_argument("--pretty", help="Pretty version string.", default=None) parser.add_argument( - "--pretty", help="Pretty version string.", default=None + "--version-fn", help="Output path for version name.", default=None ) parser.add_argument( - "--version-fn", help="Output path for version name.", default=None + "--revision", + help="The git hash of the project (placed in 'org.opencontainers.image.revision' and internal metadata for calculating changelogs).", + default=None, + ) + parser.add_argument( + "--git-dir", + help="The checked out git directory for calculating changelogs.", + default=None, + ) + parser.add_argument("--changelog", help="Changelog template.", default=None) + parser.add_argument( + "--changelog-fn", help="Output path for the generated changelog.", default=None ) # Hyperparameters @@ -118,6 +130,10 @@ def argparse_func(): pretty=args.pretty, version_fn=args.version_fn, result_fn=None, + revision=args.revision, + git_dir=args.git_dir, + changelog=args.changelog, + changelog_fn=args.changelog_fn, ) diff --git a/src/rechunk/alg.py b/src/rechunk/alg.py index 08b37ed..1f5fd51 100644 --- a/src/rechunk/alg.py +++ b/src/rechunk/alg.py @@ -10,8 +10,13 @@ from rechunk.model import MetaPackage, Package from .fedora import get_packages -from .model import INFO_KEY, Package, get_layers -from .ostree import calculate_ostree_layers, dump_ostree_contentmeta, get_ostree_map, run_with_ostree_files +from .model import INFO_KEY, Package, get_layers, get_info, ExportInfo +from .ostree import ( + calculate_ostree_layers, + dump_ostree_contentmeta, + get_ostree_map, + run_with_ostree_files, +) from .utils import get_default_meta_yaml, get_labels, get_update_matrix, tqdm logger = logging.getLogger(__name__) @@ -429,18 +434,22 @@ def load_previous_manifest( fn: str | list[str], packages: list[MetaPackage], max_layers: int ): logger.info(f"Loading previous manifest from '{fn}'.") + info = None if isinstance(fn, str): with open(fn, "r") as f: raw = json.load(f) # Since podman/skopeo do not respect layer annotations, use - # a JSON config key - layers = get_layers(raw) + # a JSON config key + info = get_info(raw) + layers = get_layers(info) # Then as a fallback use the old OSTree format if layers: - logger.info(f"Processing previous manifest with {len(layers)} layers (loaded from '{INFO_KEY}').") + logger.info( + f"Processing previous manifest with {len(layers)} layers (loaded from '{INFO_KEY}')." + ) else: layers = [] for data in raw["LayersData"]: @@ -452,11 +461,15 @@ def load_previous_manifest( if "ostree.components" not in annotations: continue layers.append(annotations["ostree.components"].split(",")) - logger.info(f"Processing previous manifest with {len(raw)} layers (loaded from 'ostree.components').") + logger.info( + f"Processing previous manifest with {len(raw)} layers (loaded from 'ostree.components')." + ) else: raw = None layers = [l.split(",") for l in fn] - logger.info(f"Processing previous manifest with {len(fn)} layers (through cache argument).") + logger.info( + f"Processing previous manifest with {len(fn)} layers (through cache argument)." + ) assert layers, "No layers found in previous manifest. Raising." @@ -476,7 +489,9 @@ def load_previous_manifest( for p in todo: if p.name == name: if pkg is not None: - logger.error(f"Duplicate package '{name}' found in previous manifest.") + logger.error( + f"Duplicate package '{name}' found in previous manifest." + ) pkg = p if pkg is None: @@ -508,7 +523,7 @@ def load_previous_manifest( if removed: logger.info(f"The following packages were removed:\n{removed}") - return todo, dedi_layers, prefill, raw + return todo, dedi_layers, prefill, raw, info def main( @@ -527,6 +542,10 @@ def main( pretty: str | None = None, version_fn: str | None = None, _cache: dict | None = None, + revision: str | None = None, + git_dir: str | None = None, + changelog: str | None = None, + changelog_fn: str | None = None, ): if not meta_fn: meta_fn = get_default_meta_yaml() @@ -593,7 +612,7 @@ def main( if previous_manifest: try: logger.info("Loading existing layer data.") - todo, dedi_layers, prefill, manifest_json = load_previous_manifest( + todo, dedi_layers, prefill, manifest_json, info = load_previous_manifest( previous_manifest, new_packages, max_layers ) found_previous_plan = True @@ -602,6 +621,7 @@ def main( if not found_previous_plan: manifest_json = None + info = None logger.warning("No existing layer data. Expect layer shifts") todo, dedi_layers, prefill = prefill_layers( new_packages, upd_matrix, max_layers, prefill_size @@ -617,11 +637,20 @@ def main( layers = fill_layers(todo, prefill, upd_matrix, max_layer_size=max_layer_size) print_results(dedi_layers, prefill, layers, upd_matrix, result_fn) - final_layers, ostree_out = calculate_ostree_layers( - dedi_layers, layers, mapping - ) + final_layers, ostree_out = calculate_ostree_layers(dedi_layers, layers, mapping) new_labels, timestamp = get_labels( - labels, version, manifest_json, version_fn, pretty, packages, final_layers + labels=labels, + version=version, + prev_manifest=manifest_json, + version_fn=version_fn, + pretty=pretty, + base_pkg=packages, + layers=final_layers, + revision=revision, + git_dir=git_dir, + changelog_template=changelog, + changelog_fn=changelog_fn, + info=info, ) if contentmeta_fn: diff --git a/src/rechunk/model.py b/src/rechunk/model.py index acce9ae..36c9454 100644 --- a/src/rechunk/model.py +++ b/src/rechunk/model.py @@ -42,9 +42,9 @@ class ExportInfoV2(TypedDict): uniq: str layers: Sequence[Sequence[str]] packages: dict[str, str] + revision: str | None - -def get_layers(manifest): +def get_info(manifest): import json if not "Labels" in manifest: @@ -55,10 +55,13 @@ def get_layers(manifest): return None try: - info = json.loads(labels[INFO_KEY]) + return json.loads(labels[INFO_KEY]) except json.JSONDecodeError: return None +ExportInfo = ExportInfoV1 | ExportInfoV2 + +def get_layers(info): if info["version"] < 2: return None @@ -72,6 +75,7 @@ def export_v2( uniq: str | None, base_pkg: Sequence[Package] | None, layers: Sequence[Sequence[str]], + revision: str | None = None, ) -> str: import json @@ -86,5 +90,6 @@ def export_v2( uniq=uniq or "", packages=packages, layers=layers, + revision=revision, ) ) diff --git a/src/rechunk/utils.py b/src/rechunk/utils.py index e07cabe..80edf08 100644 --- a/src/rechunk/utils.py +++ b/src/rechunk/utils.py @@ -1,6 +1,7 @@ import datetime import logging import os +import subprocess import sys from datetime import datetime from typing import Sequence @@ -8,13 +9,14 @@ import numpy as np from tqdm.auto import tqdm as tqdm_orig -from .model import MetaPackage, Package, export_v2, INFO_KEY +from .model import INFO_KEY, ExportInfo, MetaPackage, Package, export_v2 logger = logging.getLogger(__name__) PBAR_OFFSET = 8 PBAR_FORMAT = (" " * PBAR_OFFSET) + ">>>>>>> {l_bar}{bar}{r_bar}" VERSION_TAG = "org.opencontainers.image.version" +REVISION_TAG = "org.opencontainers.image.revision" class tqdm(tqdm_orig): @@ -134,6 +136,65 @@ def get_update_matrix(packages: list[MetaPackage], biweekly: bool = True): return p_upd +def get_commits(git_dir: str | None, revision: str | None, prev_rev: str | None): + if not git_dir or not revision or not prev_rev: + return "" + + out = "" + try: + cmd = f"git --git-dir='{git_dir}/.git' log --format=\"%t/%s\" --no-merges {prev_rev}..{revision}" + for commit in ( + subprocess.run(cmd, stdout=subprocess.PIPE, shell=True) + .stdout.decode("utf-8") + .splitlines() + ): + if not "/" in commit: + continue + idx = commit.index("/") + out += f" - **{commit[:idx]}** {commit[idx+1:]}\n" + except Exception as e: + logger.error(f"Failed to get commits: {e}") + return out + +def get_package_update_str(base_pkg: Sequence[Package] | None, info: ExportInfo | None): + if not base_pkg or not info or not info.get("packages", None): + return "" + + previous = info.get("packages", {}) + seen = set() + + out = "" + for p in base_pkg: + if p.name in seen: + continue + seen.add(p.name) + + if p.name not in previous: + out += f" - {p.name}: x → {p.version}\n" + else: + pv = previous[p.name] + # Skip release for package version updates + if "-" in pv: + prel = pv[pv.rindex("-") + 1 :] + pv = pv[:pv.rindex("-")] + else: + prel = None + + if p.version != pv: + out += f" - {p.name}: {pv} → {p.version}\n" + elif prel and p.release != prel: + out += f" - {p.name}: {pv}-{prel} → {p.version}-{p.release}\n" + + for p in previous: + if p not in seen: + pv = previous[p] + if "-" in pv: + pv = pv[:pv.rindex("-")] + out += f" - {p}: {pv} → x\n" + seen.add(p) + + return out + def get_labels( labels: Sequence[str], version: str | None, @@ -142,12 +203,18 @@ def get_labels( pretty: str | None, base_pkg: Sequence[Package] | None, layers: dict[str, Sequence[str]], + revision: str | None, + git_dir: str | None, + changelog_template: str | None, + changelog_fn: str | None, + info: ExportInfo | None, ) -> tuple[dict[str, str], str]: # Date format is YYMMDD # Timestamp format is YYYY-MM-DDTHH:MM:SSZ now = datetime.now() date = now.strftime("%y%m%d") timestamp = now.strftime("%Y-%m-%dT%H:%M:%SZ") + pkgupd = get_package_update_str(base_pkg, info) prev_labels = prev_manifest.get("Labels", {}) if prev_manifest else {} prev_version = prev_labels.get(VERSION_TAG, None) if prev_labels else None @@ -181,12 +248,68 @@ def get_labels( with open(version_fn, "w") as f: f.write(new_version) - imginfo = export_v2(uniq=new_version, base_pkg=base_pkg, layers=list(layers.values())) + imginfo = export_v2( + uniq=new_version, + base_pkg=base_pkg, + layers=list(layers.values()), + revision=revision, + ) BLACKLIST_KEY = "> IMGINFO V2 INSERTED" new_labels[INFO_KEY] = imginfo + if revision: + new_labels[REVISION_TAG] = revision blacklist = dict() blacklist[INFO_KEY] = BLACKLIST_KEY + commit_str = get_commits( + git_dir, + revision, + (info or {}).get("revision", None) or prev_labels.get(REVISION_TAG, None), + ) + + def process_label(key: str, value: str): + if "" in value: + value = value.replace("", changelog_template or "") + if "" in value and new_version: + value = value.replace("", new_version) + if "" in value: + value = value.replace("", date) + if "" in value: + value = value.replace("", timestamp) + if "" in value and pretty: + value = value.replace("", pretty) + if "" in value and prev_version: + value = value.replace("", prev_version) + if "" in value: + value = value.replace("", imginfo) + blacklist[key] = BLACKLIST_KEY + if "" in value: + value = value.replace("", commit_str or "-") + if "" in value: + value = value.replace("", pkgupd or "-") + + if base_pkg: + for pkg in base_pkg: + if not pkg.version: + continue + vkey = f"" + if vkey in value: + value = value.replace(vkey, pkg.version) + vkey = f"" + if vkey in value: + value = value.replace( + vkey, + ( + f"{pkg.version}-{pkg.release}" + if pkg.release + else pkg.version + ), + ) + return value + + if changelog_fn: + with open(changelog_fn, "w") as f: + f.write(process_label("", "")) if labels: for line in labels: @@ -196,39 +319,7 @@ def get_labels( idx = line.index("=") key = line[:idx] value = line[idx + 1 :] - if "" in value and new_version: - value = value.replace("", new_version) - if "" in value: - value = value.replace("", date) - if "" in value: - value = value.replace("", timestamp) - if "" in value and pretty: - value = value.replace("", pretty) - if "" in value and prev_version: - value = value.replace("", prev_version) - if "" in value: - value = value.replace("", imginfo) - blacklist[key] = BLACKLIST_KEY - - if base_pkg: - for pkg in base_pkg: - if not pkg.version: - continue - vkey = f"" - if vkey in value: - value = value.replace(vkey, pkg.version) - vkey = f"" - if vkey in value: - value = value.replace( - vkey, - ( - f"{pkg.version}-{pkg.release}" - if pkg.release - else pkg.version - ), - ) - - new_labels[key] = value + new_labels[key] = process_label(key, value) if new_labels: log = "Writing labels:\n"