projects: Add support for custom data

This patch implements proposal ST090. It allows to add custom
metadata into the snapcraft.yaml files without interferring
with snapcraft.

This is useful for adding extra data for tools that manage
snap files, like updatesnap.py, which searches for new versions
of the parts in a snapcraft.yaml file.

Proposal: https://docs.google.com/document/d/1QX0moP6V709D7L267pIT5AREQetbPb7KsFrpgvCLKVM

snapcraft/projects.py

@@ -32,6 +32,7 @@
     convert_architecture_deb_to_platform,
     get_effective_base,
     get_host_architecture,
+    remove_custom_data,
 )
 
 
@@ -639,6 +640,7 @@ def unmarshal(cls, data: Dict[str, Any]) -> "Project":
         if not isinstance(data, dict):
             raise TypeError("Project data is not a dictionary")
 
+        data = remove_custom_data(data)
         try:
             project = Project(**data)
         except pydantic.ValidationError as err:

snapcraft/utils.py

@@ -25,7 +25,7 @@
 from dataclasses import dataclass
 from getpass import getpass
 from pathlib import Path
-from typing import Iterable, List, Optional
+from typing import Any, Dict, Iterable, List, Optional
 
 from craft_cli import emit
 from craft_parts.sources.git_source import GitSource
@@ -457,3 +457,30 @@ def process_version(version: Optional[str]) -> str:
 def is_snapcraft_running_from_snap() -> bool:
     """Check if snapcraft is running from the snap."""
     return os.getenv("SNAP_NAME") == "snapcraft" and os.getenv("SNAP") is not None
+
+
+def _remove_custom_data_from_dict(data: Dict[str, Any]) -> Dict[str, Any]:
+    if data is None or isinstance(data, str):
+        return data
+    prog = re.compile("^custom-data-[a-zA-Z0-9][a-zA-Z0-9_-]*$")
+    for element in list(data.keys()):
+        if prog.match(element) is not None:
+            data.pop(element)
+    return data
+
+
+def remove_custom_data(data: Dict[str, Any]) -> Dict[str, Any]:
+    """Remove custom metadata from the snapcraft.yaml dictionary."""
+    if data is None or isinstance(data, str):
+        return data
+    # remove custom data from the root
+    data = _remove_custom_data_from_dict(data)
+
+    # remove custom data from apps, plugs, slots and parts
+    for entry_name in ["apps", "plugs", "slots", "parts"]:
+        if entry_name in data and isinstance(data[entry_name], dict):
+            for app in list(data[entry_name].keys()):
+                data[entry_name][app] = _remove_custom_data_from_dict(
+                    data[entry_name][app]
+                )
+    return data

snapcraft_legacy/project/_project_info.py

@@ -17,6 +17,7 @@
 from copy import deepcopy
 
 import snapcraft_legacy.yaml_utils.errors
+from snapcraft.utils import remove_custom_data
 from snapcraft_legacy import yaml_utils
 
 from . import _schema
@@ -27,7 +28,8 @@ class ProjectInfo:
 
     def __init__(self, *, snapcraft_yaml_file_path) -> None:
         self.snapcraft_yaml_file_path = snapcraft_yaml_file_path
-        self.__raw_snapcraft = yaml_utils.load_yaml_file(snapcraft_yaml_file_path)
+        self.__raw_snapcraft = remove_custom_data(
+            yaml_utils.load_yaml_file(snapcraft_yaml_file_path))
 
         try:
             self.name = self.__raw_snapcraft["name"]

tests/unit/test_utils.py

@@ -21,6 +21,7 @@
 from unittest.mock import call, patch
 
 import pytest
+import yaml
 
 from snapcraft import errors, utils
 
@@ -607,3 +608,206 @@ def fake_input(prompt):
 
     with patch("snapcraft.utils.input", fake_input):
         utils.confirm_with_user("prompt")
+
+
+def test_remove_custom_data():
+    """Tests that remove_custom_data removes correctly the custom
+    metadata from a .yaml file."""
+    test_snap = yaml.load(
+        """name: snapcraft
+base: core22
+summary: easily create snaps
+description: |
+  Snapcraft aims to make upstream developers' lives easier and as such is not
+  a single toolset, but instead is a collection of tools that enable the
+  natural workflow of an upstream to be extended with a simple release step
+  into Snappy.
+adopt-info: snapcraft
+confinement: classic
+license: GPL-3.0
+assumes:
+  - snapd2.43
+
+custom-data-tests:
+  entry1: "data1"
+  entry2: 4
+
+environment:
+  PATH: "$SNAP/libexec/snapcraft:/snap/bin:/usr/local/sbin:\
+/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+  LD_LIBRARY_PATH: "$SNAP/none"
+
+apps:
+  snapcraft:
+    custom-data-tests:
+      entry_app: "snapcraft"
+    environment:
+      PYLXD_WARNINGS: "none"
+    command: bin/python $SNAP/bin/snapcraft
+    completer: snapcraft-completion
+
+build-packages:
+  - cargo
+  - rustc
+  - sed
+
+parts:
+  bash-completion:
+    custom-data-tests:
+      entry_part: "bash-completion"
+    source: debian
+    plugin: dump
+    stage:
+      - snapcraft-completion
+
+  patchelf:
+    plugin: autotools
+    source: https://github.com/snapcore/patchelf
+    source-type: git
+    source-branch: '0.9+snapcraft'
+    custom-data-tests:
+      entry_part: "patchelf"
+    autotools-configure-parameters:
+      - --prefix=/
+    build-attributes:
+      - enable-patchelf
+    build-packages:
+      - g++
+      - git
+      - make
+    override-pull: |
+      ${SNAP}/libexec/snapcraft/craftctl default
+
+      if [ "${CRAFT_TARGET_ARCH}" = "riscv64" ]; then
+        git am "${CRAFT_PROJECT_DIR}/snap/local/patches/patchelf/\
+0001-Always-use-the-ET_DYN-codepath-avoiding-shifting-loa.patch"
+        git am "${CRAFT_PROJECT_DIR}/snap/local/patches/patchelf/\
+0002-Fix-rewriteSectionsLibrary-to-not-assume-the-base-ad.patch"
+      fi
+    override-build: |
+      ${SNAP}/libexec/snapcraft/craftctl default
+      make check
+    prime:
+      - bin/patchelf
+
+  snapcraft-libs:
+    custom-data-tests:
+      entry_part: "snapcraft-libs"
+    plugin: nil
+    stage-packages:
+        - apt
+        - apt-transport-https
+        - python3.10-minimal
+        - squashfs-tools
+        - xdelta3
+    build-attributes:
+      - enable-patchelf
+
+  snapcraft:
+    custom-data-tests:
+      entry_part: "snapcraft"
+    source: .
+    plugin: python
+    python-packages:
+        - wheel
+        - pip
+    python-requirements:
+        - requirements.txt
+    organize:
+        bin/craftctl: libexec/snapcraft/craftctl
+        bin/snapcraftctl: bin/scriptlet-bin/snapcraftctl
+        bin/snapcraftctl-compat: libexec/snapcraft/snapcraftctl
+    build-attributes:
+      - enable-patchelf
+    build-environment:
+        - "PIP_NO_BINARY": "PyNaCl"
+        - "SODIUM_INSTALL": "system"
+        - "CFLAGS": "$(pkg-config python-3.10 yaml-0.1 --cflags)"
+    after: [snapcraft-libs]
+""",
+        Loader=yaml.SafeLoader,
+    )
+    filtered_data = utils.remove_custom_data(test_snap)
+    expected = {
+        "name": "snapcraft",
+        "base": "core22",
+        "summary": "easily create snaps",
+        "description": "Snapcraft aims to make upstream developers' lives "
+        "easier and as such is not\na single toolset, but "
+        "instead is a collection of tools that enable the\n"
+        "natural workflow of an upstream to be extended with "
+        "a simple release step\ninto Snappy.\n",
+        "adopt-info": "snapcraft",
+        "confinement": "classic",
+        "license": "GPL-3.0",
+        "assumes": ["snapd2.43"],
+        "environment": {
+            "PATH": "$SNAP/libexec/snapcraft:/snap/bin:/usr/local/sbin:"
+            "/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+            "LD_LIBRARY_PATH": "$SNAP/none",
+        },
+        "apps": {
+            "snapcraft": {
+                "environment": {"PYLXD_WARNINGS": "none"},
+                "command": "bin/python $SNAP/bin/snapcraft",
+                "completer": "snapcraft-completion",
+            }
+        },
+        "build-packages": ["cargo", "rustc", "sed"],
+        "parts": {
+            "bash-completion": {
+                "source": "debian",
+                "plugin": "dump",
+                "stage": ["snapcraft-completion"],
+            },
+            "patchelf": {
+                "plugin": "autotools",
+                "source": "https://github.com/snapcore/patchelf",
+                "source-type": "git",
+                "source-branch": "0.9+snapcraft",
+                "autotools-configure-parameters": ["--prefix=/"],
+                "build-attributes": ["enable-patchelf"],
+                "build-packages": ["g++", "git", "make"],
+                "override-pull": "${SNAP}/libexec/snapcraft/craftctl default\n\n"
+                'if [ "${CRAFT_TARGET_ARCH}" = "riscv64" ]; then'
+                '\n  git am "${CRAFT_PROJECT_DIR}/snap/local/'
+                "patches/patchelf/0001-Always-use-the-ET_DYN-codepath"
+                '-avoiding-shifting-loa.patch"\n  git am '
+                '"${CRAFT_PROJECT_DIR}/snap/local/patches/patchelf/'
+                "0002-Fix-rewriteSectionsLibrary-to-not-assume-"
+                'the-base-ad.patch"\nfi\n',
+                "override-build": "${SNAP}/libexec/snapcraft/craftctl default\nmake check\n",
+                "prime": ["bin/patchelf"],
+            },
+            "snapcraft-libs": {
+                "plugin": "nil",
+                "stage-packages": [
+                    "apt",
+                    "apt-transport-https",
+                    "python3.10-minimal",
+                    "squashfs-tools",
+                    "xdelta3",
+                ],
+                "build-attributes": ["enable-patchelf"],
+            },
+            "snapcraft": {
+                "source": ".",
+                "plugin": "python",
+                "python-packages": ["wheel", "pip"],
+                "python-requirements": ["requirements.txt"],
+                "organize": {
+                    "bin/craftctl": "libexec/snapcraft/craftctl",
+                    "bin/snapcraftctl": "bin/scriptlet-bin/snapcraftctl",
+                    "bin/snapcraftctl-compat": "libexec/snapcraft/snapcraftctl",
+                },
+                "build-attributes": ["enable-patchelf"],
+                "build-environment": [
+                    {"PIP_NO_BINARY": "PyNaCl"},
+                    {"SODIUM_INSTALL": "system"},
+                    {"CFLAGS": "$(pkg-config python-3.10 yaml-0.1 --cflags)"},
+                ],
+                "after": ["snapcraft-libs"],
+            },
+        },
+    }
+    assert filtered_data == expected