tests: new approach to run tests in ubuntu and debian using snapd deb from the repo

spread.yaml

@@ -82,6 +82,8 @@ environment:
     SNAPD_NO_MEMORY_LIMIT: '$(HOST: echo "${SPREAD_SNAPD_NO_MEMORY_LIMIT:-}")'
 
     SNAPD_PUBLISHED_VERSION: '$(HOST: echo "$SPREAD_SNAPD_PUBLISHED_VERSION")'
+    # Use the snapd package from the repository when possible
+    SNAPD_DEB_FROM_REPO: '$(HOST: echo "${SPREAD_SNAPD_DEB_FROM_REPO:-true}")'
     # Build and use snapd from current branch
     BUILD_SNAPD_FROM_CURRENT: '$(HOST: echo "${SPREAD_BUILD_SNAPD_FROM_CURRENT:-true}")'
 
@@ -1430,6 +1432,10 @@ suites:
             "$TESTSLIB"/prepare-restore.sh --prepare-suite
         prepare-each: |
             "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
+            # When snapd.deb is from the repo, we just need to test when re-exec is enabled
+            if [ "$SNAP_REEXEC" = "0" ] && tests.info is-snapd-pkg-repo; then
+                tests.exec skip-test "When snapd.deb is from the repo, we just need to test when re-exec is enabled" && exit 0
+            fi
         restore-each: |
             "$TESTSLIB"/prepare-restore.sh --restore-suite-each
         restore: |

tests/bin/tests.exec

@@ -0,0 +1 @@
+../lib/tools/tests.exec

tests/bin/tests.info

@@ -0,0 +1 @@
+../lib/tools/tests.info

tests/lib/prepare-restore.sh

@@ -587,7 +587,12 @@ prepare_project() {
     # go mod runs as root and will leave strange permissions
     chown test:test -R "$SPREAD_PATH"
 
-    if [ "$BUILD_SNAPD_FROM_CURRENT" = true ]; then
+    # We are testing snapd snap on top of snapd from the archive
+    # of the tested distribution. Download snapd and snap-confine
+    # as they exist in the archive for further use.
+    if tests.info is-snapd-pkg-repo; then
+        ( cd "${GOHOME}" && tests.pkgs download snapd snap-confine)
+    elif [ "$BUILD_SNAPD_FROM_CURRENT" = true ]; then
         case "$SPREAD_SYSTEM" in
             ubuntu-*|debian-*)
                 build_deb

tests/lib/prepare.sh

@@ -154,6 +154,25 @@ setup_experimental_features() {
     fi
 }
 
+save_installed_core_snap() {
+    local target_dir="${1-}"
+
+    SNAP_MOUNT_DIR="$(os.paths snap-mount-dir)"
+    LIBEXEC_DIR="$(os.paths libexec-dir)"
+    core="$(readlink -f "$SNAP_MOUNT_DIR/core/current")"
+    snap="$(mount | grep " $core" | head -n 1 | awk '{print $1}')"
+    snap_name="$(basename "$snap")"
+
+    # make a copy for later use
+    if [ -n "$target_dir" ]; then
+        mkdir -p "$target_dir"
+
+        cp -av "$snap" "${target_dir}/${snap_name}"
+        cp "$snap" "${target_dir}/${snap_name}.orig"
+    fi
+}
+
+
 # update_core_snap_for_classic_reexec modifies the core snap for snapd re-exec
 # by injecting binaries from the installed snapd deb built from our modified code.
 # $1: directory where updated core snap should be copied (optional)
@@ -450,10 +469,17 @@ prepare_classic() {
 
         snap list | grep core
 
-        systemctl stop snapd.{service,socket}
-        # repack and also make a side copy of the core snap
-        update_core_snap_for_classic_reexec "$TESTSTMP/core_snap"
-        systemctl start snapd.{service,socket}
+        # With reexec, and on classic, the snapd snap is preferred over the core snap for reexecution target,
+        # so to be as close as possible to the actual real life scenarios, we only update the snapd snap.
+        # The tests alreday ensure that snapd snap is installed.
+        if tests.info is-snapd-pkg-repo; then
+            save_installed_core_snap "$TESTSTMP/core_snap"
+        else
+            systemctl stop snapd.{service,socket}
+            # repack and also make a side copy of the core snap
+            update_core_snap_for_classic_reexec "$TESTSTMP/core_snap"
+            systemctl start snapd.{service,socket}
+        fi
 
         prepare_reexec_override
         prepare_memory_limit_override
@@ -547,6 +573,7 @@ build_snapd_snap() {
                         ;;
                 esac
                 [ -d "${TARGET}" ] || mkdir -p "${TARGET}"
+                touch "${PROJECT_PATH}"/test-build
                 chmod -R go+r "${PROJECT_PATH}/tests"
                 # TODO: run_snapcraft does not currently guarantee or check the required version for building snapd
                 run_snapcraft --use-lxd --verbosity quiet --output="snapd_from_snapcraft.snap"
@@ -1192,7 +1219,7 @@ setup_reflash_magic() {
         # FIXME: install would be better but we don't have dpkg on
         #        the image
         # unpack our freshly build snapd into the new snapd snap
-        dpkg-deb -x "$SPREAD_PATH"/../snapd_*.deb "$UNPACK_DIR"
+        dpkg-deb -x "$GOHOME"/snapd_*.deb "$UNPACK_DIR"
         # Debian packages don't carry permissions correctly and we use
         # post-inst hooks to fix that on classic systems. Here, as a special
         # case, fix the void directory we just unpacked.

tests/lib/tools/snapd.tool

@@ -47,8 +47,15 @@ main() {
 		esac
 	done
 
-	LIBEXEC_DIR="$(os.paths libexec-dir)"
-	exec "$LIBEXEC_DIR/snapd/$tool" "$@"
+	# Use the tool from the snapd snap when re-exec is enabled
+	# Otherwise use the tool from the distro installation
+	if tests.info is-reexec-in-use; then
+		MOUNT_DIR="$(os.paths snap-mount-dir)"
+		exec "$MOUNT_DIR"/snapd/current/usr/lib/snapd/"$tool" "$@"
+	else
+		LIBEXEC_DIR="$(os.paths libexec-dir)"
+		exec "$LIBEXEC_DIR/snapd/$tool" "$@"
+	fi
 }
 
 main "$@"

tests/lib/tools/tests.exec

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -eu
+
+show_help() {
+    echo "usage: tests.exec skip-test [MSG]"
+    echo "usage: tests.exec is-skipped"
+    echo
+    echo "Supported commands:"
+    echo "    skip-test:  indicates the test has to be skipped and saves the raeson"
+    echo "    is-skipped: check if the test has to be skipped and prints the raeson"
+}
+
+skip_test() {
+    local raeson="${1:-}"
+    echo "$raeson" > tests.exec
+}
+
+is_skipped() {
+    if [ -f tests.exec ]; then
+        echo "skip raeson: $(cat tests.exec)"
+        return 0
+    fi
+    return 1
+}
+
+main() {
+    if [ $# -eq 0 ]; then
+        show_help
+        exit 0
+    fi
+
+    local subcommand="$1"
+    local action=
+    while [ $# -gt 0 ]; do
+        case "$subcommand" in
+            -h|--help)
+                show_help
+                exit 0
+                ;;
+            *)
+                action=$(echo "$subcommand" | tr '-' '_')
+                shift
+                break
+                ;;
+        esac
+    done
+
+    if [ -z "$(declare -f "$action")" ]; then
+        echo "tests.exec: no such command: $subcommand"
+        show_help
+        exit 1
+    fi
+
+    "$action" "$@"
+}
+
+main "$@"

tests/lib/tools/tests.info

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+set -eu
+
+show_help() {
+    echo "usage: tests.info <CMD>"
+    echo
+    echo "Supported commands:"
+    echo "    is-snapd-pkg-repo: indicates when the snapd package is from the repository"
+    echo "    is-reexec-enabled: indicates re-execution has been explicitly enabled through the env"
+    echo "    is-reexec-in-use:  indicates re-execution to the snapd snap is being effectively performed"
+}
+
+is_snapd_pkg_repo() {
+    [ "$SNAPD_DEB_FROM_REPO" = true ] && \
+    [ -n "$(command -v apt)" ] && \
+    [[ "$SPREAD_SYSTEM" != ubuntu-core-* ]] && \
+    not os.query is-trusty
+}
+
+is_reexec_enabled() {
+    snap debug execution snap | grep -q "is-reexec-enabled: true"
+}
+
+is_reexec_in_use() {
+    snap debug execution snap | grep -q "is-reexecd: true"
+}
+
+main() {
+    if [ $# -eq 0 ]; then
+        show_help
+        exit 0
+    fi
+
+    local subcommand="$1"
+    local action=
+    while [ $# -gt 0 ]; do
+        case "$subcommand" in
+            -h|--help)
+                show_help
+                exit 0
+                ;;
+            *)
+                action=$(echo "$subcommand" | tr '-' '_')
+                shift
+                break
+                ;;
+        esac
+    done
+
+    if [ -z "$(declare -f "$action")" ]; then
+        echo "tests.info: no such command: $subcommand"
+        show_help
+        exit 1
+    fi
+
+    "$action" "$@"
+}
+
+main "$@"

tests/main/classic-custom-device-reg/task.yaml

@@ -15,9 +15,9 @@ environment:
     SEED_DIR: /var/lib/snapd/seed
 
 prepare: |
-    if [ "$TRUST_TEST_KEYS" = "false" ]; then
-        echo "This test needs test keys to be trusted"
-        exit
+    # In this scenario, the keys from the snapd pkg are used
+    if [ "$TRUST_TEST_KEYS" = "false" ] || tests.info is-snapd-pkg-repo; then
+        tests.exec skip-test "This test needs test keys to be trusted" && exit 0
     fi
 
     snap pack "$TESTSLIB/snaps/classic-gadget"
@@ -52,20 +52,15 @@ prepare: |
     systemd-run --unit fakedevicesvc fakedevicesvc localhost:11029
 
 restore: |
-    if [ "$TRUST_TEST_KEYS" = "false" ]; then
-        echo "This test needs test keys to be trusted"
-        exit
-    fi
+    tests.exec is-skipped && exit 0
+
     systemctl stop snapd.service snapd.socket fakedevicesvc
 
     rm -rf "$SEED_DIR"
     systemctl start snapd.socket snapd.service
 
 execute: |
-    if [ "$TRUST_TEST_KEYS" = "false" ]; then
-        echo "This test needs test keys to be trusted"
-        exit
-    fi
+    tests.exec is-skipped && exit 0
 
     #shellcheck source=tests/lib/core-config.sh
     . "$TESTSLIB"/core-config.sh

tests/main/classic-firstboot/task.yaml

@@ -14,9 +14,9 @@ environment:
     SEED_DIR: /var/lib/snapd/seed
 
 prepare: |
-    if [ "$TRUST_TEST_KEYS" = "false" ]; then
-        echo "This test needs test keys to be trusted"
-        exit
+    # In this scenario, the keys from the snapd pkg are used
+    if [ "$TRUST_TEST_KEYS" = "false" ] || tests.info is-snapd-pkg-repo; then
+        tests.exec skip-test "This test needs test keys to be trusted" && exit 0
     fi
 
     snap pack "$TESTSLIB/snaps/basic"
@@ -51,19 +51,13 @@ prepare: |
     cp ./test-snapd-service_1.0_all.snap "$SEED_DIR/snaps/test-snapd-service.snap"
 
 restore: |
-    if [ "$TRUST_TEST_KEYS" = "false" ]; then
-        echo "This test needs test keys to be trusted"
-        exit
-    fi
+    tests.exec is-skipped && exit 0
 
     rm -rf "$SEED_DIR"
     systemctl start snapd.socket snapd.service
 
 execute: |
-    if [ "$TRUST_TEST_KEYS" = "false" ]; then
-        echo "This test needs test keys to be trusted"
-        exit
-    fi
+    tests.exec is-skipped && exit 0
 
     echo "Start the daemon with an empty state, this will make it import "
     echo "assertions from the $SEED_DIR/assertions subdirectory and "

tests/main/classic-prepare-image-no-core/task.yaml

@@ -20,9 +20,9 @@ environment:
     SEED_DIR: /var/lib/snapd/seed
 
 prepare: |
-    if [ "$TRUST_TEST_KEYS" = "false" ]; then
-        echo "This test needs test keys to be trusted"
-        exit
+    # In this scenario, the keys from the snapd pkg are used
+    if [ "$TRUST_TEST_KEYS" = "false" ] || tests.info is-snapd-pkg-repo; then
+        tests.exec skip-test "This test needs test keys to be trusted" && exit 0
     fi
 
     "$TESTSTOOLS"/store-state setup-fake-store "$STORE_DIR"
@@ -49,10 +49,8 @@ prepare: |
     systemd-run --unit fakedevicesvc fakedevicesvc localhost:11029
 
 restore: |
-    if [ "$TRUST_TEST_KEYS" = "false" ]; then
-        echo "This test needs test keys to be trusted"
-        exit
-    fi
+    tests.exec is-skipped && exit 0
+
     systemctl stop snapd.service snapd.socket fakedevicesvc
 
     rm -rf "$SEED_DIR"
@@ -61,10 +59,7 @@ restore: |
     rm -rf "$ROOT"
 
 execute: |
-    if [ "$TRUST_TEST_KEYS" = "false" ]; then
-        echo "This test needs test keys to be trusted"
-        exit
-    fi
+    tests.exec is-skipped && exit 0
 
     #shellcheck source=tests/lib/core-config.sh
     . "$TESTSLIB"/core-config.sh

tests/main/classic-prepare-image/task.yaml

@@ -19,9 +19,9 @@ environment:
     SEED_DIR: /var/lib/snapd/seed
 
 prepare: |
-    if [ "$TRUST_TEST_KEYS" = "false" ]; then
-        echo "This test needs test keys to be trusted"
-        exit
+    # In this scenario, the keys from the snapd pkg are used
+    if [ "$TRUST_TEST_KEYS" = "false" ] || tests.info is-snapd-pkg-repo; then
+        tests.exec skip-test "This test needs test keys to be trusted" && exit 0
     fi
 
     "$TESTSTOOLS"/store-state setup-fake-store "$STORE_DIR"
@@ -48,10 +48,8 @@ prepare: |
     systemd-run --unit fakedevicesvc fakedevicesvc localhost:11029
 
 restore: |
-    if [ "$TRUST_TEST_KEYS" = "false" ]; then
-        echo "This test needs test keys to be trusted"
-        exit
-    fi
+    tests.exec is-skipped && exit 0
+
     systemctl stop snapd.service snapd.socket fakedevicesvc
 
     rm -rf "$SEED_DIR"
@@ -60,10 +58,7 @@ restore: |
     rm -rf "$ROOT"
 
 execute: |
-    if [ "$TRUST_TEST_KEYS" = "false" ]; then
-        echo "This test needs test keys to be trusted"
-        exit
-    fi
+    tests.exec is-skipped && exit 0
 
     #shellcheck source=tests/lib/core-config.sh
     . "$TESTSLIB"/core-config.sh