Merge pull request #736 from cgwalters/tmt-split-tests

tests: Split up tmt tests into separate plans

plans/integration-run.fmf → plans/test-01-readonly.fmf

@@ -5,14 +5,18 @@ provision:
   # Generated by make test-tmt
   image: file://./target/testvm/disk.qcow2
   disk: 20
-summary: Execute booted tests
+summary: Execute booted readonly/nondestructive tests
 execute:
   how: tmt
   # There's currently two dynamic test frameworks; python and nushell.
   # python is well known and understood. nushell is less well known, but
   # is quite nice for running subprocesses and the like while making
   # it easy to parse JSON etc.
+  # All of these tests should generally be read-only - avoid any kind
+  # of persistent changes.
+  # If you need to do that, unfortunately right now that needs to be
+  # a separate plan.
   script: |
     set -xeu
-    pytest tests/booted/*.py
-    ls tests/booted/*-test-*.nu |sort -n | while read t; do nu $t; done
+    pytest tests/booted/readonly/*.py
+    ls tests/booted/readonly/*-test-*.nu |sort -n | while read t; do nu $t; done

plans/test-20-local-upgrade.fmf

@@ -0,0 +1,12 @@
+# 
+provision:
+  how: virtual
+  # Generated by make test-tmt
+  image: file://./target/testvm/disk.qcow2
+  disk: 20
+summary: Execute local upgrade tests
+execute:
+  how: tmt
+  # We avoid writing nontrivial shell script as a general rule,
+  # so this is written in nu.
+  script: exec nu tests/booted/test-image-pushpull-upgrade.nu

tests/booted/readonly/tap.nu

@@ -0,0 +1 @@
+../tap.nu

xtask/src/xtask.rs

@@ -148,10 +148,49 @@ fn update_generated(sh: &Shell) -> Result<()> {
 
 #[context("test-integration")]
 fn test_tmt(sh: &Shell) -> Result<()> {
+    // We need to split most of our tests into separate plans because tmt doesn't
+    // support automatic isolation. (xref)
+    let mut all_plan_files =
+        sh.read_dir("plans")?
+            .into_iter()
+            .try_fold(Vec::new(), |mut acc, ent| -> Result<_> {
+                let path = Utf8PathBuf::try_from(ent)?;
+                let Some(ext) = path.extension() else {
+                    return Ok(acc);
+                };
+                if ext != "fmf" {
+                    return Ok(acc);
+                }
+                let stem = path.file_stem().expect("file stem");
+                let Some((prefix, suffix)) = stem.split_once('-') else {
+                    return Ok(acc);
+                };
+                if prefix != "test" {
+                    return Ok(acc);
+                }
+                let Some((priority, _)) = suffix.split_once('-') else {
+                    anyhow::bail!("Invalid test {path}");
+                };
+                let priority: u32 = priority
+                    .parse()
+                    .with_context(|| format!("Parsing {path}"))?;
+                acc.push((priority, stem.to_string()));
+                Ok(acc)
+            })?;
+    all_plan_files.sort_by_key(|v| v.0);
+    println!("Discovered plans: {all_plan_files:?}");
+
     cmd!(sh, "cargo run -p tests-integration run-vm prepare-tmt").run()?;
     // cc https://pagure.io/testcloud/pull-request/174
     cmd!(sh, "rm -vf /var/tmp/tmt/testcloud/images/disk.qcow2").run()?;
-    cmd!(sh, "tmt run plans -n integration-run").run()?;
+
+    for (_prio, name) in all_plan_files {
+        if let Err(e) = cmd!(sh, "tmt run plans -n {name}").run() {
+            // tmt annoyingly does not output errors by default
+            let _ = cmd!(sh, "tmt run -l report -vvv").run();
+            return Err(e.into());
+        }
+    }
     Ok(())
 }