test: add integration tests for "pebble run" (#497)

Add integration tests for `pebble run`.

.github/workflows/integration-tests.yml

@@ -0,0 +1,23 @@
+name: Integration Tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    name: tests
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: 1.22
+
+    - name: Run tests
+      run: go test -count=1 -tags=integration ./tests/

HACKING.md

@@ -202,6 +202,13 @@ ok      github.com/canonical/pebble/cmd/pebble  0.165s
 ...
 ```
 
+Pebble also has a suite of integration tests for testing things like `pebble run`. To run them, use the "integration" build constraint:
+
+```
+$ go test -count=1 -tags=integration ./tests/
+ok  	github.com/canonical/pebble/tests	4.774s
+```
+
 ## Docs
 
 We use [`sphinx`](https://www.sphinx-doc.org/en/master/) to build the docs with styles preconfigured by the [Canonical Documentation Starter Pack](https://github.com/canonical/sphinx-docs-starter-pack).

tests/README.md

@@ -0,0 +1,33 @@
+# Pebble Integration Tests
+
+This directory holds a suite of integration tests for end-to-end tests of things like pebble run. They use the standard go test runner, but are only executed if you set the integration build constraint.
+
+## Run Tests
+
+```bash
+go test -count=1 -tags=integration ./tests/
+```
+
+The above command will build Pebble first, then run tests with it.
+
+To use an existing Pebble binary rather than building one, you can explicitly set the flag `-pebblebin`. For example, the following command will use a pre-built Pebble at `/home/ubuntu/pebble`:
+
+```bash
+go test -v -count=1 -tags=integration ./tests -pebblebin=/home/ubuntu/pebble
+```
+
+## Developing
+
+### Visual Studio Code Settings
+
+For VSCode Go and the gopls extention to work properly with files containing build tags, add the following:
+
+```json
+{
+    "gopls": {
+        "build.buildFlags": [
+            "-tags=integration"
+        ]
+    }
+}
+```

tests/main_test.go

@@ -0,0 +1,191 @@
+//go:build integration
+
+// Copyright (c) 2024 Canonical Ltd
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License version 3 as
+// published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package tests
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/canonical/pebble/internals/servicelog"
+)
+
+var pebbleBin = flag.String("pebblebin", "", "Path to the pre-built Pebble binary")
+
+// TestMain builds the pebble binary of `-pebblebin` flag is not set
+// before running the integration tests.
+func TestMain(m *testing.M) {
+	flag.Parse()
+
+	if *pebbleBin == "" {
+		goBuild := exec.Command("go", "build", "-o", "../pebble", "../cmd/pebble")
+		if err := goBuild.Run(); err != nil {
+			fmt.Println("Cannot build pebble binary:", err)
+			os.Exit(1)
+		}
+		*pebbleBin = "../pebble"
+	} else {
+		// Use the pre-built Pebble binary provided by the pebbleBin flag.
+		fmt.Println("Using pre-built Pebble binary at:", *pebbleBin)
+	}
+
+	exitCode := m.Run()
+	os.Exit(exitCode)
+}
+
+// createLayer creates a layer file with layerYAML under the directory "pebbleDir/layers".
+func createLayer(t *testing.T, pebbleDir, layerFileName, layerYAML string) {
+	t.Helper()
+
+	layersDir := filepath.Join(pebbleDir, "layers")
+	err := os.MkdirAll(layersDir, 0o755)
+	if err != nil {
+		t.Fatalf("Cannot create layers directory: %v", err)
+	}
+
+	layerPath := filepath.Join(layersDir, layerFileName)
+	err = os.WriteFile(layerPath, []byte(layerYAML), 0o755)
+	if err != nil {
+		t.Fatalf("Cannot create layers file: %v", err)
+	}
+}
+
+// pebbleRun starts the pebble daemon (`pebble run`) with optional arguments
+// and returns two channels for standard output and standard error.
+func pebbleRun(t *testing.T, pebbleDir string, args ...string) (stdoutCh chan servicelog.Entry, stderrCh chan servicelog.Entry) {
+	t.Helper()
+
+	stdoutCh = make(chan servicelog.Entry)
+	stderrCh = make(chan servicelog.Entry)
+
+	cmd := exec.Command(*pebbleBin, append([]string{"run"}, args...)...)
+	cmd.Env = append(os.Environ(), "PEBBLE="+pebbleDir)
+
+	stdoutPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		t.Fatalf("Cannot create stdout pipe: %v", err)
+	}
+	stderrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		t.Fatalf("Cannot create stderr pipe: %v", err)
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		t.Fatalf("Error starting 'pebble run': %v", err)
+	}
+
+	stopStdout := make(chan struct{})
+	stopStderr := make(chan struct{})
+
+	t.Cleanup(func() {
+		err := cmd.Process.Signal(os.Interrupt)
+		if err != nil {
+			t.Errorf("Error sending SIGINT/Ctrl+C to pebble: %v", err)
+		}
+		cmd.Wait()
+		close(stopStdout)
+		close(stopStderr)
+	})
+
+	readLogs := func(parser *servicelog.Parser, ch chan servicelog.Entry, stop <-chan struct{}) {
+		for parser.Next() {
+			if err := parser.Err(); err != nil {
+				t.Errorf("Cannot parse Pebble logs: %v", err)
+			}
+			select {
+			case ch <- parser.Entry():
+			case <-stop:
+				return
+			}
+		}
+	}
+
+	// Both stderr and stdout are needed, because pebble logs to stderr
+	// while with "--verbose", services output to stdout.
+	stderrParser := servicelog.NewParser(stderrPipe, 4*1024)
+	stdoutParser := servicelog.NewParser(stdoutPipe, 4*1024)
+
+	go readLogs(stdoutParser, stdoutCh, stopStdout)
+	go readLogs(stderrParser, stderrCh, stopStderr)
+
+	return stdoutCh, stderrCh
+}
+
+// waitForLog waits until an expectedLog from an expectedService appears in the logs channel, or fails the test after a
+// specified timeout if the expectedLog is still not found.
+func waitForLog(t *testing.T, logsCh <-chan servicelog.Entry, expectedService, expectedLog string, timeout time.Duration) {
+	t.Helper()
+
+	timeoutCh := time.After(timeout)
+	for {
+		select {
+		case log, ok := <-logsCh:
+			if !ok {
+				t.Error("channel closed before all expected logs were received")
+			}
+
+			if log.Service == expectedService && strings.Contains(log.Message, expectedLog) {
+				return
+			}
+
+		case <-timeoutCh:
+			t.Fatalf("timed out after %v waiting for log %s", 3*time.Second, expectedLog)
+		}
+	}
+}
+
+// waitForFile waits until a file exists, or fails the test after a specified timeout
+// if the file still doesn't exist.
+func waitForFile(t *testing.T, file string, timeout time.Duration) {
+	t.Helper()
+
+	timeoutCh := time.After(timeout)
+	ticker := time.NewTicker(time.Millisecond)
+	for {
+		select {
+		case <-timeoutCh:
+			t.Fatalf("timeout waiting for file %s", file)
+
+		case <-ticker.C:
+			stat, err := os.Stat(file)
+			if err == nil && stat.Mode().IsRegular() {
+				return
+			}
+		}
+	}
+}
+
+// runPebbleCommand runs a pebble command and returns the standard output.
+func runPebbleCommand(t *testing.T, pebbleDir string, args ...string) string {
+	t.Helper()
+
+	cmd := exec.Command(*pebbleBin, args...)
+	cmd.Env = append(os.Environ(), "PEBBLE="+pebbleDir)
+
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("error executing pebble command: %v", err)
+	}
+
+	return string(output)
+}