From 0b1c6772be4c1175e1d9d3039ed54c62fc5b3614 Mon Sep 17 00:00:00 2001
From: finn <fherzfeld@tbd.email>
Date: Tue, 31 Oct 2023 11:20:11 -0700
Subject: [PATCH] add "many" subcommand to test multiple targets and produce a
 single HTML report

previous functionality is available under the "one" subcommand
---
 .github/workflows/pages.yaml                  |  17 +++
 .github/workflows/self-test.yaml              |   4 +-
 cmd/web5-spec-test/main.go                    | 129 +----------------
 cmd/web5-spec-test/test-many.go               |  53 +++++++
 cmd/web5-spec-test/test-one.go                | 131 ++++++++++++++++++
 go.mod                                        |   6 +-
 go.sum                                        |  16 +++
 reports/report-template.html                  |  45 ++++++
 .../report-template.md                        |   0
 .../report-template.txt                       |   0
 {cmd/web5-spec-test => reports}/reports.go    |  63 ++++++++-
 11 files changed, 331 insertions(+), 133 deletions(-)
 create mode 100644 .github/workflows/pages.yaml
 create mode 100644 cmd/web5-spec-test/test-many.go
 create mode 100644 cmd/web5-spec-test/test-one.go
 create mode 100644 reports/report-template.html
 rename {cmd/web5-spec-test => reports}/report-template.md (100%)
 rename {cmd/web5-spec-test => reports}/report-template.txt (100%)
 rename {cmd/web5-spec-test => reports}/reports.go (50%)

diff --git a/.github/workflows/pages.yaml b/.github/workflows/pages.yaml
new file mode 100644
index 0000000..68ee324
--- /dev/null
+++ b/.github/workflows/pages.yaml
@@ -0,0 +1,17 @@
+on:
+  push:
+    branches: [main]
+jobs:
+  pages:
+    permissions:
+      pages: write
+      id-token: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+      - name: run for all SDKs
+        run: go run ./cmd/web5-spec-test many sdks/*
+      - uses: actions/upload-pages-artifact@v2
+      - name: deploy GitHub Pages
+        uses: actions/deploy-pages@v2
diff --git a/.github/workflows/self-test.yaml b/.github/workflows/self-test.yaml
index cd608be..62fafcb 100644
--- a/.github/workflows/self-test.yaml
+++ b/.github/workflows/self-test.yaml
@@ -9,11 +9,11 @@ jobs:
       matrix:
         SDK:
           - web5-js
-          - web5-kt
+          # - web5-kt
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
       - name: self test
-        run: go run ./cmd/web5-spec-test sdks/$SDK
+        run: go run ./cmd/web5-spec-test one sdks/$SDK
         env:
           SDK: ${{ matrix.SDK }}
diff --git a/cmd/web5-spec-test/main.go b/cmd/web5-spec-test/main.go
index d1e442b..0aa473d 100644
--- a/cmd/web5-spec-test/main.go
+++ b/cmd/web5-spec-test/main.go
@@ -1,25 +1,17 @@
 package main
 
 import (
-	"context"
-	"flag"
-	"fmt"
 	"os"
 	"os/exec"
-	"path/filepath"
-	"time"
 
-	"github.com/TBD54566975/web5-spec/openapi"
-	"github.com/TBD54566975/web5-spec/tests"
+	"github.com/spf13/cobra"
 	"golang.org/x/exp/slog"
 )
 
 var (
-	nostart = flag.Bool("no-start", false, "when set, the server is not built and is expected to be already running")
-	nostop  = flag.Bool("no-stop", false, "when set, the server is not asked to shut down")
-	server  = flag.String("server", "http://localhost:8080", "url of the server to connect to")
-
-	dir string
+	root = cobra.Command{
+		Use: "web5-spec",
+	}
 
 	dockerfiles = []string{
 		".web5-component/test.Dockerfile",
@@ -29,119 +21,10 @@ var (
 )
 
 func main() {
-	os.Exit(runTests()) // without this weird wrapper thing, the defers wont get called
-}
-
-func runTests() int {
-	flag.Parse()
-
-	dir, _ = os.Getwd()
-	if len(flag.Args()) > 0 {
-		dir = flag.Arg(0)
-	}
-
-	logger := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug})
-	slog.SetDefault(slog.New(logger))
-
-	if !*nostart {
-		var dockerfile string
-		for _, d := range dockerfiles {
-			candidate := filepath.Join(dir, d)
-			if _, err := os.Stat(candidate); !os.IsNotExist(err) {
-				dockerfile = d
-			}
-		}
-
-		if dockerfile == "" {
-			slog.Error("no dockerfile found", "paths", dockerfiles)
-			return 1
-		}
-
-		cmd := docker("build", "-t", "web5-spec:latest", "-f", dockerfile, ".")
-		if err := cmd.Run(); err != nil {
-			slog.Error("error building server", "error", err)
-			return 1
-		}
-
-		cmd = docker("run", "-p", "8080:8080", "--name", "web5-spec", "--rm", "web5-spec:latest")
-		if err := cmd.Start(); err != nil {
-			slog.Error("error running server", "error", err)
-			return 1
-		}
-
-		if !*nostop {
-			defer func() {
-				cmd := docker("stop", "web5-spec")
-				if err := cmd.Run(); err != nil {
-					slog.Error("error stopping server container", "error", err)
-				}
-			}()
-		}
-	}
-
-	ctx := context.Background()
-
-	client, err := openapi.NewClientWithResponses(*server)
-	if err != nil {
-		panic(err)
-	}
-
-	var serverID openapi.TestServerID
-	for {
-		serverIDResponse, err := client.IdentifySelfWithResponse(ctx)
-		if err != nil {
-			slog.Debug("server ID check failed, retrying in 1 second", "err", err)
-			time.Sleep(time.Second)
-			continue
-		}
-
-		if serverIDResponse.JSON200 == nil {
-			slog.Debug("server ID check failed, retrying in 1 second", "status", serverIDResponse.Status(), "body", string(serverIDResponse.Body))
-			time.Sleep(time.Second)
-			continue
-		}
-
-		serverID = *serverIDResponse.JSON200
-		break
-	}
-
-	defer func() {
-		_, err := client.ServerShutdown(context.Background())
-		if err != nil {
-			slog.Error("error shutting down server", "error", err)
-		}
-	}()
-
-	slog.Debug("server running", "sdk", serverID.Name, "url", serverID.Url)
-
-	report := Report{
-		TestServerID: serverID,
-		Results:      tests.RunTests(*server),
-	}
-
-	fmt.Println()
-	if txt, err := report.Text(); err != nil {
-		slog.Error("error generating text report", "error", err)
-	} else {
-		fmt.Println(txt)
-	}
-	fmt.Println()
-
-	stepSummaryFile := os.Getenv("GITHUB_STEP_SUMMARY")
-	if stepSummaryFile != "" {
-		if err := report.WriteMarkdown(stepSummaryFile); err != nil {
-			slog.Error("error writing github step summary", "file", stepSummaryFile, "error", err)
-		}
-	}
-
-	if !report.IsPassing() {
-		return 1
-	}
-
-	return 0
+	root.Execute()
 }
 
-func docker(args ...string) *exec.Cmd {
+func docker(dir string, args ...string) *exec.Cmd {
 	cmd := exec.Command("docker", args...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
diff --git a/cmd/web5-spec-test/test-many.go b/cmd/web5-spec-test/test-many.go
new file mode 100644
index 0000000..ced8d72
--- /dev/null
+++ b/cmd/web5-spec-test/test-many.go
@@ -0,0 +1,53 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/TBD54566975/web5-spec/reports"
+	"github.com/spf13/cobra"
+	"golang.org/x/exp/slog"
+)
+
+var (
+	testManyCmd = &cobra.Command{
+		Use:  "many dir [dir...]",
+		Args: cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			allReports := []reports.Report{}
+			for _, dir := range args {
+				report, err := testOne(dir)
+				if err != nil {
+					slog.Error("error testing server", "dir", dir, "err", err)
+					continue
+				}
+
+				allReports = append(allReports, report)
+			}
+
+			for _, report := range allReports {
+				fmt.Println()
+				if txt, err := report.Text(); err != nil {
+					slog.Error("error generating text report", "error", err)
+					continue
+				} else {
+					fmt.Println(txt)
+				}
+				fmt.Println()
+			}
+
+			if err := os.MkdirAll("_site", 0755); err != nil && err != os.ErrExist {
+				slog.Error("error creating _site/ for HTML report")
+				panic(err)
+			}
+			if err := reports.WriteHTML(allReports, "_site/index.html"); err != nil {
+				slog.Error("error rendering HTML template", "err", err)
+				os.Exit(1)
+			}
+		},
+	}
+)
+
+func init() {
+	root.AddCommand(testManyCmd)
+}
diff --git a/cmd/web5-spec-test/test-one.go b/cmd/web5-spec-test/test-one.go
new file mode 100644
index 0000000..fc6a931
--- /dev/null
+++ b/cmd/web5-spec-test/test-one.go
@@ -0,0 +1,131 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/TBD54566975/web5-spec/openapi"
+	"github.com/TBD54566975/web5-spec/reports"
+	"github.com/TBD54566975/web5-spec/tests"
+	"github.com/spf13/cobra"
+	"golang.org/x/exp/slog"
+)
+
+var (
+	testOneCmd = &cobra.Command{
+		Use: "one [dir]",
+		Run: func(cmd *cobra.Command, args []string) {
+			dir, _ := os.Getwd()
+			if len(args) > 0 {
+				dir = args[0]
+			}
+
+			report, err := testOne(dir)
+			if err != nil {
+				panic(err)
+			}
+
+			fmt.Println()
+			if txt, err := report.Text(); err != nil {
+				slog.Error("error generating text report", "error", err)
+			} else {
+				fmt.Println(txt)
+			}
+			fmt.Println()
+
+			stepSummaryFile := os.Getenv("GITHUB_STEP_SUMMARY")
+			if stepSummaryFile != "" {
+				if err := reports.WriteMarkdown(report, stepSummaryFile); err != nil {
+					slog.Error("error writing github step summary", "file", stepSummaryFile, "error", err)
+				}
+			}
+
+			if !report.IsPassing() {
+				os.Exit(1)
+			}
+
+			os.Exit(0)
+		},
+	}
+)
+
+func testOne(dir string) (reports.Report, error) {
+	logger := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug})
+	slog.SetDefault(slog.New(logger))
+
+	var dockerfile string
+	for _, d := range dockerfiles {
+		candidate := filepath.Join(dir, d)
+		if _, err := os.Stat(candidate); !os.IsNotExist(err) {
+			dockerfile = d
+		}
+	}
+
+	if dockerfile == "" {
+		return reports.Report{}, fmt.Errorf("no dockerfile found (paths=%v)", dockerfiles)
+	}
+
+	cmd := docker(dir, "build", "-t", "web5-spec:latest", "-f", dockerfile, ".")
+	if err := cmd.Run(); err != nil {
+		return reports.Report{}, fmt.Errorf("error building server: %v", err)
+	}
+
+	cmd = docker(dir, "run", "-p", "8080:8080", "--name", "web5-spec", "--rm", "web5-spec:latest")
+	if err := cmd.Start(); err != nil {
+		return reports.Report{}, fmt.Errorf("error running server: %v", err)
+	}
+
+	defer func() {
+		cmd := docker(dir, "stop", "web5-spec")
+		if err := cmd.Run(); err != nil {
+			slog.Error("error stopping server container", "error", err)
+		}
+	}()
+
+	ctx := context.Background()
+
+	client, err := openapi.NewClientWithResponses("http://localhost:8080")
+	if err != nil {
+		panic(err)
+	}
+
+	var serverID openapi.TestServerID
+	for {
+		serverIDResponse, err := client.IdentifySelfWithResponse(ctx)
+		if err != nil {
+			slog.Debug("waiting for server to come up", "err", err)
+			time.Sleep(time.Second)
+			continue
+		}
+
+		if serverIDResponse.JSON200 == nil {
+			slog.Debug("server ID check failed, retrying in 1 second", "status", serverIDResponse.Status(), "body", string(serverIDResponse.Body))
+			time.Sleep(time.Second)
+			continue
+		}
+
+		serverID = *serverIDResponse.JSON200
+		break
+	}
+
+	defer func() {
+		_, err := client.ServerShutdown(context.Background())
+		if err != nil {
+			slog.Error("error shutting down server", "error", err)
+		}
+	}()
+
+	slog.Debug("server running", "sdk", serverID.Name, "url", serverID.Url)
+
+	return reports.Report{
+		TestServerID: serverID,
+		Results:      tests.RunTests("http://localhost:8080"),
+	}, nil
+}
+
+func init() {
+	root.AddCommand(testOneCmd)
+}
diff --git a/go.mod b/go.mod
index 96807f2..643f1c5 100644
--- a/go.mod
+++ b/go.mod
@@ -4,10 +4,14 @@ go 1.20
 
 require (
 	github.com/mr-tron/base58 v1.2.0
+	github.com/spf13/cobra v1.7.0
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
+	gopkg.in/square/go-jose.v2 v2.6.0
 )
 
 require (
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.8.4 // indirect
 	golang.org/x/crypto v0.14.0 // indirect
-	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 )
diff --git a/go.sum b/go.sum
index 7455b51..79c875b 100644
--- a/go.sum
+++ b/go.sum
@@ -1,8 +1,24 @@
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
 github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
+github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
 golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI=
 gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/reports/report-template.html b/reports/report-template.html
new file mode 100644
index 0000000..d608981
--- /dev/null
+++ b/reports/report-template.html
@@ -0,0 +1,45 @@
+<!DOCTYPE html>
+<html>
+  <head>
+        <title>web5 spec compliance report</title>
+        <meta http-equiv='Content-Type' content='text/html;charset=utf-8'/>
+        <style type="text/css">
+            * {
+                font-family: sans-serif;
+            }
+            table {
+                border-collapse: collapse;
+            }
+            th {
+                border-bottom: solid black 1px;
+            }
+        </style>
+    </head>
+    <body>
+        <h1>web5 spec compliance report</h1>
+
+        {{ range $category, $tests := .Tests }}
+        <h2>{{ $category }}</h2>
+        <table>
+            <tr><th>test</th>{{ range $.Reports }}<th><a href="{{ .TestServerID.Url }}">{{ .TestServerID.Name }}</a></th>{{ end }}</tr>
+            {{ range $_, $test := $tests }}
+            <tr>
+                <td>{{ . }}</td>
+                {{ range $.Reports }}
+                <td>
+                    {{ if not (index (index .Results $category) $test) }}
+                    {{ index (index .Results $category) $test | getEmoji }}
+                    {{ else }}
+                    <details>
+                    <summary>{{ index (index .Results $category) $test | getEmoji }}</summary>
+                    <ul>
+                        {{ range index (index .Results $category) $test }}<li>{{ . }}</li>{{ end }}
+                    </ul>
+                    {{ end }}
+                </td>{{ end }}
+            </tr>
+            {{ end }}
+        </table>
+        {{ end }}
+    </body>
+</html>
\ No newline at end of file
diff --git a/cmd/web5-spec-test/report-template.md b/reports/report-template.md
similarity index 100%
rename from cmd/web5-spec-test/report-template.md
rename to reports/report-template.md
diff --git a/cmd/web5-spec-test/report-template.txt b/reports/report-template.txt
similarity index 100%
rename from cmd/web5-spec-test/report-template.txt
rename to reports/report-template.txt
diff --git a/cmd/web5-spec-test/reports.go b/reports/reports.go
similarity index 50%
rename from cmd/web5-spec-test/reports.go
rename to reports/reports.go
index 42dcd60..f035f01 100644
--- a/cmd/web5-spec-test/reports.go
+++ b/reports/reports.go
@@ -1,18 +1,21 @@
-package main
+package reports
 
 import (
 	"bytes"
 	"embed"
+	"fmt"
+	"html/template"
 	"os"
 	"strings"
-	"text/template"
+
+	"golang.org/x/exp/slog"
 
 	"github.com/TBD54566975/web5-spec/openapi"
 	"github.com/TBD54566975/web5-spec/tests"
 )
 
-//go:embed report-template.*
-var reportTemplateFS embed.FS
+//go:embed *
+var templatesFS embed.FS
 
 var templates = template.New("")
 
@@ -21,7 +24,10 @@ func init() {
 		"sanatizeHTML": sanatizeHTML,
 		"getEmoji":     getEmoji,
 	})
-	templates.ParseFS(reportTemplateFS, "report-template.*")
+	_, err := templates.ParseFS(templatesFS, "report-template.*")
+	if err != nil {
+		panic(err)
+	}
 }
 
 type Report struct {
@@ -56,14 +62,14 @@ func (r Report) Text() (string, error) {
 	return buffer.String(), nil
 }
 
-func (r Report) WriteMarkdown(filename string) error {
+func WriteMarkdown(report Report, filename string) error {
 	f, err := os.Create(filename)
 	if err != nil {
 		return err
 	}
 	defer f.Close()
 
-	err = templates.ExecuteTemplate(f, "report-template.md", r)
+	err = templates.ExecuteTemplate(f, "report-template.md", report)
 	if err != nil {
 		return err
 	}
@@ -90,3 +96,46 @@ func getEmoji(errs []error) string {
 
 	return "❌"
 }
+
+type htmlTemplateInput struct {
+	Reports []Report
+	Tests   map[string][]string
+}
+
+func WriteHTML(reports []Report, filename string) error {
+	slog.Info("writing html report")
+
+	testmap := map[string]map[string]bool{}
+	for _, report := range reports {
+		for category, tests := range report.Results {
+			if _, ok := tests[category]; !ok {
+				testmap[category] = map[string]bool{}
+			}
+
+			for test := range tests {
+				testmap[category][test] = true
+			}
+		}
+	}
+
+	templateInput := htmlTemplateInput{Reports: reports, Tests: map[string][]string{}}
+
+	for category, tests := range testmap {
+		templateInput.Tests[category] = []string{}
+		for test := range tests {
+			templateInput.Tests[category] = append(templateInput.Tests[category], test)
+		}
+	}
+
+	f, err := os.Create(filename)
+	if err != nil {
+		return fmt.Errorf("error opening %s: %v", filename, err)
+	}
+	defer f.Close()
+
+	if err := templates.ExecuteTemplate(f, "report-template.html", templateInput); err != nil {
+		return err
+	}
+
+	return nil
+}