don't count 404 as failing

TBD54566975 · Oct 19, 2023 · 84580bc · 84580bc
1 parent 8eb165e
commit 84580bc
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 5 deletions.
diff --git a/cmd/web5-spec-test/report-template.md b/cmd/web5-spec-test/report-template.md
@@ -8,6 +8,6 @@ SDK: [{{ .TestServerID.Name }}]({{ .TestServerID.Url }}) ({{ .TestServerID.Langu
 
 | Feature | Result |
 | ------- | ------ |{{ range $test, $result := $results }}
-| {{ $test }} | {{ if $result }}:x: <ul>{{ range $result }}<li><pre>{{ . | sanatizeHTML }}</pre></li>{{ end }}</ul>{{ else }}:heavy_check_mark:{{ end }} |{{ end }}
+| {{ $test }} | {{ $result | getEmoji}}{{ if $result }} <ul>{{ range $result }}<li><pre>{{ . | sanatizeHTML }}</pre></li>{{ end }}</ul>{{ end }} |{{ end }}
 
 {{ end }}
diff --git a/cmd/web5-spec-test/report-template.txt b/cmd/web5-spec-test/report-template.txt
@@ -3,5 +3,5 @@ web5 spec conformance report for {{ .TestServerID.Name }} ({{ .TestServerID.Url
 {{ range $groupName, $results := .Results }}
 {{ $groupName }}
 ======================{{ range $test, $result := $results }}
-{{ $test }}: {{ if $result }}❌ fail: {{ $result }}{{ else }}✅ pass{{ end }}{{ end }}
+{{ $test }}: {{ $result | getEmoji }} {{ if $result }}fail: {{ $result }}{{ else }}pass{{ end }}{{ end }}
 {{ end }}
diff --git a/cmd/web5-spec-test/reports.go b/cmd/web5-spec-test/reports.go
@@ -8,6 +8,7 @@ import (
 	"text/template"
 
 	"github.com/TBD54566975/web5-spec/openapi"
+	"github.com/TBD54566975/web5-spec/tests"
 )
 
 //go:embed report-template.*
@@ -18,6 +19,7 @@ var templates = template.New("")
 func init() {
 	templates.Funcs(template.FuncMap{
 		"sanatizeHTML": sanatizeHTML,
+		"getEmoji":     getEmoji,
 	})
 	templates.ParseFS(reportTemplateFS, "report-template.*")
 }
@@ -28,10 +30,17 @@ type Report struct {
 }
 
 func (r Report) IsPassing() bool {
-	for _, errs := range r.Results {
-		if len(errs) > 0 {
-			return false
+	for _, results := range r.Results {
+		for _, errs := range results {
+			if len(errs) == 1 && errs[0] == tests.ErrNotSupported {
+				continue
+			}
+
+			if len(errs) > 0 {
+				return false
+			}
 		}
+
 	}
 
 	return true
@@ -69,3 +78,15 @@ func sanatizeHTML(dirty error) string {
 
 	return clean
 }
+
+func getEmoji(errs []error) string {
+	if len(errs) == 0 {
+		return "✅"
+	}
+
+	if len(errs) == 1 && errs[0] == tests.ErrNotSupported {
+		return "🚧"
+	}
+
+	return "❌"
+}