diff --git a/.github/workflows/docs-404-checker.yml b/.github/workflows/docs-linter.yml similarity index 94% rename from .github/workflows/docs-404-checker.yml rename to .github/workflows/docs-linter.yml index fc32fdc435a..e56c1a663ea 100644 --- a/.github/workflows/docs-404-checker.yml +++ b/.github/workflows/docs-linter.yml @@ -1,4 +1,4 @@ -name: "docs / 404 checker" +name: "docs / lint" on: push: diff --git a/docs/gno-infrastructure/validators/connect-to-existing-chain.md b/docs/gno-infrastructure/validators/connect-to-existing-chain.md index 5393b05d652..f1acf06049f 100644 --- a/docs/gno-infrastructure/validators/connect-to-existing-chain.md +++ b/docs/gno-infrastructure/validators/connect-to-existing-chain.md @@ -19,7 +19,7 @@ In this tutorial, you will learn how to start a local Gno node and connect to an ## 1. Initialize the node directory To initialize a new Gno.land node working directory (configuration and secrets), make sure to -follow [Step 1](./validators-setting-up-a-new-chain#1-generate-the-node-directory-secrets--config) from the +follow [Step 1](./setting-up-a-new-chain.md#1-generate-the-node-directory-secrets--config) from the chain setup tutorial. ## 2. Obtain the `genesis.json` of the remote chain diff --git a/docs/how-to-guides/connecting-from-go.md b/docs/how-to-guides/connecting-from-go.md index 2d8ebbc7f19..d78f709201e 100644 --- a/docs/how-to-guides/connecting-from-go.md +++ b/docs/how-to-guides/connecting-from-go.md @@ -50,7 +50,7 @@ The `gnoclient` package exposes a `Client` struct containing a `Signer` and `RPCClient` connector. `Client` exposes all available functionality for talking to a Gno.land chain. -```go +```go type Client struct { Signer Signer // Signer for transaction authentication RPCClient rpcclient.Client // gnolang/gno/tm2/pkg/bft/rpc/client diff --git a/docs/reference/stdlibs/std/testing.md b/docs/reference/stdlibs/std/testing.md index 7a383478ef5..e3e87ea7262 100644 --- a/docs/reference/stdlibs/std/testing.md +++ b/docs/reference/stdlibs/std/testing.md @@ -11,8 +11,8 @@ func TestSetOrigPkgAddr(addr Address) func TestSetOrigSend(sent, spent Coins) func TestIssueCoins(addr Address, coins Coins) func TestSetRealm(realm Realm) -func NewUserRealm(address Address) -func NewCodeRealm(pkgPath string) +func NewUserRealm(address Address) Realm +func NewCodeRealm(pkgPath string) Realm ``` --- @@ -132,7 +132,7 @@ userRealm := std.NewUserRealm(addr) ## NewCodeRealm ```go -func NewCodeRealm(pkgPath string) +func NewCodeRealm(pkgPath string) Realm ``` Creates a new code realm for testing purposes. diff --git a/misc/docs-linter/errors.go b/misc/docs-linter/errors.go index 8702f6f7de9..c116df3fe16 100644 --- a/misc/docs-linter/errors.go +++ b/misc/docs-linter/errors.go @@ -3,9 +3,10 @@ package main import "errors" var ( - errEmptyPath = errors.New("you need to pass in a path to scan") - err404Link = errors.New("link returned a 404") - errFound404Links = errors.New("found links resulting in a 404 response status") - errFoundUnescapedJSXTags = errors.New("found unescaped JSX tags") - errFoundLintItems = errors.New("found items that need linting") + errEmptyPath = errors.New("you need to pass in a path to scan") + err404Link = errors.New("link returned a 404") + errFound404Links = errors.New("found links resulting in a 404 response status") + errFoundUnescapedJSXTags = errors.New("found unescaped JSX tags") + errFoundUnreachableLocalLinks = errors.New("found local links that stat fails on") + errFoundLintItems = errors.New("found items that need linting") ) diff --git a/misc/docs-linter/jsx.go b/misc/docs-linter/jsx.go index 50b16d842c0..d0307680a0c 100644 --- a/misc/docs-linter/jsx.go +++ b/misc/docs-linter/jsx.go @@ -1,21 +1,25 @@ package main import ( - "context" + "bytes" "fmt" "regexp" "strings" ) +var ( + reCodeBlocks = regexp.MustCompile("(?s)```.*?```") + reInlineCode = regexp.MustCompile("`[^`]*`") +) + +// extractJSX extracts JSX tags from given file content func extractJSX(fileContent []byte) []string { text := string(fileContent) // Remove code blocks - reCodeBlocks := regexp.MustCompile("(?s)```.*?```") contentNoCodeBlocks := reCodeBlocks.ReplaceAllString(text, "") // Remove inline code - reInlineCode := regexp.MustCompile("`[^`]*`") contentNoInlineCode := reInlineCode.ReplaceAllString(contentNoCodeBlocks, "") // Extract JSX/HTML elements @@ -34,23 +38,25 @@ func extractJSX(fileContent []byte) []string { return filteredMatches } -func lintJSX(fileUrlMap map[string][]string, ctx context.Context) error { - found := false - for filePath, tags := range fileUrlMap { - filePath := filePath +func lintJSX(filepathToJSX map[string][]string) (string, error) { + var ( + found bool + output bytes.Buffer + ) + for filePath, tags := range filepathToJSX { for _, tag := range tags { if !found { - fmt.Println("Tags that need checking:") + output.WriteString("Tags that need checking:\n") found = true } - fmt.Printf(">>> %s (found in file: %s)\n", tag, filePath) + output.WriteString(fmt.Sprintf(">>> %s (found in file: %s)\n", tag, filePath)) } } if found { - return errFoundUnescapedJSXTags + return output.String(), errFoundUnescapedJSXTags } - return nil + return "", nil } diff --git a/misc/docs-linter/links.go b/misc/docs-linter/links.go index 41b90207aa5..744917d8dfb 100644 --- a/misc/docs-linter/links.go +++ b/misc/docs-linter/links.go @@ -3,105 +3,91 @@ package main import ( "bufio" "bytes" - "context" "fmt" - "golang.org/x/sync/errgroup" - "io" - "mvdan.cc/xurls/v2" - "net/http" + "os" + "path/filepath" + "regexp" "strings" - "sync" ) -// extractUrls extracts URLs from a file and maps them to the file -func extractUrls(fileContent []byte) []string { +// Valid start to an embedmd link +const embedmd = `[embedmd]:# ` + +// Regular expression to match markdown links +var regex = regexp.MustCompile(`]\((\.\.?/.+?)\)`) + +// extractLocalLinks extracts links to local files from the given file content +func extractLocalLinks(fileContent []byte) []string { scanner := bufio.NewScanner(bytes.NewReader(fileContent)) - urls := make([]string, 0) + links := make([]string, 0) // Scan file line by line for scanner.Scan() { line := scanner.Text() - // Extract links - rxStrict := xurls.Strict() - url := rxStrict.FindString(line) + // Check for embedmd links + if embedmdPos := strings.Index(line, embedmd); embedmdPos != -1 { + link := line[embedmdPos+len(embedmd)+1:] + + // Find closing parentheses + if closePar := strings.LastIndex(link, ")"); closePar != -1 { + link = link[:closePar] + } + + // Remove space + if pos := strings.Index(link, " "); pos != -1 { + link = link[:pos] + } - // Check for empty links and skip them - if url == " " || len(url) == 0 { + // Add link to be checked + links = append(links, link) continue } - // Look for http & https only - if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") { - // Ignore localhost - if !strings.Contains(url, "localhost") && !strings.Contains(url, "127.0.0.1") { - urls = append(urls, url) + // Find all matches + matches := regex.FindAllString(line, -1) + + // Extract and print the local file links + for _, match := range matches { + // Remove ]( from the beginning and ) from end of link + match = match[2 : len(match)-1] + + // Remove markdown headers in links + if pos := strings.Index(match, "#"); pos != -1 { + match = match[:pos] } + + links = append(links, match) } } - return urls + return links } -func lintLinks(fileUrlMap map[string][]string, ctx context.Context) error { - // Filter links by prefix & ignore localhost - // Setup parallel checking for links - g, _ := errgroup.WithContext(ctx) - +func lintLocalLinks(filepathToLinks map[string][]string, docsPath string) (string, error) { var ( - lock sync.Mutex - notFoundUrls []string + found bool + output bytes.Buffer ) - for filePath, urls := range fileUrlMap { - filePath := filePath - for _, url := range urls { - url := url - g.Go(func() error { - if err := checkUrl(url); err != nil { - lock.Lock() - notFoundUrls = append(notFoundUrls, fmt.Sprintf(">>> %s (found in file: %s)", url, filePath)) - lock.Unlock() - } - - return nil - }) - } - } + for filePath, links := range filepathToLinks { + for _, link := range links { + path := filepath.Join(docsPath, filepath.Dir(filePath), link) - if err := g.Wait(); err != nil { - return err - } + if _, err := os.Stat(path); err != nil { + if !found { + output.WriteString("Could not find files with the following paths:\n") + found = true + } - // Print out the URLs that returned a 404 along with the file names - if len(notFoundUrls) > 0 { - fmt.Println("Links that need checking:") - for _, result := range notFoundUrls { - fmt.Println(result) + output.WriteString(fmt.Sprintf(">>> %s (found in file: %s)\n", link, filePath)) + } } - - return errFound404Links } - return nil -} - -// checkUrl checks if a URL is a 404 -func checkUrl(url string) error { - // Attempt to retrieve the HTTP header - resp, err := http.Get(url) - if err != nil || resp.StatusCode == http.StatusNotFound { - return err404Link - } - - // Ensure the response body is closed properly - cleanup := func(Body io.ReadCloser) error { - if err := Body.Close(); err != nil { - return fmt.Errorf("could not close response properly: %w", err) - } - - return nil + if found { + return output.String(), errFoundUnreachableLocalLinks } - return cleanup(resp.Body) + return "", nil } diff --git a/misc/docs-linter/main.go b/misc/docs-linter/main.go index 035170ca5ae..97d80316108 100644 --- a/misc/docs-linter/main.go +++ b/misc/docs-linter/main.go @@ -1,14 +1,17 @@ package main import ( + "bytes" "context" "flag" "fmt" - "github.com/gnolang/gno/tm2/pkg/commands" - "golang.org/x/sync/errgroup" "os" "path/filepath" "strings" + "sync" + + "github.com/gnolang/gno/tm2/pkg/commands" + "golang.org/x/sync/errgroup" ) type cfg struct { @@ -23,11 +26,16 @@ func main() { Name: "docs-linter", ShortUsage: "docs-linter [flags]", ShortHelp: `Lints the .md files in the given folder & subfolders. -Checks for 404 links, as well as improperly escaped JSX tags.`, +Checks for 404 links (local and remote), as well as improperly escaped JSX tags.`, }, cfg, func(ctx context.Context, args []string) error { - return execLint(cfg, ctx) + res, err := execLint(cfg, ctx) + if len(res) != 0 { + fmt.Println(res) + } + + return err }) cmd.Execute(context.Background(), os.Args[1:]) @@ -42,59 +50,93 @@ func (c *cfg) RegisterFlags(fs *flag.FlagSet) { ) } -func execLint(cfg *cfg, ctx context.Context) error { +func execLint(cfg *cfg, ctx context.Context) (string, error) { if cfg.docsPath == "" { - return errEmptyPath + return "", errEmptyPath } absPath, err := filepath.Abs(cfg.docsPath) if err != nil { - return fmt.Errorf("error getting absolute path for docs folder: %w", err) + return "", fmt.Errorf("error getting absolute path for docs folder: %w", err) } - fmt.Printf("Linting %s...\n", absPath) + // Main buffer to write to the end user after linting + var output bytes.Buffer + output.WriteString(fmt.Sprintf("Linting %s...\n", absPath)) // Find docs files to lint mdFiles, err := findFilePaths(cfg.docsPath) if err != nil { - return fmt.Errorf("error finding .md files: %w", err) + return "", fmt.Errorf("error finding .md files: %w", err) } // Make storage maps for tokens to analyze - fileUrlMap := make(map[string][]string) // file path > [urls] - fileJSXMap := make(map[string][]string) // file path > [JSX items] + filepathToURLs := make(map[string][]string) // file path > [urls] + filepathToJSX := make(map[string][]string) // file path > [JSX items] + filepathToLocalLink := make(map[string][]string) // file path > [local links] // Extract tokens from files for _, filePath := range mdFiles { // Read file content once and pass it to linters fileContents, err := os.ReadFile(filePath) if err != nil { - return err + return "", err } - fileJSXMap[filePath] = extractJSX(fileContents) + // Execute JSX extractor + filepathToJSX[filePath] = extractJSX(fileContents) // Execute URL extractor - fileUrlMap[filePath] = extractUrls(fileContents) + filepathToURLs[filePath] = extractUrls(fileContents) + + // Execute local link extractor + filepathToLocalLink[filePath] = extractLocalLinks(fileContents) } // Run linters in parallel g, _ := errgroup.WithContext(ctx) + var writeLock sync.Mutex + + g.Go(func() error { + res, err := lintJSX(filepathToJSX) + if err != nil { + writeLock.Lock() + output.WriteString(res) + writeLock.Unlock() + } + + return err + }) + g.Go(func() error { - return lintJSX(fileJSXMap, ctx) + res, err := lintURLs(filepathToURLs, ctx) + if err != nil { + writeLock.Lock() + output.WriteString(res) + writeLock.Unlock() + } + + return err }) g.Go(func() error { - return lintLinks(fileUrlMap, ctx) + res, err := lintLocalLinks(filepathToLocalLink, cfg.docsPath) + if err != nil { + writeLock.Lock() + output.WriteString(res) + writeLock.Unlock() + } + + return err }) - if err := g.Wait(); err != nil { - return errFoundLintItems + if err = g.Wait(); err != nil { + return output.String(), errFoundLintItems } - fmt.Println("Lint complete, no issues found.") - return nil + output.WriteString("Lint complete, no issues found.") + return output.String(), nil } // findFilePaths gathers the file paths for specific file types diff --git a/misc/docs-linter/main_test.go b/misc/docs-linter/main_test.go index 27393236215..f42874ea81e 100644 --- a/misc/docs-linter/main_test.go +++ b/misc/docs-linter/main_test.go @@ -2,14 +2,16 @@ package main import ( "context" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "os" "path/filepath" "sort" "strconv" + "strings" "testing" "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestEmptyPathError(t *testing.T) { @@ -22,7 +24,8 @@ func TestEmptyPathError(t *testing.T) { ctx, cancelFn := context.WithTimeout(context.Background(), time.Second*5) defer cancelFn() - assert.ErrorIs(t, execLint(cfg, ctx), errEmptyPath) + _, err := execLint(cfg, ctx) + assert.ErrorIs(t, err, errEmptyPath) } func TestExtractLinks(t *testing.T) { @@ -121,6 +124,50 @@ Returns **Promise** } } +func TestExtractLocalLinks(t *testing.T) { + t.Parallel() + + // Create mock file content with random local links + mockFileContent := ` +Here is some text with a link to a local file: [text](../concepts/file1.md) +Here is another local link: [another](./path/to/file1.md) +Here is another local link: [another](./path/to/file2.md#header-1-2) +And a link to an external website: [example](https://example.com) +And a websocket link: [websocket](ws://example.com/socket) +Here's an embedmd link: [embedmd]:# (../assets/how-to-guides/simple-library/tapas.gno go) +Here's an embedmd link: [embedmd]:# (../assets/myfile.sol go) +Here's an embedmd link: [embedmd]:# (../assets/myfi()le.gno c) +Here's an embedmd link: [embedmd]:# (../assets/)myfi(le.gno c) +Here's another link: [embedmd]:# (../folder/myfile.gno c +` + + // Expected local links + expectedLinks := []string{ + "../concepts/file1.md", + "./path/to/file1.md", + "./path/to/file2.md", + "../assets/how-to-guides/simple-library/tapas.gno", + "../assets/myfile.sol", + "../assets/myfi()le.gno", + "../assets/)myfi(le.gno", + "../folder/myfile.gno", + } + + // Extract local links tags from the mock file content + extractedLinks := extractLocalLinks([]byte(mockFileContent)) + + if len(expectedLinks) != len(extractedLinks) { + t.Fatal("did not extract the correct amount of local links") + } + + sort.Strings(extractedLinks) + sort.Strings(expectedLinks) + + for i, tag := range expectedLinks { + require.Equal(t, tag, extractedLinks[i]) + } +} + func TestFindFilePaths(t *testing.T) { t.Parallel() @@ -170,6 +217,52 @@ func TestFindFilePaths(t *testing.T) { } } +func TestFlow(t *testing.T) { + t.Parallel() + + tempDir, err := os.MkdirTemp(".", "test") + require.NoError(t, err) + t.Cleanup(removeDir(t, tempDir)) + + contents := `This is a [broken Wikipedia link](https://www.wikipedia.org/non-existent-page). +Here's an embedmd link that links to a non-existing file: [embedmd]:# (../assets/myfile.sol go) +and here is some JSX tags +and [this is a link to a non-existent](../myfolder/myfile.md) file.` + + expectedItems := []string{ + "https://www.wikipedia.org/non-existent-page", + "../assets/myfile.sol", + "", + "../myfolder/myfile.md", + } + + filePath := filepath.Join(tempDir, "examplefile.md") + + err = os.MkdirAll(filepath.Dir(filePath), os.ModePerm) + require.NoError(t, err) + + f, err := os.Create(filePath) + require.NoError(t, err) + + _, err = f.WriteString(contents) + require.NoError(t, err) + + err = f.Close() + require.NoError(t, err) + + res, err := execLint(&cfg{ + docsPath: tempDir, + }, + context.Background(), + ) + + assert.ErrorIs(t, err, errFoundLintItems) + + for _, item := range expectedItems { + assert.True(t, strings.Contains(res, item)) + } +} + func removeDir(t *testing.T, dirPath string) func() { return func() { require.NoError(t, os.RemoveAll(dirPath)) diff --git a/misc/docs-linter/urls.go b/misc/docs-linter/urls.go new file mode 100644 index 00000000000..093e624d81e --- /dev/null +++ b/misc/docs-linter/urls.go @@ -0,0 +1,108 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "net/http" + "strings" + "sync" + + "golang.org/x/sync/errgroup" + "mvdan.cc/xurls/v2" +) + +// extractUrls extracts urls from given file content +func extractUrls(fileContent []byte) []string { + scanner := bufio.NewScanner(bytes.NewReader(fileContent)) + urls := make([]string, 0) + + // Scan file line by line + for scanner.Scan() { + line := scanner.Text() + + // Extract links + rxStrict := xurls.Strict() + url := rxStrict.FindString(line) + + // Check for empty links and skip them + if url == " " || len(url) == 0 { + continue + } + + // Look for http & https only + if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") { + // Ignore localhost + if !strings.Contains(url, "localhost") && !strings.Contains(url, "127.0.0.1") { + urls = append(urls, url) + } + } + } + + return urls +} + +func lintURLs(filepathToURLs map[string][]string, ctx context.Context) (string, error) { + // Setup parallel checking for links + g, _ := errgroup.WithContext(ctx) + + var ( + lock sync.Mutex + output bytes.Buffer + found bool + ) + + for filePath, urls := range filepathToURLs { + filePath := filePath + for _, url := range urls { + url := url + g.Go(func() error { + if err := checkUrl(url); err != nil { + lock.Lock() + if !found { + output.WriteString("Remote links that need checking:\n") + found = true + } + + output.WriteString(fmt.Sprintf(">>> %s (found in file: %s)\n", url, filePath)) + lock.Unlock() + } + + return nil + }) + } + } + + // Check for possible thread errors + if err := g.Wait(); err != nil { + return "", err + } + + if found { + return output.String(), errFound404Links + } + + return "", nil +} + +// checkUrl checks if a URL is a 404 +func checkUrl(url string) error { + // Attempt to retrieve the HTTP header + resp, err := http.Get(url) + if err != nil || resp.StatusCode == http.StatusNotFound { + return err404Link + } + + // Ensure the response body is closed properly + cleanup := func(Body io.ReadCloser) error { + if err := Body.Close(); err != nil { + return fmt.Errorf("could not close response properly: %w", err) + } + + return nil + } + + return cleanup(resp.Body) +}