Merge pull request #54 from lc/lc/fix-commoncrawl

fix(gau): change error handling for commoncrawl, bump retry count

.gau.toml

@@ -1,6 +1,6 @@
 threads = 2
 verbose = false
-retries = 5
+retries = 15
 subdomains = false
 providers = ["gau","commoncrawl","otx","urlscan"]
 blacklist = ["ttf","woff","svg","png","jpg"]

cmd/gau/main.go

@@ -33,7 +33,7 @@ func main() {
 	gau := &runner.Runner{}
 
 	if err = gau.Init(config, pMap); err != nil {
-		log.Fatal(err)
+		log.Warn(err)
 	}
 
 	results := make(chan string)

pkg/providers/providers.go

@@ -5,7 +5,7 @@ import (
 	"github.com/valyala/fasthttp"
 )
 
-const Version = `2.0.5`
+const Version = `2.0.6`
 
 // Provider is a generic interface for all archive fetchers
 type Provider interface {

runner/runner.go

@@ -40,7 +40,7 @@ func (r *Runner) Init(c *providers.Config, providerMap ProvidersMap) error {
 		case "commoncrawl":
 			cc, err := commoncrawl.New(c, filters)
 			if err != nil {
-				return fmt.Errorf("error instantiating client: %v\n", err)
+				return fmt.Errorf("error instantiating commoncrawl: %v\n", err)
 			}
 			r.providers = append(r.providers, cc)
 		}
@@ -51,7 +51,7 @@ func (r *Runner) Init(c *providers.Config, providerMap ProvidersMap) error {
 
 // Starts starts the worker
 func (r *Runner) Start(domains chan string, results chan string) {
-	for i := uint(0); i < r.config.Threads; i++ { // TODO: get thread count dynamically
+	for i := uint(0); i < r.config.Threads; i++ {
 		r.wg.Add(1)
 		go func() {
 			defer r.wg.Done()