Merge pull request #73 from lc/lc/add-timeout

feat(gau): add timeout flag for http client

README.md

@@ -39,6 +39,7 @@ $ gau -h
 |`--providers`| list of providers to use (wayback,commoncrawl,otx,urlscan) | gau --providers wayback|
 |`--proxy`| http proxy to use (socks5:// or http:// | gau --proxy http://proxy.example.com:8080 |
 |`--retries`| retries for HTTP client | gau --retries 10 |
+|`--timeout`| timeout (in seconds) for HTTP client | gau --timeout 60 |
 |`--subs`| include subdomains of target domain | gau example.com --subs |
 |`--threads`| number of workers to spawn | gau example.com --threads |
 |`--to`| fetch urls to date (format: YYYYMM) | gau example.com --to 202101 |

pkg/httpclient/client.go

@@ -15,12 +15,12 @@ type Header struct {
 	Value string
 }
 
-func MakeRequest(c *fasthttp.Client, url string, maxRetries int, headers ...Header) ([]byte, error) {
+func MakeRequest(c *fasthttp.Client, url string, maxRetries uint, timeout uint, headers ...Header) ([]byte, error) {
 	var (
 		req  *fasthttp.Request
 		resp *fasthttp.Response
 	)
-	retries := maxRetries
+	retries := int(maxRetries)
 	for i := retries; i >= 0; i-- {
 		req = fasthttp.AcquireRequest()
 		defer fasthttp.ReleaseRequest(req)
@@ -35,7 +35,7 @@ func MakeRequest(c *fasthttp.Client, url string, maxRetries int, headers ...Head
 		resp = fasthttp.AcquireResponse()
 		defer fasthttp.ReleaseResponse(resp)
 
-		if err := c.DoTimeout(req, resp, time.Second*45); err != nil {
+		if err := c.DoTimeout(req, resp, time.Second*time.Duration(timeout)); err != nil {
 			fasthttp.ReleaseRequest(req)
 			if retries == 0 {
 				return nil, err

pkg/providers/commoncrawl/commoncrawl.go

@@ -31,7 +31,7 @@ type Client struct {
 func New(c *providers.Config, filters providers.Filters) (*Client, error) {
 	client := &Client{config: c, filters: filters}
 	// Fetch the list of available CommonCrawl Api URLs.
-	resp, err := httpclient.MakeRequest(c.Client, "http://index.commoncrawl.org/collinfo.json", int(c.MaxRetries))
+	resp, err := httpclient.MakeRequest(c.Client, "http://index.commoncrawl.org/collinfo.json", c.MaxRetries, c.Timeout)
 	if err != nil {
 		return nil, err
 	}
@@ -78,7 +78,7 @@ paginate:
 				logrus.WithFields(logrus.Fields{"provider": Name, "page": page}).Infof("fetching %s", domain)
 			}
 			apiURL := c.formatURL(domain, page)
-			resp, err := httpclient.MakeRequest(c.config.Client, apiURL, int(c.config.MaxRetries))
+			resp, err := httpclient.MakeRequest(c.config.Client, apiURL, c.config.MaxRetries, c.config.Timeout)
 			if err != nil {
 				return fmt.Errorf("failed to fetch commoncrawl(%d): %s", page, err)
 			}
@@ -114,7 +114,7 @@ func (c *Client) formatURL(domain string, page uint) string {
 func (c *Client) getPagination(domain string) (paginationResult, error) {
 	url := fmt.Sprintf("%s&showNumPages=true", c.formatURL(domain, 0))
 
-	resp, err := httpclient.MakeRequest(c.config.Client, url, int(c.config.MaxRetries))
+	resp, err := httpclient.MakeRequest(c.config.Client, url, c.config.MaxRetries, c.config.Timeout)
 	if err != nil {
 		return paginationResult{}, err
 	}

pkg/providers/otx/otx.go

@@ -56,7 +56,7 @@ paginate:
 				logrus.WithFields(logrus.Fields{"provider": Name, "page": page - 1}).Infof("fetching %s", domain)
 			}
 			apiURL := c.formatURL(domain, page)
-			resp, err := httpclient.MakeRequest(c.config.Client, apiURL, int(c.config.MaxRetries))
+			resp, err := httpclient.MakeRequest(c.config.Client, apiURL, c.config.MaxRetries, c.config.Timeout)
 			if err != nil {
 				return fmt.Errorf("failed to fetch alienvault(%d): %s", page, err)
 			}

pkg/providers/providers.go

@@ -5,7 +5,7 @@ import (
 	"github.com/valyala/fasthttp"
 )
 
-const Version = `2.0.9`
+const Version = `2.1.0`
 
 // Provider is a generic interface for all archive fetchers
 type Provider interface {
@@ -20,6 +20,7 @@ type URLScan struct {
 
 type Config struct {
 	Threads           uint
+	Timeout           uint
 	Verbose           bool
 	MaxRetries        uint
 	IncludeSubdomains bool

pkg/providers/urlscan/urlscan.go

@@ -51,7 +51,7 @@ paginate:
 				logrus.WithFields(logrus.Fields{"provider": Name, "page": page}).Infof("fetching %s", domain)
 			}
 			apiURL := c.formatURL(domain, searchAfter)
-			resp, err := httpclient.MakeRequest(c.config.Client, apiURL, int(c.config.MaxRetries), header)
+			resp, err := httpclient.MakeRequest(c.config.Client, apiURL, c.config.MaxRetries, c.config.Timeout, header)
 			if err != nil {
 				return fmt.Errorf("failed to fetch urlscan: %s", err)
 			}

pkg/providers/wayback/wayback.go

@@ -53,7 +53,7 @@ func (c *Client) Fetch(ctx context.Context, domain string, results chan string)
 			}
 			apiURL := c.formatURL(domain, page)
 			// make HTTP request
-			resp, err := httpclient.MakeRequest(c.config.Client, apiURL, int(c.config.MaxRetries))
+			resp, err := httpclient.MakeRequest(c.config.Client, apiURL, c.config.MaxRetries, c.config.Timeout)
 			if err != nil {
 				return fmt.Errorf("failed to fetch wayback results page %d: %s", page, err)
 			}
@@ -96,7 +96,8 @@ func (c *Client) formatURL(domain string, page uint) string {
 // getPagination returns the number of pages for Wayback
 func (c *Client) getPagination(domain string) (uint, error) {
 	url := fmt.Sprintf("%s&showNumPages=true", c.formatURL(domain, 0))
-	resp, err := httpclient.MakeRequest(c.config.Client, url, int(c.config.MaxRetries))
+	resp, err := httpclient.MakeRequest(c.config.Client, url, c.config.MaxRetries, c.config.Timeout)
+
 	if err != nil {
 		return 0, err
 	}

runner/flags/flags.go

@@ -26,6 +26,7 @@ type Config struct {
 	Filters           providers.Filters `mapstructure:"filters"`
 	Proxy             string            `mapstructure:"proxy"`
 	Threads           uint              `mapstructure:"threads"`
+	Timeout           uint              `mapstructure:"timeout"`
 	Verbose           bool              `mapstructure:"verbose"`
 	MaxRetries        uint              `mapstructure:"retries"`
 	IncludeSubdomains bool              `mapstructure:"subdomains"`
@@ -58,6 +59,7 @@ func (c *Config) ProviderConfig() (*providers.Config, error) {
 
 	pc := &providers.Config{
 		Threads:           c.Threads,
+		Timeout:           c.Timeout,
 		Verbose:           c.Verbose,
 		MaxRetries:        c.MaxRetries,
 		IncludeSubdomains: c.IncludeSubdomains,
@@ -94,7 +96,8 @@ func New() *Options {
 	v := viper.New()
 
 	pflag.String("o", "", "filename to write results to")
-	pflag.Uint("threads", 1, "number of workers to spawn, default: 1")
+	pflag.Uint("threads", 1, "number of workers to spawn")
+	pflag.Uint("timeout", 45, "timeout (in seconds) for HTTP client")
 	pflag.Uint("retries", 0, "retries for HTTP client")
 	pflag.String("proxy", "", "http proxy to use")
 	pflag.StringSlice("blacklist", []string{}, "list of extensions to skip")