diff --git a/common/httpx/title.go b/common/httpx/title.go index 1b59b1e7..c2766af2 100644 --- a/common/httpx/title.go +++ b/common/httpx/title.go @@ -9,12 +9,22 @@ import ( stringsutil "github.com/projectdiscovery/utils/strings" "golang.org/x/net/html" + "slices" ) var ( - cutset = "\n\t\v\f\r" - reTitle = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`) - reContentType = regexp.MustCompile(`(?im)\s*charset="(.*?)"|charset=(.*?)"\s*`) + cutset = "\n\t\v\f\r" + reTitle = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`) + reContentType = regexp.MustCompile(`(?im)\s*charset="(.*?)"|charset=(.*?)"\s*`) + supportedTitleMimeTypes = []string{ + "text/html", + "application/xhtml+xml", + "application/xml", + "application/rss+xml", + "application/atom+xml", + "application/xhtml+xml", + "application/vnd.wap.xhtml+xml", + } ) // ExtractTitle from a response @@ -40,6 +50,10 @@ func ExtractTitle(r *Response) (title string) { return title } +func CanHaveTitleTag(mimeType string) bool { + return slices.Contains(supportedTitleMimeTypes, mimeType) +} + func getTitleWithDom(r *Response) (*html.Node, error) { var title *html.Node var crawler func(*html.Node) diff --git a/runner/runner.go b/runner/runner.go index 1f4f6640..bd6faf17 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1590,8 +1590,12 @@ retry: builder.WriteRune(']') } - title := httpx.ExtractTitle(resp) - if scanopts.OutputTitle { + var title string + if httpx.CanHaveTitleTag(resp.GetHeaderPart("Content-Type", ";")) { + title = httpx.ExtractTitle(resp) + } + + if scanopts.OutputTitle && title != "" { builder.WriteString(" [") if !scanopts.OutputWithNoColor { builder.WriteString(aurora.Cyan(title).String()) diff --git a/runner/runner_test.go b/runner/runner_test.go index f96d3f9b..08698b20 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -89,6 +89,10 @@ func TestRunner_cidr_targets(t *testing.T) { } func TestRunner_asn_targets(t *testing.T) { + if os.Getenv("PDCP_API_KEY") == "" { + return + } + options := &Options{} r, err := New(options) require.Nil(t, err, "could not create httpx runner") @@ -131,10 +135,12 @@ func TestRunner_countTargetFromRawTarget(t *testing.T) { got = r.countTargetFromRawTarget(input) require.Equal(t, expected, got, "got wrong output") - input = "AS14421" - expected = 256 - got = r.countTargetFromRawTarget(input) - require.Equal(t, expected, got, "got wrong output") + if os.Getenv("PDCP_API_KEY") != "" { + input = "AS14421" + expected = 256 + got = r.countTargetFromRawTarget(input) + require.Equal(t, expected, got, "got wrong output") + } input = "173.0.84.0/24" expected = 256