diff --git a/cmd/httpx/httpx.go b/cmd/httpx/httpx.go index 3c890e29..4f4aa9a7 100644 --- a/cmd/httpx/httpx.go +++ b/cmd/httpx/httpx.go @@ -138,5 +138,8 @@ func setupOptionalAssetUpload(opts *runner.Options) *pdcp.UploadWriter { // silently ignore writer.SetAssetGroupName(opts.AssetName) } + if opts.TeamID != "" { + writer.SetTeamID(opts.TeamID) + } return writer } diff --git a/go.mod b/go.mod index 7dad8b49..6f54de67 100644 --- a/go.mod +++ b/go.mod @@ -18,27 +18,26 @@ require ( github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 github.com/microcosm-cc/bluemonday v1.0.26 github.com/miekg/dns v1.1.59 // indirect - github.com/mitchellh/mapstructure v1.5.0 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/asnmap v1.1.1 github.com/projectdiscovery/cdncheck v1.1.0 github.com/projectdiscovery/clistats v0.1.0 - github.com/projectdiscovery/dsl v0.1.10 - github.com/projectdiscovery/fastdialer v0.2.3 + github.com/projectdiscovery/dsl v0.2.1 + github.com/projectdiscovery/fastdialer v0.2.9 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/goconfig v0.0.1 - github.com/projectdiscovery/goflags v0.1.62 - github.com/projectdiscovery/gologger v1.1.19 - github.com/projectdiscovery/hmap v0.0.54 + github.com/projectdiscovery/goflags v0.1.64 + github.com/projectdiscovery/gologger v1.1.23 + github.com/projectdiscovery/hmap v0.0.58 github.com/projectdiscovery/mapcidr v1.1.34 github.com/projectdiscovery/networkpolicy v0.0.9 - github.com/projectdiscovery/ratelimit v0.0.49 - github.com/projectdiscovery/rawhttp v0.1.61 - github.com/projectdiscovery/retryablehttp-go v1.0.72 - github.com/projectdiscovery/tlsx v1.1.6 - github.com/projectdiscovery/useragent v0.0.65 - github.com/projectdiscovery/utils v0.2.4 - github.com/projectdiscovery/wappalyzergo v0.1.12 + github.com/projectdiscovery/ratelimit v0.0.54 + github.com/projectdiscovery/rawhttp v0.1.66 + github.com/projectdiscovery/retryablehttp-go v1.0.78 + github.com/projectdiscovery/tlsx v1.1.7 + github.com/projectdiscovery/useragent v0.0.69 + github.com/projectdiscovery/utils v0.2.10 + github.com/projectdiscovery/wappalyzergo v0.1.16 github.com/rs/xid v1.5.0 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.9.0 @@ -51,7 +50,10 @@ require ( golang.org/x/text v0.16.0 ) -require github.com/weppos/publicsuffix-go v0.30.2 +require ( + github.com/go-viper/mapstructure/v2 v2.1.0 + github.com/weppos/publicsuffix-go v0.30.2 +) require ( aead.dev/minisign v0.2.0 // indirect @@ -110,10 +112,10 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/freeport v0.0.5 // indirect + github.com/projectdiscovery/freeport v0.0.7 // indirect github.com/projectdiscovery/gostruct v0.0.2 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/retryabledns v1.0.70 // indirect + github.com/projectdiscovery/retryabledns v1.0.75 // indirect github.com/refraction-networking/utls v1.6.7 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect diff --git a/go.sum b/go.sum index 1d9a8ac0..a6287fde 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,8 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-rod/rod v0.114.0 h1:P+zLOqsj+vKf4C86SfjP6ymyPl9VXoYKm+ceCeQms6Y= github.com/go-rod/rod v0.114.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= +github.com/go-viper/mapstructure/v2 v2.1.0 h1:gHnMa2Y/pIxElCH2GlZZ1lZSsn6XMtufpGyP1XxdC/w= +github.com/go-viper/mapstructure/v2 v2.1.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -176,8 +178,6 @@ github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs= github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -224,48 +224,48 @@ github.com/projectdiscovery/cdncheck v1.1.0 h1:qDITidmJsejzpk3rMkauCh6sjI2GH9hW/ github.com/projectdiscovery/cdncheck v1.1.0/go.mod h1:sZ8U4MjHSsyaTVjBbYWHT1cwUVvUYwDX1W+WvWRicIc= github.com/projectdiscovery/clistats v0.1.0 h1:b+LF1w0xhNd7cneKWMXb+/yUmF1n5szawxP4XGpmbxs= github.com/projectdiscovery/clistats v0.1.0/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= -github.com/projectdiscovery/dsl v0.1.10 h1:FBD5Muiwj3OdkDAYvmltQ0rcVMpLZK5IqNZi9Pd/gb0= -github.com/projectdiscovery/dsl v0.1.10/go.mod h1:Kg9snSkjTPZ2qiIDr4jjFI46FgJzvqz48qcHWUWY7lw= -github.com/projectdiscovery/fastdialer v0.2.3 h1:K03x5XEXGyVWxS2rtSR104E9kHF0aphN7kOCzbh8zv0= -github.com/projectdiscovery/fastdialer v0.2.3/go.mod h1:a0BKvETrO1EAohUYp9gwtbbce0hKD1qGrTegyAUqyRo= +github.com/projectdiscovery/dsl v0.2.1 h1:TK3KD4jsg4YbvY7WJqnz1QyH4AOvAwezeBFOX97Evgk= +github.com/projectdiscovery/dsl v0.2.1/go.mod h1:IRQXsmi5/g1dDZ79//A9t2vrRtxm4frRSd5t8CZVSbI= +github.com/projectdiscovery/fastdialer v0.2.9 h1:vDCqxVMCyUu3oVEizEK1K8K+CCcLkVDW3X2HfiWaVFA= +github.com/projectdiscovery/fastdialer v0.2.9/go.mod h1:mYv5QaNBDDSHlZO9DI0niRMw+G5hUzwIhs8QixSElUI= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= -github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= -github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= +github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk= +github.com/projectdiscovery/freeport v0.0.7/go.mod h1:cOhWKvNBe9xM6dFJ3RrrLvJ5vXx2NQ36SecuwjenV2k= github.com/projectdiscovery/goconfig v0.0.1 h1:36m3QjohZvemqh9bkJAakaHsm9iEZ2AcQSS18+0QX/s= github.com/projectdiscovery/goconfig v0.0.1/go.mod h1:CPO25zR+mzTtyBrsygqsHse0sp/4vB/PjaHi9upXlDw= -github.com/projectdiscovery/goflags v0.1.62 h1:UmzKJQT+1UyqT1cZDmb3vZ8/IGhQ7LTsWfdqVcAGoJc= -github.com/projectdiscovery/goflags v0.1.62/go.mod h1:d1/D8GaTDoV332ABwceUcY1ffKODaYFlGP0Oriq3wfk= -github.com/projectdiscovery/gologger v1.1.19 h1:b7cU32XuDrDiwhr7hlDeE6mfj/nENBtHEohe51txJCE= -github.com/projectdiscovery/gologger v1.1.19/go.mod h1:DbeKwx9IEfcvnclImX5gBlhIKUuOZwOM5itdpYXl+54= +github.com/projectdiscovery/goflags v0.1.64 h1:FDfwdt9N97Hi8OuhbkDlKtVttpc/CRMIWQVa08VsHsI= +github.com/projectdiscovery/goflags v0.1.64/go.mod h1:3FyHIVQtnycNOc1LE3O1jj/XR5XuMdF9QfHd0ujhnX4= +github.com/projectdiscovery/gologger v1.1.23 h1:u9ksRgkWuUoaOqkk0Z1FngWriXH0Bnt73yYwFGmZo8E= +github.com/projectdiscovery/gologger v1.1.23/go.mod h1:zcFt3u4U0kmhXou7cTXWwwu+yqVPX3GSABD1hqU0Ur8= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= -github.com/projectdiscovery/hmap v0.0.54 h1:b3pdQZwCw4is3xiL2jBx7SJZcYaf/7vtozY7bjUzO/s= -github.com/projectdiscovery/hmap v0.0.54/go.mod h1:j0oakxYOWEfk29wRq5gQgrCv1JnfAfzGaMsRWwEas80= +github.com/projectdiscovery/hmap v0.0.58 h1:SoXmnmYS2egPSRgFKgUhHozu8QvPIUKAuoDpuii9jkw= +github.com/projectdiscovery/hmap v0.0.58/go.mod h1:nQTelqkgxU6vuuU+qmQloMrDBxYZMt2TTO0fV86yXN4= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= github.com/projectdiscovery/mapcidr v1.1.34 h1:udr83vQ7oz3kEOwlsU6NC6o08leJzSDQtls1wmXN/kM= github.com/projectdiscovery/mapcidr v1.1.34/go.mod h1:1+1R6OkKSAKtWDXE9RvxXtXPoajXTYX0eiEdkqlhQqQ= github.com/projectdiscovery/networkpolicy v0.0.9 h1:IrlDoYZagNNO8y+7iZeHT8k5izE+nek7TdtvEBwCxqk= github.com/projectdiscovery/networkpolicy v0.0.9/go.mod h1:XFJ2Lnv8BE/ziQCFjBHMsH1w6VmkPiQtk+NlBpdMU7M= -github.com/projectdiscovery/ratelimit v0.0.49 h1:PYatMp8g5OuoFsZOA90e48nLd2vB6a4Tw0FZ8h9zqkQ= -github.com/projectdiscovery/ratelimit v0.0.49/go.mod h1:Xi0LTMHg4HQlmCZFzRBIhRW6N+QW5RxQ8V/Qs+Vta4k= -github.com/projectdiscovery/rawhttp v0.1.61 h1:EbskCj6kkDSG31sO5zEUFTqHp9ltccG1DdcI+MCdahQ= -github.com/projectdiscovery/rawhttp v0.1.61/go.mod h1:5XmDAKph9pLVnh87zjL+vXDpfG5W8Gz5N2BAtYTkuVU= -github.com/projectdiscovery/retryabledns v1.0.70 h1:2yFMqQ4v3tgI9ORjlPH60h5QIs2EXxHVGCaxrOA1ZlI= -github.com/projectdiscovery/retryabledns v1.0.70/go.mod h1:Ld/RLVsG7d+wlNcye9xcuPTjGHLF9XO8w34GLRKsNis= -github.com/projectdiscovery/retryablehttp-go v1.0.72 h1:3m+9aRwC4KOMoQiOF3lVsDnPm/1+OR5r0UCHt5Edz5k= -github.com/projectdiscovery/retryablehttp-go v1.0.72/go.mod h1:tDPEpm0PlDOMB4yqHFPoLJZaK3uo+Auj/QIiUlzS+6Y= +github.com/projectdiscovery/ratelimit v0.0.54 h1:VwrFLPSfWle5Hg7AvwQkDd4EWDzYWTzd2mDTXTrqvmA= +github.com/projectdiscovery/ratelimit v0.0.54/go.mod h1:yGIqMaT8vKG+4mlCqkWOWAvBtvWUlQeanZae31E09cY= +github.com/projectdiscovery/rawhttp v0.1.66 h1:dG2rLnd1uOpj15LOmNC7Rs0CUL+ysUj0hPn9o3OdOh8= +github.com/projectdiscovery/rawhttp v0.1.66/go.mod h1:coHIiWt2PcEjTg5V5mbmvK+j7BH2LUQRR+ps20ElVQY= +github.com/projectdiscovery/retryabledns v1.0.75 h1:CIwX9PsA7GgrbcKAXYy3aBiFoQ/Futd3iA+nNaRn/6s= +github.com/projectdiscovery/retryabledns v1.0.75/go.mod h1:MnnWpXz13j2JyjwBY6Hem/C9fLZ+2UmPiB5fDu23qL4= +github.com/projectdiscovery/retryablehttp-go v1.0.78 h1:If7/XjCWk893YrnTMaW69TNMsfE1Er3i1SWOkWbEk4E= +github.com/projectdiscovery/retryablehttp-go v1.0.78/go.mod h1:NqzTdnGihSRkF9c/aXo+3qTJXEeNwnOuza6GrlPo9qw= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/tlsx v1.1.6 h1:iw2zwKbd2+kRQ8J1G4dLmS0CLyemd/tKz1UzcNsC77A= -github.com/projectdiscovery/tlsx v1.1.6/go.mod h1:s7SRRFdrwIZBK/RXXZi4CR/CubqFSvp8h5Bk1srEZIo= -github.com/projectdiscovery/useragent v0.0.65 h1:x78ZwWdqpzokOHxLITUXvq+ljkTKc19z3ILGtoV1N70= -github.com/projectdiscovery/useragent v0.0.65/go.mod h1:deOP8YLJU6SCzM8k+K8PjkcOF4Ux0spqyO4ODZGIT4A= -github.com/projectdiscovery/utils v0.2.4 h1:CHnlt2la4jr8TeL7ZK7UhQItHY7DDXqIuLnnxyAJLDY= -github.com/projectdiscovery/utils v0.2.4/go.mod h1:2Vx7geSrBfCPqknZywqbChQm8SE30mcyrlB5YsxEnhA= -github.com/projectdiscovery/wappalyzergo v0.1.12 h1:usZmlyiNDRvdYstZpFQH11cx413Ppykhh5pg6zVvcY4= -github.com/projectdiscovery/wappalyzergo v0.1.12/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= +github.com/projectdiscovery/tlsx v1.1.7 h1:eSsl/SmTDL/z2CMeSrbssk4f/9oOotMP1SgXl3yynSM= +github.com/projectdiscovery/tlsx v1.1.7/go.mod h1:g66QQ4/y4tLVjoGbzWIv+Q6xwFzxfJbEDx86Y1dYHDM= +github.com/projectdiscovery/useragent v0.0.69 h1:6TkVMKjOBDqecUBBkXlTlBUfAUM8Bgn/58E3ZasndBs= +github.com/projectdiscovery/useragent v0.0.69/go.mod h1:Ss+Kf6XEjSQqhm7LmbEtOxKvGnnnR88Fx+0PG2PXuBY= +github.com/projectdiscovery/utils v0.2.10 h1:MFx9pAaSyYGjXNi46mdAD7ceSZ80E0AGeuBj8fXaAF4= +github.com/projectdiscovery/utils v0.2.10/go.mod h1:jvTckFUQxZSiJrhACUuU63tqtd0tdBu3ng3QUVm/Ce0= +github.com/projectdiscovery/wappalyzergo v0.1.16 h1:c8HScLwkKOydIWyfjojLFzQjHdo0YYsU0MYXlx6ON5U= +github.com/projectdiscovery/wappalyzergo v0.1.16/go.mod h1:/hzgxkBFTMe2wDbA93nFfoMjULw7/vIZ9QPSAnCgUa8= github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= diff --git a/internal/pdcp/utils.go b/internal/pdcp/utils.go index 301ef2aa..3c399802 100644 --- a/internal/pdcp/utils.go +++ b/internal/pdcp/utils.go @@ -5,9 +5,17 @@ import ( urlutil "github.com/projectdiscovery/utils/url" ) -func getAssetsDashBoardURL(id string) string { +func getAssetsDashBoardURL(id, teamID string) string { ux, _ := urlutil.Parse(pdcpauth.DashBoardURL) ux.Path = "/assets/" + id + if ux.Params == nil { + ux.Params = urlutil.NewOrderedParams() + } + if teamID != "" { + ux.Params.Add("team_id", teamID) + } else { + ux.Params.Add("team_id", NoneTeamID) + } ux.Update() return ux.String() } diff --git a/internal/pdcp/writer.go b/internal/pdcp/writer.go index 5fd023ab..61327eeb 100644 --- a/internal/pdcp/writer.go +++ b/internal/pdcp/writer.go @@ -31,12 +31,11 @@ const ( MaxChunkSize = 4 * unitutils.Mega // 4 MB xidRe = `^[a-z0-9]{20}$` teamIDHeader = "X-Team-Id" + NoneTeamID = "none" ) var ( xidRegex = regexp.MustCompile(xidRe) - // teamID if given - teamID = env.GetEnvOrDefault("PDCP_TEAM_ID", "") // EnableeUpload if set to true enables the upload feature HideAutoSaveMsg = env.GetEnvOrDefault("DISABLE_CLOUD_UPLOAD_WRN", false) EnableCloudUpload = env.GetEnvOrDefault("ENABLE_CLOUD_UPLOAD", false) @@ -54,6 +53,7 @@ type UploadWriter struct { assetGroupName string counter atomic.Int32 closed atomic.Bool + TeamID string } // NewUploadWriterCallback creates a new upload writer callback @@ -63,9 +63,10 @@ func NewUploadWriterCallback(ctx context.Context, creds *pdcpauth.PDCPCredential return nil, fmt.Errorf("no credentials provided") } u := &UploadWriter{ - creds: creds, - done: make(chan struct{}, 1), - data: make(chan runner.Result, 8), // default buffer size + creds: creds, + done: make(chan struct{}, 1), + data: make(chan runner.Result, 8), // default buffer size + TeamID: "", } var err error tmp, err := urlutil.Parse(creds.Server) @@ -111,6 +112,11 @@ func (u *UploadWriter) SetAssetGroupName(name string) { u.assetGroupName = name } +// SetTeamID sets the team id for the upload writer +func (u *UploadWriter) SetTeamID(id string) { + u.TeamID = id +} + func (u *UploadWriter) autoCommit(ctx context.Context) { // wait for context to be done defer func() { @@ -120,7 +126,7 @@ func (u *UploadWriter) autoCommit(ctx context.Context) { if u.assetGroupID == "" { gologger.Verbose().Msgf("UI dashboard setup skipped, no results found to upload") } else { - gologger.Info().Msgf("Found %v results, View found results in dashboard : %v", u.counter.Load(), getAssetsDashBoardURL(u.assetGroupID)) + gologger.Info().Msgf("Found %v results, View found results in dashboard : %v", u.counter.Load(), getAssetsDashBoardURL(u.assetGroupID, u.TeamID)) } }() // temporary buffer to store the results @@ -185,7 +191,7 @@ func (u *UploadWriter) uploadChunk(buff *bytes.Buffer) error { // if successful, reset the buffer buff.Reset() // log in verbose mode - gologger.Warning().Msgf("Uploaded results chunk, you can view assets at %v", getAssetsDashBoardURL(u.assetGroupID)) + gologger.Warning().Msgf("Uploaded results chunk, you can view assets at %v", getAssetsDashBoardURL(u.assetGroupID, u.TeamID)) return nil } @@ -244,8 +250,8 @@ func (u *UploadWriter) getRequest(bin []byte) (*retryablehttp.Request, error) { req.URL.Update() req.Header.Set(pdcpauth.ApiKeyHeaderName, u.creds.APIKey) - if teamID != "" { - req.Header.Set(teamIDHeader, teamID) + if u.TeamID != "" { + req.Header.Set(teamIDHeader, u.TeamID) } req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Accept", "application/json") diff --git a/runner/options.go b/runner/options.go index ccf3cd73..325209c1 100644 --- a/runner/options.go +++ b/runner/options.go @@ -25,7 +25,7 @@ import ( "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/httpx/common/stringz" "github.com/projectdiscovery/networkpolicy" - "github.com/projectdiscovery/utils/auth/pdcp" + pdcpauth "github.com/projectdiscovery/utils/auth/pdcp" "github.com/projectdiscovery/utils/env" fileutil "github.com/projectdiscovery/utils/file" sliceutil "github.com/projectdiscovery/utils/slice" @@ -41,7 +41,10 @@ const ( DefaultOutputDirectory = "output" ) -var PDCPApiKey = "" +var ( + PDCPApiKey = "" + TeamIDEnv = env.GetEnvOrDefault("PDCP_TEAM_ID", "") +) // OnResultCallback (hostResult) type OnResultCallback func(Result) @@ -321,6 +324,7 @@ type Options struct { AssetID string // AssetFileUpload AssetFileUpload string + TeamID string // OnClose adds a callback function that is invoked when httpx is closed // to be exact at end of existing closures OnClose func() @@ -513,6 +517,7 @@ func ParseOptions() *Options { flagSet.CreateGroup("cloud", "Cloud", flagSet.DynamicVar(&options.PdcpAuth, "auth", "true", "configure projectdiscovery cloud (pdcp) api key"), flagSet.BoolVarP(&options.AssetUpload, "dashboard", "pd", false, "upload / view output in projectdiscovery cloud (pdcp) UI dashboard"), + flagSet.StringVarP(&options.TeamID, "team-id", "tid", TeamIDEnv, "upload asset results to given team id (optional)"), flagSet.StringVarP(&options.AssetID, "asset-id", "aid", "", "upload new assets to existing asset id (optional)"), flagSet.StringVarP(&options.AssetName, "asset-name", "aname", "", "assets group name to set (optional)"), flagSet.StringVarP(&options.AssetFileUpload, "dashboard-upload", "pdu", "", "upload httpx output file (jsonl) in projectdiscovery cloud (pdcp) UI dashboard"), @@ -544,9 +549,9 @@ func ParseOptions() *Options { AuthWithPDCP() } else if len(options.PdcpAuth) == 36 { PDCPApiKey = options.PdcpAuth - ph := pdcp.PDCPCredHandler{} - if _, err := ph.GetCreds(); err == pdcp.ErrNoCreds { - apiServer := env.GetEnvOrDefault("PDCP_API_SERVER", pdcp.DefaultApiServer) + ph := pdcpauth.PDCPCredHandler{} + if _, err := ph.GetCreds(); err == pdcpauth.ErrNoCreds { + apiServer := env.GetEnvOrDefault("PDCP_API_SERVER", pdcpauth.DefaultApiServer) if validatedCreds, err := ph.ValidateAPIKey(PDCPApiKey, apiServer, "httpx"); err == nil { _ = ph.SaveCreds(validatedCreds) } diff --git a/runner/types.go b/runner/types.go index da3b1cc6..a556ea96 100644 --- a/runner/types.go +++ b/runner/types.go @@ -9,7 +9,7 @@ import ( "github.com/go-faker/faker/v4" "github.com/go-faker/faker/v4/pkg/options" - "github.com/mitchellh/mapstructure" + mapstructure "github.com/go-viper/mapstructure/v2" "github.com/projectdiscovery/dsl" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/tlsx/pkg/tlsx/clients" @@ -32,74 +32,74 @@ func (o AsnResponse) String() string { // Result of a scan type Result struct { - Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"` - ASN *AsnResponse `json:"asn,omitempty" csv:"asn"` - Err error `json:"-" csv:"-"` - CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` - TLSData *clients.Response `json:"tls,omitempty" csv:"tls"` - Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash"` - ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex"` - CDNName string `json:"cdn_name,omitempty" csv:"cdn_name"` - CDNType string `json:"cdn_type,omitempty" csv:"cdn_type"` - SNI string `json:"sni,omitempty" csv:"sni"` - Port string `json:"port,omitempty" csv:"port"` - Raw string `json:"-" csv:"-"` - URL string `json:"url,omitempty" csv:"url"` - Input string `json:"input,omitempty" csv:"input"` - Location string `json:"location,omitempty" csv:"location"` - Title string `json:"title,omitempty" csv:"title"` - str string - Scheme string `json:"scheme,omitempty" csv:"scheme"` - Error string `json:"error,omitempty" csv:"error"` - WebServer string `json:"webserver,omitempty" csv:"webserver"` - ResponseBody string `json:"body,omitempty" csv:"body"` - BodyPreview string `json:"body_preview,omitempty" csv:"body_preview"` - ContentType string `json:"content_type,omitempty" csv:"content_type"` - Method string `json:"method,omitempty" csv:"method"` - Host string `json:"host,omitempty" csv:"host"` - Path string `json:"path,omitempty" csv:"path"` - FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon"` - FavIconMD5 string `json:"favicon_md5,omitempty" csv:"favicon"` - FaviconPath string `json:"favicon_path,omitempty" csv:"favicon_path"` - FaviconURL string `json:"favicon_url,omitempty" csv:"favicon_url"` - FinalURL string `json:"final_url,omitempty" csv:"final_url"` - ResponseHeaders map[string]interface{} `json:"header,omitempty" csv:"header"` - RawHeaders string `json:"raw_header,omitempty" csv:"raw_header"` - Request string `json:"request,omitempty" csv:"request"` - ResponseTime string `json:"time,omitempty" csv:"time"` - JarmHash string `json:"jarm_hash,omitempty" csv:"jarm_hash"` - ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes"` - A []string `json:"a,omitempty" csv:"a"` - AAAA []string `json:"aaaa,omitempty" csv:"aaaa"` - CNAMEs []string `json:"cname,omitempty" csv:"cname"` - Technologies []string `json:"tech,omitempty" csv:"tech"` - Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts"` - Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` - Words int `json:"words" csv:"words"` - Lines int `json:"lines" csv:"lines"` - StatusCode int `json:"status_code" csv:"status_code"` - ContentLength int `json:"content_length" csv:"content_length"` - Failed bool `json:"failed" csv:"failed"` - VHost bool `json:"vhost,omitempty" csv:"vhost"` - WebSocket bool `json:"websocket,omitempty" csv:"websocket"` - CDN bool `json:"cdn,omitempty" csv:"cdn"` - HTTP2 bool `json:"http2,omitempty" csv:"http2"` - Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` - HeadlessBody string `json:"headless_body,omitempty" csv:"headless_body"` - ScreenshotBytes []byte `json:"screenshot_bytes,omitempty" csv:"screenshot_bytes"` - StoredResponsePath string `json:"stored_response_path,omitempty" csv:"stored_response_path"` - ScreenshotPath string `json:"screenshot_path,omitempty" csv:"screenshot_path"` - ScreenshotPathRel string `json:"screenshot_path_rel,omitempty" csv:"screenshot_path_rel"` - KnowledgeBase map[string]interface{} `json:"knowledgebase,omitempty" csv:"knowledgebase"` - Resolvers []string `json:"resolvers,omitempty" csv:"resolvers"` - Fqdns []string `json:"body_fqdn,omitempty"` - Domains []string `json:"body_domains,omitempty"` + Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp" mapstructure:"timestamp"` + ASN *AsnResponse `json:"asn,omitempty" csv:"asn" mapstructure:"asn"` + Err error `json:"-" csv:"-" mapstructure:"-"` + CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp" mapstructure:"csp"` + TLSData *clients.Response `json:"tls,omitempty" csv:"tls" mapstructure:"tls"` + Hashes map[string]interface{} `json:"hash,omitempty" csv:"hash" mapstructure:"hash"` + ExtractRegex []string `json:"extract_regex,omitempty" csv:"extract_regex" mapstructure:"extract_regex"` + CDNName string `json:"cdn_name,omitempty" csv:"cdn_name" mapstructure:"cdn_name"` + CDNType string `json:"cdn_type,omitempty" csv:"cdn_type" mapstructure:"cdn_type"` + SNI string `json:"sni,omitempty" csv:"sni" mapstructure:"sni"` + Port string `json:"port,omitempty" csv:"port" mapstructure:"port"` + Raw string `json:"-" csv:"-" mapstructure:"-"` + URL string `json:"url,omitempty" csv:"url" mapstructure:"url"` + Input string `json:"input,omitempty" csv:"input" mapstructure:"input"` + Location string `json:"location,omitempty" csv:"location" mapstructure:"location"` + Title string `json:"title,omitempty" csv:"title" mapstructure:"title"` + str string `mapstructure:"-"` + Scheme string `json:"scheme,omitempty" csv:"scheme" mapstructure:"scheme"` + Error string `json:"error,omitempty" csv:"error" mapstructure:"error"` + WebServer string `json:"webserver,omitempty" csv:"webserver" mapstructure:"webserver"` + ResponseBody string `json:"body,omitempty" csv:"body" mapstructure:"body"` + BodyPreview string `json:"body_preview,omitempty" csv:"body_preview" mapstructure:"body_preview"` + ContentType string `json:"content_type,omitempty" csv:"content_type" mapstructure:"content_type"` + Method string `json:"method,omitempty" csv:"method" mapstructure:"method"` + Host string `json:"host,omitempty" csv:"host" mapstructure:"host"` + Path string `json:"path,omitempty" csv:"path" mapstructure:"path"` + FavIconMMH3 string `json:"favicon,omitempty" csv:"favicon" mapstructure:"favicon"` + FavIconMD5 string `json:"favicon_md5,omitempty" csv:"favicon" mapstructure:"favicon_md5"` + FaviconPath string `json:"favicon_path,omitempty" csv:"favicon_path" mapstructure:"favicon_path"` + FaviconURL string `json:"favicon_url,omitempty" csv:"favicon_url" mapstructure:"favicon_url"` + FinalURL string `json:"final_url,omitempty" csv:"final_url" mapstructure:"final_url"` + ResponseHeaders map[string]interface{} `json:"header,omitempty" csv:"header" mapstructure:"header"` + RawHeaders string `json:"raw_header,omitempty" csv:"raw_header" mapstructure:"raw_header"` + Request string `json:"request,omitempty" csv:"request" mapstructure:"request"` + ResponseTime string `json:"time,omitempty" csv:"time" mapstructure:"time"` + JarmHash string `json:"jarm_hash,omitempty" csv:"jarm_hash" mapstructure:"jarm_hash"` + ChainStatusCodes []int `json:"chain_status_codes,omitempty" csv:"chain_status_codes" mapstructure:"chain_status_codes"` + A []string `json:"a,omitempty" csv:"a" mapstructure:"a"` + AAAA []string `json:"aaaa,omitempty" csv:"aaaa" mapstructure:"aaaa"` + CNAMEs []string `json:"cname,omitempty" csv:"cname" mapstructure:"cname"` + Technologies []string `json:"tech,omitempty" csv:"tech" mapstructure:"tech"` + Extracts map[string][]string `json:"extracts,omitempty" csv:"extracts" mapstructure:"extracts"` + Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain" mapstructure:"chain"` + Words int `json:"words" csv:"words" mapstructure:"words"` + Lines int `json:"lines" csv:"lines" mapstructure:"lines"` + StatusCode int `json:"status_code" csv:"status_code" mapstructure:"status_code"` + ContentLength int `json:"content_length" csv:"content_length" mapstructure:"content_length"` + Failed bool `json:"failed" csv:"failed" mapstructure:"failed"` + VHost bool `json:"vhost,omitempty" csv:"vhost" mapstructure:"vhost"` + WebSocket bool `json:"websocket,omitempty" csv:"websocket" mapstructure:"websocket"` + CDN bool `json:"cdn,omitempty" csv:"cdn" mapstructure:"cdn"` + HTTP2 bool `json:"http2,omitempty" csv:"http2" mapstructure:"http2"` + Pipeline bool `json:"pipeline,omitempty" csv:"pipeline" mapstructure:"pipeline"` + HeadlessBody string `json:"headless_body,omitempty" csv:"headless_body" mapstructure:"headless_body"` + ScreenshotBytes []byte `json:"screenshot_bytes,omitempty" csv:"screenshot_bytes" mapstructure:"screenshot_bytes"` + StoredResponsePath string `json:"stored_response_path,omitempty" csv:"stored_response_path" mapstructure:"stored_response_path"` + ScreenshotPath string `json:"screenshot_path,omitempty" csv:"screenshot_path" mapstructure:"screenshot_path"` + ScreenshotPathRel string `json:"screenshot_path_rel,omitempty" csv:"screenshot_path_rel" mapstructure:"screenshot_path_rel"` + KnowledgeBase map[string]interface{} `json:"knowledgebase,omitempty" csv:"knowledgebase" mapstructure:"knowledgebase"` + Resolvers []string `json:"resolvers,omitempty" csv:"resolvers" mapstructure:"resolvers"` + Fqdns []string `json:"body_fqdn,omitempty" mapstructure:"body_fqdn"` + Domains []string `json:"body_domains,omitempty" mapstructure:"body_domains"` // Internal Fields - TechnologyDetails map[string]wappalyzer.AppInfo `json:"-" csv:"-"` - RequestRaw []byte `json:"-" csv:"-"` - Response *httpx.Response `json:"-" csv:"-"` - FaviconData []byte `json:"-" csv:"-"` + TechnologyDetails map[string]wappalyzer.AppInfo `json:"-" csv:"-" mapstructure:"-"` + RequestRaw []byte `json:"-" csv:"-" mapstructure:"-"` + Response *httpx.Response `json:"-" csv:"-" mapstructure:"-"` + FaviconData []byte `json:"-" csv:"-" mapstructure:"-"` } // function to get dsl variables from result struct @@ -139,8 +139,7 @@ func evalDslExpr(result Result, dslExpr string) bool { func resultToMap(resp Result) (map[string]any, error) { m := make(map[string]any) config := &mapstructure.DecoderConfig{ - TagName: "json", - Result: &m, + Result: &m, } decoder, err := mapstructure.NewDecoder(config) if err != nil {