diff --git a/smtp.go b/smtp.go index 58fd69d..f9f10d5 100644 --- a/smtp.go +++ b/smtp.go @@ -7,6 +7,7 @@ import ( "net" "net/smtp" "net/url" + "strings" "sync" "time" @@ -33,26 +34,35 @@ func (v *Verifier) CheckSMTP(domain, username string) (*SMTP, error) { } var ret SMTP + var err error + email := fmt.Sprintf("%s@%s", username, domain) // Dial any SMTP server that will accept a connection - client, err := newSMTPClient(domain, v.proxyURI) + client, mx, err := newSMTPClient(domain, v.proxyURI) if err != nil { return &ret, ParseSMTPError(err) } + // Defer quit the SMTP connection + defer client.Close() + + // Check by api when enabled and host recognized. + for _, apiVerifier := range v.apiVerifiers { + if apiVerifier.isSupported(strings.ToLower(mx.Host)) { + return apiVerifier.check(domain, username) + } + } + // Sets the HELO/EHLO hostname - if err := client.Hello(v.helloName); err != nil { + if err = client.Hello(v.helloName); err != nil { return &ret, ParseSMTPError(err) } // Sets the from email - if err := client.Mail(v.fromEmail); err != nil { + if err = client.Mail(v.fromEmail); err != nil { return &ret, ParseSMTPError(err) } - // Defer quit the SMTP connection - defer client.Close() - // Host exists if we've successfully formed a connection ret.HostExists = true @@ -63,7 +73,7 @@ func (v *Verifier) CheckSMTP(domain, username string) (*SMTP, error) { // Checks the deliver ability of a randomly generated address in // order to verify the existence of a catch-all and etc. randomEmail := GenerateRandomEmail(domain) - if err := client.Rcpt(randomEmail); err != nil { + if err = client.Rcpt(randomEmail); err != nil { if e := ParseSMTPError(err); e != nil { switch e.Message { case ErrFullInbox: @@ -94,8 +104,7 @@ func (v *Verifier) CheckSMTP(domain, username string) (*SMTP, error) { return &ret, nil } - email := fmt.Sprintf("%s@%s", username, domain) - if err := client.Rcpt(email); err == nil { + if err = client.Rcpt(email); err == nil { ret.Deliverable = true } @@ -103,18 +112,19 @@ func (v *Verifier) CheckSMTP(domain, username string) (*SMTP, error) { } // newSMTPClient generates a new available SMTP client -func newSMTPClient(domain, proxyURI string) (*smtp.Client, error) { +func newSMTPClient(domain, proxyURI string) (*smtp.Client, *net.MX, error) { domain = domainToASCII(domain) mxRecords, err := net.LookupMX(domain) if err != nil { - return nil, err + return nil, nil, err } if len(mxRecords) == 0 { - return nil, errors.New("No MX records found") + return nil, nil, errors.New("No MX records found") } // Create a channel for receiving response from ch := make(chan interface{}, 1) + selectedMXCh := make(chan *net.MX, 1) // Done indicates if we're still waiting on dial responses var done bool @@ -123,9 +133,9 @@ func newSMTPClient(domain, proxyURI string) (*smtp.Client, error) { var mutex sync.Mutex // Attempt to connect to all SMTP servers concurrently - for _, r := range mxRecords { + for i, r := range mxRecords { addr := r.Host + smtpPort - + index := i go func() { c, err := dialSMTP(addr, proxyURI) if err != nil { @@ -141,6 +151,7 @@ func newSMTPClient(domain, proxyURI string) (*smtp.Client, error) { case !done: done = true ch <- c + selectedMXCh <- mxRecords[index] default: c.Close() } @@ -154,14 +165,14 @@ func newSMTPClient(domain, proxyURI string) (*smtp.Client, error) { res := <-ch switch r := res.(type) { case *smtp.Client: - return r, nil + return r, <-selectedMXCh, nil case error: errs = append(errs, r) if len(errs) == len(mxRecords) { - return nil, errs[0] + return nil, nil, errs[0] } default: - return nil, errors.New("Unexpected response dialing SMTP server") + return nil, nil, errors.New("Unexpected response dialing SMTP server") } } diff --git a/smtp_by_api.go b/smtp_by_api.go new file mode 100644 index 0000000..2d73d2b --- /dev/null +++ b/smtp_by_api.go @@ -0,0 +1,13 @@ +package emailverifier + +const ( + GMAIL = "gmail" + YAHOO = "yahoo" +) + +type smtpAPIVerifier interface { + // isSupported the specific host supports the check by api. + isSupported(host string) bool + // check must be called before isSupported == true + check(domain, username string) (*SMTP, error) +} diff --git a/smtp_by_api_gmail.go b/smtp_by_api_gmail.go new file mode 100644 index 0000000..b173afa --- /dev/null +++ b/smtp_by_api_gmail.go @@ -0,0 +1,53 @@ +package emailverifier + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" +) + +const ( + glxuPageFormat = "https://mail.google.com/mail/gxlu?email=%s" +) + +// See the link below to know why we can use this way to check if a gmail exists. +// https://blog.0day.rocks/abusing-gmail-to-get-previously-unlisted-e-mail-addresses-41544b62b2 +func newGmailAPIVerifier(client *http.Client) smtpAPIVerifier { + if client == nil { + client = http.DefaultClient + } + return gmail{ + client: client, + } +} + +type gmail struct { + client *http.Client +} + +func (g gmail) isSupported(host string) bool { + return strings.HasSuffix(host, ".google.com.") +} + +func (g gmail) check(domain, username string) (*SMTP, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + email := fmt.Sprintf("%s@%s", username, domain) + request, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf(glxuPageFormat, email), nil) + if err != nil { + return nil, err + } + resp, err := g.client.Do(request) + if err != nil { + return &SMTP{}, err + } + defer resp.Body.Close() + emailExists := len(resp.Cookies()) > 0 + + return &SMTP{ + HostExists: true, + Deliverable: emailExists, + }, nil +} diff --git a/smtp_by_api_gmail_test.go b/smtp_by_api_gmail_test.go new file mode 100644 index 0000000..fc76332 --- /dev/null +++ b/smtp_by_api_gmail_test.go @@ -0,0 +1,25 @@ +package emailverifier + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGmailCheckByAPI(t *testing.T) { + gmailAPIVerifier := newGmailAPIVerifier(nil) + + t.Run("email exists", func(tt *testing.T) { + res, err := gmailAPIVerifier.check("gmail.com", "someone") + assert.NoError(t, err) + assert.Equal(t, true, res.HostExists) + assert.Equal(t, true, res.Deliverable) + }) + t.Run("invalid email not exists", func(tt *testing.T) { + // username must greater than 6 characters + res, err := gmailAPIVerifier.check("gmail.com", "hello") + assert.NoError(t, err) + assert.Equal(t, true, res.HostExists) + assert.Equal(t, false, res.Deliverable) + }) +} diff --git a/smtp_by_api_yahoo.go b/smtp_by_api_yahoo.go new file mode 100644 index 0000000..5e0119c --- /dev/null +++ b/smtp_by_api_yahoo.go @@ -0,0 +1,178 @@ +package emailverifier + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io/ioutil" + "net/http" + "regexp" + "strings" + "time" +) + +const ( + SIGNUP_PAGE = "https://login.yahoo.com/account/create?specId=yidregsimplified&lang=en-US&src=&done=https%3A%2F%2Fwww.yahoo.com&display=login" + SIGNUP_API = "https://login.yahoo.com/account/module/create?validateField=userId" + // USER_AGENT Fake one to use in API requests + USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36" +) + +// Check yahoo email exists by their login & registration page. +// See https://login.yahoo.com +// See https://login.yahoo.com/account/create +func newYahooAPIVerifier(client *http.Client) smtpAPIVerifier { + if client == nil { + client = http.DefaultClient + } + return yahoo{ + client: client, + } +} + +type yahoo struct { + client *http.Client +} + +type yahooValidateReq struct { + Domain, Username, Acrumb, SessionIndex string + Cookies []*http.Cookie +} + +type yahooErrorResp struct { + Errors []errItem `json:"errors"` +} + +type errItem struct { + Name string `json:"name"` + Error string `json:"error"` +} + +func (y yahoo) isSupported(host string) bool { + // FIXME Is this `contains` too lenient? + return strings.Contains(host, "yahoo") +} + +func (y yahoo) check(domain, username string) (*SMTP, error) { + cookies, signUpPageRespBytes, err := y.toSignUpPage() + if err != nil { + return nil, err + } + if len(cookies) == 0 { + return nil, errors.New("yahoo check by api, no cookies") + } + + acrumb := getAcrumb(cookies) + if acrumb == "" { + return nil, errors.New("yahoo check by api, no acrumb") + } + + sessionIndex := getSessionIndex(signUpPageRespBytes) + if sessionIndex == "" { + return nil, errors.New("yahoo check by api, no sessionIndex") + } + + yahooErrResp, err := y.sendValidateRequest(yahooValidateReq{ + Domain: domain, + Username: username, + Acrumb: acrumb, + SessionIndex: sessionIndex, + Cookies: cookies, + }) + if err != nil { + return nil, err + } + usernameExists := checkUsernameExists(yahooErrResp) + return &SMTP{ + HostExists: true, + Deliverable: usernameExists, + }, nil +} + +func getSessionIndex(respBytes []byte) string { + re := regexp.MustCompile(`value="([^"]+)" name="sessionIndex"`) + match := re.FindSubmatch(respBytes) + if len(match) > 1 { + return string(match[1]) + } + return "" +} + +func checkUsernameExists(resp yahooErrorResp) bool { + for _, item := range resp.Errors { + if item.Name == "userId" && item.Error == "IDENTIFIER_EXISTS" { + return true + } + } + return false +} + +func (y yahoo) sendValidateRequest(req yahooValidateReq) (yahooErrorResp, error) { + var res yahooErrorResp + data, err := json.Marshal(struct { + Acrumb string `json:"acrumb"` + SpecId string `json:"specId"` + Yid string `json:"userId"` + SessionIndex string `json:"sessionIndex"` + YidDomain string `json:"yidDomain"` + }{ + Acrumb: req.Acrumb, + SpecId: "yidregsimplified", + Yid: req.Username, + SessionIndex: req.SessionIndex, + YidDomain: req.Domain, + }) + if err != nil { + return res, err + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + request, err := http.NewRequestWithContext(ctx, http.MethodPost, SIGNUP_API, bytes.NewReader(data)) + if err != nil { + return res, err + } + for _, c := range req.Cookies { + request.AddCookie(c) + } + request.Header.Add("X-Requested-With", "XMLHttpRequest") + request.Header.Add("Content-Type", "application/json; charset=UTF-8") + resp, err := y.client.Do(request) + if err != nil { + return res, err + } + defer resp.Body.Close() + respBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return res, err + } + return res, json.Unmarshal(respBytes, &res) +} + +func (y yahoo) toSignUpPage() ([]*http.Cookie, []byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + request, err := http.NewRequestWithContext(ctx, http.MethodGet, SIGNUP_PAGE, nil) + if err != nil { + return nil, nil, err + } + request.Header.Add("User-Agent", USER_AGENT) + resp, err := y.client.Do(request) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + respBytes, err := ioutil.ReadAll(resp.Body) + return resp.Cookies(), respBytes, err +} + +func getAcrumb(cookies []*http.Cookie) string { + for _, c := range cookies { + re := regexp.MustCompile(`s=(?P[^;^&]*)`) + match := re.FindStringSubmatch(c.Value) + if len(match) > 1 { + return match[1] + } + } + return "" +} diff --git a/smtp_by_api_yahoo_test.go b/smtp_by_api_yahoo_test.go new file mode 100644 index 0000000..6a0bcfa --- /dev/null +++ b/smtp_by_api_yahoo_test.go @@ -0,0 +1,46 @@ +package emailverifier + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestYahooCheckByAPI(t *testing.T) { + yahooAPIVerifier := newYahooAPIVerifier(nil) + t.Run("email exists", func(tt *testing.T) { + res, err := yahooAPIVerifier.check("yahoo.com", "hello") + assert.NoError(t, err) + assert.Equal(t, true, res.HostExists) + assert.Equal(t, true, res.Deliverable) + }) + t.Run("invalid email not exists", func(tt *testing.T) { + res, err := yahooAPIVerifier.check("yahoo.com", "123") + assert.NoError(t, err) + assert.Equal(t, true, res.HostExists) + assert.Equal(t, false, res.Deliverable) + }) +} + +func TestGetAcrumb(t *testing.T) { + cookies0 := []*http.Cookie{ + {Value: "123321"}, + {Value: "v=1&s=gWKqrs5c&d=A6454c24b|Zt.ZFgb.2T"}, + } + acrumb := getAcrumb(cookies0) + assert.Equal(t, acrumb, "gWKqrs5c") + + cookies1 := []*http.Cookie{ + {Value: "123321"}, + {Value: "v=1&s=gWKqrs5c"}, + } + acrumb = getAcrumb(cookies1) + assert.Equal(t, acrumb, "gWKqrs5c") + + cookies2 := []*http.Cookie{ + {Value: "123321"}, + } + acrumb = getAcrumb(cookies2) + assert.Equal(t, acrumb, "") +} diff --git a/smtp_test.go b/smtp_test.go index 71ccaf2..3e92436 100644 --- a/smtp_test.go +++ b/smtp_test.go @@ -8,6 +8,87 @@ import ( "github.com/stretchr/testify/assert" ) +func TestCheckSMTPUnSupportedVendor(t *testing.T) { + err := verifier.EnableAPIVerifier("unsupported_vendor") + assert.Error(t, err) +} + +func TestCheckSMTPOK_ByApi(t *testing.T) { + cases := []struct { + name string + domain string + username string + expected *SMTP + }{ + { + name: "gmail exists", + domain: "gmail.com", + username: "someone", + expected: &SMTP{ + HostExists: true, + Deliverable: true, + }, + }, + { + name: "gmail no exists", + domain: "gmail.com", + username: "hello", + expected: &SMTP{ + HostExists: true, + Deliverable: false, + }, + }, + { + name: "yahoo exists", + domain: "yahoo.com", + username: "someone", + expected: &SMTP{ + HostExists: true, + Deliverable: true, + }, + }, + { + name: "myyahoo exists", + domain: "myyahoo.com", + username: "someone", + expected: &SMTP{ + HostExists: true, + Deliverable: true, + }, + }, + { + name: "yahoo no exists", + domain: "yahoo.com", + username: "123", + expected: &SMTP{ + HostExists: true, + Deliverable: false, + }, + }, + { + name: "myyahoo no exists", + domain: "myyahoo.com", + username: "123", + expected: &SMTP{ + HostExists: true, + Deliverable: false, + }, + }, + } + _ = verifier.EnableAPIVerifier(GMAIL) + _ = verifier.EnableAPIVerifier(YAHOO) + defer verifier.DisableAPIVerifier(GMAIL) + defer verifier.DisableAPIVerifier(YAHOO) + for _, c := range cases { + test := c + t.Run(test.name, func(tt *testing.T) { + smtp, err := verifier.CheckSMTP(test.domain, test.username) + assert.NoError(t, err) + assert.Equal(t, test.expected, smtp) + }) + } +} + func TestCheckSMTPOK_HostExists(t *testing.T) { domain := "github.com" @@ -133,7 +214,7 @@ func TestCheckSMTPOK_HostNotExists(t *testing.T) { func TestNewSMTPClientOK(t *testing.T) { domain := "gmail.com" - ret, err := newSMTPClient(domain, "") + ret, _, err := newSMTPClient(domain, "") assert.NotNil(t, ret) assert.Nil(t, err) } @@ -141,14 +222,14 @@ func TestNewSMTPClientOK(t *testing.T) { func TestNewSMTPClientFailed_WithInvalidProxy(t *testing.T) { domain := "gmail.com" proxyURI := "socks5://user:password@127.0.0.1:1080?timeout=5s" - ret, err := newSMTPClient(domain, proxyURI) + ret, _, err := newSMTPClient(domain, proxyURI) assert.Nil(t, ret) assert.Error(t, err, syscall.ECONNREFUSED) } func TestNewSMTPClientFailed(t *testing.T) { domain := "zzzz171777.com" - ret, err := newSMTPClient(domain, "") + ret, _, err := newSMTPClient(domain, "") assert.Nil(t, ret) assert.Error(t, err) } diff --git a/verifier.go b/verifier.go index eef6307..f3cb487 100644 --- a/verifier.go +++ b/verifier.go @@ -1,20 +1,22 @@ package emailverifier import ( + "fmt" + "net/http" "time" ) // Verifier is an email verifier. Create one by calling NewVerifier type Verifier struct { - smtpCheckEnabled bool // SMTP check enabled or disabled (disabled by default) - catchAllCheckEnabled bool // SMTP catchAll check enabled or disabled (enabled by default) - domainSuggestEnabled bool // whether suggest a most similar correct domain or not (disabled by default) - gravatarCheckEnabled bool // gravatar check enabled or disabled (disabled by default) - fromEmail string // name to use in the `EHLO:` SMTP command, defaults to "user@example.org" - helloName string // email to use in the `MAIL FROM:` SMTP command. defaults to `localhost` - schedule *schedule // schedule represents a job schedule - - proxyURI string // use a SOCKS5 proxy to verify the email, + smtpCheckEnabled bool // SMTP check enabled or disabled (disabled by default) + catchAllCheckEnabled bool // SMTP catchAll check enabled or disabled (enabled by default) + domainSuggestEnabled bool // whether suggest a most similar correct domain or not (disabled by default) + gravatarCheckEnabled bool // gravatar check enabled or disabled (disabled by default) + fromEmail string // name to use in the `EHLO:` SMTP command, defaults to "user@example.org" + helloName string // email to use in the `MAIL FROM:` SMTP command. defaults to `localhost` + schedule *schedule // schedule represents a job schedule + proxyURI string // use a SOCKS5 proxy to verify the email, + apiVerifiers map[string]smtpAPIVerifier // currently support gmail & yahoo, further contributions are welcomed. } // Result is the result of Email Verification @@ -47,6 +49,7 @@ func NewVerifier() *Verifier { fromEmail: defaultFromEmail, helloName: defaultHelloName, catchAllCheckEnabled: true, + apiVerifiers: map[string]smtpAPIVerifier{}, } } @@ -131,6 +134,25 @@ func (v *Verifier) EnableSMTPCheck() *Verifier { return v } +// EnableAPIVerifier API verifier is activated when EnableAPIVerifier for the target vendor. +// ** Please know ** that this is a tricky way (but relatively stable) to check if target vendor's email exists. +// If you use this feature in a production environment, please ensure that you have sufficient backup measures in place, as this may encounter rate limiting or other API issues. +func (v *Verifier) EnableAPIVerifier(name string) error { + switch name { + case GMAIL: + v.apiVerifiers[GMAIL] = newGmailAPIVerifier(http.DefaultClient) + case YAHOO: + v.apiVerifiers[YAHOO] = newYahooAPIVerifier(http.DefaultClient) + default: + return fmt.Errorf("unsupported to enable the API verifier for vendor: %s", name) + } + return nil +} + +func (v *Verifier) DisableAPIVerifier(name string) { + delete(v.apiVerifiers, name) +} + // DisableSMTPCheck disables check email by smtp func (v *Verifier) DisableSMTPCheck() *Verifier { v.smtpCheckEnabled = false