-
Notifications
You must be signed in to change notification settings - Fork 2
/
main.go
120 lines (107 loc) · 2.6 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
package main
import (
"bufio"
"encoding/csv"
"flag"
"log"
"net/http"
"net/http/cookiejar"
"os"
"strings"
"sync"
"golang.org/x/sync/errgroup"
)
var (
fMaxPages = flag.Int("max-pages", 50, "maximum result pages per domain")
fResolvers = flag.Int("resolvers", 10, "number of concurrent resovlers. More is safe but won't speed things up much")
fScanners = flag.Int("scanners", 5, "number of concurrent scanners. More will make things faster but risk rate limiting")
)
func fatalIfError(err error, msg string) {
if err != nil {
log.Fatal("error ", msg, ": ", err)
}
}
func main() {
flag.Parse()
// Need an auth cookie for requests. These aren't persisted to disk
jar, err := cookiejar.New(nil)
fatalIfError(err, "creating cookie jar")
client := &http.Client{
Jar: jar,
}
fatalIfError(getGoogleCookie(client), "getting google cookie")
scanner := Scanner{
client: client,
maxPages: *fMaxPages,
lock: &sync.Mutex{},
scanned: map[string]struct{}{},
in: make(chan string),
out: make(chan Record),
}
scanners := errgroup.Group{}
for i := 0; i < *fScanners; i++ {
// Start up multiple scanners
scanners.Go(scanner.ScanStream)
}
resolver := Resolver{
in: scanner.out,
out: make(chan Record),
lock: &sync.Mutex{},
resolved: map[string]struct{}{},
}
resolvers := errgroup.Group{}
for i := 0; i < *fResolvers; i++ {
// Start up multiple resolvers
resolvers.Go(resolver.Resolve)
}
go func() {
// when we've received everything from STDIN, close the input channel
// to the scanners to signal no more work
defer close(scanner.in)
lineScanner := bufio.NewScanner(os.Stdin)
for lineScanner.Scan() {
// read lines from standard in
line := lineScanner.Text()
line = strings.TrimSpace(line)
if line == "" || line[0] == '#' {
// skip empty lines and comments
continue
}
scanner.in <- line
}
}()
go func() {
// wait for the scanners to finish
fatalIfError(scanners.Wait(), "in scanner")
// close scanner.out/resolver.in to signal no more resolver work
close(scanner.out)
// Wait for the resolvers to finish
fatalIfError(resolvers.Wait(), "in resolver")
// close resolver.out to signal no more output work
close(resolver.out)
}()
w := csv.NewWriter(os.Stdout)
for record := range resolver.out {
var row []string
if record.Err != nil {
w.Write([]string{
record.From,
record.Name,
"",
record.Err.Error(),
})
} else {
row = []string{
record.From,
record.Name,
"",
"",
}
for _, addr := range record.Addrs {
row[2] = addr
w.Write(row)
}
}
}
w.Flush()
}