Codebase list subfinder / 726cc37
Import upstream version 2.4.5 Kali Janitor 3 years ago
125 changed file(s) with 5137 addition(s) and 4367 deletion(s). Raw diff Collapse all Expand all
+20
-0
.github/ISSUE_TEMPLATE/bug_report.md less more
0 ---
1 name: Bug report
2 about: Create a report to help us improve
3 title: "[Issue] "
4 labels: ''
5 assignees: ''
6
7 ---
8
9 **Describe the bug**
10 A clear and concise description of what the bug is.
11
12 **Subfinder version**
13 Include the version of subfinder you are using, `subfinder -version`
14
15 **Complete command you used to reproduce this**
16
17
18 **Screenshots**
19 Add screenshots of the error for a better context.
+14
-0
.github/ISSUE_TEMPLATE/feature_request.md less more
0 ---
1 name: Feature request
2 about: Suggest an idea for this project
3 title: "[Feature] "
4 labels: ''
5 assignees: ''
6
7 ---
8
9 **Is your feature request related to a problem? Please describe.**
10 A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
11
12 **Describe the solution you'd like**
13 A clear and concise description of what you want to happen.
+28
-5
.github/workflows/build.yaml less more
33 branches:
44 - master
55 pull_request:
6
7 jobs:
6
7 jobs:
8 lint:
9 name: golangci-lint
10 runs-on: ubuntu-latest
11 steps:
12 - name: Checkout code
13 uses: actions/checkout@v2
14 - name: Run golangci-lint
15 uses: golangci/golangci-lint-action@v1
16 with:
17 # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
18 version: v1.29
19 args: --timeout 5m
20 working-directory: v2/cmd/subfinder/
21
22 # Optional: working directory, useful for monorepos
23 # working-directory: somedir
24
25 # Optional: golangci-lint command line arguments.
26 # args: --issues-exit-code=0
27
28 # Optional: show only new issues if it's a pull request. The default value is `false`.
29 # only-new-issues: true
30
831 build:
932 name: Build
10 runs-on: ubuntu-latest
33 runs-on: ubuntu-latest
1134 steps:
1235 - name: Set up Go
1336 uses: actions/setup-go@v2
1942
2043 - name: Test
2144 run: go test .
22 working-directory: cmd/subfinder/
45 working-directory: v2/cmd/subfinder/
2346
2447 - name: Build
2548 run: go build .
26 working-directory: cmd/subfinder/⏎
49 working-directory: v2/cmd/subfinder/
+2
-2
.github/workflows/dockerhub-push-on-release.yml less more
00 # dockerhub-push pushes docker build to dockerhub automatically
11 # on the creation of a new release
22 name: Publish to Dockerhub on creation of a new release
3 on:
3 on:
44 release:
55 types: [published]
66 jobs:
1313 with:
1414 name: projectdiscovery/subfinder
1515 username: ${{ secrets.DOCKER_USERNAME }}
16 password: ${{ secrets.DOCKER_PASSWORD }}⏎
16 password: ${{ secrets.DOCKER_PASSWORD }}
+10
-10
.github/workflows/release.yml less more
33 tags:
44 - v*
55
6 jobs:
6 jobs:
77 release:
88 runs-on: ubuntu-latest
9 steps:
10 -
9 steps:
10 -
1111 name: "Check out code"
1212 uses: actions/checkout@v2
13 with:
13 with:
1414 fetch-depth: 0
15 -
15 -
1616 name: "Set up Go"
1717 uses: actions/setup-go@v2
18 with:
18 with:
1919 go-version: 1.14
20 -
21 env:
20 -
21 env:
2222 GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
2323 name: "Create release on GitHub"
2424 uses: goreleaser/goreleaser-action@v2
25 with:
25 with:
2626 args: "release --rm-dist"
27 version: latest⏎
27 version: latest
+1
-0
.gitignore less more
00 .DS_Store
11 cmd/subfinder/subfinder
2 v2/cmd/subfinder/subfinder
23 vendor/
34 .idea⏎
+116
-0
.golangci.yml less more
0 linters-settings:
1 dupl:
2 threshold: 100
3 exhaustive:
4 default-signifies-exhaustive: false
5 # funlen:
6 # lines: 100
7 # statements: 50
8 goconst:
9 min-len: 2
10 min-occurrences: 2
11 gocritic:
12 enabled-tags:
13 - diagnostic
14 - experimental
15 - opinionated
16 - performance
17 - style
18 disabled-checks:
19 - dupImport # https://github.com/go-critic/go-critic/issues/845
20 - ifElseChain
21 # gocyclo:
22 # min-complexity: 15
23 goimports:
24 local-prefixes: github.com/golangci/golangci-lint
25 golint:
26 min-confidence: 0
27 gomnd:
28 settings:
29 mnd:
30 # don't include the "operation" and "assign"
31 checks: argument,case,condition,return
32 govet:
33 check-shadowing: true
34 settings:
35 printf:
36 funcs:
37 - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof
38 - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf
39 - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf
40 - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf
41 # lll:
42 # line-length: 140
43 maligned:
44 suggest-new: true
45 misspell:
46 locale: US
47 nolintlint:
48 allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
49 allow-unused: false # report any unused nolint directives
50 require-explanation: false # don't require an explanation for nolint directives
51 require-specific: false # don't require nolint directives to be specific about which linter is being skipped
52
53 linters:
54 # please, do not use `enable-all`: it's deprecated and will be removed soon.
55 # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
56 disable-all: true
57 enable:
58 - bodyclose
59 - deadcode
60 - dogsled
61 - dupl
62 - errcheck
63 - exhaustive
64 - gochecknoinits
65 - goconst
66 - gocritic
67 - gofmt
68 - goimports
69 - golint
70 - gomnd
71 - goprintffuncname
72 - gosimple
73 - govet
74 - ineffassign
75 - interfacer
76 - maligned
77 - misspell
78 - nakedret
79 - noctx
80 - nolintlint
81 - rowserrcheck
82 - scopelint
83 - staticcheck
84 - structcheck
85 - stylecheck
86 - typecheck
87 - unconvert
88 - unparam
89 - unused
90 - varcheck
91 - whitespace
92
93 # don't enable:
94 # - depguard
95 # - asciicheck
96 # - funlen
97 # - gochecknoglobals
98 # - gocognit
99 # - gocyclo
100 # - godot
101 # - godox
102 # - goerr113
103 # - gosec
104 # - lll
105 # - nestif
106 # - prealloc
107 # - testpackage
108 # - wsl
109
110 # golangci.com configuration
111 # https://github.com/golangci/golangci/wiki/Configuration
112 service:
113 golangci-lint-version: 1.29.x # use the fixed version to not introduce new linters unexpectedly
114 prepare:
115 - echo "here I can run custom commands, but no preparation needed for this repo"
+1
-1
.goreleaser.yml less more
00 builds:
11 - binary: subfinder
2 main: cmd/subfinder/main.go
2 main: v2/cmd/subfinder/main.go
33 goos:
44 - linux
55 - windows
+0
-1
DISCLAIMER.md less more
55 - CommonCrawl: https://commoncrawl.org/terms-of-use/full
66 - certspotter: https://sslmate.com/terms
77 - dnsdumpster: https://hackertarget.com/terms
8 - entrust: https://www.entrustdatacard.com/pages/terms-of-use
98 - Google Transparency: https://policies.google.com/terms
109 - Threatcrowd: https://www.alienvault.com/terms/website-terms-of-use07may2018
1110
+1
-1
Dockerfile less more
55 WORKDIR /go/src/app
66
77 # Install
8 RUN go get -u github.com/projectdiscovery/subfinder/cmd/subfinder
8 RUN go get -u github.com/projectdiscovery/subfinder/v2/cmd/subfinder
99
1010 ENTRYPOINT ["subfinder"]
+0
-22
ISSUE_TEMPLATE.md less more
0 ## What's the problem (or question)?
1 <!--- If describing a bug, tell us what happens instead of the expected behavior -->
2 <!--- If suggesting a change/improvement, explain the difference from current behavior -->
3
4 ## Do you have an idea for a solution?
5 <!--- Not obligatory, but suggest a fix/reason for the bug, -->
6 <!--- or ideas how to implement the addition or change -->
7
8 ## How can we reproduce the issue?
9 <!--- Provide unambiguous set of steps to reproduce this bug. Include command to reproduce, if relevant (you can mask the sensitive data) -->
10 1.
11 2.
12 3.
13 4.
14
15 ## What are the running context details?
16 <!--- Include as many relevant details about the running context you experienced the bug/problem in -->
17 * Installation method (e.g. `pip`, `apt-get`, `git clone` or `zip`/`tar.gz`):
18 * Client OS (e.g. `Microsoft Windows 10`)
19 * Program version (see banner):
20 * Relevant console output (if any):
21 * Exception traceback (if any):
+97
-127
README.md less more
66 [![License](https://img.shields.io/badge/license-MIT-_red.svg)](https://opensource.org/licenses/MIT)
77 [![Go Report Card](https://goreportcard.com/badge/github.com/projectdiscovery/subfinder)](https://goreportcard.com/report/github.com/projectdiscovery/subfinder)
88 [![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/projectdiscovery/subfinder/issues)
9 [![GitHub Release](https://img.shields.io/github/release/projectdiscovery/subfinder)](https://github.com/projectdiscovery/subfinder/releases)
10 [![Follow on Twitter](https://img.shields.io/twitter/follow/pdiscoveryio.svg?logo=twitter)](https://twitter.com/pdiscoveryio)
11 [![Docker Images](https://img.shields.io/docker/pulls/projectdiscovery/subfinder.svg)](https://hub.docker.com/r/projectdiscovery/subfinder)
12 [![Chat on Discord](https://img.shields.io/discord/695645237418131507.svg?logo=discord)](https://discord.gg/KECAGdH)
13
914
1015
1116 subfinder is a subdomain discovery tool that discovers valid subdomains for websites by using passive online sources. It has a simple modular architecture and is optimized for speed. subfinder is built for doing one thing only - passive subdomain enumeration, and it does that very well.
3742
3843 - Simple and modular code base making it easy to contribute.
3944 - Fast And Powerful Resolution and wildcard elimination module
40 - **Curated** passive sources to maximize results (26 Sources as of now)
45 - **Curated** passive sources to maximize results (35 Sources as of now)
4146 - Multiple Output formats supported (Json, File, Stdout)
4247 - Optimized for speed, very fast and **lightweight** on resources
4348 - **Stdin** and **stdout** support for integrating in workflows
4550
4651 # Usage
4752
48 ```bash
53 ```sh
4954 subfinder -h
5055 ```
5156 This will display help for the tool. Here are all the switches it supports.
5257
5358 | Flag | Description | Example |
5459 |------|-------------|---------|
55 | -cd | Upload results to the Chaos API (api-key required) | subfinder -d uber.com -cd |
56 | -config string | Configuration file for API Keys, etc | subfinder -config config.yaml |
57 | -d | Domain to find subdomains for | subfinder -d uber.com |
58 | -dL | File containing list of domains to enumerate | subfinder -dL hackerone-hosts.txt |
59 | -exclude-sources | List of sources to exclude from enumeration | subfinder -exclude-sources archiveis |
60 | -max-time | Minutes to wait for enumeration results (default 10) | subfinder -max-time 1 |
61 | -nC | Don't Use colors in output | subfinder -nC |
62 | -nW | Remove Wildcard & Dead Subdomains from output | subfinder -nW |
63 | -ls | List all available sources | subfinder -ls |
64 | -o | File to write output to (optional) | subfinder -o output.txt |
65 | -oD | Directory to write enumeration results to (optional) | subfinder -oD ~/outputs |
60 | -all | Use all sources (slow) for enumeration | subfinder -d uber.com -all |
61 | -cd | Upload results to the Chaos API (api-key required) | subfinder -d uber.com -cd |
62 | -config string | Configuration file for API Keys, etc | subfinder -config config.yaml |
63 | -d | Domain to find subdomains for | subfinder -d uber.com |
64 | -dL | File containing list of domains to enumerate | subfinder -dL hackerone-hosts.txt |
65 | -exclude-sources | List of sources to exclude from enumeration | subfinder -exclude-sources archiveis |
66 | -max-time | Minutes to wait for enumeration results (default 10) | subfinder -max-time 1 |
67 | -nC | Don't Use colors in output | subfinder -nC |
68 | -nW | Remove Wildcard & Dead Subdomains from output | subfinder -nW |
69 | -ls | List all available sources | subfinder -ls |
70 | -o | File to write output to (optional) | subfinder -o output.txt |
71 | -oD | Directory to write enumeration results to (optional) | subfinder -oD ~/outputs |
6672 | -oI | Write output in Host,IP format | subfinder -oI |
6773 | -oJ | Write output in JSON lines Format | subfinder -oJ |
68 | -r | Comma-separated list of resolvers to use | subfinder -r 1.1.1.1,1.0.0.1 |
74 | -r | Comma-separated list of resolvers to use | subfinder -r 1.1.1.1,1.0.0.1 |
6975 | -rL | Text file containing list of resolvers to use | subfinder -rL resolvers.txt
70 | -silent | Show only subdomains in output | subfinder -silent |
71 | -sources | Comma separated list of sources to use | subfinder -sources shodan,censys |
72 | -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 |
73 | -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 |
74 | -v | Show Verbose output | subfinder -v |
75 | -version | Show current program version | subfinder -version |
76
76 | -recursive | Enumeration recursive subdomains | subfinder -d news.yahoo.com -recursive
77 | -silent | Show only subdomains in output | subfinder -silent |
78 | -sources | Comma separated list of sources to use | subfinder -sources shodan,censys |
79 | -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 |
80 | -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 |
81 | -v | Show Verbose output | subfinder -v |
82 | -version | Show current program version | subfinder -version |
83
7784
7885 # Installation Instructions
7986
8188
8289 The installation is easy. You can download the pre-built binaries for different platforms from the [releases](https://github.com/projectdiscovery/subfinder/releases/) page. Extract them using tar, move it to your `$PATH` and you're ready to go.
8390
84 ```bash
85 > tar -xzvf subfinder-linux-amd64.tar.gz
86 > mv subfinder /usr/local/local/bin/
87 > subfinder -h
91 ```sh
92 ▶ # download release from https://github.com/projectdiscovery/subfinder/releases/
93 ▶ tar -xzvf subfinder-linux-amd64.tar.gz
94 ▶ mv subfinder /usr/local/bin/
95 ▶ subfinder -h
8896 ```
8997
9098 ### From Source
9199
92 subfinder requires go1.13+ to install successfully. Run the following command to get the repo -
93
94 ```bash
95 GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/cmd/subfinder
100 subfinder requires **go1.14+** to install successfully. Run the following command to get the repo -
101
102 ```sh
103 GO111MODULE=on go get -u -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder
96104 ```
97105
98106 ### From Github
99107
100 ```bash
108 ```sh
101109 git clone https://github.com/projectdiscovery/subfinder.git
102 cd subfinder/cmd/subfinder
110 cd subfinder/v2/cmd/subfinder
103111 go build .
104112 mv subfinder /usr/local/bin/
105113 subfinder -h
106114 ```
107115
108 ### Upgrading
109 If you wish to upgrade the package you can use:
110
111 ```bash
112 GO111MODULE=on go get -u -v github.com/projectdiscovery/subfinder/cmd/subfinder
113 ```
114
115116 ## Post Installation Instructions
116117
117118 Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key:
118119
119 - [Virustotal](https://www.virustotal.com)
120 - [Passivetotal](http://passivetotal.org)
121 - [SecurityTrails](http://securitytrails.com)
120 - [Binaryedge](https://binaryedge.io)
121 - [Certspotter](https://sslmate.com/certspotter/api/)
122122 - [Censys](https://censys.io)
123 - [Binaryedge](https://binaryedge.io)
124 - [Shodan](https://shodan.io)
125 - [URLScan](https://urlscan.io)
126123 - [Chaos](https://chaos.projectdiscovery.io)
127 - [Spyse](https://spyse.com)
128124 - [DnsDB](https://api.dnsdb.info)
129 - [Zoomeye](https://www.zoomeye.org)
130125 - [Github](https://github.com)
131126 - [Intelx](https://intelx.io)
127 - [Passivetotal](http://passivetotal.org)
128 - [Recon.dev](https://recon.dev)
129 - [Robtex](https://www.robtex.com/api/)
130 - [SecurityTrails](http://securitytrails.com)
131 - [Shodan](https://shodan.io)
132 - [Spyse](https://spyse.com)
133 - [Threatbook](https://threatbook.cn/api)
134 - [Virustotal](https://www.virustotal.com)
135 - [Zoomeye](https://www.zoomeye.org)
132136
133137 Theses values are stored in the `$HOME/.config/subfinder/config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys can be specified for each of these services from which one of them will be used for enumeration.
134138
135139 For sources that require multiple keys, namely `Censys`, `Passivetotal`, they can be added by separating them via a colon (:).
136140
137 An example config file -
141 An example config file -
138142
139143 ```yaml
140144 resolvers:
152156 censys:
153157 - ac244e2f-b635-4581-878a-33f4e79a2c13:dd510d6e-1b6e-4655-83f6-f347b363def9
154158 certspotter: []
155 passivetotal:
159 passivetotal:
156160 - [email protected]:sample_password
157161 securitytrails: []
158162 shodan:
165169 # Running Subfinder
166170
167171 To run the tool on a target, just use the following command.
168 ```bash
169 > subfinder -d freelancer.com
172 ```sh
173 ▶ subfinder -d freelancer.com
170174 ```
171175
172176 This will run the tool against freelancer.com. There are a number of configuration options that you can pass along with this command. The verbose switch (-v) can be used to display verbose information.
173177
174 ```bash
175 [CERTSPOTTER] www.fi.freelancer.com
176 [DNSDUMPSTER] hosting.freelancer.com
177 [DNSDUMPSTER] support.freelancer.com
178 [DNSDUMPSTER] accounts.freelancer.com
179 [DNSDUMPSTER] phabricator.freelancer.com
180 [DNSDUMPSTER] cdn1.freelancer.com
181 [DNSDUMPSTER] t1.freelancer.com
182 [DNSDUMPSTER] wdc.t1.freelancer.com
183 [DNSDUMPSTER] dal.t1.freelancer.com
178 ```
179 [threatcrowd] ns1.hosting.freelancer.com
180 [threatcrowd] ns2.hosting.freelancer.com
181 [threatcrowd] flash.freelancer.com
182 [threatcrowd] auth.freelancer.com
183 [chaos] alertmanager.accounts.freelancer.com
184 [chaos] analytics01.freelancer.com
185 [chaos] apidocs.freelancer.com
186 [chaos] brains.freelancer.com
187 [chaos] consul.accounts.freelancer.com
184188 ```
185189
186190 The `-silent` switch can be used to show only subdomains found without any other info.
188192
189193 The `-o` command can be used to specify an output file.
190194
191 ```bash
192 > subfinder -d freelancer.com -o output.txt
195 ```sh
196 ▶ subfinder -d freelancer.com -o output.txt
193197 ```
194198
195199 To run the tool on a list of domains, `-dL` option can be used. This requires a directory to write the output files. Subdomains for each domain from the list are written in a text file in the directory specified by the `-oD` flag with their name being the domain name.
196200
197 ```bash
198 > cat domains.txt
201 ```sh
202 ▶ cat domains.txt
199203 hackerone.com
200204 google.com
201205
202 > subfinder -dL domains.txt -oD ~/path/to/output
203 > ls ~/path/to/output
206 ▶ subfinder -dL domains.txt -oD ~/path/to/output
207 ▶ ls ~/path/to/output
204208
205209 hackerone.com.txt
206210 google.com.txt
207211 ```
208212
209 If you want to save results to a single file while using a domain list, specify the `-o` flag with the name of the output file.
210
211
212 ```bash
213 > cat domains.txt
214 hackerone.com
215 google.com
216
217 > subfinder -dL domains.txt -o ~/path/to/output.txt
218 > ls ~/path/to/
219
220 output.txt
221 ```
222
223 If you want upload your data to chaos dataset, you can use `-cd` flag with your scan, chaos will resolve all the input and add valid subdomains to public dataset, which you can access on the go using [chaos-client](https://github.com/projectdiscovery/chaos-client)
224
225 ```bash
226 > subfinder -d hackerone.com -cd
227
228 root@b0x:~# subfinder -d hackerone.com -cd
229
230 www.hackerone.com
231 api.hackerone.com
232 go.hackerone.com
233 hackerone.com
234 staging.hackerone.com
235 [INF] Input processed successfully and subdomains with valid records will be updated to chaos dataset.
236 ```
237
238 You can also get output in json format using `-oJ` switch. This switch saves the output in the JSON lines format.
213 You can also get output in json format using `-oJ` switch. This switch saves the output in the JSON lines format.
239214
240215 If you use the JSON format, or the `Host:IP` format, then it becomes mandatory for you to use the **-nW** format as resolving is essential for these output format. By default, resolving the found subdomains is disabled.
241216
242 ```bash
243 > subfinder -d hackerone.com -o output.json -oJ -nW
244 > cat output.json
217 ```sh
218 ▶ subfinder -d hackerone.com -o output.json -oJ -nW
219 ▶ cat output.json
245220
246221 {"host":"www.hackerone.com","ip":"104.16.99.52"}
247222 {"host":"mta-sts.hackerone.com","ip":"185.199.108.153"}
249224 {"host":"mta-sts.managed.hackerone.com","ip":"185.199.110.153"}
250225 ```
251226
252 You can specify custom resolvers too.
253 ```bash
254 > subfinder -d freelancer.com -o result.txt -nW -v -r 8.8.8.8,1.1.1.1
255 > subfinder -d freelancer.com -o result.txt -nW -v -rL resolvers.txt
256 ```
257
258 **The new highlight of this release is the addition of stdin/stdout features.** Now, domains can be piped to subfinder and enumeration can be ran on them. For example -
259
260 ```bash
261 > echo hackerone.com | subfinder -v
262 > cat targets.txt | subfinder -v
227
228 **The new highlight of this release is the addition of stdin/stdout features.** Now, domains can be piped to subfinder and enumeration can be ran on them. For example -
229
230 ```sh
231 ▶ echo hackerone.com | subfinder
232 ▶ cat targets.txt | subfinder
263233 ```
264234
265235 The subdomains discovered can be piped to other tools too. For example, you can pipe the subdomains discovered by subfinder to httpx [httpx](https://github.com/projectdiscovery/httpx) which will then find running http servers on the host.
266236
267 ```bash
268 > echo hackerone.com | subfinder -silent | httpx -silent
237 ```sh
238 ▶ echo hackerone.com | subfinder -silent | httpx -silent
269239
270240 http://hackerone.com
271241 http://www.hackerone.com
277247
278248 ## Running in a Docker Container
279249
280 You can use the official dockerhub image at [subfinder](https://hub.docker.com/r/projectdiscovery/subfinder). Simply run -
281
282 ```bash
283 > docker pull projectdiscovery/subfinder
250 You can use the official dockerhub image at [subfinder](https://hub.docker.com/r/projectdiscovery/subfinder). Simply run -
251
252 ```sh
253 ▶ docker pull projectdiscovery/subfinder
284254 ```
285255
286256 The above command will pull the latest tagged release from the dockerhub repository.
289259
290260 - Clone the repo using `git clone https://github.com/projectdiscovery/subfinder.git`
291261 - Build your docker container
292 ```bash
262 ```sh
293263 docker build -t projectdiscovery/subfinder .
294264 ```
295265
296 - After building the container using either way, run the following -
297 ```bash
266 - After building the container using either way, run the following -
267 ```sh
298268 docker run -it projectdiscovery/subfinder
299269 ```
300 > The above command is the same as running `-h`
270 ▶ The above command is the same as running `-h`
301271
302272 If you are using docker, you need to first create your directory structure holding subfinder configuration file. After modifying the default config.yaml file, you can run:
303273
304 ```bash
305 > mkdir -p $HOME/.config/subfinder
306 > cp config.yaml $HOME/.config/subfinder/config.yaml
307 > nano $HOME/.config/subfinder/config.yaml
274 ```sh
275 ▶ mkdir -p $HOME/.config/subfinder
276 ▶ cp config.yaml $HOME/.config/subfinder/config.yaml
277 ▶ nano $HOME/.config/subfinder/config.yaml
308278 ```
309279
310280 After that, you can pass it as a volume using the following sample command.
311 ```bash
312 > docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it projectdiscovery/subfinder -d freelancer.com
281 ```sh
282 ▶ docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it projectdiscovery/subfinder -d freelancer.com
313283 ```
314284
315285 For example, this runs the tool against uber.com and output the results to your host file system:
316 ```bash
286 ```sh
317287 docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it projectdiscovery/subfinder -d uber.com > uber.com.txt
318288 ```
319289
+5
-4
THANKS.md less more
33
44 - All the contributors at [CONTRIBUTORS](https://github.com/projectdiscovery/subfinder/graphs/contributors) who made subfinder what it is.
55
6 We'd like to thank some additional amazing people, wo contributed a lot in subfinder's journey -
6 We'd like to thank some additional amazing people, who contributed a lot in subfinder's journey -
77
8 - @infosec-au - Donating to the project
9 - @codingo - Initial work on the project, managing it, lot of work!
10 - @picatz - Improving the structure of the project a lot. New ideas!⏎
8 - [@vzamanillo](https://github.com/vzamanillo) - For adding multiple features and overall project improvements.
9 - [@infosec-au](https://github.com/infosec-au) - Donating to the project.
10 - [@codingo](https://github.com/codingo) - Initial work on the project, managing it, lot of work!
11 - [@picatz](https://github.com/picatz) - Improving the structure of the project a lot. New ideas!⏎
+0
-21
cmd/subfinder/main.go less more
0 package main
1
2 import (
3 "github.com/projectdiscovery/gologger"
4 "github.com/projectdiscovery/subfinder/pkg/runner"
5 )
6
7 func main() {
8 // Parse the command line flags and read config files
9 options := runner.ParseOptions()
10
11 runner, err := runner.NewRunner(options)
12 if err != nil {
13 gologger.Fatalf("Could not create runner: %s\n", err)
14 }
15
16 err = runner.RunEnumeration()
17 if err != nil {
18 gologger.Fatalf("Could not run enumeration: %s\n", err)
19 }
20 }
+0
-68
config.yaml less more
0 resolvers:
1 - 1.1.1.1
2 - 1.0.0.1
3 - 8.8.8.8
4 - 8.8.4.4
5 - 9.9.9.9
6 - 9.9.9.10
7 - 77.88.8.8
8 - 77.88.8.1
9 - 208.67.222.222
10 - 208.67.220.220
11 sources:
12 - alienvault
13 - archiveis
14 - binaryedge
15 - bufferover
16 - censys
17 - certspotter
18 - certspotterold
19 - commoncrawl
20 - crtsh
21 - dnsdumpster
22 - dnsdb
23 - entrust
24 - github
25 - googleter
26 - hackertarget
27 - intelx
28 - ipv4info
29 - passivetotal
30 - rapiddns
31 - securitytrails
32 - shodan
33 - sitedossier
34 - sublist3r
35 - spyse
36 - threatcrowd
37 - threatminer
38 - urlscan
39 - virustotal
40 - waybackarchive
41 - zoomeye
42 censys:
43 - <key-here>
44 binaryedge:
45 - <key-here>
46 certspotter:
47 - <key-here>
48 github:
49 - <token-here>
50 intelx:
51 - <public.intelx.io:key-here>
52 passivetotal:
53 - <email:key-here>
54 securitytrails:
55 - <key-here>
56 virustotal:
57 - <key-here>
58 urlscan:
59 - <key-here>
60 chaos:
61 - <key-here>
62 spyse:
63 - <key-here>
64 shodan:
65 - <key-here>
66 dnsdb:
67 - <key-here>
+0
-15
go.mod less more
0 module github.com/projectdiscovery/subfinder
1
2 go 1.14
3
4 require (
5 github.com/json-iterator/go v1.1.9
6 github.com/lib/pq v1.6.0
7 github.com/m-mizutani/urlscan-go v1.0.0
8 github.com/miekg/dns v1.1.29
9 github.com/pkg/errors v0.9.1
10 github.com/projectdiscovery/gologger v1.0.0
11 github.com/rs/xid v1.2.1
12 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
13 gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c
14 )
+0
-87
go.sum less more
0 github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5/go.mod h1:976q2ETgjT2snVCf2ZaBnyBbVoPERGjUz+0sofzEfro=
1 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
2 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
3 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
4 github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
5 github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
6 github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
7 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
8 github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
9 github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
10 github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
11 github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
12 github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
13 github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
14 github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=
15 github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
16 github.com/jcmturner/gokrb5/v8 v8.2.0 h1:lzPl/30ZLkTveYsYZPKMcgXc8MbnE6RsTd4F9KgiLtk=
17 github.com/jcmturner/gokrb5/v8 v8.2.0/go.mod h1:T1hnNppQsBtxW0tCHMHTkAt8n/sABdzZgZdoFrZaZNM=
18 github.com/jcmturner/rpc/v2 v2.0.2 h1:gMB4IwRXYsWw4Bc6o/az2HJgFUA1ffSh90i26ZJ6Xl0=
19 github.com/jcmturner/rpc/v2 v2.0.2/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
20 github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
21 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
22 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
23 github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg=
24 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
25 github.com/lib/pq v1.6.0 h1:I5DPxhYJChW9KYc66se+oKFFQX6VuQrKiprsX6ivRZc=
26 github.com/lib/pq v1.6.0/go.mod h1:4vXEAYvW1fRQ2/FhZ78H73A60MHw1geSm145z2mdY1g=
27 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
28 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
29 github.com/m-mizutani/urlscan-go v1.0.0 h1:+fTiSRCQXdy3EM1BgO5gmAHFWbccTDdoEKy9Fa7m9xo=
30 github.com/m-mizutani/urlscan-go v1.0.0/go.mod h1:ppEBT0e/xv0bPcVWKev4cYG7Ey8933JsOzEzovxGMjI=
31 github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
32 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
33 github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg=
34 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
35 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
36 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
37 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
38 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
39 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
40 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
41 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
42 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
43 github.com/projectdiscovery/gologger v1.0.0 h1:XAQ8kHeVKXMjY4rLGh7eT5+oHU077BNEvs7X6n+vu1s=
44 github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
45 github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
46 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
47 github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME=
48 github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
49 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
50 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
51 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
52 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
53 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
54 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
55 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
56 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
57 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
58 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
59 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
60 golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
61 golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 h1:QmwruyY+bKbDDL0BaglrbZABEali68eoMFhTZpCjYVA=
62 golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
63 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
64 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
65 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
66 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
67 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA=
68 golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
69 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
70 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
71 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
72 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
73 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M=
74 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
75 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
76 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
77 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
78 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
79 gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo=
80 gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q=
81 gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4=
82 gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
83 gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
84 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
85 gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo=
86 gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+0
-4
pkg/passive/doc.go less more
0 // Package passive provides capability for doing passive subdomain
1 // enumeration on targets.
2 package passive
3
+0
-58
pkg/passive/passive.go less more
0 package passive
1
2 import (
3 "context"
4 "fmt"
5 "sync"
6 "time"
7
8 "github.com/projectdiscovery/gologger"
9 "github.com/projectdiscovery/subfinder/pkg/subscraping"
10 )
11
12 // EnumerateSubdomains enumerates all the subdomains for a given domain
13 func (a *Agent) EnumerateSubdomains(domain string, keys subscraping.Keys, timeout int, maxEnumTime time.Duration) chan subscraping.Result {
14 results := make(chan subscraping.Result)
15
16 go func() {
17 session, err := subscraping.NewSession(domain, keys, timeout)
18 if err != nil {
19 results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)}
20 }
21
22 ctx, cancel := context.WithTimeout(context.Background(), maxEnumTime)
23
24 timeTaken := make(map[string]string)
25 timeTakenMutex := &sync.Mutex{}
26
27 wg := &sync.WaitGroup{}
28 // Run each source in parallel on the target domain
29 for source, runner := range a.sources {
30 wg.Add(1)
31
32 now := time.Now()
33 go func(source string, runner subscraping.Source) {
34 for resp := range runner.Run(ctx, domain, session) {
35 results <- resp
36 }
37
38 duration := time.Now().Sub(now)
39 timeTakenMutex.Lock()
40 timeTaken[source] = fmt.Sprintf("Source took %s for enumeration\n", duration)
41 timeTakenMutex.Unlock()
42
43 wg.Done()
44 }(source, runner)
45 }
46 wg.Wait()
47
48 for source, data := range timeTaken {
49 gologger.Verbosef(data, source)
50 }
51
52 close(results)
53 cancel()
54 }()
55
56 return results
57 }
+0
-158
pkg/passive/sources.go less more
0 package passive
1
2 import (
3 "github.com/projectdiscovery/subfinder/pkg/subscraping"
4 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/alienvault"
5 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/archiveis"
6 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/binaryedge"
7 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/bufferover"
8 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/censys"
9 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/certspotter"
10 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/certspotterold"
11 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/commoncrawl"
12 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/crtsh"
13 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/dnsdb"
14 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/dnsdumpster"
15 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/entrust"
16 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/github"
17 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/hackertarget"
18 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/intelx"
19 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/ipv4info"
20 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/passivetotal"
21 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/rapiddns"
22 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/securitytrails"
23 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/shodan"
24 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/sitedossier"
25 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/spyse"
26 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/sublist3r"
27 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/threatcrowd"
28 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/threatminer"
29 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/urlscan"
30 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/virustotal"
31 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/waybackarchive"
32 "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/zoomeye"
33 )
34
35 // DefaultSources contains the list of sources used by default
36 var DefaultSources = []string{
37 "alienvault",
38 "archiveis",
39 "binaryedge",
40 "bufferover",
41 "censys",
42 "certspotter",
43 "certspotterold",
44 "commoncrawl",
45 "crtsh",
46 "dnsdumpster",
47 "dnsdb",
48 "entrust",
49 "github",
50 "hackertarget",
51 "ipv4info",
52 "intelx",
53 "passivetotal",
54 "rapiddns",
55 "securitytrails",
56 "shodan",
57 "sitedossier",
58 "spyse",
59 "sublist3r",
60 "threatcrowd",
61 "threatminer",
62 "urlscan",
63 "virustotal",
64 "waybackarchive",
65 "zoomeye",
66 }
67
68 // Agent is a struct for running passive subdomain enumeration
69 // against a given host. It wraps subscraping package and provides
70 // a layer to build upon.
71 type Agent struct {
72 sources map[string]subscraping.Source
73 }
74
75 // New creates a new agent for passive subdomain discovery
76 func New(sources []string, exclusions []string) *Agent {
77 // Create the agent, insert the sources and remove the excluded sources
78 agent := &Agent{sources: make(map[string]subscraping.Source)}
79
80 agent.addSources(sources)
81 agent.removeSources(exclusions)
82
83 return agent
84 }
85
86 // addSources adds the given list of sources to the source array
87 func (a *Agent) addSources(sources []string) {
88 for _, source := range sources {
89 switch source {
90 case "alienvault":
91 a.sources[source] = &alienvault.Source{}
92 case "archiveis":
93 a.sources[source] = &archiveis.Source{}
94 case "binaryedge":
95 a.sources[source] = &binaryedge.Source{}
96 case "bufferover":
97 a.sources[source] = &bufferover.Source{}
98 case "censys":
99 a.sources[source] = &censys.Source{}
100 case "certspotter":
101 a.sources[source] = &certspotter.Source{}
102 case "certspotterold":
103 a.sources[source] = &certspotterold.Source{}
104 case "commoncrawl":
105 a.sources[source] = &commoncrawl.Source{}
106 case "crtsh":
107 a.sources[source] = &crtsh.Source{}
108 case "dnsdumpster":
109 a.sources[source] = &dnsdumpster.Source{}
110 case "dnsdb":
111 a.sources[source] = &dnsdb.Source{}
112 case "entrust":
113 a.sources[source] = &entrust.Source{}
114 case "github":
115 a.sources[source] = &github.Source{}
116 case "hackertarget":
117 a.sources[source] = &hackertarget.Source{}
118 case "ipv4info":
119 a.sources[source] = &ipv4info.Source{}
120 case "intelx":
121 a.sources[source] = &intelx.Source{}
122 case "passivetotal":
123 a.sources[source] = &passivetotal.Source{}
124 case "rapiddns":
125 a.sources[source] = &rapiddns.Source{}
126 case "securitytrails":
127 a.sources[source] = &securitytrails.Source{}
128 case "shodan":
129 a.sources[source] = &shodan.Source{}
130 case "sitedossier":
131 a.sources[source] = &sitedossier.Source{}
132 case "spyse":
133 a.sources[source] = &spyse.Source{}
134 case "sublist3r":
135 a.sources[source] = &sublist3r.Source{}
136 case "threatcrowd":
137 a.sources[source] = &threatcrowd.Source{}
138 case "threatminer":
139 a.sources[source] = &threatminer.Source{}
140 case "urlscan":
141 a.sources[source] = &urlscan.Source{}
142 case "virustotal":
143 a.sources[source] = &virustotal.Source{}
144 case "waybackarchive":
145 a.sources[source] = &waybackarchive.Source{}
146 case "zoomeye":
147 a.sources[source] = &zoomeye.Source{}
148 }
149 }
150 }
151
152 // removeSources deletes the given sources from the source map
153 func (a *Agent) removeSources(sources []string) {
154 for _, source := range sources {
155 delete(a.sources, source)
156 }
157 }
+0
-59
pkg/resolve/client.go less more
0 package resolve
1
2 import (
3 "bufio"
4 "math/rand"
5 "os"
6 "time"
7 )
8
9 // DefaultResolvers contains the default list of resolvers known to be good
10 var DefaultResolvers = []string{
11 "1.1.1.1", // Cloudflare primary
12 "1.0.0.1", // Cloudlfare secondary
13 "8.8.8.8", // Google primary
14 "8.8.4.4", // Google secondary
15 "9.9.9.9", // Quad9 Primary
16 "9.9.9.10", // Quad9 Secondary
17 "77.88.8.8", // Yandex Primary
18 "77.88.8.1", // Yandex Secondary
19 "208.67.222.222", // OpenDNS Primary
20 "208.67.220.220", // OpenDNS Secondary
21 }
22
23 // Resolver is a struct for resolving DNS names
24 type Resolver struct {
25 resolvers []string
26 rand *rand.Rand
27 }
28
29 // New creates a new resolver struct with the default resolvers
30 func New() *Resolver {
31 return &Resolver{
32 resolvers: []string{},
33 rand: rand.New(rand.NewSource(time.Now().UnixNano())),
34 }
35 }
36
37 // AppendResolversFromFile appends the resolvers read from a file to the list of resolvers
38 func (r *Resolver) AppendResolversFromFile(file string) error {
39 f, err := os.Open(file)
40 if err != nil {
41 return err
42 }
43 scanner := bufio.NewScanner(f)
44 for scanner.Scan() {
45 text := scanner.Text()
46 if text == "" {
47 continue
48 }
49 r.resolvers = append(r.resolvers, text)
50 }
51 f.Close()
52 return scanner.Err()
53 }
54
55 // AppendResolversFromSlice appends the slice to the list of resolvers
56 func (r *Resolver) AppendResolversFromSlice(list []string) {
57 r.resolvers = append(r.resolvers, list...)
58 }
+0
-3
pkg/resolve/doc.go less more
0 // Package resolve is used to handle resolving records
1 // It also handles wildcard subdomains and rotating resolvers.
2 package resolve
+0
-150
pkg/resolve/resolve.go less more
0 package resolve
1
2 import (
3 "sync"
4
5 "github.com/miekg/dns"
6 "github.com/rs/xid"
7 )
8
9 const (
10 maxResolveRetries = 5
11 maxWildcardChecks = 3
12 )
13
14 // ResolutionPool is a pool of resolvers created for resolving subdomains
15 // for a given host.
16 type ResolutionPool struct {
17 *Resolver
18 Tasks chan string
19 Results chan Result
20 wg *sync.WaitGroup
21 removeWildcard bool
22
23 wildcardIPs map[string]struct{}
24 }
25
26 // Result contains the result for a host resolution
27 type Result struct {
28 Type ResultType
29 Host string
30 IP string
31 Error error
32 }
33
34 // ResultType is the type of result found
35 type ResultType int
36
37 // Types of data result can return
38 const (
39 Subdomain ResultType = iota
40 Error
41 )
42
43 // NewResolutionPool creates a pool of resolvers for resolving subdomains of a given domain
44 func (r *Resolver) NewResolutionPool(workers int, removeWildcard bool) *ResolutionPool {
45 resolutionPool := &ResolutionPool{
46 Resolver: r,
47 Tasks: make(chan string),
48 Results: make(chan Result),
49 wg: &sync.WaitGroup{},
50 removeWildcard: removeWildcard,
51 wildcardIPs: make(map[string]struct{}),
52 }
53
54 go func() {
55 for i := 0; i < workers; i++ {
56 resolutionPool.wg.Add(1)
57 go resolutionPool.resolveWorker()
58 }
59 resolutionPool.wg.Wait()
60 close(resolutionPool.Results)
61 }()
62
63 return resolutionPool
64 }
65
66 // InitWildcards inits the wildcard ips array
67 func (r *ResolutionPool) InitWildcards(domain string) error {
68 for i := 0; i < maxWildcardChecks; i++ {
69 uid := xid.New().String()
70
71 hosts, err := r.getARecords(uid + "." + domain)
72 if err != nil {
73 return err
74 }
75
76 // Append all wildcard ips found for domains
77 for _, host := range hosts {
78 r.wildcardIPs[host] = struct{}{}
79 }
80 }
81 return nil
82 }
83
84 func (r *ResolutionPool) resolveWorker() {
85 for task := range r.Tasks {
86 if !r.removeWildcard {
87 r.Results <- Result{Type: Subdomain, Host: task, IP: ""}
88 continue
89 }
90
91 hosts, err := r.getARecords(task)
92 if err != nil {
93 r.Results <- Result{Type: Error, Error: err}
94 continue
95 }
96
97 if len(hosts) == 0 {
98 continue
99 }
100
101 for _, host := range hosts {
102 // Ignore the host if it exists in wildcard ips map
103 if _, ok := r.wildcardIPs[host]; ok {
104 continue
105 }
106 }
107
108 r.Results <- Result{Type: Subdomain, Host: task, IP: hosts[0]}
109 }
110 r.wg.Done()
111 }
112
113 // getARecords gets all the A records for a given host
114 func (r *ResolutionPool) getARecords(host string) ([]string, error) {
115 var iteration int
116
117 m := new(dns.Msg)
118 m.Id = dns.Id()
119 m.RecursionDesired = true
120 m.Question = make([]dns.Question, 1)
121 m.Question[0] = dns.Question{
122 Name: dns.Fqdn(host),
123 Qtype: dns.TypeA,
124 Qclass: dns.ClassINET,
125 }
126 exchange:
127 iteration++
128 in, err := dns.Exchange(m, r.resolvers[r.rand.Intn(len(r.resolvers))]+":53")
129 if err != nil {
130 // Retry in case of I/O error
131 if iteration <= maxResolveRetries {
132 goto exchange
133 }
134 return nil, err
135 }
136 // Ignore the error in case we have bad result
137 if in != nil && in.Rcode != dns.RcodeSuccess {
138 return nil, nil
139 }
140
141 var hosts []string
142 for _, record := range in.Answer {
143 if t, ok := record.(*dns.A); ok {
144 hosts = append(hosts, t.A.String())
145 }
146 }
147
148 return hosts, nil
149 }
+0
-57
pkg/runner/banners.go less more
0 package runner
1
2 import (
3 "github.com/projectdiscovery/gologger"
4 "github.com/projectdiscovery/subfinder/pkg/passive"
5 "github.com/projectdiscovery/subfinder/pkg/resolve"
6 )
7
8 const banner = `
9 _ __ _ _
10 ____ _| |__ / _(_)_ _ __| |___ _ _
11 (_-< || | '_ \ _| | ' \/ _ / -_) '_|
12 /__/\_,_|_.__/_| |_|_||_\__,_\___|_| v2
13 `
14
15 // Version is the current version of subfinder
16 const Version = `2.3.8`
17
18 // showBanner is used to show the banner to the user
19 func showBanner() {
20 gologger.Printf("%s\n", banner)
21 gologger.Printf("\t\tprojectdiscovery.io\n\n")
22
23 gologger.Labelf("Use with caution. You are responsible for your actions\n")
24 gologger.Labelf("Developers assume no liability and are not responsible for any misuse or damage.\n")
25 gologger.Labelf("By using subfinder, you also agree to the terms of the APIs used.\n\n")
26 }
27
28 // normalRunTasks runs the normal startup tasks
29 func (options *Options) normalRunTasks() {
30 configFile, err := UnmarshalRead(options.ConfigFile)
31 if err != nil {
32 gologger.Fatalf("Could not read configuration file %s: %s\n", options.ConfigFile, err)
33 }
34 options.YAMLConfig = configFile
35 }
36
37 // firstRunTasks runs some housekeeping tasks done
38 // when the program is ran for the first time
39 func (options *Options) firstRunTasks() {
40 // Create the configuration file and display information
41 // about it to the user.
42 config := ConfigFile{
43 // Use the default list of resolvers by marshalling it to the config
44 Resolvers: resolve.DefaultResolvers,
45 // Use the default list of passive sources
46 Sources: passive.DefaultSources,
47 }
48
49 err := config.MarshalWrite(options.ConfigFile)
50 if err != nil {
51 gologger.Fatalf("Could not write configuration file to %s: %s\n", options.ConfigFile, err)
52 }
53 options.YAMLConfig = config
54
55 gologger.Infof("Configuration file saved to %s\n", options.ConfigFile)
56 }
+0
-169
pkg/runner/config.go less more
0 package runner
1
2 import (
3 "math/rand"
4 "os"
5 "strings"
6 "time"
7
8 "github.com/projectdiscovery/subfinder/pkg/subscraping"
9 "gopkg.in/yaml.v3"
10 )
11
12 // ConfigFile contains the fields stored in the configuration file
13 type ConfigFile struct {
14 // Resolvers contains the list of resolvers to use while resolving
15 Resolvers []string `yaml:"resolvers,omitempty"`
16 // Sources contains a list of sources to use for enumeration
17 Sources []string `yaml:"sources,omitempty"`
18 // ExcludeSources contains the sources to not include in the enumeration process
19 ExcludeSources []string `yaml:"exclude-sources,omitempty"`
20 // API keys for different sources
21 Binaryedge []string `yaml:"binaryedge"`
22 Censys []string `yaml:"censys"`
23 Certspotter []string `yaml:"certspotter"`
24 Chaos []string `yaml:"chaos"`
25 DNSDB []string `yaml:"dnsdb"`
26 GitHub []string `yaml:"github"`
27 IntelX []string `yaml:"intelx"`
28 PassiveTotal []string `yaml:"passivetotal"`
29 SecurityTrails []string `yaml:"securitytrails"`
30 Shodan []string `yaml:"shodan"`
31 Spyse []string `yaml:"spyse"`
32 URLScan []string `yaml:"urlscan"`
33 Virustotal []string `yaml:"virustotal"`
34 ZoomEye []string `yaml:"zoomeye"`
35 }
36
37 // GetConfigDirectory gets the subfinder config directory for a user
38 func GetConfigDirectory() (string, error) {
39 // Seed the random number generator
40 rand.Seed(time.Now().UnixNano())
41
42 var config string
43
44 directory, err := os.UserHomeDir()
45 if err != nil {
46 return config, err
47 }
48 config = directory + "/.config/subfinder"
49 // Create All directory for subfinder even if they exist
50 os.MkdirAll(config, os.ModePerm)
51
52 return config, nil
53 }
54
55 // CheckConfigExists checks if the config file exists in the given path
56 func CheckConfigExists(configPath string) bool {
57 if _, err := os.Stat(configPath); err == nil {
58 return true
59 } else if os.IsNotExist(err) {
60 return false
61 }
62 return false
63 }
64
65 // MarshalWrite writes the marshalled yaml config to disk
66 func (c ConfigFile) MarshalWrite(file string) error {
67 f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE, 0755)
68 if err != nil {
69 return err
70 }
71
72 // Indent the spaces too
73 enc := yaml.NewEncoder(f)
74 enc.SetIndent(4)
75 err = enc.Encode(&c)
76 f.Close()
77 return err
78 }
79
80 // UnmarshalRead reads the unmarshalled config yaml file from disk
81 func UnmarshalRead(file string) (ConfigFile, error) {
82 config := ConfigFile{}
83
84 f, err := os.Open(file)
85 if err != nil {
86 return config, err
87 }
88 err = yaml.NewDecoder(f).Decode(&config)
89 f.Close()
90 return config, err
91 }
92
93 // GetKeys gets the API keys from config file and creates a Keys struct
94 // We use random selection of api keys from the list of keys supplied.
95 // Keys that require 2 options are separated by colon (:).
96 func (c ConfigFile) GetKeys() subscraping.Keys {
97 keys := subscraping.Keys{}
98
99 if len(c.Binaryedge) > 0 {
100 keys.Binaryedge = c.Binaryedge[rand.Intn(len(c.Binaryedge))]
101 }
102
103 if len(c.Censys) > 0 {
104 censysKeys := c.Censys[rand.Intn(len(c.Censys))]
105 parts := strings.Split(censysKeys, ":")
106 if len(parts) == 2 {
107 keys.CensysToken = parts[0]
108 keys.CensysSecret = parts[1]
109 }
110 }
111
112 if len(c.Certspotter) > 0 {
113 keys.Certspotter = c.Certspotter[rand.Intn(len(c.Certspotter))]
114 }
115 if len(c.Chaos) > 0 {
116 keys.Chaos = c.Chaos[rand.Intn(len(c.Chaos))]
117 }
118 if (len(c.DNSDB)) > 0 {
119 keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))]
120 }
121 if (len(c.GitHub)) > 0 {
122 keys.GitHub = c.GitHub
123 }
124
125 if len(c.IntelX) > 0 {
126 intelxKeys := c.IntelX[rand.Intn(len(c.IntelX))]
127 parts := strings.Split(intelxKeys, ":")
128 if len(parts) == 2 {
129 keys.IntelXHost = parts[0]
130 keys.IntelXKey = parts[1]
131 }
132 }
133
134 if len(c.PassiveTotal) > 0 {
135 passiveTotalKeys := c.PassiveTotal[rand.Intn(len(c.PassiveTotal))]
136 parts := strings.Split(passiveTotalKeys, ":")
137 if len(parts) == 2 {
138 keys.PassiveTotalUsername = parts[0]
139 keys.PassiveTotalPassword = parts[1]
140 }
141 }
142
143 if len(c.SecurityTrails) > 0 {
144 keys.Securitytrails = c.SecurityTrails[rand.Intn(len(c.SecurityTrails))]
145 }
146 if len(c.Shodan) > 0 {
147 keys.Shodan = c.Shodan[rand.Intn(len(c.Shodan))]
148 }
149 if len(c.Spyse) > 0 {
150 keys.Spyse = c.Spyse[rand.Intn(len(c.Spyse))]
151 }
152 if len(c.URLScan) > 0 {
153 keys.URLScan = c.URLScan[rand.Intn(len(c.URLScan))]
154 }
155 if len(c.Virustotal) > 0 {
156 keys.Virustotal = c.Virustotal[rand.Intn(len(c.Virustotal))]
157 }
158 if len(c.ZoomEye) > 0 {
159 zoomEyeKeys := c.ZoomEye[rand.Intn(len(c.ZoomEye))]
160 parts := strings.Split(zoomEyeKeys, ":")
161 if len(parts) == 2 {
162 keys.ZoomEyeUsername = parts[0]
163 keys.ZoomEyePassword = parts[1]
164 }
165 }
166
167 return keys
168 }
+0
-22
pkg/runner/config_test.go less more
0 package runner
1
2 import (
3 "os"
4 "testing"
5
6 "github.com/stretchr/testify/assert"
7 )
8
9 func TestConfigGetDirectory(t *testing.T) {
10 directory, err := GetConfigDirectory()
11 if err != nil {
12 t.Fatalf("Expected nil got %v while getting home\n", err)
13 }
14 home, err := os.UserHomeDir()
15 if err != nil {
16 t.Fatalf("Expected nil got %v while getting dir\n", err)
17 }
18 config := home + "/.config/subfinder"
19
20 assert.Equal(t, directory, config, "Directory and config should be equal")
21 }
+0
-3
pkg/runner/doc.go less more
0 // Package runner implements the mechanism to drive the
1 // subdomain enumeration process
2 package runner
+0
-180
pkg/runner/enumerate.go less more
0 package runner
1
2 import (
3 "bytes"
4 "os"
5 "strings"
6 "sync"
7 "time"
8
9 "github.com/projectdiscovery/gologger"
10 "github.com/projectdiscovery/subfinder/pkg/resolve"
11 "github.com/projectdiscovery/subfinder/pkg/subscraping"
12 )
13
14 // EnumerateSingleDomain performs subdomain enumeration against a single domain
15 func (r *Runner) EnumerateSingleDomain(domain, output string, append bool) error {
16 gologger.Infof("Enumerating subdomains for %s\n", domain)
17
18 // Get the API keys for sources from the configuration
19 // and also create the active resolving engine for the domain.
20 keys := r.options.YAMLConfig.GetKeys()
21
22 // Check if the user has asked to remove wildcards explicitly.
23 // If yes, create the resolution pool and get the wildcards for the current domain
24 var resolutionPool *resolve.ResolutionPool
25 if r.options.RemoveWildcard {
26 resolutionPool = r.resolverClient.NewResolutionPool(r.options.Threads, r.options.RemoveWildcard)
27 err := resolutionPool.InitWildcards(domain)
28 if err != nil {
29 // Log the error but don't quit.
30 gologger.Warningf("Could not get wildcards for domain %s: %s\n", domain, err)
31 }
32 }
33
34 // Run the passive subdomain enumeration
35 passiveResults := r.passiveAgent.EnumerateSubdomains(domain, keys, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute)
36
37 wg := &sync.WaitGroup{}
38 wg.Add(1)
39 // Create a unique map for filtering duplicate subdomains out
40 uniqueMap := make(map[string]struct{})
41 // Process the results in a separate goroutine
42 go func() {
43 for result := range passiveResults {
44 switch result.Type {
45 case subscraping.Error:
46 gologger.Warningf("Could not run source %s: %s\n", result.Source, result.Error)
47 case subscraping.Subdomain:
48 // Validate the subdomain found and remove wildcards from
49 if !strings.HasSuffix(result.Value, "."+domain) {
50 continue
51 }
52 subdomain := strings.ReplaceAll(strings.ToLower(result.Value), "*.", "")
53
54 // Check if the subdomain is a duplicate. If not,
55 // send the subdomain for resolution.
56 if _, ok := uniqueMap[subdomain]; ok {
57 continue
58 }
59 uniqueMap[subdomain] = struct{}{}
60
61 // Log the verbose message about the found subdomain and send the
62 // host for resolution to the resolution pool
63 gologger.Verbosef("%s\n", result.Source, subdomain)
64
65 // If the user asked to remove wildcard then send on the resolve
66 // queue. Otherwise, if mode is not verbose print the results on
67 // the screen as they are discovered.
68 if r.options.RemoveWildcard {
69 resolutionPool.Tasks <- subdomain
70 }
71
72 if !r.options.Verbose {
73 gologger.Silentf("%s\n", subdomain)
74 }
75 }
76 }
77 // Close the task channel only if wildcards are asked to be removed
78 if r.options.RemoveWildcard {
79 close(resolutionPool.Tasks)
80 }
81 wg.Done()
82 }()
83
84 // If the user asked to remove wildcards, listen from the results
85 // queue and write to the map. At the end, print the found results to the screen
86 foundResults := make(map[string]string)
87 if r.options.RemoveWildcard {
88 // Process the results coming from the resolutions pool
89 for result := range resolutionPool.Results {
90 switch result.Type {
91 case resolve.Error:
92 gologger.Warningf("Could not resolve host: %s\n", result.Error)
93 case resolve.Subdomain:
94 // Add the found subdomain to a map.
95 if _, ok := foundResults[result.Host]; !ok {
96 foundResults[result.Host] = result.IP
97 }
98 }
99 }
100 }
101 wg.Wait()
102
103 // If verbose mode was used, then now print all the
104 // found subdomains on the screen together.
105 if r.options.Verbose {
106 if r.options.RemoveWildcard {
107 for result := range foundResults {
108 gologger.Silentf("%s\n", result)
109 }
110 } else {
111 for result := range uniqueMap {
112 gologger.Silentf("%s\n", result)
113 }
114 }
115 }
116 // In case the user has specified to upload to chaos, write everything to a temporary buffer and upload
117 if r.options.ChaosUpload {
118 var buf = &bytes.Buffer{}
119 err := WriteHostOutput(uniqueMap, buf)
120 // If an error occurs, do not interrupt, continue to check if user specifed an output file
121 if err != nil {
122 gologger.Errorf("Could not prepare results for chaos %s\n", err)
123 } else {
124 // no error in writing host output, upload to chaos
125 err = r.UploadToChaos(buf)
126 if err != nil {
127 gologger.Errorf("Could not upload results to chaos %s\n", err)
128 } else {
129 gologger.Infof("Input processed successfully and subdomains with valid records will be updated to chaos dataset.\n")
130 }
131 // clear buffer
132 buf = nil
133 }
134 }
135 // In case the user has given an output file, write all the found
136 // subdomains to the output file.
137 if output != "" {
138 // If the output format is json, append .json
139 // else append .txt
140 if r.options.OutputDirectory != "" {
141 if r.options.JSON {
142 output = output + ".json"
143 } else {
144 output = output + ".txt"
145 }
146 }
147
148 var file *os.File
149 var err error
150 if append {
151 file, err = os.OpenFile(output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
152 } else {
153 file, err = os.Create(output)
154 }
155 if err != nil {
156 gologger.Errorf("Could not create file %s for %s: %s\n", output, domain, err)
157 return err
158 }
159
160 // Write the output to the file depending upon user requirement
161 if r.options.HostIP {
162 err = WriteHostIPOutput(foundResults, file)
163 } else if r.options.JSON {
164 err = WriteJSONOutput(foundResults, file)
165 } else {
166 if r.options.RemoveWildcard {
167 err = WriteHostOutputNoWildcard(foundResults, file)
168 } else {
169 err = WriteHostOutput(uniqueMap, file)
170 }
171 }
172 if err != nil {
173 gologger.Errorf("Could not write results to file %s for %s: %s\n", output, domain, err)
174 }
175 file.Close()
176 return err
177 }
178 return nil
179 }
+0
-52
pkg/runner/initialize.go less more
0 package runner
1
2 import (
3 "strings"
4
5 "github.com/projectdiscovery/subfinder/pkg/passive"
6 "github.com/projectdiscovery/subfinder/pkg/resolve"
7 )
8
9 // initializePassiveEngine creates the passive engine and loads sources etc
10 func (r *Runner) initializePassiveEngine() {
11 var sources, exclusions []string
12
13 // If there are any sources from CLI, only use them
14 // Otherwise, use the yaml file sources
15 if r.options.Sources != "" {
16 sources = append(sources, strings.Split(r.options.Sources, ",")...)
17 } else {
18 sources = append(sources, r.options.YAMLConfig.Sources...)
19 }
20
21 if r.options.ExcludeSources != "" {
22 exclusions = append(exclusions, strings.Split(r.options.ExcludeSources, ",")...)
23 } else {
24 exclusions = append(exclusions, r.options.YAMLConfig.ExcludeSources...)
25 }
26
27 r.passiveAgent = passive.New(sources, exclusions)
28 }
29
30 // initializeActiveEngine creates the resolver used to resolve the found subdomains
31 func (r *Runner) initializeActiveEngine() error {
32 r.resolverClient = resolve.New()
33
34 // If the file has been provided, read resolvers from the file
35 if r.options.ResolverList != "" {
36 err := r.resolverClient.AppendResolversFromFile(r.options.ResolverList)
37 if err != nil {
38 return err
39 }
40 }
41
42 var resolvers []string
43
44 if r.options.Resolvers != "" {
45 resolvers = append(resolvers, strings.Split(r.options.Resolvers, ",")...)
46 } else {
47 resolvers = append(resolvers, r.options.YAMLConfig.Resolvers...)
48 }
49 r.resolverClient.AppendResolversFromSlice(resolvers)
50 return nil
51 }
+0
-143
pkg/runner/options.go less more
0 package runner
1
2 import (
3 "flag"
4 "os"
5 "path"
6 "reflect"
7 "strings"
8
9 "github.com/projectdiscovery/gologger"
10 )
11
12 // Options contains the configuration options for tuning
13 // the subdomain enumeration process.
14 type Options struct {
15 Verbose bool // Verbose flag indicates whether to show verbose output or not
16 NoColor bool // No-Color disables the colored output
17 Threads int // Thread controls the number of threads to use for active enumerations
18 Timeout int // Timeout is the seconds to wait for sources to respond
19 MaxEnumerationTime int // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration
20 Domain string // Domain is the domain to find subdomains for
21 DomainsFile string // DomainsFile is the file containing list of domains to find subdomains for
22 ChaosUpload bool // ChaosUpload indicates whether to upload results to the Chaos API
23 Output string // Output is the file to write found subdomains to.
24 OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given
25 JSON bool // JSON specifies whether to use json for output format or text file
26 HostIP bool // HostIP specifies whether to write subdomains in host:ip format
27 Silent bool // Silent suppresses any extra text and only writes subdomains to screen
28 Sources string // Sources contains a comma-separated list of sources to use for enumeration
29 ListSources bool // ListSources specifies whether to list all available sources
30 ExcludeSources string // ExcludeSources contains the comma-separated sources to not include in the enumeration process
31 Resolvers string // Resolvers is the comma-separated resolvers to use for enumeration
32 ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration
33 RemoveWildcard bool // RemoveWildcard specifies whether to remove potential wildcard or dead subdomains from the results.
34 ConfigFile string // ConfigFile contains the location of the config file
35 Stdin bool // Stdin specifies whether stdin input was given to the process
36 Version bool // Version specifies if we should just show version and exit
37
38 YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file
39 }
40
41 // ParseOptions parses the command line flags provided by a user
42 func ParseOptions() *Options {
43 options := &Options{}
44
45 config, err := GetConfigDirectory()
46 if err != nil {
47 // This should never be reached
48 gologger.Fatalf("Could not get user home: %s\n", err)
49 }
50
51 flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output")
52 flag.BoolVar(&options.NoColor, "nC", false, "Don't Use colors in output")
53 flag.IntVar(&options.Threads, "t", 10, "Number of concurrent goroutines for resolving")
54 flag.IntVar(&options.Timeout, "timeout", 30, "Seconds to wait before timing out")
55 flag.IntVar(&options.MaxEnumerationTime, "max-time", 10, "Minutes to wait for enumeration results")
56 flag.StringVar(&options.Domain, "d", "", "Domain to find subdomains for")
57 flag.StringVar(&options.DomainsFile, "dL", "", "File containing list of domains to enumerate")
58 flag.BoolVar(&options.ChaosUpload, "cd", false, "Upload results to the Chaos API (api-key required)")
59 flag.StringVar(&options.Output, "o", "", "File to write output to (optional)")
60 flag.StringVar(&options.OutputDirectory, "oD", "", "Directory to write enumeration results to (optional)")
61 flag.BoolVar(&options.JSON, "oJ", false, "Write output in JSON lines Format")
62 flag.BoolVar(&options.HostIP, "oI", false, "Write output in Host,IP format")
63 flag.BoolVar(&options.Silent, "silent", false, "Show only subdomains in output")
64 flag.StringVar(&options.Sources, "sources", "", "Comma separated list of sources to use")
65 flag.BoolVar(&options.ListSources, "ls", false, "List all available sources")
66 flag.StringVar(&options.ExcludeSources, "exclude-sources", "", "List of sources to exclude from enumeration")
67 flag.StringVar(&options.Resolvers, "r", "", "Comma-separated list of resolvers to use")
68 flag.StringVar(&options.ResolverList, "rL", "", "Text file containing list of resolvers to use")
69 flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output")
70 flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc")
71 flag.BoolVar(&options.Version, "version", false, "Show version of subfinder")
72 flag.Parse()
73
74 // Check if stdin pipe was given
75 options.Stdin = hasStdin()
76
77 // Read the inputs and configure the logging
78 options.configureOutput()
79
80 // Show the user the banner
81 showBanner()
82
83 if options.Version {
84 gologger.Infof("Current Version: %s\n", Version)
85 os.Exit(0)
86 }
87
88 // Check if the config file exists. If not, it means this is the
89 // first run of the program. Show the first run notices and initialize the config file.
90 // Else show the normal banners and read the yaml fiile to the config
91 if !CheckConfigExists(options.ConfigFile) {
92 options.firstRunTasks()
93 } else {
94 options.normalRunTasks()
95 }
96
97 if options.ListSources {
98 listSources(options)
99 os.Exit(0)
100 }
101
102 // Validate the options passed by the user and if any
103 // invalid options have been used, exit.
104 err = options.validateOptions()
105 if err != nil {
106 gologger.Fatalf("Program exiting: %s\n", err)
107 }
108
109 return options
110 }
111
112 func hasStdin() bool {
113 fi, err := os.Stdin.Stat()
114 if err != nil {
115 return false
116 }
117 if fi.Mode()&os.ModeNamedPipe == 0 {
118 return false
119 }
120 return true
121 }
122
123 func listSources(options *Options) {
124 gologger.Infof("Current list of available sources. [%d]\n", len(options.YAMLConfig.Sources))
125 gologger.Infof("Sources marked with an * needs key or token in order to work.\n")
126 gologger.Infof("You can modify %s to configure your keys / tokens.\n\n", options.ConfigFile)
127
128 keys := options.YAMLConfig.GetKeys()
129 needsKey := make(map[string]interface{})
130 keysElem := reflect.ValueOf(&keys).Elem()
131 for i := 0; i < keysElem.NumField(); i++ {
132 needsKey[strings.ToLower(keysElem.Type().Field(i).Name)] = keysElem.Field(i).Interface()
133 }
134
135 for _, source := range options.YAMLConfig.Sources {
136 message := "%s\n"
137 if _, ok := needsKey[source]; ok {
138 message = "%s *\n"
139 }
140 gologger.Silentf(message, source)
141 }
142 }
+0
-91
pkg/runner/runner.go less more
0 package runner
1
2 import (
3 "bufio"
4 "io"
5 "os"
6 "path"
7
8 "github.com/projectdiscovery/subfinder/pkg/passive"
9 "github.com/projectdiscovery/subfinder/pkg/resolve"
10 )
11
12 // Runner is an instance of the subdomain enumeration
13 // client used to orchestrate the whole process.
14 type Runner struct {
15 options *Options
16 passiveAgent *passive.Agent
17 resolverClient *resolve.Resolver
18 }
19
20 // NewRunner creates a new runner struct instance by parsing
21 // the configuration options, configuring sources, reading lists
22 // and setting up loggers, etc.
23 func NewRunner(options *Options) (*Runner, error) {
24 runner := &Runner{options: options}
25
26 // Initialize the passive subdomain enumeration engine
27 runner.initializePassiveEngine()
28
29 // Initialize the active subdomain enumeration engine
30 err := runner.initializeActiveEngine()
31 if err != nil {
32 return nil, err
33 }
34
35 return runner, nil
36 }
37
38 // RunEnumeration runs the subdomain enumeration flow on the targets specified
39 func (r *Runner) RunEnumeration() error {
40 // Check if only a single domain is sent as input. Process the domain now.
41 if r.options.Domain != "" {
42 return r.EnumerateSingleDomain(r.options.Domain, r.options.Output, false)
43 }
44
45 // If we have multiple domains as input,
46 if r.options.DomainsFile != "" {
47 f, err := os.Open(r.options.DomainsFile)
48 if err != nil {
49 return err
50 }
51 err = r.EnumerateMultipleDomains(f)
52 f.Close()
53 return err
54 }
55
56 // If we have STDIN input, treat it as multiple domains
57 if r.options.Stdin {
58 return r.EnumerateMultipleDomains(os.Stdin)
59 }
60 return nil
61 }
62
63 // EnumerateMultipleDomains enumerates subdomains for multiple domains
64 // We keep enumerating subdomains for a given domain until we reach an error
65 func (r *Runner) EnumerateMultipleDomains(reader io.Reader) error {
66 scanner := bufio.NewScanner(reader)
67 for scanner.Scan() {
68 domain := scanner.Text()
69 if domain == "" {
70 continue
71 }
72
73 var err error
74 // If the user has specifed an output file, use that output file instead
75 // of creating a new output file for each domain. Else create a new file
76 // for each domain in the directory.
77 if r.options.Output != "" {
78 err = r.EnumerateSingleDomain(domain, r.options.Output, true)
79 } else if r.options.OutputDirectory != "" {
80 outputFile := path.Join(r.options.OutputDirectory, domain)
81 err = r.EnumerateSingleDomain(domain, outputFile, false)
82 } else {
83 err = r.EnumerateSingleDomain(domain, "", true)
84 }
85 if err != nil {
86 return err
87 }
88 }
89 return nil
90 }
+0
-131
pkg/runner/utils.go less more
0 package runner
1
2 import (
3 "bufio"
4 "crypto/tls"
5 "fmt"
6 "io"
7 "io/ioutil"
8 "net/http"
9 "strings"
10 "time"
11
12 jsoniter "github.com/json-iterator/go"
13 "github.com/pkg/errors"
14 )
15
16 // JSONResult contains the result for a host in JSON format
17 type JSONResult struct {
18 Host string `json:"host"`
19 IP string `json:"ip"`
20 }
21
22 func (r *Runner) UploadToChaos(reader io.Reader) error {
23 httpClient := &http.Client{
24 Transport: &http.Transport{
25 MaxIdleConnsPerHost: 100,
26 MaxIdleConns: 100,
27 TLSClientConfig: &tls.Config{
28 InsecureSkipVerify: true,
29 },
30 },
31 Timeout: time.Duration(600) * time.Second, // 10 minutes - uploads may take long
32 }
33
34 request, err := http.NewRequest("POST", "https://dns.projectdiscovery.io/dns/add", reader)
35 if err != nil {
36 return errors.Wrap(err, "could not create request")
37 }
38 request.Header.Set("Authorization", r.options.YAMLConfig.GetKeys().Chaos)
39
40 resp, err := httpClient.Do(request)
41 if err != nil {
42 return errors.Wrap(err, "could not make request")
43 }
44 defer func() {
45 io.Copy(ioutil.Discard, resp.Body)
46 resp.Body.Close()
47 }()
48
49 if resp.StatusCode != 200 {
50 return fmt.Errorf("invalid status code received: %d", resp.StatusCode)
51 }
52 return nil
53 }
54
55 // WriteHostOutput writes the output list of subdomain to an io.Writer
56 func WriteHostOutput(results map[string]struct{}, writer io.Writer) error {
57 bufwriter := bufio.NewWriter(writer)
58 sb := &strings.Builder{}
59
60 for host := range results {
61 sb.WriteString(host)
62 sb.WriteString("\n")
63
64 _, err := bufwriter.WriteString(sb.String())
65 if err != nil {
66 bufwriter.Flush()
67 return err
68 }
69 sb.Reset()
70 }
71 return bufwriter.Flush()
72 }
73
74 // WriteHostOutputNoWildcard writes the output list of subdomain with nW flag to an io.Writer
75 func WriteHostOutputNoWildcard(results map[string]string, writer io.Writer) error {
76 bufwriter := bufio.NewWriter(writer)
77 sb := &strings.Builder{}
78
79 for host := range results {
80 sb.WriteString(host)
81 sb.WriteString("\n")
82
83 _, err := bufwriter.WriteString(sb.String())
84 if err != nil {
85 bufwriter.Flush()
86 return err
87 }
88 sb.Reset()
89 }
90 return bufwriter.Flush()
91 }
92
93 // WriteJSONOutput writes the output list of subdomain in JSON to an io.Writer
94 func WriteJSONOutput(results map[string]string, writer io.Writer) error {
95 encoder := jsoniter.NewEncoder(writer)
96
97 data := JSONResult{}
98
99 for host, ip := range results {
100 data.Host = host
101 data.IP = ip
102
103 err := encoder.Encode(&data)
104 if err != nil {
105 return err
106 }
107 }
108 return nil
109 }
110
111 // WriteHostIPOutput writes the output list of subdomain to an io.Writer
112 func WriteHostIPOutput(results map[string]string, writer io.Writer) error {
113 bufwriter := bufio.NewWriter(writer)
114 sb := &strings.Builder{}
115
116 for host, ip := range results {
117 sb.WriteString(host)
118 sb.WriteString(",")
119 sb.WriteString(ip)
120 sb.WriteString("\n")
121
122 _, err := bufwriter.WriteString(sb.String())
123 if err != nil {
124 bufwriter.Flush()
125 return err
126 }
127 sb.Reset()
128 }
129 return bufwriter.Flush()
130 }
+0
-58
pkg/runner/validate.go less more
0 package runner
1
2 import (
3 "errors"
4
5 "github.com/projectdiscovery/gologger"
6 )
7
8 // validateOptions validates the configuration options passed
9 func (options *Options) validateOptions() error {
10 // Check if domain, list of domains, or stdin info was provided.
11 // If none was provided, then return.
12 if options.Domain == "" && options.DomainsFile == "" && !options.Stdin {
13 return errors.New("no input list provided")
14 }
15
16 // Both verbose and silent flags were used
17 if options.Verbose && options.Silent {
18 return errors.New("both verbose and silent mode specified")
19 }
20
21 // Validate threads and options
22 if options.Threads == 0 {
23 return errors.New("threads cannot be zero")
24 }
25 if options.Timeout == 0 {
26 return errors.New("timeout cannot be zero")
27 }
28
29 // JSON cannot be used with hostIP
30 if options.JSON && options.HostIP {
31 return errors.New("hostip flag cannot be used with json flag")
32 }
33
34 // Always remove wildcard with hostip and json
35 if options.HostIP && !options.RemoveWildcard {
36 return errors.New("hostip flag must be used with RemoveWildcard option")
37 }
38 if options.JSON && !options.RemoveWildcard {
39 return errors.New("JSON flag must be used with RemoveWildcard option")
40 }
41
42 return nil
43 }
44
45 // configureOutput configures the output on the screen
46 func (options *Options) configureOutput() {
47 // If the user desires verbose output, show verbose output
48 if options.Verbose {
49 gologger.MaxLevel = gologger.Verbose
50 }
51 if options.NoColor {
52 gologger.UseColors = false
53 }
54 if options.Silent {
55 gologger.MaxLevel = gologger.Silent
56 }
57 }
+0
-96
pkg/subscraping/agent.go less more
0 package subscraping
1
2 import (
3 "context"
4 "crypto/tls"
5 "fmt"
6 "io"
7 "io/ioutil"
8 "net/http"
9 "net/url"
10 "time"
11 )
12
13 // NewSession creates a new session object for a domain
14 func NewSession(domain string, keys Keys, timeout int) (*Session, error) {
15 client := &http.Client{
16 Transport: &http.Transport{
17 MaxIdleConns: 100,
18 MaxIdleConnsPerHost: 100,
19 TLSClientConfig: &tls.Config{
20 InsecureSkipVerify: true,
21 },
22 },
23 Timeout: time.Duration(timeout) * time.Second,
24 }
25
26 session := &Session{
27 Client: client,
28 Keys: keys,
29 }
30
31 // Create a new extractor object for the current domain
32 extractor, err := NewSubdomainExtractor(domain)
33 session.Extractor = extractor
34
35 return session, err
36 }
37
38 // NormalGetWithContext makes a normal GET request to a URL with context
39 func (s *Session) NormalGetWithContext(ctx context.Context, url string) (*http.Response, error) {
40 req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
41 if err != nil {
42 return nil, err
43 }
44
45 // Don't randomize user agents, as they cause issues sometimes
46 req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36")
47 req.Header.Set("Accept", "*/*")
48 req.Header.Set("Accept-Language", "en")
49
50 return httpRequestWrapper(s.Client, req)
51 }
52
53 // Get makes a GET request to a URL
54 func (s *Session) Get(ctx context.Context, url string, cookies string, headers map[string]string) (*http.Response, error) {
55 req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
56 if err != nil {
57 return nil, err
58 }
59
60 req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36")
61 req.Header.Set("Accept", "*/*")
62 req.Header.Set("Accept-Language", "en")
63
64 if cookies != "" {
65 req.Header.Set("Cookie", cookies)
66 }
67
68 if headers != nil {
69 for key, value := range headers {
70 req.Header.Set(key, value)
71 }
72 }
73
74 return httpRequestWrapper(s.Client, req)
75 }
76
77 func (s *Session) DiscardHttpResponse(response *http.Response) {
78 if response != nil {
79 io.Copy(ioutil.Discard, response.Body)
80 response.Body.Close()
81 }
82 }
83
84 func httpRequestWrapper(client *http.Client, request *http.Request) (*http.Response, error) {
85 resp, err := client.Do(request)
86 if err != nil {
87 return nil, err
88 }
89
90 if resp.StatusCode != http.StatusOK {
91 requestUrl, _ := url.QueryUnescape(request.URL.String())
92 return resp, fmt.Errorf("Unexpected status code %d received from %s", resp.StatusCode, requestUrl)
93 }
94 return resp, nil
95 }
+0
-55
pkg/subscraping/sources/alienvault/alienvault.go less more
0 package alienvault
1
2 import (
3 "context"
4 "encoding/json"
5 "fmt"
6
7 "github.com/projectdiscovery/subfinder/pkg/subscraping"
8 )
9
10 type alienvaultResponse struct {
11 PassiveDNS []struct {
12 Hostname string `json:"hostname"`
13 } `json:"passive_dns"`
14 }
15
16 // Source is the passive scraping agent
17 type Source struct{}
18
19 // Run function returns all subdomains found with the service
20 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
21 results := make(chan subscraping.Result)
22
23 go func() {
24 resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain))
25 if err != nil {
26 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
27 session.DiscardHttpResponse(resp)
28 close(results)
29 return
30 }
31
32 otxResp := &alienvaultResponse{}
33 // Get the response body and decode
34 err = json.NewDecoder(resp.Body).Decode(&otxResp)
35 if err != nil {
36 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
37 resp.Body.Close()
38 close(results)
39 return
40 }
41 resp.Body.Close()
42 for _, record := range otxResp.PassiveDNS {
43 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}
44 }
45 close(results)
46 }()
47
48 return results
49 }
50
51 // Name returns the name of the source
52 func (s *Source) Name() string {
53 return "alienvault"
54 }
+0
-77
pkg/subscraping/sources/archiveis/archiveis.go less more
0 // Package archiveis is a Archiveis Scraping Engine in Golang
1 package archiveis
2
3 import (
4 "context"
5 "io/ioutil"
6 "regexp"
7
8 "github.com/projectdiscovery/subfinder/pkg/subscraping"
9 )
10
11 // ArchiveIs is a struct for archiveurlsagent
12 type ArchiveIs struct {
13 Results chan subscraping.Result
14 Session *subscraping.Session
15 }
16
17 var reNext = regexp.MustCompile("<a id=\"next\" style=\".*\" href=\"(.*)\">&rarr;</a>")
18
19 func (a *ArchiveIs) enumerate(ctx context.Context, baseURL string) {
20 select {
21 case <-ctx.Done():
22 return
23 default:
24 }
25
26 resp, err := a.Session.NormalGetWithContext(ctx, baseURL)
27 if err != nil {
28 a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err}
29 a.Session.DiscardHttpResponse(resp)
30 return
31 }
32
33 // Get the response body
34 body, err := ioutil.ReadAll(resp.Body)
35 resp.Body.Close()
36 if err != nil {
37 a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err}
38 return
39 }
40
41 src := string(body)
42
43 for _, subdomain := range a.Session.Extractor.FindAllString(src, -1) {
44 a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Subdomain, Value: subdomain}
45 }
46
47 match1 := reNext.FindStringSubmatch(src)
48 if len(match1) > 0 {
49 a.enumerate(ctx, match1[1])
50 }
51 }
52
53 // Source is the passive scraping agent
54 type Source struct{}
55
56 // Run function returns all subdomains found with the service
57 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
58 results := make(chan subscraping.Result)
59
60 aInstance := ArchiveIs{
61 Session: session,
62 Results: results,
63 }
64
65 go func() {
66 aInstance.enumerate(ctx, "http://archive.is/*."+domain)
67 close(aInstance.Results)
68 }()
69
70 return aInstance.Results
71 }
72
73 // Name returns the name of the source
74 func (s *Source) Name() string {
75 return "archiveis"
76 }
+0
-104
pkg/subscraping/sources/binaryedge/binaryedge.go less more
0 package binaryedge
1
2 import (
3 "context"
4 "fmt"
5
6 jsoniter "github.com/json-iterator/go"
7 "github.com/projectdiscovery/subfinder/pkg/subscraping"
8 )
9
10 type binaryedgeResponse struct {
11 Subdomains []string `json:"events"`
12 Total int `json:"total"`
13 }
14
15 // Source is the passive scraping agent
16 type Source struct{}
17
18 // Run function returns all subdomains found with the service
19 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
20 results := make(chan subscraping.Result)
21
22 go func() {
23 if session.Keys.Binaryedge == "" {
24 close(results)
25 return
26 }
27
28 resp, err := session.Get(ctx, fmt.Sprintf("https://api.binaryedge.io/v2/query/domains/subdomain/%s", domain), "", map[string]string{"X-Key": session.Keys.Binaryedge})
29 if err != nil {
30 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
31 session.DiscardHttpResponse(resp)
32 close(results)
33 return
34 }
35
36 response := new(binaryedgeResponse)
37 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
38 if err != nil {
39 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
40 resp.Body.Close()
41 close(results)
42 return
43 }
44 resp.Body.Close()
45
46 for _, subdomain := range response.Subdomains {
47 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
48 }
49
50 remaining := response.Total - 100
51 currentPage := 2
52
53 for {
54 further := s.getSubdomains(ctx, domain, &remaining, &currentPage, session, results)
55 if !further {
56 break
57 }
58 }
59 close(results)
60 }()
61
62 return results
63 }
64
65 // Name returns the name of the source
66 func (s *Source) Name() string {
67 return "binaryedge"
68 }
69
70 func (s *Source) getSubdomains(ctx context.Context, domain string, remaining, currentPage *int, session *subscraping.Session, results chan subscraping.Result) bool {
71 for {
72 select {
73 case <-ctx.Done():
74 return false
75 default:
76 resp, err := session.Get(ctx, fmt.Sprintf("https://api.binaryedge.io/v2/query/domains/subdomain/%s?page=%d", domain, *currentPage), "", map[string]string{"X-Key": session.Keys.Binaryedge})
77 if err != nil {
78 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
79 return false
80 }
81
82 response := binaryedgeResponse{}
83 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
84 if err != nil {
85 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
86 resp.Body.Close()
87 return false
88 }
89 resp.Body.Close()
90
91 for _, subdomain := range response.Subdomains {
92 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
93 }
94
95 *remaining = *remaining - 100
96 if *remaining <= 0 {
97 return false
98 }
99 *currentPage++
100 return true
101 }
102 }
103 }
+0
-57
pkg/subscraping/sources/bufferover/bufferover.go less more
0 // Package bufferover is a bufferover Scraping Engine in Golang
1 package bufferover
2
3 import (
4 "context"
5 "fmt"
6 "io/ioutil"
7
8 "github.com/projectdiscovery/subfinder/pkg/subscraping"
9 )
10
11 // Source is the passive scraping agent
12 type Source struct{}
13
14 // Run function returns all subdomains found with the service
15 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
16 results := make(chan subscraping.Result)
17
18 go func() {
19 // Run enumeration on subdomain dataset for historical SONAR datasets
20 s.getData(ctx, fmt.Sprintf("https://dns.bufferover.run/dns?q=.%s", domain), session, results)
21 s.getData(ctx, fmt.Sprintf("https://tls.bufferover.run/dns?q=.%s", domain), session, results)
22
23 close(results)
24 }()
25
26 return results
27 }
28
29 func (s *Source) getData(ctx context.Context, URL string, session *subscraping.Session, results chan subscraping.Result) {
30 resp, err := session.NormalGetWithContext(ctx, URL)
31 if err != nil {
32 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
33 session.DiscardHttpResponse(resp)
34 return
35 }
36
37 body, err := ioutil.ReadAll(resp.Body)
38 if err != nil {
39 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
40 resp.Body.Close()
41 return
42 }
43 resp.Body.Close()
44
45 src := string(body)
46
47 for _, subdomain := range session.Extractor.FindAllString(src, -1) {
48 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
49 }
50 return
51 }
52
53 // Name returns the name of the source
54 func (s *Source) Name() string {
55 return "bufferover"
56 }
+0
-96
pkg/subscraping/sources/censys/censys.go less more
0 package censys
1
2 import (
3 "bytes"
4 "context"
5 "net/http"
6 "strconv"
7
8 jsoniter "github.com/json-iterator/go"
9 "github.com/projectdiscovery/subfinder/pkg/subscraping"
10 )
11
12 const maxCensysPages = 10
13
14 type resultsq struct {
15 Data []string `json:"parsed.extensions.subject_alt_name.dns_names"`
16 Data1 []string `json:"parsed.names"`
17 }
18
19 type response struct {
20 Results []resultsq `json:"results"`
21 Metadata struct {
22 Pages int `json:"pages"`
23 } `json:"metadata"`
24 }
25
26 // Source is the passive scraping agent
27 type Source struct{}
28
29 // Run function returns all subdomains found with the service
30 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
31 results := make(chan subscraping.Result)
32
33 go func() {
34 if session.Keys.CensysToken == "" || session.Keys.CensysSecret == "" {
35 close(results)
36 return
37 }
38 var response response
39
40 currentPage := 1
41 for {
42 var request = []byte(`{"query":"` + domain + `", "page":` + strconv.Itoa(currentPage) + `, "fields":["parsed.names","parsed.extensions.subject_alt_name.dns_names"], "flatten":true}`)
43
44 req, err := http.NewRequestWithContext(ctx, "POST", "https://www.censys.io/api/v1/search/certificates", bytes.NewReader(request))
45 if err != nil {
46 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
47 close(results)
48 return
49 }
50 req.SetBasicAuth(session.Keys.CensysToken, session.Keys.CensysSecret)
51 req.Header.Set("Content-Type", "application/json")
52 req.Header.Set("Accept", "application/json")
53
54 resp, err := session.Client.Do(req)
55 if err != nil {
56 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
57 close(results)
58 return
59 }
60
61 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
62 if err != nil {
63 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
64 resp.Body.Close()
65 close(results)
66 return
67 }
68 resp.Body.Close()
69
70 // Exit the censys enumeration if max pages is reached
71 if currentPage >= response.Metadata.Pages || currentPage >= maxCensysPages {
72 break
73 }
74
75 for _, res := range response.Results {
76 for _, part := range res.Data {
77 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: part}
78 }
79 for _, part := range res.Data1 {
80 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: part}
81 }
82 }
83
84 currentPage++
85 }
86 close(results)
87 }()
88
89 return results
90 }
91
92 // Name returns the name of the source
93 func (s *Source) Name() string {
94 return "censys"
95 }
+0
-101
pkg/subscraping/sources/certspotter/certspotter.go less more
0 package certspotter
1
2 import (
3 "context"
4 "fmt"
5
6 jsoniter "github.com/json-iterator/go"
7 "github.com/projectdiscovery/subfinder/pkg/subscraping"
8 )
9
10 type certspotterObject struct {
11 ID string `json:"id"`
12 DNSNames []string `json:"dns_names"`
13 }
14
15 // Source is the passive scraping agent
16 type Source struct{}
17
18 // Run function returns all subdomains found with the service
19 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
20 results := make(chan subscraping.Result)
21
22 go func() {
23 if session.Keys.Certspotter == "" {
24 close(results)
25 return
26 }
27
28 resp, err := session.Get(ctx, fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain), "", map[string]string{"Authorization": "Bearer " + session.Keys.Certspotter})
29 if err != nil {
30 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
31 session.DiscardHttpResponse(resp)
32 close(results)
33 return
34 }
35
36 response := []certspotterObject{}
37 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
38 if err != nil {
39 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
40 resp.Body.Close()
41 close(results)
42 return
43 }
44 resp.Body.Close()
45
46 for _, cert := range response {
47 for _, subdomain := range cert.DNSNames {
48 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
49 }
50 }
51
52 // if the number of responses is zero, close the channel and return.
53 if len(response) == 0 {
54 close(results)
55 return
56 }
57
58 id := response[len(response)-1].ID
59 for {
60 reqURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, id)
61
62 resp, err := session.Get(ctx, reqURL, "", map[string]string{"Authorization": "Bearer " + session.Keys.Certspotter})
63 if err != nil {
64 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
65 close(results)
66 return
67 }
68
69 response := []certspotterObject{}
70 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
71 if err != nil {
72 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
73 resp.Body.Close()
74 close(results)
75 return
76 }
77 resp.Body.Close()
78
79 if len(response) == 0 {
80 break
81 }
82
83 for _, cert := range response {
84 for _, subdomain := range cert.DNSNames {
85 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
86 }
87 }
88
89 id = response[len(response)-1].ID
90 }
91 close(results)
92 }()
93
94 return results
95 }
96
97 // Name returns the name of the source
98 func (s *Source) Name() string {
99 return "certspotter"
100 }
+0
-50
pkg/subscraping/sources/certspotterold/certspotterold.go less more
0 package certspotterold
1
2 import (
3 "context"
4 "fmt"
5 "io/ioutil"
6
7 "github.com/projectdiscovery/subfinder/pkg/subscraping"
8 )
9
10 // Source is the passive scraping agent
11 type Source struct{}
12
13 // Run function returns all subdomains found with the service
14 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
15 results := make(chan subscraping.Result)
16
17 go func() {
18 resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://certspotter.com/api/v0/certs?domain=%s", domain))
19 if err != nil {
20 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
21 session.DiscardHttpResponse(resp)
22 close(results)
23 return
24 }
25
26 body, err := ioutil.ReadAll(resp.Body)
27 if err != nil {
28 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
29 resp.Body.Close()
30 close(results)
31 return
32 }
33 resp.Body.Close()
34
35 src := string(body)
36
37 for _, subdomain := range session.Extractor.FindAllString(src, -1) {
38 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
39 }
40 close(results)
41 }()
42
43 return results
44 }
45
46 // Name returns the name of the source
47 func (s *Source) Name() string {
48 return "certspotterold"
49 }
+0
-108
pkg/subscraping/sources/commoncrawl/commoncrawl.go less more
0 package commoncrawl
1
2 import (
3 "context"
4 "fmt"
5 "io/ioutil"
6 "net/url"
7 "strings"
8
9 jsoniter "github.com/json-iterator/go"
10 "github.com/projectdiscovery/subfinder/pkg/subscraping"
11 )
12
13 const indexURL = "https://index.commoncrawl.org/collinfo.json"
14
15 type indexResponse struct {
16 ID string `json:"id"`
17 APIURL string `json:"cdx-api"`
18 }
19
20 // Source is the passive scraping agent
21 type Source struct{}
22
23 var years = [...]string{"2020", "2019", "2018", "2017"}
24
25 // Run function returns all subdomains found with the service
26 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
27 results := make(chan subscraping.Result)
28
29 go func() {
30 resp, err := session.NormalGetWithContext(ctx, indexURL)
31 if err != nil {
32 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
33 session.DiscardHttpResponse(resp)
34 close(results)
35 return
36 }
37
38 indexes := []indexResponse{}
39 err = jsoniter.NewDecoder(resp.Body).Decode(&indexes)
40 if err != nil {
41 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
42 resp.Body.Close()
43 close(results)
44 return
45 }
46 resp.Body.Close()
47
48 searchIndexes := make(map[string]string)
49 for _, year := range years {
50 for _, index := range indexes {
51 if strings.Contains(index.ID, year) {
52 if _, ok := searchIndexes[year]; !ok {
53 searchIndexes[year] = index.APIURL
54 break
55 }
56 }
57 }
58 }
59
60 for _, apiURL := range searchIndexes {
61 further := s.getSubdomains(ctx, apiURL, domain, session, results)
62 if !further {
63 break
64 }
65 }
66 close(results)
67 }()
68
69 return results
70 }
71
72 // Name returns the name of the source
73 func (s *Source) Name() string {
74 return "commoncrawl"
75 }
76
77 func (s *Source) getSubdomains(ctx context.Context, searchURL string, domain string, session *subscraping.Session, results chan subscraping.Result) bool {
78 for {
79 select {
80 case <-ctx.Done():
81 return false
82 default:
83 resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("%s?url=*.%s&output=json", searchURL, domain))
84 if err != nil {
85 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
86 return false
87 }
88
89 body, err := ioutil.ReadAll(resp.Body)
90 if err != nil {
91 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
92 resp.Body.Close()
93 return false
94 }
95 resp.Body.Close()
96
97 src, _ := url.QueryUnescape(string(body))
98
99 for _, subdomain := range session.Extractor.FindAllString(src, -1) {
100 subdomain = strings.TrimPrefix(subdomain, "25")
101
102 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
103 }
104 return true
105 }
106 }
107 }
+0
-93
pkg/subscraping/sources/crtsh/crtsh.go less more
0 package crtsh
1
2 import (
3 "context"
4 "database/sql"
5 "fmt"
6 "io/ioutil"
7 "strings"
8
9 // postgres driver
10 _ "github.com/lib/pq"
11 "github.com/projectdiscovery/subfinder/pkg/subscraping"
12 )
13
14 // Source is the passive scraping agent
15 type Source struct{}
16
17 // Run function returns all subdomains found with the service
18 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
19 results := make(chan subscraping.Result)
20
21 go func() {
22 found := s.getSubdomainsFromSQL(ctx, domain, session, results)
23 if found {
24 close(results)
25 return
26 }
27 _ = s.getSubdomainsFromHTTP(ctx, domain, session, results)
28 close(results)
29 }()
30
31 return results
32 }
33
34 func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) bool {
35 db, err := sql.Open("postgres", "host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes")
36 if err != nil {
37 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
38 return false
39 }
40
41 pattern := "%." + domain
42 rows, err := db.Query(`SELECT DISTINCT ci.NAME_VALUE as domain
43 FROM certificate_identity ci
44 WHERE reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower($1))
45 ORDER BY ci.NAME_VALUE`, pattern)
46 if err != nil {
47 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
48 return false
49 }
50
51 var data string
52 // Parse all the rows getting subdomains
53 for rows.Next() {
54 err := rows.Scan(&data)
55 if err != nil {
56 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
57 return false
58 }
59 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data}
60 }
61 return true
62 }
63
64 func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) bool {
65 resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain))
66 if err != nil {
67 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
68 session.DiscardHttpResponse(resp)
69 return false
70 }
71
72 body, err := ioutil.ReadAll(resp.Body)
73 if err != nil {
74 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
75 resp.Body.Close()
76 return false
77 }
78 resp.Body.Close()
79
80 // Also replace all newlines
81 src := strings.Replace(string(body), "\\n", " ", -1)
82
83 for _, subdomain := range session.Extractor.FindAllString(src, -1) {
84 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
85 }
86 return true
87 }
88
89 // Name returns the name of the source
90 func (s *Source) Name() string {
91 return "crtsh"
92 }
+0
-70
pkg/subscraping/sources/dnsdb/dnsdb.go less more
0 package dnsdb
1
2 import (
3 "bufio"
4 "context"
5 "encoding/json"
6 "fmt"
7 "strings"
8
9 "github.com/projectdiscovery/subfinder/pkg/subscraping"
10 )
11
12 type dnsdbResponse struct {
13 Name string `json:"rrname"`
14 }
15
16 // Source is the passive scraping agent
17 type Source struct{}
18
19 // Run function returns all subdomains found with the service
20 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
21 results := make(chan subscraping.Result)
22
23 if session.Keys.DNSDB == "" {
24 close(results)
25 } else {
26 headers := map[string]string{
27 "X-API-KEY": session.Keys.DNSDB,
28 "Accept": "application/json",
29 "Content-Type": "application/json",
30 }
31
32 go func() {
33 resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdb.info/lookup/rrset/name/*.%s?limit=1000000000000", domain), "", headers)
34 if err != nil {
35 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
36 session.DiscardHttpResponse(resp)
37 close(results)
38 return
39 }
40
41 defer resp.Body.Close()
42 // Get the response body
43 scanner := bufio.NewScanner(resp.Body)
44 for scanner.Scan() {
45 line := scanner.Text()
46 if line == "" {
47 continue
48 }
49 out := &dnsdbResponse{}
50 err := json.Unmarshal([]byte(line), out)
51 if err != nil {
52 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
53 resp.Body.Close()
54 close(results)
55 return
56 }
57 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(out.Name, ".")}
58 out = nil
59 }
60 close(results)
61 }()
62 }
63 return results
64 }
65
66 // Name returns the name of the source
67 func (s *Source) Name() string {
68 return "DNSDB"
69 }
+0
-113
pkg/subscraping/sources/dnsdumpster/dnsdumpster.go less more
0 package dnsdumpster
1
2 import (
3 "context"
4 "io/ioutil"
5 "net"
6 "net/http"
7 "net/url"
8 "regexp"
9 "strings"
10 "time"
11
12 "github.com/projectdiscovery/subfinder/pkg/subscraping"
13 )
14
15 var re = regexp.MustCompile("<input type=\"hidden\" name=\"csrfmiddlewaretoken\" value=\"(.*)\">")
16
17 // getCSRFToken gets the CSRF Token from the page
18 func getCSRFToken(page string) string {
19 if subs := re.FindStringSubmatch(page); len(subs) == 2 {
20 return strings.TrimSpace(subs[1])
21 }
22 return ""
23 }
24
25 // postForm posts a form for a domain and returns the response
26 func postForm(token, domain string) (string, error) {
27 dial := net.Dialer{}
28 client := &http.Client{
29 Transport: &http.Transport{
30 DialContext: dial.DialContext,
31 TLSHandshakeTimeout: 10 * time.Second,
32 },
33 }
34 params := url.Values{
35 "csrfmiddlewaretoken": {token},
36 "targetip": {domain},
37 }
38
39 req, err := http.NewRequest("POST", "https://dnsdumpster.com/", strings.NewReader(params.Encode()))
40 if err != nil {
41 return "", err
42 }
43
44 // The CSRF token needs to be sent as a cookie
45 cookie := &http.Cookie{
46 Name: "csrftoken",
47 Domain: "dnsdumpster.com",
48 Value: token,
49 }
50 req.AddCookie(cookie)
51
52 req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36")
53 req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
54 req.Header.Set("Referer", "https://dnsdumpster.com")
55 req.Header.Set("X-CSRF-Token", token)
56
57 resp, err := client.Do(req)
58 if err != nil {
59 return "", err
60 }
61 // Now, grab the entire page
62 in, err := ioutil.ReadAll(resp.Body)
63 resp.Body.Close()
64 return string(in), err
65 }
66
67 // Source is the passive scraping agent
68 type Source struct{}
69
70 // Run function returns all subdomains found with the service
71 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
72 results := make(chan subscraping.Result)
73
74 go func() {
75 resp, err := session.NormalGetWithContext(ctx, "https://dnsdumpster.com/")
76 if err != nil {
77 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
78 session.DiscardHttpResponse(resp)
79 close(results)
80 return
81 }
82
83 body, err := ioutil.ReadAll(resp.Body)
84 if err != nil {
85 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
86 resp.Body.Close()
87 close(results)
88 return
89 }
90 resp.Body.Close()
91 csrfToken := getCSRFToken(string(body))
92
93 data, err := postForm(csrfToken, domain)
94 if err != nil {
95 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
96 close(results)
97 return
98 }
99
100 for _, subdomain := range session.Extractor.FindAllString(data, -1) {
101 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
102 }
103 close(results)
104 }()
105
106 return results
107 }
108
109 // Name returns the name of the source
110 func (s *Source) Name() string {
111 return "dnsdumpster"
112 }
+0
-53
pkg/subscraping/sources/entrust/entrust.go less more
0 package entrust
1
2 import (
3 "context"
4 "fmt"
5 "io/ioutil"
6 "strings"
7
8 "github.com/projectdiscovery/subfinder/pkg/subscraping"
9 )
10
11 // Source is the passive scraping agent
12 type Source struct{}
13
14 // Run function returns all subdomains found with the service
15 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
16 results := make(chan subscraping.Result)
17
18 go func() {
19 resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://ctsearch.entrust.com/api/v1/certificates?fields=issuerCN,subjectO,issuerDN,issuerO,subjectDN,signAlg,san,publicKeyType,publicKeySize,validFrom,validTo,sn,ev,logEntries.logName,subjectCNReversed,cert&domain=%s&includeExpired=true&exactMatch=false&limit=5000", domain))
20 if err != nil {
21 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
22 session.DiscardHttpResponse(resp)
23 close(results)
24 return
25 }
26
27 body, err := ioutil.ReadAll(resp.Body)
28 if err != nil {
29 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
30 resp.Body.Close()
31 close(results)
32 return
33 }
34 resp.Body.Close()
35
36 src := string(body)
37
38 for _, subdomain := range session.Extractor.FindAllString(src, -1) {
39 subdomain = strings.TrimPrefix(subdomain, "u003d")
40
41 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
42 }
43 close(results)
44 }()
45
46 return results
47 }
48
49 // Name returns the name of the source
50 func (s *Source) Name() string {
51 return "entrust"
52 }
+0
-211
pkg/subscraping/sources/github/github.go less more
0 // GitHub search package, based on gwen001's https://github.com/gwen001/github-search github-subdomains
1 package github
2
3 import (
4 "context"
5 "fmt"
6 "io/ioutil"
7 "net/http"
8 "net/url"
9 "regexp"
10 "strconv"
11 "strings"
12 "time"
13
14 jsoniter "github.com/json-iterator/go"
15
16 "github.com/projectdiscovery/gologger"
17 "github.com/projectdiscovery/subfinder/pkg/subscraping"
18 "github.com/tomnomnom/linkheader"
19 )
20
21 type textMatch struct {
22 Fragment string `json:"fragment"`
23 }
24
25 type item struct {
26 Name string `json:"name"`
27 HtmlUrl string `json:"html_url"`
28 TextMatches []textMatch `json:"text_matches"`
29 }
30
31 type response struct {
32 TotalCount int `json:"total_count"`
33 Items []item `json:"items"`
34 }
35
36 // Source is the passive scraping agent
37 type Source struct{}
38
39 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
40 results := make(chan subscraping.Result)
41
42 go func() {
43 if len(session.Keys.GitHub) == 0 {
44 close(results)
45 return
46 }
47
48 tokens := NewTokenManager(session.Keys.GitHub)
49
50 // search on GitHub with exact match
51 searchURL := fmt.Sprintf("https://api.github.com/search/code?per_page=100&q=\"%s\"", domain)
52 s.enumerate(ctx, searchURL, s.DomainRegexp(domain), tokens, session, results)
53 close(results)
54 }()
55
56 return results
57 }
58
59 func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *regexp.Regexp, tokens *Tokens, session *subscraping.Session, results chan subscraping.Result) {
60 select {
61 case <-ctx.Done():
62 return
63 default:
64 }
65
66 token := tokens.Get()
67
68 if token.RetryAfter > 0 {
69 if len(tokens.pool) == 1 {
70 gologger.Verbosef("GitHub Search request rate limit exceeded, waiting for %d seconds before retry... \n", s.Name(), token.RetryAfter)
71 time.Sleep(time.Duration(token.RetryAfter) * time.Second)
72 } else {
73 token = tokens.Get()
74 }
75 }
76
77 headers := map[string]string{
78 "Accept": "application/vnd.github.v3.text-match+json",
79 "Authorization": "token " + token.Hash,
80 }
81
82 // Initial request to GitHub search
83 resp, err := session.Get(ctx, searchURL, "", headers)
84 isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden
85
86 if err != nil && !isForbidden {
87 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
88 session.DiscardHttpResponse(resp)
89 return
90 } else {
91 // Retry enumerarion after Retry-After seconds on rate limit abuse detected
92 ratelimitRemaining, _ := strconv.ParseInt(resp.Header.Get("X-Ratelimit-Remaining"), 10, 64)
93 if isForbidden && ratelimitRemaining == 0 {
94 retryAfterSeconds, _ := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64)
95 tokens.setCurrentTokenExceeded(retryAfterSeconds)
96
97 s.enumerate(ctx, searchURL, domainRegexp, tokens, session, results)
98 } else {
99 // Links header, first, next, last...
100 linksHeader := linkheader.Parse(resp.Header.Get("Link"))
101
102 data := response{}
103
104 // Marshall json reponse
105 err = jsoniter.NewDecoder(resp.Body).Decode(&data)
106 resp.Body.Close()
107 if err != nil {
108 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
109 return
110 }
111
112 // Response items iteration
113 for _, item := range data.Items {
114 resp, err := session.NormalGetWithContext(ctx, rawUrl(item.HtmlUrl))
115 if err != nil {
116 if resp != nil && resp.StatusCode != http.StatusNotFound {
117 session.DiscardHttpResponse(resp)
118 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
119 return
120 }
121 }
122
123 var subdomains []string
124
125 if resp.StatusCode == http.StatusOK {
126 // Get the item code from the raw file url
127 code, err := ioutil.ReadAll(resp.Body)
128 resp.Body.Close()
129 if err != nil {
130 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
131 return
132 }
133 // Search for domain matches in the code
134 subdomains = append(subdomains, matches(domainRegexp, normalizeContent(string(code)))...)
135 }
136
137 // Text matches iteration per item
138 for _, textMatch := range item.TextMatches {
139 // Search for domain matches in the text fragment
140 subdomains = append(subdomains, matches(domainRegexp, normalizeContent(textMatch.Fragment))...)
141 }
142
143 for _, subdomain := range unique(subdomains) {
144 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
145 }
146 }
147
148 // Proccess the next link recursively
149 for _, link := range linksHeader {
150 if link.Rel == "next" {
151 nextUrl, err := url.QueryUnescape(link.URL)
152 if err != nil {
153 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
154 return
155 }
156 s.enumerate(ctx, nextUrl, domainRegexp, tokens, session, results)
157 }
158 }
159 }
160 }
161
162 }
163
164 // Normalize content before matching, query unescape, remove tabs and new line chars
165 func normalizeContent(content string) string {
166 normalizedContent, _ := url.QueryUnescape(content)
167 normalizedContent = strings.Replace(normalizedContent, "\\t", "", -1)
168 normalizedContent = strings.Replace(normalizedContent, "\\n", "", -1)
169 return normalizedContent
170 }
171
172 // Remove duplicates from string array
173 func unique(arr []string) []string {
174 occured := map[string]bool{}
175 result := []string{}
176 for e := range arr {
177 if occured[arr[e]] != true {
178 occured[arr[e]] = true
179 result = append(result, arr[e])
180 }
181 }
182 return result
183 }
184
185 // Find matches by regular expression in any content
186 func matches(regexp *regexp.Regexp, content string) []string {
187 var matches []string
188 match := regexp.FindAllString(content, -1)
189 if len(match) > 0 {
190 matches = unique(match)
191 }
192 return matches
193 }
194
195 // Raw URL to get the files code and match for subdomains
196 func rawUrl(htmlUrl string) string {
197 domain := strings.Replace(htmlUrl, "https://github.com/", "https://raw.githubusercontent.com/", -1)
198 return strings.Replace(domain, "/blob/", "/", -1)
199 }
200
201 // Domain regular expression to match subdomains in github files code
202 func (s *Source) DomainRegexp(domain string) *regexp.Regexp {
203 rdomain := strings.Replace(domain, ".", "\\.", -1)
204 return regexp.MustCompile("(\\w+[.])*" + rdomain)
205 }
206
207 // Name returns the name of the source
208 func (s *Source) Name() string {
209 return "github"
210 }
+0
-61
pkg/subscraping/sources/github/tokenmanager.go less more
0 package github
1
2 import "time"
3
4 type token struct {
5 Hash string
6 RetryAfter int64
7 ExceededTime time.Time
8 }
9
10 type Tokens struct {
11 current int
12 pool []token
13 }
14
15 func NewTokenManager(keys []string) *Tokens {
16 pool := []token{}
17 for _, key := range keys {
18 t := token{Hash: key, ExceededTime: time.Time{}, RetryAfter: 0}
19 pool = append(pool, t)
20 }
21
22 return &Tokens{
23 current: 0,
24 pool: pool,
25 }
26 }
27
28 func (r *Tokens) setCurrentTokenExceeded(retryAfter int64) {
29 if r.current >= len(r.pool) {
30 r.current = r.current % len(r.pool)
31 }
32 if r.pool[r.current].RetryAfter == 0 {
33 r.pool[r.current].ExceededTime = time.Now()
34 r.pool[r.current].RetryAfter = retryAfter
35 }
36 }
37
38 func (r *Tokens) Get() token {
39 resetExceededTokens(r)
40
41 if r.current >= len(r.pool) {
42 r.current = r.current % len(r.pool)
43 }
44
45 result := r.pool[r.current]
46 r.current++
47
48 return result
49 }
50
51 func resetExceededTokens(r *Tokens) {
52 for i, token := range r.pool {
53 if token.RetryAfter > 0 {
54 if int64(time.Since(token.ExceededTime)/time.Second) > token.RetryAfter {
55 r.pool[i].ExceededTime = time.Time{}
56 r.pool[i].RetryAfter = 0
57 }
58 }
59 }
60 }
+0
-50
pkg/subscraping/sources/hackertarget/hackertarget.go less more
0 package hackertarget
1
2 import (
3 "context"
4 "fmt"
5 "io/ioutil"
6
7 "github.com/projectdiscovery/subfinder/pkg/subscraping"
8 )
9
10 // Source is the passive scraping agent
11 type Source struct{}
12
13 // Run function returns all subdomains found with the service
14 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
15 results := make(chan subscraping.Result)
16
17 go func() {
18 resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("http://api.hackertarget.com/hostsearch/?q=%s", domain))
19 if err != nil {
20 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
21 session.DiscardHttpResponse(resp)
22 close(results)
23 return
24 }
25
26 // Get the response body
27 body, err := ioutil.ReadAll(resp.Body)
28 if err != nil {
29 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
30 resp.Body.Close()
31 close(results)
32 return
33 }
34 resp.Body.Close()
35 src := string(body)
36
37 for _, match := range session.Extractor.FindAllString(src, -1) {
38 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match}
39 }
40 close(results)
41 }()
42
43 return results
44 }
45
46 // Name returns the name of the source
47 func (s *Source) Name() string {
48 return "hackertarget"
49 }
+0
-114
pkg/subscraping/sources/intelx/intelx.go less more
0 package intelx
1
2 import (
3 "bytes"
4 "context"
5 "encoding/json"
6 "fmt"
7 "io/ioutil"
8 "net/http"
9
10 jsoniter "github.com/json-iterator/go"
11 "github.com/projectdiscovery/subfinder/pkg/subscraping"
12 )
13
14 type searchResponseType struct {
15 Id string `json:"id"`
16 Status int `json:"status"`
17 }
18
19 type selectorType struct {
20 Selectvalue string `json:"selectorvalue"`
21 }
22
23 type searchResultType struct {
24 Selectors []selectorType `json:"selectors"`
25 Status int `json:"status"`
26 }
27
28 type requestBody struct {
29 Term string
30 Maxresults int
31 Media int
32 Target int
33 Terminate []int
34 Timeout int
35 }
36
37 // Source is the passive scraping agent
38 type Source struct{}
39
40 // Run function returns all subdomains found with the service
41 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
42 results := make(chan subscraping.Result)
43
44 go func() {
45 defer close(results)
46 if session.Keys.IntelXKey == "" || session.Keys.IntelXHost == "" {
47 return
48 }
49
50 searchURL := fmt.Sprintf("https://%s/phonebook/search?k=%s", session.Keys.IntelXHost, session.Keys.IntelXKey)
51 reqBody := requestBody{
52 Term: domain,
53 Maxresults: 100000,
54 Media: 0,
55 Target: 1,
56 Timeout: 20,
57 }
58
59 body, err := json.Marshal(reqBody)
60 if err != nil {
61 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
62 return
63 }
64
65 resp, err := http.Post(searchURL, "application/json", bytes.NewBuffer(body))
66 if err != nil {
67 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
68 session.DiscardHttpResponse(resp)
69 return
70 }
71
72 var response searchResponseType
73 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
74 if err != nil {
75 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
76 close(results)
77 return
78 }
79
80 resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", session.Keys.IntelXHost, session.Keys.IntelXKey, response.Id)
81 status := 0
82 for status == 0 || status == 3 {
83 resp, err = session.Get(ctx, resultsURL, "", nil)
84 if err != nil {
85 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
86 return
87 }
88 var response searchResultType
89 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
90 if err != nil {
91 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
92 return
93 }
94 body, err = ioutil.ReadAll(resp.Body)
95 if err != nil {
96 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
97 return
98 }
99 resp.Body.Close()
100 status = response.Status
101 for _, hostname := range response.Selectors {
102 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue}
103 }
104 }
105 }()
106
107 return results
108 }
109
110 // Name returns the name of the source
111 func (s *Source) Name() string {
112 return "intelx"
113 }
+0
-169
pkg/subscraping/sources/ipv4info/ipv4info.go less more
0 package ipv4info
1
2 import (
3 "context"
4 "io/ioutil"
5 "regexp"
6 "strconv"
7
8 "github.com/projectdiscovery/subfinder/pkg/subscraping"
9 )
10
11 // Source is the passive scraping agent
12 type Source struct{}
13
14 // Run function returns all subdomains found with the service
15 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
16 results := make(chan subscraping.Result)
17
18 go func() {
19 resp, err := session.NormalGetWithContext(ctx, "http://ipv4info.com/search/"+domain)
20 if err != nil {
21 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
22 session.DiscardHttpResponse(resp)
23 close(results)
24 return
25 }
26
27 body, err := ioutil.ReadAll(resp.Body)
28 if err != nil {
29 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
30 resp.Body.Close()
31 close(results)
32 return
33 }
34 resp.Body.Close()
35 src := string(body)
36
37 regxTokens := regexp.MustCompile("/ip-address/(.*)/" + domain)
38 matchTokens := regxTokens.FindAllString(src, -1)
39
40 if len(matchTokens) <= 0 {
41 close(results)
42 return
43 }
44 token := matchTokens[0]
45
46 resp, err = session.NormalGetWithContext(ctx, "http://ipv4info.com"+token)
47 if err != nil {
48 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
49 close(results)
50 return
51 }
52
53 body, err = ioutil.ReadAll(resp.Body)
54 if err != nil {
55 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
56 resp.Body.Close()
57 close(results)
58 return
59 }
60 resp.Body.Close()
61 src = string(body)
62
63 regxTokens = regexp.MustCompile("/dns/(.*?)/" + domain)
64 matchTokens = regxTokens.FindAllString(src, -1)
65 if len(matchTokens) <= 0 {
66 close(results)
67 return
68 }
69 token = matchTokens[0]
70
71 resp, err = session.NormalGetWithContext(ctx, "http://ipv4info.com"+token)
72 if err != nil {
73 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
74 close(results)
75 return
76 }
77
78 body, err = ioutil.ReadAll(resp.Body)
79 if err != nil {
80 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
81 resp.Body.Close()
82 close(results)
83 return
84 }
85 resp.Body.Close()
86 src = string(body)
87
88 regxTokens = regexp.MustCompile("/subdomains/(.*?)/" + domain)
89 matchTokens = regxTokens.FindAllString(src, -1)
90 if len(matchTokens) <= 0 {
91 close(results)
92 return
93 }
94 token = matchTokens[0]
95
96 resp, err = session.NormalGetWithContext(ctx, "http://ipv4info.com"+token)
97 if err != nil {
98 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
99 close(results)
100 return
101 }
102
103 body, err = ioutil.ReadAll(resp.Body)
104 if err != nil {
105 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
106 resp.Body.Close()
107 close(results)
108 return
109 }
110 resp.Body.Close()
111 src = string(body)
112
113 for _, match := range session.Extractor.FindAllString(src, -1) {
114 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match}
115 }
116 nextPage := 1
117
118 for {
119 further := s.getSubdomains(ctx, domain, &nextPage, src, session, results)
120 if !further {
121 break
122 }
123 }
124 close(results)
125 }()
126
127 return results
128 }
129
130 // Name returns the name of the source
131 func (s *Source) Name() string {
132 return "ipv4info"
133 }
134
135 func (s *Source) getSubdomains(ctx context.Context, domain string, nextPage *int, src string, session *subscraping.Session, results chan subscraping.Result) bool {
136 for {
137 select {
138 case <-ctx.Done():
139 return false
140 default:
141 regxTokens := regexp.MustCompile("/subdomains/.*/page" + strconv.Itoa(*nextPage) + "/" + domain + ".html")
142 matchTokens := regxTokens.FindAllString(src, -1)
143 if len(matchTokens) == 0 {
144 return false
145 }
146 token := matchTokens[0]
147
148 resp, err := session.NormalGetWithContext(ctx, "http://ipv4info.com"+token)
149 if err != nil {
150 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
151 return false
152 }
153 body, err := ioutil.ReadAll(resp.Body)
154 if err != nil {
155 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
156 resp.Body.Close()
157 return false
158 }
159 resp.Body.Close()
160 src = string(body)
161 for _, match := range session.Extractor.FindAllString(src, -1) {
162 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match}
163 }
164 *nextPage++
165 return true
166 }
167 }
168 }
+0
-72
pkg/subscraping/sources/passivetotal/passivetotal.go less more
0 package passivetotal
1
2 import (
3 "bytes"
4 "context"
5 "net/http"
6
7 jsoniter "github.com/json-iterator/go"
8 "github.com/projectdiscovery/subfinder/pkg/subscraping"
9 )
10
11 type response struct {
12 Subdomains []string `json:"subdomains"`
13 }
14
15 // Source is the passive scraping agent
16 type Source struct{}
17
18 // Run function returns all subdomains found with the service
19 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
20 results := make(chan subscraping.Result)
21
22 go func() {
23 if session.Keys.PassiveTotalUsername == "" || session.Keys.PassiveTotalPassword == "" {
24 close(results)
25 return
26 }
27
28 // Create JSON Get body
29 var request = []byte(`{"query":"` + domain + `"}`)
30
31 req, err := http.NewRequestWithContext(ctx, "GET", "https://api.passivetotal.org/v2/enrichment/subdomains", bytes.NewBuffer(request))
32 if err != nil {
33 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
34 close(results)
35 return
36 }
37
38 req.SetBasicAuth(session.Keys.PassiveTotalUsername, session.Keys.PassiveTotalPassword)
39 req.Header.Set("Content-Type", "application/json")
40
41 resp, err := session.Client.Do(req)
42 if err != nil {
43 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
44 close(results)
45 return
46 }
47
48 data := response{}
49 err = jsoniter.NewDecoder(resp.Body).Decode(&data)
50 if err != nil {
51 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
52 resp.Body.Close()
53 close(results)
54 return
55 }
56 resp.Body.Close()
57
58 for _, subdomain := range data.Subdomains {
59 finalSubdomain := subdomain + "." + domain
60 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: finalSubdomain}
61 }
62 close(results)
63 }()
64
65 return results
66 }
67
68 // Name returns the name of the source
69 func (s *Source) Name() string {
70 return "passivetotal"
71 }
+0
-46
pkg/subscraping/sources/rapiddns/rapiddns.go less more
0 // Package rapiddns is a RapidDNS Scraping Engine in Golang
1 package rapiddns
2
3 import (
4 "context"
5 "io/ioutil"
6
7 "github.com/projectdiscovery/subfinder/pkg/subscraping"
8 )
9
10 // Source is the passive scraping agent
11 type Source struct{}
12
13 // Run function returns all subdomains found with the service
14 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
15 results := make(chan subscraping.Result)
16
17 go func() {
18 defer close(results)
19 resp, err := session.NormalGetWithContext(ctx, "https://rapiddns.io/subdomain/"+domain+"?full=1")
20 if err != nil {
21 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
22 session.DiscardHttpResponse(resp)
23 return
24 }
25
26 body, err := ioutil.ReadAll(resp.Body)
27 resp.Body.Close()
28 if err != nil {
29 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
30 return
31 }
32
33 src := string(body)
34 for _, subdomain := range session.Extractor.FindAllString(src, -1) {
35 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
36 }
37 }()
38
39 return results
40 }
41
42 // Name returns the name of the source
43 func (s *Source) Name() string {
44 return "rapiddns"
45 }
+0
-65
pkg/subscraping/sources/securitytrails/securitytrails.go less more
0 package securitytrails
1
2 import (
3 "context"
4 "fmt"
5 "strings"
6
7 jsoniter "github.com/json-iterator/go"
8 "github.com/projectdiscovery/subfinder/pkg/subscraping"
9 )
10
11 type response struct {
12 Subdomains []string `json:"subdomains"`
13 }
14
15 // Source is the passive scraping agent
16 type Source struct{}
17
18 // Run function returns all subdomains found with the service
19 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
20 results := make(chan subscraping.Result)
21
22 go func() {
23 if session.Keys.Securitytrails == "" {
24 close(results)
25 return
26 }
27
28 resp, err := session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", map[string]string{"APIKEY": session.Keys.Securitytrails})
29 if err != nil {
30 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
31 session.DiscardHttpResponse(resp)
32 close(results)
33 return
34 }
35
36 response := response{}
37 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
38 if err != nil {
39 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
40 resp.Body.Close()
41 close(results)
42 return
43 }
44 resp.Body.Close()
45
46 for _, subdomain := range response.Subdomains {
47 if strings.HasSuffix(subdomain, ".") {
48 subdomain = subdomain + domain
49 } else {
50 subdomain = subdomain + "." + domain
51 }
52
53 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
54 }
55 close(results)
56 }()
57
58 return results
59 }
60
61 // Name returns the name of the source
62 func (s *Source) Name() string {
63 return "securitytrails"
64 }
+0
-73
pkg/subscraping/sources/shodan/shodan.go less more
0 package shodan
1
2 import (
3 "context"
4 "strconv"
5
6 jsoniter "github.com/json-iterator/go"
7 "github.com/projectdiscovery/subfinder/pkg/subscraping"
8 )
9
10 type shodanResult struct {
11 Matches []shodanObject `json:"matches"`
12 Result int `json:"result"`
13 Error string `json:"error"`
14 }
15
16 type shodanObject struct {
17 Hostnames []string `json:"hostnames"`
18 }
19
20 // Source is the passive scraping agent
21 type Source struct{}
22
23 // Run function returns all subdomains found with the service
24 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
25 results := make(chan subscraping.Result)
26
27 go func() {
28 if session.Keys.Shodan == "" {
29 close(results)
30 return
31 }
32
33 for currentPage := 0; currentPage <= 10; currentPage++ {
34 resp, err := session.NormalGetWithContext(ctx, "https://api.shodan.io/shodan/host/search?query=hostname:"+domain+"&page="+strconv.Itoa(currentPage)+"&key="+session.Keys.Shodan)
35 if err != nil {
36 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
37 session.DiscardHttpResponse(resp)
38 close(results)
39 return
40 }
41
42 var response shodanResult
43 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
44 if err != nil {
45 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
46 resp.Body.Close()
47 close(results)
48 return
49 }
50 resp.Body.Close()
51
52 if response.Error != "" || len(response.Matches) == 0 {
53 close(results)
54 return
55 }
56
57 for _, block := range response.Matches {
58 for _, hostname := range block.Hostnames {
59 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname}
60 }
61 }
62 }
63 close(results)
64 }()
65
66 return results
67 }
68
69 // Name returns the name of the source
70 func (s *Source) Name() string {
71 return "shodan"
72 }
+0
-84
pkg/subscraping/sources/sitedossier/sitedossier.go less more
0 package sitedossier
1
2 import (
3 "context"
4 "fmt"
5 "io/ioutil"
6 "math/rand"
7 "regexp"
8 "time"
9
10 "github.com/projectdiscovery/subfinder/pkg/subscraping"
11 )
12
13 var reNext = regexp.MustCompile("<a href=\"([A-Za-z0-9\\/.]+)\"><b>")
14
15 type agent struct {
16 results chan subscraping.Result
17 session *subscraping.Session
18 }
19
20 func (a *agent) enumerate(ctx context.Context, baseURL string) error {
21 for {
22 select {
23 case <-ctx.Done():
24 return nil
25 default:
26 resp, err := a.session.NormalGetWithContext(ctx, baseURL)
27 if err != nil {
28 a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err}
29 a.session.DiscardHttpResponse(resp)
30 close(a.results)
31 return err
32 }
33
34 body, err := ioutil.ReadAll(resp.Body)
35 if err != nil {
36 a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err}
37 resp.Body.Close()
38 close(a.results)
39 return err
40 }
41 resp.Body.Close()
42 src := string(body)
43
44 for _, match := range a.session.Extractor.FindAllString(src, -1) {
45 a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: match}
46 }
47
48 match1 := reNext.FindStringSubmatch(src)
49 time.Sleep(time.Duration((3 + rand.Intn(5))) * time.Second)
50
51 if len(match1) > 0 {
52 a.enumerate(ctx, "http://www.sitedossier.com"+match1[1])
53 }
54 return nil
55 }
56 }
57 }
58
59 // Source is the passive scraping agent
60 type Source struct{}
61
62 // Run function returns all subdomains found with the service
63 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
64 results := make(chan subscraping.Result)
65
66 a := agent{
67 session: session,
68 results: results,
69 }
70
71 go func() {
72 err := a.enumerate(ctx, fmt.Sprintf("http://www.sitedossier.com/parentdomain/%s", domain))
73 if err == nil {
74 close(a.results)
75 }
76 }()
77 return results
78 }
79
80 // Name returns the name of the source
81 func (s *Source) Name() string {
82 return "sitedossier"
83 }
+0
-90
pkg/subscraping/sources/spyse/spyse.go less more
0 package spyse
1
2 import (
3 "context"
4 "strconv"
5 "fmt"
6
7 jsoniter "github.com/json-iterator/go"
8 "github.com/projectdiscovery/subfinder/pkg/subscraping"
9 )
10
11
12 type resultObject struct {
13 Name string `json:"name"`
14 }
15
16 type dataObject struct {
17 Items []resultObject `json:"items"`
18 Total_Count int `json:"total_count"`
19 }
20
21 type errorObject struct {
22 Code string `json:"code"`
23 Message string `json:"message"`
24 }
25
26
27 type spyseResult struct {
28 Data dataObject `json:"data"`
29 Error []errorObject `json:"error"`
30 }
31
32
33 type Source struct{}
34
35 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
36 results := make(chan subscraping.Result)
37
38 go func() {
39 if session.Keys.Spyse == "" {
40 close(results)
41 return
42 }
43
44 maxCount := 100;
45
46 for offSet := 0; offSet <= maxCount; offSet += 100 {
47 resp, err := session.Get(ctx, fmt.Sprintf("https://api.spyse.com/v3/data/domain/subdomain?domain=%s&limit=100&offset=%s", domain, strconv.Itoa(offSet)), "", map[string]string{"Authorization": "Bearer " + session.Keys.Spyse})
48 if err != nil {
49 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
50 session.DiscardHttpResponse(resp)
51 close(results)
52 return
53 }
54
55
56 var response spyseResult;
57
58 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
59
60 if err != nil {
61 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
62 resp.Body.Close()
63 close(results)
64 return
65 }
66 resp.Body.Close()
67
68 if response.Data.Total_Count == 0 {
69 close(results)
70 return
71 }
72
73 maxCount = response.Data.Total_Count;
74
75 for _, hostname := range response.Data.Items {
76 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Name}
77 }
78 }
79 close(results)
80 }()
81
82 return results
83 }
84
85
86 // Name returns the name of the source
87 func (s *Source) Name() string {
88 return "spyse"
89 }
+0
-49
pkg/subscraping/sources/sublist3r/subllist3r.go less more
0 package sublist3r
1
2 import (
3 "context"
4 "encoding/json"
5 "fmt"
6
7 "github.com/projectdiscovery/subfinder/pkg/subscraping"
8 )
9
10 // Source is the passive scraping agent
11 type Source struct{}
12
13 // Run function returns all subdomains found with the service
14 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
15 results := make(chan subscraping.Result)
16
17 go func() {
18 resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://api.sublist3r.com/search.php?domain=%s", domain))
19 if err != nil {
20 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
21 session.DiscardHttpResponse(resp)
22 close(results)
23 return
24 }
25 defer resp.Body.Close()
26 var subdomains []string
27 // Get the response body and unmarshal
28 err = json.NewDecoder(resp.Body).Decode(&subdomains)
29 if err != nil {
30 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
31 resp.Body.Close()
32 close(results)
33 return
34 }
35
36 for _, subdomain := range subdomains {
37 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
38 }
39 close(results)
40 }()
41
42 return results
43 }
44
45 // Name returns the name of the source
46 func (s *Source) Name() string {
47 return "sublist3r"
48 }
+0
-51
pkg/subscraping/sources/threatcrowd/threatcrowd.go less more
0 package threatcrowd
1
2 import (
3 "context"
4 "fmt"
5 "io/ioutil"
6
7 "github.com/projectdiscovery/subfinder/pkg/subscraping"
8 )
9
10 // Source is the passive scraping agent
11 type Source struct{}
12
13 // Run function returns all subdomains found with the service
14 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
15 results := make(chan subscraping.Result)
16
17 go func() {
18 resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://www.threatcrowd.org/searchApi/v2/domain/report/?domain=%s", domain))
19 if err != nil {
20 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
21 session.DiscardHttpResponse(resp)
22 close(results)
23 return
24 }
25
26 // Get the response body
27 body, err := ioutil.ReadAll(resp.Body)
28 if err != nil {
29 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
30 resp.Body.Close()
31 close(results)
32 return
33 }
34 resp.Body.Close()
35
36 src := string(body)
37
38 for _, match := range session.Extractor.FindAllString(src, -1) {
39 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match}
40 }
41 close(results)
42 }()
43
44 return results
45 }
46
47 // Name returns the name of the source
48 func (s *Source) Name() string {
49 return "threatcrowd"
50 }
+0
-51
pkg/subscraping/sources/threatminer/threatminer.go less more
0 package threatminer
1
2 import (
3 "context"
4 "fmt"
5 "io/ioutil"
6
7 "github.com/projectdiscovery/subfinder/pkg/subscraping"
8 )
9
10 // Source is the passive scraping agent
11 type Source struct{}
12
13 // Run function returns all subdomains found with the service
14 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
15 results := make(chan subscraping.Result)
16
17 go func() {
18 resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://api.threatminer.org/v2/domain.php?q=%s&rt=5", domain))
19 if err != nil {
20 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
21 session.DiscardHttpResponse(resp)
22 close(results)
23 return
24 }
25
26 // Get the response body
27 body, err := ioutil.ReadAll(resp.Body)
28 if err != nil {
29 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
30 resp.Body.Close()
31 close(results)
32 return
33 }
34 resp.Body.Close()
35
36 src := string(body)
37
38 for _, match := range session.Extractor.FindAllString(src, -1) {
39 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match}
40 }
41 close(results)
42 }()
43
44 return results
45 }
46
47 // Name returns the name of the source
48 func (s *Source) Name() string {
49 return "threatminer"
50 }
+0
-60
pkg/subscraping/sources/urlscan/urlscan.go less more
0 package urlscan
1
2 import (
3 "context"
4 "fmt"
5
6 jsoniter "github.com/json-iterator/go"
7 "github.com/m-mizutani/urlscan-go/urlscan"
8 "github.com/projectdiscovery/subfinder/pkg/subscraping"
9 )
10
11 // Source is the passive scraping agent
12 type Source struct{}
13
14 // Run function returns all subdomains found with the service
15 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
16 results := make(chan subscraping.Result)
17
18 go func() {
19 if session.Keys.URLScan == "" {
20 close(results)
21 return
22 }
23
24 client := urlscan.NewClient(session.Keys.URLScan)
25 task, err := client.Submit(urlscan.SubmitArguments{URL: fmt.Sprintf("https://%s", domain)})
26 if err != nil {
27 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
28 close(results)
29 return
30 }
31
32 err = task.Wait()
33 if err != nil {
34 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
35 close(results)
36 return
37 }
38
39 data, err := jsoniter.Marshal(task.Result.Data)
40 if err != nil {
41 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
42 close(results)
43 return
44 }
45
46 match := session.Extractor.FindAllString(string(data), -1)
47 for _, m := range match {
48 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: m}
49 }
50 close(results)
51 }()
52
53 return results
54 }
55
56 // Name returns the name of the source
57 func (s *Source) Name() string {
58 return "urlscan"
59 }
+0
-58
pkg/subscraping/sources/virustotal/virustotal.go less more
0 package virustotal
1
2 import (
3 "context"
4 "fmt"
5
6 jsoniter "github.com/json-iterator/go"
7 "github.com/projectdiscovery/subfinder/pkg/subscraping"
8 )
9
10 type response struct {
11 Subdomains []string `json:"subdomains"`
12 }
13
14 // Source is the passive scraping agent
15 type Source struct{}
16
17 // Run function returns all subdomains found with the service
18 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
19 results := make(chan subscraping.Result)
20
21 go func() {
22 if session.Keys.Virustotal == "" {
23 close(results)
24 return
25 }
26
27 resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://www.virustotal.com/vtapi/v2/domain/report?apikey=%s&domain=%s", session.Keys.Virustotal, domain))
28 if err != nil {
29 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
30 session.DiscardHttpResponse(resp)
31 close(results)
32 return
33 }
34
35 data := response{}
36 err = jsoniter.NewDecoder(resp.Body).Decode(&data)
37 if err != nil {
38 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
39 resp.Body.Close()
40 close(results)
41 return
42 }
43 resp.Body.Close()
44
45 for _, subdomain := range data.Subdomains {
46 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
47 }
48 close(results)
49 }()
50
51 return results
52 }
53
54 // Name returns the name of the source
55 func (s *Source) Name() string {
56 return "virustotal"
57 }
+0
-53
pkg/subscraping/sources/waybackarchive/waybackarchive.go less more
0 package waybackarchive
1
2 import (
3 "context"
4 "fmt"
5 "io/ioutil"
6 "strings"
7
8 "github.com/projectdiscovery/subfinder/pkg/subscraping"
9 )
10
11 // Source is the passive scraping agent
12 type Source struct{}
13
14 // Run function returns all subdomains found with the service
15 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
16 results := make(chan subscraping.Result)
17
18 go func() {
19 pagesResp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=json&fl=original&collapse=urlkey", domain))
20 if err != nil {
21 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
22 session.DiscardHttpResponse(pagesResp)
23 close(results)
24 return
25 }
26
27 body, err := ioutil.ReadAll(pagesResp.Body)
28 if err != nil {
29 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
30 pagesResp.Body.Close()
31 close(results)
32 return
33 }
34 pagesResp.Body.Close()
35
36 match := session.Extractor.FindAllString(string(body), -1)
37 for _, subdomain := range match {
38 subdomain = strings.TrimPrefix(subdomain, "25")
39 subdomain = strings.TrimPrefix(subdomain, "2F")
40
41 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
42 }
43 close(results)
44 }()
45
46 return results
47 }
48
49 // Name returns the name of the source
50 func (s *Source) Name() string {
51 return "waybackarchive"
52 }
+0
-138
pkg/subscraping/sources/zoomeye/zoomeye.go less more
0 package zoomeye
1
2 import (
3 "bytes"
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "io"
9 "io/ioutil"
10 "net/http"
11
12 "github.com/projectdiscovery/subfinder/pkg/subscraping"
13 )
14
15 // zoomAuth holds the ZoomEye credentials
16 type zoomAuth struct {
17 User string `json:"username"`
18 Pass string `json:"password"`
19 }
20
21 type loginResp struct {
22 JWT string `json:"access_token"`
23 }
24
25 // search results
26 type zoomeyeResults struct {
27 Matches []struct {
28 Site string `json:"site"`
29 Domains []string `json:"domains"`
30 } `json:"matches"`
31 }
32
33 // Source is the passive scraping agent
34 type Source struct{}
35
36 // Run function returns all subdomains found with the service
37 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
38 results := make(chan subscraping.Result)
39
40 go func() {
41 if session.Keys.ZoomEyeUsername == "" || session.Keys.ZoomEyePassword == "" {
42 close(results)
43 return
44 }
45 jwt, err := doLogin(session)
46 if err != nil {
47 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
48 close(results)
49 return
50 }
51 // check if jwt is null
52 if jwt == "" {
53 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errors.New("could not log into zoomeye")}
54 close(results)
55 return
56 }
57 headers := map[string]string{
58 "Authorization": fmt.Sprintf("JWT %s", jwt),
59 "Accept": "application/json",
60 "Content-Type": "application/json",
61 }
62 for currentPage := 0; currentPage <= 100; currentPage++ {
63 api := fmt.Sprintf("https://api.zoomeye.org/web/search?query=hostname:%s&page=%d", domain, currentPage)
64 resp, err := session.Get(ctx, api, "", headers)
65 isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden
66 if err != nil {
67 if !isForbidden && currentPage == 0 {
68 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
69 session.DiscardHttpResponse(resp)
70 }
71 close(results)
72 return
73 }
74
75 defer resp.Body.Close()
76 res := &zoomeyeResults{}
77 err = json.NewDecoder(resp.Body).Decode(res)
78 if err != nil {
79 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
80 resp.Body.Close()
81 close(results)
82 return
83 }
84 resp.Body.Close()
85 for _, r := range res.Matches {
86 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Site}
87 for _, domain := range r.Domains {
88 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domain}
89 }
90 }
91 currentPage++
92 }
93 close(results)
94 }()
95
96 return results
97 }
98
99 // doLogin performs authentication on the ZoomEye API
100 func doLogin(session *subscraping.Session) (string, error) {
101 creds := &zoomAuth{
102 User: session.Keys.ZoomEyeUsername,
103 Pass: session.Keys.ZoomEyePassword,
104 }
105 body, err := json.Marshal(&creds)
106 if err != nil {
107 return "", err
108 }
109 req, err := http.NewRequest("POST", "https://api.zoomeye.org/user/login", bytes.NewBuffer(body))
110 if err != nil {
111 return "", err
112 }
113 req.Header.Add("Content-Type", "application/json")
114 resp, err := session.Client.Do(req)
115 if err != nil {
116 return "", err
117 }
118 // if not 200, bad credentials
119 if resp.StatusCode != 200 {
120 io.Copy(ioutil.Discard, resp.Body)
121 resp.Body.Close()
122 return "", fmt.Errorf("login failed, non-200 response from zoomeye")
123 }
124
125 defer resp.Body.Close()
126 login := &loginResp{}
127 err = json.NewDecoder(resp.Body).Decode(login)
128 if err != nil {
129 return "", err
130 }
131 return login.JWT, nil
132 }
133
134 // Name returns the name of the source
135 func (s *Source) Name() string {
136 return "zoomeye"
137 }
+0
-67
pkg/subscraping/types.go less more
0 package subscraping
1
2 import (
3 "context"
4 "net/http"
5 "regexp"
6 )
7
8 // Source is an interface inherited by each passive source
9 type Source interface {
10 // Run takes a domain as argument and a session object
11 // which contains the extractor for subdomains, http client
12 // and other stuff.
13 Run(context.Context, string, *Session) <-chan Result
14 // Name returns the name of the source
15 Name() string
16 }
17
18 // Session is the option passed to the source, an option is created
19 // uniquely for eac source.
20 type Session struct {
21 // Extractor is the regex for subdomains created for each domain
22 Extractor *regexp.Regexp
23 // Keys is the API keys for the application
24 Keys Keys
25 // Client is the current http client
26 Client *http.Client
27 }
28
29 // Keys contains the current API Keys we have in store
30 type Keys struct {
31 Binaryedge string `json:"binaryedge"`
32 CensysToken string `json:"censysUsername"`
33 CensysSecret string `json:"censysPassword"`
34 Certspotter string `json:"certspotter"`
35 Chaos string `json:"chaos"`
36 DNSDB string `json:"dnsdb"`
37 GitHub []string `json:"github"`
38 IntelXHost string `json:"intelXHost"`
39 IntelXKey string `json:"intelXKey"`
40 PassiveTotalUsername string `json:"passivetotal_username"`
41 PassiveTotalPassword string `json:"passivetotal_password"`
42 Securitytrails string `json:"securitytrails"`
43 Shodan string `json:"shodan"`
44 Spyse string `json:"spyse"`
45 URLScan string `json:"urlscan"`
46 Virustotal string `json:"virustotal"`
47 ZoomEyeUsername string `json:"zoomeye_username"`
48 ZoomEyePassword string `json:"zoomeye_password"`
49 }
50
51 // Result is a result structure returned by a source
52 type Result struct {
53 Type ResultType
54 Source string
55 Value string
56 Error error
57 }
58
59 // ResultType is the type of result returned by the source
60 type ResultType int
61
62 // Types of results returned by the source
63 const (
64 Subdomain ResultType = iota
65 Error
66 )
+0
-30
pkg/subscraping/utils.go less more
0 package subscraping
1
2 import (
3 "regexp"
4 "sync"
5 )
6
7 var subdomainExtractorMutex = &sync.Mutex{}
8
9 // NewSubdomainExtractor creates a new regular expression to extract
10 // subdomains from text based on the given domain.
11 func NewSubdomainExtractor(domain string) (*regexp.Regexp, error) {
12 subdomainExtractorMutex.Lock()
13 defer subdomainExtractorMutex.Unlock()
14 extractor, err := regexp.Compile(`[a-zA-Z0-9\*_.-]+\.` + domain)
15 if err != nil {
16 return nil, err
17 }
18 return extractor, nil
19 }
20
21 // Exists check if a key exist in a slice
22 func Exists(values []string, key string) bool {
23 for _, v := range values {
24 if v == key {
25 return true
26 }
27 }
28 return false
29 }
+25
-0
v2/cmd/subfinder/main.go less more
0 package main
1
2 import (
3 "context"
4
5 // Attempts to increase the OS file descriptors - Fail silently
6 _ "github.com/projectdiscovery/fdmax/autofdmax"
7 "github.com/projectdiscovery/gologger"
8 "github.com/projectdiscovery/subfinder/v2/pkg/runner"
9 )
10
11 func main() {
12 // Parse the command line flags and read config files
13 options := runner.ParseOptions()
14
15 newRunner, err := runner.NewRunner(options)
16 if err != nil {
17 gologger.Fatalf("Could not create runner: %s\n", err)
18 }
19
20 err = newRunner.RunEnumeration(context.Background())
21 if err != nil {
22 gologger.Fatalf("Could not run enumeration: %s\n", err)
23 }
24 }
+23
-0
v2/go.mod less more
0 module github.com/projectdiscovery/subfinder/v2
1
2 go 1.14
3
4 require (
5 github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026
6 github.com/json-iterator/go v1.1.10
7 github.com/lib/pq v1.8.0
8 github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
9 github.com/miekg/dns v1.1.31
10 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
11 github.com/modern-go/reflect2 v1.0.1 // indirect
12 github.com/pkg/errors v0.9.1
13 github.com/projectdiscovery/chaos-client v0.1.6
14 github.com/projectdiscovery/fdmax v0.0.2
15 github.com/projectdiscovery/gologger v1.0.1
16 github.com/rs/xid v1.2.1
17 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80
18 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect
19 golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321 // indirect
20 golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d // indirect
21 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
22 )
+68
-0
v2/go.sum less more
0 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
1 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
2 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
3 github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 h1:BpJ2o0OR5FV7vrkDYfXYVJQeMNWa8RhklZOpW2ITAIQ=
4 github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE=
5 github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
6 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
7 github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg=
8 github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
9 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
10 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
11 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
12 github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
13 github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo=
14 github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
15 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
16 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
17 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
18 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
19 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
20 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
21 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
22 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
23 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
24 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
25 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
26 github.com/projectdiscovery/chaos-client v0.1.6 h1:AbIN7xUszjUi7FxI4qUVSqJ3um+6eImE/xstbNS0A1M=
27 github.com/projectdiscovery/chaos-client v0.1.6/go.mod h1:F5omaoJh/vMvWnZhKD4zFFA5ti+RPwUletwepKSyfxk=
28 github.com/projectdiscovery/fdmax v0.0.2 h1:d0HqNC4kbrMWT669u9W7ksFS7UBvnW0zmgY6FBU45UY=
29 github.com/projectdiscovery/fdmax v0.0.2/go.mod h1:mbR7lJ9EONyxEfcsL2LlGtOSlzCQ5VraLzoJa/VTrAs=
30 github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
31 github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ=
32 github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
33 github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
34 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
35 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
36 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
37 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
38 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y=
39 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE=
40 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
41 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
42 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
43 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
44 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
45 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
46 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
47 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
48 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
49 golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
50 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
51 golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321 h1:lleNcKRbcaC8MqgLwghIkzZ2JBQAb7QQ9MiwRt1BisA=
52 golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
53 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
54 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
55 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
56 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M=
57 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
58 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
59 golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d h1:L/IKR6COd7ubZrs2oTnTi73IhgqJ71c9s80WsQnh0Es=
60 golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
61 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
62 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
63 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
64 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
65 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
66 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
67 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+3
-0
v2/pkg/passive/doc.go less more
0 // Package passive provides capability for doing passive subdomain
1 // enumeration on targets.
2 package passive
+58
-0
v2/pkg/passive/passive.go less more
0 package passive
1
2 import (
3 "context"
4 "fmt"
5 "sync"
6 "time"
7
8 "github.com/projectdiscovery/gologger"
9 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
10 )
11
12 // EnumerateSubdomains enumerates all the subdomains for a given domain
13 func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, timeout int, maxEnumTime time.Duration) chan subscraping.Result {
14 results := make(chan subscraping.Result)
15
16 go func() {
17 session, err := subscraping.NewSession(domain, keys, timeout)
18 if err != nil {
19 results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)}
20 }
21
22 ctx, cancel := context.WithTimeout(context.Background(), maxEnumTime)
23
24 timeTaken := make(map[string]string)
25 timeTakenMutex := &sync.Mutex{}
26
27 wg := &sync.WaitGroup{}
28 // Run each source in parallel on the target domain
29 for source, runner := range a.sources {
30 wg.Add(1)
31
32 now := time.Now()
33 go func(source string, runner subscraping.Source) {
34 for resp := range runner.Run(ctx, domain, session) {
35 results <- resp
36 }
37
38 duration := time.Since(now)
39 timeTakenMutex.Lock()
40 timeTaken[source] = fmt.Sprintf("Source took %s for enumeration\n", duration)
41 timeTakenMutex.Unlock()
42
43 wg.Done()
44 }(source, runner)
45 }
46 wg.Wait()
47
48 for source, data := range timeTaken {
49 gologger.Verbosef(data, source)
50 }
51
52 close(results)
53 cancel()
54 }()
55
56 return results
57 }
+228
-0
v2/pkg/passive/sources.go less more
0 package passive
1
2 import (
3 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
4 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/alienvault"
5 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/anubis"
6 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/archiveis"
7 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/binaryedge"
8 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover"
9 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/cebaidu"
10 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys"
11 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter"
12 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotterold"
13 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos"
14 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl"
15 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh"
16 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb"
17 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster"
18 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github"
19 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget"
20 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx"
21 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/ipv4info"
22 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/passivetotal"
23 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns"
24 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/recon"
25 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/riddler"
26 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/robtex"
27 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails"
28 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan"
29 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier"
30 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/spyse"
31 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sublist3r"
32 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook"
33 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd"
34 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatminer"
35 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal"
36 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive"
37 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/ximcx"
38 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeye"
39 )
40
41 // DefaultSources contains the list of fast sources used by default.
42 var DefaultSources = []string{
43 "alienvault",
44 "anubis",
45 "bufferover",
46 "cebaidu",
47 "certspotter",
48 "certspotterold",
49 "censys",
50 "chaos",
51 "crtsh",
52 "dnsdumpster",
53 "hackertarget",
54 "intelx",
55 "ipv4info",
56 "passivetotal",
57 "robtex",
58 "riddler",
59 "securitytrails",
60 "shodan",
61 "spyse",
62 "sublist3r",
63 "threatcrowd",
64 "threatminer",
65 "virustotal",
66 }
67
68 // DefaultRecursiveSources contains list of default recursive sources
69 var DefaultRecursiveSources = []string{
70 "alienvault",
71 "binaryedge",
72 "bufferover",
73 "cebaidu",
74 "certspotter",
75 "certspotterold",
76 "crtsh",
77 "dnsdumpster",
78 "hackertarget",
79 "ipv4info",
80 "passivetotal",
81 "securitytrails",
82 "sublist3r",
83 "virustotal",
84 "ximcx",
85 }
86
87 // DefaultAllSources contains list of all sources
88 var DefaultAllSources = []string{
89 "alienvault",
90 "anubis",
91 "archiveis",
92 "binaryedge",
93 "bufferover",
94 "cebaidu",
95 "censys",
96 "certspotter",
97 "certspotterold",
98 "chaos",
99 "commoncrawl",
100 "crtsh",
101 "dnsdumpster",
102 "dnsdb",
103 "github",
104 "hackertarget",
105 "ipv4info",
106 "intelx",
107 "passivetotal",
108 "rapiddns",
109 "riddler",
110 "recon",
111 "robtex",
112 "securitytrails",
113 "shodan",
114 "sitedossier",
115 "spyse",
116 "sublist3r",
117 "threatbook",
118 "threatcrowd",
119 "threatminer",
120 "virustotal",
121 "waybackarchive",
122 "ximcx",
123 "zoomeye",
124 }
125
126 // Agent is a struct for running passive subdomain enumeration
127 // against a given host. It wraps subscraping package and provides
128 // a layer to build upon.
129 type Agent struct {
130 sources map[string]subscraping.Source
131 }
132
133 // New creates a new agent for passive subdomain discovery
134 func New(sources, exclusions []string) *Agent {
135 // Create the agent, insert the sources and remove the excluded sources
136 agent := &Agent{sources: make(map[string]subscraping.Source)}
137
138 agent.addSources(sources)
139 agent.removeSources(exclusions)
140
141 return agent
142 }
143
144 // addSources adds the given list of sources to the source array
145 func (a *Agent) addSources(sources []string) {
146 for _, source := range sources {
147 switch source {
148 case "alienvault":
149 a.sources[source] = &alienvault.Source{}
150 case "anubis":
151 a.sources[source] = &anubis.Source{}
152 case "archiveis":
153 a.sources[source] = &archiveis.Source{}
154 case "binaryedge":
155 a.sources[source] = &binaryedge.Source{}
156 case "bufferover":
157 a.sources[source] = &bufferover.Source{}
158 case "cebaidu":
159 a.sources[source] = &cebaidu.Source{}
160 case "censys":
161 a.sources[source] = &censys.Source{}
162 case "certspotter":
163 a.sources[source] = &certspotter.Source{}
164 case "certspotterold":
165 a.sources[source] = &certspotterold.Source{}
166 case "chaos":
167 a.sources[source] = &chaos.Source{}
168 case "commoncrawl":
169 a.sources[source] = &commoncrawl.Source{}
170 case "crtsh":
171 a.sources[source] = &crtsh.Source{}
172 case "dnsdumpster":
173 a.sources[source] = &dnsdumpster.Source{}
174 case "dnsdb":
175 a.sources[source] = &dnsdb.Source{}
176 case "github":
177 a.sources[source] = &github.Source{}
178 case "hackertarget":
179 a.sources[source] = &hackertarget.Source{}
180 case "ipv4info":
181 a.sources[source] = &ipv4info.Source{}
182 case "intelx":
183 a.sources[source] = &intelx.Source{}
184 case "passivetotal":
185 a.sources[source] = &passivetotal.Source{}
186 case "rapiddns":
187 a.sources[source] = &rapiddns.Source{}
188 case "recon":
189 a.sources[source] = &recon.Source{}
190 case "riddler":
191 a.sources[source] = &riddler.Source{}
192 case "robtex":
193 a.sources[source] = &robtex.Source{}
194 case "securitytrails":
195 a.sources[source] = &securitytrails.Source{}
196 case "shodan":
197 a.sources[source] = &shodan.Source{}
198 case "sitedossier":
199 a.sources[source] = &sitedossier.Source{}
200 case "spyse":
201 a.sources[source] = &spyse.Source{}
202 case "sublist3r":
203 a.sources[source] = &sublist3r.Source{}
204 case "threatbook":
205 a.sources[source] = &threatbook.Source{}
206 case "threatcrowd":
207 a.sources[source] = &threatcrowd.Source{}
208 case "threatminer":
209 a.sources[source] = &threatminer.Source{}
210 case "virustotal":
211 a.sources[source] = &virustotal.Source{}
212 case "waybackarchive":
213 a.sources[source] = &waybackarchive.Source{}
214 case "ximcx":
215 a.sources[source] = &ximcx.Source{}
216 case "zoomeye":
217 a.sources[source] = &zoomeye.Source{}
218 }
219 }
220 }
221
222 // removeSources deletes the given sources from the source map
223 func (a *Agent) removeSources(sources []string) {
224 for _, source := range sources {
225 delete(a.sources, source)
226 }
227 }
+59
-0
v2/pkg/resolve/client.go less more
0 package resolve
1
2 import (
3 "bufio"
4 "math/rand"
5 "os"
6 "time"
7 )
8
9 // DefaultResolvers contains the default list of resolvers known to be good
10 var DefaultResolvers = []string{
11 "1.1.1.1", // Cloudflare primary
12 "1.0.0.1", // Cloudlfare secondary
13 "8.8.8.8", // Google primary
14 "8.8.4.4", // Google secondary
15 "9.9.9.9", // Quad9 Primary
16 "9.9.9.10", // Quad9 Secondary
17 "77.88.8.8", // Yandex Primary
18 "77.88.8.1", // Yandex Secondary
19 "208.67.222.222", // OpenDNS Primary
20 "208.67.220.220", // OpenDNS Secondary
21 }
22
23 // Resolver is a struct for resolving DNS names
24 type Resolver struct {
25 resolvers []string
26 rand *rand.Rand
27 }
28
29 // New creates a new resolver struct with the default resolvers
30 func New() *Resolver {
31 return &Resolver{
32 resolvers: []string{},
33 rand: rand.New(rand.NewSource(time.Now().UnixNano())),
34 }
35 }
36
37 // AppendResolversFromFile appends the resolvers read from a file to the list of resolvers
38 func (r *Resolver) AppendResolversFromFile(file string) error {
39 f, err := os.Open(file)
40 if err != nil {
41 return err
42 }
43 scanner := bufio.NewScanner(f)
44 for scanner.Scan() {
45 text := scanner.Text()
46 if text == "" {
47 continue
48 }
49 r.resolvers = append(r.resolvers, text)
50 }
51 f.Close()
52 return scanner.Err()
53 }
54
55 // AppendResolversFromSlice appends the slice to the list of resolvers
56 func (r *Resolver) AppendResolversFromSlice(list []string) {
57 r.resolvers = append(r.resolvers, list...)
58 }
+3
-0
v2/pkg/resolve/doc.go less more
0 // Package resolve is used to handle resolving records
1 // It also handles wildcard subdomains and rotating resolvers.
2 package resolve
+157
-0
v2/pkg/resolve/resolve.go less more
0 package resolve
1
2 import (
3 "sync"
4
5 "github.com/miekg/dns"
6 "github.com/rs/xid"
7 )
8
9 const (
10 maxResolveRetries = 5
11 maxWildcardChecks = 3
12 )
13
14 // ResolutionPool is a pool of resolvers created for resolving subdomains
15 // for a given host.
16 type ResolutionPool struct {
17 *Resolver
18 Tasks chan HostEntry
19 Results chan Result
20 wg *sync.WaitGroup
21 removeWildcard bool
22
23 wildcardIPs map[string]struct{}
24 }
25
26 // HostEntry defines a host with the source
27 type HostEntry struct {
28 Host string `json:"host"`
29 Source string `json:"source"`
30 }
31
32 // Result contains the result for a host resolution
33 type Result struct {
34 Type ResultType
35 Host string
36 IP string
37 Error error
38 Source string
39 }
40
41 // ResultType is the type of result found
42 type ResultType int
43
44 // Types of data result can return
45 const (
46 Subdomain ResultType = iota
47 Error
48 )
49
50 // NewResolutionPool creates a pool of resolvers for resolving subdomains of a given domain
51 func (r *Resolver) NewResolutionPool(workers int, removeWildcard bool) *ResolutionPool {
52 resolutionPool := &ResolutionPool{
53 Resolver: r,
54 Tasks: make(chan HostEntry),
55 Results: make(chan Result),
56 wg: &sync.WaitGroup{},
57 removeWildcard: removeWildcard,
58 wildcardIPs: make(map[string]struct{}),
59 }
60
61 go func() {
62 for i := 0; i < workers; i++ {
63 resolutionPool.wg.Add(1)
64 go resolutionPool.resolveWorker()
65 }
66 resolutionPool.wg.Wait()
67 close(resolutionPool.Results)
68 }()
69
70 return resolutionPool
71 }
72
73 // InitWildcards inits the wildcard ips array
74 func (r *ResolutionPool) InitWildcards(domain string) error {
75 for i := 0; i < maxWildcardChecks; i++ {
76 uid := xid.New().String()
77
78 hosts, err := r.getARecords(HostEntry{Host: uid + "." + domain})
79 if err != nil {
80 return err
81 }
82
83 // Append all wildcard ips found for domains
84 for _, host := range hosts {
85 r.wildcardIPs[host] = struct{}{}
86 }
87 }
88 return nil
89 }
90
91 func (r *ResolutionPool) resolveWorker() {
92 for task := range r.Tasks {
93 if !r.removeWildcard {
94 r.Results <- Result{Type: Subdomain, Host: task.Host, IP: "", Source: task.Source}
95 continue
96 }
97
98 hosts, err := r.getARecords(task)
99 if err != nil {
100 r.Results <- Result{Type: Error, Host: task.Host, Source: task.Source, Error: err}
101 continue
102 }
103
104 if len(hosts) == 0 {
105 continue
106 }
107
108 for _, host := range hosts {
109 // Ignore the host if it exists in wildcard ips map
110 if _, ok := r.wildcardIPs[host]; ok { //nolint:staticcheck //search alternatives for "comma ok"
111 continue
112 }
113 }
114
115 r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source}
116 }
117 r.wg.Done()
118 }
119
120 // getARecords gets all the A records for a given host
121 func (r *ResolutionPool) getARecords(hostEntry HostEntry) ([]string, error) {
122 var iteration int
123
124 m := new(dns.Msg)
125 m.Id = dns.Id()
126 m.RecursionDesired = true
127 m.Question = make([]dns.Question, 1)
128 m.Question[0] = dns.Question{
129 Name: dns.Fqdn(hostEntry.Host),
130 Qtype: dns.TypeA,
131 Qclass: dns.ClassINET,
132 }
133 exchange:
134 iteration++
135 in, err := dns.Exchange(m, r.resolvers[r.rand.Intn(len(r.resolvers))]+":53")
136 if err != nil {
137 // Retry in case of I/O error
138 if iteration <= maxResolveRetries {
139 goto exchange
140 }
141 return nil, err
142 }
143 // Ignore the error in case we have bad result
144 if in != nil && in.Rcode != dns.RcodeSuccess {
145 return nil, nil
146 }
147
148 var hosts []string
149 for _, record := range in.Answer {
150 if t, ok := record.(*dns.A); ok {
151 hosts = append(hosts, t.A.String())
152 }
153 }
154
155 return hosts, nil
156 }
+75
-0
v2/pkg/runner/banners.go less more
0 package runner
1
2 import (
3 "github.com/projectdiscovery/gologger"
4 "github.com/projectdiscovery/subfinder/v2/pkg/passive"
5 "github.com/projectdiscovery/subfinder/v2/pkg/resolve"
6 )
7
8 const banner = `
9 _ __ _ _
10 ____ _| |__ / _(_)_ _ __| |___ _ _
11 (_-< || | '_ \ _| | ' \/ _ / -_) '_|
12 /__/\_,_|_.__/_| |_|_||_\__,_\___|_| v2.4.5
13 `
14
15 // Version is the current version of subfinder
16 const Version = `2.4.5`
17
18 // showBanner is used to show the banner to the user
19 func showBanner() {
20 gologger.Printf("%s\n", banner)
21 gologger.Printf("\t\tprojectdiscovery.io\n\n")
22
23 gologger.Labelf("Use with caution. You are responsible for your actions\n")
24 gologger.Labelf("Developers assume no liability and are not responsible for any misuse or damage.\n")
25 gologger.Labelf("By using subfinder, you also agree to the terms of the APIs used.\n\n")
26 }
27
28 // normalRunTasks runs the normal startup tasks
29 func (options *Options) normalRunTasks() {
30 configFile, err := UnmarshalRead(options.ConfigFile)
31 if err != nil {
32 gologger.Fatalf("Could not read configuration file %s: %s\n", options.ConfigFile, err)
33 }
34
35 // If we have a different version of subfinder installed
36 // previously, use the new iteration of config file.
37 if configFile.Version != Version {
38 configFile.Sources = passive.DefaultSources
39 configFile.AllSources = passive.DefaultAllSources
40 configFile.Recursive = passive.DefaultRecursiveSources
41 configFile.Version = Version
42
43 err = configFile.MarshalWrite(options.ConfigFile)
44 if err != nil {
45 gologger.Fatalf("Could not update configuration file to %s: %s\n", options.ConfigFile, err)
46 }
47 }
48 options.YAMLConfig = configFile
49 }
50
51 // firstRunTasks runs some housekeeping tasks done
52 // when the program is ran for the first time
53 func (options *Options) firstRunTasks() {
54 // Create the configuration file and display information
55 // about it to the user.
56 config := ConfigFile{
57 // Use the default list of resolvers by marshaling it to the config
58 Resolvers: resolve.DefaultResolvers,
59 // Use the default list of passive sources
60 Sources: passive.DefaultSources,
61 // Use the default list of all passive sources
62 AllSources: passive.DefaultAllSources,
63 // Use the default list of recursive sources
64 Recursive: passive.DefaultRecursiveSources,
65 }
66
67 err := config.MarshalWrite(options.ConfigFile)
68 if err != nil {
69 gologger.Fatalf("Could not write configuration file to %s: %s\n", options.ConfigFile, err)
70 }
71 options.YAMLConfig = config
72
73 gologger.Infof("Configuration file saved to %s\n", options.ConfigFile)
74 }
+55
-0
v2/pkg/runner/chaosuploader.go less more
0 package runner
1
2 import (
3 "context"
4 "crypto/tls"
5 "fmt"
6 "io"
7 "io/ioutil"
8 "net/http"
9 "time"
10
11 "github.com/pkg/errors"
12 "github.com/projectdiscovery/gologger"
13 )
14
15 // UploadToChaosTimeoutNano timeout to upload to Chaos in nanoseconds
16 const UploadToChaosTimeoutNano = 600
17
18 // UploadToChaos upload new data to Chaos dataset
19 func (r *Runner) UploadToChaos(ctx context.Context, reader io.Reader) error {
20 httpClient := &http.Client{
21 Transport: &http.Transport{
22 MaxIdleConnsPerHost: 100,
23 MaxIdleConns: 100,
24 TLSClientConfig: &tls.Config{
25 InsecureSkipVerify: true,
26 },
27 },
28 Timeout: time.Duration(UploadToChaosTimeoutNano) * time.Second, // 10 minutes - uploads may take long
29 }
30
31 request, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://dns.projectdiscovery.io/dns/add", reader)
32 if err != nil {
33 return errors.Wrap(err, "could not create request")
34 }
35 request.Header.Set("Authorization", r.options.YAMLConfig.GetKeys().Chaos)
36
37 resp, err := httpClient.Do(request)
38 if err != nil {
39 return errors.Wrap(err, "could not make request")
40 }
41 defer func() {
42 _, err := io.Copy(ioutil.Discard, resp.Body)
43 if err != nil {
44 gologger.Warningf("Could not discard response body: %s\n", err)
45 return
46 }
47 resp.Body.Close()
48 }()
49
50 if resp.StatusCode != http.StatusOK {
51 return fmt.Errorf("invalid status code received: %d", resp.StatusCode)
52 }
53 return nil
54 }
+199
-0
v2/pkg/runner/config.go less more
0 package runner
1
2 import (
3 "math/rand"
4 "os"
5 "strings"
6 "time"
7
8 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
9 "gopkg.in/yaml.v3"
10 )
11
12 // MultipleKeyPartsLength is the max length for multiple keys
13 const MultipleKeyPartsLength = 2
14
15 // YAMLIndentCharLength number of chars for identation on write YAML to file
16 const YAMLIndentCharLength = 4
17
18 // ConfigFile contains the fields stored in the configuration file
19 type ConfigFile struct {
20 // Resolvers contains the list of resolvers to use while resolving
21 Resolvers []string `yaml:"resolvers,omitempty"`
22 // Sources contains a list of sources to use for enumeration
23 Sources []string `yaml:"sources,omitempty"`
24 // AllSources contains the list of all sources for enumeration (slow)
25 AllSources []string `yaml:"all-sources,omitempty"`
26 // Recrusive contains the list of recursive subdomain enum sources
27 Recursive []string `yaml:"recursive,omitempty"`
28 // ExcludeSources contains the sources to not include in the enumeration process
29 ExcludeSources []string `yaml:"exclude-sources,omitempty"`
30 // API keys for different sources
31 Binaryedge []string `yaml:"binaryedge"`
32 Censys []string `yaml:"censys"`
33 Certspotter []string `yaml:"certspotter"`
34 Chaos []string `yaml:"chaos"`
35 DNSDB []string `yaml:"dnsdb"`
36 GitHub []string `yaml:"github"`
37 IntelX []string `yaml:"intelx"`
38 PassiveTotal []string `yaml:"passivetotal"`
39 Recon []string `yaml:"recon"`
40 Robtex []string `yaml:"robtex"`
41 SecurityTrails []string `yaml:"securitytrails"`
42 Shodan []string `yaml:"shodan"`
43 Spyse []string `yaml:"spyse"`
44 ThreatBook []string `yaml:"threatbook"`
45 URLScan []string `yaml:"urlscan"`
46 Virustotal []string `yaml:"virustotal"`
47 ZoomEye []string `yaml:"zoomeye"`
48 // Version indicates the version of subfinder installed.
49 Version string `yaml:"subfinder-version"`
50 }
51
52 // GetConfigDirectory gets the subfinder config directory for a user
53 func GetConfigDirectory() (string, error) {
54 // Seed the random number generator
55 rand.Seed(time.Now().UnixNano())
56
57 var config string
58
59 directory, err := os.UserHomeDir()
60 if err != nil {
61 return config, err
62 }
63 config = directory + "/.config/subfinder"
64
65 // Create All directory for subfinder even if they exist
66 err = os.MkdirAll(config, os.ModePerm)
67 if err != nil {
68 return config, err
69 }
70
71 return config, nil
72 }
73
74 // CheckConfigExists checks if the config file exists in the given path
75 func CheckConfigExists(configPath string) bool {
76 if _, err := os.Stat(configPath); err == nil {
77 return true
78 } else if os.IsNotExist(err) {
79 return false
80 }
81 return false
82 }
83
84 // MarshalWrite writes the marshaled yaml config to disk
85 func (c *ConfigFile) MarshalWrite(file string) error {
86 f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
87 if err != nil {
88 return err
89 }
90
91 // Indent the spaces too
92 enc := yaml.NewEncoder(f)
93 enc.SetIndent(YAMLIndentCharLength)
94 err = enc.Encode(&c)
95 f.Close()
96 return err
97 }
98
99 // UnmarshalRead reads the unmarshalled config yaml file from disk
100 func UnmarshalRead(file string) (ConfigFile, error) {
101 config := ConfigFile{}
102
103 f, err := os.Open(file)
104 if err != nil {
105 return config, err
106 }
107 err = yaml.NewDecoder(f).Decode(&config)
108 f.Close()
109 return config, err
110 }
111
112 // GetKeys gets the API keys from config file and creates a Keys struct
113 // We use random selection of api keys from the list of keys supplied.
114 // Keys that require 2 options are separated by colon (:).
115 func (c *ConfigFile) GetKeys() subscraping.Keys {
116 keys := subscraping.Keys{}
117
118 if len(c.Binaryedge) > 0 {
119 keys.Binaryedge = c.Binaryedge[rand.Intn(len(c.Binaryedge))]
120 }
121
122 if len(c.Censys) > 0 {
123 censysKeys := c.Censys[rand.Intn(len(c.Censys))]
124 parts := strings.Split(censysKeys, ":")
125 if len(parts) == MultipleKeyPartsLength {
126 keys.CensysToken = parts[0]
127 keys.CensysSecret = parts[1]
128 }
129 }
130
131 if len(c.Certspotter) > 0 {
132 keys.Certspotter = c.Certspotter[rand.Intn(len(c.Certspotter))]
133 }
134 if len(c.Chaos) > 0 {
135 keys.Chaos = c.Chaos[rand.Intn(len(c.Chaos))]
136 }
137 if (len(c.DNSDB)) > 0 {
138 keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))]
139 }
140 if (len(c.GitHub)) > 0 {
141 keys.GitHub = c.GitHub
142 }
143
144 if len(c.IntelX) > 0 {
145 intelxKeys := c.IntelX[rand.Intn(len(c.IntelX))]
146 parts := strings.Split(intelxKeys, ":")
147 if len(parts) == MultipleKeyPartsLength {
148 keys.IntelXHost = parts[0]
149 keys.IntelXKey = parts[1]
150 }
151 }
152
153 if len(c.PassiveTotal) > 0 {
154 passiveTotalKeys := c.PassiveTotal[rand.Intn(len(c.PassiveTotal))]
155 parts := strings.Split(passiveTotalKeys, ":")
156 if len(parts) == MultipleKeyPartsLength {
157 keys.PassiveTotalUsername = parts[0]
158 keys.PassiveTotalPassword = parts[1]
159 }
160 }
161
162 if len(c.Recon) > 0 {
163 keys.Recon = c.Recon[rand.Intn(len(c.Recon))]
164 }
165
166 if len(c.Robtex) > 0 {
167 keys.Robtex = c.Robtex[rand.Intn(len(c.Robtex))]
168 }
169
170 if len(c.SecurityTrails) > 0 {
171 keys.Securitytrails = c.SecurityTrails[rand.Intn(len(c.SecurityTrails))]
172 }
173 if len(c.Shodan) > 0 {
174 keys.Shodan = c.Shodan[rand.Intn(len(c.Shodan))]
175 }
176 if len(c.Spyse) > 0 {
177 keys.Spyse = c.Spyse[rand.Intn(len(c.Spyse))]
178 }
179 if len(c.ThreatBook) > 0 {
180 keys.ThreatBook = c.ThreatBook[rand.Intn(len(c.ThreatBook))]
181 }
182 if len(c.URLScan) > 0 {
183 keys.URLScan = c.URLScan[rand.Intn(len(c.URLScan))]
184 }
185 if len(c.Virustotal) > 0 {
186 keys.Virustotal = c.Virustotal[rand.Intn(len(c.Virustotal))]
187 }
188 if len(c.ZoomEye) > 0 {
189 zoomEyeKeys := c.ZoomEye[rand.Intn(len(c.ZoomEye))]
190 parts := strings.Split(zoomEyeKeys, ":")
191 if len(parts) == MultipleKeyPartsLength {
192 keys.ZoomEyeUsername = parts[0]
193 keys.ZoomEyePassword = parts[1]
194 }
195 }
196
197 return keys
198 }
+22
-0
v2/pkg/runner/config_test.go less more
0 package runner
1
2 import (
3 "os"
4 "testing"
5
6 "github.com/stretchr/testify/assert"
7 )
8
9 func TestConfigGetDirectory(t *testing.T) {
10 directory, err := GetConfigDirectory()
11 if err != nil {
12 t.Fatalf("Expected nil got %v while getting home\n", err)
13 }
14 home, err := os.UserHomeDir()
15 if err != nil {
16 t.Fatalf("Expected nil got %v while getting dir\n", err)
17 }
18 config := home + "/.config/subfinder"
19
20 assert.Equal(t, directory, config, "Directory and config should be equal")
21 }
+3
-0
v2/pkg/runner/doc.go less more
0 // Package runner implements the mechanism to drive the
1 // subdomain enumeration process
2 package runner
+180
-0
v2/pkg/runner/enumerate.go less more
0 package runner
1
2 import (
3 "bytes"
4 "context"
5 "os"
6 "strings"
7 "sync"
8 "time"
9
10 "github.com/hako/durafmt"
11 "github.com/projectdiscovery/gologger"
12 "github.com/projectdiscovery/subfinder/v2/pkg/resolve"
13 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
14 )
15
16 const maxNumCount = 2
17
18 // EnumerateSingleDomain performs subdomain enumeration against a single domain
19 func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output string, appendToFile bool) error {
20 gologger.Infof("Enumerating subdomains for %s\n", domain)
21
22 // Get the API keys for sources from the configuration
23 // and also create the active resolving engine for the domain.
24 keys := r.options.YAMLConfig.GetKeys()
25
26 // Check if the user has asked to remove wildcards explicitly.
27 // If yes, create the resolution pool and get the wildcards for the current domain
28 var resolutionPool *resolve.ResolutionPool
29 if r.options.RemoveWildcard {
30 resolutionPool = r.resolverClient.NewResolutionPool(r.options.Threads, r.options.RemoveWildcard)
31 err := resolutionPool.InitWildcards(domain)
32 if err != nil {
33 // Log the error but don't quit.
34 gologger.Warningf("Could not get wildcards for domain %s: %s\n", domain, err)
35 }
36 }
37
38 // Run the passive subdomain enumeration
39 now := time.Now()
40 passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute)
41
42 wg := &sync.WaitGroup{}
43 wg.Add(1)
44 // Create a unique map for filtering duplicate subdomains out
45 uniqueMap := make(map[string]resolve.HostEntry)
46 // Process the results in a separate goroutine
47 go func() {
48 for result := range passiveResults {
49 switch result.Type {
50 case subscraping.Error:
51 gologger.Warningf("Could not run source %s: %s\n", result.Source, result.Error)
52 case subscraping.Subdomain:
53 // Validate the subdomain found and remove wildcards from
54 if !strings.HasSuffix(result.Value, "."+domain) {
55 continue
56 }
57 subdomain := strings.ReplaceAll(strings.ToLower(result.Value), "*.", "")
58
59 // Check if the subdomain is a duplicate. If not,
60 // send the subdomain for resolution.
61 if _, ok := uniqueMap[subdomain]; ok {
62 continue
63 }
64
65 hostEntry := resolve.HostEntry{Host: subdomain, Source: result.Source}
66
67 uniqueMap[subdomain] = hostEntry
68
69 // Log the verbose message about the found subdomain and send the
70 // host for resolution to the resolution pool
71 gologger.Verbosef("%s\n", result.Source, subdomain)
72
73 // If the user asked to remove wildcard then send on the resolve
74 // queue. Otherwise, if mode is not verbose print the results on
75 // the screen as they are discovered.
76 if r.options.RemoveWildcard {
77 resolutionPool.Tasks <- hostEntry
78 }
79 }
80 }
81 // Close the task channel only if wildcards are asked to be removed
82 if r.options.RemoveWildcard {
83 close(resolutionPool.Tasks)
84 }
85 wg.Done()
86 }()
87
88 // If the user asked to remove wildcards, listen from the results
89 // queue and write to the map. At the end, print the found results to the screen
90 foundResults := make(map[string]resolve.Result)
91 if r.options.RemoveWildcard {
92 // Process the results coming from the resolutions pool
93 for result := range resolutionPool.Results {
94 switch result.Type {
95 case resolve.Error:
96 gologger.Warningf("Could not resolve host: %s\n", result.Error)
97 case resolve.Subdomain:
98 // Add the found subdomain to a map.
99 if _, ok := foundResults[result.Host]; !ok {
100 foundResults[result.Host] = result
101 }
102 }
103 }
104 }
105 wg.Wait()
106
107 outputter := NewOutputter(r.options.JSON)
108
109 // If verbose mode was used, then now print all the
110 // found subdomains on the screen together.
111 var err error
112 if r.options.HostIP {
113 err = outputter.WriteHostIP(foundResults, os.Stdout)
114 } else {
115 if r.options.RemoveWildcard {
116 err = outputter.WriteHostNoWildcard(foundResults, os.Stdout)
117 } else {
118 err = outputter.WriteHost(uniqueMap, os.Stdout)
119 }
120 }
121 if err != nil {
122 gologger.Errorf("Could not verbose results for %s: %s\n", domain, err)
123 return err
124 }
125
126 // Show found subdomain count in any case.
127 duration := durafmt.Parse(time.Since(now)).LimitFirstN(maxNumCount).String()
128 if r.options.RemoveWildcard {
129 gologger.Infof("Found %d subdomains for %s in %s\n", len(foundResults), domain, duration)
130 } else {
131 gologger.Infof("Found %d subdomains for %s in %s\n", len(uniqueMap), domain, duration)
132 }
133
134 // In case the user has specified to upload to chaos, write everything to a temporary buffer and upload
135 if r.options.ChaosUpload {
136 var buf = &bytes.Buffer{}
137 err := outputter.WriteForChaos(uniqueMap, buf)
138 // If an error occurs, do not interrupt, continue to check if user specified an output file
139 if err != nil {
140 gologger.Errorf("Could not prepare results for chaos %s\n", err)
141 } else {
142 // no error in writing host output, upload to chaos
143 err = r.UploadToChaos(ctx, buf)
144 if err != nil {
145 gologger.Errorf("Could not upload results to chaos %s\n", err)
146 } else {
147 gologger.Infof("Input processed successfully and subdomains with valid records will be updated to chaos dataset.\n")
148 }
149 // clear buffer
150 buf.Reset()
151 }
152 }
153
154 if output != "" {
155 file, err := outputter.createFile(output, appendToFile)
156 if err != nil {
157 gologger.Errorf("Could not create file %s for %s: %s\n", output, domain, err)
158 return err
159 }
160
161 defer file.Close()
162
163 if r.options.HostIP {
164 err = outputter.WriteHostIP(foundResults, file)
165 } else {
166 if r.options.RemoveWildcard {
167 err = outputter.WriteHostNoWildcard(foundResults, file)
168 } else {
169 err = outputter.WriteHost(uniqueMap, file)
170 }
171 }
172 if err != nil {
173 gologger.Errorf("Could not write results to file %s for %s: %s\n", output, domain, err)
174 return err
175 }
176 }
177
178 return nil
179 }
+67
-0
v2/pkg/runner/initialize.go less more
0 package runner
1
2 import (
3 "strings"
4
5 "github.com/projectdiscovery/subfinder/v2/pkg/passive"
6 "github.com/projectdiscovery/subfinder/v2/pkg/resolve"
7 )
8
9 // initializePassiveEngine creates the passive engine and loads sources etc
10 func (r *Runner) initializePassiveEngine() {
11 var sources, exclusions []string
12
13 if r.options.ExcludeSources != "" {
14 exclusions = append(exclusions, strings.Split(r.options.ExcludeSources, ",")...)
15 } else {
16 exclusions = append(exclusions, r.options.YAMLConfig.ExcludeSources...)
17 }
18
19 // Use all sources if asked by the user
20 if r.options.All {
21 sources = append(sources, r.options.YAMLConfig.AllSources...)
22 }
23
24 // If only recursive sources are wanted, use them only.
25 if r.options.Recursive {
26 sources = append(sources, r.options.YAMLConfig.Recursive...)
27 }
28
29 // If there are any sources from CLI, only use them
30 // Otherwise, use the yaml file sources
31 if !r.options.All && !r.options.Recursive {
32 if r.options.Sources != "" {
33 sources = append(sources, strings.Split(r.options.Sources, ",")...)
34 } else {
35 sources = append(sources, r.options.YAMLConfig.Sources...)
36 }
37 }
38 r.passiveAgent = passive.New(sources, exclusions)
39 }
40
41 // initializeActiveEngine creates the resolver used to resolve the found subdomains
42 func (r *Runner) initializeActiveEngine() error {
43 r.resolverClient = resolve.New()
44
45 // If the file has been provided, read resolvers from the file
46 if r.options.ResolverList != "" {
47 err := r.resolverClient.AppendResolversFromFile(r.options.ResolverList)
48 if err != nil {
49 return err
50 }
51 }
52
53 var resolvers []string
54
55 if r.options.Resolvers != "" {
56 resolvers = append(resolvers, strings.Split(r.options.Resolvers, ",")...)
57 } else if len(r.options.YAMLConfig.Resolvers) > 0 {
58 resolvers = append(resolvers, r.options.YAMLConfig.Resolvers...)
59 } else {
60 resolvers = append(resolvers, resolve.DefaultResolvers...)
61 }
62
63 r.resolverClient.AppendResolversFromSlice(resolvers)
64
65 return nil
66 }
+148
-0
v2/pkg/runner/options.go less more
0 package runner
1
2 import (
3 "flag"
4 "os"
5 "path"
6 "reflect"
7 "strings"
8
9 "github.com/projectdiscovery/gologger"
10 )
11
12 // Options contains the configuration options for tuning
13 // the subdomain enumeration process.
14 type Options struct {
15 Verbose bool // Verbose flag indicates whether to show verbose output or not
16 NoColor bool // No-Color disables the colored output
17 ChaosUpload bool // ChaosUpload indicates whether to upload results to the Chaos API
18 JSON bool // JSON specifies whether to use json for output format or text file
19 HostIP bool // HostIP specifies whether to write subdomains in host:ip format
20 Silent bool // Silent suppresses any extra text and only writes subdomains to screen
21 ListSources bool // ListSources specifies whether to list all available sources
22 RemoveWildcard bool // RemoveWildcard specifies whether to remove potential wildcard or dead subdomains from the results.
23 Stdin bool // Stdin specifies whether stdin input was given to the process
24 Version bool // Version specifies if we should just show version and exit
25 Recursive bool // Recursive specifies whether to use only recursive subdomain enumeration sources
26 All bool // All specifies whether to use all (slow) sources.
27 Threads int // Thread controls the number of threads to use for active enumerations
28 Timeout int // Timeout is the seconds to wait for sources to respond
29 MaxEnumerationTime int // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration
30 Domain string // Domain is the domain to find subdomains for
31 DomainsFile string // DomainsFile is the file containing list of domains to find subdomains for
32 Output string // Output is the file to write found subdomains to.
33 OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given
34 Sources string // Sources contains a comma-separated list of sources to use for enumeration
35 ExcludeSources string // ExcludeSources contains the comma-separated sources to not include in the enumeration process
36 Resolvers string // Resolvers is the comma-separated resolvers to use for enumeration
37 ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration
38 ConfigFile string // ConfigFile contains the location of the config file
39
40 YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file
41 }
42
43 // ParseOptions parses the command line flags provided by a user
44 func ParseOptions() *Options {
45 options := &Options{}
46
47 config, err := GetConfigDirectory()
48 if err != nil {
49 // This should never be reached
50 gologger.Fatalf("Could not get user home: %s\n", err)
51 }
52
53 flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output")
54 flag.BoolVar(&options.NoColor, "nC", false, "Don't Use colors in output")
55 flag.IntVar(&options.Threads, "t", 10, "Number of concurrent goroutines for resolving")
56 flag.IntVar(&options.Timeout, "timeout", 30, "Seconds to wait before timing out")
57 flag.IntVar(&options.MaxEnumerationTime, "max-time", 10, "Minutes to wait for enumeration results")
58 flag.StringVar(&options.Domain, "d", "", "Domain to find subdomains for")
59 flag.StringVar(&options.DomainsFile, "dL", "", "File containing list of domains to enumerate")
60 flag.BoolVar(&options.ChaosUpload, "cd", false, "Upload results to the Chaos API (api-key required)")
61 flag.StringVar(&options.Output, "o", "", "File to write output to (optional)")
62 flag.StringVar(&options.OutputDirectory, "oD", "", "Directory to write enumeration results to (optional)")
63 flag.BoolVar(&options.JSON, "json", false, "Write output in JSON lines Format")
64 flag.BoolVar(&options.JSON, "oJ", false, "Write output in JSON lines Format")
65 flag.BoolVar(&options.HostIP, "oI", false, "Write output in Host,IP format")
66 flag.BoolVar(&options.Silent, "silent", false, "Show only subdomains in output")
67 flag.BoolVar(&options.Recursive, "recursive", false, "Use only recursive subdomain enumeration sources")
68 flag.BoolVar(&options.All, "all", false, "Use all sources (slow) for enumeration")
69 flag.StringVar(&options.Sources, "sources", "", "Comma separated list of sources to use")
70 flag.BoolVar(&options.ListSources, "ls", false, "List all available sources")
71 flag.StringVar(&options.ExcludeSources, "exclude-sources", "", "List of sources to exclude from enumeration")
72 flag.StringVar(&options.Resolvers, "r", "", "Comma-separated list of resolvers to use")
73 flag.StringVar(&options.ResolverList, "rL", "", "Text file containing list of resolvers to use")
74 flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output")
75 flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc")
76 flag.BoolVar(&options.Version, "version", false, "Show version of subfinder")
77 flag.Parse()
78
79 // Check if stdin pipe was given
80 options.Stdin = hasStdin()
81
82 // Read the inputs and configure the logging
83 options.configureOutput()
84
85 // Show the user the banner
86 showBanner()
87
88 if options.Version {
89 gologger.Infof("Current Version: %s\n", Version)
90 os.Exit(0)
91 }
92
93 // Check if the config file exists. If not, it means this is the
94 // first run of the program. Show the first run notices and initialize the config file.
95 // Else show the normal banners and read the yaml fiile to the config
96 if !CheckConfigExists(options.ConfigFile) {
97 options.firstRunTasks()
98 } else {
99 options.normalRunTasks()
100 }
101
102 if options.ListSources {
103 listSources(options)
104 os.Exit(0)
105 }
106
107 // Validate the options passed by the user and if any
108 // invalid options have been used, exit.
109 err = options.validateOptions()
110 if err != nil {
111 gologger.Fatalf("Program exiting: %s\n", err)
112 }
113
114 return options
115 }
116
117 func hasStdin() bool {
118 fi, err := os.Stdin.Stat()
119 if err != nil {
120 return false
121 }
122 if fi.Mode()&os.ModeNamedPipe == 0 {
123 return false
124 }
125 return true
126 }
127
128 func listSources(options *Options) {
129 gologger.Infof("Current list of available sources. [%d]\n", len(options.YAMLConfig.AllSources))
130 gologger.Infof("Sources marked with an * needs key or token in order to work.\n")
131 gologger.Infof("You can modify %s to configure your keys / tokens.\n\n", options.ConfigFile)
132
133 keys := options.YAMLConfig.GetKeys()
134 needsKey := make(map[string]interface{})
135 keysElem := reflect.ValueOf(&keys).Elem()
136 for i := 0; i < keysElem.NumField(); i++ {
137 needsKey[strings.ToLower(keysElem.Type().Field(i).Name)] = keysElem.Field(i).Interface()
138 }
139
140 for _, source := range options.YAMLConfig.AllSources {
141 message := "%s\n"
142 if _, ok := needsKey[source]; ok {
143 message = "%s *\n"
144 }
145 gologger.Silentf(message, source)
146 }
147 }
+180
-0
v2/pkg/runner/outputter.go less more
0 package runner
1
2 import (
3 "bufio"
4 "errors"
5 "io"
6 "os"
7 "path/filepath"
8 "strings"
9
10 jsoniter "github.com/json-iterator/go"
11 "github.com/projectdiscovery/subfinder/v2/pkg/resolve"
12 )
13
14 // OutPutter outputs content to writers.
15 type OutPutter struct {
16 JSON bool
17 }
18
19 type jsonResult struct {
20 Host string `json:"host"`
21 IP string `json:"ip"`
22 Source string `json:"source"`
23 }
24
25 // NewOutputter creates a new Outputter
26 func NewOutputter(json bool) *OutPutter {
27 return &OutPutter{JSON: json}
28 }
29
30 func (o *OutPutter) createFile(filename string, appendtoFile bool) (*os.File, error) {
31 if filename == "" {
32 return nil, errors.New("empty filename")
33 }
34
35 dir := filepath.Dir(filename)
36
37 if dir != "" {
38 if _, err := os.Stat(dir); os.IsNotExist(err) {
39 err := os.MkdirAll(dir, os.ModePerm)
40 if err != nil {
41 return nil, err
42 }
43 }
44 }
45
46 var file *os.File
47 var err error
48 if appendtoFile {
49 file, err = os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
50 } else {
51 file, err = os.Create(filename)
52 }
53 if err != nil {
54 return nil, err
55 }
56
57 return file, nil
58 }
59
60 // WriteForChaos prepares the buffer to upload to Chaos
61 func (o *OutPutter) WriteForChaos(results map[string]resolve.HostEntry, writer io.Writer) error {
62 bufwriter := bufio.NewWriter(writer)
63 sb := &strings.Builder{}
64
65 for _, result := range results {
66 sb.WriteString(result.Host)
67 sb.WriteString("\n")
68
69 _, err := bufwriter.WriteString(sb.String())
70 if err != nil {
71 bufwriter.Flush()
72 return err
73 }
74 sb.Reset()
75 }
76 return bufwriter.Flush()
77 }
78
79 // WriteHostIP writes the output list of subdomain to an io.Writer
80 func (o *OutPutter) WriteHostIP(results map[string]resolve.Result, writer io.Writer) error {
81 var err error
82 if o.JSON {
83 err = writeJSONHostIP(results, writer)
84 } else {
85 err = writePlainHostIP(results, writer)
86 }
87 return err
88 }
89
90 func writePlainHostIP(results map[string]resolve.Result, writer io.Writer) error {
91 bufwriter := bufio.NewWriter(writer)
92 sb := &strings.Builder{}
93
94 for _, result := range results {
95 sb.WriteString(result.Host)
96 sb.WriteString(",")
97 sb.WriteString(result.IP)
98 sb.WriteString(",")
99 sb.WriteString(result.Source)
100 sb.WriteString("\n")
101
102 _, err := bufwriter.WriteString(sb.String())
103 if err != nil {
104 bufwriter.Flush()
105 return err
106 }
107 sb.Reset()
108 }
109 return bufwriter.Flush()
110 }
111
112 func writeJSONHostIP(results map[string]resolve.Result, writer io.Writer) error {
113 encoder := jsoniter.NewEncoder(writer)
114
115 var data jsonResult
116
117 for _, result := range results {
118 data.Host = result.Host
119 data.IP = result.IP
120 data.Source = result.Source
121
122 err := encoder.Encode(&data)
123 if err != nil {
124 return err
125 }
126 }
127 return nil
128 }
129
130 // WriteHostNoWildcard writes the output list of subdomain with nW flag to an io.Writer
131 func (o *OutPutter) WriteHostNoWildcard(results map[string]resolve.Result, writer io.Writer) error {
132 hosts := make(map[string]resolve.HostEntry)
133 for host, result := range results {
134 hosts[host] = resolve.HostEntry{Host: result.Host, Source: result.Source}
135 }
136
137 return o.WriteHost(hosts, writer)
138 }
139
140 // WriteHost writes the output list of subdomain to an io.Writer
141 func (o *OutPutter) WriteHost(results map[string]resolve.HostEntry, writer io.Writer) error {
142 var err error
143 if o.JSON {
144 err = writeJSONHost(results, writer)
145 } else {
146 err = writePlainHost(results, writer)
147 }
148 return err
149 }
150
151 func writePlainHost(results map[string]resolve.HostEntry, writer io.Writer) error {
152 bufwriter := bufio.NewWriter(writer)
153 sb := &strings.Builder{}
154
155 for _, result := range results {
156 sb.WriteString(result.Host)
157 sb.WriteString("\n")
158
159 _, err := bufwriter.WriteString(sb.String())
160 if err != nil {
161 bufwriter.Flush()
162 return err
163 }
164 sb.Reset()
165 }
166 return bufwriter.Flush()
167 }
168
169 func writeJSONHost(results map[string]resolve.HostEntry, writer io.Writer) error {
170 encoder := jsoniter.NewEncoder(writer)
171
172 for _, result := range results {
173 err := encoder.Encode(result)
174 if err != nil {
175 return err
176 }
177 }
178 return nil
179 }
+97
-0
v2/pkg/runner/runner.go less more
0 package runner
1
2 import (
3 "bufio"
4 "context"
5 "io"
6 "os"
7 "path"
8
9 "github.com/projectdiscovery/subfinder/v2/pkg/passive"
10 "github.com/projectdiscovery/subfinder/v2/pkg/resolve"
11 )
12
13 // Runner is an instance of the subdomain enumeration
14 // client used to orchestrate the whole process.
15 type Runner struct {
16 options *Options
17 passiveAgent *passive.Agent
18 resolverClient *resolve.Resolver
19 }
20
21 // NewRunner creates a new runner struct instance by parsing
22 // the configuration options, configuring sources, reading lists
23 // and setting up loggers, etc.
24 func NewRunner(options *Options) (*Runner, error) {
25 runner := &Runner{options: options}
26
27 // Initialize the passive subdomain enumeration engine
28 runner.initializePassiveEngine()
29
30 // Initialize the active subdomain enumeration engine
31 err := runner.initializeActiveEngine()
32 if err != nil {
33 return nil, err
34 }
35
36 return runner, nil
37 }
38
39 // RunEnumeration runs the subdomain enumeration flow on the targets specified
40 func (r *Runner) RunEnumeration(ctx context.Context) error {
41 // Check if only a single domain is sent as input. Process the domain now.
42 if r.options.Domain != "" {
43 return r.EnumerateSingleDomain(ctx, r.options.Domain, r.options.Output, false)
44 }
45
46 // If we have multiple domains as input,
47 if r.options.DomainsFile != "" {
48 f, err := os.Open(r.options.DomainsFile)
49 if err != nil {
50 return err
51 }
52 err = r.EnumerateMultipleDomains(ctx, f)
53 f.Close()
54 return err
55 }
56
57 // If we have STDIN input, treat it as multiple domains
58 if r.options.Stdin {
59 return r.EnumerateMultipleDomains(ctx, os.Stdin)
60 }
61 return nil
62 }
63
64 // EnumerateMultipleDomains enumerates subdomains for multiple domains
65 // We keep enumerating subdomains for a given domain until we reach an error
66 func (r *Runner) EnumerateMultipleDomains(ctx context.Context, reader io.Reader) error {
67 scanner := bufio.NewScanner(reader)
68 for scanner.Scan() {
69 domain := scanner.Text()
70 if domain == "" {
71 continue
72 }
73
74 var err error
75 // If the user has specified an output file, use that output file instead
76 // of creating a new output file for each domain. Else create a new file
77 // for each domain in the directory.
78 if r.options.Output != "" {
79 err = r.EnumerateSingleDomain(ctx, domain, r.options.Output, true)
80 } else if r.options.OutputDirectory != "" {
81 outputFile := path.Join(r.options.OutputDirectory, domain)
82 if r.options.JSON {
83 outputFile += ".json"
84 } else {
85 outputFile += ".txt"
86 }
87 err = r.EnumerateSingleDomain(ctx, domain, outputFile, false)
88 } else {
89 err = r.EnumerateSingleDomain(ctx, domain, "", true)
90 }
91 if err != nil {
92 return err
93 }
94 }
95 return nil
96 }
+50
-0
v2/pkg/runner/validate.go less more
0 package runner
1
2 import (
3 "errors"
4
5 "github.com/projectdiscovery/gologger"
6 )
7
8 // validateOptions validates the configuration options passed
9 func (options *Options) validateOptions() error {
10 // Check if domain, list of domains, or stdin info was provided.
11 // If none was provided, then return.
12 if options.Domain == "" && options.DomainsFile == "" && !options.Stdin {
13 return errors.New("no input list provided")
14 }
15
16 // Both verbose and silent flags were used
17 if options.Verbose && options.Silent {
18 return errors.New("both verbose and silent mode specified")
19 }
20
21 // Validate threads and options
22 if options.Threads == 0 {
23 return errors.New("threads cannot be zero")
24 }
25 if options.Timeout == 0 {
26 return errors.New("timeout cannot be zero")
27 }
28
29 // Always remove wildcard with hostip
30 if options.HostIP && !options.RemoveWildcard {
31 return errors.New("hostip flag must be used with RemoveWildcard option")
32 }
33
34 return nil
35 }
36
37 // configureOutput configures the output on the screen
38 func (options *Options) configureOutput() {
39 // If the user desires verbose output, show verbose output
40 if options.Verbose {
41 gologger.MaxLevel = gologger.Verbose
42 }
43 if options.NoColor {
44 gologger.UseColors = false
45 }
46 if options.Silent {
47 gologger.MaxLevel = gologger.Silent
48 }
49 }
+111
-0
v2/pkg/subscraping/agent.go less more
0 package subscraping
1
2 import (
3 "context"
4 "crypto/tls"
5 "fmt"
6 "io"
7 "io/ioutil"
8 "net/http"
9 "net/url"
10 "time"
11
12 "github.com/projectdiscovery/gologger"
13 )
14
15 // NewSession creates a new session object for a domain
16 func NewSession(domain string, keys *Keys, timeout int) (*Session, error) {
17 client := &http.Client{
18 Transport: &http.Transport{
19 MaxIdleConns: 100,
20 MaxIdleConnsPerHost: 100,
21 TLSClientConfig: &tls.Config{
22 InsecureSkipVerify: true,
23 },
24 },
25 Timeout: time.Duration(timeout) * time.Second,
26 }
27
28 session := &Session{
29 Client: client,
30 Keys: keys,
31 }
32
33 // Create a new extractor object for the current domain
34 extractor, err := NewSubdomainExtractor(domain)
35 session.Extractor = extractor
36
37 return session, err
38 }
39
40 // Get makes a GET request to a URL with extended parameters
41 func (s *Session) Get(ctx context.Context, getURL, cookies string, headers map[string]string) (*http.Response, error) {
42 return s.HTTPRequest(ctx, http.MethodGet, getURL, cookies, headers, nil, BasicAuth{})
43 }
44
45 // SimpleGet makes a simple GET request to a URL
46 func (s *Session) SimpleGet(ctx context.Context, getURL string) (*http.Response, error) {
47 return s.HTTPRequest(ctx, http.MethodGet, getURL, "", map[string]string{}, nil, BasicAuth{})
48 }
49
50 // Post makes a POST request to a URL with extended parameters
51 func (s *Session) Post(ctx context.Context, postURL, cookies string, headers map[string]string, body io.Reader) (*http.Response, error) {
52 return s.HTTPRequest(ctx, http.MethodPost, postURL, cookies, headers, body, BasicAuth{})
53 }
54
55 // SimplePost makes a simple POST request to a URL
56 func (s *Session) SimplePost(ctx context.Context, postURL, contentType string, body io.Reader) (*http.Response, error) {
57 return s.HTTPRequest(ctx, http.MethodPost, postURL, "", map[string]string{"Content-Type": contentType}, body, BasicAuth{})
58 }
59
60 // HTTPRequest makes any HTTP request to a URL with extended parameters
61 func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies string, headers map[string]string, body io.Reader, basicAuth BasicAuth) (*http.Response, error) {
62 req, err := http.NewRequestWithContext(ctx, method, requestURL, body)
63 if err != nil {
64 return nil, err
65 }
66
67 req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36")
68 req.Header.Set("Accept", "*/*")
69 req.Header.Set("Accept-Language", "en")
70 req.Header.Set("Connection", "close")
71
72 if basicAuth.Username != "" || basicAuth.Password != "" {
73 req.SetBasicAuth(basicAuth.Username, basicAuth.Password)
74 }
75
76 if cookies != "" {
77 req.Header.Set("Cookie", cookies)
78 }
79
80 for key, value := range headers {
81 req.Header.Set(key, value)
82 }
83
84 return httpRequestWrapper(s.Client, req)
85 }
86
87 // DiscardHTTPResponse discards the response content by demand
88 func (s *Session) DiscardHTTPResponse(response *http.Response) {
89 if response != nil {
90 _, err := io.Copy(ioutil.Discard, response.Body)
91 if err != nil {
92 gologger.Warningf("Could not discard response body: %s\n", err)
93 return
94 }
95 response.Body.Close()
96 }
97 }
98
99 func httpRequestWrapper(client *http.Client, request *http.Request) (*http.Response, error) {
100 resp, err := client.Do(request)
101 if err != nil {
102 return nil, err
103 }
104
105 if resp.StatusCode != http.StatusOK {
106 requestURL, _ := url.QueryUnescape(request.URL.String())
107 return resp, fmt.Errorf("unexpected status code %d received from %s", resp.StatusCode, requestURL)
108 }
109 return resp, nil
110 }
+62
-0
v2/pkg/subscraping/sources/alienvault/alienvault.go less more
0 package alienvault
1
2 import (
3 "context"
4 "encoding/json"
5 "fmt"
6
7 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
8 )
9
10 type alienvaultResponse struct {
11 Detail string `json:"detail"`
12 Error string `json:"error"`
13 PassiveDNS []struct {
14 Hostname string `json:"hostname"`
15 } `json:"passive_dns"`
16 }
17
18 // Source is the passive scraping agent
19 type Source struct{}
20
21 // Run function returns all subdomains found with the service
22 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
23 results := make(chan subscraping.Result)
24
25 go func() {
26 defer close(results)
27
28 resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain))
29 if err != nil && resp == nil {
30 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
31 session.DiscardHTTPResponse(resp)
32 return
33 }
34
35 var response alienvaultResponse
36 // Get the response body and decode
37 err = json.NewDecoder(resp.Body).Decode(&response)
38 if err != nil {
39 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
40 resp.Body.Close()
41 return
42 }
43 resp.Body.Close()
44
45 if response.Error != "" {
46 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s, %s", response.Detail, response.Error)}
47 return
48 }
49
50 for _, record := range response.PassiveDNS {
51 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname}
52 }
53 }()
54
55 return results
56 }
57
58 // Name returns the name of the source
59 func (s *Source) Name() string {
60 return "alienvault"
61 }
+49
-0
v2/pkg/subscraping/sources/anubis/anubis.go less more
0 package anubis
1
2 import (
3 "context"
4 "fmt"
5
6 jsoniter "github.com/json-iterator/go"
7 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
8 )
9
10 // Source is the passive scraping agent
11 type Source struct{}
12
13 // Run function returns all subdomains found with the service
14 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
15 results := make(chan subscraping.Result)
16
17 go func() {
18 defer close(results)
19
20 resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://jldc.me/anubis/subdomains/%s", domain))
21 if err != nil {
22 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
23 session.DiscardHTTPResponse(resp)
24 return
25 }
26
27 var subdomains []string
28 err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains)
29 if err != nil {
30 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
31 resp.Body.Close()
32 return
33 }
34
35 resp.Body.Close()
36
37 for _, record := range subdomains {
38 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record}
39 }
40 }()
41
42 return results
43 }
44
45 // Name returns the name of the source
46 func (s *Source) Name() string {
47 return "anubis"
48 }
+78
-0
v2/pkg/subscraping/sources/archiveis/archiveis.go less more
0 // Package archiveis is a Archiveis Scraping Engine in Golang
1 package archiveis
2
3 import (
4 "context"
5 "fmt"
6 "io/ioutil"
7 "regexp"
8
9 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
10 )
11
12 type agent struct {
13 Results chan subscraping.Result
14 Session *subscraping.Session
15 }
16
17 var reNext = regexp.MustCompile("<a id=\"next\" style=\".*\" href=\"(.*)\">&rarr;</a>")
18
19 func (a *agent) enumerate(ctx context.Context, baseURL string) {
20 select {
21 case <-ctx.Done():
22 return
23 default:
24 }
25
26 resp, err := a.Session.SimpleGet(ctx, baseURL)
27 if err != nil {
28 a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err}
29 a.Session.DiscardHTTPResponse(resp)
30 return
31 }
32
33 // Get the response body
34 body, err := ioutil.ReadAll(resp.Body)
35 if err != nil {
36 a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err}
37 resp.Body.Close()
38 return
39 }
40
41 resp.Body.Close()
42
43 src := string(body)
44 for _, subdomain := range a.Session.Extractor.FindAllString(src, -1) {
45 a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Subdomain, Value: subdomain}
46 }
47
48 match1 := reNext.FindStringSubmatch(src)
49 if len(match1) > 0 {
50 a.enumerate(ctx, match1[1])
51 }
52 }
53
54 // Source is the passive scraping agent
55 type Source struct{}
56
57 // Run function returns all subdomains found with the service
58 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
59 results := make(chan subscraping.Result)
60
61 a := agent{
62 Session: session,
63 Results: results,
64 }
65
66 go func() {
67 a.enumerate(ctx, fmt.Sprintf("http://archive.is/*.%s", domain))
68 close(a.Results)
69 }()
70
71 return a.Results
72 }
73
74 // Name returns the name of the source
75 func (s *Source) Name() string {
76 return "archiveis"
77 }
+143
-0
v2/pkg/subscraping/sources/binaryedge/binaryedge.go less more
0 package binaryedge
1
2 import (
3 "context"
4 "fmt"
5 "math"
6 "net/url"
7 "strconv"
8
9 jsoniter "github.com/json-iterator/go"
10 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 )
12
13 const (
14 v1 = "v1"
15 v2 = "v2"
16 baseAPIURLFmt = "https://api.binaryedge.io/%s/query/domains/subdomain/%s"
17 v2SubscriptionURL = "https://api.binaryedge.io/v2/user/subscription"
18 v1PageSizeParam = "pagesize"
19 pageParam = "page"
20 firstPage = 1
21 maxV1PageSize = 10000
22 )
23
24 type subdomainsResponse struct {
25 Message string `json:"message"`
26 Title string `json:"title"`
27 Status interface{} `json:"status"` // string for v1, int for v2
28 Subdomains []string `json:"events"`
29 Page int `json:"page"`
30 PageSize int `json:"pagesize"`
31 Total int `json:"total"`
32 }
33
34 // Source is the passive scraping agent
35 type Source struct{}
36
37 // Run function returns all subdomains found with the service
38 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
39 results := make(chan subscraping.Result)
40
41 go func() {
42 defer close(results)
43
44 if session.Keys.Binaryedge == "" {
45 return
46 }
47
48 var baseURL string
49
50 authHeader := map[string]string{"X-Key": session.Keys.Binaryedge}
51
52 if isV2(ctx, session, authHeader) {
53 baseURL = fmt.Sprintf(baseAPIURLFmt, v2, domain)
54 } else {
55 authHeader = map[string]string{"X-Token": session.Keys.Binaryedge}
56 v1URLWithPageSize, err := addURLParam(fmt.Sprintf(baseAPIURLFmt, v1, domain), v1PageSizeParam, strconv.Itoa(maxV1PageSize))
57 if err != nil {
58 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
59 return
60 }
61 baseURL = v1URLWithPageSize.String()
62 }
63
64 if baseURL == "" {
65 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("can't get API URL")}
66 return
67 }
68
69 s.enumerate(ctx, session, baseURL, firstPage, authHeader, results)
70 }()
71
72 return results
73 }
74
75 func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, baseURL string, page int, authHeader map[string]string, results chan subscraping.Result) {
76 pageURL, err := addURLParam(baseURL, pageParam, strconv.Itoa(page))
77 if err != nil {
78 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
79 return
80 }
81
82 resp, err := session.Get(ctx, pageURL.String(), "", authHeader)
83 if err != nil && resp == nil {
84 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
85 session.DiscardHTTPResponse(resp)
86 return
87 }
88
89 var response subdomainsResponse
90 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
91 if err != nil {
92 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
93 resp.Body.Close()
94 return
95 }
96
97 // Check error messages
98 if response.Message != "" && response.Status != nil {
99 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf(response.Message)}
100 }
101
102 resp.Body.Close()
103
104 for _, subdomain := range response.Subdomains {
105 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
106 }
107
108 totalPages := int(math.Ceil(float64(response.Total) / float64(response.PageSize)))
109 nextPage := response.Page + 1
110 for currentPage := nextPage; currentPage <= totalPages; currentPage++ {
111 s.enumerate(ctx, session, baseURL, currentPage, authHeader, results)
112 }
113 }
114
115 // Name returns the name of the source
116 func (s *Source) Name() string {
117 return "binaryedge"
118 }
119
120 func isV2(ctx context.Context, session *subscraping.Session, authHeader map[string]string) bool {
121 resp, err := session.Get(ctx, v2SubscriptionURL, "", authHeader)
122 if err != nil {
123 session.DiscardHTTPResponse(resp)
124 return false
125 }
126
127 resp.Body.Close()
128
129 return true
130 }
131
132 func addURLParam(targetURL, name, value string) (*url.URL, error) {
133 u, err := url.Parse(targetURL)
134 if err != nil {
135 return u, err
136 }
137 q, _ := url.ParseQuery(u.RawQuery)
138 q.Add(name, value)
139 u.RawQuery = q.Encode()
140
141 return u, nil
142 }
+85
-0
v2/pkg/subscraping/sources/bufferover/bufferover.go less more
0 // Package bufferover is a bufferover Scraping Engine in Golang
1 package bufferover
2
3 import (
4 "context"
5 "fmt"
6 "strings"
7
8 jsoniter "github.com/json-iterator/go"
9
10 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 )
12
13 type response struct {
14 Meta struct {
15 Errors []string `json:"Errors"`
16 } `json:"Meta"`
17 FDNSA []string `json:"FDNS_A"`
18 RDNS []string `json:"RDNS"`
19 Results []string `json:"Results"`
20 }
21
22 // Source is the passive scraping agent
23 type Source struct{}
24
25 // Run function returns all subdomains found with the service
26 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
27 results := make(chan subscraping.Result)
28
29 go func() {
30 // Run enumeration on subdomain dataset for historical SONAR datasets
31 s.getData(ctx, fmt.Sprintf("https://dns.bufferover.run/dns?q=.%s", domain), session, results)
32 s.getData(ctx, fmt.Sprintf("https://tls.bufferover.run/dns?q=.%s", domain), session, results)
33
34 close(results)
35 }()
36
37 return results
38 }
39
40 func (s *Source) getData(ctx context.Context, sourceURL string, session *subscraping.Session, results chan subscraping.Result) {
41 resp, err := session.SimpleGet(ctx, sourceURL)
42 if err != nil && resp == nil {
43 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
44 session.DiscardHTTPResponse(resp)
45 return
46 }
47
48 var bufforesponse response
49 err = jsoniter.NewDecoder(resp.Body).Decode(&bufforesponse)
50 if err != nil {
51 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
52 resp.Body.Close()
53 return
54 }
55
56 resp.Body.Close()
57
58 metaErrors := bufforesponse.Meta.Errors
59
60 if len(metaErrors) > 0 {
61 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", strings.Join(metaErrors, ", "))}
62 return
63 }
64
65 var subdomains []string
66
67 if len(bufforesponse.FDNSA) > 0 {
68 subdomains = bufforesponse.FDNSA
69 subdomains = append(subdomains, bufforesponse.RDNS...)
70 } else if len(bufforesponse.Results) > 0 {
71 subdomains = bufforesponse.Results
72 }
73
74 for _, subdomain := range subdomains {
75 for _, value := range session.Extractor.FindAllString(subdomain, -1) {
76 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value}
77 }
78 }
79 }
80
81 // Name returns the name of the source
82 func (s *Source) Name() string {
83 return "bufferover"
84 }
+63
-0
v2/pkg/subscraping/sources/cebaidu/cebaidu.go less more
0 package cebaidu
1
2 import (
3 "context"
4 "fmt"
5
6 jsoniter "github.com/json-iterator/go"
7 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
8 )
9
10 // Source is the passive scraping agent
11 type Source struct{}
12
13 type domain struct {
14 Domain string `json:"domain"`
15 }
16
17 type cebaiduResponse struct {
18 Code int64 `json:"code"`
19 Message string `json:"message"`
20 Data []domain `json:"data"`
21 }
22
23 // Run function returns all subdomains found with the service
24 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
25 results := make(chan subscraping.Result)
26
27 go func() {
28 defer close(results)
29
30 resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://ce.baidu.com/index/getRelatedSites?site_address=%s", domain))
31 if err != nil {
32 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
33 session.DiscardHTTPResponse(resp)
34 return
35 }
36
37 var response cebaiduResponse
38 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
39 if err != nil {
40 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
41 resp.Body.Close()
42 return
43 }
44 resp.Body.Close()
45
46 if response.Code > 0 {
47 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%d, %s", response.Code, response.Message)}
48 return
49 }
50
51 for _, result := range response.Data {
52 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Domain}
53 }
54 }()
55
56 return results
57 }
58
59 // Name returns the name of the source
60 func (s *Source) Name() string {
61 return "cebaidu"
62 }
+94
-0
v2/pkg/subscraping/sources/censys/censys.go less more
0 package censys
1
2 import (
3 "bytes"
4 "context"
5 "strconv"
6
7 jsoniter "github.com/json-iterator/go"
8 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
9 )
10
11 const maxCensysPages = 10
12
13 type resultsq struct {
14 Data []string `json:"parsed.extensions.subject_alt_name.dns_names"`
15 Data1 []string `json:"parsed.names"`
16 }
17
18 type response struct {
19 Results []resultsq `json:"results"`
20 Metadata struct {
21 Pages int `json:"pages"`
22 } `json:"metadata"`
23 }
24
25 // Source is the passive scraping agent
26 type Source struct{}
27
28 // Run function returns all subdomains found with the service
29 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
30 results := make(chan subscraping.Result)
31
32 go func() {
33 defer close(results)
34
35 if session.Keys.CensysToken == "" || session.Keys.CensysSecret == "" {
36 return
37 }
38
39 currentPage := 1
40 for {
41 var request = []byte(`{"query":"` + domain + `", "page":` + strconv.Itoa(currentPage) + `, "fields":["parsed.names","parsed.extensions.subject_alt_name.dns_names"], "flatten":true}`)
42
43 resp, err := session.HTTPRequest(
44 ctx,
45 "POST",
46 "https://www.censys.io/api/v1/search/certificates",
47 "",
48 map[string]string{"Content-Type": "application/json", "Accept": "application/json"},
49 bytes.NewReader(request),
50 subscraping.BasicAuth{Username: session.Keys.CensysToken, Password: session.Keys.CensysSecret},
51 )
52
53 if err != nil {
54 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
55 session.DiscardHTTPResponse(resp)
56 return
57 }
58
59 var censysResponse response
60 err = jsoniter.NewDecoder(resp.Body).Decode(&censysResponse)
61 if err != nil {
62 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
63 resp.Body.Close()
64 return
65 }
66
67 resp.Body.Close()
68
69 // Exit the censys enumeration if max pages is reached
70 if currentPage >= censysResponse.Metadata.Pages || currentPage >= maxCensysPages {
71 break
72 }
73
74 for _, res := range censysResponse.Results {
75 for _, part := range res.Data {
76 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: part}
77 }
78 for _, part := range res.Data1 {
79 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: part}
80 }
81 }
82
83 currentPage++
84 }
85 }()
86
87 return results
88 }
89
90 // Name returns the name of the source
91 func (s *Source) Name() string {
92 return "censys"
93 }
+96
-0
v2/pkg/subscraping/sources/certspotter/certspotter.go less more
0 package certspotter
1
2 import (
3 "context"
4 "fmt"
5
6 jsoniter "github.com/json-iterator/go"
7 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
8 )
9
10 type certspotterObject struct {
11 ID string `json:"id"`
12 DNSNames []string `json:"dns_names"`
13 }
14
15 // Source is the passive scraping agent
16 type Source struct{}
17
18 // Run function returns all subdomains found with the service
19 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
20 results := make(chan subscraping.Result)
21
22 go func() {
23 defer close(results)
24
25 if session.Keys.Certspotter == "" {
26 return
27 }
28
29 resp, err := session.Get(ctx, fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain), "", map[string]string{"Authorization": "Bearer " + session.Keys.Certspotter})
30 if err != nil {
31 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
32 session.DiscardHTTPResponse(resp)
33 return
34 }
35
36 var response []certspotterObject
37 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
38 if err != nil {
39 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
40 resp.Body.Close()
41 return
42 }
43 resp.Body.Close()
44
45 for _, cert := range response {
46 for _, subdomain := range cert.DNSNames {
47 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
48 }
49 }
50
51 // if the number of responses is zero, close the channel and return.
52 if len(response) == 0 {
53 return
54 }
55
56 id := response[len(response)-1].ID
57 for {
58 reqURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, id)
59
60 resp, err := session.Get(ctx, reqURL, "", map[string]string{"Authorization": "Bearer " + session.Keys.Certspotter})
61 if err != nil {
62 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
63 return
64 }
65
66 var response []certspotterObject
67 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
68 if err != nil {
69 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
70 resp.Body.Close()
71 return
72 }
73 resp.Body.Close()
74
75 if len(response) == 0 {
76 break
77 }
78
79 for _, cert := range response {
80 for _, subdomain := range cert.DNSNames {
81 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
82 }
83 }
84
85 id = response[len(response)-1].ID
86 }
87 }()
88
89 return results
90 }
91
92 // Name returns the name of the source
93 func (s *Source) Name() string {
94 return "certspotter"
95 }
+75
-0
v2/pkg/subscraping/sources/certspotterold/certspotterold.go less more
0 package certspotterold
1
2 import (
3 "context"
4 "fmt"
5 "net/http"
6
7 jsoniter "github.com/json-iterator/go"
8
9 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
10 )
11
12 type errorResponse struct {
13 Code string `json:"code"`
14 Message string `json:"Message"`
15 }
16
17 type subdomain struct {
18 DNSNames []string `json:"dns_names"`
19 }
20
21 // Source is the passive scraping agent
22 type Source struct{}
23
24 // Run function returns all subdomains found with the service
25 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
26 results := make(chan subscraping.Result)
27
28 go func() {
29 defer close(results)
30
31 resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://certspotter.com/api/v0/certs?domain=%s", domain))
32 if err != nil && resp == nil {
33 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
34 session.DiscardHTTPResponse(resp)
35 return
36 }
37
38 if resp.StatusCode != http.StatusOK {
39 var errResponse errorResponse
40 err = jsoniter.NewDecoder(resp.Body).Decode(&errResponse)
41 if err != nil {
42 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
43 resp.Body.Close()
44 return
45 }
46
47 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s: %s", errResponse.Code, errResponse.Message)}
48 resp.Body.Close()
49 return
50 }
51
52 var subdomains []subdomain
53 err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains)
54 if err != nil {
55 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
56 resp.Body.Close()
57 return
58 }
59
60 resp.Body.Close()
61
62 for _, subdomain := range subdomains {
63 for _, dnsname := range subdomain.DNSNames {
64 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: dnsname}
65 }
66 }
67 }()
68 return results
69 }
70
71 // Name returns the name of the source
72 func (s *Source) Name() string {
73 return "certspotterold"
74 }
+43
-0
v2/pkg/subscraping/sources/chaos/chaos.go less more
0 package chaos
1
2 import (
3 "context"
4 "fmt"
5
6 "github.com/projectdiscovery/chaos-client/pkg/chaos"
7 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
8 )
9
10 // Source is the passive scraping agent
11 type Source struct{}
12
13 // Run function returns all subdomains found with the service
14 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
15 results := make(chan subscraping.Result)
16
17 go func() {
18 defer close(results)
19
20 if session.Keys.Chaos == "" {
21 return
22 }
23
24 chaosClient := chaos.New(session.Keys.Chaos)
25 for result := range chaosClient.GetSubdomains(&chaos.SubdomainsRequest{
26 Domain: domain,
27 }) {
28 if result.Error != nil {
29 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: result.Error}
30 break
31 }
32 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", result.Subdomain, domain)}
33 }
34 }()
35
36 return results
37 }
38
39 // Name returns the name of the source
40 func (s *Source) Name() string {
41 return "chaos"
42 }
+112
-0
v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go less more
0 package commoncrawl
1
2 import (
3 "bufio"
4 "context"
5 "fmt"
6 "net/url"
7 "strings"
8
9 jsoniter "github.com/json-iterator/go"
10 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 )
12
13 const indexURL = "https://index.commoncrawl.org/collinfo.json"
14
15 type indexResponse struct {
16 ID string `json:"id"`
17 APIURL string `json:"cdx-api"`
18 }
19
20 // Source is the passive scraping agent
21 type Source struct{}
22
23 var years = [...]string{"2020", "2019", "2018", "2017"}
24
25 // Run function returns all subdomains found with the service
26 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
27 results := make(chan subscraping.Result)
28
29 go func() {
30 defer close(results)
31
32 resp, err := session.SimpleGet(ctx, indexURL)
33 if err != nil {
34 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
35 session.DiscardHTTPResponse(resp)
36 return
37 }
38
39 var indexes []indexResponse
40 err = jsoniter.NewDecoder(resp.Body).Decode(&indexes)
41 if err != nil {
42 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
43 resp.Body.Close()
44 return
45 }
46 resp.Body.Close()
47
48 searchIndexes := make(map[string]string)
49 for _, year := range years {
50 for _, index := range indexes {
51 if strings.Contains(index.ID, year) {
52 if _, ok := searchIndexes[year]; !ok {
53 searchIndexes[year] = index.APIURL
54 break
55 }
56 }
57 }
58 }
59
60 for _, apiURL := range searchIndexes {
61 further := s.getSubdomains(ctx, apiURL, domain, session, results)
62 if !further {
63 break
64 }
65 }
66 }()
67
68 return results
69 }
70
71 // Name returns the name of the source
72 func (s *Source) Name() string {
73 return "commoncrawl"
74 }
75
76 func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, session *subscraping.Session, results chan subscraping.Result) bool {
77 for {
78 select {
79 case <-ctx.Done():
80 return false
81 default:
82 var headers = map[string]string{"Host": "index.commoncrawl.org"}
83 resp, err := session.Get(ctx, fmt.Sprintf("%s?url=*.%s", searchURL, domain), "", headers)
84 if err != nil {
85 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
86 session.DiscardHTTPResponse(resp)
87 return false
88 }
89
90 scanner := bufio.NewScanner(resp.Body)
91 for scanner.Scan() {
92 line := scanner.Text()
93 if line == "" {
94 continue
95 }
96 line, _ = url.QueryUnescape(line)
97 subdomain := session.Extractor.FindString(line)
98 if subdomain != "" {
99 // fix for triple encoded URL
100 subdomain = strings.ToLower(subdomain)
101 subdomain = strings.TrimPrefix(subdomain, "25")
102 subdomain = strings.TrimPrefix(subdomain, "2f")
103
104 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
105 }
106 }
107 resp.Body.Close()
108 return true
109 }
110 }
111 }
+101
-0
v2/pkg/subscraping/sources/crtsh/crtsh.go less more
0 package crtsh
1
2 import (
3 "context"
4 "database/sql"
5 "fmt"
6
7 jsoniter "github.com/json-iterator/go"
8
9 // postgres driver
10 _ "github.com/lib/pq"
11 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
12 )
13
14 type subdomain struct {
15 ID int `json:"id"`
16 NameValue string `json:"name_value"`
17 }
18
19 // Source is the passive scraping agent
20 type Source struct{}
21
22 // Run function returns all subdomains found with the service
23 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
24 results := make(chan subscraping.Result)
25
26 go func() {
27 defer close(results)
28 found := s.getSubdomainsFromSQL(domain, results)
29 if found {
30 return
31 }
32 _ = s.getSubdomainsFromHTTP(ctx, domain, session, results)
33 }()
34
35 return results
36 }
37
38 func (s *Source) getSubdomainsFromSQL(domain string, results chan subscraping.Result) bool {
39 db, err := sql.Open("postgres", "host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes")
40 if err != nil {
41 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
42 return false
43 }
44
45 pattern := "%." + domain
46 query := `SELECT DISTINCT ci.NAME_VALUE as domain FROM certificate_identity ci
47 WHERE reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower($1))
48 ORDER BY ci.NAME_VALUE`
49 rows, err := db.Query(query, pattern)
50 if err != nil {
51 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
52 return false
53 }
54 if err := rows.Err(); err != nil {
55 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
56 return false
57 }
58
59 var data string
60 // Parse all the rows getting subdomains
61 for rows.Next() {
62 err := rows.Scan(&data)
63 if err != nil {
64 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
65 return false
66 }
67 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data}
68 }
69 return true
70 }
71
72 func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) bool {
73 resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain))
74 if err != nil {
75 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
76 session.DiscardHTTPResponse(resp)
77 return false
78 }
79
80 var subdomains []subdomain
81 err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains)
82 if err != nil {
83 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
84 resp.Body.Close()
85 return false
86 }
87
88 resp.Body.Close()
89
90 for _, subdomain := range subdomains {
91 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.NameValue}
92 }
93
94 return true
95 }
96
97 // Name returns the name of the source
98 func (s *Source) Name() string {
99 return "crtsh"
100 }
+67
-0
v2/pkg/subscraping/sources/dnsdb/dnsdb.go less more
0 package dnsdb
1
2 import (
3 "bufio"
4 "bytes"
5 "context"
6 "fmt"
7 "strings"
8
9 jsoniter "github.com/json-iterator/go"
10 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 )
12
13 type dnsdbResponse struct {
14 Name string `json:"rrname"`
15 }
16
17 // Source is the passive scraping agent
18 type Source struct{}
19
20 // Run function returns all subdomains found with the service
21 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
22 results := make(chan subscraping.Result)
23
24 go func() {
25 defer close(results)
26
27 if session.Keys.DNSDB == "" {
28 return
29 }
30
31 headers := map[string]string{
32 "X-API-KEY": session.Keys.DNSDB,
33 "Accept": "application/json",
34 "Content-Type": "application/json",
35 }
36
37 resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdb.info/lookup/rrset/name/*.%s?limit=1000000000000", domain), "", headers)
38 if err != nil {
39 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
40 session.DiscardHTTPResponse(resp)
41 return
42 }
43
44 scanner := bufio.NewScanner(resp.Body)
45 for scanner.Scan() {
46 line := scanner.Text()
47 if line == "" {
48 continue
49 }
50 var response dnsdbResponse
51 err = jsoniter.NewDecoder(bytes.NewBufferString(line)).Decode(&response)
52 if err != nil {
53 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
54 return
55 }
56 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(response.Name, ".")}
57 }
58 resp.Body.Close()
59 }()
60 return results
61 }
62
63 // Name returns the name of the source
64 func (s *Source) Name() string {
65 return "DNSDB"
66 }
+102
-0
v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go less more
0 package dnsdumpster
1
2 import (
3 "context"
4 "fmt"
5 "io/ioutil"
6 "net/url"
7 "regexp"
8 "strings"
9
10 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 )
12
13 // CSRFSubMatchLength CSRF regex submatch length
14 const CSRFSubMatchLength = 2
15
16 var re = regexp.MustCompile("<input type=\"hidden\" name=\"csrfmiddlewaretoken\" value=\"(.*)\">")
17
18 // getCSRFToken gets the CSRF Token from the page
19 func getCSRFToken(page string) string {
20 if subs := re.FindStringSubmatch(page); len(subs) == CSRFSubMatchLength {
21 return strings.TrimSpace(subs[1])
22 }
23 return ""
24 }
25
26 // postForm posts a form for a domain and returns the response
27 func postForm(ctx context.Context, session *subscraping.Session, token, domain string) (string, error) {
28 params := url.Values{
29 "csrfmiddlewaretoken": {token},
30 "targetip": {domain},
31 }
32
33 resp, err := session.HTTPRequest(
34 ctx,
35 "POST",
36 "https://dnsdumpster.com/",
37 fmt.Sprintf("csrftoken=%s; Domain=dnsdumpster.com", token),
38 map[string]string{
39 "Content-Type": "application/x-www-form-urlencoded",
40 "Referer": "https://dnsdumpster.com",
41 "X-CSRF-Token": token,
42 },
43 strings.NewReader(params.Encode()),
44 subscraping.BasicAuth{},
45 )
46
47 if err != nil {
48 session.DiscardHTTPResponse(resp)
49 return "", err
50 }
51
52 // Now, grab the entire page
53 in, err := ioutil.ReadAll(resp.Body)
54 resp.Body.Close()
55 return string(in), err
56 }
57
58 // Source is the passive scraping agent
59 type Source struct{}
60
61 // Run function returns all subdomains found with the service
62 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
63 results := make(chan subscraping.Result)
64
65 go func() {
66 defer close(results)
67
68 resp, err := session.SimpleGet(ctx, "https://dnsdumpster.com/")
69 if err != nil {
70 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
71 session.DiscardHTTPResponse(resp)
72 return
73 }
74
75 body, err := ioutil.ReadAll(resp.Body)
76 if err != nil {
77 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
78 resp.Body.Close()
79 return
80 }
81 resp.Body.Close()
82
83 csrfToken := getCSRFToken(string(body))
84 data, err := postForm(ctx, session, csrfToken, domain)
85 if err != nil {
86 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
87 return
88 }
89
90 for _, subdomain := range session.Extractor.FindAllString(data, -1) {
91 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
92 }
93 }()
94
95 return results
96 }
97
98 // Name returns the name of the source
99 func (s *Source) Name() string {
100 return "dnsdumpster"
101 }
+192
-0
v2/pkg/subscraping/sources/github/github.go less more
0 // Package github GitHub search package
1 // Based on gwen001's https://github.com/gwen001/github-search github-subdomains
2 package github
3
4 import (
5 "bufio"
6 "context"
7 "fmt"
8 "net/http"
9 "net/url"
10 "regexp"
11 "strconv"
12 "strings"
13 "time"
14
15 jsoniter "github.com/json-iterator/go"
16
17 "github.com/projectdiscovery/gologger"
18 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
19 "github.com/tomnomnom/linkheader"
20 )
21
22 type textMatch struct {
23 Fragment string `json:"fragment"`
24 }
25
26 type item struct {
27 Name string `json:"name"`
28 HTMLURL string `json:"html_url"`
29 TextMatches []textMatch `json:"text_matches"`
30 }
31
32 type response struct {
33 TotalCount int `json:"total_count"`
34 Items []item `json:"items"`
35 }
36
37 // Source is the passive scraping agent
38 type Source struct{}
39
40 // Run function returns all subdomains found with the service
41 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
42 results := make(chan subscraping.Result)
43
44 go func() {
45 defer close(results)
46
47 if len(session.Keys.GitHub) == 0 {
48 return
49 }
50
51 tokens := NewTokenManager(session.Keys.GitHub)
52
53 searchURL := fmt.Sprintf("https://api.github.com/search/code?per_page=100&q=%s&sort=created&order=asc", domain)
54 s.enumerate(ctx, searchURL, domainRegexp(domain), tokens, session, results)
55 }()
56
57 return results
58 }
59
60 func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *regexp.Regexp, tokens *Tokens, session *subscraping.Session, results chan subscraping.Result) {
61 select {
62 case <-ctx.Done():
63 return
64 default:
65 }
66
67 token := tokens.Get()
68
69 if token.RetryAfter > 0 {
70 if len(tokens.pool) == 1 {
71 gologger.Verbosef("GitHub Search request rate limit exceeded, waiting for %d seconds before retry... \n", s.Name(), token.RetryAfter)
72 time.Sleep(time.Duration(token.RetryAfter) * time.Second)
73 } else {
74 token = tokens.Get()
75 }
76 }
77
78 headers := map[string]string{"Accept": "application/vnd.github.v3.text-match+json", "Authorization": "token " + token.Hash}
79
80 // Initial request to GitHub search
81 resp, err := session.Get(ctx, searchURL, "", headers)
82 isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden
83 if err != nil && !isForbidden {
84 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
85 session.DiscardHTTPResponse(resp)
86 return
87 }
88
89 // Retry enumerarion after Retry-After seconds on rate limit abuse detected
90 ratelimitRemaining, _ := strconv.ParseInt(resp.Header.Get("X-Ratelimit-Remaining"), 10, 64)
91 if isForbidden && ratelimitRemaining == 0 {
92 retryAfterSeconds, _ := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64)
93 tokens.setCurrentTokenExceeded(retryAfterSeconds)
94 resp.Body.Close()
95
96 s.enumerate(ctx, searchURL, domainRegexp, tokens, session, results)
97 }
98
99 var data response
100
101 // Marshall json response
102 err = jsoniter.NewDecoder(resp.Body).Decode(&data)
103 if err != nil {
104 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
105 resp.Body.Close()
106 return
107 }
108
109 resp.Body.Close()
110
111 err = proccesItems(ctx, data.Items, domainRegexp, s.Name(), session, results)
112 if err != nil {
113 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
114 return
115 }
116
117 // Links header, first, next, last...
118 linksHeader := linkheader.Parse(resp.Header.Get("Link"))
119 // Process the next link recursively
120 for _, link := range linksHeader {
121 if link.Rel == "next" {
122 nextURL, err := url.QueryUnescape(link.URL)
123 if err != nil {
124 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
125 return
126 }
127 s.enumerate(ctx, nextURL, domainRegexp, tokens, session, results)
128 }
129 }
130 }
131
132 // proccesItems procceses github response items
133 func proccesItems(ctx context.Context, items []item, domainRegexp *regexp.Regexp, name string, session *subscraping.Session, results chan subscraping.Result) error {
134 for _, item := range items {
135 // find subdomains in code
136 resp, err := session.SimpleGet(ctx, rawURL(item.HTMLURL))
137 if err != nil {
138 if resp != nil && resp.StatusCode != http.StatusNotFound {
139 session.DiscardHTTPResponse(resp)
140 }
141 return err
142 }
143
144 if resp.StatusCode == http.StatusOK {
145 scanner := bufio.NewScanner(resp.Body)
146 for scanner.Scan() {
147 line := scanner.Text()
148 if line == "" {
149 continue
150 }
151 for _, subdomain := range domainRegexp.FindAllString(normalizeContent(line), -1) {
152 results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}
153 }
154 }
155 resp.Body.Close()
156 }
157
158 // find subdomains in text matches
159 for _, textMatch := range item.TextMatches {
160 for _, subdomain := range domainRegexp.FindAllString(normalizeContent(textMatch.Fragment), -1) {
161 results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}
162 }
163 }
164 }
165 return nil
166 }
167
168 // Normalize content before matching, query unescape, remove tabs and new line chars
169 func normalizeContent(content string) string {
170 normalizedContent, _ := url.QueryUnescape(content)
171 normalizedContent = strings.ReplaceAll(normalizedContent, "\\t", "")
172 normalizedContent = strings.ReplaceAll(normalizedContent, "\\n", "")
173 return normalizedContent
174 }
175
176 // Raw URL to get the files code and match for subdomains
177 func rawURL(htmlURL string) string {
178 domain := strings.ReplaceAll(htmlURL, "https://github.com/", "https://raw.githubusercontent.com/")
179 return strings.ReplaceAll(domain, "/blob/", "/")
180 }
181
182 // DomainRegexp regular expression to match subdomains in github files code
183 func domainRegexp(domain string) *regexp.Regexp {
184 rdomain := strings.ReplaceAll(domain, ".", "\\.")
185 return regexp.MustCompile("(\\w+[.])*" + rdomain)
186 }
187
188 // Name returns the name of the source
189 func (s *Source) Name() string {
190 return "github"
191 }
+66
-0
v2/pkg/subscraping/sources/github/tokenmanager.go less more
0 package github
1
2 import "time"
3
4 // Token struct
5 type Token struct {
6 Hash string
7 RetryAfter int64
8 ExceededTime time.Time
9 }
10
11 // Tokens is the internal struct to manage the current token
12 // and the pool
13 type Tokens struct {
14 current int
15 pool []Token
16 }
17
18 // NewTokenManager initialize the tokens pool
19 func NewTokenManager(keys []string) *Tokens {
20 pool := []Token{}
21 for _, key := range keys {
22 t := Token{Hash: key, ExceededTime: time.Time{}, RetryAfter: 0}
23 pool = append(pool, t)
24 }
25
26 return &Tokens{
27 current: 0,
28 pool: pool,
29 }
30 }
31
32 func (r *Tokens) setCurrentTokenExceeded(retryAfter int64) {
33 if r.current >= len(r.pool) {
34 r.current %= len(r.pool)
35 }
36 if r.pool[r.current].RetryAfter == 0 {
37 r.pool[r.current].ExceededTime = time.Now()
38 r.pool[r.current].RetryAfter = retryAfter
39 }
40 }
41
42 // Get returns a new token from the token pool
43 func (r *Tokens) Get() *Token {
44 resetExceededTokens(r)
45
46 if r.current >= len(r.pool) {
47 r.current %= len(r.pool)
48 }
49
50 result := &r.pool[r.current]
51 r.current++
52
53 return result
54 }
55
56 func resetExceededTokens(r *Tokens) {
57 for i, token := range r.pool {
58 if token.RetryAfter > 0 {
59 if int64(time.Since(token.ExceededTime)/time.Second) > token.RetryAfter {
60 r.pool[i].ExceededTime = time.Time{}
61 r.pool[i].RetryAfter = 0
62 }
63 }
64 }
65 }
+49
-0
v2/pkg/subscraping/sources/hackertarget/hackertarget.go less more
0 package hackertarget
1
2 import (
3 "bufio"
4 "context"
5 "fmt"
6
7 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
8 )
9
10 // Source is the passive scraping agent
11 type Source struct{}
12
13 // Run function returns all subdomains found with the service
14 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
15 results := make(chan subscraping.Result)
16
17 go func() {
18 defer close(results)
19
20 resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://api.hackertarget.com/hostsearch/?q=%s", domain))
21 if err != nil {
22 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
23 session.DiscardHTTPResponse(resp)
24 return
25 }
26
27 defer resp.Body.Close()
28
29 scanner := bufio.NewScanner(resp.Body)
30 for scanner.Scan() {
31 line := scanner.Text()
32 if line == "" {
33 continue
34 }
35 match := session.Extractor.FindAllString(line, -1)
36 for _, subdomain := range match {
37 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
38 }
39 }
40 }()
41
42 return results
43 }
44
45 // Name returns the name of the source
46 func (s *Source) Name() string {
47 return "hackertarget"
48 }
+121
-0
v2/pkg/subscraping/sources/intelx/intelx.go less more
0 package intelx
1
2 import (
3 "bytes"
4 "context"
5 "encoding/json"
6 "fmt"
7 "io/ioutil"
8
9 jsoniter "github.com/json-iterator/go"
10 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 )
12
13 type searchResponseType struct {
14 ID string `json:"id"`
15 Status int `json:"status"`
16 }
17
18 type selectorType struct {
19 Selectvalue string `json:"selectorvalue"`
20 }
21
22 type searchResultType struct {
23 Selectors []selectorType `json:"selectors"`
24 Status int `json:"status"`
25 }
26
27 type requestBody struct {
28 Term string
29 Maxresults int
30 Media int
31 Target int
32 Terminate []int
33 Timeout int
34 }
35
36 // Source is the passive scraping agent
37 type Source struct{}
38
39 // Run function returns all subdomains found with the service
40 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
41 results := make(chan subscraping.Result)
42
43 go func() {
44 defer close(results)
45
46 if session.Keys.IntelXKey == "" || session.Keys.IntelXHost == "" {
47 return
48 }
49
50 searchURL := fmt.Sprintf("https://%s/phonebook/search?k=%s", session.Keys.IntelXHost, session.Keys.IntelXKey)
51 reqBody := requestBody{
52 Term: domain,
53 Maxresults: 100000,
54 Media: 0,
55 Target: 1,
56 Timeout: 20,
57 }
58
59 body, err := json.Marshal(reqBody)
60 if err != nil {
61 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
62 return
63 }
64
65 resp, err := session.SimplePost(ctx, searchURL, "application/json", bytes.NewBuffer(body))
66 if err != nil {
67 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
68 session.DiscardHTTPResponse(resp)
69 return
70 }
71
72 var response searchResponseType
73 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
74 if err != nil {
75 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
76 resp.Body.Close()
77 return
78 }
79
80 resp.Body.Close()
81
82 resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", session.Keys.IntelXHost, session.Keys.IntelXKey, response.ID)
83 status := 0
84 for status == 0 || status == 3 {
85 resp, err = session.Get(ctx, resultsURL, "", nil)
86 if err != nil {
87 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
88 session.DiscardHTTPResponse(resp)
89 return
90 }
91 var response searchResultType
92 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
93 if err != nil {
94 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
95 resp.Body.Close()
96 return
97 }
98
99 _, err = ioutil.ReadAll(resp.Body)
100 if err != nil {
101 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
102 resp.Body.Close()
103 return
104 }
105 resp.Body.Close()
106
107 status = response.Status
108 for _, hostname := range response.Selectors {
109 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue}
110 }
111 }
112 }()
113
114 return results
115 }
116
117 // Name returns the name of the source
118 func (s *Source) Name() string {
119 return "intelx"
120 }
+171
-0
v2/pkg/subscraping/sources/ipv4info/ipv4info.go less more
0 package ipv4info
1
2 import (
3 "context"
4 "fmt"
5 "io/ioutil"
6 "net/http"
7 "regexp"
8 "strconv"
9
10 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 )
12
13 // Source is the passive scraping agent
14 type Source struct{}
15
16 // Run function returns all subdomains found with the service
17 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
18 results := make(chan subscraping.Result)
19
20 go func() {
21 defer close(results)
22
23 resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://ipv4info.com/search/%s", domain))
24 if err != nil && resp == nil {
25 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
26 session.DiscardHTTPResponse(resp)
27 return
28 }
29
30 body, err := ioutil.ReadAll(resp.Body)
31 if err != nil {
32 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
33 resp.Body.Close()
34 return
35 }
36 resp.Body.Close()
37
38 src := string(body)
39
40 if resp.StatusCode != http.StatusOK {
41 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", src)}
42 return
43 }
44
45 regxTokens := regexp.MustCompile("/ip-address/(.*)/" + domain)
46 matchTokens := regxTokens.FindAllString(src, -1)
47
48 if len(matchTokens) == 0 {
49 return
50 }
51
52 token := matchTokens[0]
53
54 resp, err = session.SimpleGet(ctx, "http://ipv4info.com"+token)
55 if err != nil {
56 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
57 session.DiscardHTTPResponse(resp)
58 return
59 }
60
61 body, err = ioutil.ReadAll(resp.Body)
62 if err != nil {
63 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
64 session.DiscardHTTPResponse(resp)
65 return
66 }
67 resp.Body.Close()
68
69 src = string(body)
70 regxTokens = regexp.MustCompile("/dns/(.*?)/" + domain)
71 matchTokens = regxTokens.FindAllString(src, -1)
72 if len(matchTokens) == 0 {
73 return
74 }
75
76 token = matchTokens[0]
77 resp, err = session.SimpleGet(ctx, "http://ipv4info.com"+token)
78 if err != nil {
79 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
80 session.DiscardHTTPResponse(resp)
81 return
82 }
83
84 body, err = ioutil.ReadAll(resp.Body)
85 if err != nil {
86 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
87 resp.Body.Close()
88 return
89 }
90 resp.Body.Close()
91
92 src = string(body)
93 regxTokens = regexp.MustCompile("/subdomains/(.*?)/" + domain)
94 matchTokens = regxTokens.FindAllString(src, -1)
95 if len(matchTokens) == 0 {
96 return
97 }
98
99 token = matchTokens[0]
100 resp, err = session.SimpleGet(ctx, "http://ipv4info.com"+token)
101 if err != nil {
102 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
103 session.DiscardHTTPResponse(resp)
104 return
105 }
106
107 body, err = ioutil.ReadAll(resp.Body)
108 if err != nil {
109 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
110 resp.Body.Close()
111 return
112 }
113 resp.Body.Close()
114
115 src = string(body)
116 for _, match := range session.Extractor.FindAllString(src, -1) {
117 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match}
118 }
119 nextPage := 1
120
121 for {
122 further := s.getSubdomains(ctx, domain, &nextPage, src, session, results)
123 if !further {
124 break
125 }
126 }
127 }()
128
129 return results
130 }
131
132 // Name returns the name of the source
133 func (s *Source) Name() string {
134 return "ipv4info"
135 }
136
137 func (s *Source) getSubdomains(ctx context.Context, domain string, nextPage *int, src string, session *subscraping.Session, results chan subscraping.Result) bool {
138 for {
139 select {
140 case <-ctx.Done():
141 return false
142 default:
143 regxTokens := regexp.MustCompile("/subdomains/.*/page" + strconv.Itoa(*nextPage) + "/" + domain + ".html")
144 matchTokens := regxTokens.FindAllString(src, -1)
145 if len(matchTokens) == 0 {
146 return false
147 }
148 token := matchTokens[0]
149
150 resp, err := session.SimpleGet(ctx, "http://ipv4info.com"+token)
151 if err != nil {
152 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
153 return false
154 }
155 body, err := ioutil.ReadAll(resp.Body)
156 if err != nil {
157 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
158 resp.Body.Close()
159 return false
160 }
161 resp.Body.Close()
162 src = string(body)
163 for _, match := range session.Extractor.FindAllString(src, -1) {
164 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match}
165 }
166 *nextPage++
167 return true
168 }
169 }
170 }
+68
-0
v2/pkg/subscraping/sources/passivetotal/passivetotal.go less more
0 package passivetotal
1
2 import (
3 "bytes"
4 "context"
5
6 jsoniter "github.com/json-iterator/go"
7 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
8 )
9
10 type response struct {
11 Subdomains []string `json:"subdomains"`
12 }
13
14 // Source is the passive scraping agent
15 type Source struct{}
16
17 // Run function returns all subdomains found with the service
18 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
19 results := make(chan subscraping.Result)
20
21 go func() {
22 defer close(results)
23
24 if session.Keys.PassiveTotalUsername == "" || session.Keys.PassiveTotalPassword == "" {
25 return
26 }
27
28 // Create JSON Get body
29 var request = []byte(`{"query":"` + domain + `"}`)
30
31 resp, err := session.HTTPRequest(
32 ctx,
33 "GET",
34 "https://api.passivetotal.org/v2/enrichment/subdomains",
35 "",
36 map[string]string{"Content-Type": "application/json"},
37 bytes.NewBuffer(request),
38 subscraping.BasicAuth{Username: session.Keys.PassiveTotalUsername, Password: session.Keys.PassiveTotalPassword},
39 )
40 if err != nil {
41 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
42 session.DiscardHTTPResponse(resp)
43 return
44 }
45
46 var data response
47 err = jsoniter.NewDecoder(resp.Body).Decode(&data)
48 if err != nil {
49 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
50 resp.Body.Close()
51 return
52 }
53 resp.Body.Close()
54
55 for _, subdomain := range data.Subdomains {
56 finalSubdomain := subdomain + "." + domain
57 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: finalSubdomain}
58 }
59 }()
60
61 return results
62 }
63
64 // Name returns the name of the source
65 func (s *Source) Name() string {
66 return "passivetotal"
67 }
+49
-0
v2/pkg/subscraping/sources/rapiddns/rapiddns.go less more
0 // Package rapiddns is a RapidDNS Scraping Engine in Golang
1 package rapiddns
2
3 import (
4 "context"
5 "io/ioutil"
6
7 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
8 )
9
10 // Source is the passive scraping agent
11 type Source struct{}
12
13 // Run function returns all subdomains found with the service
14 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
15 results := make(chan subscraping.Result)
16
17 go func() {
18 defer close(results)
19
20 resp, err := session.SimpleGet(ctx, "https://rapiddns.io/subdomain/"+domain+"?full=1")
21 if err != nil {
22 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
23 session.DiscardHTTPResponse(resp)
24 return
25 }
26
27 body, err := ioutil.ReadAll(resp.Body)
28 if err != nil {
29 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
30 resp.Body.Close()
31 return
32 }
33
34 resp.Body.Close()
35
36 src := string(body)
37 for _, subdomain := range session.Extractor.FindAllString(src, -1) {
38 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
39 }
40 }()
41
42 return results
43 }
44
45 // Name returns the name of the source
46 func (s *Source) Name() string {
47 return "rapiddns"
48 }
+56
-0
v2/pkg/subscraping/sources/recon/recon.go less more
0 package recon
1
2 import (
3 "context"
4 "encoding/json"
5 "fmt"
6
7 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
8 )
9
10 type subdomain struct {
11 RawDomain string `json:"rawDomain"`
12 }
13
14 // Source is the passive scraping agent
15 type Source struct{}
16
17 // Run function returns all subdomains found with the service
18 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
19 results := make(chan subscraping.Result)
20
21 go func() {
22 defer close(results)
23
24 if session.Keys.Recon == "" {
25 return
26 }
27
28 resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://recon.dev/api/search?key=%s&domain=%s", session.Keys.Recon, domain))
29 if err != nil {
30 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
31 session.DiscardHTTPResponse(resp)
32 return
33 }
34
35 var subdomains []subdomain
36 err = json.NewDecoder(resp.Body).Decode(&subdomains)
37 if err != nil {
38 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
39 resp.Body.Close()
40 return
41 }
42 resp.Body.Close()
43
44 for _, subdomain := range subdomains {
45 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.RawDomain}
46 }
47 }()
48
49 return results
50 }
51
52 // Name returns the name of the source
53 func (s *Source) Name() string {
54 return "recon"
55 }
+48
-0
v2/pkg/subscraping/sources/riddler/riddler.go less more
0 package riddler
1
2 import (
3 "bufio"
4 "context"
5 "fmt"
6
7 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
8 )
9
10 // Source is the passive scraping agent
11 type Source struct{}
12
13 // Run function returns all subdomains found with the service
14 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
15 results := make(chan subscraping.Result)
16
17 go func() {
18 defer close(results)
19
20 resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://riddler.io/search?q=pld:%s&view_type=data_table", domain))
21 if err != nil {
22 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
23 session.DiscardHTTPResponse(resp)
24 return
25 }
26
27 scanner := bufio.NewScanner(resp.Body)
28 for scanner.Scan() {
29 line := scanner.Text()
30 if line == "" {
31 continue
32 }
33 subdomain := session.Extractor.FindString(line)
34 if subdomain != "" {
35 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
36 }
37 }
38 resp.Body.Close()
39 }()
40
41 return results
42 }
43
44 // Name returns the name of the source
45 func (s *Source) Name() string {
46 return "riddler"
47 }
+95
-0
v2/pkg/subscraping/sources/robtex/robtext.go less more
0 package robtex
1
2 import (
3 "bufio"
4 "bytes"
5 "context"
6 "fmt"
7
8 jsoniter "github.com/json-iterator/go"
9 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
10 )
11
12 const (
13 addrRecord = "A"
14 iPv6AddrRecord = "AAAA"
15 baseURL = "https://proapi.robtex.com/pdns"
16 )
17
18 // Source is the passive scraping agent
19 type Source struct{}
20
21 type result struct {
22 Rrname string `json:"rrname"`
23 Rrdata string `json:"rrdata"`
24 Rrtype string `json:"rrtype"`
25 }
26
27 // Run function returns all subdomains found with the service
28 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
29 results := make(chan subscraping.Result)
30
31 go func() {
32 defer close(results)
33
34 if session.Keys.Robtex == "" {
35 return
36 }
37
38 headers := map[string]string{"Content-Type": "application/x-ndjson"}
39
40 ips, err := enumerate(ctx, session, fmt.Sprintf("%s/forward/%s?key=%s", baseURL, domain, session.Keys.Robtex), headers)
41 if err != nil {
42 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
43 return
44 }
45
46 for _, result := range ips {
47 if result.Rrtype == addrRecord || result.Rrtype == iPv6AddrRecord {
48 domains, err := enumerate(ctx, session, fmt.Sprintf("%s/reverse/%s?key=%s", baseURL, result.Rrdata, session.Keys.Robtex), headers)
49 if err != nil {
50 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
51 return
52 }
53 for _, result := range domains {
54 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Rrdata}
55 }
56 }
57 }
58 }()
59 return results
60 }
61
62 func enumerate(ctx context.Context, session *subscraping.Session, targetURL string, headers map[string]string) ([]result, error) {
63 var results []result
64
65 resp, err := session.Get(ctx, targetURL, "", headers)
66 if err != nil {
67 session.DiscardHTTPResponse(resp)
68 return results, err
69 }
70
71 scanner := bufio.NewScanner(resp.Body)
72 for scanner.Scan() {
73 line := scanner.Text()
74 if line == "" {
75 continue
76 }
77 var response result
78 err = jsoniter.NewDecoder(bytes.NewBufferString(line)).Decode(&response)
79 if err != nil {
80 return results, err
81 }
82
83 results = append(results, response)
84 }
85
86 resp.Body.Close()
87
88 return results, nil
89 }
90
91 // Name returns the name of the source
92 func (s *Source) Name() string {
93 return "robtex"
94 }
+64
-0
v2/pkg/subscraping/sources/securitytrails/securitytrails.go less more
0 package securitytrails
1
2 import (
3 "context"
4 "fmt"
5 "strings"
6
7 jsoniter "github.com/json-iterator/go"
8 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
9 )
10
11 type response struct {
12 Subdomains []string `json:"subdomains"`
13 }
14
15 // Source is the passive scraping agent
16 type Source struct{}
17
18 // Run function returns all subdomains found with the service
19 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
20 results := make(chan subscraping.Result)
21
22 go func() {
23 defer close(results)
24
25 if session.Keys.Securitytrails == "" {
26 return
27 }
28
29 resp, err := session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", map[string]string{"APIKEY": session.Keys.Securitytrails})
30 if err != nil {
31 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
32 session.DiscardHTTPResponse(resp)
33 return
34 }
35
36 var securityTrailsResponse response
37 err = jsoniter.NewDecoder(resp.Body).Decode(&securityTrailsResponse)
38 if err != nil {
39 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
40 resp.Body.Close()
41 return
42 }
43
44 resp.Body.Close()
45
46 for _, subdomain := range securityTrailsResponse.Subdomains {
47 if strings.HasSuffix(subdomain, ".") {
48 subdomain += domain
49 } else {
50 subdomain = subdomain + "." + domain
51 }
52
53 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
54 }
55 }()
56
57 return results
58 }
59
60 // Name returns the name of the source
61 func (s *Source) Name() string {
62 return "securitytrails"
63 }
+74
-0
v2/pkg/subscraping/sources/shodan/shodan.go less more
0 package shodan
1
2 import (
3 "context"
4 "fmt"
5
6 jsoniter "github.com/json-iterator/go"
7 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
8 )
9
10 // Source is the passive scraping agent
11 type Source struct{}
12
13 type dnsdbLookupResponse struct {
14 Domain string `json:"domain"`
15 Data []struct {
16 Subdomain string `json:"subdomain"`
17 Type string `json:"type"`
18 Value string `json:"value"`
19 } `json:"data"`
20 Result int `json:"result"`
21 Error string `json:"error"`
22 }
23
24 // Run function returns all subdomains found with the service
25 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
26 results := make(chan subscraping.Result)
27
28 go func() {
29 defer close(results)
30
31 if session.Keys.Shodan == "" {
32 return
33 }
34
35 searchURL := fmt.Sprintf("https://api.shodan.io/dns/domain/%s?key=%s", domain, session.Keys.Shodan)
36 resp, err := session.SimpleGet(ctx, searchURL)
37 if err != nil {
38 session.DiscardHTTPResponse(resp)
39 return
40 }
41
42 defer resp.Body.Close()
43
44 var response dnsdbLookupResponse
45 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
46 if err != nil {
47 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
48 return
49 }
50
51 if response.Error != "" {
52 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error)}
53 return
54 }
55
56 for _, data := range response.Data {
57 if data.Subdomain != "" {
58 if data.Type == "CNAME" {
59 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Value}
60 } else if data.Type == "A" {
61 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", data.Subdomain, domain)}
62 }
63 }
64 }
65 }()
66
67 return results
68 }
69
70 // Name returns the name of the source
71 func (s *Source) Name() string {
72 return "shodan"
73 }
+83
-0
v2/pkg/subscraping/sources/sitedossier/sitedossier.go less more
0 package sitedossier
1
2 import (
3 "context"
4 "fmt"
5 "io/ioutil"
6 "math/rand"
7 "net/http"
8 "regexp"
9 "time"
10
11 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
12 )
13
14 // SleepRandIntn is the integer value to get the pseudo-random number
15 // to sleep before find the next match
16 const SleepRandIntn = 5
17
18 var reNext = regexp.MustCompile(`<a href="([A-Za-z0-9/.]+)"><b>`)
19
20 type agent struct {
21 results chan subscraping.Result
22 session *subscraping.Session
23 }
24
25 func (a *agent) enumerate(ctx context.Context, baseURL string) {
26 select {
27 case <-ctx.Done():
28 return
29 default:
30 }
31
32 resp, err := a.session.SimpleGet(ctx, baseURL)
33 isnotfound := resp != nil && resp.StatusCode == http.StatusNotFound
34 if err != nil && !isnotfound {
35 a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err}
36 a.session.DiscardHTTPResponse(resp)
37 }
38
39 body, err := ioutil.ReadAll(resp.Body)
40 if err != nil {
41 a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err}
42 resp.Body.Close()
43 }
44 resp.Body.Close()
45
46 src := string(body)
47 for _, match := range a.session.Extractor.FindAllString(src, -1) {
48 a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: match}
49 }
50
51 match1 := reNext.FindStringSubmatch(src)
52 time.Sleep(time.Duration((3 + rand.Intn(SleepRandIntn))) * time.Second)
53
54 if len(match1) > 0 {
55 a.enumerate(ctx, "http://www.sitedossier.com"+match1[1])
56 }
57 }
58
59 // Source is the passive scraping agent
60 type Source struct{}
61
62 // Run function returns all subdomains found with the service
63 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
64 results := make(chan subscraping.Result)
65
66 a := agent{
67 session: session,
68 results: results,
69 }
70
71 go func() {
72 a.enumerate(ctx, fmt.Sprintf("http://www.sitedossier.com/parentdomain/%s", domain))
73 close(a.results)
74 }()
75
76 return a.results
77 }
78
79 // Name returns the name of the source
80 func (s *Source) Name() string {
81 return "sitedossier"
82 }
+82
-0
v2/pkg/subscraping/sources/spyse/spyse.go less more
0 package spyse
1
2 import (
3 "context"
4 "fmt"
5 "strconv"
6
7 jsoniter "github.com/json-iterator/go"
8 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
9 )
10
11 type resultObject struct {
12 Name string `json:"name"`
13 }
14
15 type dataObject struct {
16 Items []resultObject `json:"items"`
17 TotalCount int `json:"total_count"`
18 }
19
20 type errorObject struct {
21 Code string `json:"code"`
22 Message string `json:"message"`
23 }
24
25 type spyseResult struct {
26 Data dataObject `json:"data"`
27 Error []errorObject `json:"error"`
28 }
29
30 // Source is the passive scraping agent
31 type Source struct{}
32
33 // Run function returns all subdomains found with the service
34 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
35 results := make(chan subscraping.Result)
36
37 go func() {
38 defer close(results)
39
40 if session.Keys.Spyse == "" {
41 return
42 }
43
44 maxCount := 100
45
46 for offSet := 0; offSet <= maxCount; offSet += 100 {
47 resp, err := session.Get(ctx, fmt.Sprintf("https://api.spyse.com/v3/data/domain/subdomain?domain=%s&limit=100&offset=%s", domain, strconv.Itoa(offSet)), "", map[string]string{"Authorization": "Bearer " + session.Keys.Spyse})
48 if err != nil {
49 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
50 session.DiscardHTTPResponse(resp)
51 return
52 }
53
54 var response spyseResult
55 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
56 if err != nil {
57 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
58 resp.Body.Close()
59 return
60 }
61 resp.Body.Close()
62
63 if response.Data.TotalCount == 0 {
64 return
65 }
66
67 maxCount = response.Data.TotalCount
68
69 for _, hostname := range response.Data.Items {
70 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Name}
71 }
72 }
73 }()
74
75 return results
76 }
77
78 // Name returns the name of the source
79 func (s *Source) Name() string {
80 return "spyse"
81 }
+49
-0
v2/pkg/subscraping/sources/sublist3r/subllist3r.go less more
0 package sublist3r
1
2 import (
3 "context"
4 "encoding/json"
5 "fmt"
6
7 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
8 )
9
10 // Source is the passive scraping agent
11 type Source struct{}
12
13 // Run function returns all subdomains found with the service
14 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
15 results := make(chan subscraping.Result)
16
17 go func() {
18 defer close(results)
19
20 resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.sublist3r.com/search.php?domain=%s", domain))
21 if err != nil {
22 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
23 session.DiscardHTTPResponse(resp)
24 return
25 }
26
27 var subdomains []string
28 err = json.NewDecoder(resp.Body).Decode(&subdomains)
29 if err != nil {
30 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
31 resp.Body.Close()
32 return
33 }
34
35 resp.Body.Close()
36
37 for _, subdomain := range subdomains {
38 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
39 }
40 }()
41
42 return results
43 }
44
45 // Name returns the name of the source
46 func (s *Source) Name() string {
47 return "sublist3r"
48 }
+78
-0
v2/pkg/subscraping/sources/threatbook/threatbook.go less more
0 package threatbook
1
2 import (
3 "context"
4 "fmt"
5 "strconv"
6
7 jsoniter "github.com/json-iterator/go"
8 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
9 )
10
11 type threatBookResponse struct {
12 ResponseCode int64 `json:"response_code"`
13 VerboseMsg string `json:"verbose_msg"`
14 Data struct {
15 Domain string `json:"domain"`
16 SubDomains struct {
17 Total string `json:"total"`
18 Data []string `json:"data"`
19 } `json:"sub_domains"`
20 } `json:"data"`
21 }
22
23 // Source is the passive scraping agent
24 type Source struct{}
25
26 // Run function returns all subdomains found with the service
27 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
28 results := make(chan subscraping.Result)
29
30 go func() {
31 defer close(results)
32
33 if session.Keys.ThreatBook == "" {
34 return
35 }
36
37 resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatbook.cn/v3/domain/sub_domains?apikey=%s&resource=%s", session.Keys.ThreatBook, domain))
38 if err != nil && resp == nil {
39 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
40 session.DiscardHTTPResponse(resp)
41 return
42 }
43
44 var response threatBookResponse
45 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
46 if err != nil {
47 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
48 resp.Body.Close()
49 return
50 }
51 resp.Body.Close()
52
53 if response.ResponseCode != 0 {
54 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("code %d, %s", response.ResponseCode, response.VerboseMsg)}
55 return
56 }
57
58 total, err := strconv.ParseInt(response.Data.SubDomains.Total, 10, 64)
59 if err != nil {
60 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
61 return
62 }
63
64 if total > 0 {
65 for _, subdomain := range response.Data.SubDomains.Data {
66 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
67 }
68 }
69 }()
70
71 return results
72 }
73
74 // Name returns the name of the source
75 func (s *Source) Name() string {
76 return "threatbook"
77 }
+53
-0
v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go less more
0 package threatcrowd
1
2 import (
3 "context"
4 "fmt"
5
6 jsoniter "github.com/json-iterator/go"
7
8 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
9 )
10
11 type response struct {
12 Subdomains []string `json:"subdomains"`
13 }
14
15 // Source is the passive scraping agent
16 type Source struct{}
17
18 // Run function returns all subdomains found with the service
19 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
20 results := make(chan subscraping.Result)
21
22 go func() {
23 defer close(results)
24
25 resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://www.threatcrowd.org/searchApi/v2/domain/report/?domain=%s", domain))
26 if err != nil {
27 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
28 session.DiscardHTTPResponse(resp)
29 return
30 }
31
32 defer resp.Body.Close()
33
34 var data response
35 err = jsoniter.NewDecoder(resp.Body).Decode(&data)
36 if err != nil {
37 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
38 return
39 }
40
41 for _, subdomain := range data.Subdomains {
42 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
43 }
44 }()
45
46 return results
47 }
48
49 // Name returns the name of the source
50 func (s *Source) Name() string {
51 return "threatcrowd"
52 }
+55
-0
v2/pkg/subscraping/sources/threatminer/threatminer.go less more
0 package threatminer
1
2 import (
3 "context"
4 "fmt"
5
6 jsoniter "github.com/json-iterator/go"
7
8 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
9 )
10
11 type response struct {
12 StatusCode string `json:"status_code"`
13 StatusMessage string `json:"status_message"`
14 Results []string `json:"results"`
15 }
16
17 // Source is the passive scraping agent
18 type Source struct{}
19
20 // Run function returns all subdomains found with the service
21 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
22 results := make(chan subscraping.Result)
23
24 go func() {
25 defer close(results)
26
27 resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatminer.org/v2/domain.php?q=%s&rt=5", domain))
28 if err != nil {
29 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
30 session.DiscardHTTPResponse(resp)
31 return
32 }
33
34 defer resp.Body.Close()
35
36 var data response
37 err = jsoniter.NewDecoder(resp.Body).Decode(&data)
38 if err != nil {
39 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
40 return
41 }
42
43 for _, subdomain := range data.Results {
44 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
45 }
46 }()
47
48 return results
49 }
50
51 // Name returns the name of the source
52 func (s *Source) Name() string {
53 return "threatminer"
54 }
+57
-0
v2/pkg/subscraping/sources/virustotal/virustotal.go less more
0 package virustotal
1
2 import (
3 "context"
4 "fmt"
5
6 jsoniter "github.com/json-iterator/go"
7 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
8 )
9
10 type response struct {
11 Subdomains []string `json:"subdomains"`
12 }
13
14 // Source is the passive scraping agent
15 type Source struct{}
16
17 // Run function returns all subdomains found with the service
18 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
19 results := make(chan subscraping.Result)
20
21 go func() {
22 defer close(results)
23
24 if session.Keys.Virustotal == "" {
25 return
26 }
27
28 resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://www.virustotal.com/vtapi/v2/domain/report?apikey=%s&domain=%s", session.Keys.Virustotal, domain))
29 if err != nil {
30 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
31 session.DiscardHTTPResponse(resp)
32 return
33 }
34
35 var data response
36 err = jsoniter.NewDecoder(resp.Body).Decode(&data)
37 if err != nil {
38 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
39 resp.Body.Close()
40 return
41 }
42
43 resp.Body.Close()
44
45 for _, subdomain := range data.Subdomains {
46 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
47 }
48 }()
49
50 return results
51 }
52
53 // Name returns the name of the source
54 func (s *Source) Name() string {
55 return "virustotal"
56 }
+57
-0
v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go less more
0 package waybackarchive
1
2 import (
3 "bufio"
4 "context"
5 "fmt"
6 "net/url"
7 "strings"
8
9 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
10 )
11
12 // Source is the passive scraping agent
13 type Source struct{}
14
15 // Run function returns all subdomains found with the service
16 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
17 results := make(chan subscraping.Result)
18
19 go func() {
20 defer close(results)
21
22 resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey", domain))
23 if err != nil {
24 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
25 session.DiscardHTTPResponse(resp)
26 return
27 }
28
29 defer resp.Body.Close()
30
31 scanner := bufio.NewScanner(resp.Body)
32 for scanner.Scan() {
33 line := scanner.Text()
34 if line == "" {
35 continue
36 }
37 line, _ = url.QueryUnescape(line)
38 subdomain := session.Extractor.FindString(line)
39 if subdomain != "" {
40 // fix for triple encoded URL
41 subdomain = strings.ToLower(subdomain)
42 subdomain = strings.TrimPrefix(subdomain, "25")
43 subdomain = strings.TrimPrefix(subdomain, "2f")
44
45 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
46 }
47 }
48 }()
49
50 return results
51 }
52
53 // Name returns the name of the source
54 func (s *Source) Name() string {
55 return "waybackarchive"
56 }
+63
-0
v2/pkg/subscraping/sources/ximcx/ximcx.go less more
0 package ximcx
1
2 import (
3 "context"
4 "fmt"
5
6 jsoniter "github.com/json-iterator/go"
7 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
8 )
9
10 // Source is the passive scraping agent
11 type Source struct{}
12
13 type domain struct {
14 Domain string `json:"domain"`
15 }
16
17 type ximcxResponse struct {
18 Code int64 `json:"code"`
19 Message string `json:"message"`
20 Data []domain `json:"data"`
21 }
22
23 // Run function returns all subdomains found with the service
24 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
25 results := make(chan subscraping.Result)
26
27 go func() {
28 defer close(results)
29
30 resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://sbd.ximcx.cn/DomainServlet?domain=%s", domain))
31 if err != nil {
32 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
33 session.DiscardHTTPResponse(resp)
34 return
35 }
36
37 var response ximcxResponse
38 err = jsoniter.NewDecoder(resp.Body).Decode(&response)
39 if err != nil {
40 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
41 resp.Body.Close()
42 return
43 }
44 resp.Body.Close()
45
46 if response.Code > 0 {
47 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%d, %s", response.Code, response.Message)}
48 return
49 }
50
51 for _, result := range response.Data {
52 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Domain}
53 }
54 }()
55
56 return results
57 }
58
59 // Name returns the name of the source
60 func (s *Source) Name() string {
61 return "ximcx"
62 }
+125
-0
v2/pkg/subscraping/sources/zoomeye/zoomeye.go less more
0 package zoomeye
1
2 import (
3 "bytes"
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "net/http"
9
10 "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
11 )
12
13 // zoomAuth holds the ZoomEye credentials
14 type zoomAuth struct {
15 User string `json:"username"`
16 Pass string `json:"password"`
17 }
18
19 type loginResp struct {
20 JWT string `json:"access_token"`
21 }
22
23 // search results
24 type zoomeyeResults struct {
25 Matches []struct {
26 Site string `json:"site"`
27 Domains []string `json:"domains"`
28 } `json:"matches"`
29 }
30
31 // Source is the passive scraping agent
32 type Source struct{}
33
34 // Run function returns all subdomains found with the service
35 func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
36 results := make(chan subscraping.Result)
37
38 go func() {
39 defer close(results)
40
41 if session.Keys.ZoomEyeUsername == "" || session.Keys.ZoomEyePassword == "" {
42 return
43 }
44
45 jwt, err := doLogin(ctx, session)
46 if err != nil {
47 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
48 return
49 }
50 // check if jwt is null
51 if jwt == "" {
52 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errors.New("could not log into zoomeye")}
53 return
54 }
55
56 headers := map[string]string{
57 "Authorization": fmt.Sprintf("JWT %s", jwt),
58 "Accept": "application/json",
59 "Content-Type": "application/json",
60 }
61 for currentPage := 0; currentPage <= 100; currentPage++ {
62 api := fmt.Sprintf("https://api.zoomeye.org/web/search?query=hostname:%s&page=%d", domain, currentPage)
63 resp, err := session.Get(ctx, api, "", headers)
64 isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden
65 if err != nil {
66 if !isForbidden && currentPage == 0 {
67 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
68 session.DiscardHTTPResponse(resp)
69 }
70 return
71 }
72
73 var res zoomeyeResults
74 err = json.NewDecoder(resp.Body).Decode(&res)
75 if err != nil {
76 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
77 resp.Body.Close()
78 return
79 }
80 resp.Body.Close()
81
82 for _, r := range res.Matches {
83 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Site}
84 for _, domain := range r.Domains {
85 results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domain}
86 }
87 }
88 currentPage++
89 }
90 }()
91
92 return results
93 }
94
95 // doLogin performs authentication on the ZoomEye API
96 func doLogin(ctx context.Context, session *subscraping.Session) (string, error) {
97 creds := &zoomAuth{
98 User: session.Keys.ZoomEyeUsername,
99 Pass: session.Keys.ZoomEyePassword,
100 }
101 body, err := json.Marshal(&creds)
102 if err != nil {
103 return "", err
104 }
105 resp, err := session.SimplePost(ctx, "https://api.zoomeye.org/user/login", "application/json", bytes.NewBuffer(body))
106 if err != nil {
107 session.DiscardHTTPResponse(resp)
108 return "", err
109 }
110
111 defer resp.Body.Close()
112
113 var login loginResp
114 err = json.NewDecoder(resp.Body).Decode(&login)
115 if err != nil {
116 return "", err
117 }
118 return login.JWT, nil
119 }
120
121 // Name returns the name of the source
122 func (s *Source) Name() string {
123 return "zoomeye"
124 }
+76
-0
v2/pkg/subscraping/types.go less more
0 package subscraping
1
2 import (
3 "context"
4 "net/http"
5 "regexp"
6 )
7
8 // BasicAuth request's Authorization header
9 type BasicAuth struct {
10 Username string
11 Password string
12 }
13
14 // Source is an interface inherited by each passive source
15 type Source interface {
16 // Run takes a domain as argument and a session object
17 // which contains the extractor for subdomains, http client
18 // and other stuff.
19 Run(context.Context, string, *Session) <-chan Result
20 // Name returns the name of the source
21 Name() string
22 }
23
24 // Session is the option passed to the source, an option is created
25 // uniquely for eac source.
26 type Session struct {
27 // Extractor is the regex for subdomains created for each domain
28 Extractor *regexp.Regexp
29 // Keys is the API keys for the application
30 Keys *Keys
31 // Client is the current http client
32 Client *http.Client
33 }
34
35 // Keys contains the current API Keys we have in store
36 type Keys struct {
37 Binaryedge string `json:"binaryedge"`
38 CensysToken string `json:"censysUsername"`
39 CensysSecret string `json:"censysPassword"`
40 Certspotter string `json:"certspotter"`
41 Chaos string `json:"chaos"`
42 DNSDB string `json:"dnsdb"`
43 GitHub []string `json:"github"`
44 IntelXHost string `json:"intelXHost"`
45 IntelXKey string `json:"intelXKey"`
46 PassiveTotalUsername string `json:"passivetotal_username"`
47 PassiveTotalPassword string `json:"passivetotal_password"`
48 Recon string `json:"recon"`
49 Robtex string `json:"robtex"`
50 Securitytrails string `json:"securitytrails"`
51 Shodan string `json:"shodan"`
52 Spyse string `json:"spyse"`
53 ThreatBook string `json:"threatbook"`
54 URLScan string `json:"urlscan"`
55 Virustotal string `json:"virustotal"`
56 ZoomEyeUsername string `json:"zoomeye_username"`
57 ZoomEyePassword string `json:"zoomeye_password"`
58 }
59
60 // Result is a result structure returned by a source
61 type Result struct {
62 Type ResultType
63 Source string
64 Value string
65 Error error
66 }
67
68 // ResultType is the type of result returned by the source
69 type ResultType int
70
71 // Types of results returned by the source
72 const (
73 Subdomain ResultType = iota
74 Error
75 )
+30
-0
v2/pkg/subscraping/utils.go less more
0 package subscraping
1
2 import (
3 "regexp"
4 "sync"
5 )
6
7 var subdomainExtractorMutex = &sync.Mutex{}
8
9 // NewSubdomainExtractor creates a new regular expression to extract
10 // subdomains from text based on the given domain.
11 func NewSubdomainExtractor(domain string) (*regexp.Regexp, error) {
12 subdomainExtractorMutex.Lock()
13 defer subdomainExtractorMutex.Unlock()
14 extractor, err := regexp.Compile(`[a-zA-Z0-9\*_.-]+\.` + domain)
15 if err != nil {
16 return nil, err
17 }
18 return extractor, nil
19 }
20
21 // Exists check if a key exist in a slice
22 func Exists(values []string, key string) bool {
23 for _, v := range values {
24 if v == key {
25 return true
26 }
27 }
28 return false
29 }