Import upstream version 2.4.5
Kali Janitor
3 years ago
0 | --- | |
1 | name: Bug report | |
2 | about: Create a report to help us improve | |
3 | title: "[Issue] " | |
4 | labels: '' | |
5 | assignees: '' | |
6 | ||
7 | --- | |
8 | ||
9 | **Describe the bug** | |
10 | A clear and concise description of what the bug is. | |
11 | ||
12 | **Subfinder version** | |
13 | Include the version of subfinder you are using, `subfinder -version` | |
14 | ||
15 | **Complete command you used to reproduce this** | |
16 | ||
17 | ||
18 | **Screenshots** | |
19 | Add screenshots of the error for a better context. |
0 | --- | |
1 | name: Feature request | |
2 | about: Suggest an idea for this project | |
3 | title: "[Feature] " | |
4 | labels: '' | |
5 | assignees: '' | |
6 | ||
7 | --- | |
8 | ||
9 | **Is your feature request related to a problem? Please describe.** | |
10 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] | |
11 | ||
12 | **Describe the solution you'd like** | |
13 | A clear and concise description of what you want to happen. |
3 | 3 | branches: |
4 | 4 | - master |
5 | 5 | pull_request: |
6 | ||
7 | jobs: | |
6 | ||
7 | jobs: | |
8 | lint: | |
9 | name: golangci-lint | |
10 | runs-on: ubuntu-latest | |
11 | steps: | |
12 | - name: Checkout code | |
13 | uses: actions/checkout@v2 | |
14 | - name: Run golangci-lint | |
15 | uses: golangci/golangci-lint-action@v1 | |
16 | with: | |
17 | # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. | |
18 | version: v1.29 | |
19 | args: --timeout 5m | |
20 | working-directory: v2/cmd/subfinder/ | |
21 | ||
22 | # Optional: working directory, useful for monorepos | |
23 | # working-directory: somedir | |
24 | ||
25 | # Optional: golangci-lint command line arguments. | |
26 | # args: --issues-exit-code=0 | |
27 | ||
28 | # Optional: show only new issues if it's a pull request. The default value is `false`. | |
29 | # only-new-issues: true | |
30 | ||
8 | 31 | build: |
9 | 32 | name: Build |
10 | runs-on: ubuntu-latest | |
33 | runs-on: ubuntu-latest | |
11 | 34 | steps: |
12 | 35 | - name: Set up Go |
13 | 36 | uses: actions/setup-go@v2 |
19 | 42 | |
20 | 43 | - name: Test |
21 | 44 | run: go test . |
22 | working-directory: cmd/subfinder/ | |
45 | working-directory: v2/cmd/subfinder/ | |
23 | 46 | |
24 | 47 | - name: Build |
25 | 48 | run: go build . |
26 | working-directory: cmd/subfinder/⏎ | |
49 | working-directory: v2/cmd/subfinder/ |
0 | 0 | # dockerhub-push pushes docker build to dockerhub automatically |
1 | 1 | # on the creation of a new release |
2 | 2 | name: Publish to Dockerhub on creation of a new release |
3 | on: | |
3 | on: | |
4 | 4 | release: |
5 | 5 | types: [published] |
6 | 6 | jobs: |
13 | 13 | with: |
14 | 14 | name: projectdiscovery/subfinder |
15 | 15 | username: ${{ secrets.DOCKER_USERNAME }} |
16 | password: ${{ secrets.DOCKER_PASSWORD }}⏎ | |
16 | password: ${{ secrets.DOCKER_PASSWORD }} |
3 | 3 | tags: |
4 | 4 | - v* |
5 | 5 | |
6 | jobs: | |
6 | jobs: | |
7 | 7 | release: |
8 | 8 | runs-on: ubuntu-latest |
9 | steps: | |
10 | - | |
9 | steps: | |
10 | - | |
11 | 11 | name: "Check out code" |
12 | 12 | uses: actions/checkout@v2 |
13 | with: | |
13 | with: | |
14 | 14 | fetch-depth: 0 |
15 | - | |
15 | - | |
16 | 16 | name: "Set up Go" |
17 | 17 | uses: actions/setup-go@v2 |
18 | with: | |
18 | with: | |
19 | 19 | go-version: 1.14 |
20 | - | |
21 | env: | |
20 | - | |
21 | env: | |
22 | 22 | GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" |
23 | 23 | name: "Create release on GitHub" |
24 | 24 | uses: goreleaser/goreleaser-action@v2 |
25 | with: | |
25 | with: | |
26 | 26 | args: "release --rm-dist" |
27 | version: latest⏎ | |
27 | version: latest |
0 | linters-settings: | |
1 | dupl: | |
2 | threshold: 100 | |
3 | exhaustive: | |
4 | default-signifies-exhaustive: false | |
5 | # funlen: | |
6 | # lines: 100 | |
7 | # statements: 50 | |
8 | goconst: | |
9 | min-len: 2 | |
10 | min-occurrences: 2 | |
11 | gocritic: | |
12 | enabled-tags: | |
13 | - diagnostic | |
14 | - experimental | |
15 | - opinionated | |
16 | - performance | |
17 | - style | |
18 | disabled-checks: | |
19 | - dupImport # https://github.com/go-critic/go-critic/issues/845 | |
20 | - ifElseChain | |
21 | # gocyclo: | |
22 | # min-complexity: 15 | |
23 | goimports: | |
24 | local-prefixes: github.com/golangci/golangci-lint | |
25 | golint: | |
26 | min-confidence: 0 | |
27 | gomnd: | |
28 | settings: | |
29 | mnd: | |
30 | # don't include the "operation" and "assign" | |
31 | checks: argument,case,condition,return | |
32 | govet: | |
33 | check-shadowing: true | |
34 | settings: | |
35 | printf: | |
36 | funcs: | |
37 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof | |
38 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf | |
39 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf | |
40 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf | |
41 | # lll: | |
42 | # line-length: 140 | |
43 | maligned: | |
44 | suggest-new: true | |
45 | misspell: | |
46 | locale: US | |
47 | nolintlint: | |
48 | allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space) | |
49 | allow-unused: false # report any unused nolint directives | |
50 | require-explanation: false # don't require an explanation for nolint directives | |
51 | require-specific: false # don't require nolint directives to be specific about which linter is being skipped | |
52 | ||
53 | linters: | |
54 | # please, do not use `enable-all`: it's deprecated and will be removed soon. | |
55 | # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint | |
56 | disable-all: true | |
57 | enable: | |
58 | - bodyclose | |
59 | - deadcode | |
60 | - dogsled | |
61 | - dupl | |
62 | - errcheck | |
63 | - exhaustive | |
64 | - gochecknoinits | |
65 | - goconst | |
66 | - gocritic | |
67 | - gofmt | |
68 | - goimports | |
69 | - golint | |
70 | - gomnd | |
71 | - goprintffuncname | |
72 | - gosimple | |
73 | - govet | |
74 | - ineffassign | |
75 | - interfacer | |
76 | - maligned | |
77 | - misspell | |
78 | - nakedret | |
79 | - noctx | |
80 | - nolintlint | |
81 | - rowserrcheck | |
82 | - scopelint | |
83 | - staticcheck | |
84 | - structcheck | |
85 | - stylecheck | |
86 | - typecheck | |
87 | - unconvert | |
88 | - unparam | |
89 | - unused | |
90 | - varcheck | |
91 | - whitespace | |
92 | ||
93 | # don't enable: | |
94 | # - depguard | |
95 | # - asciicheck | |
96 | # - funlen | |
97 | # - gochecknoglobals | |
98 | # - gocognit | |
99 | # - gocyclo | |
100 | # - godot | |
101 | # - godox | |
102 | # - goerr113 | |
103 | # - gosec | |
104 | # - lll | |
105 | # - nestif | |
106 | # - prealloc | |
107 | # - testpackage | |
108 | # - wsl | |
109 | ||
110 | # golangci.com configuration | |
111 | # https://github.com/golangci/golangci/wiki/Configuration | |
112 | service: | |
113 | golangci-lint-version: 1.29.x # use the fixed version to not introduce new linters unexpectedly | |
114 | prepare: | |
115 | - echo "here I can run custom commands, but no preparation needed for this repo" |
0 | 0 | builds: |
1 | 1 | - binary: subfinder |
2 | main: cmd/subfinder/main.go | |
2 | main: v2/cmd/subfinder/main.go | |
3 | 3 | goos: |
4 | 4 | - linux |
5 | 5 | - windows |
5 | 5 | - CommonCrawl: https://commoncrawl.org/terms-of-use/full |
6 | 6 | - certspotter: https://sslmate.com/terms |
7 | 7 | - dnsdumpster: https://hackertarget.com/terms |
8 | - entrust: https://www.entrustdatacard.com/pages/terms-of-use | |
9 | 8 | - Google Transparency: https://policies.google.com/terms |
10 | 9 | - Threatcrowd: https://www.alienvault.com/terms/website-terms-of-use07may2018 |
11 | 10 |
5 | 5 | WORKDIR /go/src/app |
6 | 6 | |
7 | 7 | # Install |
8 | RUN go get -u github.com/projectdiscovery/subfinder/cmd/subfinder | |
8 | RUN go get -u github.com/projectdiscovery/subfinder/v2/cmd/subfinder | |
9 | 9 | |
10 | 10 | ENTRYPOINT ["subfinder"] |
0 | ## What's the problem (or question)? | |
1 | <!--- If describing a bug, tell us what happens instead of the expected behavior --> | |
2 | <!--- If suggesting a change/improvement, explain the difference from current behavior --> | |
3 | ||
4 | ## Do you have an idea for a solution? | |
5 | <!--- Not obligatory, but suggest a fix/reason for the bug, --> | |
6 | <!--- or ideas how to implement the addition or change --> | |
7 | ||
8 | ## How can we reproduce the issue? | |
9 | <!--- Provide unambiguous set of steps to reproduce this bug. Include command to reproduce, if relevant (you can mask the sensitive data) --> | |
10 | 1. | |
11 | 2. | |
12 | 3. | |
13 | 4. | |
14 | ||
15 | ## What are the running context details? | |
16 | <!--- Include as many relevant details about the running context you experienced the bug/problem in --> | |
17 | * Installation method (e.g. `pip`, `apt-get`, `git clone` or `zip`/`tar.gz`): | |
18 | * Client OS (e.g. `Microsoft Windows 10`) | |
19 | * Program version (see banner): | |
20 | * Relevant console output (if any): | |
21 | * Exception traceback (if any): |
6 | 6 | [![License](https://img.shields.io/badge/license-MIT-_red.svg)](https://opensource.org/licenses/MIT) |
7 | 7 | [![Go Report Card](https://goreportcard.com/badge/github.com/projectdiscovery/subfinder)](https://goreportcard.com/report/github.com/projectdiscovery/subfinder) |
8 | 8 | [![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/projectdiscovery/subfinder/issues) |
9 | [![GitHub Release](https://img.shields.io/github/release/projectdiscovery/subfinder)](https://github.com/projectdiscovery/subfinder/releases) | |
10 | [![Follow on Twitter](https://img.shields.io/twitter/follow/pdiscoveryio.svg?logo=twitter)](https://twitter.com/pdiscoveryio) | |
11 | [![Docker Images](https://img.shields.io/docker/pulls/projectdiscovery/subfinder.svg)](https://hub.docker.com/r/projectdiscovery/subfinder) | |
12 | [![Chat on Discord](https://img.shields.io/discord/695645237418131507.svg?logo=discord)](https://discord.gg/KECAGdH) | |
13 | ||
9 | 14 | |
10 | 15 | |
11 | 16 | subfinder is a subdomain discovery tool that discovers valid subdomains for websites by using passive online sources. It has a simple modular architecture and is optimized for speed. subfinder is built for doing one thing only - passive subdomain enumeration, and it does that very well. |
37 | 42 | |
38 | 43 | - Simple and modular code base making it easy to contribute. |
39 | 44 | - Fast And Powerful Resolution and wildcard elimination module |
40 | - **Curated** passive sources to maximize results (26 Sources as of now) | |
45 | - **Curated** passive sources to maximize results (35 Sources as of now) | |
41 | 46 | - Multiple Output formats supported (Json, File, Stdout) |
42 | 47 | - Optimized for speed, very fast and **lightweight** on resources |
43 | 48 | - **Stdin** and **stdout** support for integrating in workflows |
45 | 50 | |
46 | 51 | # Usage |
47 | 52 | |
48 | ```bash | |
53 | ```sh | |
49 | 54 | subfinder -h |
50 | 55 | ``` |
51 | 56 | This will display help for the tool. Here are all the switches it supports. |
52 | 57 | |
53 | 58 | | Flag | Description | Example | |
54 | 59 | |------|-------------|---------| |
55 | | -cd | Upload results to the Chaos API (api-key required) | subfinder -d uber.com -cd | | |
56 | | -config string | Configuration file for API Keys, etc | subfinder -config config.yaml | | |
57 | | -d | Domain to find subdomains for | subfinder -d uber.com | | |
58 | | -dL | File containing list of domains to enumerate | subfinder -dL hackerone-hosts.txt | | |
59 | | -exclude-sources | List of sources to exclude from enumeration | subfinder -exclude-sources archiveis | | |
60 | | -max-time | Minutes to wait for enumeration results (default 10) | subfinder -max-time 1 | | |
61 | | -nC | Don't Use colors in output | subfinder -nC | | |
62 | | -nW | Remove Wildcard & Dead Subdomains from output | subfinder -nW | | |
63 | | -ls | List all available sources | subfinder -ls | | |
64 | | -o | File to write output to (optional) | subfinder -o output.txt | | |
65 | | -oD | Directory to write enumeration results to (optional) | subfinder -oD ~/outputs | | |
60 | | -all | Use all sources (slow) for enumeration | subfinder -d uber.com -all | | |
61 | | -cd | Upload results to the Chaos API (api-key required) | subfinder -d uber.com -cd | | |
62 | | -config string | Configuration file for API Keys, etc | subfinder -config config.yaml | | |
63 | | -d | Domain to find subdomains for | subfinder -d uber.com | | |
64 | | -dL | File containing list of domains to enumerate | subfinder -dL hackerone-hosts.txt | | |
65 | | -exclude-sources | List of sources to exclude from enumeration | subfinder -exclude-sources archiveis | | |
66 | | -max-time | Minutes to wait for enumeration results (default 10) | subfinder -max-time 1 | | |
67 | | -nC | Don't Use colors in output | subfinder -nC | | |
68 | | -nW | Remove Wildcard & Dead Subdomains from output | subfinder -nW | | |
69 | | -ls | List all available sources | subfinder -ls | | |
70 | | -o | File to write output to (optional) | subfinder -o output.txt | | |
71 | | -oD | Directory to write enumeration results to (optional) | subfinder -oD ~/outputs | | |
66 | 72 | | -oI | Write output in Host,IP format | subfinder -oI | |
67 | 73 | | -oJ | Write output in JSON lines Format | subfinder -oJ | |
68 | | -r | Comma-separated list of resolvers to use | subfinder -r 1.1.1.1,1.0.0.1 | | |
74 | | -r | Comma-separated list of resolvers to use | subfinder -r 1.1.1.1,1.0.0.1 | | |
69 | 75 | | -rL | Text file containing list of resolvers to use | subfinder -rL resolvers.txt |
70 | | -silent | Show only subdomains in output | subfinder -silent | | |
71 | | -sources | Comma separated list of sources to use | subfinder -sources shodan,censys | | |
72 | | -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | | |
73 | | -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | | |
74 | | -v | Show Verbose output | subfinder -v | | |
75 | | -version | Show current program version | subfinder -version | | |
76 | ||
76 | | -recursive | Enumeration recursive subdomains | subfinder -d news.yahoo.com -recursive | |
77 | | -silent | Show only subdomains in output | subfinder -silent | | |
78 | | -sources | Comma separated list of sources to use | subfinder -sources shodan,censys | | |
79 | | -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | | |
80 | | -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | | |
81 | | -v | Show Verbose output | subfinder -v | | |
82 | | -version | Show current program version | subfinder -version | | |
83 | ||
77 | 84 | |
78 | 85 | # Installation Instructions |
79 | 86 | |
81 | 88 | |
82 | 89 | The installation is easy. You can download the pre-built binaries for different platforms from the [releases](https://github.com/projectdiscovery/subfinder/releases/) page. Extract them using tar, move it to your `$PATH` and you're ready to go. |
83 | 90 | |
84 | ```bash | |
85 | > tar -xzvf subfinder-linux-amd64.tar.gz | |
86 | > mv subfinder /usr/local/local/bin/ | |
87 | > subfinder -h | |
91 | ```sh | |
92 | ▶ # download release from https://github.com/projectdiscovery/subfinder/releases/ | |
93 | ▶ tar -xzvf subfinder-linux-amd64.tar.gz | |
94 | ▶ mv subfinder /usr/local/bin/ | |
95 | ▶ subfinder -h | |
88 | 96 | ``` |
89 | 97 | |
90 | 98 | ### From Source |
91 | 99 | |
92 | subfinder requires go1.13+ to install successfully. Run the following command to get the repo - | |
93 | ||
94 | ```bash | |
95 | GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/cmd/subfinder | |
100 | subfinder requires **go1.14+** to install successfully. Run the following command to get the repo - | |
101 | ||
102 | ```sh | |
103 | GO111MODULE=on go get -u -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder | |
96 | 104 | ``` |
97 | 105 | |
98 | 106 | ### From Github |
99 | 107 | |
100 | ```bash | |
108 | ```sh | |
101 | 109 | git clone https://github.com/projectdiscovery/subfinder.git |
102 | cd subfinder/cmd/subfinder | |
110 | cd subfinder/v2/cmd/subfinder | |
103 | 111 | go build . |
104 | 112 | mv subfinder /usr/local/bin/ |
105 | 113 | subfinder -h |
106 | 114 | ``` |
107 | 115 | |
108 | ### Upgrading | |
109 | If you wish to upgrade the package you can use: | |
110 | ||
111 | ```bash | |
112 | GO111MODULE=on go get -u -v github.com/projectdiscovery/subfinder/cmd/subfinder | |
113 | ``` | |
114 | ||
115 | 116 | ## Post Installation Instructions |
116 | 117 | |
117 | 118 | Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key: |
118 | 119 | |
119 | - [Virustotal](https://www.virustotal.com) | |
120 | - [Passivetotal](http://passivetotal.org) | |
121 | - [SecurityTrails](http://securitytrails.com) | |
120 | - [Binaryedge](https://binaryedge.io) | |
121 | - [Certspotter](https://sslmate.com/certspotter/api/) | |
122 | 122 | - [Censys](https://censys.io) |
123 | - [Binaryedge](https://binaryedge.io) | |
124 | - [Shodan](https://shodan.io) | |
125 | - [URLScan](https://urlscan.io) | |
126 | 123 | - [Chaos](https://chaos.projectdiscovery.io) |
127 | - [Spyse](https://spyse.com) | |
128 | 124 | - [DnsDB](https://api.dnsdb.info) |
129 | - [Zoomeye](https://www.zoomeye.org) | |
130 | 125 | - [Github](https://github.com) |
131 | 126 | - [Intelx](https://intelx.io) |
127 | - [Passivetotal](http://passivetotal.org) | |
128 | - [Recon.dev](https://recon.dev) | |
129 | - [Robtex](https://www.robtex.com/api/) | |
130 | - [SecurityTrails](http://securitytrails.com) | |
131 | - [Shodan](https://shodan.io) | |
132 | - [Spyse](https://spyse.com) | |
133 | - [Threatbook](https://threatbook.cn/api) | |
134 | - [Virustotal](https://www.virustotal.com) | |
135 | - [Zoomeye](https://www.zoomeye.org) | |
132 | 136 | |
133 | 137 | Theses values are stored in the `$HOME/.config/subfinder/config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys can be specified for each of these services from which one of them will be used for enumeration. |
134 | 138 | |
135 | 139 | For sources that require multiple keys, namely `Censys`, `Passivetotal`, they can be added by separating them via a colon (:). |
136 | 140 | |
137 | An example config file - | |
141 | An example config file - | |
138 | 142 | |
139 | 143 | ```yaml |
140 | 144 | resolvers: |
152 | 156 | censys: |
153 | 157 | - ac244e2f-b635-4581-878a-33f4e79a2c13:dd510d6e-1b6e-4655-83f6-f347b363def9 |
154 | 158 | certspotter: [] |
155 | passivetotal: | |
159 | passivetotal: | |
156 | 160 | - [email protected]:sample_password |
157 | 161 | securitytrails: [] |
158 | 162 | shodan: |
165 | 169 | # Running Subfinder |
166 | 170 | |
167 | 171 | To run the tool on a target, just use the following command. |
168 | ```bash | |
169 | > subfinder -d freelancer.com | |
172 | ```sh | |
173 | ▶ subfinder -d freelancer.com | |
170 | 174 | ``` |
171 | 175 | |
172 | 176 | This will run the tool against freelancer.com. There are a number of configuration options that you can pass along with this command. The verbose switch (-v) can be used to display verbose information. |
173 | 177 | |
174 | ```bash | |
175 | [CERTSPOTTER] www.fi.freelancer.com | |
176 | [DNSDUMPSTER] hosting.freelancer.com | |
177 | [DNSDUMPSTER] support.freelancer.com | |
178 | [DNSDUMPSTER] accounts.freelancer.com | |
179 | [DNSDUMPSTER] phabricator.freelancer.com | |
180 | [DNSDUMPSTER] cdn1.freelancer.com | |
181 | [DNSDUMPSTER] t1.freelancer.com | |
182 | [DNSDUMPSTER] wdc.t1.freelancer.com | |
183 | [DNSDUMPSTER] dal.t1.freelancer.com | |
178 | ``` | |
179 | [threatcrowd] ns1.hosting.freelancer.com | |
180 | [threatcrowd] ns2.hosting.freelancer.com | |
181 | [threatcrowd] flash.freelancer.com | |
182 | [threatcrowd] auth.freelancer.com | |
183 | [chaos] alertmanager.accounts.freelancer.com | |
184 | [chaos] analytics01.freelancer.com | |
185 | [chaos] apidocs.freelancer.com | |
186 | [chaos] brains.freelancer.com | |
187 | [chaos] consul.accounts.freelancer.com | |
184 | 188 | ``` |
185 | 189 | |
186 | 190 | The `-silent` switch can be used to show only subdomains found without any other info. |
188 | 192 | |
189 | 193 | The `-o` command can be used to specify an output file. |
190 | 194 | |
191 | ```bash | |
192 | > subfinder -d freelancer.com -o output.txt | |
195 | ```sh | |
196 | ▶ subfinder -d freelancer.com -o output.txt | |
193 | 197 | ``` |
194 | 198 | |
195 | 199 | To run the tool on a list of domains, `-dL` option can be used. This requires a directory to write the output files. Subdomains for each domain from the list are written in a text file in the directory specified by the `-oD` flag with their name being the domain name. |
196 | 200 | |
197 | ```bash | |
198 | > cat domains.txt | |
201 | ```sh | |
202 | ▶ cat domains.txt | |
199 | 203 | hackerone.com |
200 | 204 | google.com |
201 | 205 | |
202 | > subfinder -dL domains.txt -oD ~/path/to/output | |
203 | > ls ~/path/to/output | |
206 | ▶ subfinder -dL domains.txt -oD ~/path/to/output | |
207 | ▶ ls ~/path/to/output | |
204 | 208 | |
205 | 209 | hackerone.com.txt |
206 | 210 | google.com.txt |
207 | 211 | ``` |
208 | 212 | |
209 | If you want to save results to a single file while using a domain list, specify the `-o` flag with the name of the output file. | |
210 | ||
211 | ||
212 | ```bash | |
213 | > cat domains.txt | |
214 | hackerone.com | |
215 | google.com | |
216 | ||
217 | > subfinder -dL domains.txt -o ~/path/to/output.txt | |
218 | > ls ~/path/to/ | |
219 | ||
220 | output.txt | |
221 | ``` | |
222 | ||
223 | If you want upload your data to chaos dataset, you can use `-cd` flag with your scan, chaos will resolve all the input and add valid subdomains to public dataset, which you can access on the go using [chaos-client](https://github.com/projectdiscovery/chaos-client) | |
224 | ||
225 | ```bash | |
226 | > subfinder -d hackerone.com -cd | |
227 | ||
228 | root@b0x:~# subfinder -d hackerone.com -cd | |
229 | ||
230 | www.hackerone.com | |
231 | api.hackerone.com | |
232 | go.hackerone.com | |
233 | hackerone.com | |
234 | staging.hackerone.com | |
235 | [INF] Input processed successfully and subdomains with valid records will be updated to chaos dataset. | |
236 | ``` | |
237 | ||
238 | You can also get output in json format using `-oJ` switch. This switch saves the output in the JSON lines format. | |
213 | You can also get output in json format using `-oJ` switch. This switch saves the output in the JSON lines format. | |
239 | 214 | |
240 | 215 | If you use the JSON format, or the `Host:IP` format, then it becomes mandatory for you to use the **-nW** format as resolving is essential for these output format. By default, resolving the found subdomains is disabled. |
241 | 216 | |
242 | ```bash | |
243 | > subfinder -d hackerone.com -o output.json -oJ -nW | |
244 | > cat output.json | |
217 | ```sh | |
218 | ▶ subfinder -d hackerone.com -o output.json -oJ -nW | |
219 | ▶ cat output.json | |
245 | 220 | |
246 | 221 | {"host":"www.hackerone.com","ip":"104.16.99.52"} |
247 | 222 | {"host":"mta-sts.hackerone.com","ip":"185.199.108.153"} |
249 | 224 | {"host":"mta-sts.managed.hackerone.com","ip":"185.199.110.153"} |
250 | 225 | ``` |
251 | 226 | |
252 | You can specify custom resolvers too. | |
253 | ```bash | |
254 | > subfinder -d freelancer.com -o result.txt -nW -v -r 8.8.8.8,1.1.1.1 | |
255 | > subfinder -d freelancer.com -o result.txt -nW -v -rL resolvers.txt | |
256 | ``` | |
257 | ||
258 | **The new highlight of this release is the addition of stdin/stdout features.** Now, domains can be piped to subfinder and enumeration can be ran on them. For example - | |
259 | ||
260 | ```bash | |
261 | > echo hackerone.com | subfinder -v | |
262 | > cat targets.txt | subfinder -v | |
227 | ||
228 | **The new highlight of this release is the addition of stdin/stdout features.** Now, domains can be piped to subfinder and enumeration can be ran on them. For example - | |
229 | ||
230 | ```sh | |
231 | ▶ echo hackerone.com | subfinder | |
232 | ▶ cat targets.txt | subfinder | |
263 | 233 | ``` |
264 | 234 | |
265 | 235 | The subdomains discovered can be piped to other tools too. For example, you can pipe the subdomains discovered by subfinder to httpx [httpx](https://github.com/projectdiscovery/httpx) which will then find running http servers on the host. |
266 | 236 | |
267 | ```bash | |
268 | > echo hackerone.com | subfinder -silent | httpx -silent | |
237 | ```sh | |
238 | ▶ echo hackerone.com | subfinder -silent | httpx -silent | |
269 | 239 | |
270 | 240 | http://hackerone.com |
271 | 241 | http://www.hackerone.com |
277 | 247 | |
278 | 248 | ## Running in a Docker Container |
279 | 249 | |
280 | You can use the official dockerhub image at [subfinder](https://hub.docker.com/r/projectdiscovery/subfinder). Simply run - | |
281 | ||
282 | ```bash | |
283 | > docker pull projectdiscovery/subfinder | |
250 | You can use the official dockerhub image at [subfinder](https://hub.docker.com/r/projectdiscovery/subfinder). Simply run - | |
251 | ||
252 | ```sh | |
253 | ▶ docker pull projectdiscovery/subfinder | |
284 | 254 | ``` |
285 | 255 | |
286 | 256 | The above command will pull the latest tagged release from the dockerhub repository. |
289 | 259 | |
290 | 260 | - Clone the repo using `git clone https://github.com/projectdiscovery/subfinder.git` |
291 | 261 | - Build your docker container |
292 | ```bash | |
262 | ```sh | |
293 | 263 | docker build -t projectdiscovery/subfinder . |
294 | 264 | ``` |
295 | 265 | |
296 | - After building the container using either way, run the following - | |
297 | ```bash | |
266 | - After building the container using either way, run the following - | |
267 | ```sh | |
298 | 268 | docker run -it projectdiscovery/subfinder |
299 | 269 | ``` |
300 | > The above command is the same as running `-h` | |
270 | ▶ The above command is the same as running `-h` | |
301 | 271 | |
302 | 272 | If you are using docker, you need to first create your directory structure holding subfinder configuration file. After modifying the default config.yaml file, you can run: |
303 | 273 | |
304 | ```bash | |
305 | > mkdir -p $HOME/.config/subfinder | |
306 | > cp config.yaml $HOME/.config/subfinder/config.yaml | |
307 | > nano $HOME/.config/subfinder/config.yaml | |
274 | ```sh | |
275 | ▶ mkdir -p $HOME/.config/subfinder | |
276 | ▶ cp config.yaml $HOME/.config/subfinder/config.yaml | |
277 | ▶ nano $HOME/.config/subfinder/config.yaml | |
308 | 278 | ``` |
309 | 279 | |
310 | 280 | After that, you can pass it as a volume using the following sample command. |
311 | ```bash | |
312 | > docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it projectdiscovery/subfinder -d freelancer.com | |
281 | ```sh | |
282 | ▶ docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it projectdiscovery/subfinder -d freelancer.com | |
313 | 283 | ``` |
314 | 284 | |
315 | 285 | For example, this runs the tool against uber.com and output the results to your host file system: |
316 | ```bash | |
286 | ```sh | |
317 | 287 | docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it projectdiscovery/subfinder -d uber.com > uber.com.txt |
318 | 288 | ``` |
319 | 289 |
3 | 3 | |
4 | 4 | - All the contributors at [CONTRIBUTORS](https://github.com/projectdiscovery/subfinder/graphs/contributors) who made subfinder what it is. |
5 | 5 | |
6 | We'd like to thank some additional amazing people, wo contributed a lot in subfinder's journey - | |
6 | We'd like to thank some additional amazing people, who contributed a lot in subfinder's journey - | |
7 | 7 | |
8 | - @infosec-au - Donating to the project | |
9 | - @codingo - Initial work on the project, managing it, lot of work! | |
10 | - @picatz - Improving the structure of the project a lot. New ideas!⏎ | |
8 | - [@vzamanillo](https://github.com/vzamanillo) - For adding multiple features and overall project improvements. | |
9 | - [@infosec-au](https://github.com/infosec-au) - Donating to the project. | |
10 | - [@codingo](https://github.com/codingo) - Initial work on the project, managing it, lot of work! | |
11 | - [@picatz](https://github.com/picatz) - Improving the structure of the project a lot. New ideas!⏎ |
0 | package main | |
1 | ||
2 | import ( | |
3 | "github.com/projectdiscovery/gologger" | |
4 | "github.com/projectdiscovery/subfinder/pkg/runner" | |
5 | ) | |
6 | ||
7 | func main() { | |
8 | // Parse the command line flags and read config files | |
9 | options := runner.ParseOptions() | |
10 | ||
11 | runner, err := runner.NewRunner(options) | |
12 | if err != nil { | |
13 | gologger.Fatalf("Could not create runner: %s\n", err) | |
14 | } | |
15 | ||
16 | err = runner.RunEnumeration() | |
17 | if err != nil { | |
18 | gologger.Fatalf("Could not run enumeration: %s\n", err) | |
19 | } | |
20 | } |
0 | resolvers: | |
1 | - 1.1.1.1 | |
2 | - 1.0.0.1 | |
3 | - 8.8.8.8 | |
4 | - 8.8.4.4 | |
5 | - 9.9.9.9 | |
6 | - 9.9.9.10 | |
7 | - 77.88.8.8 | |
8 | - 77.88.8.1 | |
9 | - 208.67.222.222 | |
10 | - 208.67.220.220 | |
11 | sources: | |
12 | - alienvault | |
13 | - archiveis | |
14 | - binaryedge | |
15 | - bufferover | |
16 | - censys | |
17 | - certspotter | |
18 | - certspotterold | |
19 | - commoncrawl | |
20 | - crtsh | |
21 | - dnsdumpster | |
22 | - dnsdb | |
23 | - entrust | |
24 | - github | |
25 | - googleter | |
26 | - hackertarget | |
27 | - intelx | |
28 | - ipv4info | |
29 | - passivetotal | |
30 | - rapiddns | |
31 | - securitytrails | |
32 | - shodan | |
33 | - sitedossier | |
34 | - sublist3r | |
35 | - spyse | |
36 | - threatcrowd | |
37 | - threatminer | |
38 | - urlscan | |
39 | - virustotal | |
40 | - waybackarchive | |
41 | - zoomeye | |
42 | censys: | |
43 | - <key-here> | |
44 | binaryedge: | |
45 | - <key-here> | |
46 | certspotter: | |
47 | - <key-here> | |
48 | github: | |
49 | - <token-here> | |
50 | intelx: | |
51 | - <public.intelx.io:key-here> | |
52 | passivetotal: | |
53 | - <email:key-here> | |
54 | securitytrails: | |
55 | - <key-here> | |
56 | virustotal: | |
57 | - <key-here> | |
58 | urlscan: | |
59 | - <key-here> | |
60 | chaos: | |
61 | - <key-here> | |
62 | spyse: | |
63 | - <key-here> | |
64 | shodan: | |
65 | - <key-here> | |
66 | dnsdb: | |
67 | - <key-here> |
0 | module github.com/projectdiscovery/subfinder | |
1 | ||
2 | go 1.14 | |
3 | ||
4 | require ( | |
5 | github.com/json-iterator/go v1.1.9 | |
6 | github.com/lib/pq v1.6.0 | |
7 | github.com/m-mizutani/urlscan-go v1.0.0 | |
8 | github.com/miekg/dns v1.1.29 | |
9 | github.com/pkg/errors v0.9.1 | |
10 | github.com/projectdiscovery/gologger v1.0.0 | |
11 | github.com/rs/xid v1.2.1 | |
12 | github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 | |
13 | gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c | |
14 | ) |
0 | github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5/go.mod h1:976q2ETgjT2snVCf2ZaBnyBbVoPERGjUz+0sofzEfro= | |
1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | |
2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | |
3 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= | |
4 | github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= | |
5 | github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= | |
6 | github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= | |
7 | github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= | |
8 | github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= | |
9 | github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= | |
10 | github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= | |
11 | github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= | |
12 | github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= | |
13 | github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= | |
14 | github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= | |
15 | github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= | |
16 | github.com/jcmturner/gokrb5/v8 v8.2.0 h1:lzPl/30ZLkTveYsYZPKMcgXc8MbnE6RsTd4F9KgiLtk= | |
17 | github.com/jcmturner/gokrb5/v8 v8.2.0/go.mod h1:T1hnNppQsBtxW0tCHMHTkAt8n/sABdzZgZdoFrZaZNM= | |
18 | github.com/jcmturner/rpc/v2 v2.0.2 h1:gMB4IwRXYsWw4Bc6o/az2HJgFUA1ffSh90i26ZJ6Xl0= | |
19 | github.com/jcmturner/rpc/v2 v2.0.2/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= | |
20 | github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= | |
21 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= | |
22 | github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= | |
23 | github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= | |
24 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= | |
25 | github.com/lib/pq v1.6.0 h1:I5DPxhYJChW9KYc66se+oKFFQX6VuQrKiprsX6ivRZc= | |
26 | github.com/lib/pq v1.6.0/go.mod h1:4vXEAYvW1fRQ2/FhZ78H73A60MHw1geSm145z2mdY1g= | |
27 | github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs= | |
28 | github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= | |
29 | github.com/m-mizutani/urlscan-go v1.0.0 h1:+fTiSRCQXdy3EM1BgO5gmAHFWbccTDdoEKy9Fa7m9xo= | |
30 | github.com/m-mizutani/urlscan-go v1.0.0/go.mod h1:ppEBT0e/xv0bPcVWKev4cYG7Ey8933JsOzEzovxGMjI= | |
31 | github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= | |
32 | github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= | |
33 | github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg= | |
34 | github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= | |
35 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= | |
36 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= | |
37 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= | |
38 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= | |
39 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= | |
40 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= | |
41 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= | |
42 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= | |
43 | github.com/projectdiscovery/gologger v1.0.0 h1:XAQ8kHeVKXMjY4rLGh7eT5+oHU077BNEvs7X6n+vu1s= | |
44 | github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= | |
45 | github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= | |
46 | github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= | |
47 | github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= | |
48 | github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= | |
49 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= | |
50 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= | |
51 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= | |
52 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= | |
53 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= | |
54 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= | |
55 | github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= | |
56 | github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= | |
57 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= | |
58 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= | |
59 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= | |
60 | golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= | |
61 | golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 h1:QmwruyY+bKbDDL0BaglrbZABEali68eoMFhTZpCjYVA= | |
62 | golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= | |
63 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= | |
64 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= | |
65 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= | |
66 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= | |
67 | golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA= | |
68 | golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= | |
69 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | |
70 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | |
71 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | |
72 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | |
73 | golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M= | |
74 | golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | |
75 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= | |
76 | golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= | |
77 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= | |
78 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= | |
79 | gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= | |
80 | gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= | |
81 | gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= | |
82 | gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= | |
83 | gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= | |
84 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= | |
85 | gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo= | |
86 | gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= |
0 | // Package passive provides capability for doing passive subdomain | |
1 | // enumeration on targets. | |
2 | package passive | |
3 |
0 | package passive | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "sync" | |
6 | "time" | |
7 | ||
8 | "github.com/projectdiscovery/gologger" | |
9 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
10 | ) | |
11 | ||
12 | // EnumerateSubdomains enumerates all the subdomains for a given domain | |
13 | func (a *Agent) EnumerateSubdomains(domain string, keys subscraping.Keys, timeout int, maxEnumTime time.Duration) chan subscraping.Result { | |
14 | results := make(chan subscraping.Result) | |
15 | ||
16 | go func() { | |
17 | session, err := subscraping.NewSession(domain, keys, timeout) | |
18 | if err != nil { | |
19 | results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} | |
20 | } | |
21 | ||
22 | ctx, cancel := context.WithTimeout(context.Background(), maxEnumTime) | |
23 | ||
24 | timeTaken := make(map[string]string) | |
25 | timeTakenMutex := &sync.Mutex{} | |
26 | ||
27 | wg := &sync.WaitGroup{} | |
28 | // Run each source in parallel on the target domain | |
29 | for source, runner := range a.sources { | |
30 | wg.Add(1) | |
31 | ||
32 | now := time.Now() | |
33 | go func(source string, runner subscraping.Source) { | |
34 | for resp := range runner.Run(ctx, domain, session) { | |
35 | results <- resp | |
36 | } | |
37 | ||
38 | duration := time.Now().Sub(now) | |
39 | timeTakenMutex.Lock() | |
40 | timeTaken[source] = fmt.Sprintf("Source took %s for enumeration\n", duration) | |
41 | timeTakenMutex.Unlock() | |
42 | ||
43 | wg.Done() | |
44 | }(source, runner) | |
45 | } | |
46 | wg.Wait() | |
47 | ||
48 | for source, data := range timeTaken { | |
49 | gologger.Verbosef(data, source) | |
50 | } | |
51 | ||
52 | close(results) | |
53 | cancel() | |
54 | }() | |
55 | ||
56 | return results | |
57 | } |
0 | package passive | |
1 | ||
2 | import ( | |
3 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
4 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/alienvault" | |
5 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/archiveis" | |
6 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/binaryedge" | |
7 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/bufferover" | |
8 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/censys" | |
9 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/certspotter" | |
10 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/certspotterold" | |
11 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/commoncrawl" | |
12 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/crtsh" | |
13 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/dnsdb" | |
14 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/dnsdumpster" | |
15 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/entrust" | |
16 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/github" | |
17 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/hackertarget" | |
18 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/intelx" | |
19 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/ipv4info" | |
20 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/passivetotal" | |
21 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/rapiddns" | |
22 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/securitytrails" | |
23 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/shodan" | |
24 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/sitedossier" | |
25 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/spyse" | |
26 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/sublist3r" | |
27 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/threatcrowd" | |
28 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/threatminer" | |
29 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/urlscan" | |
30 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/virustotal" | |
31 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/waybackarchive" | |
32 | "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/zoomeye" | |
33 | ) | |
34 | ||
35 | // DefaultSources contains the list of sources used by default | |
36 | var DefaultSources = []string{ | |
37 | "alienvault", | |
38 | "archiveis", | |
39 | "binaryedge", | |
40 | "bufferover", | |
41 | "censys", | |
42 | "certspotter", | |
43 | "certspotterold", | |
44 | "commoncrawl", | |
45 | "crtsh", | |
46 | "dnsdumpster", | |
47 | "dnsdb", | |
48 | "entrust", | |
49 | "github", | |
50 | "hackertarget", | |
51 | "ipv4info", | |
52 | "intelx", | |
53 | "passivetotal", | |
54 | "rapiddns", | |
55 | "securitytrails", | |
56 | "shodan", | |
57 | "sitedossier", | |
58 | "spyse", | |
59 | "sublist3r", | |
60 | "threatcrowd", | |
61 | "threatminer", | |
62 | "urlscan", | |
63 | "virustotal", | |
64 | "waybackarchive", | |
65 | "zoomeye", | |
66 | } | |
67 | ||
68 | // Agent is a struct for running passive subdomain enumeration | |
69 | // against a given host. It wraps subscraping package and provides | |
70 | // a layer to build upon. | |
71 | type Agent struct { | |
72 | sources map[string]subscraping.Source | |
73 | } | |
74 | ||
75 | // New creates a new agent for passive subdomain discovery | |
76 | func New(sources []string, exclusions []string) *Agent { | |
77 | // Create the agent, insert the sources and remove the excluded sources | |
78 | agent := &Agent{sources: make(map[string]subscraping.Source)} | |
79 | ||
80 | agent.addSources(sources) | |
81 | agent.removeSources(exclusions) | |
82 | ||
83 | return agent | |
84 | } | |
85 | ||
86 | // addSources adds the given list of sources to the source array | |
87 | func (a *Agent) addSources(sources []string) { | |
88 | for _, source := range sources { | |
89 | switch source { | |
90 | case "alienvault": | |
91 | a.sources[source] = &alienvault.Source{} | |
92 | case "archiveis": | |
93 | a.sources[source] = &archiveis.Source{} | |
94 | case "binaryedge": | |
95 | a.sources[source] = &binaryedge.Source{} | |
96 | case "bufferover": | |
97 | a.sources[source] = &bufferover.Source{} | |
98 | case "censys": | |
99 | a.sources[source] = &censys.Source{} | |
100 | case "certspotter": | |
101 | a.sources[source] = &certspotter.Source{} | |
102 | case "certspotterold": | |
103 | a.sources[source] = &certspotterold.Source{} | |
104 | case "commoncrawl": | |
105 | a.sources[source] = &commoncrawl.Source{} | |
106 | case "crtsh": | |
107 | a.sources[source] = &crtsh.Source{} | |
108 | case "dnsdumpster": | |
109 | a.sources[source] = &dnsdumpster.Source{} | |
110 | case "dnsdb": | |
111 | a.sources[source] = &dnsdb.Source{} | |
112 | case "entrust": | |
113 | a.sources[source] = &entrust.Source{} | |
114 | case "github": | |
115 | a.sources[source] = &github.Source{} | |
116 | case "hackertarget": | |
117 | a.sources[source] = &hackertarget.Source{} | |
118 | case "ipv4info": | |
119 | a.sources[source] = &ipv4info.Source{} | |
120 | case "intelx": | |
121 | a.sources[source] = &intelx.Source{} | |
122 | case "passivetotal": | |
123 | a.sources[source] = &passivetotal.Source{} | |
124 | case "rapiddns": | |
125 | a.sources[source] = &rapiddns.Source{} | |
126 | case "securitytrails": | |
127 | a.sources[source] = &securitytrails.Source{} | |
128 | case "shodan": | |
129 | a.sources[source] = &shodan.Source{} | |
130 | case "sitedossier": | |
131 | a.sources[source] = &sitedossier.Source{} | |
132 | case "spyse": | |
133 | a.sources[source] = &spyse.Source{} | |
134 | case "sublist3r": | |
135 | a.sources[source] = &sublist3r.Source{} | |
136 | case "threatcrowd": | |
137 | a.sources[source] = &threatcrowd.Source{} | |
138 | case "threatminer": | |
139 | a.sources[source] = &threatminer.Source{} | |
140 | case "urlscan": | |
141 | a.sources[source] = &urlscan.Source{} | |
142 | case "virustotal": | |
143 | a.sources[source] = &virustotal.Source{} | |
144 | case "waybackarchive": | |
145 | a.sources[source] = &waybackarchive.Source{} | |
146 | case "zoomeye": | |
147 | a.sources[source] = &zoomeye.Source{} | |
148 | } | |
149 | } | |
150 | } | |
151 | ||
152 | // removeSources deletes the given sources from the source map | |
153 | func (a *Agent) removeSources(sources []string) { | |
154 | for _, source := range sources { | |
155 | delete(a.sources, source) | |
156 | } | |
157 | } |
0 | package resolve | |
1 | ||
2 | import ( | |
3 | "bufio" | |
4 | "math/rand" | |
5 | "os" | |
6 | "time" | |
7 | ) | |
8 | ||
9 | // DefaultResolvers contains the default list of resolvers known to be good | |
10 | var DefaultResolvers = []string{ | |
11 | "1.1.1.1", // Cloudflare primary | |
12 | "1.0.0.1", // Cloudlfare secondary | |
13 | "8.8.8.8", // Google primary | |
14 | "8.8.4.4", // Google secondary | |
15 | "9.9.9.9", // Quad9 Primary | |
16 | "9.9.9.10", // Quad9 Secondary | |
17 | "77.88.8.8", // Yandex Primary | |
18 | "77.88.8.1", // Yandex Secondary | |
19 | "208.67.222.222", // OpenDNS Primary | |
20 | "208.67.220.220", // OpenDNS Secondary | |
21 | } | |
22 | ||
23 | // Resolver is a struct for resolving DNS names | |
24 | type Resolver struct { | |
25 | resolvers []string | |
26 | rand *rand.Rand | |
27 | } | |
28 | ||
29 | // New creates a new resolver struct with the default resolvers | |
30 | func New() *Resolver { | |
31 | return &Resolver{ | |
32 | resolvers: []string{}, | |
33 | rand: rand.New(rand.NewSource(time.Now().UnixNano())), | |
34 | } | |
35 | } | |
36 | ||
37 | // AppendResolversFromFile appends the resolvers read from a file to the list of resolvers | |
38 | func (r *Resolver) AppendResolversFromFile(file string) error { | |
39 | f, err := os.Open(file) | |
40 | if err != nil { | |
41 | return err | |
42 | } | |
43 | scanner := bufio.NewScanner(f) | |
44 | for scanner.Scan() { | |
45 | text := scanner.Text() | |
46 | if text == "" { | |
47 | continue | |
48 | } | |
49 | r.resolvers = append(r.resolvers, text) | |
50 | } | |
51 | f.Close() | |
52 | return scanner.Err() | |
53 | } | |
54 | ||
55 | // AppendResolversFromSlice appends the slice to the list of resolvers | |
56 | func (r *Resolver) AppendResolversFromSlice(list []string) { | |
57 | r.resolvers = append(r.resolvers, list...) | |
58 | } |
0 | // Package resolve is used to handle resolving records | |
1 | // It also handles wildcard subdomains and rotating resolvers. | |
2 | package resolve |
0 | package resolve | |
1 | ||
2 | import ( | |
3 | "sync" | |
4 | ||
5 | "github.com/miekg/dns" | |
6 | "github.com/rs/xid" | |
7 | ) | |
8 | ||
9 | const ( | |
10 | maxResolveRetries = 5 | |
11 | maxWildcardChecks = 3 | |
12 | ) | |
13 | ||
14 | // ResolutionPool is a pool of resolvers created for resolving subdomains | |
15 | // for a given host. | |
16 | type ResolutionPool struct { | |
17 | *Resolver | |
18 | Tasks chan string | |
19 | Results chan Result | |
20 | wg *sync.WaitGroup | |
21 | removeWildcard bool | |
22 | ||
23 | wildcardIPs map[string]struct{} | |
24 | } | |
25 | ||
26 | // Result contains the result for a host resolution | |
27 | type Result struct { | |
28 | Type ResultType | |
29 | Host string | |
30 | IP string | |
31 | Error error | |
32 | } | |
33 | ||
34 | // ResultType is the type of result found | |
35 | type ResultType int | |
36 | ||
37 | // Types of data result can return | |
38 | const ( | |
39 | Subdomain ResultType = iota | |
40 | Error | |
41 | ) | |
42 | ||
43 | // NewResolutionPool creates a pool of resolvers for resolving subdomains of a given domain | |
44 | func (r *Resolver) NewResolutionPool(workers int, removeWildcard bool) *ResolutionPool { | |
45 | resolutionPool := &ResolutionPool{ | |
46 | Resolver: r, | |
47 | Tasks: make(chan string), | |
48 | Results: make(chan Result), | |
49 | wg: &sync.WaitGroup{}, | |
50 | removeWildcard: removeWildcard, | |
51 | wildcardIPs: make(map[string]struct{}), | |
52 | } | |
53 | ||
54 | go func() { | |
55 | for i := 0; i < workers; i++ { | |
56 | resolutionPool.wg.Add(1) | |
57 | go resolutionPool.resolveWorker() | |
58 | } | |
59 | resolutionPool.wg.Wait() | |
60 | close(resolutionPool.Results) | |
61 | }() | |
62 | ||
63 | return resolutionPool | |
64 | } | |
65 | ||
66 | // InitWildcards inits the wildcard ips array | |
67 | func (r *ResolutionPool) InitWildcards(domain string) error { | |
68 | for i := 0; i < maxWildcardChecks; i++ { | |
69 | uid := xid.New().String() | |
70 | ||
71 | hosts, err := r.getARecords(uid + "." + domain) | |
72 | if err != nil { | |
73 | return err | |
74 | } | |
75 | ||
76 | // Append all wildcard ips found for domains | |
77 | for _, host := range hosts { | |
78 | r.wildcardIPs[host] = struct{}{} | |
79 | } | |
80 | } | |
81 | return nil | |
82 | } | |
83 | ||
84 | func (r *ResolutionPool) resolveWorker() { | |
85 | for task := range r.Tasks { | |
86 | if !r.removeWildcard { | |
87 | r.Results <- Result{Type: Subdomain, Host: task, IP: ""} | |
88 | continue | |
89 | } | |
90 | ||
91 | hosts, err := r.getARecords(task) | |
92 | if err != nil { | |
93 | r.Results <- Result{Type: Error, Error: err} | |
94 | continue | |
95 | } | |
96 | ||
97 | if len(hosts) == 0 { | |
98 | continue | |
99 | } | |
100 | ||
101 | for _, host := range hosts { | |
102 | // Ignore the host if it exists in wildcard ips map | |
103 | if _, ok := r.wildcardIPs[host]; ok { | |
104 | continue | |
105 | } | |
106 | } | |
107 | ||
108 | r.Results <- Result{Type: Subdomain, Host: task, IP: hosts[0]} | |
109 | } | |
110 | r.wg.Done() | |
111 | } | |
112 | ||
113 | // getARecords gets all the A records for a given host | |
114 | func (r *ResolutionPool) getARecords(host string) ([]string, error) { | |
115 | var iteration int | |
116 | ||
117 | m := new(dns.Msg) | |
118 | m.Id = dns.Id() | |
119 | m.RecursionDesired = true | |
120 | m.Question = make([]dns.Question, 1) | |
121 | m.Question[0] = dns.Question{ | |
122 | Name: dns.Fqdn(host), | |
123 | Qtype: dns.TypeA, | |
124 | Qclass: dns.ClassINET, | |
125 | } | |
126 | exchange: | |
127 | iteration++ | |
128 | in, err := dns.Exchange(m, r.resolvers[r.rand.Intn(len(r.resolvers))]+":53") | |
129 | if err != nil { | |
130 | // Retry in case of I/O error | |
131 | if iteration <= maxResolveRetries { | |
132 | goto exchange | |
133 | } | |
134 | return nil, err | |
135 | } | |
136 | // Ignore the error in case we have bad result | |
137 | if in != nil && in.Rcode != dns.RcodeSuccess { | |
138 | return nil, nil | |
139 | } | |
140 | ||
141 | var hosts []string | |
142 | for _, record := range in.Answer { | |
143 | if t, ok := record.(*dns.A); ok { | |
144 | hosts = append(hosts, t.A.String()) | |
145 | } | |
146 | } | |
147 | ||
148 | return hosts, nil | |
149 | } |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "github.com/projectdiscovery/gologger" | |
4 | "github.com/projectdiscovery/subfinder/pkg/passive" | |
5 | "github.com/projectdiscovery/subfinder/pkg/resolve" | |
6 | ) | |
7 | ||
8 | const banner = ` | |
9 | _ __ _ _ | |
10 | ____ _| |__ / _(_)_ _ __| |___ _ _ | |
11 | (_-< || | '_ \ _| | ' \/ _ / -_) '_| | |
12 | /__/\_,_|_.__/_| |_|_||_\__,_\___|_| v2 | |
13 | ` | |
14 | ||
15 | // Version is the current version of subfinder | |
16 | const Version = `2.3.8` | |
17 | ||
18 | // showBanner is used to show the banner to the user | |
19 | func showBanner() { | |
20 | gologger.Printf("%s\n", banner) | |
21 | gologger.Printf("\t\tprojectdiscovery.io\n\n") | |
22 | ||
23 | gologger.Labelf("Use with caution. You are responsible for your actions\n") | |
24 | gologger.Labelf("Developers assume no liability and are not responsible for any misuse or damage.\n") | |
25 | gologger.Labelf("By using subfinder, you also agree to the terms of the APIs used.\n\n") | |
26 | } | |
27 | ||
28 | // normalRunTasks runs the normal startup tasks | |
29 | func (options *Options) normalRunTasks() { | |
30 | configFile, err := UnmarshalRead(options.ConfigFile) | |
31 | if err != nil { | |
32 | gologger.Fatalf("Could not read configuration file %s: %s\n", options.ConfigFile, err) | |
33 | } | |
34 | options.YAMLConfig = configFile | |
35 | } | |
36 | ||
37 | // firstRunTasks runs some housekeeping tasks done | |
38 | // when the program is ran for the first time | |
39 | func (options *Options) firstRunTasks() { | |
40 | // Create the configuration file and display information | |
41 | // about it to the user. | |
42 | config := ConfigFile{ | |
43 | // Use the default list of resolvers by marshalling it to the config | |
44 | Resolvers: resolve.DefaultResolvers, | |
45 | // Use the default list of passive sources | |
46 | Sources: passive.DefaultSources, | |
47 | } | |
48 | ||
49 | err := config.MarshalWrite(options.ConfigFile) | |
50 | if err != nil { | |
51 | gologger.Fatalf("Could not write configuration file to %s: %s\n", options.ConfigFile, err) | |
52 | } | |
53 | options.YAMLConfig = config | |
54 | ||
55 | gologger.Infof("Configuration file saved to %s\n", options.ConfigFile) | |
56 | } |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "math/rand" | |
4 | "os" | |
5 | "strings" | |
6 | "time" | |
7 | ||
8 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
9 | "gopkg.in/yaml.v3" | |
10 | ) | |
11 | ||
12 | // ConfigFile contains the fields stored in the configuration file | |
13 | type ConfigFile struct { | |
14 | // Resolvers contains the list of resolvers to use while resolving | |
15 | Resolvers []string `yaml:"resolvers,omitempty"` | |
16 | // Sources contains a list of sources to use for enumeration | |
17 | Sources []string `yaml:"sources,omitempty"` | |
18 | // ExcludeSources contains the sources to not include in the enumeration process | |
19 | ExcludeSources []string `yaml:"exclude-sources,omitempty"` | |
20 | // API keys for different sources | |
21 | Binaryedge []string `yaml:"binaryedge"` | |
22 | Censys []string `yaml:"censys"` | |
23 | Certspotter []string `yaml:"certspotter"` | |
24 | Chaos []string `yaml:"chaos"` | |
25 | DNSDB []string `yaml:"dnsdb"` | |
26 | GitHub []string `yaml:"github"` | |
27 | IntelX []string `yaml:"intelx"` | |
28 | PassiveTotal []string `yaml:"passivetotal"` | |
29 | SecurityTrails []string `yaml:"securitytrails"` | |
30 | Shodan []string `yaml:"shodan"` | |
31 | Spyse []string `yaml:"spyse"` | |
32 | URLScan []string `yaml:"urlscan"` | |
33 | Virustotal []string `yaml:"virustotal"` | |
34 | ZoomEye []string `yaml:"zoomeye"` | |
35 | } | |
36 | ||
37 | // GetConfigDirectory gets the subfinder config directory for a user | |
38 | func GetConfigDirectory() (string, error) { | |
39 | // Seed the random number generator | |
40 | rand.Seed(time.Now().UnixNano()) | |
41 | ||
42 | var config string | |
43 | ||
44 | directory, err := os.UserHomeDir() | |
45 | if err != nil { | |
46 | return config, err | |
47 | } | |
48 | config = directory + "/.config/subfinder" | |
49 | // Create All directory for subfinder even if they exist | |
50 | os.MkdirAll(config, os.ModePerm) | |
51 | ||
52 | return config, nil | |
53 | } | |
54 | ||
55 | // CheckConfigExists checks if the config file exists in the given path | |
56 | func CheckConfigExists(configPath string) bool { | |
57 | if _, err := os.Stat(configPath); err == nil { | |
58 | return true | |
59 | } else if os.IsNotExist(err) { | |
60 | return false | |
61 | } | |
62 | return false | |
63 | } | |
64 | ||
65 | // MarshalWrite writes the marshalled yaml config to disk | |
66 | func (c ConfigFile) MarshalWrite(file string) error { | |
67 | f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE, 0755) | |
68 | if err != nil { | |
69 | return err | |
70 | } | |
71 | ||
72 | // Indent the spaces too | |
73 | enc := yaml.NewEncoder(f) | |
74 | enc.SetIndent(4) | |
75 | err = enc.Encode(&c) | |
76 | f.Close() | |
77 | return err | |
78 | } | |
79 | ||
80 | // UnmarshalRead reads the unmarshalled config yaml file from disk | |
81 | func UnmarshalRead(file string) (ConfigFile, error) { | |
82 | config := ConfigFile{} | |
83 | ||
84 | f, err := os.Open(file) | |
85 | if err != nil { | |
86 | return config, err | |
87 | } | |
88 | err = yaml.NewDecoder(f).Decode(&config) | |
89 | f.Close() | |
90 | return config, err | |
91 | } | |
92 | ||
93 | // GetKeys gets the API keys from config file and creates a Keys struct | |
94 | // We use random selection of api keys from the list of keys supplied. | |
95 | // Keys that require 2 options are separated by colon (:). | |
96 | func (c ConfigFile) GetKeys() subscraping.Keys { | |
97 | keys := subscraping.Keys{} | |
98 | ||
99 | if len(c.Binaryedge) > 0 { | |
100 | keys.Binaryedge = c.Binaryedge[rand.Intn(len(c.Binaryedge))] | |
101 | } | |
102 | ||
103 | if len(c.Censys) > 0 { | |
104 | censysKeys := c.Censys[rand.Intn(len(c.Censys))] | |
105 | parts := strings.Split(censysKeys, ":") | |
106 | if len(parts) == 2 { | |
107 | keys.CensysToken = parts[0] | |
108 | keys.CensysSecret = parts[1] | |
109 | } | |
110 | } | |
111 | ||
112 | if len(c.Certspotter) > 0 { | |
113 | keys.Certspotter = c.Certspotter[rand.Intn(len(c.Certspotter))] | |
114 | } | |
115 | if len(c.Chaos) > 0 { | |
116 | keys.Chaos = c.Chaos[rand.Intn(len(c.Chaos))] | |
117 | } | |
118 | if (len(c.DNSDB)) > 0 { | |
119 | keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))] | |
120 | } | |
121 | if (len(c.GitHub)) > 0 { | |
122 | keys.GitHub = c.GitHub | |
123 | } | |
124 | ||
125 | if len(c.IntelX) > 0 { | |
126 | intelxKeys := c.IntelX[rand.Intn(len(c.IntelX))] | |
127 | parts := strings.Split(intelxKeys, ":") | |
128 | if len(parts) == 2 { | |
129 | keys.IntelXHost = parts[0] | |
130 | keys.IntelXKey = parts[1] | |
131 | } | |
132 | } | |
133 | ||
134 | if len(c.PassiveTotal) > 0 { | |
135 | passiveTotalKeys := c.PassiveTotal[rand.Intn(len(c.PassiveTotal))] | |
136 | parts := strings.Split(passiveTotalKeys, ":") | |
137 | if len(parts) == 2 { | |
138 | keys.PassiveTotalUsername = parts[0] | |
139 | keys.PassiveTotalPassword = parts[1] | |
140 | } | |
141 | } | |
142 | ||
143 | if len(c.SecurityTrails) > 0 { | |
144 | keys.Securitytrails = c.SecurityTrails[rand.Intn(len(c.SecurityTrails))] | |
145 | } | |
146 | if len(c.Shodan) > 0 { | |
147 | keys.Shodan = c.Shodan[rand.Intn(len(c.Shodan))] | |
148 | } | |
149 | if len(c.Spyse) > 0 { | |
150 | keys.Spyse = c.Spyse[rand.Intn(len(c.Spyse))] | |
151 | } | |
152 | if len(c.URLScan) > 0 { | |
153 | keys.URLScan = c.URLScan[rand.Intn(len(c.URLScan))] | |
154 | } | |
155 | if len(c.Virustotal) > 0 { | |
156 | keys.Virustotal = c.Virustotal[rand.Intn(len(c.Virustotal))] | |
157 | } | |
158 | if len(c.ZoomEye) > 0 { | |
159 | zoomEyeKeys := c.ZoomEye[rand.Intn(len(c.ZoomEye))] | |
160 | parts := strings.Split(zoomEyeKeys, ":") | |
161 | if len(parts) == 2 { | |
162 | keys.ZoomEyeUsername = parts[0] | |
163 | keys.ZoomEyePassword = parts[1] | |
164 | } | |
165 | } | |
166 | ||
167 | return keys | |
168 | } |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "os" | |
4 | "testing" | |
5 | ||
6 | "github.com/stretchr/testify/assert" | |
7 | ) | |
8 | ||
9 | func TestConfigGetDirectory(t *testing.T) { | |
10 | directory, err := GetConfigDirectory() | |
11 | if err != nil { | |
12 | t.Fatalf("Expected nil got %v while getting home\n", err) | |
13 | } | |
14 | home, err := os.UserHomeDir() | |
15 | if err != nil { | |
16 | t.Fatalf("Expected nil got %v while getting dir\n", err) | |
17 | } | |
18 | config := home + "/.config/subfinder" | |
19 | ||
20 | assert.Equal(t, directory, config, "Directory and config should be equal") | |
21 | } |
0 | // Package runner implements the mechanism to drive the | |
1 | // subdomain enumeration process | |
2 | package runner |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "bytes" | |
4 | "os" | |
5 | "strings" | |
6 | "sync" | |
7 | "time" | |
8 | ||
9 | "github.com/projectdiscovery/gologger" | |
10 | "github.com/projectdiscovery/subfinder/pkg/resolve" | |
11 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
12 | ) | |
13 | ||
14 | // EnumerateSingleDomain performs subdomain enumeration against a single domain | |
15 | func (r *Runner) EnumerateSingleDomain(domain, output string, append bool) error { | |
16 | gologger.Infof("Enumerating subdomains for %s\n", domain) | |
17 | ||
18 | // Get the API keys for sources from the configuration | |
19 | // and also create the active resolving engine for the domain. | |
20 | keys := r.options.YAMLConfig.GetKeys() | |
21 | ||
22 | // Check if the user has asked to remove wildcards explicitly. | |
23 | // If yes, create the resolution pool and get the wildcards for the current domain | |
24 | var resolutionPool *resolve.ResolutionPool | |
25 | if r.options.RemoveWildcard { | |
26 | resolutionPool = r.resolverClient.NewResolutionPool(r.options.Threads, r.options.RemoveWildcard) | |
27 | err := resolutionPool.InitWildcards(domain) | |
28 | if err != nil { | |
29 | // Log the error but don't quit. | |
30 | gologger.Warningf("Could not get wildcards for domain %s: %s\n", domain, err) | |
31 | } | |
32 | } | |
33 | ||
34 | // Run the passive subdomain enumeration | |
35 | passiveResults := r.passiveAgent.EnumerateSubdomains(domain, keys, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) | |
36 | ||
37 | wg := &sync.WaitGroup{} | |
38 | wg.Add(1) | |
39 | // Create a unique map for filtering duplicate subdomains out | |
40 | uniqueMap := make(map[string]struct{}) | |
41 | // Process the results in a separate goroutine | |
42 | go func() { | |
43 | for result := range passiveResults { | |
44 | switch result.Type { | |
45 | case subscraping.Error: | |
46 | gologger.Warningf("Could not run source %s: %s\n", result.Source, result.Error) | |
47 | case subscraping.Subdomain: | |
48 | // Validate the subdomain found and remove wildcards from | |
49 | if !strings.HasSuffix(result.Value, "."+domain) { | |
50 | continue | |
51 | } | |
52 | subdomain := strings.ReplaceAll(strings.ToLower(result.Value), "*.", "") | |
53 | ||
54 | // Check if the subdomain is a duplicate. If not, | |
55 | // send the subdomain for resolution. | |
56 | if _, ok := uniqueMap[subdomain]; ok { | |
57 | continue | |
58 | } | |
59 | uniqueMap[subdomain] = struct{}{} | |
60 | ||
61 | // Log the verbose message about the found subdomain and send the | |
62 | // host for resolution to the resolution pool | |
63 | gologger.Verbosef("%s\n", result.Source, subdomain) | |
64 | ||
65 | // If the user asked to remove wildcard then send on the resolve | |
66 | // queue. Otherwise, if mode is not verbose print the results on | |
67 | // the screen as they are discovered. | |
68 | if r.options.RemoveWildcard { | |
69 | resolutionPool.Tasks <- subdomain | |
70 | } | |
71 | ||
72 | if !r.options.Verbose { | |
73 | gologger.Silentf("%s\n", subdomain) | |
74 | } | |
75 | } | |
76 | } | |
77 | // Close the task channel only if wildcards are asked to be removed | |
78 | if r.options.RemoveWildcard { | |
79 | close(resolutionPool.Tasks) | |
80 | } | |
81 | wg.Done() | |
82 | }() | |
83 | ||
84 | // If the user asked to remove wildcards, listen from the results | |
85 | // queue and write to the map. At the end, print the found results to the screen | |
86 | foundResults := make(map[string]string) | |
87 | if r.options.RemoveWildcard { | |
88 | // Process the results coming from the resolutions pool | |
89 | for result := range resolutionPool.Results { | |
90 | switch result.Type { | |
91 | case resolve.Error: | |
92 | gologger.Warningf("Could not resolve host: %s\n", result.Error) | |
93 | case resolve.Subdomain: | |
94 | // Add the found subdomain to a map. | |
95 | if _, ok := foundResults[result.Host]; !ok { | |
96 | foundResults[result.Host] = result.IP | |
97 | } | |
98 | } | |
99 | } | |
100 | } | |
101 | wg.Wait() | |
102 | ||
103 | // If verbose mode was used, then now print all the | |
104 | // found subdomains on the screen together. | |
105 | if r.options.Verbose { | |
106 | if r.options.RemoveWildcard { | |
107 | for result := range foundResults { | |
108 | gologger.Silentf("%s\n", result) | |
109 | } | |
110 | } else { | |
111 | for result := range uniqueMap { | |
112 | gologger.Silentf("%s\n", result) | |
113 | } | |
114 | } | |
115 | } | |
116 | // In case the user has specified to upload to chaos, write everything to a temporary buffer and upload | |
117 | if r.options.ChaosUpload { | |
118 | var buf = &bytes.Buffer{} | |
119 | err := WriteHostOutput(uniqueMap, buf) | |
120 | // If an error occurs, do not interrupt, continue to check if user specifed an output file | |
121 | if err != nil { | |
122 | gologger.Errorf("Could not prepare results for chaos %s\n", err) | |
123 | } else { | |
124 | // no error in writing host output, upload to chaos | |
125 | err = r.UploadToChaos(buf) | |
126 | if err != nil { | |
127 | gologger.Errorf("Could not upload results to chaos %s\n", err) | |
128 | } else { | |
129 | gologger.Infof("Input processed successfully and subdomains with valid records will be updated to chaos dataset.\n") | |
130 | } | |
131 | // clear buffer | |
132 | buf = nil | |
133 | } | |
134 | } | |
135 | // In case the user has given an output file, write all the found | |
136 | // subdomains to the output file. | |
137 | if output != "" { | |
138 | // If the output format is json, append .json | |
139 | // else append .txt | |
140 | if r.options.OutputDirectory != "" { | |
141 | if r.options.JSON { | |
142 | output = output + ".json" | |
143 | } else { | |
144 | output = output + ".txt" | |
145 | } | |
146 | } | |
147 | ||
148 | var file *os.File | |
149 | var err error | |
150 | if append { | |
151 | file, err = os.OpenFile(output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) | |
152 | } else { | |
153 | file, err = os.Create(output) | |
154 | } | |
155 | if err != nil { | |
156 | gologger.Errorf("Could not create file %s for %s: %s\n", output, domain, err) | |
157 | return err | |
158 | } | |
159 | ||
160 | // Write the output to the file depending upon user requirement | |
161 | if r.options.HostIP { | |
162 | err = WriteHostIPOutput(foundResults, file) | |
163 | } else if r.options.JSON { | |
164 | err = WriteJSONOutput(foundResults, file) | |
165 | } else { | |
166 | if r.options.RemoveWildcard { | |
167 | err = WriteHostOutputNoWildcard(foundResults, file) | |
168 | } else { | |
169 | err = WriteHostOutput(uniqueMap, file) | |
170 | } | |
171 | } | |
172 | if err != nil { | |
173 | gologger.Errorf("Could not write results to file %s for %s: %s\n", output, domain, err) | |
174 | } | |
175 | file.Close() | |
176 | return err | |
177 | } | |
178 | return nil | |
179 | } |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "strings" | |
4 | ||
5 | "github.com/projectdiscovery/subfinder/pkg/passive" | |
6 | "github.com/projectdiscovery/subfinder/pkg/resolve" | |
7 | ) | |
8 | ||
9 | // initializePassiveEngine creates the passive engine and loads sources etc | |
10 | func (r *Runner) initializePassiveEngine() { | |
11 | var sources, exclusions []string | |
12 | ||
13 | // If there are any sources from CLI, only use them | |
14 | // Otherwise, use the yaml file sources | |
15 | if r.options.Sources != "" { | |
16 | sources = append(sources, strings.Split(r.options.Sources, ",")...) | |
17 | } else { | |
18 | sources = append(sources, r.options.YAMLConfig.Sources...) | |
19 | } | |
20 | ||
21 | if r.options.ExcludeSources != "" { | |
22 | exclusions = append(exclusions, strings.Split(r.options.ExcludeSources, ",")...) | |
23 | } else { | |
24 | exclusions = append(exclusions, r.options.YAMLConfig.ExcludeSources...) | |
25 | } | |
26 | ||
27 | r.passiveAgent = passive.New(sources, exclusions) | |
28 | } | |
29 | ||
30 | // initializeActiveEngine creates the resolver used to resolve the found subdomains | |
31 | func (r *Runner) initializeActiveEngine() error { | |
32 | r.resolverClient = resolve.New() | |
33 | ||
34 | // If the file has been provided, read resolvers from the file | |
35 | if r.options.ResolverList != "" { | |
36 | err := r.resolverClient.AppendResolversFromFile(r.options.ResolverList) | |
37 | if err != nil { | |
38 | return err | |
39 | } | |
40 | } | |
41 | ||
42 | var resolvers []string | |
43 | ||
44 | if r.options.Resolvers != "" { | |
45 | resolvers = append(resolvers, strings.Split(r.options.Resolvers, ",")...) | |
46 | } else { | |
47 | resolvers = append(resolvers, r.options.YAMLConfig.Resolvers...) | |
48 | } | |
49 | r.resolverClient.AppendResolversFromSlice(resolvers) | |
50 | return nil | |
51 | } |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "flag" | |
4 | "os" | |
5 | "path" | |
6 | "reflect" | |
7 | "strings" | |
8 | ||
9 | "github.com/projectdiscovery/gologger" | |
10 | ) | |
11 | ||
12 | // Options contains the configuration options for tuning | |
13 | // the subdomain enumeration process. | |
14 | type Options struct { | |
15 | Verbose bool // Verbose flag indicates whether to show verbose output or not | |
16 | NoColor bool // No-Color disables the colored output | |
17 | Threads int // Thread controls the number of threads to use for active enumerations | |
18 | Timeout int // Timeout is the seconds to wait for sources to respond | |
19 | MaxEnumerationTime int // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration | |
20 | Domain string // Domain is the domain to find subdomains for | |
21 | DomainsFile string // DomainsFile is the file containing list of domains to find subdomains for | |
22 | ChaosUpload bool // ChaosUpload indicates whether to upload results to the Chaos API | |
23 | Output string // Output is the file to write found subdomains to. | |
24 | OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given | |
25 | JSON bool // JSON specifies whether to use json for output format or text file | |
26 | HostIP bool // HostIP specifies whether to write subdomains in host:ip format | |
27 | Silent bool // Silent suppresses any extra text and only writes subdomains to screen | |
28 | Sources string // Sources contains a comma-separated list of sources to use for enumeration | |
29 | ListSources bool // ListSources specifies whether to list all available sources | |
30 | ExcludeSources string // ExcludeSources contains the comma-separated sources to not include in the enumeration process | |
31 | Resolvers string // Resolvers is the comma-separated resolvers to use for enumeration | |
32 | ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration | |
33 | RemoveWildcard bool // RemoveWildcard specifies whether to remove potential wildcard or dead subdomains from the results. | |
34 | ConfigFile string // ConfigFile contains the location of the config file | |
35 | Stdin bool // Stdin specifies whether stdin input was given to the process | |
36 | Version bool // Version specifies if we should just show version and exit | |
37 | ||
38 | YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file | |
39 | } | |
40 | ||
41 | // ParseOptions parses the command line flags provided by a user | |
42 | func ParseOptions() *Options { | |
43 | options := &Options{} | |
44 | ||
45 | config, err := GetConfigDirectory() | |
46 | if err != nil { | |
47 | // This should never be reached | |
48 | gologger.Fatalf("Could not get user home: %s\n", err) | |
49 | } | |
50 | ||
51 | flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output") | |
52 | flag.BoolVar(&options.NoColor, "nC", false, "Don't Use colors in output") | |
53 | flag.IntVar(&options.Threads, "t", 10, "Number of concurrent goroutines for resolving") | |
54 | flag.IntVar(&options.Timeout, "timeout", 30, "Seconds to wait before timing out") | |
55 | flag.IntVar(&options.MaxEnumerationTime, "max-time", 10, "Minutes to wait for enumeration results") | |
56 | flag.StringVar(&options.Domain, "d", "", "Domain to find subdomains for") | |
57 | flag.StringVar(&options.DomainsFile, "dL", "", "File containing list of domains to enumerate") | |
58 | flag.BoolVar(&options.ChaosUpload, "cd", false, "Upload results to the Chaos API (api-key required)") | |
59 | flag.StringVar(&options.Output, "o", "", "File to write output to (optional)") | |
60 | flag.StringVar(&options.OutputDirectory, "oD", "", "Directory to write enumeration results to (optional)") | |
61 | flag.BoolVar(&options.JSON, "oJ", false, "Write output in JSON lines Format") | |
62 | flag.BoolVar(&options.HostIP, "oI", false, "Write output in Host,IP format") | |
63 | flag.BoolVar(&options.Silent, "silent", false, "Show only subdomains in output") | |
64 | flag.StringVar(&options.Sources, "sources", "", "Comma separated list of sources to use") | |
65 | flag.BoolVar(&options.ListSources, "ls", false, "List all available sources") | |
66 | flag.StringVar(&options.ExcludeSources, "exclude-sources", "", "List of sources to exclude from enumeration") | |
67 | flag.StringVar(&options.Resolvers, "r", "", "Comma-separated list of resolvers to use") | |
68 | flag.StringVar(&options.ResolverList, "rL", "", "Text file containing list of resolvers to use") | |
69 | flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") | |
70 | flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") | |
71 | flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") | |
72 | flag.Parse() | |
73 | ||
74 | // Check if stdin pipe was given | |
75 | options.Stdin = hasStdin() | |
76 | ||
77 | // Read the inputs and configure the logging | |
78 | options.configureOutput() | |
79 | ||
80 | // Show the user the banner | |
81 | showBanner() | |
82 | ||
83 | if options.Version { | |
84 | gologger.Infof("Current Version: %s\n", Version) | |
85 | os.Exit(0) | |
86 | } | |
87 | ||
88 | // Check if the config file exists. If not, it means this is the | |
89 | // first run of the program. Show the first run notices and initialize the config file. | |
90 | // Else show the normal banners and read the yaml fiile to the config | |
91 | if !CheckConfigExists(options.ConfigFile) { | |
92 | options.firstRunTasks() | |
93 | } else { | |
94 | options.normalRunTasks() | |
95 | } | |
96 | ||
97 | if options.ListSources { | |
98 | listSources(options) | |
99 | os.Exit(0) | |
100 | } | |
101 | ||
102 | // Validate the options passed by the user and if any | |
103 | // invalid options have been used, exit. | |
104 | err = options.validateOptions() | |
105 | if err != nil { | |
106 | gologger.Fatalf("Program exiting: %s\n", err) | |
107 | } | |
108 | ||
109 | return options | |
110 | } | |
111 | ||
112 | func hasStdin() bool { | |
113 | fi, err := os.Stdin.Stat() | |
114 | if err != nil { | |
115 | return false | |
116 | } | |
117 | if fi.Mode()&os.ModeNamedPipe == 0 { | |
118 | return false | |
119 | } | |
120 | return true | |
121 | } | |
122 | ||
123 | func listSources(options *Options) { | |
124 | gologger.Infof("Current list of available sources. [%d]\n", len(options.YAMLConfig.Sources)) | |
125 | gologger.Infof("Sources marked with an * needs key or token in order to work.\n") | |
126 | gologger.Infof("You can modify %s to configure your keys / tokens.\n\n", options.ConfigFile) | |
127 | ||
128 | keys := options.YAMLConfig.GetKeys() | |
129 | needsKey := make(map[string]interface{}) | |
130 | keysElem := reflect.ValueOf(&keys).Elem() | |
131 | for i := 0; i < keysElem.NumField(); i++ { | |
132 | needsKey[strings.ToLower(keysElem.Type().Field(i).Name)] = keysElem.Field(i).Interface() | |
133 | } | |
134 | ||
135 | for _, source := range options.YAMLConfig.Sources { | |
136 | message := "%s\n" | |
137 | if _, ok := needsKey[source]; ok { | |
138 | message = "%s *\n" | |
139 | } | |
140 | gologger.Silentf(message, source) | |
141 | } | |
142 | } |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "bufio" | |
4 | "io" | |
5 | "os" | |
6 | "path" | |
7 | ||
8 | "github.com/projectdiscovery/subfinder/pkg/passive" | |
9 | "github.com/projectdiscovery/subfinder/pkg/resolve" | |
10 | ) | |
11 | ||
12 | // Runner is an instance of the subdomain enumeration | |
13 | // client used to orchestrate the whole process. | |
14 | type Runner struct { | |
15 | options *Options | |
16 | passiveAgent *passive.Agent | |
17 | resolverClient *resolve.Resolver | |
18 | } | |
19 | ||
20 | // NewRunner creates a new runner struct instance by parsing | |
21 | // the configuration options, configuring sources, reading lists | |
22 | // and setting up loggers, etc. | |
23 | func NewRunner(options *Options) (*Runner, error) { | |
24 | runner := &Runner{options: options} | |
25 | ||
26 | // Initialize the passive subdomain enumeration engine | |
27 | runner.initializePassiveEngine() | |
28 | ||
29 | // Initialize the active subdomain enumeration engine | |
30 | err := runner.initializeActiveEngine() | |
31 | if err != nil { | |
32 | return nil, err | |
33 | } | |
34 | ||
35 | return runner, nil | |
36 | } | |
37 | ||
38 | // RunEnumeration runs the subdomain enumeration flow on the targets specified | |
39 | func (r *Runner) RunEnumeration() error { | |
40 | // Check if only a single domain is sent as input. Process the domain now. | |
41 | if r.options.Domain != "" { | |
42 | return r.EnumerateSingleDomain(r.options.Domain, r.options.Output, false) | |
43 | } | |
44 | ||
45 | // If we have multiple domains as input, | |
46 | if r.options.DomainsFile != "" { | |
47 | f, err := os.Open(r.options.DomainsFile) | |
48 | if err != nil { | |
49 | return err | |
50 | } | |
51 | err = r.EnumerateMultipleDomains(f) | |
52 | f.Close() | |
53 | return err | |
54 | } | |
55 | ||
56 | // If we have STDIN input, treat it as multiple domains | |
57 | if r.options.Stdin { | |
58 | return r.EnumerateMultipleDomains(os.Stdin) | |
59 | } | |
60 | return nil | |
61 | } | |
62 | ||
63 | // EnumerateMultipleDomains enumerates subdomains for multiple domains | |
64 | // We keep enumerating subdomains for a given domain until we reach an error | |
65 | func (r *Runner) EnumerateMultipleDomains(reader io.Reader) error { | |
66 | scanner := bufio.NewScanner(reader) | |
67 | for scanner.Scan() { | |
68 | domain := scanner.Text() | |
69 | if domain == "" { | |
70 | continue | |
71 | } | |
72 | ||
73 | var err error | |
74 | // If the user has specifed an output file, use that output file instead | |
75 | // of creating a new output file for each domain. Else create a new file | |
76 | // for each domain in the directory. | |
77 | if r.options.Output != "" { | |
78 | err = r.EnumerateSingleDomain(domain, r.options.Output, true) | |
79 | } else if r.options.OutputDirectory != "" { | |
80 | outputFile := path.Join(r.options.OutputDirectory, domain) | |
81 | err = r.EnumerateSingleDomain(domain, outputFile, false) | |
82 | } else { | |
83 | err = r.EnumerateSingleDomain(domain, "", true) | |
84 | } | |
85 | if err != nil { | |
86 | return err | |
87 | } | |
88 | } | |
89 | return nil | |
90 | } |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "bufio" | |
4 | "crypto/tls" | |
5 | "fmt" | |
6 | "io" | |
7 | "io/ioutil" | |
8 | "net/http" | |
9 | "strings" | |
10 | "time" | |
11 | ||
12 | jsoniter "github.com/json-iterator/go" | |
13 | "github.com/pkg/errors" | |
14 | ) | |
15 | ||
16 | // JSONResult contains the result for a host in JSON format | |
17 | type JSONResult struct { | |
18 | Host string `json:"host"` | |
19 | IP string `json:"ip"` | |
20 | } | |
21 | ||
22 | func (r *Runner) UploadToChaos(reader io.Reader) error { | |
23 | httpClient := &http.Client{ | |
24 | Transport: &http.Transport{ | |
25 | MaxIdleConnsPerHost: 100, | |
26 | MaxIdleConns: 100, | |
27 | TLSClientConfig: &tls.Config{ | |
28 | InsecureSkipVerify: true, | |
29 | }, | |
30 | }, | |
31 | Timeout: time.Duration(600) * time.Second, // 10 minutes - uploads may take long | |
32 | } | |
33 | ||
34 | request, err := http.NewRequest("POST", "https://dns.projectdiscovery.io/dns/add", reader) | |
35 | if err != nil { | |
36 | return errors.Wrap(err, "could not create request") | |
37 | } | |
38 | request.Header.Set("Authorization", r.options.YAMLConfig.GetKeys().Chaos) | |
39 | ||
40 | resp, err := httpClient.Do(request) | |
41 | if err != nil { | |
42 | return errors.Wrap(err, "could not make request") | |
43 | } | |
44 | defer func() { | |
45 | io.Copy(ioutil.Discard, resp.Body) | |
46 | resp.Body.Close() | |
47 | }() | |
48 | ||
49 | if resp.StatusCode != 200 { | |
50 | return fmt.Errorf("invalid status code received: %d", resp.StatusCode) | |
51 | } | |
52 | return nil | |
53 | } | |
54 | ||
55 | // WriteHostOutput writes the output list of subdomain to an io.Writer | |
56 | func WriteHostOutput(results map[string]struct{}, writer io.Writer) error { | |
57 | bufwriter := bufio.NewWriter(writer) | |
58 | sb := &strings.Builder{} | |
59 | ||
60 | for host := range results { | |
61 | sb.WriteString(host) | |
62 | sb.WriteString("\n") | |
63 | ||
64 | _, err := bufwriter.WriteString(sb.String()) | |
65 | if err != nil { | |
66 | bufwriter.Flush() | |
67 | return err | |
68 | } | |
69 | sb.Reset() | |
70 | } | |
71 | return bufwriter.Flush() | |
72 | } | |
73 | ||
74 | // WriteHostOutputNoWildcard writes the output list of subdomain with nW flag to an io.Writer | |
75 | func WriteHostOutputNoWildcard(results map[string]string, writer io.Writer) error { | |
76 | bufwriter := bufio.NewWriter(writer) | |
77 | sb := &strings.Builder{} | |
78 | ||
79 | for host := range results { | |
80 | sb.WriteString(host) | |
81 | sb.WriteString("\n") | |
82 | ||
83 | _, err := bufwriter.WriteString(sb.String()) | |
84 | if err != nil { | |
85 | bufwriter.Flush() | |
86 | return err | |
87 | } | |
88 | sb.Reset() | |
89 | } | |
90 | return bufwriter.Flush() | |
91 | } | |
92 | ||
93 | // WriteJSONOutput writes the output list of subdomain in JSON to an io.Writer | |
94 | func WriteJSONOutput(results map[string]string, writer io.Writer) error { | |
95 | encoder := jsoniter.NewEncoder(writer) | |
96 | ||
97 | data := JSONResult{} | |
98 | ||
99 | for host, ip := range results { | |
100 | data.Host = host | |
101 | data.IP = ip | |
102 | ||
103 | err := encoder.Encode(&data) | |
104 | if err != nil { | |
105 | return err | |
106 | } | |
107 | } | |
108 | return nil | |
109 | } | |
110 | ||
111 | // WriteHostIPOutput writes the output list of subdomain to an io.Writer | |
112 | func WriteHostIPOutput(results map[string]string, writer io.Writer) error { | |
113 | bufwriter := bufio.NewWriter(writer) | |
114 | sb := &strings.Builder{} | |
115 | ||
116 | for host, ip := range results { | |
117 | sb.WriteString(host) | |
118 | sb.WriteString(",") | |
119 | sb.WriteString(ip) | |
120 | sb.WriteString("\n") | |
121 | ||
122 | _, err := bufwriter.WriteString(sb.String()) | |
123 | if err != nil { | |
124 | bufwriter.Flush() | |
125 | return err | |
126 | } | |
127 | sb.Reset() | |
128 | } | |
129 | return bufwriter.Flush() | |
130 | } |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "errors" | |
4 | ||
5 | "github.com/projectdiscovery/gologger" | |
6 | ) | |
7 | ||
8 | // validateOptions validates the configuration options passed | |
9 | func (options *Options) validateOptions() error { | |
10 | // Check if domain, list of domains, or stdin info was provided. | |
11 | // If none was provided, then return. | |
12 | if options.Domain == "" && options.DomainsFile == "" && !options.Stdin { | |
13 | return errors.New("no input list provided") | |
14 | } | |
15 | ||
16 | // Both verbose and silent flags were used | |
17 | if options.Verbose && options.Silent { | |
18 | return errors.New("both verbose and silent mode specified") | |
19 | } | |
20 | ||
21 | // Validate threads and options | |
22 | if options.Threads == 0 { | |
23 | return errors.New("threads cannot be zero") | |
24 | } | |
25 | if options.Timeout == 0 { | |
26 | return errors.New("timeout cannot be zero") | |
27 | } | |
28 | ||
29 | // JSON cannot be used with hostIP | |
30 | if options.JSON && options.HostIP { | |
31 | return errors.New("hostip flag cannot be used with json flag") | |
32 | } | |
33 | ||
34 | // Always remove wildcard with hostip and json | |
35 | if options.HostIP && !options.RemoveWildcard { | |
36 | return errors.New("hostip flag must be used with RemoveWildcard option") | |
37 | } | |
38 | if options.JSON && !options.RemoveWildcard { | |
39 | return errors.New("JSON flag must be used with RemoveWildcard option") | |
40 | } | |
41 | ||
42 | return nil | |
43 | } | |
44 | ||
45 | // configureOutput configures the output on the screen | |
46 | func (options *Options) configureOutput() { | |
47 | // If the user desires verbose output, show verbose output | |
48 | if options.Verbose { | |
49 | gologger.MaxLevel = gologger.Verbose | |
50 | } | |
51 | if options.NoColor { | |
52 | gologger.UseColors = false | |
53 | } | |
54 | if options.Silent { | |
55 | gologger.MaxLevel = gologger.Silent | |
56 | } | |
57 | } |
0 | package subscraping | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "crypto/tls" | |
5 | "fmt" | |
6 | "io" | |
7 | "io/ioutil" | |
8 | "net/http" | |
9 | "net/url" | |
10 | "time" | |
11 | ) | |
12 | ||
13 | // NewSession creates a new session object for a domain | |
14 | func NewSession(domain string, keys Keys, timeout int) (*Session, error) { | |
15 | client := &http.Client{ | |
16 | Transport: &http.Transport{ | |
17 | MaxIdleConns: 100, | |
18 | MaxIdleConnsPerHost: 100, | |
19 | TLSClientConfig: &tls.Config{ | |
20 | InsecureSkipVerify: true, | |
21 | }, | |
22 | }, | |
23 | Timeout: time.Duration(timeout) * time.Second, | |
24 | } | |
25 | ||
26 | session := &Session{ | |
27 | Client: client, | |
28 | Keys: keys, | |
29 | } | |
30 | ||
31 | // Create a new extractor object for the current domain | |
32 | extractor, err := NewSubdomainExtractor(domain) | |
33 | session.Extractor = extractor | |
34 | ||
35 | return session, err | |
36 | } | |
37 | ||
38 | // NormalGetWithContext makes a normal GET request to a URL with context | |
39 | func (s *Session) NormalGetWithContext(ctx context.Context, url string) (*http.Response, error) { | |
40 | req, err := http.NewRequestWithContext(ctx, "GET", url, nil) | |
41 | if err != nil { | |
42 | return nil, err | |
43 | } | |
44 | ||
45 | // Don't randomize user agents, as they cause issues sometimes | |
46 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36") | |
47 | req.Header.Set("Accept", "*/*") | |
48 | req.Header.Set("Accept-Language", "en") | |
49 | ||
50 | return httpRequestWrapper(s.Client, req) | |
51 | } | |
52 | ||
53 | // Get makes a GET request to a URL | |
54 | func (s *Session) Get(ctx context.Context, url string, cookies string, headers map[string]string) (*http.Response, error) { | |
55 | req, err := http.NewRequestWithContext(ctx, "GET", url, nil) | |
56 | if err != nil { | |
57 | return nil, err | |
58 | } | |
59 | ||
60 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36") | |
61 | req.Header.Set("Accept", "*/*") | |
62 | req.Header.Set("Accept-Language", "en") | |
63 | ||
64 | if cookies != "" { | |
65 | req.Header.Set("Cookie", cookies) | |
66 | } | |
67 | ||
68 | if headers != nil { | |
69 | for key, value := range headers { | |
70 | req.Header.Set(key, value) | |
71 | } | |
72 | } | |
73 | ||
74 | return httpRequestWrapper(s.Client, req) | |
75 | } | |
76 | ||
77 | func (s *Session) DiscardHttpResponse(response *http.Response) { | |
78 | if response != nil { | |
79 | io.Copy(ioutil.Discard, response.Body) | |
80 | response.Body.Close() | |
81 | } | |
82 | } | |
83 | ||
84 | func httpRequestWrapper(client *http.Client, request *http.Request) (*http.Response, error) { | |
85 | resp, err := client.Do(request) | |
86 | if err != nil { | |
87 | return nil, err | |
88 | } | |
89 | ||
90 | if resp.StatusCode != http.StatusOK { | |
91 | requestUrl, _ := url.QueryUnescape(request.URL.String()) | |
92 | return resp, fmt.Errorf("Unexpected status code %d received from %s", resp.StatusCode, requestUrl) | |
93 | } | |
94 | return resp, nil | |
95 | } |
0 | package alienvault | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "encoding/json" | |
5 | "fmt" | |
6 | ||
7 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | type alienvaultResponse struct { | |
11 | PassiveDNS []struct { | |
12 | Hostname string `json:"hostname"` | |
13 | } `json:"passive_dns"` | |
14 | } | |
15 | ||
16 | // Source is the passive scraping agent | |
17 | type Source struct{} | |
18 | ||
19 | // Run function returns all subdomains found with the service | |
20 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
21 | results := make(chan subscraping.Result) | |
22 | ||
23 | go func() { | |
24 | resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain)) | |
25 | if err != nil { | |
26 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
27 | session.DiscardHttpResponse(resp) | |
28 | close(results) | |
29 | return | |
30 | } | |
31 | ||
32 | otxResp := &alienvaultResponse{} | |
33 | // Get the response body and decode | |
34 | err = json.NewDecoder(resp.Body).Decode(&otxResp) | |
35 | if err != nil { | |
36 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
37 | resp.Body.Close() | |
38 | close(results) | |
39 | return | |
40 | } | |
41 | resp.Body.Close() | |
42 | for _, record := range otxResp.PassiveDNS { | |
43 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} | |
44 | } | |
45 | close(results) | |
46 | }() | |
47 | ||
48 | return results | |
49 | } | |
50 | ||
51 | // Name returns the name of the source | |
52 | func (s *Source) Name() string { | |
53 | return "alienvault" | |
54 | } |
0 | // Package archiveis is a Archiveis Scraping Engine in Golang | |
1 | package archiveis | |
2 | ||
3 | import ( | |
4 | "context" | |
5 | "io/ioutil" | |
6 | "regexp" | |
7 | ||
8 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
9 | ) | |
10 | ||
11 | // ArchiveIs is a struct for archiveurlsagent | |
12 | type ArchiveIs struct { | |
13 | Results chan subscraping.Result | |
14 | Session *subscraping.Session | |
15 | } | |
16 | ||
17 | var reNext = regexp.MustCompile("<a id=\"next\" style=\".*\" href=\"(.*)\">→</a>") | |
18 | ||
19 | func (a *ArchiveIs) enumerate(ctx context.Context, baseURL string) { | |
20 | select { | |
21 | case <-ctx.Done(): | |
22 | return | |
23 | default: | |
24 | } | |
25 | ||
26 | resp, err := a.Session.NormalGetWithContext(ctx, baseURL) | |
27 | if err != nil { | |
28 | a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} | |
29 | a.Session.DiscardHttpResponse(resp) | |
30 | return | |
31 | } | |
32 | ||
33 | // Get the response body | |
34 | body, err := ioutil.ReadAll(resp.Body) | |
35 | resp.Body.Close() | |
36 | if err != nil { | |
37 | a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} | |
38 | return | |
39 | } | |
40 | ||
41 | src := string(body) | |
42 | ||
43 | for _, subdomain := range a.Session.Extractor.FindAllString(src, -1) { | |
44 | a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Subdomain, Value: subdomain} | |
45 | } | |
46 | ||
47 | match1 := reNext.FindStringSubmatch(src) | |
48 | if len(match1) > 0 { | |
49 | a.enumerate(ctx, match1[1]) | |
50 | } | |
51 | } | |
52 | ||
53 | // Source is the passive scraping agent | |
54 | type Source struct{} | |
55 | ||
56 | // Run function returns all subdomains found with the service | |
57 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
58 | results := make(chan subscraping.Result) | |
59 | ||
60 | aInstance := ArchiveIs{ | |
61 | Session: session, | |
62 | Results: results, | |
63 | } | |
64 | ||
65 | go func() { | |
66 | aInstance.enumerate(ctx, "http://archive.is/*."+domain) | |
67 | close(aInstance.Results) | |
68 | }() | |
69 | ||
70 | return aInstance.Results | |
71 | } | |
72 | ||
73 | // Name returns the name of the source | |
74 | func (s *Source) Name() string { | |
75 | return "archiveis" | |
76 | } |
0 | package binaryedge | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | ||
6 | jsoniter "github.com/json-iterator/go" | |
7 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | type binaryedgeResponse struct { | |
11 | Subdomains []string `json:"events"` | |
12 | Total int `json:"total"` | |
13 | } | |
14 | ||
15 | // Source is the passive scraping agent | |
16 | type Source struct{} | |
17 | ||
18 | // Run function returns all subdomains found with the service | |
19 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
20 | results := make(chan subscraping.Result) | |
21 | ||
22 | go func() { | |
23 | if session.Keys.Binaryedge == "" { | |
24 | close(results) | |
25 | return | |
26 | } | |
27 | ||
28 | resp, err := session.Get(ctx, fmt.Sprintf("https://api.binaryedge.io/v2/query/domains/subdomain/%s", domain), "", map[string]string{"X-Key": session.Keys.Binaryedge}) | |
29 | if err != nil { | |
30 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
31 | session.DiscardHttpResponse(resp) | |
32 | close(results) | |
33 | return | |
34 | } | |
35 | ||
36 | response := new(binaryedgeResponse) | |
37 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
38 | if err != nil { | |
39 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
40 | resp.Body.Close() | |
41 | close(results) | |
42 | return | |
43 | } | |
44 | resp.Body.Close() | |
45 | ||
46 | for _, subdomain := range response.Subdomains { | |
47 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
48 | } | |
49 | ||
50 | remaining := response.Total - 100 | |
51 | currentPage := 2 | |
52 | ||
53 | for { | |
54 | further := s.getSubdomains(ctx, domain, &remaining, ¤tPage, session, results) | |
55 | if !further { | |
56 | break | |
57 | } | |
58 | } | |
59 | close(results) | |
60 | }() | |
61 | ||
62 | return results | |
63 | } | |
64 | ||
65 | // Name returns the name of the source | |
66 | func (s *Source) Name() string { | |
67 | return "binaryedge" | |
68 | } | |
69 | ||
70 | func (s *Source) getSubdomains(ctx context.Context, domain string, remaining, currentPage *int, session *subscraping.Session, results chan subscraping.Result) bool { | |
71 | for { | |
72 | select { | |
73 | case <-ctx.Done(): | |
74 | return false | |
75 | default: | |
76 | resp, err := session.Get(ctx, fmt.Sprintf("https://api.binaryedge.io/v2/query/domains/subdomain/%s?page=%d", domain, *currentPage), "", map[string]string{"X-Key": session.Keys.Binaryedge}) | |
77 | if err != nil { | |
78 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
79 | return false | |
80 | } | |
81 | ||
82 | response := binaryedgeResponse{} | |
83 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
84 | if err != nil { | |
85 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
86 | resp.Body.Close() | |
87 | return false | |
88 | } | |
89 | resp.Body.Close() | |
90 | ||
91 | for _, subdomain := range response.Subdomains { | |
92 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
93 | } | |
94 | ||
95 | *remaining = *remaining - 100 | |
96 | if *remaining <= 0 { | |
97 | return false | |
98 | } | |
99 | *currentPage++ | |
100 | return true | |
101 | } | |
102 | } | |
103 | } |
0 | // Package bufferover is a bufferover Scraping Engine in Golang | |
1 | package bufferover | |
2 | ||
3 | import ( | |
4 | "context" | |
5 | "fmt" | |
6 | "io/ioutil" | |
7 | ||
8 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
9 | ) | |
10 | ||
11 | // Source is the passive scraping agent | |
12 | type Source struct{} | |
13 | ||
14 | // Run function returns all subdomains found with the service | |
15 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
16 | results := make(chan subscraping.Result) | |
17 | ||
18 | go func() { | |
19 | // Run enumeration on subdomain dataset for historical SONAR datasets | |
20 | s.getData(ctx, fmt.Sprintf("https://dns.bufferover.run/dns?q=.%s", domain), session, results) | |
21 | s.getData(ctx, fmt.Sprintf("https://tls.bufferover.run/dns?q=.%s", domain), session, results) | |
22 | ||
23 | close(results) | |
24 | }() | |
25 | ||
26 | return results | |
27 | } | |
28 | ||
29 | func (s *Source) getData(ctx context.Context, URL string, session *subscraping.Session, results chan subscraping.Result) { | |
30 | resp, err := session.NormalGetWithContext(ctx, URL) | |
31 | if err != nil { | |
32 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
33 | session.DiscardHttpResponse(resp) | |
34 | return | |
35 | } | |
36 | ||
37 | body, err := ioutil.ReadAll(resp.Body) | |
38 | if err != nil { | |
39 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
40 | resp.Body.Close() | |
41 | return | |
42 | } | |
43 | resp.Body.Close() | |
44 | ||
45 | src := string(body) | |
46 | ||
47 | for _, subdomain := range session.Extractor.FindAllString(src, -1) { | |
48 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
49 | } | |
50 | return | |
51 | } | |
52 | ||
53 | // Name returns the name of the source | |
54 | func (s *Source) Name() string { | |
55 | return "bufferover" | |
56 | } |
0 | package censys | |
1 | ||
2 | import ( | |
3 | "bytes" | |
4 | "context" | |
5 | "net/http" | |
6 | "strconv" | |
7 | ||
8 | jsoniter "github.com/json-iterator/go" | |
9 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
10 | ) | |
11 | ||
12 | const maxCensysPages = 10 | |
13 | ||
14 | type resultsq struct { | |
15 | Data []string `json:"parsed.extensions.subject_alt_name.dns_names"` | |
16 | Data1 []string `json:"parsed.names"` | |
17 | } | |
18 | ||
19 | type response struct { | |
20 | Results []resultsq `json:"results"` | |
21 | Metadata struct { | |
22 | Pages int `json:"pages"` | |
23 | } `json:"metadata"` | |
24 | } | |
25 | ||
26 | // Source is the passive scraping agent | |
27 | type Source struct{} | |
28 | ||
29 | // Run function returns all subdomains found with the service | |
30 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
31 | results := make(chan subscraping.Result) | |
32 | ||
33 | go func() { | |
34 | if session.Keys.CensysToken == "" || session.Keys.CensysSecret == "" { | |
35 | close(results) | |
36 | return | |
37 | } | |
38 | var response response | |
39 | ||
40 | currentPage := 1 | |
41 | for { | |
42 | var request = []byte(`{"query":"` + domain + `", "page":` + strconv.Itoa(currentPage) + `, "fields":["parsed.names","parsed.extensions.subject_alt_name.dns_names"], "flatten":true}`) | |
43 | ||
44 | req, err := http.NewRequestWithContext(ctx, "POST", "https://www.censys.io/api/v1/search/certificates", bytes.NewReader(request)) | |
45 | if err != nil { | |
46 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
47 | close(results) | |
48 | return | |
49 | } | |
50 | req.SetBasicAuth(session.Keys.CensysToken, session.Keys.CensysSecret) | |
51 | req.Header.Set("Content-Type", "application/json") | |
52 | req.Header.Set("Accept", "application/json") | |
53 | ||
54 | resp, err := session.Client.Do(req) | |
55 | if err != nil { | |
56 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
57 | close(results) | |
58 | return | |
59 | } | |
60 | ||
61 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
62 | if err != nil { | |
63 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
64 | resp.Body.Close() | |
65 | close(results) | |
66 | return | |
67 | } | |
68 | resp.Body.Close() | |
69 | ||
70 | // Exit the censys enumeration if max pages is reached | |
71 | if currentPage >= response.Metadata.Pages || currentPage >= maxCensysPages { | |
72 | break | |
73 | } | |
74 | ||
75 | for _, res := range response.Results { | |
76 | for _, part := range res.Data { | |
77 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: part} | |
78 | } | |
79 | for _, part := range res.Data1 { | |
80 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: part} | |
81 | } | |
82 | } | |
83 | ||
84 | currentPage++ | |
85 | } | |
86 | close(results) | |
87 | }() | |
88 | ||
89 | return results | |
90 | } | |
91 | ||
92 | // Name returns the name of the source | |
93 | func (s *Source) Name() string { | |
94 | return "censys" | |
95 | } |
0 | package certspotter | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | ||
6 | jsoniter "github.com/json-iterator/go" | |
7 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | type certspotterObject struct { | |
11 | ID string `json:"id"` | |
12 | DNSNames []string `json:"dns_names"` | |
13 | } | |
14 | ||
15 | // Source is the passive scraping agent | |
16 | type Source struct{} | |
17 | ||
18 | // Run function returns all subdomains found with the service | |
19 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
20 | results := make(chan subscraping.Result) | |
21 | ||
22 | go func() { | |
23 | if session.Keys.Certspotter == "" { | |
24 | close(results) | |
25 | return | |
26 | } | |
27 | ||
28 | resp, err := session.Get(ctx, fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain), "", map[string]string{"Authorization": "Bearer " + session.Keys.Certspotter}) | |
29 | if err != nil { | |
30 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
31 | session.DiscardHttpResponse(resp) | |
32 | close(results) | |
33 | return | |
34 | } | |
35 | ||
36 | response := []certspotterObject{} | |
37 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
38 | if err != nil { | |
39 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
40 | resp.Body.Close() | |
41 | close(results) | |
42 | return | |
43 | } | |
44 | resp.Body.Close() | |
45 | ||
46 | for _, cert := range response { | |
47 | for _, subdomain := range cert.DNSNames { | |
48 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
49 | } | |
50 | } | |
51 | ||
52 | // if the number of responses is zero, close the channel and return. | |
53 | if len(response) == 0 { | |
54 | close(results) | |
55 | return | |
56 | } | |
57 | ||
58 | id := response[len(response)-1].ID | |
59 | for { | |
60 | reqURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, id) | |
61 | ||
62 | resp, err := session.Get(ctx, reqURL, "", map[string]string{"Authorization": "Bearer " + session.Keys.Certspotter}) | |
63 | if err != nil { | |
64 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
65 | close(results) | |
66 | return | |
67 | } | |
68 | ||
69 | response := []certspotterObject{} | |
70 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
71 | if err != nil { | |
72 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
73 | resp.Body.Close() | |
74 | close(results) | |
75 | return | |
76 | } | |
77 | resp.Body.Close() | |
78 | ||
79 | if len(response) == 0 { | |
80 | break | |
81 | } | |
82 | ||
83 | for _, cert := range response { | |
84 | for _, subdomain := range cert.DNSNames { | |
85 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
86 | } | |
87 | } | |
88 | ||
89 | id = response[len(response)-1].ID | |
90 | } | |
91 | close(results) | |
92 | }() | |
93 | ||
94 | return results | |
95 | } | |
96 | ||
97 | // Name returns the name of the source | |
98 | func (s *Source) Name() string { | |
99 | return "certspotter" | |
100 | } |
0 | package certspotterold | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "io/ioutil" | |
6 | ||
7 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | // Source is the passive scraping agent | |
11 | type Source struct{} | |
12 | ||
13 | // Run function returns all subdomains found with the service | |
14 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
15 | results := make(chan subscraping.Result) | |
16 | ||
17 | go func() { | |
18 | resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://certspotter.com/api/v0/certs?domain=%s", domain)) | |
19 | if err != nil { | |
20 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
21 | session.DiscardHttpResponse(resp) | |
22 | close(results) | |
23 | return | |
24 | } | |
25 | ||
26 | body, err := ioutil.ReadAll(resp.Body) | |
27 | if err != nil { | |
28 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
29 | resp.Body.Close() | |
30 | close(results) | |
31 | return | |
32 | } | |
33 | resp.Body.Close() | |
34 | ||
35 | src := string(body) | |
36 | ||
37 | for _, subdomain := range session.Extractor.FindAllString(src, -1) { | |
38 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
39 | } | |
40 | close(results) | |
41 | }() | |
42 | ||
43 | return results | |
44 | } | |
45 | ||
46 | // Name returns the name of the source | |
47 | func (s *Source) Name() string { | |
48 | return "certspotterold" | |
49 | } |
0 | package commoncrawl | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "io/ioutil" | |
6 | "net/url" | |
7 | "strings" | |
8 | ||
9 | jsoniter "github.com/json-iterator/go" | |
10 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
11 | ) | |
12 | ||
13 | const indexURL = "https://index.commoncrawl.org/collinfo.json" | |
14 | ||
15 | type indexResponse struct { | |
16 | ID string `json:"id"` | |
17 | APIURL string `json:"cdx-api"` | |
18 | } | |
19 | ||
20 | // Source is the passive scraping agent | |
21 | type Source struct{} | |
22 | ||
23 | var years = [...]string{"2020", "2019", "2018", "2017"} | |
24 | ||
25 | // Run function returns all subdomains found with the service | |
26 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
27 | results := make(chan subscraping.Result) | |
28 | ||
29 | go func() { | |
30 | resp, err := session.NormalGetWithContext(ctx, indexURL) | |
31 | if err != nil { | |
32 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
33 | session.DiscardHttpResponse(resp) | |
34 | close(results) | |
35 | return | |
36 | } | |
37 | ||
38 | indexes := []indexResponse{} | |
39 | err = jsoniter.NewDecoder(resp.Body).Decode(&indexes) | |
40 | if err != nil { | |
41 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
42 | resp.Body.Close() | |
43 | close(results) | |
44 | return | |
45 | } | |
46 | resp.Body.Close() | |
47 | ||
48 | searchIndexes := make(map[string]string) | |
49 | for _, year := range years { | |
50 | for _, index := range indexes { | |
51 | if strings.Contains(index.ID, year) { | |
52 | if _, ok := searchIndexes[year]; !ok { | |
53 | searchIndexes[year] = index.APIURL | |
54 | break | |
55 | } | |
56 | } | |
57 | } | |
58 | } | |
59 | ||
60 | for _, apiURL := range searchIndexes { | |
61 | further := s.getSubdomains(ctx, apiURL, domain, session, results) | |
62 | if !further { | |
63 | break | |
64 | } | |
65 | } | |
66 | close(results) | |
67 | }() | |
68 | ||
69 | return results | |
70 | } | |
71 | ||
72 | // Name returns the name of the source | |
73 | func (s *Source) Name() string { | |
74 | return "commoncrawl" | |
75 | } | |
76 | ||
77 | func (s *Source) getSubdomains(ctx context.Context, searchURL string, domain string, session *subscraping.Session, results chan subscraping.Result) bool { | |
78 | for { | |
79 | select { | |
80 | case <-ctx.Done(): | |
81 | return false | |
82 | default: | |
83 | resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("%s?url=*.%s&output=json", searchURL, domain)) | |
84 | if err != nil { | |
85 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
86 | return false | |
87 | } | |
88 | ||
89 | body, err := ioutil.ReadAll(resp.Body) | |
90 | if err != nil { | |
91 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
92 | resp.Body.Close() | |
93 | return false | |
94 | } | |
95 | resp.Body.Close() | |
96 | ||
97 | src, _ := url.QueryUnescape(string(body)) | |
98 | ||
99 | for _, subdomain := range session.Extractor.FindAllString(src, -1) { | |
100 | subdomain = strings.TrimPrefix(subdomain, "25") | |
101 | ||
102 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
103 | } | |
104 | return true | |
105 | } | |
106 | } | |
107 | } |
0 | package crtsh | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "database/sql" | |
5 | "fmt" | |
6 | "io/ioutil" | |
7 | "strings" | |
8 | ||
9 | // postgres driver | |
10 | _ "github.com/lib/pq" | |
11 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
12 | ) | |
13 | ||
14 | // Source is the passive scraping agent | |
15 | type Source struct{} | |
16 | ||
17 | // Run function returns all subdomains found with the service | |
18 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
19 | results := make(chan subscraping.Result) | |
20 | ||
21 | go func() { | |
22 | found := s.getSubdomainsFromSQL(ctx, domain, session, results) | |
23 | if found { | |
24 | close(results) | |
25 | return | |
26 | } | |
27 | _ = s.getSubdomainsFromHTTP(ctx, domain, session, results) | |
28 | close(results) | |
29 | }() | |
30 | ||
31 | return results | |
32 | } | |
33 | ||
34 | func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) bool { | |
35 | db, err := sql.Open("postgres", "host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes") | |
36 | if err != nil { | |
37 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
38 | return false | |
39 | } | |
40 | ||
41 | pattern := "%." + domain | |
42 | rows, err := db.Query(`SELECT DISTINCT ci.NAME_VALUE as domain | |
43 | FROM certificate_identity ci | |
44 | WHERE reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower($1)) | |
45 | ORDER BY ci.NAME_VALUE`, pattern) | |
46 | if err != nil { | |
47 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
48 | return false | |
49 | } | |
50 | ||
51 | var data string | |
52 | // Parse all the rows getting subdomains | |
53 | for rows.Next() { | |
54 | err := rows.Scan(&data) | |
55 | if err != nil { | |
56 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
57 | return false | |
58 | } | |
59 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data} | |
60 | } | |
61 | return true | |
62 | } | |
63 | ||
64 | func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) bool { | |
65 | resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain)) | |
66 | if err != nil { | |
67 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
68 | session.DiscardHttpResponse(resp) | |
69 | return false | |
70 | } | |
71 | ||
72 | body, err := ioutil.ReadAll(resp.Body) | |
73 | if err != nil { | |
74 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
75 | resp.Body.Close() | |
76 | return false | |
77 | } | |
78 | resp.Body.Close() | |
79 | ||
80 | // Also replace all newlines | |
81 | src := strings.Replace(string(body), "\\n", " ", -1) | |
82 | ||
83 | for _, subdomain := range session.Extractor.FindAllString(src, -1) { | |
84 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
85 | } | |
86 | return true | |
87 | } | |
88 | ||
89 | // Name returns the name of the source | |
90 | func (s *Source) Name() string { | |
91 | return "crtsh" | |
92 | } |
0 | package dnsdb | |
1 | ||
2 | import ( | |
3 | "bufio" | |
4 | "context" | |
5 | "encoding/json" | |
6 | "fmt" | |
7 | "strings" | |
8 | ||
9 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
10 | ) | |
11 | ||
12 | type dnsdbResponse struct { | |
13 | Name string `json:"rrname"` | |
14 | } | |
15 | ||
16 | // Source is the passive scraping agent | |
17 | type Source struct{} | |
18 | ||
19 | // Run function returns all subdomains found with the service | |
20 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
21 | results := make(chan subscraping.Result) | |
22 | ||
23 | if session.Keys.DNSDB == "" { | |
24 | close(results) | |
25 | } else { | |
26 | headers := map[string]string{ | |
27 | "X-API-KEY": session.Keys.DNSDB, | |
28 | "Accept": "application/json", | |
29 | "Content-Type": "application/json", | |
30 | } | |
31 | ||
32 | go func() { | |
33 | resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdb.info/lookup/rrset/name/*.%s?limit=1000000000000", domain), "", headers) | |
34 | if err != nil { | |
35 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
36 | session.DiscardHttpResponse(resp) | |
37 | close(results) | |
38 | return | |
39 | } | |
40 | ||
41 | defer resp.Body.Close() | |
42 | // Get the response body | |
43 | scanner := bufio.NewScanner(resp.Body) | |
44 | for scanner.Scan() { | |
45 | line := scanner.Text() | |
46 | if line == "" { | |
47 | continue | |
48 | } | |
49 | out := &dnsdbResponse{} | |
50 | err := json.Unmarshal([]byte(line), out) | |
51 | if err != nil { | |
52 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
53 | resp.Body.Close() | |
54 | close(results) | |
55 | return | |
56 | } | |
57 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(out.Name, ".")} | |
58 | out = nil | |
59 | } | |
60 | close(results) | |
61 | }() | |
62 | } | |
63 | return results | |
64 | } | |
65 | ||
66 | // Name returns the name of the source | |
67 | func (s *Source) Name() string { | |
68 | return "DNSDB" | |
69 | } |
0 | package dnsdumpster | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "io/ioutil" | |
5 | "net" | |
6 | "net/http" | |
7 | "net/url" | |
8 | "regexp" | |
9 | "strings" | |
10 | "time" | |
11 | ||
12 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
13 | ) | |
14 | ||
15 | var re = regexp.MustCompile("<input type=\"hidden\" name=\"csrfmiddlewaretoken\" value=\"(.*)\">") | |
16 | ||
17 | // getCSRFToken gets the CSRF Token from the page | |
18 | func getCSRFToken(page string) string { | |
19 | if subs := re.FindStringSubmatch(page); len(subs) == 2 { | |
20 | return strings.TrimSpace(subs[1]) | |
21 | } | |
22 | return "" | |
23 | } | |
24 | ||
25 | // postForm posts a form for a domain and returns the response | |
26 | func postForm(token, domain string) (string, error) { | |
27 | dial := net.Dialer{} | |
28 | client := &http.Client{ | |
29 | Transport: &http.Transport{ | |
30 | DialContext: dial.DialContext, | |
31 | TLSHandshakeTimeout: 10 * time.Second, | |
32 | }, | |
33 | } | |
34 | params := url.Values{ | |
35 | "csrfmiddlewaretoken": {token}, | |
36 | "targetip": {domain}, | |
37 | } | |
38 | ||
39 | req, err := http.NewRequest("POST", "https://dnsdumpster.com/", strings.NewReader(params.Encode())) | |
40 | if err != nil { | |
41 | return "", err | |
42 | } | |
43 | ||
44 | // The CSRF token needs to be sent as a cookie | |
45 | cookie := &http.Cookie{ | |
46 | Name: "csrftoken", | |
47 | Domain: "dnsdumpster.com", | |
48 | Value: token, | |
49 | } | |
50 | req.AddCookie(cookie) | |
51 | ||
52 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36") | |
53 | req.Header.Set("Content-Type", "application/x-www-form-urlencoded") | |
54 | req.Header.Set("Referer", "https://dnsdumpster.com") | |
55 | req.Header.Set("X-CSRF-Token", token) | |
56 | ||
57 | resp, err := client.Do(req) | |
58 | if err != nil { | |
59 | return "", err | |
60 | } | |
61 | // Now, grab the entire page | |
62 | in, err := ioutil.ReadAll(resp.Body) | |
63 | resp.Body.Close() | |
64 | return string(in), err | |
65 | } | |
66 | ||
67 | // Source is the passive scraping agent | |
68 | type Source struct{} | |
69 | ||
70 | // Run function returns all subdomains found with the service | |
71 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
72 | results := make(chan subscraping.Result) | |
73 | ||
74 | go func() { | |
75 | resp, err := session.NormalGetWithContext(ctx, "https://dnsdumpster.com/") | |
76 | if err != nil { | |
77 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
78 | session.DiscardHttpResponse(resp) | |
79 | close(results) | |
80 | return | |
81 | } | |
82 | ||
83 | body, err := ioutil.ReadAll(resp.Body) | |
84 | if err != nil { | |
85 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
86 | resp.Body.Close() | |
87 | close(results) | |
88 | return | |
89 | } | |
90 | resp.Body.Close() | |
91 | csrfToken := getCSRFToken(string(body)) | |
92 | ||
93 | data, err := postForm(csrfToken, domain) | |
94 | if err != nil { | |
95 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
96 | close(results) | |
97 | return | |
98 | } | |
99 | ||
100 | for _, subdomain := range session.Extractor.FindAllString(data, -1) { | |
101 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
102 | } | |
103 | close(results) | |
104 | }() | |
105 | ||
106 | return results | |
107 | } | |
108 | ||
109 | // Name returns the name of the source | |
110 | func (s *Source) Name() string { | |
111 | return "dnsdumpster" | |
112 | } |
0 | package entrust | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "io/ioutil" | |
6 | "strings" | |
7 | ||
8 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
9 | ) | |
10 | ||
11 | // Source is the passive scraping agent | |
12 | type Source struct{} | |
13 | ||
14 | // Run function returns all subdomains found with the service | |
15 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
16 | results := make(chan subscraping.Result) | |
17 | ||
18 | go func() { | |
19 | resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://ctsearch.entrust.com/api/v1/certificates?fields=issuerCN,subjectO,issuerDN,issuerO,subjectDN,signAlg,san,publicKeyType,publicKeySize,validFrom,validTo,sn,ev,logEntries.logName,subjectCNReversed,cert&domain=%s&includeExpired=true&exactMatch=false&limit=5000", domain)) | |
20 | if err != nil { | |
21 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
22 | session.DiscardHttpResponse(resp) | |
23 | close(results) | |
24 | return | |
25 | } | |
26 | ||
27 | body, err := ioutil.ReadAll(resp.Body) | |
28 | if err != nil { | |
29 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
30 | resp.Body.Close() | |
31 | close(results) | |
32 | return | |
33 | } | |
34 | resp.Body.Close() | |
35 | ||
36 | src := string(body) | |
37 | ||
38 | for _, subdomain := range session.Extractor.FindAllString(src, -1) { | |
39 | subdomain = strings.TrimPrefix(subdomain, "u003d") | |
40 | ||
41 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
42 | } | |
43 | close(results) | |
44 | }() | |
45 | ||
46 | return results | |
47 | } | |
48 | ||
49 | // Name returns the name of the source | |
50 | func (s *Source) Name() string { | |
51 | return "entrust" | |
52 | } |
0 | // GitHub search package, based on gwen001's https://github.com/gwen001/github-search github-subdomains | |
1 | package github | |
2 | ||
3 | import ( | |
4 | "context" | |
5 | "fmt" | |
6 | "io/ioutil" | |
7 | "net/http" | |
8 | "net/url" | |
9 | "regexp" | |
10 | "strconv" | |
11 | "strings" | |
12 | "time" | |
13 | ||
14 | jsoniter "github.com/json-iterator/go" | |
15 | ||
16 | "github.com/projectdiscovery/gologger" | |
17 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
18 | "github.com/tomnomnom/linkheader" | |
19 | ) | |
20 | ||
21 | type textMatch struct { | |
22 | Fragment string `json:"fragment"` | |
23 | } | |
24 | ||
25 | type item struct { | |
26 | Name string `json:"name"` | |
27 | HtmlUrl string `json:"html_url"` | |
28 | TextMatches []textMatch `json:"text_matches"` | |
29 | } | |
30 | ||
31 | type response struct { | |
32 | TotalCount int `json:"total_count"` | |
33 | Items []item `json:"items"` | |
34 | } | |
35 | ||
36 | // Source is the passive scraping agent | |
37 | type Source struct{} | |
38 | ||
39 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
40 | results := make(chan subscraping.Result) | |
41 | ||
42 | go func() { | |
43 | if len(session.Keys.GitHub) == 0 { | |
44 | close(results) | |
45 | return | |
46 | } | |
47 | ||
48 | tokens := NewTokenManager(session.Keys.GitHub) | |
49 | ||
50 | // search on GitHub with exact match | |
51 | searchURL := fmt.Sprintf("https://api.github.com/search/code?per_page=100&q=\"%s\"", domain) | |
52 | s.enumerate(ctx, searchURL, s.DomainRegexp(domain), tokens, session, results) | |
53 | close(results) | |
54 | }() | |
55 | ||
56 | return results | |
57 | } | |
58 | ||
59 | func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *regexp.Regexp, tokens *Tokens, session *subscraping.Session, results chan subscraping.Result) { | |
60 | select { | |
61 | case <-ctx.Done(): | |
62 | return | |
63 | default: | |
64 | } | |
65 | ||
66 | token := tokens.Get() | |
67 | ||
68 | if token.RetryAfter > 0 { | |
69 | if len(tokens.pool) == 1 { | |
70 | gologger.Verbosef("GitHub Search request rate limit exceeded, waiting for %d seconds before retry... \n", s.Name(), token.RetryAfter) | |
71 | time.Sleep(time.Duration(token.RetryAfter) * time.Second) | |
72 | } else { | |
73 | token = tokens.Get() | |
74 | } | |
75 | } | |
76 | ||
77 | headers := map[string]string{ | |
78 | "Accept": "application/vnd.github.v3.text-match+json", | |
79 | "Authorization": "token " + token.Hash, | |
80 | } | |
81 | ||
82 | // Initial request to GitHub search | |
83 | resp, err := session.Get(ctx, searchURL, "", headers) | |
84 | isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden | |
85 | ||
86 | if err != nil && !isForbidden { | |
87 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
88 | session.DiscardHttpResponse(resp) | |
89 | return | |
90 | } else { | |
91 | // Retry enumerarion after Retry-After seconds on rate limit abuse detected | |
92 | ratelimitRemaining, _ := strconv.ParseInt(resp.Header.Get("X-Ratelimit-Remaining"), 10, 64) | |
93 | if isForbidden && ratelimitRemaining == 0 { | |
94 | retryAfterSeconds, _ := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64) | |
95 | tokens.setCurrentTokenExceeded(retryAfterSeconds) | |
96 | ||
97 | s.enumerate(ctx, searchURL, domainRegexp, tokens, session, results) | |
98 | } else { | |
99 | // Links header, first, next, last... | |
100 | linksHeader := linkheader.Parse(resp.Header.Get("Link")) | |
101 | ||
102 | data := response{} | |
103 | ||
104 | // Marshall json reponse | |
105 | err = jsoniter.NewDecoder(resp.Body).Decode(&data) | |
106 | resp.Body.Close() | |
107 | if err != nil { | |
108 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
109 | return | |
110 | } | |
111 | ||
112 | // Response items iteration | |
113 | for _, item := range data.Items { | |
114 | resp, err := session.NormalGetWithContext(ctx, rawUrl(item.HtmlUrl)) | |
115 | if err != nil { | |
116 | if resp != nil && resp.StatusCode != http.StatusNotFound { | |
117 | session.DiscardHttpResponse(resp) | |
118 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
119 | return | |
120 | } | |
121 | } | |
122 | ||
123 | var subdomains []string | |
124 | ||
125 | if resp.StatusCode == http.StatusOK { | |
126 | // Get the item code from the raw file url | |
127 | code, err := ioutil.ReadAll(resp.Body) | |
128 | resp.Body.Close() | |
129 | if err != nil { | |
130 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
131 | return | |
132 | } | |
133 | // Search for domain matches in the code | |
134 | subdomains = append(subdomains, matches(domainRegexp, normalizeContent(string(code)))...) | |
135 | } | |
136 | ||
137 | // Text matches iteration per item | |
138 | for _, textMatch := range item.TextMatches { | |
139 | // Search for domain matches in the text fragment | |
140 | subdomains = append(subdomains, matches(domainRegexp, normalizeContent(textMatch.Fragment))...) | |
141 | } | |
142 | ||
143 | for _, subdomain := range unique(subdomains) { | |
144 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
145 | } | |
146 | } | |
147 | ||
148 | // Proccess the next link recursively | |
149 | for _, link := range linksHeader { | |
150 | if link.Rel == "next" { | |
151 | nextUrl, err := url.QueryUnescape(link.URL) | |
152 | if err != nil { | |
153 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
154 | return | |
155 | } | |
156 | s.enumerate(ctx, nextUrl, domainRegexp, tokens, session, results) | |
157 | } | |
158 | } | |
159 | } | |
160 | } | |
161 | ||
162 | } | |
163 | ||
164 | // Normalize content before matching, query unescape, remove tabs and new line chars | |
165 | func normalizeContent(content string) string { | |
166 | normalizedContent, _ := url.QueryUnescape(content) | |
167 | normalizedContent = strings.Replace(normalizedContent, "\\t", "", -1) | |
168 | normalizedContent = strings.Replace(normalizedContent, "\\n", "", -1) | |
169 | return normalizedContent | |
170 | } | |
171 | ||
172 | // Remove duplicates from string array | |
173 | func unique(arr []string) []string { | |
174 | occured := map[string]bool{} | |
175 | result := []string{} | |
176 | for e := range arr { | |
177 | if occured[arr[e]] != true { | |
178 | occured[arr[e]] = true | |
179 | result = append(result, arr[e]) | |
180 | } | |
181 | } | |
182 | return result | |
183 | } | |
184 | ||
185 | // Find matches by regular expression in any content | |
186 | func matches(regexp *regexp.Regexp, content string) []string { | |
187 | var matches []string | |
188 | match := regexp.FindAllString(content, -1) | |
189 | if len(match) > 0 { | |
190 | matches = unique(match) | |
191 | } | |
192 | return matches | |
193 | } | |
194 | ||
195 | // Raw URL to get the files code and match for subdomains | |
196 | func rawUrl(htmlUrl string) string { | |
197 | domain := strings.Replace(htmlUrl, "https://github.com/", "https://raw.githubusercontent.com/", -1) | |
198 | return strings.Replace(domain, "/blob/", "/", -1) | |
199 | } | |
200 | ||
201 | // Domain regular expression to match subdomains in github files code | |
202 | func (s *Source) DomainRegexp(domain string) *regexp.Regexp { | |
203 | rdomain := strings.Replace(domain, ".", "\\.", -1) | |
204 | return regexp.MustCompile("(\\w+[.])*" + rdomain) | |
205 | } | |
206 | ||
207 | // Name returns the name of the source | |
208 | func (s *Source) Name() string { | |
209 | return "github" | |
210 | } |
0 | package github | |
1 | ||
2 | import "time" | |
3 | ||
4 | type token struct { | |
5 | Hash string | |
6 | RetryAfter int64 | |
7 | ExceededTime time.Time | |
8 | } | |
9 | ||
10 | type Tokens struct { | |
11 | current int | |
12 | pool []token | |
13 | } | |
14 | ||
15 | func NewTokenManager(keys []string) *Tokens { | |
16 | pool := []token{} | |
17 | for _, key := range keys { | |
18 | t := token{Hash: key, ExceededTime: time.Time{}, RetryAfter: 0} | |
19 | pool = append(pool, t) | |
20 | } | |
21 | ||
22 | return &Tokens{ | |
23 | current: 0, | |
24 | pool: pool, | |
25 | } | |
26 | } | |
27 | ||
28 | func (r *Tokens) setCurrentTokenExceeded(retryAfter int64) { | |
29 | if r.current >= len(r.pool) { | |
30 | r.current = r.current % len(r.pool) | |
31 | } | |
32 | if r.pool[r.current].RetryAfter == 0 { | |
33 | r.pool[r.current].ExceededTime = time.Now() | |
34 | r.pool[r.current].RetryAfter = retryAfter | |
35 | } | |
36 | } | |
37 | ||
38 | func (r *Tokens) Get() token { | |
39 | resetExceededTokens(r) | |
40 | ||
41 | if r.current >= len(r.pool) { | |
42 | r.current = r.current % len(r.pool) | |
43 | } | |
44 | ||
45 | result := r.pool[r.current] | |
46 | r.current++ | |
47 | ||
48 | return result | |
49 | } | |
50 | ||
51 | func resetExceededTokens(r *Tokens) { | |
52 | for i, token := range r.pool { | |
53 | if token.RetryAfter > 0 { | |
54 | if int64(time.Since(token.ExceededTime)/time.Second) > token.RetryAfter { | |
55 | r.pool[i].ExceededTime = time.Time{} | |
56 | r.pool[i].RetryAfter = 0 | |
57 | } | |
58 | } | |
59 | } | |
60 | } |
0 | package hackertarget | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "io/ioutil" | |
6 | ||
7 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | // Source is the passive scraping agent | |
11 | type Source struct{} | |
12 | ||
13 | // Run function returns all subdomains found with the service | |
14 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
15 | results := make(chan subscraping.Result) | |
16 | ||
17 | go func() { | |
18 | resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("http://api.hackertarget.com/hostsearch/?q=%s", domain)) | |
19 | if err != nil { | |
20 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
21 | session.DiscardHttpResponse(resp) | |
22 | close(results) | |
23 | return | |
24 | } | |
25 | ||
26 | // Get the response body | |
27 | body, err := ioutil.ReadAll(resp.Body) | |
28 | if err != nil { | |
29 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
30 | resp.Body.Close() | |
31 | close(results) | |
32 | return | |
33 | } | |
34 | resp.Body.Close() | |
35 | src := string(body) | |
36 | ||
37 | for _, match := range session.Extractor.FindAllString(src, -1) { | |
38 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match} | |
39 | } | |
40 | close(results) | |
41 | }() | |
42 | ||
43 | return results | |
44 | } | |
45 | ||
46 | // Name returns the name of the source | |
47 | func (s *Source) Name() string { | |
48 | return "hackertarget" | |
49 | } |
0 | package intelx | |
1 | ||
2 | import ( | |
3 | "bytes" | |
4 | "context" | |
5 | "encoding/json" | |
6 | "fmt" | |
7 | "io/ioutil" | |
8 | "net/http" | |
9 | ||
10 | jsoniter "github.com/json-iterator/go" | |
11 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
12 | ) | |
13 | ||
14 | type searchResponseType struct { | |
15 | Id string `json:"id"` | |
16 | Status int `json:"status"` | |
17 | } | |
18 | ||
19 | type selectorType struct { | |
20 | Selectvalue string `json:"selectorvalue"` | |
21 | } | |
22 | ||
23 | type searchResultType struct { | |
24 | Selectors []selectorType `json:"selectors"` | |
25 | Status int `json:"status"` | |
26 | } | |
27 | ||
28 | type requestBody struct { | |
29 | Term string | |
30 | Maxresults int | |
31 | Media int | |
32 | Target int | |
33 | Terminate []int | |
34 | Timeout int | |
35 | } | |
36 | ||
37 | // Source is the passive scraping agent | |
38 | type Source struct{} | |
39 | ||
40 | // Run function returns all subdomains found with the service | |
41 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
42 | results := make(chan subscraping.Result) | |
43 | ||
44 | go func() { | |
45 | defer close(results) | |
46 | if session.Keys.IntelXKey == "" || session.Keys.IntelXHost == "" { | |
47 | return | |
48 | } | |
49 | ||
50 | searchURL := fmt.Sprintf("https://%s/phonebook/search?k=%s", session.Keys.IntelXHost, session.Keys.IntelXKey) | |
51 | reqBody := requestBody{ | |
52 | Term: domain, | |
53 | Maxresults: 100000, | |
54 | Media: 0, | |
55 | Target: 1, | |
56 | Timeout: 20, | |
57 | } | |
58 | ||
59 | body, err := json.Marshal(reqBody) | |
60 | if err != nil { | |
61 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
62 | return | |
63 | } | |
64 | ||
65 | resp, err := http.Post(searchURL, "application/json", bytes.NewBuffer(body)) | |
66 | if err != nil { | |
67 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
68 | session.DiscardHttpResponse(resp) | |
69 | return | |
70 | } | |
71 | ||
72 | var response searchResponseType | |
73 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
74 | if err != nil { | |
75 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
76 | close(results) | |
77 | return | |
78 | } | |
79 | ||
80 | resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", session.Keys.IntelXHost, session.Keys.IntelXKey, response.Id) | |
81 | status := 0 | |
82 | for status == 0 || status == 3 { | |
83 | resp, err = session.Get(ctx, resultsURL, "", nil) | |
84 | if err != nil { | |
85 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
86 | return | |
87 | } | |
88 | var response searchResultType | |
89 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
90 | if err != nil { | |
91 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
92 | return | |
93 | } | |
94 | body, err = ioutil.ReadAll(resp.Body) | |
95 | if err != nil { | |
96 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
97 | return | |
98 | } | |
99 | resp.Body.Close() | |
100 | status = response.Status | |
101 | for _, hostname := range response.Selectors { | |
102 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue} | |
103 | } | |
104 | } | |
105 | }() | |
106 | ||
107 | return results | |
108 | } | |
109 | ||
110 | // Name returns the name of the source | |
111 | func (s *Source) Name() string { | |
112 | return "intelx" | |
113 | } |
0 | package ipv4info | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "io/ioutil" | |
5 | "regexp" | |
6 | "strconv" | |
7 | ||
8 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
9 | ) | |
10 | ||
11 | // Source is the passive scraping agent | |
12 | type Source struct{} | |
13 | ||
14 | // Run function returns all subdomains found with the service | |
15 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
16 | results := make(chan subscraping.Result) | |
17 | ||
18 | go func() { | |
19 | resp, err := session.NormalGetWithContext(ctx, "http://ipv4info.com/search/"+domain) | |
20 | if err != nil { | |
21 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
22 | session.DiscardHttpResponse(resp) | |
23 | close(results) | |
24 | return | |
25 | } | |
26 | ||
27 | body, err := ioutil.ReadAll(resp.Body) | |
28 | if err != nil { | |
29 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
30 | resp.Body.Close() | |
31 | close(results) | |
32 | return | |
33 | } | |
34 | resp.Body.Close() | |
35 | src := string(body) | |
36 | ||
37 | regxTokens := regexp.MustCompile("/ip-address/(.*)/" + domain) | |
38 | matchTokens := regxTokens.FindAllString(src, -1) | |
39 | ||
40 | if len(matchTokens) <= 0 { | |
41 | close(results) | |
42 | return | |
43 | } | |
44 | token := matchTokens[0] | |
45 | ||
46 | resp, err = session.NormalGetWithContext(ctx, "http://ipv4info.com"+token) | |
47 | if err != nil { | |
48 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
49 | close(results) | |
50 | return | |
51 | } | |
52 | ||
53 | body, err = ioutil.ReadAll(resp.Body) | |
54 | if err != nil { | |
55 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
56 | resp.Body.Close() | |
57 | close(results) | |
58 | return | |
59 | } | |
60 | resp.Body.Close() | |
61 | src = string(body) | |
62 | ||
63 | regxTokens = regexp.MustCompile("/dns/(.*?)/" + domain) | |
64 | matchTokens = regxTokens.FindAllString(src, -1) | |
65 | if len(matchTokens) <= 0 { | |
66 | close(results) | |
67 | return | |
68 | } | |
69 | token = matchTokens[0] | |
70 | ||
71 | resp, err = session.NormalGetWithContext(ctx, "http://ipv4info.com"+token) | |
72 | if err != nil { | |
73 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
74 | close(results) | |
75 | return | |
76 | } | |
77 | ||
78 | body, err = ioutil.ReadAll(resp.Body) | |
79 | if err != nil { | |
80 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
81 | resp.Body.Close() | |
82 | close(results) | |
83 | return | |
84 | } | |
85 | resp.Body.Close() | |
86 | src = string(body) | |
87 | ||
88 | regxTokens = regexp.MustCompile("/subdomains/(.*?)/" + domain) | |
89 | matchTokens = regxTokens.FindAllString(src, -1) | |
90 | if len(matchTokens) <= 0 { | |
91 | close(results) | |
92 | return | |
93 | } | |
94 | token = matchTokens[0] | |
95 | ||
96 | resp, err = session.NormalGetWithContext(ctx, "http://ipv4info.com"+token) | |
97 | if err != nil { | |
98 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
99 | close(results) | |
100 | return | |
101 | } | |
102 | ||
103 | body, err = ioutil.ReadAll(resp.Body) | |
104 | if err != nil { | |
105 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
106 | resp.Body.Close() | |
107 | close(results) | |
108 | return | |
109 | } | |
110 | resp.Body.Close() | |
111 | src = string(body) | |
112 | ||
113 | for _, match := range session.Extractor.FindAllString(src, -1) { | |
114 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match} | |
115 | } | |
116 | nextPage := 1 | |
117 | ||
118 | for { | |
119 | further := s.getSubdomains(ctx, domain, &nextPage, src, session, results) | |
120 | if !further { | |
121 | break | |
122 | } | |
123 | } | |
124 | close(results) | |
125 | }() | |
126 | ||
127 | return results | |
128 | } | |
129 | ||
130 | // Name returns the name of the source | |
131 | func (s *Source) Name() string { | |
132 | return "ipv4info" | |
133 | } | |
134 | ||
135 | func (s *Source) getSubdomains(ctx context.Context, domain string, nextPage *int, src string, session *subscraping.Session, results chan subscraping.Result) bool { | |
136 | for { | |
137 | select { | |
138 | case <-ctx.Done(): | |
139 | return false | |
140 | default: | |
141 | regxTokens := regexp.MustCompile("/subdomains/.*/page" + strconv.Itoa(*nextPage) + "/" + domain + ".html") | |
142 | matchTokens := regxTokens.FindAllString(src, -1) | |
143 | if len(matchTokens) == 0 { | |
144 | return false | |
145 | } | |
146 | token := matchTokens[0] | |
147 | ||
148 | resp, err := session.NormalGetWithContext(ctx, "http://ipv4info.com"+token) | |
149 | if err != nil { | |
150 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
151 | return false | |
152 | } | |
153 | body, err := ioutil.ReadAll(resp.Body) | |
154 | if err != nil { | |
155 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
156 | resp.Body.Close() | |
157 | return false | |
158 | } | |
159 | resp.Body.Close() | |
160 | src = string(body) | |
161 | for _, match := range session.Extractor.FindAllString(src, -1) { | |
162 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match} | |
163 | } | |
164 | *nextPage++ | |
165 | return true | |
166 | } | |
167 | } | |
168 | } |
0 | package passivetotal | |
1 | ||
2 | import ( | |
3 | "bytes" | |
4 | "context" | |
5 | "net/http" | |
6 | ||
7 | jsoniter "github.com/json-iterator/go" | |
8 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
9 | ) | |
10 | ||
11 | type response struct { | |
12 | Subdomains []string `json:"subdomains"` | |
13 | } | |
14 | ||
15 | // Source is the passive scraping agent | |
16 | type Source struct{} | |
17 | ||
18 | // Run function returns all subdomains found with the service | |
19 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
20 | results := make(chan subscraping.Result) | |
21 | ||
22 | go func() { | |
23 | if session.Keys.PassiveTotalUsername == "" || session.Keys.PassiveTotalPassword == "" { | |
24 | close(results) | |
25 | return | |
26 | } | |
27 | ||
28 | // Create JSON Get body | |
29 | var request = []byte(`{"query":"` + domain + `"}`) | |
30 | ||
31 | req, err := http.NewRequestWithContext(ctx, "GET", "https://api.passivetotal.org/v2/enrichment/subdomains", bytes.NewBuffer(request)) | |
32 | if err != nil { | |
33 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
34 | close(results) | |
35 | return | |
36 | } | |
37 | ||
38 | req.SetBasicAuth(session.Keys.PassiveTotalUsername, session.Keys.PassiveTotalPassword) | |
39 | req.Header.Set("Content-Type", "application/json") | |
40 | ||
41 | resp, err := session.Client.Do(req) | |
42 | if err != nil { | |
43 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
44 | close(results) | |
45 | return | |
46 | } | |
47 | ||
48 | data := response{} | |
49 | err = jsoniter.NewDecoder(resp.Body).Decode(&data) | |
50 | if err != nil { | |
51 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
52 | resp.Body.Close() | |
53 | close(results) | |
54 | return | |
55 | } | |
56 | resp.Body.Close() | |
57 | ||
58 | for _, subdomain := range data.Subdomains { | |
59 | finalSubdomain := subdomain + "." + domain | |
60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: finalSubdomain} | |
61 | } | |
62 | close(results) | |
63 | }() | |
64 | ||
65 | return results | |
66 | } | |
67 | ||
68 | // Name returns the name of the source | |
69 | func (s *Source) Name() string { | |
70 | return "passivetotal" | |
71 | } |
0 | // Package rapiddns is a RapidDNS Scraping Engine in Golang | |
1 | package rapiddns | |
2 | ||
3 | import ( | |
4 | "context" | |
5 | "io/ioutil" | |
6 | ||
7 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | // Source is the passive scraping agent | |
11 | type Source struct{} | |
12 | ||
13 | // Run function returns all subdomains found with the service | |
14 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
15 | results := make(chan subscraping.Result) | |
16 | ||
17 | go func() { | |
18 | defer close(results) | |
19 | resp, err := session.NormalGetWithContext(ctx, "https://rapiddns.io/subdomain/"+domain+"?full=1") | |
20 | if err != nil { | |
21 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
22 | session.DiscardHttpResponse(resp) | |
23 | return | |
24 | } | |
25 | ||
26 | body, err := ioutil.ReadAll(resp.Body) | |
27 | resp.Body.Close() | |
28 | if err != nil { | |
29 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
30 | return | |
31 | } | |
32 | ||
33 | src := string(body) | |
34 | for _, subdomain := range session.Extractor.FindAllString(src, -1) { | |
35 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
36 | } | |
37 | }() | |
38 | ||
39 | return results | |
40 | } | |
41 | ||
42 | // Name returns the name of the source | |
43 | func (s *Source) Name() string { | |
44 | return "rapiddns" | |
45 | } |
0 | package securitytrails | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "strings" | |
6 | ||
7 | jsoniter "github.com/json-iterator/go" | |
8 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
9 | ) | |
10 | ||
11 | type response struct { | |
12 | Subdomains []string `json:"subdomains"` | |
13 | } | |
14 | ||
15 | // Source is the passive scraping agent | |
16 | type Source struct{} | |
17 | ||
18 | // Run function returns all subdomains found with the service | |
19 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
20 | results := make(chan subscraping.Result) | |
21 | ||
22 | go func() { | |
23 | if session.Keys.Securitytrails == "" { | |
24 | close(results) | |
25 | return | |
26 | } | |
27 | ||
28 | resp, err := session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", map[string]string{"APIKEY": session.Keys.Securitytrails}) | |
29 | if err != nil { | |
30 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
31 | session.DiscardHttpResponse(resp) | |
32 | close(results) | |
33 | return | |
34 | } | |
35 | ||
36 | response := response{} | |
37 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
38 | if err != nil { | |
39 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
40 | resp.Body.Close() | |
41 | close(results) | |
42 | return | |
43 | } | |
44 | resp.Body.Close() | |
45 | ||
46 | for _, subdomain := range response.Subdomains { | |
47 | if strings.HasSuffix(subdomain, ".") { | |
48 | subdomain = subdomain + domain | |
49 | } else { | |
50 | subdomain = subdomain + "." + domain | |
51 | } | |
52 | ||
53 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
54 | } | |
55 | close(results) | |
56 | }() | |
57 | ||
58 | return results | |
59 | } | |
60 | ||
61 | // Name returns the name of the source | |
62 | func (s *Source) Name() string { | |
63 | return "securitytrails" | |
64 | } |
0 | package shodan | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "strconv" | |
5 | ||
6 | jsoniter "github.com/json-iterator/go" | |
7 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | type shodanResult struct { | |
11 | Matches []shodanObject `json:"matches"` | |
12 | Result int `json:"result"` | |
13 | Error string `json:"error"` | |
14 | } | |
15 | ||
16 | type shodanObject struct { | |
17 | Hostnames []string `json:"hostnames"` | |
18 | } | |
19 | ||
20 | // Source is the passive scraping agent | |
21 | type Source struct{} | |
22 | ||
23 | // Run function returns all subdomains found with the service | |
24 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
25 | results := make(chan subscraping.Result) | |
26 | ||
27 | go func() { | |
28 | if session.Keys.Shodan == "" { | |
29 | close(results) | |
30 | return | |
31 | } | |
32 | ||
33 | for currentPage := 0; currentPage <= 10; currentPage++ { | |
34 | resp, err := session.NormalGetWithContext(ctx, "https://api.shodan.io/shodan/host/search?query=hostname:"+domain+"&page="+strconv.Itoa(currentPage)+"&key="+session.Keys.Shodan) | |
35 | if err != nil { | |
36 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
37 | session.DiscardHttpResponse(resp) | |
38 | close(results) | |
39 | return | |
40 | } | |
41 | ||
42 | var response shodanResult | |
43 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
44 | if err != nil { | |
45 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
46 | resp.Body.Close() | |
47 | close(results) | |
48 | return | |
49 | } | |
50 | resp.Body.Close() | |
51 | ||
52 | if response.Error != "" || len(response.Matches) == 0 { | |
53 | close(results) | |
54 | return | |
55 | } | |
56 | ||
57 | for _, block := range response.Matches { | |
58 | for _, hostname := range block.Hostnames { | |
59 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname} | |
60 | } | |
61 | } | |
62 | } | |
63 | close(results) | |
64 | }() | |
65 | ||
66 | return results | |
67 | } | |
68 | ||
69 | // Name returns the name of the source | |
70 | func (s *Source) Name() string { | |
71 | return "shodan" | |
72 | } |
0 | package sitedossier | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "io/ioutil" | |
6 | "math/rand" | |
7 | "regexp" | |
8 | "time" | |
9 | ||
10 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
11 | ) | |
12 | ||
13 | var reNext = regexp.MustCompile("<a href=\"([A-Za-z0-9\\/.]+)\"><b>") | |
14 | ||
15 | type agent struct { | |
16 | results chan subscraping.Result | |
17 | session *subscraping.Session | |
18 | } | |
19 | ||
20 | func (a *agent) enumerate(ctx context.Context, baseURL string) error { | |
21 | for { | |
22 | select { | |
23 | case <-ctx.Done(): | |
24 | return nil | |
25 | default: | |
26 | resp, err := a.session.NormalGetWithContext(ctx, baseURL) | |
27 | if err != nil { | |
28 | a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} | |
29 | a.session.DiscardHttpResponse(resp) | |
30 | close(a.results) | |
31 | return err | |
32 | } | |
33 | ||
34 | body, err := ioutil.ReadAll(resp.Body) | |
35 | if err != nil { | |
36 | a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} | |
37 | resp.Body.Close() | |
38 | close(a.results) | |
39 | return err | |
40 | } | |
41 | resp.Body.Close() | |
42 | src := string(body) | |
43 | ||
44 | for _, match := range a.session.Extractor.FindAllString(src, -1) { | |
45 | a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: match} | |
46 | } | |
47 | ||
48 | match1 := reNext.FindStringSubmatch(src) | |
49 | time.Sleep(time.Duration((3 + rand.Intn(5))) * time.Second) | |
50 | ||
51 | if len(match1) > 0 { | |
52 | a.enumerate(ctx, "http://www.sitedossier.com"+match1[1]) | |
53 | } | |
54 | return nil | |
55 | } | |
56 | } | |
57 | } | |
58 | ||
59 | // Source is the passive scraping agent | |
60 | type Source struct{} | |
61 | ||
62 | // Run function returns all subdomains found with the service | |
63 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
64 | results := make(chan subscraping.Result) | |
65 | ||
66 | a := agent{ | |
67 | session: session, | |
68 | results: results, | |
69 | } | |
70 | ||
71 | go func() { | |
72 | err := a.enumerate(ctx, fmt.Sprintf("http://www.sitedossier.com/parentdomain/%s", domain)) | |
73 | if err == nil { | |
74 | close(a.results) | |
75 | } | |
76 | }() | |
77 | return results | |
78 | } | |
79 | ||
80 | // Name returns the name of the source | |
81 | func (s *Source) Name() string { | |
82 | return "sitedossier" | |
83 | } |
0 | package spyse | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "strconv" | |
5 | "fmt" | |
6 | ||
7 | jsoniter "github.com/json-iterator/go" | |
8 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
9 | ) | |
10 | ||
11 | ||
12 | type resultObject struct { | |
13 | Name string `json:"name"` | |
14 | } | |
15 | ||
16 | type dataObject struct { | |
17 | Items []resultObject `json:"items"` | |
18 | Total_Count int `json:"total_count"` | |
19 | } | |
20 | ||
21 | type errorObject struct { | |
22 | Code string `json:"code"` | |
23 | Message string `json:"message"` | |
24 | } | |
25 | ||
26 | ||
27 | type spyseResult struct { | |
28 | Data dataObject `json:"data"` | |
29 | Error []errorObject `json:"error"` | |
30 | } | |
31 | ||
32 | ||
33 | type Source struct{} | |
34 | ||
35 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
36 | results := make(chan subscraping.Result) | |
37 | ||
38 | go func() { | |
39 | if session.Keys.Spyse == "" { | |
40 | close(results) | |
41 | return | |
42 | } | |
43 | ||
44 | maxCount := 100; | |
45 | ||
46 | for offSet := 0; offSet <= maxCount; offSet += 100 { | |
47 | resp, err := session.Get(ctx, fmt.Sprintf("https://api.spyse.com/v3/data/domain/subdomain?domain=%s&limit=100&offset=%s", domain, strconv.Itoa(offSet)), "", map[string]string{"Authorization": "Bearer " + session.Keys.Spyse}) | |
48 | if err != nil { | |
49 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
50 | session.DiscardHttpResponse(resp) | |
51 | close(results) | |
52 | return | |
53 | } | |
54 | ||
55 | ||
56 | var response spyseResult; | |
57 | ||
58 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
59 | ||
60 | if err != nil { | |
61 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
62 | resp.Body.Close() | |
63 | close(results) | |
64 | return | |
65 | } | |
66 | resp.Body.Close() | |
67 | ||
68 | if response.Data.Total_Count == 0 { | |
69 | close(results) | |
70 | return | |
71 | } | |
72 | ||
73 | maxCount = response.Data.Total_Count; | |
74 | ||
75 | for _, hostname := range response.Data.Items { | |
76 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Name} | |
77 | } | |
78 | } | |
79 | close(results) | |
80 | }() | |
81 | ||
82 | return results | |
83 | } | |
84 | ||
85 | ||
86 | // Name returns the name of the source | |
87 | func (s *Source) Name() string { | |
88 | return "spyse" | |
89 | } |
0 | package sublist3r | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "encoding/json" | |
5 | "fmt" | |
6 | ||
7 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | // Source is the passive scraping agent | |
11 | type Source struct{} | |
12 | ||
13 | // Run function returns all subdomains found with the service | |
14 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
15 | results := make(chan subscraping.Result) | |
16 | ||
17 | go func() { | |
18 | resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://api.sublist3r.com/search.php?domain=%s", domain)) | |
19 | if err != nil { | |
20 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
21 | session.DiscardHttpResponse(resp) | |
22 | close(results) | |
23 | return | |
24 | } | |
25 | defer resp.Body.Close() | |
26 | var subdomains []string | |
27 | // Get the response body and unmarshal | |
28 | err = json.NewDecoder(resp.Body).Decode(&subdomains) | |
29 | if err != nil { | |
30 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
31 | resp.Body.Close() | |
32 | close(results) | |
33 | return | |
34 | } | |
35 | ||
36 | for _, subdomain := range subdomains { | |
37 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
38 | } | |
39 | close(results) | |
40 | }() | |
41 | ||
42 | return results | |
43 | } | |
44 | ||
45 | // Name returns the name of the source | |
46 | func (s *Source) Name() string { | |
47 | return "sublist3r" | |
48 | } |
0 | package threatcrowd | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "io/ioutil" | |
6 | ||
7 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | // Source is the passive scraping agent | |
11 | type Source struct{} | |
12 | ||
13 | // Run function returns all subdomains found with the service | |
14 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
15 | results := make(chan subscraping.Result) | |
16 | ||
17 | go func() { | |
18 | resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://www.threatcrowd.org/searchApi/v2/domain/report/?domain=%s", domain)) | |
19 | if err != nil { | |
20 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
21 | session.DiscardHttpResponse(resp) | |
22 | close(results) | |
23 | return | |
24 | } | |
25 | ||
26 | // Get the response body | |
27 | body, err := ioutil.ReadAll(resp.Body) | |
28 | if err != nil { | |
29 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
30 | resp.Body.Close() | |
31 | close(results) | |
32 | return | |
33 | } | |
34 | resp.Body.Close() | |
35 | ||
36 | src := string(body) | |
37 | ||
38 | for _, match := range session.Extractor.FindAllString(src, -1) { | |
39 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match} | |
40 | } | |
41 | close(results) | |
42 | }() | |
43 | ||
44 | return results | |
45 | } | |
46 | ||
47 | // Name returns the name of the source | |
48 | func (s *Source) Name() string { | |
49 | return "threatcrowd" | |
50 | } |
0 | package threatminer | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "io/ioutil" | |
6 | ||
7 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | // Source is the passive scraping agent | |
11 | type Source struct{} | |
12 | ||
13 | // Run function returns all subdomains found with the service | |
14 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
15 | results := make(chan subscraping.Result) | |
16 | ||
17 | go func() { | |
18 | resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://api.threatminer.org/v2/domain.php?q=%s&rt=5", domain)) | |
19 | if err != nil { | |
20 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
21 | session.DiscardHttpResponse(resp) | |
22 | close(results) | |
23 | return | |
24 | } | |
25 | ||
26 | // Get the response body | |
27 | body, err := ioutil.ReadAll(resp.Body) | |
28 | if err != nil { | |
29 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
30 | resp.Body.Close() | |
31 | close(results) | |
32 | return | |
33 | } | |
34 | resp.Body.Close() | |
35 | ||
36 | src := string(body) | |
37 | ||
38 | for _, match := range session.Extractor.FindAllString(src, -1) { | |
39 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match} | |
40 | } | |
41 | close(results) | |
42 | }() | |
43 | ||
44 | return results | |
45 | } | |
46 | ||
47 | // Name returns the name of the source | |
48 | func (s *Source) Name() string { | |
49 | return "threatminer" | |
50 | } |
0 | package urlscan | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | ||
6 | jsoniter "github.com/json-iterator/go" | |
7 | "github.com/m-mizutani/urlscan-go/urlscan" | |
8 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
9 | ) | |
10 | ||
11 | // Source is the passive scraping agent | |
12 | type Source struct{} | |
13 | ||
14 | // Run function returns all subdomains found with the service | |
15 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
16 | results := make(chan subscraping.Result) | |
17 | ||
18 | go func() { | |
19 | if session.Keys.URLScan == "" { | |
20 | close(results) | |
21 | return | |
22 | } | |
23 | ||
24 | client := urlscan.NewClient(session.Keys.URLScan) | |
25 | task, err := client.Submit(urlscan.SubmitArguments{URL: fmt.Sprintf("https://%s", domain)}) | |
26 | if err != nil { | |
27 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
28 | close(results) | |
29 | return | |
30 | } | |
31 | ||
32 | err = task.Wait() | |
33 | if err != nil { | |
34 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
35 | close(results) | |
36 | return | |
37 | } | |
38 | ||
39 | data, err := jsoniter.Marshal(task.Result.Data) | |
40 | if err != nil { | |
41 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
42 | close(results) | |
43 | return | |
44 | } | |
45 | ||
46 | match := session.Extractor.FindAllString(string(data), -1) | |
47 | for _, m := range match { | |
48 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: m} | |
49 | } | |
50 | close(results) | |
51 | }() | |
52 | ||
53 | return results | |
54 | } | |
55 | ||
56 | // Name returns the name of the source | |
57 | func (s *Source) Name() string { | |
58 | return "urlscan" | |
59 | } |
0 | package virustotal | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | ||
6 | jsoniter "github.com/json-iterator/go" | |
7 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | type response struct { | |
11 | Subdomains []string `json:"subdomains"` | |
12 | } | |
13 | ||
14 | // Source is the passive scraping agent | |
15 | type Source struct{} | |
16 | ||
17 | // Run function returns all subdomains found with the service | |
18 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
19 | results := make(chan subscraping.Result) | |
20 | ||
21 | go func() { | |
22 | if session.Keys.Virustotal == "" { | |
23 | close(results) | |
24 | return | |
25 | } | |
26 | ||
27 | resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://www.virustotal.com/vtapi/v2/domain/report?apikey=%s&domain=%s", session.Keys.Virustotal, domain)) | |
28 | if err != nil { | |
29 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
30 | session.DiscardHttpResponse(resp) | |
31 | close(results) | |
32 | return | |
33 | } | |
34 | ||
35 | data := response{} | |
36 | err = jsoniter.NewDecoder(resp.Body).Decode(&data) | |
37 | if err != nil { | |
38 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
39 | resp.Body.Close() | |
40 | close(results) | |
41 | return | |
42 | } | |
43 | resp.Body.Close() | |
44 | ||
45 | for _, subdomain := range data.Subdomains { | |
46 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
47 | } | |
48 | close(results) | |
49 | }() | |
50 | ||
51 | return results | |
52 | } | |
53 | ||
54 | // Name returns the name of the source | |
55 | func (s *Source) Name() string { | |
56 | return "virustotal" | |
57 | } |
0 | package waybackarchive | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "io/ioutil" | |
6 | "strings" | |
7 | ||
8 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
9 | ) | |
10 | ||
11 | // Source is the passive scraping agent | |
12 | type Source struct{} | |
13 | ||
14 | // Run function returns all subdomains found with the service | |
15 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
16 | results := make(chan subscraping.Result) | |
17 | ||
18 | go func() { | |
19 | pagesResp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=json&fl=original&collapse=urlkey", domain)) | |
20 | if err != nil { | |
21 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
22 | session.DiscardHttpResponse(pagesResp) | |
23 | close(results) | |
24 | return | |
25 | } | |
26 | ||
27 | body, err := ioutil.ReadAll(pagesResp.Body) | |
28 | if err != nil { | |
29 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
30 | pagesResp.Body.Close() | |
31 | close(results) | |
32 | return | |
33 | } | |
34 | pagesResp.Body.Close() | |
35 | ||
36 | match := session.Extractor.FindAllString(string(body), -1) | |
37 | for _, subdomain := range match { | |
38 | subdomain = strings.TrimPrefix(subdomain, "25") | |
39 | subdomain = strings.TrimPrefix(subdomain, "2F") | |
40 | ||
41 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
42 | } | |
43 | close(results) | |
44 | }() | |
45 | ||
46 | return results | |
47 | } | |
48 | ||
49 | // Name returns the name of the source | |
50 | func (s *Source) Name() string { | |
51 | return "waybackarchive" | |
52 | } |
0 | package zoomeye | |
1 | ||
2 | import ( | |
3 | "bytes" | |
4 | "context" | |
5 | "encoding/json" | |
6 | "errors" | |
7 | "fmt" | |
8 | "io" | |
9 | "io/ioutil" | |
10 | "net/http" | |
11 | ||
12 | "github.com/projectdiscovery/subfinder/pkg/subscraping" | |
13 | ) | |
14 | ||
15 | // zoomAuth holds the ZoomEye credentials | |
16 | type zoomAuth struct { | |
17 | User string `json:"username"` | |
18 | Pass string `json:"password"` | |
19 | } | |
20 | ||
21 | type loginResp struct { | |
22 | JWT string `json:"access_token"` | |
23 | } | |
24 | ||
25 | // search results | |
26 | type zoomeyeResults struct { | |
27 | Matches []struct { | |
28 | Site string `json:"site"` | |
29 | Domains []string `json:"domains"` | |
30 | } `json:"matches"` | |
31 | } | |
32 | ||
33 | // Source is the passive scraping agent | |
34 | type Source struct{} | |
35 | ||
36 | // Run function returns all subdomains found with the service | |
37 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
38 | results := make(chan subscraping.Result) | |
39 | ||
40 | go func() { | |
41 | if session.Keys.ZoomEyeUsername == "" || session.Keys.ZoomEyePassword == "" { | |
42 | close(results) | |
43 | return | |
44 | } | |
45 | jwt, err := doLogin(session) | |
46 | if err != nil { | |
47 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
48 | close(results) | |
49 | return | |
50 | } | |
51 | // check if jwt is null | |
52 | if jwt == "" { | |
53 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errors.New("could not log into zoomeye")} | |
54 | close(results) | |
55 | return | |
56 | } | |
57 | headers := map[string]string{ | |
58 | "Authorization": fmt.Sprintf("JWT %s", jwt), | |
59 | "Accept": "application/json", | |
60 | "Content-Type": "application/json", | |
61 | } | |
62 | for currentPage := 0; currentPage <= 100; currentPage++ { | |
63 | api := fmt.Sprintf("https://api.zoomeye.org/web/search?query=hostname:%s&page=%d", domain, currentPage) | |
64 | resp, err := session.Get(ctx, api, "", headers) | |
65 | isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden | |
66 | if err != nil { | |
67 | if !isForbidden && currentPage == 0 { | |
68 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
69 | session.DiscardHttpResponse(resp) | |
70 | } | |
71 | close(results) | |
72 | return | |
73 | } | |
74 | ||
75 | defer resp.Body.Close() | |
76 | res := &zoomeyeResults{} | |
77 | err = json.NewDecoder(resp.Body).Decode(res) | |
78 | if err != nil { | |
79 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
80 | resp.Body.Close() | |
81 | close(results) | |
82 | return | |
83 | } | |
84 | resp.Body.Close() | |
85 | for _, r := range res.Matches { | |
86 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Site} | |
87 | for _, domain := range r.Domains { | |
88 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domain} | |
89 | } | |
90 | } | |
91 | currentPage++ | |
92 | } | |
93 | close(results) | |
94 | }() | |
95 | ||
96 | return results | |
97 | } | |
98 | ||
99 | // doLogin performs authentication on the ZoomEye API | |
100 | func doLogin(session *subscraping.Session) (string, error) { | |
101 | creds := &zoomAuth{ | |
102 | User: session.Keys.ZoomEyeUsername, | |
103 | Pass: session.Keys.ZoomEyePassword, | |
104 | } | |
105 | body, err := json.Marshal(&creds) | |
106 | if err != nil { | |
107 | return "", err | |
108 | } | |
109 | req, err := http.NewRequest("POST", "https://api.zoomeye.org/user/login", bytes.NewBuffer(body)) | |
110 | if err != nil { | |
111 | return "", err | |
112 | } | |
113 | req.Header.Add("Content-Type", "application/json") | |
114 | resp, err := session.Client.Do(req) | |
115 | if err != nil { | |
116 | return "", err | |
117 | } | |
118 | // if not 200, bad credentials | |
119 | if resp.StatusCode != 200 { | |
120 | io.Copy(ioutil.Discard, resp.Body) | |
121 | resp.Body.Close() | |
122 | return "", fmt.Errorf("login failed, non-200 response from zoomeye") | |
123 | } | |
124 | ||
125 | defer resp.Body.Close() | |
126 | login := &loginResp{} | |
127 | err = json.NewDecoder(resp.Body).Decode(login) | |
128 | if err != nil { | |
129 | return "", err | |
130 | } | |
131 | return login.JWT, nil | |
132 | } | |
133 | ||
134 | // Name returns the name of the source | |
135 | func (s *Source) Name() string { | |
136 | return "zoomeye" | |
137 | } |
0 | package subscraping | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "net/http" | |
5 | "regexp" | |
6 | ) | |
7 | ||
8 | // Source is an interface inherited by each passive source | |
9 | type Source interface { | |
10 | // Run takes a domain as argument and a session object | |
11 | // which contains the extractor for subdomains, http client | |
12 | // and other stuff. | |
13 | Run(context.Context, string, *Session) <-chan Result | |
14 | // Name returns the name of the source | |
15 | Name() string | |
16 | } | |
17 | ||
18 | // Session is the option passed to the source, an option is created | |
19 | // uniquely for eac source. | |
20 | type Session struct { | |
21 | // Extractor is the regex for subdomains created for each domain | |
22 | Extractor *regexp.Regexp | |
23 | // Keys is the API keys for the application | |
24 | Keys Keys | |
25 | // Client is the current http client | |
26 | Client *http.Client | |
27 | } | |
28 | ||
29 | // Keys contains the current API Keys we have in store | |
30 | type Keys struct { | |
31 | Binaryedge string `json:"binaryedge"` | |
32 | CensysToken string `json:"censysUsername"` | |
33 | CensysSecret string `json:"censysPassword"` | |
34 | Certspotter string `json:"certspotter"` | |
35 | Chaos string `json:"chaos"` | |
36 | DNSDB string `json:"dnsdb"` | |
37 | GitHub []string `json:"github"` | |
38 | IntelXHost string `json:"intelXHost"` | |
39 | IntelXKey string `json:"intelXKey"` | |
40 | PassiveTotalUsername string `json:"passivetotal_username"` | |
41 | PassiveTotalPassword string `json:"passivetotal_password"` | |
42 | Securitytrails string `json:"securitytrails"` | |
43 | Shodan string `json:"shodan"` | |
44 | Spyse string `json:"spyse"` | |
45 | URLScan string `json:"urlscan"` | |
46 | Virustotal string `json:"virustotal"` | |
47 | ZoomEyeUsername string `json:"zoomeye_username"` | |
48 | ZoomEyePassword string `json:"zoomeye_password"` | |
49 | } | |
50 | ||
51 | // Result is a result structure returned by a source | |
52 | type Result struct { | |
53 | Type ResultType | |
54 | Source string | |
55 | Value string | |
56 | Error error | |
57 | } | |
58 | ||
59 | // ResultType is the type of result returned by the source | |
60 | type ResultType int | |
61 | ||
62 | // Types of results returned by the source | |
63 | const ( | |
64 | Subdomain ResultType = iota | |
65 | Error | |
66 | ) |
0 | package subscraping | |
1 | ||
2 | import ( | |
3 | "regexp" | |
4 | "sync" | |
5 | ) | |
6 | ||
7 | var subdomainExtractorMutex = &sync.Mutex{} | |
8 | ||
9 | // NewSubdomainExtractor creates a new regular expression to extract | |
10 | // subdomains from text based on the given domain. | |
11 | func NewSubdomainExtractor(domain string) (*regexp.Regexp, error) { | |
12 | subdomainExtractorMutex.Lock() | |
13 | defer subdomainExtractorMutex.Unlock() | |
14 | extractor, err := regexp.Compile(`[a-zA-Z0-9\*_.-]+\.` + domain) | |
15 | if err != nil { | |
16 | return nil, err | |
17 | } | |
18 | return extractor, nil | |
19 | } | |
20 | ||
21 | // Exists check if a key exist in a slice | |
22 | func Exists(values []string, key string) bool { | |
23 | for _, v := range values { | |
24 | if v == key { | |
25 | return true | |
26 | } | |
27 | } | |
28 | return false | |
29 | } |
0 | package main | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | ||
5 | // Attempts to increase the OS file descriptors - Fail silently | |
6 | _ "github.com/projectdiscovery/fdmax/autofdmax" | |
7 | "github.com/projectdiscovery/gologger" | |
8 | "github.com/projectdiscovery/subfinder/v2/pkg/runner" | |
9 | ) | |
10 | ||
11 | func main() { | |
12 | // Parse the command line flags and read config files | |
13 | options := runner.ParseOptions() | |
14 | ||
15 | newRunner, err := runner.NewRunner(options) | |
16 | if err != nil { | |
17 | gologger.Fatalf("Could not create runner: %s\n", err) | |
18 | } | |
19 | ||
20 | err = newRunner.RunEnumeration(context.Background()) | |
21 | if err != nil { | |
22 | gologger.Fatalf("Could not run enumeration: %s\n", err) | |
23 | } | |
24 | } |
0 | module github.com/projectdiscovery/subfinder/v2 | |
1 | ||
2 | go 1.14 | |
3 | ||
4 | require ( | |
5 | github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 | |
6 | github.com/json-iterator/go v1.1.10 | |
7 | github.com/lib/pq v1.8.0 | |
8 | github.com/logrusorgru/aurora v2.0.3+incompatible // indirect | |
9 | github.com/miekg/dns v1.1.31 | |
10 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect | |
11 | github.com/modern-go/reflect2 v1.0.1 // indirect | |
12 | github.com/pkg/errors v0.9.1 | |
13 | github.com/projectdiscovery/chaos-client v0.1.6 | |
14 | github.com/projectdiscovery/fdmax v0.0.2 | |
15 | github.com/projectdiscovery/gologger v1.0.1 | |
16 | github.com/rs/xid v1.2.1 | |
17 | github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 | |
18 | golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect | |
19 | golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321 // indirect | |
20 | golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d // indirect | |
21 | gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 | |
22 | ) |
0 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | |
1 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | |
2 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= | |
3 | github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 h1:BpJ2o0OR5FV7vrkDYfXYVJQeMNWa8RhklZOpW2ITAIQ= | |
4 | github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE= | |
5 | github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= | |
6 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= | |
7 | github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= | |
8 | github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= | |
9 | github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs= | |
10 | github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= | |
11 | github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= | |
12 | github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= | |
13 | github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo= | |
14 | github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= | |
15 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= | |
16 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= | |
17 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= | |
18 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= | |
19 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= | |
20 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= | |
21 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= | |
22 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= | |
23 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= | |
24 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= | |
25 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= | |
26 | github.com/projectdiscovery/chaos-client v0.1.6 h1:AbIN7xUszjUi7FxI4qUVSqJ3um+6eImE/xstbNS0A1M= | |
27 | github.com/projectdiscovery/chaos-client v0.1.6/go.mod h1:F5omaoJh/vMvWnZhKD4zFFA5ti+RPwUletwepKSyfxk= | |
28 | github.com/projectdiscovery/fdmax v0.0.2 h1:d0HqNC4kbrMWT669u9W7ksFS7UBvnW0zmgY6FBU45UY= | |
29 | github.com/projectdiscovery/fdmax v0.0.2/go.mod h1:mbR7lJ9EONyxEfcsL2LlGtOSlzCQ5VraLzoJa/VTrAs= | |
30 | github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= | |
31 | github.com/projectdiscovery/gologger v1.0.1 h1:FzoYQZnxz9DCvSi/eg5A6+ET4CQ0CDUs27l6Exr8zMQ= | |
32 | github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= | |
33 | github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= | |
34 | github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= | |
35 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= | |
36 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= | |
37 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= | |
38 | github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= | |
39 | github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= | |
40 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= | |
41 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= | |
42 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= | |
43 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= | |
44 | golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= | |
45 | golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= | |
46 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= | |
47 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= | |
48 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= | |
49 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= | |
50 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= | |
51 | golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321 h1:lleNcKRbcaC8MqgLwghIkzZ2JBQAb7QQ9MiwRt1BisA= | |
52 | golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= | |
53 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | |
54 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | |
55 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | |
56 | golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M= | |
57 | golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | |
58 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | |
59 | golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d h1:L/IKR6COd7ubZrs2oTnTi73IhgqJ71c9s80WsQnh0Es= | |
60 | golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | |
61 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= | |
62 | golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= | |
63 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= | |
64 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= | |
65 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= | |
66 | gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= | |
67 | gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= |
0 | // Package passive provides capability for doing passive subdomain | |
1 | // enumeration on targets. | |
2 | package passive |
0 | package passive | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "sync" | |
6 | "time" | |
7 | ||
8 | "github.com/projectdiscovery/gologger" | |
9 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
10 | ) | |
11 | ||
12 | // EnumerateSubdomains enumerates all the subdomains for a given domain | |
13 | func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, timeout int, maxEnumTime time.Duration) chan subscraping.Result { | |
14 | results := make(chan subscraping.Result) | |
15 | ||
16 | go func() { | |
17 | session, err := subscraping.NewSession(domain, keys, timeout) | |
18 | if err != nil { | |
19 | results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} | |
20 | } | |
21 | ||
22 | ctx, cancel := context.WithTimeout(context.Background(), maxEnumTime) | |
23 | ||
24 | timeTaken := make(map[string]string) | |
25 | timeTakenMutex := &sync.Mutex{} | |
26 | ||
27 | wg := &sync.WaitGroup{} | |
28 | // Run each source in parallel on the target domain | |
29 | for source, runner := range a.sources { | |
30 | wg.Add(1) | |
31 | ||
32 | now := time.Now() | |
33 | go func(source string, runner subscraping.Source) { | |
34 | for resp := range runner.Run(ctx, domain, session) { | |
35 | results <- resp | |
36 | } | |
37 | ||
38 | duration := time.Since(now) | |
39 | timeTakenMutex.Lock() | |
40 | timeTaken[source] = fmt.Sprintf("Source took %s for enumeration\n", duration) | |
41 | timeTakenMutex.Unlock() | |
42 | ||
43 | wg.Done() | |
44 | }(source, runner) | |
45 | } | |
46 | wg.Wait() | |
47 | ||
48 | for source, data := range timeTaken { | |
49 | gologger.Verbosef(data, source) | |
50 | } | |
51 | ||
52 | close(results) | |
53 | cancel() | |
54 | }() | |
55 | ||
56 | return results | |
57 | } |
0 | package passive | |
1 | ||
2 | import ( | |
3 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
4 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/alienvault" | |
5 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/anubis" | |
6 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/archiveis" | |
7 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/binaryedge" | |
8 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover" | |
9 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/cebaidu" | |
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys" | |
11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter" | |
12 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotterold" | |
13 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos" | |
14 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl" | |
15 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh" | |
16 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" | |
17 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster" | |
18 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github" | |
19 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget" | |
20 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" | |
21 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/ipv4info" | |
22 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/passivetotal" | |
23 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" | |
24 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/recon" | |
25 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/riddler" | |
26 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/robtex" | |
27 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails" | |
28 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan" | |
29 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier" | |
30 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/spyse" | |
31 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sublist3r" | |
32 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook" | |
33 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatcrowd" | |
34 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatminer" | |
35 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal" | |
36 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" | |
37 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/ximcx" | |
38 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeye" | |
39 | ) | |
40 | ||
41 | // DefaultSources contains the list of fast sources used by default. | |
42 | var DefaultSources = []string{ | |
43 | "alienvault", | |
44 | "anubis", | |
45 | "bufferover", | |
46 | "cebaidu", | |
47 | "certspotter", | |
48 | "certspotterold", | |
49 | "censys", | |
50 | "chaos", | |
51 | "crtsh", | |
52 | "dnsdumpster", | |
53 | "hackertarget", | |
54 | "intelx", | |
55 | "ipv4info", | |
56 | "passivetotal", | |
57 | "robtex", | |
58 | "riddler", | |
59 | "securitytrails", | |
60 | "shodan", | |
61 | "spyse", | |
62 | "sublist3r", | |
63 | "threatcrowd", | |
64 | "threatminer", | |
65 | "virustotal", | |
66 | } | |
67 | ||
68 | // DefaultRecursiveSources contains list of default recursive sources | |
69 | var DefaultRecursiveSources = []string{ | |
70 | "alienvault", | |
71 | "binaryedge", | |
72 | "bufferover", | |
73 | "cebaidu", | |
74 | "certspotter", | |
75 | "certspotterold", | |
76 | "crtsh", | |
77 | "dnsdumpster", | |
78 | "hackertarget", | |
79 | "ipv4info", | |
80 | "passivetotal", | |
81 | "securitytrails", | |
82 | "sublist3r", | |
83 | "virustotal", | |
84 | "ximcx", | |
85 | } | |
86 | ||
87 | // DefaultAllSources contains list of all sources | |
88 | var DefaultAllSources = []string{ | |
89 | "alienvault", | |
90 | "anubis", | |
91 | "archiveis", | |
92 | "binaryedge", | |
93 | "bufferover", | |
94 | "cebaidu", | |
95 | "censys", | |
96 | "certspotter", | |
97 | "certspotterold", | |
98 | "chaos", | |
99 | "commoncrawl", | |
100 | "crtsh", | |
101 | "dnsdumpster", | |
102 | "dnsdb", | |
103 | "github", | |
104 | "hackertarget", | |
105 | "ipv4info", | |
106 | "intelx", | |
107 | "passivetotal", | |
108 | "rapiddns", | |
109 | "riddler", | |
110 | "recon", | |
111 | "robtex", | |
112 | "securitytrails", | |
113 | "shodan", | |
114 | "sitedossier", | |
115 | "spyse", | |
116 | "sublist3r", | |
117 | "threatbook", | |
118 | "threatcrowd", | |
119 | "threatminer", | |
120 | "virustotal", | |
121 | "waybackarchive", | |
122 | "ximcx", | |
123 | "zoomeye", | |
124 | } | |
125 | ||
126 | // Agent is a struct for running passive subdomain enumeration | |
127 | // against a given host. It wraps subscraping package and provides | |
128 | // a layer to build upon. | |
129 | type Agent struct { | |
130 | sources map[string]subscraping.Source | |
131 | } | |
132 | ||
133 | // New creates a new agent for passive subdomain discovery | |
134 | func New(sources, exclusions []string) *Agent { | |
135 | // Create the agent, insert the sources and remove the excluded sources | |
136 | agent := &Agent{sources: make(map[string]subscraping.Source)} | |
137 | ||
138 | agent.addSources(sources) | |
139 | agent.removeSources(exclusions) | |
140 | ||
141 | return agent | |
142 | } | |
143 | ||
144 | // addSources adds the given list of sources to the source array | |
145 | func (a *Agent) addSources(sources []string) { | |
146 | for _, source := range sources { | |
147 | switch source { | |
148 | case "alienvault": | |
149 | a.sources[source] = &alienvault.Source{} | |
150 | case "anubis": | |
151 | a.sources[source] = &anubis.Source{} | |
152 | case "archiveis": | |
153 | a.sources[source] = &archiveis.Source{} | |
154 | case "binaryedge": | |
155 | a.sources[source] = &binaryedge.Source{} | |
156 | case "bufferover": | |
157 | a.sources[source] = &bufferover.Source{} | |
158 | case "cebaidu": | |
159 | a.sources[source] = &cebaidu.Source{} | |
160 | case "censys": | |
161 | a.sources[source] = &censys.Source{} | |
162 | case "certspotter": | |
163 | a.sources[source] = &certspotter.Source{} | |
164 | case "certspotterold": | |
165 | a.sources[source] = &certspotterold.Source{} | |
166 | case "chaos": | |
167 | a.sources[source] = &chaos.Source{} | |
168 | case "commoncrawl": | |
169 | a.sources[source] = &commoncrawl.Source{} | |
170 | case "crtsh": | |
171 | a.sources[source] = &crtsh.Source{} | |
172 | case "dnsdumpster": | |
173 | a.sources[source] = &dnsdumpster.Source{} | |
174 | case "dnsdb": | |
175 | a.sources[source] = &dnsdb.Source{} | |
176 | case "github": | |
177 | a.sources[source] = &github.Source{} | |
178 | case "hackertarget": | |
179 | a.sources[source] = &hackertarget.Source{} | |
180 | case "ipv4info": | |
181 | a.sources[source] = &ipv4info.Source{} | |
182 | case "intelx": | |
183 | a.sources[source] = &intelx.Source{} | |
184 | case "passivetotal": | |
185 | a.sources[source] = &passivetotal.Source{} | |
186 | case "rapiddns": | |
187 | a.sources[source] = &rapiddns.Source{} | |
188 | case "recon": | |
189 | a.sources[source] = &recon.Source{} | |
190 | case "riddler": | |
191 | a.sources[source] = &riddler.Source{} | |
192 | case "robtex": | |
193 | a.sources[source] = &robtex.Source{} | |
194 | case "securitytrails": | |
195 | a.sources[source] = &securitytrails.Source{} | |
196 | case "shodan": | |
197 | a.sources[source] = &shodan.Source{} | |
198 | case "sitedossier": | |
199 | a.sources[source] = &sitedossier.Source{} | |
200 | case "spyse": | |
201 | a.sources[source] = &spyse.Source{} | |
202 | case "sublist3r": | |
203 | a.sources[source] = &sublist3r.Source{} | |
204 | case "threatbook": | |
205 | a.sources[source] = &threatbook.Source{} | |
206 | case "threatcrowd": | |
207 | a.sources[source] = &threatcrowd.Source{} | |
208 | case "threatminer": | |
209 | a.sources[source] = &threatminer.Source{} | |
210 | case "virustotal": | |
211 | a.sources[source] = &virustotal.Source{} | |
212 | case "waybackarchive": | |
213 | a.sources[source] = &waybackarchive.Source{} | |
214 | case "ximcx": | |
215 | a.sources[source] = &ximcx.Source{} | |
216 | case "zoomeye": | |
217 | a.sources[source] = &zoomeye.Source{} | |
218 | } | |
219 | } | |
220 | } | |
221 | ||
222 | // removeSources deletes the given sources from the source map | |
223 | func (a *Agent) removeSources(sources []string) { | |
224 | for _, source := range sources { | |
225 | delete(a.sources, source) | |
226 | } | |
227 | } |
0 | package resolve | |
1 | ||
2 | import ( | |
3 | "bufio" | |
4 | "math/rand" | |
5 | "os" | |
6 | "time" | |
7 | ) | |
8 | ||
9 | // DefaultResolvers contains the default list of resolvers known to be good | |
10 | var DefaultResolvers = []string{ | |
11 | "1.1.1.1", // Cloudflare primary | |
12 | "1.0.0.1", // Cloudlfare secondary | |
13 | "8.8.8.8", // Google primary | |
14 | "8.8.4.4", // Google secondary | |
15 | "9.9.9.9", // Quad9 Primary | |
16 | "9.9.9.10", // Quad9 Secondary | |
17 | "77.88.8.8", // Yandex Primary | |
18 | "77.88.8.1", // Yandex Secondary | |
19 | "208.67.222.222", // OpenDNS Primary | |
20 | "208.67.220.220", // OpenDNS Secondary | |
21 | } | |
22 | ||
23 | // Resolver is a struct for resolving DNS names | |
24 | type Resolver struct { | |
25 | resolvers []string | |
26 | rand *rand.Rand | |
27 | } | |
28 | ||
29 | // New creates a new resolver struct with the default resolvers | |
30 | func New() *Resolver { | |
31 | return &Resolver{ | |
32 | resolvers: []string{}, | |
33 | rand: rand.New(rand.NewSource(time.Now().UnixNano())), | |
34 | } | |
35 | } | |
36 | ||
37 | // AppendResolversFromFile appends the resolvers read from a file to the list of resolvers | |
38 | func (r *Resolver) AppendResolversFromFile(file string) error { | |
39 | f, err := os.Open(file) | |
40 | if err != nil { | |
41 | return err | |
42 | } | |
43 | scanner := bufio.NewScanner(f) | |
44 | for scanner.Scan() { | |
45 | text := scanner.Text() | |
46 | if text == "" { | |
47 | continue | |
48 | } | |
49 | r.resolvers = append(r.resolvers, text) | |
50 | } | |
51 | f.Close() | |
52 | return scanner.Err() | |
53 | } | |
54 | ||
55 | // AppendResolversFromSlice appends the slice to the list of resolvers | |
56 | func (r *Resolver) AppendResolversFromSlice(list []string) { | |
57 | r.resolvers = append(r.resolvers, list...) | |
58 | } |
0 | // Package resolve is used to handle resolving records | |
1 | // It also handles wildcard subdomains and rotating resolvers. | |
2 | package resolve |
0 | package resolve | |
1 | ||
2 | import ( | |
3 | "sync" | |
4 | ||
5 | "github.com/miekg/dns" | |
6 | "github.com/rs/xid" | |
7 | ) | |
8 | ||
9 | const ( | |
10 | maxResolveRetries = 5 | |
11 | maxWildcardChecks = 3 | |
12 | ) | |
13 | ||
14 | // ResolutionPool is a pool of resolvers created for resolving subdomains | |
15 | // for a given host. | |
16 | type ResolutionPool struct { | |
17 | *Resolver | |
18 | Tasks chan HostEntry | |
19 | Results chan Result | |
20 | wg *sync.WaitGroup | |
21 | removeWildcard bool | |
22 | ||
23 | wildcardIPs map[string]struct{} | |
24 | } | |
25 | ||
26 | // HostEntry defines a host with the source | |
27 | type HostEntry struct { | |
28 | Host string `json:"host"` | |
29 | Source string `json:"source"` | |
30 | } | |
31 | ||
32 | // Result contains the result for a host resolution | |
33 | type Result struct { | |
34 | Type ResultType | |
35 | Host string | |
36 | IP string | |
37 | Error error | |
38 | Source string | |
39 | } | |
40 | ||
41 | // ResultType is the type of result found | |
42 | type ResultType int | |
43 | ||
44 | // Types of data result can return | |
45 | const ( | |
46 | Subdomain ResultType = iota | |
47 | Error | |
48 | ) | |
49 | ||
50 | // NewResolutionPool creates a pool of resolvers for resolving subdomains of a given domain | |
51 | func (r *Resolver) NewResolutionPool(workers int, removeWildcard bool) *ResolutionPool { | |
52 | resolutionPool := &ResolutionPool{ | |
53 | Resolver: r, | |
54 | Tasks: make(chan HostEntry), | |
55 | Results: make(chan Result), | |
56 | wg: &sync.WaitGroup{}, | |
57 | removeWildcard: removeWildcard, | |
58 | wildcardIPs: make(map[string]struct{}), | |
59 | } | |
60 | ||
61 | go func() { | |
62 | for i := 0; i < workers; i++ { | |
63 | resolutionPool.wg.Add(1) | |
64 | go resolutionPool.resolveWorker() | |
65 | } | |
66 | resolutionPool.wg.Wait() | |
67 | close(resolutionPool.Results) | |
68 | }() | |
69 | ||
70 | return resolutionPool | |
71 | } | |
72 | ||
73 | // InitWildcards inits the wildcard ips array | |
74 | func (r *ResolutionPool) InitWildcards(domain string) error { | |
75 | for i := 0; i < maxWildcardChecks; i++ { | |
76 | uid := xid.New().String() | |
77 | ||
78 | hosts, err := r.getARecords(HostEntry{Host: uid + "." + domain}) | |
79 | if err != nil { | |
80 | return err | |
81 | } | |
82 | ||
83 | // Append all wildcard ips found for domains | |
84 | for _, host := range hosts { | |
85 | r.wildcardIPs[host] = struct{}{} | |
86 | } | |
87 | } | |
88 | return nil | |
89 | } | |
90 | ||
91 | func (r *ResolutionPool) resolveWorker() { | |
92 | for task := range r.Tasks { | |
93 | if !r.removeWildcard { | |
94 | r.Results <- Result{Type: Subdomain, Host: task.Host, IP: "", Source: task.Source} | |
95 | continue | |
96 | } | |
97 | ||
98 | hosts, err := r.getARecords(task) | |
99 | if err != nil { | |
100 | r.Results <- Result{Type: Error, Host: task.Host, Source: task.Source, Error: err} | |
101 | continue | |
102 | } | |
103 | ||
104 | if len(hosts) == 0 { | |
105 | continue | |
106 | } | |
107 | ||
108 | for _, host := range hosts { | |
109 | // Ignore the host if it exists in wildcard ips map | |
110 | if _, ok := r.wildcardIPs[host]; ok { //nolint:staticcheck //search alternatives for "comma ok" | |
111 | continue | |
112 | } | |
113 | } | |
114 | ||
115 | r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source} | |
116 | } | |
117 | r.wg.Done() | |
118 | } | |
119 | ||
120 | // getARecords gets all the A records for a given host | |
121 | func (r *ResolutionPool) getARecords(hostEntry HostEntry) ([]string, error) { | |
122 | var iteration int | |
123 | ||
124 | m := new(dns.Msg) | |
125 | m.Id = dns.Id() | |
126 | m.RecursionDesired = true | |
127 | m.Question = make([]dns.Question, 1) | |
128 | m.Question[0] = dns.Question{ | |
129 | Name: dns.Fqdn(hostEntry.Host), | |
130 | Qtype: dns.TypeA, | |
131 | Qclass: dns.ClassINET, | |
132 | } | |
133 | exchange: | |
134 | iteration++ | |
135 | in, err := dns.Exchange(m, r.resolvers[r.rand.Intn(len(r.resolvers))]+":53") | |
136 | if err != nil { | |
137 | // Retry in case of I/O error | |
138 | if iteration <= maxResolveRetries { | |
139 | goto exchange | |
140 | } | |
141 | return nil, err | |
142 | } | |
143 | // Ignore the error in case we have bad result | |
144 | if in != nil && in.Rcode != dns.RcodeSuccess { | |
145 | return nil, nil | |
146 | } | |
147 | ||
148 | var hosts []string | |
149 | for _, record := range in.Answer { | |
150 | if t, ok := record.(*dns.A); ok { | |
151 | hosts = append(hosts, t.A.String()) | |
152 | } | |
153 | } | |
154 | ||
155 | return hosts, nil | |
156 | } |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "github.com/projectdiscovery/gologger" | |
4 | "github.com/projectdiscovery/subfinder/v2/pkg/passive" | |
5 | "github.com/projectdiscovery/subfinder/v2/pkg/resolve" | |
6 | ) | |
7 | ||
8 | const banner = ` | |
9 | _ __ _ _ | |
10 | ____ _| |__ / _(_)_ _ __| |___ _ _ | |
11 | (_-< || | '_ \ _| | ' \/ _ / -_) '_| | |
12 | /__/\_,_|_.__/_| |_|_||_\__,_\___|_| v2.4.5 | |
13 | ` | |
14 | ||
15 | // Version is the current version of subfinder | |
16 | const Version = `2.4.5` | |
17 | ||
18 | // showBanner is used to show the banner to the user | |
19 | func showBanner() { | |
20 | gologger.Printf("%s\n", banner) | |
21 | gologger.Printf("\t\tprojectdiscovery.io\n\n") | |
22 | ||
23 | gologger.Labelf("Use with caution. You are responsible for your actions\n") | |
24 | gologger.Labelf("Developers assume no liability and are not responsible for any misuse or damage.\n") | |
25 | gologger.Labelf("By using subfinder, you also agree to the terms of the APIs used.\n\n") | |
26 | } | |
27 | ||
28 | // normalRunTasks runs the normal startup tasks | |
29 | func (options *Options) normalRunTasks() { | |
30 | configFile, err := UnmarshalRead(options.ConfigFile) | |
31 | if err != nil { | |
32 | gologger.Fatalf("Could not read configuration file %s: %s\n", options.ConfigFile, err) | |
33 | } | |
34 | ||
35 | // If we have a different version of subfinder installed | |
36 | // previously, use the new iteration of config file. | |
37 | if configFile.Version != Version { | |
38 | configFile.Sources = passive.DefaultSources | |
39 | configFile.AllSources = passive.DefaultAllSources | |
40 | configFile.Recursive = passive.DefaultRecursiveSources | |
41 | configFile.Version = Version | |
42 | ||
43 | err = configFile.MarshalWrite(options.ConfigFile) | |
44 | if err != nil { | |
45 | gologger.Fatalf("Could not update configuration file to %s: %s\n", options.ConfigFile, err) | |
46 | } | |
47 | } | |
48 | options.YAMLConfig = configFile | |
49 | } | |
50 | ||
51 | // firstRunTasks runs some housekeeping tasks done | |
52 | // when the program is ran for the first time | |
53 | func (options *Options) firstRunTasks() { | |
54 | // Create the configuration file and display information | |
55 | // about it to the user. | |
56 | config := ConfigFile{ | |
57 | // Use the default list of resolvers by marshaling it to the config | |
58 | Resolvers: resolve.DefaultResolvers, | |
59 | // Use the default list of passive sources | |
60 | Sources: passive.DefaultSources, | |
61 | // Use the default list of all passive sources | |
62 | AllSources: passive.DefaultAllSources, | |
63 | // Use the default list of recursive sources | |
64 | Recursive: passive.DefaultRecursiveSources, | |
65 | } | |
66 | ||
67 | err := config.MarshalWrite(options.ConfigFile) | |
68 | if err != nil { | |
69 | gologger.Fatalf("Could not write configuration file to %s: %s\n", options.ConfigFile, err) | |
70 | } | |
71 | options.YAMLConfig = config | |
72 | ||
73 | gologger.Infof("Configuration file saved to %s\n", options.ConfigFile) | |
74 | } |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "crypto/tls" | |
5 | "fmt" | |
6 | "io" | |
7 | "io/ioutil" | |
8 | "net/http" | |
9 | "time" | |
10 | ||
11 | "github.com/pkg/errors" | |
12 | "github.com/projectdiscovery/gologger" | |
13 | ) | |
14 | ||
15 | // UploadToChaosTimeoutNano timeout to upload to Chaos in nanoseconds | |
16 | const UploadToChaosTimeoutNano = 600 | |
17 | ||
18 | // UploadToChaos upload new data to Chaos dataset | |
19 | func (r *Runner) UploadToChaos(ctx context.Context, reader io.Reader) error { | |
20 | httpClient := &http.Client{ | |
21 | Transport: &http.Transport{ | |
22 | MaxIdleConnsPerHost: 100, | |
23 | MaxIdleConns: 100, | |
24 | TLSClientConfig: &tls.Config{ | |
25 | InsecureSkipVerify: true, | |
26 | }, | |
27 | }, | |
28 | Timeout: time.Duration(UploadToChaosTimeoutNano) * time.Second, // 10 minutes - uploads may take long | |
29 | } | |
30 | ||
31 | request, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://dns.projectdiscovery.io/dns/add", reader) | |
32 | if err != nil { | |
33 | return errors.Wrap(err, "could not create request") | |
34 | } | |
35 | request.Header.Set("Authorization", r.options.YAMLConfig.GetKeys().Chaos) | |
36 | ||
37 | resp, err := httpClient.Do(request) | |
38 | if err != nil { | |
39 | return errors.Wrap(err, "could not make request") | |
40 | } | |
41 | defer func() { | |
42 | _, err := io.Copy(ioutil.Discard, resp.Body) | |
43 | if err != nil { | |
44 | gologger.Warningf("Could not discard response body: %s\n", err) | |
45 | return | |
46 | } | |
47 | resp.Body.Close() | |
48 | }() | |
49 | ||
50 | if resp.StatusCode != http.StatusOK { | |
51 | return fmt.Errorf("invalid status code received: %d", resp.StatusCode) | |
52 | } | |
53 | return nil | |
54 | } |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "math/rand" | |
4 | "os" | |
5 | "strings" | |
6 | "time" | |
7 | ||
8 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
9 | "gopkg.in/yaml.v3" | |
10 | ) | |
11 | ||
12 | // MultipleKeyPartsLength is the max length for multiple keys | |
13 | const MultipleKeyPartsLength = 2 | |
14 | ||
15 | // YAMLIndentCharLength number of chars for identation on write YAML to file | |
16 | const YAMLIndentCharLength = 4 | |
17 | ||
18 | // ConfigFile contains the fields stored in the configuration file | |
19 | type ConfigFile struct { | |
20 | // Resolvers contains the list of resolvers to use while resolving | |
21 | Resolvers []string `yaml:"resolvers,omitempty"` | |
22 | // Sources contains a list of sources to use for enumeration | |
23 | Sources []string `yaml:"sources,omitempty"` | |
24 | // AllSources contains the list of all sources for enumeration (slow) | |
25 | AllSources []string `yaml:"all-sources,omitempty"` | |
26 | // Recrusive contains the list of recursive subdomain enum sources | |
27 | Recursive []string `yaml:"recursive,omitempty"` | |
28 | // ExcludeSources contains the sources to not include in the enumeration process | |
29 | ExcludeSources []string `yaml:"exclude-sources,omitempty"` | |
30 | // API keys for different sources | |
31 | Binaryedge []string `yaml:"binaryedge"` | |
32 | Censys []string `yaml:"censys"` | |
33 | Certspotter []string `yaml:"certspotter"` | |
34 | Chaos []string `yaml:"chaos"` | |
35 | DNSDB []string `yaml:"dnsdb"` | |
36 | GitHub []string `yaml:"github"` | |
37 | IntelX []string `yaml:"intelx"` | |
38 | PassiveTotal []string `yaml:"passivetotal"` | |
39 | Recon []string `yaml:"recon"` | |
40 | Robtex []string `yaml:"robtex"` | |
41 | SecurityTrails []string `yaml:"securitytrails"` | |
42 | Shodan []string `yaml:"shodan"` | |
43 | Spyse []string `yaml:"spyse"` | |
44 | ThreatBook []string `yaml:"threatbook"` | |
45 | URLScan []string `yaml:"urlscan"` | |
46 | Virustotal []string `yaml:"virustotal"` | |
47 | ZoomEye []string `yaml:"zoomeye"` | |
48 | // Version indicates the version of subfinder installed. | |
49 | Version string `yaml:"subfinder-version"` | |
50 | } | |
51 | ||
52 | // GetConfigDirectory gets the subfinder config directory for a user | |
53 | func GetConfigDirectory() (string, error) { | |
54 | // Seed the random number generator | |
55 | rand.Seed(time.Now().UnixNano()) | |
56 | ||
57 | var config string | |
58 | ||
59 | directory, err := os.UserHomeDir() | |
60 | if err != nil { | |
61 | return config, err | |
62 | } | |
63 | config = directory + "/.config/subfinder" | |
64 | ||
65 | // Create All directory for subfinder even if they exist | |
66 | err = os.MkdirAll(config, os.ModePerm) | |
67 | if err != nil { | |
68 | return config, err | |
69 | } | |
70 | ||
71 | return config, nil | |
72 | } | |
73 | ||
74 | // CheckConfigExists checks if the config file exists in the given path | |
75 | func CheckConfigExists(configPath string) bool { | |
76 | if _, err := os.Stat(configPath); err == nil { | |
77 | return true | |
78 | } else if os.IsNotExist(err) { | |
79 | return false | |
80 | } | |
81 | return false | |
82 | } | |
83 | ||
84 | // MarshalWrite writes the marshaled yaml config to disk | |
85 | func (c *ConfigFile) MarshalWrite(file string) error { | |
86 | f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) | |
87 | if err != nil { | |
88 | return err | |
89 | } | |
90 | ||
91 | // Indent the spaces too | |
92 | enc := yaml.NewEncoder(f) | |
93 | enc.SetIndent(YAMLIndentCharLength) | |
94 | err = enc.Encode(&c) | |
95 | f.Close() | |
96 | return err | |
97 | } | |
98 | ||
99 | // UnmarshalRead reads the unmarshalled config yaml file from disk | |
100 | func UnmarshalRead(file string) (ConfigFile, error) { | |
101 | config := ConfigFile{} | |
102 | ||
103 | f, err := os.Open(file) | |
104 | if err != nil { | |
105 | return config, err | |
106 | } | |
107 | err = yaml.NewDecoder(f).Decode(&config) | |
108 | f.Close() | |
109 | return config, err | |
110 | } | |
111 | ||
112 | // GetKeys gets the API keys from config file and creates a Keys struct | |
113 | // We use random selection of api keys from the list of keys supplied. | |
114 | // Keys that require 2 options are separated by colon (:). | |
115 | func (c *ConfigFile) GetKeys() subscraping.Keys { | |
116 | keys := subscraping.Keys{} | |
117 | ||
118 | if len(c.Binaryedge) > 0 { | |
119 | keys.Binaryedge = c.Binaryedge[rand.Intn(len(c.Binaryedge))] | |
120 | } | |
121 | ||
122 | if len(c.Censys) > 0 { | |
123 | censysKeys := c.Censys[rand.Intn(len(c.Censys))] | |
124 | parts := strings.Split(censysKeys, ":") | |
125 | if len(parts) == MultipleKeyPartsLength { | |
126 | keys.CensysToken = parts[0] | |
127 | keys.CensysSecret = parts[1] | |
128 | } | |
129 | } | |
130 | ||
131 | if len(c.Certspotter) > 0 { | |
132 | keys.Certspotter = c.Certspotter[rand.Intn(len(c.Certspotter))] | |
133 | } | |
134 | if len(c.Chaos) > 0 { | |
135 | keys.Chaos = c.Chaos[rand.Intn(len(c.Chaos))] | |
136 | } | |
137 | if (len(c.DNSDB)) > 0 { | |
138 | keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))] | |
139 | } | |
140 | if (len(c.GitHub)) > 0 { | |
141 | keys.GitHub = c.GitHub | |
142 | } | |
143 | ||
144 | if len(c.IntelX) > 0 { | |
145 | intelxKeys := c.IntelX[rand.Intn(len(c.IntelX))] | |
146 | parts := strings.Split(intelxKeys, ":") | |
147 | if len(parts) == MultipleKeyPartsLength { | |
148 | keys.IntelXHost = parts[0] | |
149 | keys.IntelXKey = parts[1] | |
150 | } | |
151 | } | |
152 | ||
153 | if len(c.PassiveTotal) > 0 { | |
154 | passiveTotalKeys := c.PassiveTotal[rand.Intn(len(c.PassiveTotal))] | |
155 | parts := strings.Split(passiveTotalKeys, ":") | |
156 | if len(parts) == MultipleKeyPartsLength { | |
157 | keys.PassiveTotalUsername = parts[0] | |
158 | keys.PassiveTotalPassword = parts[1] | |
159 | } | |
160 | } | |
161 | ||
162 | if len(c.Recon) > 0 { | |
163 | keys.Recon = c.Recon[rand.Intn(len(c.Recon))] | |
164 | } | |
165 | ||
166 | if len(c.Robtex) > 0 { | |
167 | keys.Robtex = c.Robtex[rand.Intn(len(c.Robtex))] | |
168 | } | |
169 | ||
170 | if len(c.SecurityTrails) > 0 { | |
171 | keys.Securitytrails = c.SecurityTrails[rand.Intn(len(c.SecurityTrails))] | |
172 | } | |
173 | if len(c.Shodan) > 0 { | |
174 | keys.Shodan = c.Shodan[rand.Intn(len(c.Shodan))] | |
175 | } | |
176 | if len(c.Spyse) > 0 { | |
177 | keys.Spyse = c.Spyse[rand.Intn(len(c.Spyse))] | |
178 | } | |
179 | if len(c.ThreatBook) > 0 { | |
180 | keys.ThreatBook = c.ThreatBook[rand.Intn(len(c.ThreatBook))] | |
181 | } | |
182 | if len(c.URLScan) > 0 { | |
183 | keys.URLScan = c.URLScan[rand.Intn(len(c.URLScan))] | |
184 | } | |
185 | if len(c.Virustotal) > 0 { | |
186 | keys.Virustotal = c.Virustotal[rand.Intn(len(c.Virustotal))] | |
187 | } | |
188 | if len(c.ZoomEye) > 0 { | |
189 | zoomEyeKeys := c.ZoomEye[rand.Intn(len(c.ZoomEye))] | |
190 | parts := strings.Split(zoomEyeKeys, ":") | |
191 | if len(parts) == MultipleKeyPartsLength { | |
192 | keys.ZoomEyeUsername = parts[0] | |
193 | keys.ZoomEyePassword = parts[1] | |
194 | } | |
195 | } | |
196 | ||
197 | return keys | |
198 | } |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "os" | |
4 | "testing" | |
5 | ||
6 | "github.com/stretchr/testify/assert" | |
7 | ) | |
8 | ||
9 | func TestConfigGetDirectory(t *testing.T) { | |
10 | directory, err := GetConfigDirectory() | |
11 | if err != nil { | |
12 | t.Fatalf("Expected nil got %v while getting home\n", err) | |
13 | } | |
14 | home, err := os.UserHomeDir() | |
15 | if err != nil { | |
16 | t.Fatalf("Expected nil got %v while getting dir\n", err) | |
17 | } | |
18 | config := home + "/.config/subfinder" | |
19 | ||
20 | assert.Equal(t, directory, config, "Directory and config should be equal") | |
21 | } |
0 | // Package runner implements the mechanism to drive the | |
1 | // subdomain enumeration process | |
2 | package runner |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "bytes" | |
4 | "context" | |
5 | "os" | |
6 | "strings" | |
7 | "sync" | |
8 | "time" | |
9 | ||
10 | "github.com/hako/durafmt" | |
11 | "github.com/projectdiscovery/gologger" | |
12 | "github.com/projectdiscovery/subfinder/v2/pkg/resolve" | |
13 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
14 | ) | |
15 | ||
16 | const maxNumCount = 2 | |
17 | ||
18 | // EnumerateSingleDomain performs subdomain enumeration against a single domain | |
19 | func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain, output string, appendToFile bool) error { | |
20 | gologger.Infof("Enumerating subdomains for %s\n", domain) | |
21 | ||
22 | // Get the API keys for sources from the configuration | |
23 | // and also create the active resolving engine for the domain. | |
24 | keys := r.options.YAMLConfig.GetKeys() | |
25 | ||
26 | // Check if the user has asked to remove wildcards explicitly. | |
27 | // If yes, create the resolution pool and get the wildcards for the current domain | |
28 | var resolutionPool *resolve.ResolutionPool | |
29 | if r.options.RemoveWildcard { | |
30 | resolutionPool = r.resolverClient.NewResolutionPool(r.options.Threads, r.options.RemoveWildcard) | |
31 | err := resolutionPool.InitWildcards(domain) | |
32 | if err != nil { | |
33 | // Log the error but don't quit. | |
34 | gologger.Warningf("Could not get wildcards for domain %s: %s\n", domain, err) | |
35 | } | |
36 | } | |
37 | ||
38 | // Run the passive subdomain enumeration | |
39 | now := time.Now() | |
40 | passiveResults := r.passiveAgent.EnumerateSubdomains(domain, &keys, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) | |
41 | ||
42 | wg := &sync.WaitGroup{} | |
43 | wg.Add(1) | |
44 | // Create a unique map for filtering duplicate subdomains out | |
45 | uniqueMap := make(map[string]resolve.HostEntry) | |
46 | // Process the results in a separate goroutine | |
47 | go func() { | |
48 | for result := range passiveResults { | |
49 | switch result.Type { | |
50 | case subscraping.Error: | |
51 | gologger.Warningf("Could not run source %s: %s\n", result.Source, result.Error) | |
52 | case subscraping.Subdomain: | |
53 | // Validate the subdomain found and remove wildcards from | |
54 | if !strings.HasSuffix(result.Value, "."+domain) { | |
55 | continue | |
56 | } | |
57 | subdomain := strings.ReplaceAll(strings.ToLower(result.Value), "*.", "") | |
58 | ||
59 | // Check if the subdomain is a duplicate. If not, | |
60 | // send the subdomain for resolution. | |
61 | if _, ok := uniqueMap[subdomain]; ok { | |
62 | continue | |
63 | } | |
64 | ||
65 | hostEntry := resolve.HostEntry{Host: subdomain, Source: result.Source} | |
66 | ||
67 | uniqueMap[subdomain] = hostEntry | |
68 | ||
69 | // Log the verbose message about the found subdomain and send the | |
70 | // host for resolution to the resolution pool | |
71 | gologger.Verbosef("%s\n", result.Source, subdomain) | |
72 | ||
73 | // If the user asked to remove wildcard then send on the resolve | |
74 | // queue. Otherwise, if mode is not verbose print the results on | |
75 | // the screen as they are discovered. | |
76 | if r.options.RemoveWildcard { | |
77 | resolutionPool.Tasks <- hostEntry | |
78 | } | |
79 | } | |
80 | } | |
81 | // Close the task channel only if wildcards are asked to be removed | |
82 | if r.options.RemoveWildcard { | |
83 | close(resolutionPool.Tasks) | |
84 | } | |
85 | wg.Done() | |
86 | }() | |
87 | ||
88 | // If the user asked to remove wildcards, listen from the results | |
89 | // queue and write to the map. At the end, print the found results to the screen | |
90 | foundResults := make(map[string]resolve.Result) | |
91 | if r.options.RemoveWildcard { | |
92 | // Process the results coming from the resolutions pool | |
93 | for result := range resolutionPool.Results { | |
94 | switch result.Type { | |
95 | case resolve.Error: | |
96 | gologger.Warningf("Could not resolve host: %s\n", result.Error) | |
97 | case resolve.Subdomain: | |
98 | // Add the found subdomain to a map. | |
99 | if _, ok := foundResults[result.Host]; !ok { | |
100 | foundResults[result.Host] = result | |
101 | } | |
102 | } | |
103 | } | |
104 | } | |
105 | wg.Wait() | |
106 | ||
107 | outputter := NewOutputter(r.options.JSON) | |
108 | ||
109 | // If verbose mode was used, then now print all the | |
110 | // found subdomains on the screen together. | |
111 | var err error | |
112 | if r.options.HostIP { | |
113 | err = outputter.WriteHostIP(foundResults, os.Stdout) | |
114 | } else { | |
115 | if r.options.RemoveWildcard { | |
116 | err = outputter.WriteHostNoWildcard(foundResults, os.Stdout) | |
117 | } else { | |
118 | err = outputter.WriteHost(uniqueMap, os.Stdout) | |
119 | } | |
120 | } | |
121 | if err != nil { | |
122 | gologger.Errorf("Could not verbose results for %s: %s\n", domain, err) | |
123 | return err | |
124 | } | |
125 | ||
126 | // Show found subdomain count in any case. | |
127 | duration := durafmt.Parse(time.Since(now)).LimitFirstN(maxNumCount).String() | |
128 | if r.options.RemoveWildcard { | |
129 | gologger.Infof("Found %d subdomains for %s in %s\n", len(foundResults), domain, duration) | |
130 | } else { | |
131 | gologger.Infof("Found %d subdomains for %s in %s\n", len(uniqueMap), domain, duration) | |
132 | } | |
133 | ||
134 | // In case the user has specified to upload to chaos, write everything to a temporary buffer and upload | |
135 | if r.options.ChaosUpload { | |
136 | var buf = &bytes.Buffer{} | |
137 | err := outputter.WriteForChaos(uniqueMap, buf) | |
138 | // If an error occurs, do not interrupt, continue to check if user specified an output file | |
139 | if err != nil { | |
140 | gologger.Errorf("Could not prepare results for chaos %s\n", err) | |
141 | } else { | |
142 | // no error in writing host output, upload to chaos | |
143 | err = r.UploadToChaos(ctx, buf) | |
144 | if err != nil { | |
145 | gologger.Errorf("Could not upload results to chaos %s\n", err) | |
146 | } else { | |
147 | gologger.Infof("Input processed successfully and subdomains with valid records will be updated to chaos dataset.\n") | |
148 | } | |
149 | // clear buffer | |
150 | buf.Reset() | |
151 | } | |
152 | } | |
153 | ||
154 | if output != "" { | |
155 | file, err := outputter.createFile(output, appendToFile) | |
156 | if err != nil { | |
157 | gologger.Errorf("Could not create file %s for %s: %s\n", output, domain, err) | |
158 | return err | |
159 | } | |
160 | ||
161 | defer file.Close() | |
162 | ||
163 | if r.options.HostIP { | |
164 | err = outputter.WriteHostIP(foundResults, file) | |
165 | } else { | |
166 | if r.options.RemoveWildcard { | |
167 | err = outputter.WriteHostNoWildcard(foundResults, file) | |
168 | } else { | |
169 | err = outputter.WriteHost(uniqueMap, file) | |
170 | } | |
171 | } | |
172 | if err != nil { | |
173 | gologger.Errorf("Could not write results to file %s for %s: %s\n", output, domain, err) | |
174 | return err | |
175 | } | |
176 | } | |
177 | ||
178 | return nil | |
179 | } |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "strings" | |
4 | ||
5 | "github.com/projectdiscovery/subfinder/v2/pkg/passive" | |
6 | "github.com/projectdiscovery/subfinder/v2/pkg/resolve" | |
7 | ) | |
8 | ||
9 | // initializePassiveEngine creates the passive engine and loads sources etc | |
10 | func (r *Runner) initializePassiveEngine() { | |
11 | var sources, exclusions []string | |
12 | ||
13 | if r.options.ExcludeSources != "" { | |
14 | exclusions = append(exclusions, strings.Split(r.options.ExcludeSources, ",")...) | |
15 | } else { | |
16 | exclusions = append(exclusions, r.options.YAMLConfig.ExcludeSources...) | |
17 | } | |
18 | ||
19 | // Use all sources if asked by the user | |
20 | if r.options.All { | |
21 | sources = append(sources, r.options.YAMLConfig.AllSources...) | |
22 | } | |
23 | ||
24 | // If only recursive sources are wanted, use them only. | |
25 | if r.options.Recursive { | |
26 | sources = append(sources, r.options.YAMLConfig.Recursive...) | |
27 | } | |
28 | ||
29 | // If there are any sources from CLI, only use them | |
30 | // Otherwise, use the yaml file sources | |
31 | if !r.options.All && !r.options.Recursive { | |
32 | if r.options.Sources != "" { | |
33 | sources = append(sources, strings.Split(r.options.Sources, ",")...) | |
34 | } else { | |
35 | sources = append(sources, r.options.YAMLConfig.Sources...) | |
36 | } | |
37 | } | |
38 | r.passiveAgent = passive.New(sources, exclusions) | |
39 | } | |
40 | ||
41 | // initializeActiveEngine creates the resolver used to resolve the found subdomains | |
42 | func (r *Runner) initializeActiveEngine() error { | |
43 | r.resolverClient = resolve.New() | |
44 | ||
45 | // If the file has been provided, read resolvers from the file | |
46 | if r.options.ResolverList != "" { | |
47 | err := r.resolverClient.AppendResolversFromFile(r.options.ResolverList) | |
48 | if err != nil { | |
49 | return err | |
50 | } | |
51 | } | |
52 | ||
53 | var resolvers []string | |
54 | ||
55 | if r.options.Resolvers != "" { | |
56 | resolvers = append(resolvers, strings.Split(r.options.Resolvers, ",")...) | |
57 | } else if len(r.options.YAMLConfig.Resolvers) > 0 { | |
58 | resolvers = append(resolvers, r.options.YAMLConfig.Resolvers...) | |
59 | } else { | |
60 | resolvers = append(resolvers, resolve.DefaultResolvers...) | |
61 | } | |
62 | ||
63 | r.resolverClient.AppendResolversFromSlice(resolvers) | |
64 | ||
65 | return nil | |
66 | } |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "flag" | |
4 | "os" | |
5 | "path" | |
6 | "reflect" | |
7 | "strings" | |
8 | ||
9 | "github.com/projectdiscovery/gologger" | |
10 | ) | |
11 | ||
12 | // Options contains the configuration options for tuning | |
13 | // the subdomain enumeration process. | |
14 | type Options struct { | |
15 | Verbose bool // Verbose flag indicates whether to show verbose output or not | |
16 | NoColor bool // No-Color disables the colored output | |
17 | ChaosUpload bool // ChaosUpload indicates whether to upload results to the Chaos API | |
18 | JSON bool // JSON specifies whether to use json for output format or text file | |
19 | HostIP bool // HostIP specifies whether to write subdomains in host:ip format | |
20 | Silent bool // Silent suppresses any extra text and only writes subdomains to screen | |
21 | ListSources bool // ListSources specifies whether to list all available sources | |
22 | RemoveWildcard bool // RemoveWildcard specifies whether to remove potential wildcard or dead subdomains from the results. | |
23 | Stdin bool // Stdin specifies whether stdin input was given to the process | |
24 | Version bool // Version specifies if we should just show version and exit | |
25 | Recursive bool // Recursive specifies whether to use only recursive subdomain enumeration sources | |
26 | All bool // All specifies whether to use all (slow) sources. | |
27 | Threads int // Thread controls the number of threads to use for active enumerations | |
28 | Timeout int // Timeout is the seconds to wait for sources to respond | |
29 | MaxEnumerationTime int // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration | |
30 | Domain string // Domain is the domain to find subdomains for | |
31 | DomainsFile string // DomainsFile is the file containing list of domains to find subdomains for | |
32 | Output string // Output is the file to write found subdomains to. | |
33 | OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given | |
34 | Sources string // Sources contains a comma-separated list of sources to use for enumeration | |
35 | ExcludeSources string // ExcludeSources contains the comma-separated sources to not include in the enumeration process | |
36 | Resolvers string // Resolvers is the comma-separated resolvers to use for enumeration | |
37 | ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration | |
38 | ConfigFile string // ConfigFile contains the location of the config file | |
39 | ||
40 | YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file | |
41 | } | |
42 | ||
43 | // ParseOptions parses the command line flags provided by a user | |
44 | func ParseOptions() *Options { | |
45 | options := &Options{} | |
46 | ||
47 | config, err := GetConfigDirectory() | |
48 | if err != nil { | |
49 | // This should never be reached | |
50 | gologger.Fatalf("Could not get user home: %s\n", err) | |
51 | } | |
52 | ||
53 | flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output") | |
54 | flag.BoolVar(&options.NoColor, "nC", false, "Don't Use colors in output") | |
55 | flag.IntVar(&options.Threads, "t", 10, "Number of concurrent goroutines for resolving") | |
56 | flag.IntVar(&options.Timeout, "timeout", 30, "Seconds to wait before timing out") | |
57 | flag.IntVar(&options.MaxEnumerationTime, "max-time", 10, "Minutes to wait for enumeration results") | |
58 | flag.StringVar(&options.Domain, "d", "", "Domain to find subdomains for") | |
59 | flag.StringVar(&options.DomainsFile, "dL", "", "File containing list of domains to enumerate") | |
60 | flag.BoolVar(&options.ChaosUpload, "cd", false, "Upload results to the Chaos API (api-key required)") | |
61 | flag.StringVar(&options.Output, "o", "", "File to write output to (optional)") | |
62 | flag.StringVar(&options.OutputDirectory, "oD", "", "Directory to write enumeration results to (optional)") | |
63 | flag.BoolVar(&options.JSON, "json", false, "Write output in JSON lines Format") | |
64 | flag.BoolVar(&options.JSON, "oJ", false, "Write output in JSON lines Format") | |
65 | flag.BoolVar(&options.HostIP, "oI", false, "Write output in Host,IP format") | |
66 | flag.BoolVar(&options.Silent, "silent", false, "Show only subdomains in output") | |
67 | flag.BoolVar(&options.Recursive, "recursive", false, "Use only recursive subdomain enumeration sources") | |
68 | flag.BoolVar(&options.All, "all", false, "Use all sources (slow) for enumeration") | |
69 | flag.StringVar(&options.Sources, "sources", "", "Comma separated list of sources to use") | |
70 | flag.BoolVar(&options.ListSources, "ls", false, "List all available sources") | |
71 | flag.StringVar(&options.ExcludeSources, "exclude-sources", "", "List of sources to exclude from enumeration") | |
72 | flag.StringVar(&options.Resolvers, "r", "", "Comma-separated list of resolvers to use") | |
73 | flag.StringVar(&options.ResolverList, "rL", "", "Text file containing list of resolvers to use") | |
74 | flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") | |
75 | flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") | |
76 | flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") | |
77 | flag.Parse() | |
78 | ||
79 | // Check if stdin pipe was given | |
80 | options.Stdin = hasStdin() | |
81 | ||
82 | // Read the inputs and configure the logging | |
83 | options.configureOutput() | |
84 | ||
85 | // Show the user the banner | |
86 | showBanner() | |
87 | ||
88 | if options.Version { | |
89 | gologger.Infof("Current Version: %s\n", Version) | |
90 | os.Exit(0) | |
91 | } | |
92 | ||
93 | // Check if the config file exists. If not, it means this is the | |
94 | // first run of the program. Show the first run notices and initialize the config file. | |
95 | // Else show the normal banners and read the yaml fiile to the config | |
96 | if !CheckConfigExists(options.ConfigFile) { | |
97 | options.firstRunTasks() | |
98 | } else { | |
99 | options.normalRunTasks() | |
100 | } | |
101 | ||
102 | if options.ListSources { | |
103 | listSources(options) | |
104 | os.Exit(0) | |
105 | } | |
106 | ||
107 | // Validate the options passed by the user and if any | |
108 | // invalid options have been used, exit. | |
109 | err = options.validateOptions() | |
110 | if err != nil { | |
111 | gologger.Fatalf("Program exiting: %s\n", err) | |
112 | } | |
113 | ||
114 | return options | |
115 | } | |
116 | ||
117 | func hasStdin() bool { | |
118 | fi, err := os.Stdin.Stat() | |
119 | if err != nil { | |
120 | return false | |
121 | } | |
122 | if fi.Mode()&os.ModeNamedPipe == 0 { | |
123 | return false | |
124 | } | |
125 | return true | |
126 | } | |
127 | ||
128 | func listSources(options *Options) { | |
129 | gologger.Infof("Current list of available sources. [%d]\n", len(options.YAMLConfig.AllSources)) | |
130 | gologger.Infof("Sources marked with an * needs key or token in order to work.\n") | |
131 | gologger.Infof("You can modify %s to configure your keys / tokens.\n\n", options.ConfigFile) | |
132 | ||
133 | keys := options.YAMLConfig.GetKeys() | |
134 | needsKey := make(map[string]interface{}) | |
135 | keysElem := reflect.ValueOf(&keys).Elem() | |
136 | for i := 0; i < keysElem.NumField(); i++ { | |
137 | needsKey[strings.ToLower(keysElem.Type().Field(i).Name)] = keysElem.Field(i).Interface() | |
138 | } | |
139 | ||
140 | for _, source := range options.YAMLConfig.AllSources { | |
141 | message := "%s\n" | |
142 | if _, ok := needsKey[source]; ok { | |
143 | message = "%s *\n" | |
144 | } | |
145 | gologger.Silentf(message, source) | |
146 | } | |
147 | } |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "bufio" | |
4 | "errors" | |
5 | "io" | |
6 | "os" | |
7 | "path/filepath" | |
8 | "strings" | |
9 | ||
10 | jsoniter "github.com/json-iterator/go" | |
11 | "github.com/projectdiscovery/subfinder/v2/pkg/resolve" | |
12 | ) | |
13 | ||
14 | // OutPutter outputs content to writers. | |
15 | type OutPutter struct { | |
16 | JSON bool | |
17 | } | |
18 | ||
19 | type jsonResult struct { | |
20 | Host string `json:"host"` | |
21 | IP string `json:"ip"` | |
22 | Source string `json:"source"` | |
23 | } | |
24 | ||
25 | // NewOutputter creates a new Outputter | |
26 | func NewOutputter(json bool) *OutPutter { | |
27 | return &OutPutter{JSON: json} | |
28 | } | |
29 | ||
30 | func (o *OutPutter) createFile(filename string, appendtoFile bool) (*os.File, error) { | |
31 | if filename == "" { | |
32 | return nil, errors.New("empty filename") | |
33 | } | |
34 | ||
35 | dir := filepath.Dir(filename) | |
36 | ||
37 | if dir != "" { | |
38 | if _, err := os.Stat(dir); os.IsNotExist(err) { | |
39 | err := os.MkdirAll(dir, os.ModePerm) | |
40 | if err != nil { | |
41 | return nil, err | |
42 | } | |
43 | } | |
44 | } | |
45 | ||
46 | var file *os.File | |
47 | var err error | |
48 | if appendtoFile { | |
49 | file, err = os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) | |
50 | } else { | |
51 | file, err = os.Create(filename) | |
52 | } | |
53 | if err != nil { | |
54 | return nil, err | |
55 | } | |
56 | ||
57 | return file, nil | |
58 | } | |
59 | ||
60 | // WriteForChaos prepares the buffer to upload to Chaos | |
61 | func (o *OutPutter) WriteForChaos(results map[string]resolve.HostEntry, writer io.Writer) error { | |
62 | bufwriter := bufio.NewWriter(writer) | |
63 | sb := &strings.Builder{} | |
64 | ||
65 | for _, result := range results { | |
66 | sb.WriteString(result.Host) | |
67 | sb.WriteString("\n") | |
68 | ||
69 | _, err := bufwriter.WriteString(sb.String()) | |
70 | if err != nil { | |
71 | bufwriter.Flush() | |
72 | return err | |
73 | } | |
74 | sb.Reset() | |
75 | } | |
76 | return bufwriter.Flush() | |
77 | } | |
78 | ||
79 | // WriteHostIP writes the output list of subdomain to an io.Writer | |
80 | func (o *OutPutter) WriteHostIP(results map[string]resolve.Result, writer io.Writer) error { | |
81 | var err error | |
82 | if o.JSON { | |
83 | err = writeJSONHostIP(results, writer) | |
84 | } else { | |
85 | err = writePlainHostIP(results, writer) | |
86 | } | |
87 | return err | |
88 | } | |
89 | ||
90 | func writePlainHostIP(results map[string]resolve.Result, writer io.Writer) error { | |
91 | bufwriter := bufio.NewWriter(writer) | |
92 | sb := &strings.Builder{} | |
93 | ||
94 | for _, result := range results { | |
95 | sb.WriteString(result.Host) | |
96 | sb.WriteString(",") | |
97 | sb.WriteString(result.IP) | |
98 | sb.WriteString(",") | |
99 | sb.WriteString(result.Source) | |
100 | sb.WriteString("\n") | |
101 | ||
102 | _, err := bufwriter.WriteString(sb.String()) | |
103 | if err != nil { | |
104 | bufwriter.Flush() | |
105 | return err | |
106 | } | |
107 | sb.Reset() | |
108 | } | |
109 | return bufwriter.Flush() | |
110 | } | |
111 | ||
112 | func writeJSONHostIP(results map[string]resolve.Result, writer io.Writer) error { | |
113 | encoder := jsoniter.NewEncoder(writer) | |
114 | ||
115 | var data jsonResult | |
116 | ||
117 | for _, result := range results { | |
118 | data.Host = result.Host | |
119 | data.IP = result.IP | |
120 | data.Source = result.Source | |
121 | ||
122 | err := encoder.Encode(&data) | |
123 | if err != nil { | |
124 | return err | |
125 | } | |
126 | } | |
127 | return nil | |
128 | } | |
129 | ||
130 | // WriteHostNoWildcard writes the output list of subdomain with nW flag to an io.Writer | |
131 | func (o *OutPutter) WriteHostNoWildcard(results map[string]resolve.Result, writer io.Writer) error { | |
132 | hosts := make(map[string]resolve.HostEntry) | |
133 | for host, result := range results { | |
134 | hosts[host] = resolve.HostEntry{Host: result.Host, Source: result.Source} | |
135 | } | |
136 | ||
137 | return o.WriteHost(hosts, writer) | |
138 | } | |
139 | ||
140 | // WriteHost writes the output list of subdomain to an io.Writer | |
141 | func (o *OutPutter) WriteHost(results map[string]resolve.HostEntry, writer io.Writer) error { | |
142 | var err error | |
143 | if o.JSON { | |
144 | err = writeJSONHost(results, writer) | |
145 | } else { | |
146 | err = writePlainHost(results, writer) | |
147 | } | |
148 | return err | |
149 | } | |
150 | ||
151 | func writePlainHost(results map[string]resolve.HostEntry, writer io.Writer) error { | |
152 | bufwriter := bufio.NewWriter(writer) | |
153 | sb := &strings.Builder{} | |
154 | ||
155 | for _, result := range results { | |
156 | sb.WriteString(result.Host) | |
157 | sb.WriteString("\n") | |
158 | ||
159 | _, err := bufwriter.WriteString(sb.String()) | |
160 | if err != nil { | |
161 | bufwriter.Flush() | |
162 | return err | |
163 | } | |
164 | sb.Reset() | |
165 | } | |
166 | return bufwriter.Flush() | |
167 | } | |
168 | ||
169 | func writeJSONHost(results map[string]resolve.HostEntry, writer io.Writer) error { | |
170 | encoder := jsoniter.NewEncoder(writer) | |
171 | ||
172 | for _, result := range results { | |
173 | err := encoder.Encode(result) | |
174 | if err != nil { | |
175 | return err | |
176 | } | |
177 | } | |
178 | return nil | |
179 | } |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "bufio" | |
4 | "context" | |
5 | "io" | |
6 | "os" | |
7 | "path" | |
8 | ||
9 | "github.com/projectdiscovery/subfinder/v2/pkg/passive" | |
10 | "github.com/projectdiscovery/subfinder/v2/pkg/resolve" | |
11 | ) | |
12 | ||
13 | // Runner is an instance of the subdomain enumeration | |
14 | // client used to orchestrate the whole process. | |
15 | type Runner struct { | |
16 | options *Options | |
17 | passiveAgent *passive.Agent | |
18 | resolverClient *resolve.Resolver | |
19 | } | |
20 | ||
21 | // NewRunner creates a new runner struct instance by parsing | |
22 | // the configuration options, configuring sources, reading lists | |
23 | // and setting up loggers, etc. | |
24 | func NewRunner(options *Options) (*Runner, error) { | |
25 | runner := &Runner{options: options} | |
26 | ||
27 | // Initialize the passive subdomain enumeration engine | |
28 | runner.initializePassiveEngine() | |
29 | ||
30 | // Initialize the active subdomain enumeration engine | |
31 | err := runner.initializeActiveEngine() | |
32 | if err != nil { | |
33 | return nil, err | |
34 | } | |
35 | ||
36 | return runner, nil | |
37 | } | |
38 | ||
39 | // RunEnumeration runs the subdomain enumeration flow on the targets specified | |
40 | func (r *Runner) RunEnumeration(ctx context.Context) error { | |
41 | // Check if only a single domain is sent as input. Process the domain now. | |
42 | if r.options.Domain != "" { | |
43 | return r.EnumerateSingleDomain(ctx, r.options.Domain, r.options.Output, false) | |
44 | } | |
45 | ||
46 | // If we have multiple domains as input, | |
47 | if r.options.DomainsFile != "" { | |
48 | f, err := os.Open(r.options.DomainsFile) | |
49 | if err != nil { | |
50 | return err | |
51 | } | |
52 | err = r.EnumerateMultipleDomains(ctx, f) | |
53 | f.Close() | |
54 | return err | |
55 | } | |
56 | ||
57 | // If we have STDIN input, treat it as multiple domains | |
58 | if r.options.Stdin { | |
59 | return r.EnumerateMultipleDomains(ctx, os.Stdin) | |
60 | } | |
61 | return nil | |
62 | } | |
63 | ||
64 | // EnumerateMultipleDomains enumerates subdomains for multiple domains | |
65 | // We keep enumerating subdomains for a given domain until we reach an error | |
66 | func (r *Runner) EnumerateMultipleDomains(ctx context.Context, reader io.Reader) error { | |
67 | scanner := bufio.NewScanner(reader) | |
68 | for scanner.Scan() { | |
69 | domain := scanner.Text() | |
70 | if domain == "" { | |
71 | continue | |
72 | } | |
73 | ||
74 | var err error | |
75 | // If the user has specified an output file, use that output file instead | |
76 | // of creating a new output file for each domain. Else create a new file | |
77 | // for each domain in the directory. | |
78 | if r.options.Output != "" { | |
79 | err = r.EnumerateSingleDomain(ctx, domain, r.options.Output, true) | |
80 | } else if r.options.OutputDirectory != "" { | |
81 | outputFile := path.Join(r.options.OutputDirectory, domain) | |
82 | if r.options.JSON { | |
83 | outputFile += ".json" | |
84 | } else { | |
85 | outputFile += ".txt" | |
86 | } | |
87 | err = r.EnumerateSingleDomain(ctx, domain, outputFile, false) | |
88 | } else { | |
89 | err = r.EnumerateSingleDomain(ctx, domain, "", true) | |
90 | } | |
91 | if err != nil { | |
92 | return err | |
93 | } | |
94 | } | |
95 | return nil | |
96 | } |
0 | package runner | |
1 | ||
2 | import ( | |
3 | "errors" | |
4 | ||
5 | "github.com/projectdiscovery/gologger" | |
6 | ) | |
7 | ||
8 | // validateOptions validates the configuration options passed | |
9 | func (options *Options) validateOptions() error { | |
10 | // Check if domain, list of domains, or stdin info was provided. | |
11 | // If none was provided, then return. | |
12 | if options.Domain == "" && options.DomainsFile == "" && !options.Stdin { | |
13 | return errors.New("no input list provided") | |
14 | } | |
15 | ||
16 | // Both verbose and silent flags were used | |
17 | if options.Verbose && options.Silent { | |
18 | return errors.New("both verbose and silent mode specified") | |
19 | } | |
20 | ||
21 | // Validate threads and options | |
22 | if options.Threads == 0 { | |
23 | return errors.New("threads cannot be zero") | |
24 | } | |
25 | if options.Timeout == 0 { | |
26 | return errors.New("timeout cannot be zero") | |
27 | } | |
28 | ||
29 | // Always remove wildcard with hostip | |
30 | if options.HostIP && !options.RemoveWildcard { | |
31 | return errors.New("hostip flag must be used with RemoveWildcard option") | |
32 | } | |
33 | ||
34 | return nil | |
35 | } | |
36 | ||
37 | // configureOutput configures the output on the screen | |
38 | func (options *Options) configureOutput() { | |
39 | // If the user desires verbose output, show verbose output | |
40 | if options.Verbose { | |
41 | gologger.MaxLevel = gologger.Verbose | |
42 | } | |
43 | if options.NoColor { | |
44 | gologger.UseColors = false | |
45 | } | |
46 | if options.Silent { | |
47 | gologger.MaxLevel = gologger.Silent | |
48 | } | |
49 | } |
0 | package subscraping | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "crypto/tls" | |
5 | "fmt" | |
6 | "io" | |
7 | "io/ioutil" | |
8 | "net/http" | |
9 | "net/url" | |
10 | "time" | |
11 | ||
12 | "github.com/projectdiscovery/gologger" | |
13 | ) | |
14 | ||
15 | // NewSession creates a new session object for a domain | |
16 | func NewSession(domain string, keys *Keys, timeout int) (*Session, error) { | |
17 | client := &http.Client{ | |
18 | Transport: &http.Transport{ | |
19 | MaxIdleConns: 100, | |
20 | MaxIdleConnsPerHost: 100, | |
21 | TLSClientConfig: &tls.Config{ | |
22 | InsecureSkipVerify: true, | |
23 | }, | |
24 | }, | |
25 | Timeout: time.Duration(timeout) * time.Second, | |
26 | } | |
27 | ||
28 | session := &Session{ | |
29 | Client: client, | |
30 | Keys: keys, | |
31 | } | |
32 | ||
33 | // Create a new extractor object for the current domain | |
34 | extractor, err := NewSubdomainExtractor(domain) | |
35 | session.Extractor = extractor | |
36 | ||
37 | return session, err | |
38 | } | |
39 | ||
40 | // Get makes a GET request to a URL with extended parameters | |
41 | func (s *Session) Get(ctx context.Context, getURL, cookies string, headers map[string]string) (*http.Response, error) { | |
42 | return s.HTTPRequest(ctx, http.MethodGet, getURL, cookies, headers, nil, BasicAuth{}) | |
43 | } | |
44 | ||
45 | // SimpleGet makes a simple GET request to a URL | |
46 | func (s *Session) SimpleGet(ctx context.Context, getURL string) (*http.Response, error) { | |
47 | return s.HTTPRequest(ctx, http.MethodGet, getURL, "", map[string]string{}, nil, BasicAuth{}) | |
48 | } | |
49 | ||
50 | // Post makes a POST request to a URL with extended parameters | |
51 | func (s *Session) Post(ctx context.Context, postURL, cookies string, headers map[string]string, body io.Reader) (*http.Response, error) { | |
52 | return s.HTTPRequest(ctx, http.MethodPost, postURL, cookies, headers, body, BasicAuth{}) | |
53 | } | |
54 | ||
55 | // SimplePost makes a simple POST request to a URL | |
56 | func (s *Session) SimplePost(ctx context.Context, postURL, contentType string, body io.Reader) (*http.Response, error) { | |
57 | return s.HTTPRequest(ctx, http.MethodPost, postURL, "", map[string]string{"Content-Type": contentType}, body, BasicAuth{}) | |
58 | } | |
59 | ||
60 | // HTTPRequest makes any HTTP request to a URL with extended parameters | |
61 | func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies string, headers map[string]string, body io.Reader, basicAuth BasicAuth) (*http.Response, error) { | |
62 | req, err := http.NewRequestWithContext(ctx, method, requestURL, body) | |
63 | if err != nil { | |
64 | return nil, err | |
65 | } | |
66 | ||
67 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36") | |
68 | req.Header.Set("Accept", "*/*") | |
69 | req.Header.Set("Accept-Language", "en") | |
70 | req.Header.Set("Connection", "close") | |
71 | ||
72 | if basicAuth.Username != "" || basicAuth.Password != "" { | |
73 | req.SetBasicAuth(basicAuth.Username, basicAuth.Password) | |
74 | } | |
75 | ||
76 | if cookies != "" { | |
77 | req.Header.Set("Cookie", cookies) | |
78 | } | |
79 | ||
80 | for key, value := range headers { | |
81 | req.Header.Set(key, value) | |
82 | } | |
83 | ||
84 | return httpRequestWrapper(s.Client, req) | |
85 | } | |
86 | ||
87 | // DiscardHTTPResponse discards the response content by demand | |
88 | func (s *Session) DiscardHTTPResponse(response *http.Response) { | |
89 | if response != nil { | |
90 | _, err := io.Copy(ioutil.Discard, response.Body) | |
91 | if err != nil { | |
92 | gologger.Warningf("Could not discard response body: %s\n", err) | |
93 | return | |
94 | } | |
95 | response.Body.Close() | |
96 | } | |
97 | } | |
98 | ||
99 | func httpRequestWrapper(client *http.Client, request *http.Request) (*http.Response, error) { | |
100 | resp, err := client.Do(request) | |
101 | if err != nil { | |
102 | return nil, err | |
103 | } | |
104 | ||
105 | if resp.StatusCode != http.StatusOK { | |
106 | requestURL, _ := url.QueryUnescape(request.URL.String()) | |
107 | return resp, fmt.Errorf("unexpected status code %d received from %s", resp.StatusCode, requestURL) | |
108 | } | |
109 | return resp, nil | |
110 | } |
0 | package alienvault | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "encoding/json" | |
5 | "fmt" | |
6 | ||
7 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | type alienvaultResponse struct { | |
11 | Detail string `json:"detail"` | |
12 | Error string `json:"error"` | |
13 | PassiveDNS []struct { | |
14 | Hostname string `json:"hostname"` | |
15 | } `json:"passive_dns"` | |
16 | } | |
17 | ||
18 | // Source is the passive scraping agent | |
19 | type Source struct{} | |
20 | ||
21 | // Run function returns all subdomains found with the service | |
22 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
23 | results := make(chan subscraping.Result) | |
24 | ||
25 | go func() { | |
26 | defer close(results) | |
27 | ||
28 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain)) | |
29 | if err != nil && resp == nil { | |
30 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
31 | session.DiscardHTTPResponse(resp) | |
32 | return | |
33 | } | |
34 | ||
35 | var response alienvaultResponse | |
36 | // Get the response body and decode | |
37 | err = json.NewDecoder(resp.Body).Decode(&response) | |
38 | if err != nil { | |
39 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
40 | resp.Body.Close() | |
41 | return | |
42 | } | |
43 | resp.Body.Close() | |
44 | ||
45 | if response.Error != "" { | |
46 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s, %s", response.Detail, response.Error)} | |
47 | return | |
48 | } | |
49 | ||
50 | for _, record := range response.PassiveDNS { | |
51 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} | |
52 | } | |
53 | }() | |
54 | ||
55 | return results | |
56 | } | |
57 | ||
58 | // Name returns the name of the source | |
59 | func (s *Source) Name() string { | |
60 | return "alienvault" | |
61 | } |
0 | package anubis | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | ||
6 | jsoniter "github.com/json-iterator/go" | |
7 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | // Source is the passive scraping agent | |
11 | type Source struct{} | |
12 | ||
13 | // Run function returns all subdomains found with the service | |
14 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
15 | results := make(chan subscraping.Result) | |
16 | ||
17 | go func() { | |
18 | defer close(results) | |
19 | ||
20 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://jldc.me/anubis/subdomains/%s", domain)) | |
21 | if err != nil { | |
22 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
23 | session.DiscardHTTPResponse(resp) | |
24 | return | |
25 | } | |
26 | ||
27 | var subdomains []string | |
28 | err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) | |
29 | if err != nil { | |
30 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
31 | resp.Body.Close() | |
32 | return | |
33 | } | |
34 | ||
35 | resp.Body.Close() | |
36 | ||
37 | for _, record := range subdomains { | |
38 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record} | |
39 | } | |
40 | }() | |
41 | ||
42 | return results | |
43 | } | |
44 | ||
45 | // Name returns the name of the source | |
46 | func (s *Source) Name() string { | |
47 | return "anubis" | |
48 | } |
0 | // Package archiveis is a Archiveis Scraping Engine in Golang | |
1 | package archiveis | |
2 | ||
3 | import ( | |
4 | "context" | |
5 | "fmt" | |
6 | "io/ioutil" | |
7 | "regexp" | |
8 | ||
9 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
10 | ) | |
11 | ||
12 | type agent struct { | |
13 | Results chan subscraping.Result | |
14 | Session *subscraping.Session | |
15 | } | |
16 | ||
17 | var reNext = regexp.MustCompile("<a id=\"next\" style=\".*\" href=\"(.*)\">→</a>") | |
18 | ||
19 | func (a *agent) enumerate(ctx context.Context, baseURL string) { | |
20 | select { | |
21 | case <-ctx.Done(): | |
22 | return | |
23 | default: | |
24 | } | |
25 | ||
26 | resp, err := a.Session.SimpleGet(ctx, baseURL) | |
27 | if err != nil { | |
28 | a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} | |
29 | a.Session.DiscardHTTPResponse(resp) | |
30 | return | |
31 | } | |
32 | ||
33 | // Get the response body | |
34 | body, err := ioutil.ReadAll(resp.Body) | |
35 | if err != nil { | |
36 | a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} | |
37 | resp.Body.Close() | |
38 | return | |
39 | } | |
40 | ||
41 | resp.Body.Close() | |
42 | ||
43 | src := string(body) | |
44 | for _, subdomain := range a.Session.Extractor.FindAllString(src, -1) { | |
45 | a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Subdomain, Value: subdomain} | |
46 | } | |
47 | ||
48 | match1 := reNext.FindStringSubmatch(src) | |
49 | if len(match1) > 0 { | |
50 | a.enumerate(ctx, match1[1]) | |
51 | } | |
52 | } | |
53 | ||
54 | // Source is the passive scraping agent | |
55 | type Source struct{} | |
56 | ||
57 | // Run function returns all subdomains found with the service | |
58 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
59 | results := make(chan subscraping.Result) | |
60 | ||
61 | a := agent{ | |
62 | Session: session, | |
63 | Results: results, | |
64 | } | |
65 | ||
66 | go func() { | |
67 | a.enumerate(ctx, fmt.Sprintf("http://archive.is/*.%s", domain)) | |
68 | close(a.Results) | |
69 | }() | |
70 | ||
71 | return a.Results | |
72 | } | |
73 | ||
74 | // Name returns the name of the source | |
75 | func (s *Source) Name() string { | |
76 | return "archiveis" | |
77 | } |
0 | package binaryedge | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "math" | |
6 | "net/url" | |
7 | "strconv" | |
8 | ||
9 | jsoniter "github.com/json-iterator/go" | |
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
11 | ) | |
12 | ||
13 | const ( | |
14 | v1 = "v1" | |
15 | v2 = "v2" | |
16 | baseAPIURLFmt = "https://api.binaryedge.io/%s/query/domains/subdomain/%s" | |
17 | v2SubscriptionURL = "https://api.binaryedge.io/v2/user/subscription" | |
18 | v1PageSizeParam = "pagesize" | |
19 | pageParam = "page" | |
20 | firstPage = 1 | |
21 | maxV1PageSize = 10000 | |
22 | ) | |
23 | ||
24 | type subdomainsResponse struct { | |
25 | Message string `json:"message"` | |
26 | Title string `json:"title"` | |
27 | Status interface{} `json:"status"` // string for v1, int for v2 | |
28 | Subdomains []string `json:"events"` | |
29 | Page int `json:"page"` | |
30 | PageSize int `json:"pagesize"` | |
31 | Total int `json:"total"` | |
32 | } | |
33 | ||
34 | // Source is the passive scraping agent | |
35 | type Source struct{} | |
36 | ||
37 | // Run function returns all subdomains found with the service | |
38 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
39 | results := make(chan subscraping.Result) | |
40 | ||
41 | go func() { | |
42 | defer close(results) | |
43 | ||
44 | if session.Keys.Binaryedge == "" { | |
45 | return | |
46 | } | |
47 | ||
48 | var baseURL string | |
49 | ||
50 | authHeader := map[string]string{"X-Key": session.Keys.Binaryedge} | |
51 | ||
52 | if isV2(ctx, session, authHeader) { | |
53 | baseURL = fmt.Sprintf(baseAPIURLFmt, v2, domain) | |
54 | } else { | |
55 | authHeader = map[string]string{"X-Token": session.Keys.Binaryedge} | |
56 | v1URLWithPageSize, err := addURLParam(fmt.Sprintf(baseAPIURLFmt, v1, domain), v1PageSizeParam, strconv.Itoa(maxV1PageSize)) | |
57 | if err != nil { | |
58 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
59 | return | |
60 | } | |
61 | baseURL = v1URLWithPageSize.String() | |
62 | } | |
63 | ||
64 | if baseURL == "" { | |
65 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("can't get API URL")} | |
66 | return | |
67 | } | |
68 | ||
69 | s.enumerate(ctx, session, baseURL, firstPage, authHeader, results) | |
70 | }() | |
71 | ||
72 | return results | |
73 | } | |
74 | ||
75 | func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, baseURL string, page int, authHeader map[string]string, results chan subscraping.Result) { | |
76 | pageURL, err := addURLParam(baseURL, pageParam, strconv.Itoa(page)) | |
77 | if err != nil { | |
78 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
79 | return | |
80 | } | |
81 | ||
82 | resp, err := session.Get(ctx, pageURL.String(), "", authHeader) | |
83 | if err != nil && resp == nil { | |
84 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
85 | session.DiscardHTTPResponse(resp) | |
86 | return | |
87 | } | |
88 | ||
89 | var response subdomainsResponse | |
90 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
91 | if err != nil { | |
92 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
93 | resp.Body.Close() | |
94 | return | |
95 | } | |
96 | ||
97 | // Check error messages | |
98 | if response.Message != "" && response.Status != nil { | |
99 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf(response.Message)} | |
100 | } | |
101 | ||
102 | resp.Body.Close() | |
103 | ||
104 | for _, subdomain := range response.Subdomains { | |
105 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
106 | } | |
107 | ||
108 | totalPages := int(math.Ceil(float64(response.Total) / float64(response.PageSize))) | |
109 | nextPage := response.Page + 1 | |
110 | for currentPage := nextPage; currentPage <= totalPages; currentPage++ { | |
111 | s.enumerate(ctx, session, baseURL, currentPage, authHeader, results) | |
112 | } | |
113 | } | |
114 | ||
115 | // Name returns the name of the source | |
116 | func (s *Source) Name() string { | |
117 | return "binaryedge" | |
118 | } | |
119 | ||
120 | func isV2(ctx context.Context, session *subscraping.Session, authHeader map[string]string) bool { | |
121 | resp, err := session.Get(ctx, v2SubscriptionURL, "", authHeader) | |
122 | if err != nil { | |
123 | session.DiscardHTTPResponse(resp) | |
124 | return false | |
125 | } | |
126 | ||
127 | resp.Body.Close() | |
128 | ||
129 | return true | |
130 | } | |
131 | ||
132 | func addURLParam(targetURL, name, value string) (*url.URL, error) { | |
133 | u, err := url.Parse(targetURL) | |
134 | if err != nil { | |
135 | return u, err | |
136 | } | |
137 | q, _ := url.ParseQuery(u.RawQuery) | |
138 | q.Add(name, value) | |
139 | u.RawQuery = q.Encode() | |
140 | ||
141 | return u, nil | |
142 | } |
0 | // Package bufferover is a bufferover Scraping Engine in Golang | |
1 | package bufferover | |
2 | ||
3 | import ( | |
4 | "context" | |
5 | "fmt" | |
6 | "strings" | |
7 | ||
8 | jsoniter "github.com/json-iterator/go" | |
9 | ||
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
11 | ) | |
12 | ||
13 | type response struct { | |
14 | Meta struct { | |
15 | Errors []string `json:"Errors"` | |
16 | } `json:"Meta"` | |
17 | FDNSA []string `json:"FDNS_A"` | |
18 | RDNS []string `json:"RDNS"` | |
19 | Results []string `json:"Results"` | |
20 | } | |
21 | ||
22 | // Source is the passive scraping agent | |
23 | type Source struct{} | |
24 | ||
25 | // Run function returns all subdomains found with the service | |
26 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
27 | results := make(chan subscraping.Result) | |
28 | ||
29 | go func() { | |
30 | // Run enumeration on subdomain dataset for historical SONAR datasets | |
31 | s.getData(ctx, fmt.Sprintf("https://dns.bufferover.run/dns?q=.%s", domain), session, results) | |
32 | s.getData(ctx, fmt.Sprintf("https://tls.bufferover.run/dns?q=.%s", domain), session, results) | |
33 | ||
34 | close(results) | |
35 | }() | |
36 | ||
37 | return results | |
38 | } | |
39 | ||
40 | func (s *Source) getData(ctx context.Context, sourceURL string, session *subscraping.Session, results chan subscraping.Result) { | |
41 | resp, err := session.SimpleGet(ctx, sourceURL) | |
42 | if err != nil && resp == nil { | |
43 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
44 | session.DiscardHTTPResponse(resp) | |
45 | return | |
46 | } | |
47 | ||
48 | var bufforesponse response | |
49 | err = jsoniter.NewDecoder(resp.Body).Decode(&bufforesponse) | |
50 | if err != nil { | |
51 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
52 | resp.Body.Close() | |
53 | return | |
54 | } | |
55 | ||
56 | resp.Body.Close() | |
57 | ||
58 | metaErrors := bufforesponse.Meta.Errors | |
59 | ||
60 | if len(metaErrors) > 0 { | |
61 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", strings.Join(metaErrors, ", "))} | |
62 | return | |
63 | } | |
64 | ||
65 | var subdomains []string | |
66 | ||
67 | if len(bufforesponse.FDNSA) > 0 { | |
68 | subdomains = bufforesponse.FDNSA | |
69 | subdomains = append(subdomains, bufforesponse.RDNS...) | |
70 | } else if len(bufforesponse.Results) > 0 { | |
71 | subdomains = bufforesponse.Results | |
72 | } | |
73 | ||
74 | for _, subdomain := range subdomains { | |
75 | for _, value := range session.Extractor.FindAllString(subdomain, -1) { | |
76 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} | |
77 | } | |
78 | } | |
79 | } | |
80 | ||
81 | // Name returns the name of the source | |
82 | func (s *Source) Name() string { | |
83 | return "bufferover" | |
84 | } |
0 | package cebaidu | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | ||
6 | jsoniter "github.com/json-iterator/go" | |
7 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | // Source is the passive scraping agent | |
11 | type Source struct{} | |
12 | ||
13 | type domain struct { | |
14 | Domain string `json:"domain"` | |
15 | } | |
16 | ||
17 | type cebaiduResponse struct { | |
18 | Code int64 `json:"code"` | |
19 | Message string `json:"message"` | |
20 | Data []domain `json:"data"` | |
21 | } | |
22 | ||
23 | // Run function returns all subdomains found with the service | |
24 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
25 | results := make(chan subscraping.Result) | |
26 | ||
27 | go func() { | |
28 | defer close(results) | |
29 | ||
30 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://ce.baidu.com/index/getRelatedSites?site_address=%s", domain)) | |
31 | if err != nil { | |
32 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
33 | session.DiscardHTTPResponse(resp) | |
34 | return | |
35 | } | |
36 | ||
37 | var response cebaiduResponse | |
38 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
39 | if err != nil { | |
40 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
41 | resp.Body.Close() | |
42 | return | |
43 | } | |
44 | resp.Body.Close() | |
45 | ||
46 | if response.Code > 0 { | |
47 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%d, %s", response.Code, response.Message)} | |
48 | return | |
49 | } | |
50 | ||
51 | for _, result := range response.Data { | |
52 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Domain} | |
53 | } | |
54 | }() | |
55 | ||
56 | return results | |
57 | } | |
58 | ||
59 | // Name returns the name of the source | |
60 | func (s *Source) Name() string { | |
61 | return "cebaidu" | |
62 | } |
0 | package censys | |
1 | ||
2 | import ( | |
3 | "bytes" | |
4 | "context" | |
5 | "strconv" | |
6 | ||
7 | jsoniter "github.com/json-iterator/go" | |
8 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
9 | ) | |
10 | ||
11 | const maxCensysPages = 10 | |
12 | ||
13 | type resultsq struct { | |
14 | Data []string `json:"parsed.extensions.subject_alt_name.dns_names"` | |
15 | Data1 []string `json:"parsed.names"` | |
16 | } | |
17 | ||
18 | type response struct { | |
19 | Results []resultsq `json:"results"` | |
20 | Metadata struct { | |
21 | Pages int `json:"pages"` | |
22 | } `json:"metadata"` | |
23 | } | |
24 | ||
25 | // Source is the passive scraping agent | |
26 | type Source struct{} | |
27 | ||
28 | // Run function returns all subdomains found with the service | |
29 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
30 | results := make(chan subscraping.Result) | |
31 | ||
32 | go func() { | |
33 | defer close(results) | |
34 | ||
35 | if session.Keys.CensysToken == "" || session.Keys.CensysSecret == "" { | |
36 | return | |
37 | } | |
38 | ||
39 | currentPage := 1 | |
40 | for { | |
41 | var request = []byte(`{"query":"` + domain + `", "page":` + strconv.Itoa(currentPage) + `, "fields":["parsed.names","parsed.extensions.subject_alt_name.dns_names"], "flatten":true}`) | |
42 | ||
43 | resp, err := session.HTTPRequest( | |
44 | ctx, | |
45 | "POST", | |
46 | "https://www.censys.io/api/v1/search/certificates", | |
47 | "", | |
48 | map[string]string{"Content-Type": "application/json", "Accept": "application/json"}, | |
49 | bytes.NewReader(request), | |
50 | subscraping.BasicAuth{Username: session.Keys.CensysToken, Password: session.Keys.CensysSecret}, | |
51 | ) | |
52 | ||
53 | if err != nil { | |
54 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
55 | session.DiscardHTTPResponse(resp) | |
56 | return | |
57 | } | |
58 | ||
59 | var censysResponse response | |
60 | err = jsoniter.NewDecoder(resp.Body).Decode(&censysResponse) | |
61 | if err != nil { | |
62 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
63 | resp.Body.Close() | |
64 | return | |
65 | } | |
66 | ||
67 | resp.Body.Close() | |
68 | ||
69 | // Exit the censys enumeration if max pages is reached | |
70 | if currentPage >= censysResponse.Metadata.Pages || currentPage >= maxCensysPages { | |
71 | break | |
72 | } | |
73 | ||
74 | for _, res := range censysResponse.Results { | |
75 | for _, part := range res.Data { | |
76 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: part} | |
77 | } | |
78 | for _, part := range res.Data1 { | |
79 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: part} | |
80 | } | |
81 | } | |
82 | ||
83 | currentPage++ | |
84 | } | |
85 | }() | |
86 | ||
87 | return results | |
88 | } | |
89 | ||
90 | // Name returns the name of the source | |
91 | func (s *Source) Name() string { | |
92 | return "censys" | |
93 | } |
0 | package certspotter | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | ||
6 | jsoniter "github.com/json-iterator/go" | |
7 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | type certspotterObject struct { | |
11 | ID string `json:"id"` | |
12 | DNSNames []string `json:"dns_names"` | |
13 | } | |
14 | ||
15 | // Source is the passive scraping agent | |
16 | type Source struct{} | |
17 | ||
18 | // Run function returns all subdomains found with the service | |
19 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
20 | results := make(chan subscraping.Result) | |
21 | ||
22 | go func() { | |
23 | defer close(results) | |
24 | ||
25 | if session.Keys.Certspotter == "" { | |
26 | return | |
27 | } | |
28 | ||
29 | resp, err := session.Get(ctx, fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain), "", map[string]string{"Authorization": "Bearer " + session.Keys.Certspotter}) | |
30 | if err != nil { | |
31 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
32 | session.DiscardHTTPResponse(resp) | |
33 | return | |
34 | } | |
35 | ||
36 | var response []certspotterObject | |
37 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
38 | if err != nil { | |
39 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
40 | resp.Body.Close() | |
41 | return | |
42 | } | |
43 | resp.Body.Close() | |
44 | ||
45 | for _, cert := range response { | |
46 | for _, subdomain := range cert.DNSNames { | |
47 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
48 | } | |
49 | } | |
50 | ||
51 | // if the number of responses is zero, close the channel and return. | |
52 | if len(response) == 0 { | |
53 | return | |
54 | } | |
55 | ||
56 | id := response[len(response)-1].ID | |
57 | for { | |
58 | reqURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, id) | |
59 | ||
60 | resp, err := session.Get(ctx, reqURL, "", map[string]string{"Authorization": "Bearer " + session.Keys.Certspotter}) | |
61 | if err != nil { | |
62 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
63 | return | |
64 | } | |
65 | ||
66 | var response []certspotterObject | |
67 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
68 | if err != nil { | |
69 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
70 | resp.Body.Close() | |
71 | return | |
72 | } | |
73 | resp.Body.Close() | |
74 | ||
75 | if len(response) == 0 { | |
76 | break | |
77 | } | |
78 | ||
79 | for _, cert := range response { | |
80 | for _, subdomain := range cert.DNSNames { | |
81 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
82 | } | |
83 | } | |
84 | ||
85 | id = response[len(response)-1].ID | |
86 | } | |
87 | }() | |
88 | ||
89 | return results | |
90 | } | |
91 | ||
92 | // Name returns the name of the source | |
93 | func (s *Source) Name() string { | |
94 | return "certspotter" | |
95 | } |
0 | package certspotterold | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "net/http" | |
6 | ||
7 | jsoniter "github.com/json-iterator/go" | |
8 | ||
9 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
10 | ) | |
11 | ||
12 | type errorResponse struct { | |
13 | Code string `json:"code"` | |
14 | Message string `json:"Message"` | |
15 | } | |
16 | ||
17 | type subdomain struct { | |
18 | DNSNames []string `json:"dns_names"` | |
19 | } | |
20 | ||
21 | // Source is the passive scraping agent | |
22 | type Source struct{} | |
23 | ||
24 | // Run function returns all subdomains found with the service | |
25 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
26 | results := make(chan subscraping.Result) | |
27 | ||
28 | go func() { | |
29 | defer close(results) | |
30 | ||
31 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://certspotter.com/api/v0/certs?domain=%s", domain)) | |
32 | if err != nil && resp == nil { | |
33 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
34 | session.DiscardHTTPResponse(resp) | |
35 | return | |
36 | } | |
37 | ||
38 | if resp.StatusCode != http.StatusOK { | |
39 | var errResponse errorResponse | |
40 | err = jsoniter.NewDecoder(resp.Body).Decode(&errResponse) | |
41 | if err != nil { | |
42 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
43 | resp.Body.Close() | |
44 | return | |
45 | } | |
46 | ||
47 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s: %s", errResponse.Code, errResponse.Message)} | |
48 | resp.Body.Close() | |
49 | return | |
50 | } | |
51 | ||
52 | var subdomains []subdomain | |
53 | err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) | |
54 | if err != nil { | |
55 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
56 | resp.Body.Close() | |
57 | return | |
58 | } | |
59 | ||
60 | resp.Body.Close() | |
61 | ||
62 | for _, subdomain := range subdomains { | |
63 | for _, dnsname := range subdomain.DNSNames { | |
64 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: dnsname} | |
65 | } | |
66 | } | |
67 | }() | |
68 | return results | |
69 | } | |
70 | ||
71 | // Name returns the name of the source | |
72 | func (s *Source) Name() string { | |
73 | return "certspotterold" | |
74 | } |
0 | package chaos | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | ||
6 | "github.com/projectdiscovery/chaos-client/pkg/chaos" | |
7 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | // Source is the passive scraping agent | |
11 | type Source struct{} | |
12 | ||
13 | // Run function returns all subdomains found with the service | |
14 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
15 | results := make(chan subscraping.Result) | |
16 | ||
17 | go func() { | |
18 | defer close(results) | |
19 | ||
20 | if session.Keys.Chaos == "" { | |
21 | return | |
22 | } | |
23 | ||
24 | chaosClient := chaos.New(session.Keys.Chaos) | |
25 | for result := range chaosClient.GetSubdomains(&chaos.SubdomainsRequest{ | |
26 | Domain: domain, | |
27 | }) { | |
28 | if result.Error != nil { | |
29 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: result.Error} | |
30 | break | |
31 | } | |
32 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", result.Subdomain, domain)} | |
33 | } | |
34 | }() | |
35 | ||
36 | return results | |
37 | } | |
38 | ||
39 | // Name returns the name of the source | |
40 | func (s *Source) Name() string { | |
41 | return "chaos" | |
42 | } |
0 | package commoncrawl | |
1 | ||
2 | import ( | |
3 | "bufio" | |
4 | "context" | |
5 | "fmt" | |
6 | "net/url" | |
7 | "strings" | |
8 | ||
9 | jsoniter "github.com/json-iterator/go" | |
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
11 | ) | |
12 | ||
13 | const indexURL = "https://index.commoncrawl.org/collinfo.json" | |
14 | ||
15 | type indexResponse struct { | |
16 | ID string `json:"id"` | |
17 | APIURL string `json:"cdx-api"` | |
18 | } | |
19 | ||
20 | // Source is the passive scraping agent | |
21 | type Source struct{} | |
22 | ||
23 | var years = [...]string{"2020", "2019", "2018", "2017"} | |
24 | ||
25 | // Run function returns all subdomains found with the service | |
26 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
27 | results := make(chan subscraping.Result) | |
28 | ||
29 | go func() { | |
30 | defer close(results) | |
31 | ||
32 | resp, err := session.SimpleGet(ctx, indexURL) | |
33 | if err != nil { | |
34 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
35 | session.DiscardHTTPResponse(resp) | |
36 | return | |
37 | } | |
38 | ||
39 | var indexes []indexResponse | |
40 | err = jsoniter.NewDecoder(resp.Body).Decode(&indexes) | |
41 | if err != nil { | |
42 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
43 | resp.Body.Close() | |
44 | return | |
45 | } | |
46 | resp.Body.Close() | |
47 | ||
48 | searchIndexes := make(map[string]string) | |
49 | for _, year := range years { | |
50 | for _, index := range indexes { | |
51 | if strings.Contains(index.ID, year) { | |
52 | if _, ok := searchIndexes[year]; !ok { | |
53 | searchIndexes[year] = index.APIURL | |
54 | break | |
55 | } | |
56 | } | |
57 | } | |
58 | } | |
59 | ||
60 | for _, apiURL := range searchIndexes { | |
61 | further := s.getSubdomains(ctx, apiURL, domain, session, results) | |
62 | if !further { | |
63 | break | |
64 | } | |
65 | } | |
66 | }() | |
67 | ||
68 | return results | |
69 | } | |
70 | ||
71 | // Name returns the name of the source | |
72 | func (s *Source) Name() string { | |
73 | return "commoncrawl" | |
74 | } | |
75 | ||
76 | func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, session *subscraping.Session, results chan subscraping.Result) bool { | |
77 | for { | |
78 | select { | |
79 | case <-ctx.Done(): | |
80 | return false | |
81 | default: | |
82 | var headers = map[string]string{"Host": "index.commoncrawl.org"} | |
83 | resp, err := session.Get(ctx, fmt.Sprintf("%s?url=*.%s", searchURL, domain), "", headers) | |
84 | if err != nil { | |
85 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
86 | session.DiscardHTTPResponse(resp) | |
87 | return false | |
88 | } | |
89 | ||
90 | scanner := bufio.NewScanner(resp.Body) | |
91 | for scanner.Scan() { | |
92 | line := scanner.Text() | |
93 | if line == "" { | |
94 | continue | |
95 | } | |
96 | line, _ = url.QueryUnescape(line) | |
97 | subdomain := session.Extractor.FindString(line) | |
98 | if subdomain != "" { | |
99 | // fix for triple encoded URL | |
100 | subdomain = strings.ToLower(subdomain) | |
101 | subdomain = strings.TrimPrefix(subdomain, "25") | |
102 | subdomain = strings.TrimPrefix(subdomain, "2f") | |
103 | ||
104 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
105 | } | |
106 | } | |
107 | resp.Body.Close() | |
108 | return true | |
109 | } | |
110 | } | |
111 | } |
0 | package crtsh | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "database/sql" | |
5 | "fmt" | |
6 | ||
7 | jsoniter "github.com/json-iterator/go" | |
8 | ||
9 | // postgres driver | |
10 | _ "github.com/lib/pq" | |
11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
12 | ) | |
13 | ||
14 | type subdomain struct { | |
15 | ID int `json:"id"` | |
16 | NameValue string `json:"name_value"` | |
17 | } | |
18 | ||
19 | // Source is the passive scraping agent | |
20 | type Source struct{} | |
21 | ||
22 | // Run function returns all subdomains found with the service | |
23 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
24 | results := make(chan subscraping.Result) | |
25 | ||
26 | go func() { | |
27 | defer close(results) | |
28 | found := s.getSubdomainsFromSQL(domain, results) | |
29 | if found { | |
30 | return | |
31 | } | |
32 | _ = s.getSubdomainsFromHTTP(ctx, domain, session, results) | |
33 | }() | |
34 | ||
35 | return results | |
36 | } | |
37 | ||
38 | func (s *Source) getSubdomainsFromSQL(domain string, results chan subscraping.Result) bool { | |
39 | db, err := sql.Open("postgres", "host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes") | |
40 | if err != nil { | |
41 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
42 | return false | |
43 | } | |
44 | ||
45 | pattern := "%." + domain | |
46 | query := `SELECT DISTINCT ci.NAME_VALUE as domain FROM certificate_identity ci | |
47 | WHERE reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower($1)) | |
48 | ORDER BY ci.NAME_VALUE` | |
49 | rows, err := db.Query(query, pattern) | |
50 | if err != nil { | |
51 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
52 | return false | |
53 | } | |
54 | if err := rows.Err(); err != nil { | |
55 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
56 | return false | |
57 | } | |
58 | ||
59 | var data string | |
60 | // Parse all the rows getting subdomains | |
61 | for rows.Next() { | |
62 | err := rows.Scan(&data) | |
63 | if err != nil { | |
64 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
65 | return false | |
66 | } | |
67 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data} | |
68 | } | |
69 | return true | |
70 | } | |
71 | ||
72 | func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) bool { | |
73 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain)) | |
74 | if err != nil { | |
75 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
76 | session.DiscardHTTPResponse(resp) | |
77 | return false | |
78 | } | |
79 | ||
80 | var subdomains []subdomain | |
81 | err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) | |
82 | if err != nil { | |
83 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
84 | resp.Body.Close() | |
85 | return false | |
86 | } | |
87 | ||
88 | resp.Body.Close() | |
89 | ||
90 | for _, subdomain := range subdomains { | |
91 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.NameValue} | |
92 | } | |
93 | ||
94 | return true | |
95 | } | |
96 | ||
97 | // Name returns the name of the source | |
98 | func (s *Source) Name() string { | |
99 | return "crtsh" | |
100 | } |
0 | package dnsdb | |
1 | ||
2 | import ( | |
3 | "bufio" | |
4 | "bytes" | |
5 | "context" | |
6 | "fmt" | |
7 | "strings" | |
8 | ||
9 | jsoniter "github.com/json-iterator/go" | |
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
11 | ) | |
12 | ||
13 | type dnsdbResponse struct { | |
14 | Name string `json:"rrname"` | |
15 | } | |
16 | ||
17 | // Source is the passive scraping agent | |
18 | type Source struct{} | |
19 | ||
20 | // Run function returns all subdomains found with the service | |
21 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
22 | results := make(chan subscraping.Result) | |
23 | ||
24 | go func() { | |
25 | defer close(results) | |
26 | ||
27 | if session.Keys.DNSDB == "" { | |
28 | return | |
29 | } | |
30 | ||
31 | headers := map[string]string{ | |
32 | "X-API-KEY": session.Keys.DNSDB, | |
33 | "Accept": "application/json", | |
34 | "Content-Type": "application/json", | |
35 | } | |
36 | ||
37 | resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdb.info/lookup/rrset/name/*.%s?limit=1000000000000", domain), "", headers) | |
38 | if err != nil { | |
39 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
40 | session.DiscardHTTPResponse(resp) | |
41 | return | |
42 | } | |
43 | ||
44 | scanner := bufio.NewScanner(resp.Body) | |
45 | for scanner.Scan() { | |
46 | line := scanner.Text() | |
47 | if line == "" { | |
48 | continue | |
49 | } | |
50 | var response dnsdbResponse | |
51 | err = jsoniter.NewDecoder(bytes.NewBufferString(line)).Decode(&response) | |
52 | if err != nil { | |
53 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
54 | return | |
55 | } | |
56 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(response.Name, ".")} | |
57 | } | |
58 | resp.Body.Close() | |
59 | }() | |
60 | return results | |
61 | } | |
62 | ||
63 | // Name returns the name of the source | |
64 | func (s *Source) Name() string { | |
65 | return "DNSDB" | |
66 | } |
0 | package dnsdumpster | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "io/ioutil" | |
6 | "net/url" | |
7 | "regexp" | |
8 | "strings" | |
9 | ||
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
11 | ) | |
12 | ||
13 | // CSRFSubMatchLength CSRF regex submatch length | |
14 | const CSRFSubMatchLength = 2 | |
15 | ||
16 | var re = regexp.MustCompile("<input type=\"hidden\" name=\"csrfmiddlewaretoken\" value=\"(.*)\">") | |
17 | ||
18 | // getCSRFToken gets the CSRF Token from the page | |
19 | func getCSRFToken(page string) string { | |
20 | if subs := re.FindStringSubmatch(page); len(subs) == CSRFSubMatchLength { | |
21 | return strings.TrimSpace(subs[1]) | |
22 | } | |
23 | return "" | |
24 | } | |
25 | ||
26 | // postForm posts a form for a domain and returns the response | |
27 | func postForm(ctx context.Context, session *subscraping.Session, token, domain string) (string, error) { | |
28 | params := url.Values{ | |
29 | "csrfmiddlewaretoken": {token}, | |
30 | "targetip": {domain}, | |
31 | } | |
32 | ||
33 | resp, err := session.HTTPRequest( | |
34 | ctx, | |
35 | "POST", | |
36 | "https://dnsdumpster.com/", | |
37 | fmt.Sprintf("csrftoken=%s; Domain=dnsdumpster.com", token), | |
38 | map[string]string{ | |
39 | "Content-Type": "application/x-www-form-urlencoded", | |
40 | "Referer": "https://dnsdumpster.com", | |
41 | "X-CSRF-Token": token, | |
42 | }, | |
43 | strings.NewReader(params.Encode()), | |
44 | subscraping.BasicAuth{}, | |
45 | ) | |
46 | ||
47 | if err != nil { | |
48 | session.DiscardHTTPResponse(resp) | |
49 | return "", err | |
50 | } | |
51 | ||
52 | // Now, grab the entire page | |
53 | in, err := ioutil.ReadAll(resp.Body) | |
54 | resp.Body.Close() | |
55 | return string(in), err | |
56 | } | |
57 | ||
58 | // Source is the passive scraping agent | |
59 | type Source struct{} | |
60 | ||
61 | // Run function returns all subdomains found with the service | |
62 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
63 | results := make(chan subscraping.Result) | |
64 | ||
65 | go func() { | |
66 | defer close(results) | |
67 | ||
68 | resp, err := session.SimpleGet(ctx, "https://dnsdumpster.com/") | |
69 | if err != nil { | |
70 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
71 | session.DiscardHTTPResponse(resp) | |
72 | return | |
73 | } | |
74 | ||
75 | body, err := ioutil.ReadAll(resp.Body) | |
76 | if err != nil { | |
77 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
78 | resp.Body.Close() | |
79 | return | |
80 | } | |
81 | resp.Body.Close() | |
82 | ||
83 | csrfToken := getCSRFToken(string(body)) | |
84 | data, err := postForm(ctx, session, csrfToken, domain) | |
85 | if err != nil { | |
86 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
87 | return | |
88 | } | |
89 | ||
90 | for _, subdomain := range session.Extractor.FindAllString(data, -1) { | |
91 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
92 | } | |
93 | }() | |
94 | ||
95 | return results | |
96 | } | |
97 | ||
98 | // Name returns the name of the source | |
99 | func (s *Source) Name() string { | |
100 | return "dnsdumpster" | |
101 | } |
0 | // Package github GitHub search package | |
1 | // Based on gwen001's https://github.com/gwen001/github-search github-subdomains | |
2 | package github | |
3 | ||
4 | import ( | |
5 | "bufio" | |
6 | "context" | |
7 | "fmt" | |
8 | "net/http" | |
9 | "net/url" | |
10 | "regexp" | |
11 | "strconv" | |
12 | "strings" | |
13 | "time" | |
14 | ||
15 | jsoniter "github.com/json-iterator/go" | |
16 | ||
17 | "github.com/projectdiscovery/gologger" | |
18 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
19 | "github.com/tomnomnom/linkheader" | |
20 | ) | |
21 | ||
22 | type textMatch struct { | |
23 | Fragment string `json:"fragment"` | |
24 | } | |
25 | ||
26 | type item struct { | |
27 | Name string `json:"name"` | |
28 | HTMLURL string `json:"html_url"` | |
29 | TextMatches []textMatch `json:"text_matches"` | |
30 | } | |
31 | ||
32 | type response struct { | |
33 | TotalCount int `json:"total_count"` | |
34 | Items []item `json:"items"` | |
35 | } | |
36 | ||
37 | // Source is the passive scraping agent | |
38 | type Source struct{} | |
39 | ||
40 | // Run function returns all subdomains found with the service | |
41 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
42 | results := make(chan subscraping.Result) | |
43 | ||
44 | go func() { | |
45 | defer close(results) | |
46 | ||
47 | if len(session.Keys.GitHub) == 0 { | |
48 | return | |
49 | } | |
50 | ||
51 | tokens := NewTokenManager(session.Keys.GitHub) | |
52 | ||
53 | searchURL := fmt.Sprintf("https://api.github.com/search/code?per_page=100&q=%s&sort=created&order=asc", domain) | |
54 | s.enumerate(ctx, searchURL, domainRegexp(domain), tokens, session, results) | |
55 | }() | |
56 | ||
57 | return results | |
58 | } | |
59 | ||
60 | func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *regexp.Regexp, tokens *Tokens, session *subscraping.Session, results chan subscraping.Result) { | |
61 | select { | |
62 | case <-ctx.Done(): | |
63 | return | |
64 | default: | |
65 | } | |
66 | ||
67 | token := tokens.Get() | |
68 | ||
69 | if token.RetryAfter > 0 { | |
70 | if len(tokens.pool) == 1 { | |
71 | gologger.Verbosef("GitHub Search request rate limit exceeded, waiting for %d seconds before retry... \n", s.Name(), token.RetryAfter) | |
72 | time.Sleep(time.Duration(token.RetryAfter) * time.Second) | |
73 | } else { | |
74 | token = tokens.Get() | |
75 | } | |
76 | } | |
77 | ||
78 | headers := map[string]string{"Accept": "application/vnd.github.v3.text-match+json", "Authorization": "token " + token.Hash} | |
79 | ||
80 | // Initial request to GitHub search | |
81 | resp, err := session.Get(ctx, searchURL, "", headers) | |
82 | isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden | |
83 | if err != nil && !isForbidden { | |
84 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
85 | session.DiscardHTTPResponse(resp) | |
86 | return | |
87 | } | |
88 | ||
89 | // Retry enumerarion after Retry-After seconds on rate limit abuse detected | |
90 | ratelimitRemaining, _ := strconv.ParseInt(resp.Header.Get("X-Ratelimit-Remaining"), 10, 64) | |
91 | if isForbidden && ratelimitRemaining == 0 { | |
92 | retryAfterSeconds, _ := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64) | |
93 | tokens.setCurrentTokenExceeded(retryAfterSeconds) | |
94 | resp.Body.Close() | |
95 | ||
96 | s.enumerate(ctx, searchURL, domainRegexp, tokens, session, results) | |
97 | } | |
98 | ||
99 | var data response | |
100 | ||
101 | // Marshall json response | |
102 | err = jsoniter.NewDecoder(resp.Body).Decode(&data) | |
103 | if err != nil { | |
104 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
105 | resp.Body.Close() | |
106 | return | |
107 | } | |
108 | ||
109 | resp.Body.Close() | |
110 | ||
111 | err = proccesItems(ctx, data.Items, domainRegexp, s.Name(), session, results) | |
112 | if err != nil { | |
113 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
114 | return | |
115 | } | |
116 | ||
117 | // Links header, first, next, last... | |
118 | linksHeader := linkheader.Parse(resp.Header.Get("Link")) | |
119 | // Process the next link recursively | |
120 | for _, link := range linksHeader { | |
121 | if link.Rel == "next" { | |
122 | nextURL, err := url.QueryUnescape(link.URL) | |
123 | if err != nil { | |
124 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
125 | return | |
126 | } | |
127 | s.enumerate(ctx, nextURL, domainRegexp, tokens, session, results) | |
128 | } | |
129 | } | |
130 | } | |
131 | ||
132 | // proccesItems procceses github response items | |
133 | func proccesItems(ctx context.Context, items []item, domainRegexp *regexp.Regexp, name string, session *subscraping.Session, results chan subscraping.Result) error { | |
134 | for _, item := range items { | |
135 | // find subdomains in code | |
136 | resp, err := session.SimpleGet(ctx, rawURL(item.HTMLURL)) | |
137 | if err != nil { | |
138 | if resp != nil && resp.StatusCode != http.StatusNotFound { | |
139 | session.DiscardHTTPResponse(resp) | |
140 | } | |
141 | return err | |
142 | } | |
143 | ||
144 | if resp.StatusCode == http.StatusOK { | |
145 | scanner := bufio.NewScanner(resp.Body) | |
146 | for scanner.Scan() { | |
147 | line := scanner.Text() | |
148 | if line == "" { | |
149 | continue | |
150 | } | |
151 | for _, subdomain := range domainRegexp.FindAllString(normalizeContent(line), -1) { | |
152 | results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain} | |
153 | } | |
154 | } | |
155 | resp.Body.Close() | |
156 | } | |
157 | ||
158 | // find subdomains in text matches | |
159 | for _, textMatch := range item.TextMatches { | |
160 | for _, subdomain := range domainRegexp.FindAllString(normalizeContent(textMatch.Fragment), -1) { | |
161 | results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain} | |
162 | } | |
163 | } | |
164 | } | |
165 | return nil | |
166 | } | |
167 | ||
168 | // Normalize content before matching, query unescape, remove tabs and new line chars | |
169 | func normalizeContent(content string) string { | |
170 | normalizedContent, _ := url.QueryUnescape(content) | |
171 | normalizedContent = strings.ReplaceAll(normalizedContent, "\\t", "") | |
172 | normalizedContent = strings.ReplaceAll(normalizedContent, "\\n", "") | |
173 | return normalizedContent | |
174 | } | |
175 | ||
176 | // Raw URL to get the files code and match for subdomains | |
177 | func rawURL(htmlURL string) string { | |
178 | domain := strings.ReplaceAll(htmlURL, "https://github.com/", "https://raw.githubusercontent.com/") | |
179 | return strings.ReplaceAll(domain, "/blob/", "/") | |
180 | } | |
181 | ||
182 | // DomainRegexp regular expression to match subdomains in github files code | |
183 | func domainRegexp(domain string) *regexp.Regexp { | |
184 | rdomain := strings.ReplaceAll(domain, ".", "\\.") | |
185 | return regexp.MustCompile("(\\w+[.])*" + rdomain) | |
186 | } | |
187 | ||
188 | // Name returns the name of the source | |
189 | func (s *Source) Name() string { | |
190 | return "github" | |
191 | } |
0 | package github | |
1 | ||
2 | import "time" | |
3 | ||
4 | // Token struct | |
5 | type Token struct { | |
6 | Hash string | |
7 | RetryAfter int64 | |
8 | ExceededTime time.Time | |
9 | } | |
10 | ||
11 | // Tokens is the internal struct to manage the current token | |
12 | // and the pool | |
13 | type Tokens struct { | |
14 | current int | |
15 | pool []Token | |
16 | } | |
17 | ||
18 | // NewTokenManager initialize the tokens pool | |
19 | func NewTokenManager(keys []string) *Tokens { | |
20 | pool := []Token{} | |
21 | for _, key := range keys { | |
22 | t := Token{Hash: key, ExceededTime: time.Time{}, RetryAfter: 0} | |
23 | pool = append(pool, t) | |
24 | } | |
25 | ||
26 | return &Tokens{ | |
27 | current: 0, | |
28 | pool: pool, | |
29 | } | |
30 | } | |
31 | ||
32 | func (r *Tokens) setCurrentTokenExceeded(retryAfter int64) { | |
33 | if r.current >= len(r.pool) { | |
34 | r.current %= len(r.pool) | |
35 | } | |
36 | if r.pool[r.current].RetryAfter == 0 { | |
37 | r.pool[r.current].ExceededTime = time.Now() | |
38 | r.pool[r.current].RetryAfter = retryAfter | |
39 | } | |
40 | } | |
41 | ||
42 | // Get returns a new token from the token pool | |
43 | func (r *Tokens) Get() *Token { | |
44 | resetExceededTokens(r) | |
45 | ||
46 | if r.current >= len(r.pool) { | |
47 | r.current %= len(r.pool) | |
48 | } | |
49 | ||
50 | result := &r.pool[r.current] | |
51 | r.current++ | |
52 | ||
53 | return result | |
54 | } | |
55 | ||
56 | func resetExceededTokens(r *Tokens) { | |
57 | for i, token := range r.pool { | |
58 | if token.RetryAfter > 0 { | |
59 | if int64(time.Since(token.ExceededTime)/time.Second) > token.RetryAfter { | |
60 | r.pool[i].ExceededTime = time.Time{} | |
61 | r.pool[i].RetryAfter = 0 | |
62 | } | |
63 | } | |
64 | } | |
65 | } |
0 | package hackertarget | |
1 | ||
2 | import ( | |
3 | "bufio" | |
4 | "context" | |
5 | "fmt" | |
6 | ||
7 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | // Source is the passive scraping agent | |
11 | type Source struct{} | |
12 | ||
13 | // Run function returns all subdomains found with the service | |
14 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
15 | results := make(chan subscraping.Result) | |
16 | ||
17 | go func() { | |
18 | defer close(results) | |
19 | ||
20 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://api.hackertarget.com/hostsearch/?q=%s", domain)) | |
21 | if err != nil { | |
22 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
23 | session.DiscardHTTPResponse(resp) | |
24 | return | |
25 | } | |
26 | ||
27 | defer resp.Body.Close() | |
28 | ||
29 | scanner := bufio.NewScanner(resp.Body) | |
30 | for scanner.Scan() { | |
31 | line := scanner.Text() | |
32 | if line == "" { | |
33 | continue | |
34 | } | |
35 | match := session.Extractor.FindAllString(line, -1) | |
36 | for _, subdomain := range match { | |
37 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
38 | } | |
39 | } | |
40 | }() | |
41 | ||
42 | return results | |
43 | } | |
44 | ||
45 | // Name returns the name of the source | |
46 | func (s *Source) Name() string { | |
47 | return "hackertarget" | |
48 | } |
0 | package intelx | |
1 | ||
2 | import ( | |
3 | "bytes" | |
4 | "context" | |
5 | "encoding/json" | |
6 | "fmt" | |
7 | "io/ioutil" | |
8 | ||
9 | jsoniter "github.com/json-iterator/go" | |
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
11 | ) | |
12 | ||
13 | type searchResponseType struct { | |
14 | ID string `json:"id"` | |
15 | Status int `json:"status"` | |
16 | } | |
17 | ||
18 | type selectorType struct { | |
19 | Selectvalue string `json:"selectorvalue"` | |
20 | } | |
21 | ||
22 | type searchResultType struct { | |
23 | Selectors []selectorType `json:"selectors"` | |
24 | Status int `json:"status"` | |
25 | } | |
26 | ||
27 | type requestBody struct { | |
28 | Term string | |
29 | Maxresults int | |
30 | Media int | |
31 | Target int | |
32 | Terminate []int | |
33 | Timeout int | |
34 | } | |
35 | ||
36 | // Source is the passive scraping agent | |
37 | type Source struct{} | |
38 | ||
39 | // Run function returns all subdomains found with the service | |
40 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
41 | results := make(chan subscraping.Result) | |
42 | ||
43 | go func() { | |
44 | defer close(results) | |
45 | ||
46 | if session.Keys.IntelXKey == "" || session.Keys.IntelXHost == "" { | |
47 | return | |
48 | } | |
49 | ||
50 | searchURL := fmt.Sprintf("https://%s/phonebook/search?k=%s", session.Keys.IntelXHost, session.Keys.IntelXKey) | |
51 | reqBody := requestBody{ | |
52 | Term: domain, | |
53 | Maxresults: 100000, | |
54 | Media: 0, | |
55 | Target: 1, | |
56 | Timeout: 20, | |
57 | } | |
58 | ||
59 | body, err := json.Marshal(reqBody) | |
60 | if err != nil { | |
61 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
62 | return | |
63 | } | |
64 | ||
65 | resp, err := session.SimplePost(ctx, searchURL, "application/json", bytes.NewBuffer(body)) | |
66 | if err != nil { | |
67 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
68 | session.DiscardHTTPResponse(resp) | |
69 | return | |
70 | } | |
71 | ||
72 | var response searchResponseType | |
73 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
74 | if err != nil { | |
75 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
76 | resp.Body.Close() | |
77 | return | |
78 | } | |
79 | ||
80 | resp.Body.Close() | |
81 | ||
82 | resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", session.Keys.IntelXHost, session.Keys.IntelXKey, response.ID) | |
83 | status := 0 | |
84 | for status == 0 || status == 3 { | |
85 | resp, err = session.Get(ctx, resultsURL, "", nil) | |
86 | if err != nil { | |
87 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
88 | session.DiscardHTTPResponse(resp) | |
89 | return | |
90 | } | |
91 | var response searchResultType | |
92 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
93 | if err != nil { | |
94 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
95 | resp.Body.Close() | |
96 | return | |
97 | } | |
98 | ||
99 | _, err = ioutil.ReadAll(resp.Body) | |
100 | if err != nil { | |
101 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
102 | resp.Body.Close() | |
103 | return | |
104 | } | |
105 | resp.Body.Close() | |
106 | ||
107 | status = response.Status | |
108 | for _, hostname := range response.Selectors { | |
109 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue} | |
110 | } | |
111 | } | |
112 | }() | |
113 | ||
114 | return results | |
115 | } | |
116 | ||
117 | // Name returns the name of the source | |
118 | func (s *Source) Name() string { | |
119 | return "intelx" | |
120 | } |
0 | package ipv4info | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "io/ioutil" | |
6 | "net/http" | |
7 | "regexp" | |
8 | "strconv" | |
9 | ||
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
11 | ) | |
12 | ||
13 | // Source is the passive scraping agent | |
14 | type Source struct{} | |
15 | ||
16 | // Run function returns all subdomains found with the service | |
17 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
18 | results := make(chan subscraping.Result) | |
19 | ||
20 | go func() { | |
21 | defer close(results) | |
22 | ||
23 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://ipv4info.com/search/%s", domain)) | |
24 | if err != nil && resp == nil { | |
25 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
26 | session.DiscardHTTPResponse(resp) | |
27 | return | |
28 | } | |
29 | ||
30 | body, err := ioutil.ReadAll(resp.Body) | |
31 | if err != nil { | |
32 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
33 | resp.Body.Close() | |
34 | return | |
35 | } | |
36 | resp.Body.Close() | |
37 | ||
38 | src := string(body) | |
39 | ||
40 | if resp.StatusCode != http.StatusOK { | |
41 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", src)} | |
42 | return | |
43 | } | |
44 | ||
45 | regxTokens := regexp.MustCompile("/ip-address/(.*)/" + domain) | |
46 | matchTokens := regxTokens.FindAllString(src, -1) | |
47 | ||
48 | if len(matchTokens) == 0 { | |
49 | return | |
50 | } | |
51 | ||
52 | token := matchTokens[0] | |
53 | ||
54 | resp, err = session.SimpleGet(ctx, "http://ipv4info.com"+token) | |
55 | if err != nil { | |
56 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
57 | session.DiscardHTTPResponse(resp) | |
58 | return | |
59 | } | |
60 | ||
61 | body, err = ioutil.ReadAll(resp.Body) | |
62 | if err != nil { | |
63 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
64 | session.DiscardHTTPResponse(resp) | |
65 | return | |
66 | } | |
67 | resp.Body.Close() | |
68 | ||
69 | src = string(body) | |
70 | regxTokens = regexp.MustCompile("/dns/(.*?)/" + domain) | |
71 | matchTokens = regxTokens.FindAllString(src, -1) | |
72 | if len(matchTokens) == 0 { | |
73 | return | |
74 | } | |
75 | ||
76 | token = matchTokens[0] | |
77 | resp, err = session.SimpleGet(ctx, "http://ipv4info.com"+token) | |
78 | if err != nil { | |
79 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
80 | session.DiscardHTTPResponse(resp) | |
81 | return | |
82 | } | |
83 | ||
84 | body, err = ioutil.ReadAll(resp.Body) | |
85 | if err != nil { | |
86 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
87 | resp.Body.Close() | |
88 | return | |
89 | } | |
90 | resp.Body.Close() | |
91 | ||
92 | src = string(body) | |
93 | regxTokens = regexp.MustCompile("/subdomains/(.*?)/" + domain) | |
94 | matchTokens = regxTokens.FindAllString(src, -1) | |
95 | if len(matchTokens) == 0 { | |
96 | return | |
97 | } | |
98 | ||
99 | token = matchTokens[0] | |
100 | resp, err = session.SimpleGet(ctx, "http://ipv4info.com"+token) | |
101 | if err != nil { | |
102 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
103 | session.DiscardHTTPResponse(resp) | |
104 | return | |
105 | } | |
106 | ||
107 | body, err = ioutil.ReadAll(resp.Body) | |
108 | if err != nil { | |
109 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
110 | resp.Body.Close() | |
111 | return | |
112 | } | |
113 | resp.Body.Close() | |
114 | ||
115 | src = string(body) | |
116 | for _, match := range session.Extractor.FindAllString(src, -1) { | |
117 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match} | |
118 | } | |
119 | nextPage := 1 | |
120 | ||
121 | for { | |
122 | further := s.getSubdomains(ctx, domain, &nextPage, src, session, results) | |
123 | if !further { | |
124 | break | |
125 | } | |
126 | } | |
127 | }() | |
128 | ||
129 | return results | |
130 | } | |
131 | ||
132 | // Name returns the name of the source | |
133 | func (s *Source) Name() string { | |
134 | return "ipv4info" | |
135 | } | |
136 | ||
137 | func (s *Source) getSubdomains(ctx context.Context, domain string, nextPage *int, src string, session *subscraping.Session, results chan subscraping.Result) bool { | |
138 | for { | |
139 | select { | |
140 | case <-ctx.Done(): | |
141 | return false | |
142 | default: | |
143 | regxTokens := regexp.MustCompile("/subdomains/.*/page" + strconv.Itoa(*nextPage) + "/" + domain + ".html") | |
144 | matchTokens := regxTokens.FindAllString(src, -1) | |
145 | if len(matchTokens) == 0 { | |
146 | return false | |
147 | } | |
148 | token := matchTokens[0] | |
149 | ||
150 | resp, err := session.SimpleGet(ctx, "http://ipv4info.com"+token) | |
151 | if err != nil { | |
152 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
153 | return false | |
154 | } | |
155 | body, err := ioutil.ReadAll(resp.Body) | |
156 | if err != nil { | |
157 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
158 | resp.Body.Close() | |
159 | return false | |
160 | } | |
161 | resp.Body.Close() | |
162 | src = string(body) | |
163 | for _, match := range session.Extractor.FindAllString(src, -1) { | |
164 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match} | |
165 | } | |
166 | *nextPage++ | |
167 | return true | |
168 | } | |
169 | } | |
170 | } |
0 | package passivetotal | |
1 | ||
2 | import ( | |
3 | "bytes" | |
4 | "context" | |
5 | ||
6 | jsoniter "github.com/json-iterator/go" | |
7 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | type response struct { | |
11 | Subdomains []string `json:"subdomains"` | |
12 | } | |
13 | ||
14 | // Source is the passive scraping agent | |
15 | type Source struct{} | |
16 | ||
17 | // Run function returns all subdomains found with the service | |
18 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
19 | results := make(chan subscraping.Result) | |
20 | ||
21 | go func() { | |
22 | defer close(results) | |
23 | ||
24 | if session.Keys.PassiveTotalUsername == "" || session.Keys.PassiveTotalPassword == "" { | |
25 | return | |
26 | } | |
27 | ||
28 | // Create JSON Get body | |
29 | var request = []byte(`{"query":"` + domain + `"}`) | |
30 | ||
31 | resp, err := session.HTTPRequest( | |
32 | ctx, | |
33 | "GET", | |
34 | "https://api.passivetotal.org/v2/enrichment/subdomains", | |
35 | "", | |
36 | map[string]string{"Content-Type": "application/json"}, | |
37 | bytes.NewBuffer(request), | |
38 | subscraping.BasicAuth{Username: session.Keys.PassiveTotalUsername, Password: session.Keys.PassiveTotalPassword}, | |
39 | ) | |
40 | if err != nil { | |
41 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
42 | session.DiscardHTTPResponse(resp) | |
43 | return | |
44 | } | |
45 | ||
46 | var data response | |
47 | err = jsoniter.NewDecoder(resp.Body).Decode(&data) | |
48 | if err != nil { | |
49 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
50 | resp.Body.Close() | |
51 | return | |
52 | } | |
53 | resp.Body.Close() | |
54 | ||
55 | for _, subdomain := range data.Subdomains { | |
56 | finalSubdomain := subdomain + "." + domain | |
57 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: finalSubdomain} | |
58 | } | |
59 | }() | |
60 | ||
61 | return results | |
62 | } | |
63 | ||
64 | // Name returns the name of the source | |
65 | func (s *Source) Name() string { | |
66 | return "passivetotal" | |
67 | } |
0 | // Package rapiddns is a RapidDNS Scraping Engine in Golang | |
1 | package rapiddns | |
2 | ||
3 | import ( | |
4 | "context" | |
5 | "io/ioutil" | |
6 | ||
7 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | // Source is the passive scraping agent | |
11 | type Source struct{} | |
12 | ||
13 | // Run function returns all subdomains found with the service | |
14 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
15 | results := make(chan subscraping.Result) | |
16 | ||
17 | go func() { | |
18 | defer close(results) | |
19 | ||
20 | resp, err := session.SimpleGet(ctx, "https://rapiddns.io/subdomain/"+domain+"?full=1") | |
21 | if err != nil { | |
22 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
23 | session.DiscardHTTPResponse(resp) | |
24 | return | |
25 | } | |
26 | ||
27 | body, err := ioutil.ReadAll(resp.Body) | |
28 | if err != nil { | |
29 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
30 | resp.Body.Close() | |
31 | return | |
32 | } | |
33 | ||
34 | resp.Body.Close() | |
35 | ||
36 | src := string(body) | |
37 | for _, subdomain := range session.Extractor.FindAllString(src, -1) { | |
38 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
39 | } | |
40 | }() | |
41 | ||
42 | return results | |
43 | } | |
44 | ||
45 | // Name returns the name of the source | |
46 | func (s *Source) Name() string { | |
47 | return "rapiddns" | |
48 | } |
0 | package recon | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "encoding/json" | |
5 | "fmt" | |
6 | ||
7 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | type subdomain struct { | |
11 | RawDomain string `json:"rawDomain"` | |
12 | } | |
13 | ||
14 | // Source is the passive scraping agent | |
15 | type Source struct{} | |
16 | ||
17 | // Run function returns all subdomains found with the service | |
18 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
19 | results := make(chan subscraping.Result) | |
20 | ||
21 | go func() { | |
22 | defer close(results) | |
23 | ||
24 | if session.Keys.Recon == "" { | |
25 | return | |
26 | } | |
27 | ||
28 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://recon.dev/api/search?key=%s&domain=%s", session.Keys.Recon, domain)) | |
29 | if err != nil { | |
30 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
31 | session.DiscardHTTPResponse(resp) | |
32 | return | |
33 | } | |
34 | ||
35 | var subdomains []subdomain | |
36 | err = json.NewDecoder(resp.Body).Decode(&subdomains) | |
37 | if err != nil { | |
38 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
39 | resp.Body.Close() | |
40 | return | |
41 | } | |
42 | resp.Body.Close() | |
43 | ||
44 | for _, subdomain := range subdomains { | |
45 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.RawDomain} | |
46 | } | |
47 | }() | |
48 | ||
49 | return results | |
50 | } | |
51 | ||
52 | // Name returns the name of the source | |
53 | func (s *Source) Name() string { | |
54 | return "recon" | |
55 | } |
0 | package riddler | |
1 | ||
2 | import ( | |
3 | "bufio" | |
4 | "context" | |
5 | "fmt" | |
6 | ||
7 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | // Source is the passive scraping agent | |
11 | type Source struct{} | |
12 | ||
13 | // Run function returns all subdomains found with the service | |
14 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
15 | results := make(chan subscraping.Result) | |
16 | ||
17 | go func() { | |
18 | defer close(results) | |
19 | ||
20 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://riddler.io/search?q=pld:%s&view_type=data_table", domain)) | |
21 | if err != nil { | |
22 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
23 | session.DiscardHTTPResponse(resp) | |
24 | return | |
25 | } | |
26 | ||
27 | scanner := bufio.NewScanner(resp.Body) | |
28 | for scanner.Scan() { | |
29 | line := scanner.Text() | |
30 | if line == "" { | |
31 | continue | |
32 | } | |
33 | subdomain := session.Extractor.FindString(line) | |
34 | if subdomain != "" { | |
35 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
36 | } | |
37 | } | |
38 | resp.Body.Close() | |
39 | }() | |
40 | ||
41 | return results | |
42 | } | |
43 | ||
44 | // Name returns the name of the source | |
45 | func (s *Source) Name() string { | |
46 | return "riddler" | |
47 | } |
0 | package robtex | |
1 | ||
2 | import ( | |
3 | "bufio" | |
4 | "bytes" | |
5 | "context" | |
6 | "fmt" | |
7 | ||
8 | jsoniter "github.com/json-iterator/go" | |
9 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
10 | ) | |
11 | ||
12 | const ( | |
13 | addrRecord = "A" | |
14 | iPv6AddrRecord = "AAAA" | |
15 | baseURL = "https://proapi.robtex.com/pdns" | |
16 | ) | |
17 | ||
18 | // Source is the passive scraping agent | |
19 | type Source struct{} | |
20 | ||
21 | type result struct { | |
22 | Rrname string `json:"rrname"` | |
23 | Rrdata string `json:"rrdata"` | |
24 | Rrtype string `json:"rrtype"` | |
25 | } | |
26 | ||
27 | // Run function returns all subdomains found with the service | |
28 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
29 | results := make(chan subscraping.Result) | |
30 | ||
31 | go func() { | |
32 | defer close(results) | |
33 | ||
34 | if session.Keys.Robtex == "" { | |
35 | return | |
36 | } | |
37 | ||
38 | headers := map[string]string{"Content-Type": "application/x-ndjson"} | |
39 | ||
40 | ips, err := enumerate(ctx, session, fmt.Sprintf("%s/forward/%s?key=%s", baseURL, domain, session.Keys.Robtex), headers) | |
41 | if err != nil { | |
42 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
43 | return | |
44 | } | |
45 | ||
46 | for _, result := range ips { | |
47 | if result.Rrtype == addrRecord || result.Rrtype == iPv6AddrRecord { | |
48 | domains, err := enumerate(ctx, session, fmt.Sprintf("%s/reverse/%s?key=%s", baseURL, result.Rrdata, session.Keys.Robtex), headers) | |
49 | if err != nil { | |
50 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
51 | return | |
52 | } | |
53 | for _, result := range domains { | |
54 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Rrdata} | |
55 | } | |
56 | } | |
57 | } | |
58 | }() | |
59 | return results | |
60 | } | |
61 | ||
62 | func enumerate(ctx context.Context, session *subscraping.Session, targetURL string, headers map[string]string) ([]result, error) { | |
63 | var results []result | |
64 | ||
65 | resp, err := session.Get(ctx, targetURL, "", headers) | |
66 | if err != nil { | |
67 | session.DiscardHTTPResponse(resp) | |
68 | return results, err | |
69 | } | |
70 | ||
71 | scanner := bufio.NewScanner(resp.Body) | |
72 | for scanner.Scan() { | |
73 | line := scanner.Text() | |
74 | if line == "" { | |
75 | continue | |
76 | } | |
77 | var response result | |
78 | err = jsoniter.NewDecoder(bytes.NewBufferString(line)).Decode(&response) | |
79 | if err != nil { | |
80 | return results, err | |
81 | } | |
82 | ||
83 | results = append(results, response) | |
84 | } | |
85 | ||
86 | resp.Body.Close() | |
87 | ||
88 | return results, nil | |
89 | } | |
90 | ||
91 | // Name returns the name of the source | |
92 | func (s *Source) Name() string { | |
93 | return "robtex" | |
94 | } |
0 | package securitytrails | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "strings" | |
6 | ||
7 | jsoniter "github.com/json-iterator/go" | |
8 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
9 | ) | |
10 | ||
11 | type response struct { | |
12 | Subdomains []string `json:"subdomains"` | |
13 | } | |
14 | ||
15 | // Source is the passive scraping agent | |
16 | type Source struct{} | |
17 | ||
18 | // Run function returns all subdomains found with the service | |
19 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
20 | results := make(chan subscraping.Result) | |
21 | ||
22 | go func() { | |
23 | defer close(results) | |
24 | ||
25 | if session.Keys.Securitytrails == "" { | |
26 | return | |
27 | } | |
28 | ||
29 | resp, err := session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", map[string]string{"APIKEY": session.Keys.Securitytrails}) | |
30 | if err != nil { | |
31 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
32 | session.DiscardHTTPResponse(resp) | |
33 | return | |
34 | } | |
35 | ||
36 | var securityTrailsResponse response | |
37 | err = jsoniter.NewDecoder(resp.Body).Decode(&securityTrailsResponse) | |
38 | if err != nil { | |
39 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
40 | resp.Body.Close() | |
41 | return | |
42 | } | |
43 | ||
44 | resp.Body.Close() | |
45 | ||
46 | for _, subdomain := range securityTrailsResponse.Subdomains { | |
47 | if strings.HasSuffix(subdomain, ".") { | |
48 | subdomain += domain | |
49 | } else { | |
50 | subdomain = subdomain + "." + domain | |
51 | } | |
52 | ||
53 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
54 | } | |
55 | }() | |
56 | ||
57 | return results | |
58 | } | |
59 | ||
60 | // Name returns the name of the source | |
61 | func (s *Source) Name() string { | |
62 | return "securitytrails" | |
63 | } |
0 | package shodan | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | ||
6 | jsoniter "github.com/json-iterator/go" | |
7 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | // Source is the passive scraping agent | |
11 | type Source struct{} | |
12 | ||
13 | type dnsdbLookupResponse struct { | |
14 | Domain string `json:"domain"` | |
15 | Data []struct { | |
16 | Subdomain string `json:"subdomain"` | |
17 | Type string `json:"type"` | |
18 | Value string `json:"value"` | |
19 | } `json:"data"` | |
20 | Result int `json:"result"` | |
21 | Error string `json:"error"` | |
22 | } | |
23 | ||
24 | // Run function returns all subdomains found with the service | |
25 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
26 | results := make(chan subscraping.Result) | |
27 | ||
28 | go func() { | |
29 | defer close(results) | |
30 | ||
31 | if session.Keys.Shodan == "" { | |
32 | return | |
33 | } | |
34 | ||
35 | searchURL := fmt.Sprintf("https://api.shodan.io/dns/domain/%s?key=%s", domain, session.Keys.Shodan) | |
36 | resp, err := session.SimpleGet(ctx, searchURL) | |
37 | if err != nil { | |
38 | session.DiscardHTTPResponse(resp) | |
39 | return | |
40 | } | |
41 | ||
42 | defer resp.Body.Close() | |
43 | ||
44 | var response dnsdbLookupResponse | |
45 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
46 | if err != nil { | |
47 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
48 | return | |
49 | } | |
50 | ||
51 | if response.Error != "" { | |
52 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error)} | |
53 | return | |
54 | } | |
55 | ||
56 | for _, data := range response.Data { | |
57 | if data.Subdomain != "" { | |
58 | if data.Type == "CNAME" { | |
59 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Value} | |
60 | } else if data.Type == "A" { | |
61 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", data.Subdomain, domain)} | |
62 | } | |
63 | } | |
64 | } | |
65 | }() | |
66 | ||
67 | return results | |
68 | } | |
69 | ||
70 | // Name returns the name of the source | |
71 | func (s *Source) Name() string { | |
72 | return "shodan" | |
73 | } |
0 | package sitedossier | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "io/ioutil" | |
6 | "math/rand" | |
7 | "net/http" | |
8 | "regexp" | |
9 | "time" | |
10 | ||
11 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
12 | ) | |
13 | ||
14 | // SleepRandIntn is the integer value to get the pseudo-random number | |
15 | // to sleep before find the next match | |
16 | const SleepRandIntn = 5 | |
17 | ||
18 | var reNext = regexp.MustCompile(`<a href="([A-Za-z0-9/.]+)"><b>`) | |
19 | ||
20 | type agent struct { | |
21 | results chan subscraping.Result | |
22 | session *subscraping.Session | |
23 | } | |
24 | ||
25 | func (a *agent) enumerate(ctx context.Context, baseURL string) { | |
26 | select { | |
27 | case <-ctx.Done(): | |
28 | return | |
29 | default: | |
30 | } | |
31 | ||
32 | resp, err := a.session.SimpleGet(ctx, baseURL) | |
33 | isnotfound := resp != nil && resp.StatusCode == http.StatusNotFound | |
34 | if err != nil && !isnotfound { | |
35 | a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} | |
36 | a.session.DiscardHTTPResponse(resp) | |
37 | } | |
38 | ||
39 | body, err := ioutil.ReadAll(resp.Body) | |
40 | if err != nil { | |
41 | a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} | |
42 | resp.Body.Close() | |
43 | } | |
44 | resp.Body.Close() | |
45 | ||
46 | src := string(body) | |
47 | for _, match := range a.session.Extractor.FindAllString(src, -1) { | |
48 | a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: match} | |
49 | } | |
50 | ||
51 | match1 := reNext.FindStringSubmatch(src) | |
52 | time.Sleep(time.Duration((3 + rand.Intn(SleepRandIntn))) * time.Second) | |
53 | ||
54 | if len(match1) > 0 { | |
55 | a.enumerate(ctx, "http://www.sitedossier.com"+match1[1]) | |
56 | } | |
57 | } | |
58 | ||
59 | // Source is the passive scraping agent | |
60 | type Source struct{} | |
61 | ||
62 | // Run function returns all subdomains found with the service | |
63 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
64 | results := make(chan subscraping.Result) | |
65 | ||
66 | a := agent{ | |
67 | session: session, | |
68 | results: results, | |
69 | } | |
70 | ||
71 | go func() { | |
72 | a.enumerate(ctx, fmt.Sprintf("http://www.sitedossier.com/parentdomain/%s", domain)) | |
73 | close(a.results) | |
74 | }() | |
75 | ||
76 | return a.results | |
77 | } | |
78 | ||
79 | // Name returns the name of the source | |
80 | func (s *Source) Name() string { | |
81 | return "sitedossier" | |
82 | } |
0 | package spyse | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "strconv" | |
6 | ||
7 | jsoniter "github.com/json-iterator/go" | |
8 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
9 | ) | |
10 | ||
11 | type resultObject struct { | |
12 | Name string `json:"name"` | |
13 | } | |
14 | ||
15 | type dataObject struct { | |
16 | Items []resultObject `json:"items"` | |
17 | TotalCount int `json:"total_count"` | |
18 | } | |
19 | ||
20 | type errorObject struct { | |
21 | Code string `json:"code"` | |
22 | Message string `json:"message"` | |
23 | } | |
24 | ||
25 | type spyseResult struct { | |
26 | Data dataObject `json:"data"` | |
27 | Error []errorObject `json:"error"` | |
28 | } | |
29 | ||
30 | // Source is the passive scraping agent | |
31 | type Source struct{} | |
32 | ||
33 | // Run function returns all subdomains found with the service | |
34 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
35 | results := make(chan subscraping.Result) | |
36 | ||
37 | go func() { | |
38 | defer close(results) | |
39 | ||
40 | if session.Keys.Spyse == "" { | |
41 | return | |
42 | } | |
43 | ||
44 | maxCount := 100 | |
45 | ||
46 | for offSet := 0; offSet <= maxCount; offSet += 100 { | |
47 | resp, err := session.Get(ctx, fmt.Sprintf("https://api.spyse.com/v3/data/domain/subdomain?domain=%s&limit=100&offset=%s", domain, strconv.Itoa(offSet)), "", map[string]string{"Authorization": "Bearer " + session.Keys.Spyse}) | |
48 | if err != nil { | |
49 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
50 | session.DiscardHTTPResponse(resp) | |
51 | return | |
52 | } | |
53 | ||
54 | var response spyseResult | |
55 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
56 | if err != nil { | |
57 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
58 | resp.Body.Close() | |
59 | return | |
60 | } | |
61 | resp.Body.Close() | |
62 | ||
63 | if response.Data.TotalCount == 0 { | |
64 | return | |
65 | } | |
66 | ||
67 | maxCount = response.Data.TotalCount | |
68 | ||
69 | for _, hostname := range response.Data.Items { | |
70 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Name} | |
71 | } | |
72 | } | |
73 | }() | |
74 | ||
75 | return results | |
76 | } | |
77 | ||
78 | // Name returns the name of the source | |
79 | func (s *Source) Name() string { | |
80 | return "spyse" | |
81 | } |
0 | package sublist3r | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "encoding/json" | |
5 | "fmt" | |
6 | ||
7 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | // Source is the passive scraping agent | |
11 | type Source struct{} | |
12 | ||
13 | // Run function returns all subdomains found with the service | |
14 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
15 | results := make(chan subscraping.Result) | |
16 | ||
17 | go func() { | |
18 | defer close(results) | |
19 | ||
20 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.sublist3r.com/search.php?domain=%s", domain)) | |
21 | if err != nil { | |
22 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
23 | session.DiscardHTTPResponse(resp) | |
24 | return | |
25 | } | |
26 | ||
27 | var subdomains []string | |
28 | err = json.NewDecoder(resp.Body).Decode(&subdomains) | |
29 | if err != nil { | |
30 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
31 | resp.Body.Close() | |
32 | return | |
33 | } | |
34 | ||
35 | resp.Body.Close() | |
36 | ||
37 | for _, subdomain := range subdomains { | |
38 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
39 | } | |
40 | }() | |
41 | ||
42 | return results | |
43 | } | |
44 | ||
45 | // Name returns the name of the source | |
46 | func (s *Source) Name() string { | |
47 | return "sublist3r" | |
48 | } |
0 | package threatbook | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | "strconv" | |
6 | ||
7 | jsoniter "github.com/json-iterator/go" | |
8 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
9 | ) | |
10 | ||
11 | type threatBookResponse struct { | |
12 | ResponseCode int64 `json:"response_code"` | |
13 | VerboseMsg string `json:"verbose_msg"` | |
14 | Data struct { | |
15 | Domain string `json:"domain"` | |
16 | SubDomains struct { | |
17 | Total string `json:"total"` | |
18 | Data []string `json:"data"` | |
19 | } `json:"sub_domains"` | |
20 | } `json:"data"` | |
21 | } | |
22 | ||
23 | // Source is the passive scraping agent | |
24 | type Source struct{} | |
25 | ||
26 | // Run function returns all subdomains found with the service | |
27 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
28 | results := make(chan subscraping.Result) | |
29 | ||
30 | go func() { | |
31 | defer close(results) | |
32 | ||
33 | if session.Keys.ThreatBook == "" { | |
34 | return | |
35 | } | |
36 | ||
37 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatbook.cn/v3/domain/sub_domains?apikey=%s&resource=%s", session.Keys.ThreatBook, domain)) | |
38 | if err != nil && resp == nil { | |
39 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
40 | session.DiscardHTTPResponse(resp) | |
41 | return | |
42 | } | |
43 | ||
44 | var response threatBookResponse | |
45 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
46 | if err != nil { | |
47 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
48 | resp.Body.Close() | |
49 | return | |
50 | } | |
51 | resp.Body.Close() | |
52 | ||
53 | if response.ResponseCode != 0 { | |
54 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("code %d, %s", response.ResponseCode, response.VerboseMsg)} | |
55 | return | |
56 | } | |
57 | ||
58 | total, err := strconv.ParseInt(response.Data.SubDomains.Total, 10, 64) | |
59 | if err != nil { | |
60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
61 | return | |
62 | } | |
63 | ||
64 | if total > 0 { | |
65 | for _, subdomain := range response.Data.SubDomains.Data { | |
66 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
67 | } | |
68 | } | |
69 | }() | |
70 | ||
71 | return results | |
72 | } | |
73 | ||
74 | // Name returns the name of the source | |
75 | func (s *Source) Name() string { | |
76 | return "threatbook" | |
77 | } |
0 | package threatcrowd | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | ||
6 | jsoniter "github.com/json-iterator/go" | |
7 | ||
8 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
9 | ) | |
10 | ||
11 | type response struct { | |
12 | Subdomains []string `json:"subdomains"` | |
13 | } | |
14 | ||
15 | // Source is the passive scraping agent | |
16 | type Source struct{} | |
17 | ||
18 | // Run function returns all subdomains found with the service | |
19 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
20 | results := make(chan subscraping.Result) | |
21 | ||
22 | go func() { | |
23 | defer close(results) | |
24 | ||
25 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://www.threatcrowd.org/searchApi/v2/domain/report/?domain=%s", domain)) | |
26 | if err != nil { | |
27 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
28 | session.DiscardHTTPResponse(resp) | |
29 | return | |
30 | } | |
31 | ||
32 | defer resp.Body.Close() | |
33 | ||
34 | var data response | |
35 | err = jsoniter.NewDecoder(resp.Body).Decode(&data) | |
36 | if err != nil { | |
37 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
38 | return | |
39 | } | |
40 | ||
41 | for _, subdomain := range data.Subdomains { | |
42 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
43 | } | |
44 | }() | |
45 | ||
46 | return results | |
47 | } | |
48 | ||
49 | // Name returns the name of the source | |
50 | func (s *Source) Name() string { | |
51 | return "threatcrowd" | |
52 | } |
0 | package threatminer | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | ||
6 | jsoniter "github.com/json-iterator/go" | |
7 | ||
8 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
9 | ) | |
10 | ||
11 | type response struct { | |
12 | StatusCode string `json:"status_code"` | |
13 | StatusMessage string `json:"status_message"` | |
14 | Results []string `json:"results"` | |
15 | } | |
16 | ||
17 | // Source is the passive scraping agent | |
18 | type Source struct{} | |
19 | ||
20 | // Run function returns all subdomains found with the service | |
21 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
22 | results := make(chan subscraping.Result) | |
23 | ||
24 | go func() { | |
25 | defer close(results) | |
26 | ||
27 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatminer.org/v2/domain.php?q=%s&rt=5", domain)) | |
28 | if err != nil { | |
29 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
30 | session.DiscardHTTPResponse(resp) | |
31 | return | |
32 | } | |
33 | ||
34 | defer resp.Body.Close() | |
35 | ||
36 | var data response | |
37 | err = jsoniter.NewDecoder(resp.Body).Decode(&data) | |
38 | if err != nil { | |
39 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
40 | return | |
41 | } | |
42 | ||
43 | for _, subdomain := range data.Results { | |
44 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
45 | } | |
46 | }() | |
47 | ||
48 | return results | |
49 | } | |
50 | ||
51 | // Name returns the name of the source | |
52 | func (s *Source) Name() string { | |
53 | return "threatminer" | |
54 | } |
0 | package virustotal | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | ||
6 | jsoniter "github.com/json-iterator/go" | |
7 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | type response struct { | |
11 | Subdomains []string `json:"subdomains"` | |
12 | } | |
13 | ||
14 | // Source is the passive scraping agent | |
15 | type Source struct{} | |
16 | ||
17 | // Run function returns all subdomains found with the service | |
18 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
19 | results := make(chan subscraping.Result) | |
20 | ||
21 | go func() { | |
22 | defer close(results) | |
23 | ||
24 | if session.Keys.Virustotal == "" { | |
25 | return | |
26 | } | |
27 | ||
28 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://www.virustotal.com/vtapi/v2/domain/report?apikey=%s&domain=%s", session.Keys.Virustotal, domain)) | |
29 | if err != nil { | |
30 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
31 | session.DiscardHTTPResponse(resp) | |
32 | return | |
33 | } | |
34 | ||
35 | var data response | |
36 | err = jsoniter.NewDecoder(resp.Body).Decode(&data) | |
37 | if err != nil { | |
38 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
39 | resp.Body.Close() | |
40 | return | |
41 | } | |
42 | ||
43 | resp.Body.Close() | |
44 | ||
45 | for _, subdomain := range data.Subdomains { | |
46 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
47 | } | |
48 | }() | |
49 | ||
50 | return results | |
51 | } | |
52 | ||
53 | // Name returns the name of the source | |
54 | func (s *Source) Name() string { | |
55 | return "virustotal" | |
56 | } |
0 | package waybackarchive | |
1 | ||
2 | import ( | |
3 | "bufio" | |
4 | "context" | |
5 | "fmt" | |
6 | "net/url" | |
7 | "strings" | |
8 | ||
9 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
10 | ) | |
11 | ||
12 | // Source is the passive scraping agent | |
13 | type Source struct{} | |
14 | ||
15 | // Run function returns all subdomains found with the service | |
16 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
17 | results := make(chan subscraping.Result) | |
18 | ||
19 | go func() { | |
20 | defer close(results) | |
21 | ||
22 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey", domain)) | |
23 | if err != nil { | |
24 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
25 | session.DiscardHTTPResponse(resp) | |
26 | return | |
27 | } | |
28 | ||
29 | defer resp.Body.Close() | |
30 | ||
31 | scanner := bufio.NewScanner(resp.Body) | |
32 | for scanner.Scan() { | |
33 | line := scanner.Text() | |
34 | if line == "" { | |
35 | continue | |
36 | } | |
37 | line, _ = url.QueryUnescape(line) | |
38 | subdomain := session.Extractor.FindString(line) | |
39 | if subdomain != "" { | |
40 | // fix for triple encoded URL | |
41 | subdomain = strings.ToLower(subdomain) | |
42 | subdomain = strings.TrimPrefix(subdomain, "25") | |
43 | subdomain = strings.TrimPrefix(subdomain, "2f") | |
44 | ||
45 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} | |
46 | } | |
47 | } | |
48 | }() | |
49 | ||
50 | return results | |
51 | } | |
52 | ||
53 | // Name returns the name of the source | |
54 | func (s *Source) Name() string { | |
55 | return "waybackarchive" | |
56 | } |
0 | package ximcx | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "fmt" | |
5 | ||
6 | jsoniter "github.com/json-iterator/go" | |
7 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
8 | ) | |
9 | ||
10 | // Source is the passive scraping agent | |
11 | type Source struct{} | |
12 | ||
13 | type domain struct { | |
14 | Domain string `json:"domain"` | |
15 | } | |
16 | ||
17 | type ximcxResponse struct { | |
18 | Code int64 `json:"code"` | |
19 | Message string `json:"message"` | |
20 | Data []domain `json:"data"` | |
21 | } | |
22 | ||
23 | // Run function returns all subdomains found with the service | |
24 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
25 | results := make(chan subscraping.Result) | |
26 | ||
27 | go func() { | |
28 | defer close(results) | |
29 | ||
30 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://sbd.ximcx.cn/DomainServlet?domain=%s", domain)) | |
31 | if err != nil { | |
32 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
33 | session.DiscardHTTPResponse(resp) | |
34 | return | |
35 | } | |
36 | ||
37 | var response ximcxResponse | |
38 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) | |
39 | if err != nil { | |
40 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
41 | resp.Body.Close() | |
42 | return | |
43 | } | |
44 | resp.Body.Close() | |
45 | ||
46 | if response.Code > 0 { | |
47 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%d, %s", response.Code, response.Message)} | |
48 | return | |
49 | } | |
50 | ||
51 | for _, result := range response.Data { | |
52 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Domain} | |
53 | } | |
54 | }() | |
55 | ||
56 | return results | |
57 | } | |
58 | ||
59 | // Name returns the name of the source | |
60 | func (s *Source) Name() string { | |
61 | return "ximcx" | |
62 | } |
0 | package zoomeye | |
1 | ||
2 | import ( | |
3 | "bytes" | |
4 | "context" | |
5 | "encoding/json" | |
6 | "errors" | |
7 | "fmt" | |
8 | "net/http" | |
9 | ||
10 | "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" | |
11 | ) | |
12 | ||
13 | // zoomAuth holds the ZoomEye credentials | |
14 | type zoomAuth struct { | |
15 | User string `json:"username"` | |
16 | Pass string `json:"password"` | |
17 | } | |
18 | ||
19 | type loginResp struct { | |
20 | JWT string `json:"access_token"` | |
21 | } | |
22 | ||
23 | // search results | |
24 | type zoomeyeResults struct { | |
25 | Matches []struct { | |
26 | Site string `json:"site"` | |
27 | Domains []string `json:"domains"` | |
28 | } `json:"matches"` | |
29 | } | |
30 | ||
31 | // Source is the passive scraping agent | |
32 | type Source struct{} | |
33 | ||
34 | // Run function returns all subdomains found with the service | |
35 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { | |
36 | results := make(chan subscraping.Result) | |
37 | ||
38 | go func() { | |
39 | defer close(results) | |
40 | ||
41 | if session.Keys.ZoomEyeUsername == "" || session.Keys.ZoomEyePassword == "" { | |
42 | return | |
43 | } | |
44 | ||
45 | jwt, err := doLogin(ctx, session) | |
46 | if err != nil { | |
47 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
48 | return | |
49 | } | |
50 | // check if jwt is null | |
51 | if jwt == "" { | |
52 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errors.New("could not log into zoomeye")} | |
53 | return | |
54 | } | |
55 | ||
56 | headers := map[string]string{ | |
57 | "Authorization": fmt.Sprintf("JWT %s", jwt), | |
58 | "Accept": "application/json", | |
59 | "Content-Type": "application/json", | |
60 | } | |
61 | for currentPage := 0; currentPage <= 100; currentPage++ { | |
62 | api := fmt.Sprintf("https://api.zoomeye.org/web/search?query=hostname:%s&page=%d", domain, currentPage) | |
63 | resp, err := session.Get(ctx, api, "", headers) | |
64 | isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden | |
65 | if err != nil { | |
66 | if !isForbidden && currentPage == 0 { | |
67 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
68 | session.DiscardHTTPResponse(resp) | |
69 | } | |
70 | return | |
71 | } | |
72 | ||
73 | var res zoomeyeResults | |
74 | err = json.NewDecoder(resp.Body).Decode(&res) | |
75 | if err != nil { | |
76 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} | |
77 | resp.Body.Close() | |
78 | return | |
79 | } | |
80 | resp.Body.Close() | |
81 | ||
82 | for _, r := range res.Matches { | |
83 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Site} | |
84 | for _, domain := range r.Domains { | |
85 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domain} | |
86 | } | |
87 | } | |
88 | currentPage++ | |
89 | } | |
90 | }() | |
91 | ||
92 | return results | |
93 | } | |
94 | ||
95 | // doLogin performs authentication on the ZoomEye API | |
96 | func doLogin(ctx context.Context, session *subscraping.Session) (string, error) { | |
97 | creds := &zoomAuth{ | |
98 | User: session.Keys.ZoomEyeUsername, | |
99 | Pass: session.Keys.ZoomEyePassword, | |
100 | } | |
101 | body, err := json.Marshal(&creds) | |
102 | if err != nil { | |
103 | return "", err | |
104 | } | |
105 | resp, err := session.SimplePost(ctx, "https://api.zoomeye.org/user/login", "application/json", bytes.NewBuffer(body)) | |
106 | if err != nil { | |
107 | session.DiscardHTTPResponse(resp) | |
108 | return "", err | |
109 | } | |
110 | ||
111 | defer resp.Body.Close() | |
112 | ||
113 | var login loginResp | |
114 | err = json.NewDecoder(resp.Body).Decode(&login) | |
115 | if err != nil { | |
116 | return "", err | |
117 | } | |
118 | return login.JWT, nil | |
119 | } | |
120 | ||
121 | // Name returns the name of the source | |
122 | func (s *Source) Name() string { | |
123 | return "zoomeye" | |
124 | } |
0 | package subscraping | |
1 | ||
2 | import ( | |
3 | "context" | |
4 | "net/http" | |
5 | "regexp" | |
6 | ) | |
7 | ||
8 | // BasicAuth request's Authorization header | |
9 | type BasicAuth struct { | |
10 | Username string | |
11 | Password string | |
12 | } | |
13 | ||
14 | // Source is an interface inherited by each passive source | |
15 | type Source interface { | |
16 | // Run takes a domain as argument and a session object | |
17 | // which contains the extractor for subdomains, http client | |
18 | // and other stuff. | |
19 | Run(context.Context, string, *Session) <-chan Result | |
20 | // Name returns the name of the source | |
21 | Name() string | |
22 | } | |
23 | ||
24 | // Session is the option passed to the source, an option is created | |
25 | // uniquely for eac source. | |
26 | type Session struct { | |
27 | // Extractor is the regex for subdomains created for each domain | |
28 | Extractor *regexp.Regexp | |
29 | // Keys is the API keys for the application | |
30 | Keys *Keys | |
31 | // Client is the current http client | |
32 | Client *http.Client | |
33 | } | |
34 | ||
35 | // Keys contains the current API Keys we have in store | |
36 | type Keys struct { | |
37 | Binaryedge string `json:"binaryedge"` | |
38 | CensysToken string `json:"censysUsername"` | |
39 | CensysSecret string `json:"censysPassword"` | |
40 | Certspotter string `json:"certspotter"` | |
41 | Chaos string `json:"chaos"` | |
42 | DNSDB string `json:"dnsdb"` | |
43 | GitHub []string `json:"github"` | |
44 | IntelXHost string `json:"intelXHost"` | |
45 | IntelXKey string `json:"intelXKey"` | |
46 | PassiveTotalUsername string `json:"passivetotal_username"` | |
47 | PassiveTotalPassword string `json:"passivetotal_password"` | |
48 | Recon string `json:"recon"` | |
49 | Robtex string `json:"robtex"` | |
50 | Securitytrails string `json:"securitytrails"` | |
51 | Shodan string `json:"shodan"` | |
52 | Spyse string `json:"spyse"` | |
53 | ThreatBook string `json:"threatbook"` | |
54 | URLScan string `json:"urlscan"` | |
55 | Virustotal string `json:"virustotal"` | |
56 | ZoomEyeUsername string `json:"zoomeye_username"` | |
57 | ZoomEyePassword string `json:"zoomeye_password"` | |
58 | } | |
59 | ||
60 | // Result is a result structure returned by a source | |
61 | type Result struct { | |
62 | Type ResultType | |
63 | Source string | |
64 | Value string | |
65 | Error error | |
66 | } | |
67 | ||
68 | // ResultType is the type of result returned by the source | |
69 | type ResultType int | |
70 | ||
71 | // Types of results returned by the source | |
72 | const ( | |
73 | Subdomain ResultType = iota | |
74 | Error | |
75 | ) |
0 | package subscraping | |
1 | ||
2 | import ( | |
3 | "regexp" | |
4 | "sync" | |
5 | ) | |
6 | ||
7 | var subdomainExtractorMutex = &sync.Mutex{} | |
8 | ||
9 | // NewSubdomainExtractor creates a new regular expression to extract | |
10 | // subdomains from text based on the given domain. | |
11 | func NewSubdomainExtractor(domain string) (*regexp.Regexp, error) { | |
12 | subdomainExtractorMutex.Lock() | |
13 | defer subdomainExtractorMutex.Unlock() | |
14 | extractor, err := regexp.Compile(`[a-zA-Z0-9\*_.-]+\.` + domain) | |
15 | if err != nil { | |
16 | return nil, err | |
17 | } | |
18 | return extractor, nil | |
19 | } | |
20 | ||
21 | // Exists check if a key exist in a slice | |
22 | func Exists(values []string, key string) bool { | |
23 | for _, v := range values { | |
24 | if v == key { | |
25 | return true | |
26 | } | |
27 | } | |
28 | return false | |
29 | } |