diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..8a7cb14 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,20 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[Issue] " +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**Subfinder version** +Include the version of subfinder you are using, `subfinder -version` + +**Complete command you used to reproduce this** + + +**Screenshots** +Add screenshots of the error for a better context. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..4a4ea46 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,14 @@ +blank_issues_enabled: false + +contact_links: + - name: Ask an question / advise on using subfinder + url: https://github.com/projectdiscovery/subfinder/discussions/categories/q-a + about: Ask a question or request support for using subfinder + + - name: Share idea / feature to discuss for subfinder + url: https://github.com/projectdiscovery/subfinder/discussions/categories/ideas + about: Share idea / feature to discuss for subfinder + + - name: Connect with PD Team (Discord) + url: https://discord.gg/projectdiscovery + about: Connect with PD Team for direct communication \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..12248b8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,18 @@ +--- +name: Feature request +about: Request feature to implement in this project +labels: 'Type: Enhancement' +--- + + + +### Please describe your feature request: + + +### Describe the use case of this feature: + diff --git a/.github/ISSUE_TEMPLATE/issue-report.md b/.github/ISSUE_TEMPLATE/issue-report.md new file mode 100644 index 0000000..c256706 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/issue-report.md @@ -0,0 +1,36 @@ +--- +name: Issue report +about: Create a report to help us to improve the project +labels: 'Type: Bug' + +--- + + + + + +### Subfinder version: + + + + +### Current Behavior: + + +### Expected Behavior: + + +### Steps To Reproduce: + + + +### Anything else: + \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..89561d7 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,43 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + target-branch: "dev" + commit-message: + prefix: "chore" + include: "scope" + labels: + - "Type: Maintenance" + + # Maintain dependencies for go modules + - package-ecosystem: "gomod" + directory: "v2/" + schedule: + interval: "daily" + target-branch: "dev" + commit-message: + prefix: "chore" + include: "scope" + labels: + - "Type: Maintenance" + + # Maintain dependencies for docker + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + target-branch: "dev" + commit-message: + prefix: "chore" + include: "scope" + labels: + - "Type: Maintenance" \ No newline at end of file diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml new file mode 100644 index 0000000..d05d6e2 --- /dev/null +++ b/.github/workflows/build-test.yml @@ -0,0 +1,40 @@ +name: 🔨 Build Test +on: + push: + pull_request: + workflow_dispatch: + +jobs: + build: + name: Test Builds + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + steps: + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.18 + + - name: Check out code + uses: actions/checkout@v3 + + - name: Build + run: go build ./... + working-directory: v2/ + + - name: Test + run: go test ./... + working-directory: v2/ + + - name: Integration Tests + env: + GH_ACTION: true + DNSREPO_API_KEY: ${{secrets.DNSREPO_API}} + run: bash run.sh + working-directory: v2/cmd/integration-test/ + + - name: Race Condition Tests + run: go build -race ./... + working-directory: v2/ \ No newline at end of file diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml deleted file mode 100644 index 9fbedbb..0000000 --- a/.github/workflows/build.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: Build -on: - push: - branches: - - master - pull_request: - -jobs: - build: - name: Build - runs-on: ubuntu-latest - steps: - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.13 - - - name: Check out code - uses: actions/checkout@v2 - - - name: Test - run: go test . - working-directory: cmd/subfinder/ - - - name: Build - run: go build . - working-directory: cmd/subfinder/ \ No newline at end of file diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..9f533f8 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,38 @@ +name: 🚨 CodeQL Analysis + +on: + workflow_dispatch: + pull_request: + branches: + - dev + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 \ No newline at end of file diff --git a/.github/workflows/dockerhub-push-on-release.yml b/.github/workflows/dockerhub-push-on-release.yml deleted file mode 100644 index 09ab9cc..0000000 --- a/.github/workflows/dockerhub-push-on-release.yml +++ /dev/null @@ -1,17 +0,0 @@ -# dockerhub-push pushes docker build to dockerhub automatically -# on the creation of a new release -name: Publish to Dockerhub on creation of a new release -on: - release: - types: [published] -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Publish to Dockerhub Registry - uses: elgohr/Publish-Docker-Github-Action@master - with: - name: projectdiscovery/subfinder - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} \ No newline at end of file diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml new file mode 100644 index 0000000..35f2ba5 --- /dev/null +++ b/.github/workflows/dockerhub-push.yml @@ -0,0 +1,41 @@ +name: 🌥 Docker Push + +on: + workflow_run: + workflows: ["🎉 Release Binary"] + types: + - completed + workflow_dispatch: + +jobs: + docker: + runs-on: ubuntu-latest + steps: + + - name: Checkout + uses: actions/checkout@v3 + + - name: Get Github tag + id: meta + run: | + echo "::set-output name=tag::$(curl --silent "https://api.github.com/repos/projectdiscovery/subfinder/releases/latest" | jq -r .tag_name)" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64,linux/arm64,linux/arm + push: true + tags: projectdiscovery/subfinder:latest,projectdiscovery/subfinder:${{ steps.meta.outputs.tag }} \ No newline at end of file diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml new file mode 100644 index 0000000..86aaaaf --- /dev/null +++ b/.github/workflows/lint-test.yml @@ -0,0 +1,21 @@ +name: 🙏🏻 Lint Test +on: + push: + pull_request: + workflow_dispatch: + +jobs: + lint: + name: Lint Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: 1.18 + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v3.3.1 + with: + version: latest + args: --timeout 5m + working-directory: v2/ \ No newline at end of file diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml new file mode 100644 index 0000000..703acf7 --- /dev/null +++ b/.github/workflows/release-binary.yml @@ -0,0 +1,31 @@ +name: 🎉 Release Binary + +on: + create: + tags: + - v* + workflow_dispatch: + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: "Check out code" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: "Set up Go" + uses: actions/setup-go@v3 + with: + go-version: 1.18 + + - name: "Create release on GitHub" + uses: goreleaser/goreleaser-action@v3 + with: + args: "release --rm-dist" + version: latest + workdir: v2/ + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}" \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 70cb60a..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Release -on: - create: - tags: - - v* - -jobs: - release: - runs-on: ubuntu-latest - steps: - - - name: "Check out code" - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: "Set up Go" - uses: actions/setup-go@v2 - with: - go-version: 1.14 - - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - name: "Create release on GitHub" - uses: goreleaser/goreleaser-action@v2 - with: - args: "release --rm-dist" - version: latest \ No newline at end of file diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml new file mode 100644 index 0000000..e8c0344 --- /dev/null +++ b/.github/workflows/sonarcloud.yml @@ -0,0 +1,40 @@ +name: 👮🏼‍♂️ Sonarcloud +on: + push: + branches: + - master + - dev + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: + +jobs: + sonarcloud: + name: SonarCloud + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + + - name: "Set up Go" + uses: actions/setup-go@v3 + with: + go-version: 1.18 + + - name: Run unit Tests + working-directory: v2/ + run: | + go test -coverprofile=./cov.out ./... + + - name: Run Gosec Security Scanner + working-directory: v2/ + run: | + go install github.com/securego/gosec/cmd/gosec@latest + gosec -no-fail -fmt=sonarqube -out report.json ./... + + - name: SonarCloud Scan + uses: SonarSource/sonarcloud-github-action@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index d2af926..0925ef5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,9 @@ .DS_Store cmd/subfinder/subfinder +# subfinder binary when built with `go build` +v2/cmd/subfinder/subfinder +# subfinder binary when built with `make` +v2/subfinder vendor/ -.idea \ No newline at end of file +.idea +.devcontainer diff --git a/.goreleaser.yml b/.goreleaser.yml deleted file mode 100644 index 47d83c7..0000000 --- a/.goreleaser.yml +++ /dev/null @@ -1,21 +0,0 @@ -builds: - - binary: subfinder - main: cmd/subfinder/main.go - goos: - - linux - - windows - - darwin - goarch: - - amd64 - - 386 - - arm - - arm64 - -archives: - - id: tgz - format: tar.gz - replacements: - darwin: macOS - format_overrides: - - goos: windows - format: zip \ No newline at end of file diff --git a/DISCLAIMER.md b/DISCLAIMER.md index 2b2d986..114a152 100644 --- a/DISCLAIMER.md +++ b/DISCLAIMER.md @@ -2,13 +2,12 @@ Subfinder leverages multiple open APIs, it is developed for individuals to help them for research or internal work. If you wish to incorporate this tool into a commercial offering or purposes, you must agree to the Terms of the leveraged services: -- Project Sonar / Bufferover: https://opendata.rapid7.com/about +- Bufferover: https://tls.bufferover.run - CommonCrawl: https://commoncrawl.org/terms-of-use/full - certspotter: https://sslmate.com/terms - dnsdumpster: https://hackertarget.com/terms -- entrust: https://www.entrustdatacard.com/pages/terms-of-use - Google Transparency: https://policies.google.com/terms -- Threatcrowd: https://www.alienvault.com/terms/website-terms-of-use07may2018 +- Alienvault: https://www.alienvault.com/terms/website-terms-of-use07may2018 --- diff --git a/Dockerfile b/Dockerfile index b03ce55..bbec2b6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,13 @@ -# Build Container -FROM golang:1.13.4-alpine3.10 AS build-env -MAINTAINER Ice3man (nizamul@projectdiscovery.io) -RUN apk add --no-cache --upgrade git openssh-client ca-certificates -RUN go get -u github.com/golang/dep/cmd/dep -WORKDIR /go/src/app +# Build -# Install -RUN go get -u github.com/projectdiscovery/subfinder/cmd/subfinder +FROM golang:1.19.2-alpine AS build-env +RUN apk add build-base +RUN go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest + +# Release +FROM alpine:3.16.3 +RUN apk -U upgrade --no-cache \ + && apk add --no-cache bind-tools ca-certificates +COPY --from=build-env /go/bin/subfinder /usr/local/bin/subfinder ENTRYPOINT ["subfinder"] diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md deleted file mode 100644 index a84de56..0000000 --- a/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,22 +0,0 @@ -## What's the problem (or question)? - - - -## Do you have an idea for a solution? - - - -## How can we reproduce the issue? - -1. -2. -3. -4. - -## What are the running context details? - -* Installation method (e.g. `pip`, `apt-get`, `git clone` or `zip`/`tar.gz`): -* Client OS (e.g. `Microsoft Windows 10`) -* Program version (see banner): -* Relevant console output (if any): -* Exception traceback (if any): diff --git a/LICENSE.md b/LICENSE.md index 5e85035..b22968b 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ MIT License -Copyright (c) Exposed Atoms Pvt Ltd +Copyright (c) 2021 ProjectDiscovery, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 2d7f5f7..9e9cff1 100644 --- a/README.md +++ b/README.md @@ -1,272 +1,199 @@ -

- subfinder +

+ subfinder

- -[![License](https://img.shields.io/badge/license-MIT-_red.svg)](https://opensource.org/licenses/MIT) -[![Go Report Card](https://goreportcard.com/badge/github.com/projectdiscovery/subfinder)](https://goreportcard.com/report/github.com/projectdiscovery/subfinder) -[![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/projectdiscovery/subfinder/issues) - - -subfinder is a subdomain discovery tool that discovers valid subdomains for websites by using passive online sources. It has a simple modular architecture and is optimized for speed. subfinder is built for doing one thing only - passive subdomain enumeration, and it does that very well. - -We have designed subfinder to comply with all passive sources licenses, and usage restrictions, as well as maintained a consistently passive model to make it useful to both penetration testers and bug bounty hunters alike. - - -# Resources -- [Features](#features) -- [Usage](#usage) -- [Installation Instuctions (direct)](#direct-installation) -- [Installation Instructions](#installation-instructions) - - [From Binary](#from-binary) - - [From Source](#from-source) - - [From Github](#from-github) -- [Upgrading](#upgrading) -- [Post Installation Instructions](#post-installation-instructions) -- [Running subfinder](#running-subfinder) -- [Running in a Docker Container](#running-in-a-docker-container) - - - # Features +

Fast passive subdomain enumeration tool.

+ + +

+ + + + + +

+ +

+ Features • + Install • + Usage • + API Setup • + Library • + Join Discord +

+ +--- + + +`subfinder` is a subdomain discovery tool that returns valid subdomains for websites, using passive online sources. It has a simple, modular architecture and is optimized for speed. `subfinder` is built for +doing one thing only - passive subdomain enumeration, and it does that very well. + +We have made it to comply with all the used passive source licenses and usage restrictions. The passive model guarantees speed and stealthiness that can be leveraged by both penetration testers and bug bounty +hunters alike. + +# Features

subfinder

- - - Simple and modular code base making it easy to contribute. - - Fast And Powerful Resolution and wildcard elimination module - - **Curated** passive sources to maximize results (26 Sources as of now) - - Multiple Output formats supported (Json, File, Stdout) - - Optimized for speed, very fast and **lightweight** on resources - - **Stdin** and **stdout** support for integrating in workflows - +- Fast and powerful resolution and wildcard elimination modules +- **Curated** passive sources to maximize results +- Multiple output formats supported (JSON, file, stdout) +- Optimized for speed and **lightweight** on resources +- **STDIN/OUT** support enables easy integration into workflows # Usage -```bash +```sh subfinder -h ``` + This will display help for the tool. Here are all the switches it supports. -| Flag | Description | Example | -|------|-------------|---------| -| -cd | Upload results to the Chaos API (api-key required) | subfinder -d uber.com -cd | -| -config string | Configuration file for API Keys, etc | subfinder -config config.yaml | -| -d | Domain to find subdomains for | subfinder -d uber.com | -| -dL | File containing list of domains to enumerate | subfinder -dL hackerone-hosts.txt | -| -exclude-sources | List of sources to exclude from enumeration | subfinder -exclude-sources archiveis | -| -max-time | Minutes to wait for enumeration results (default 10) | subfinder -max-time 1 | -| -nC | Don't Use colors in output | subfinder -nC | -| -nW | Remove Wildcard & Dead Subdomains from output | subfinder -nW | -| -ls | List all available sources | subfinder -ls | -| -o | File to write output to (optional) | subfinder -o output.txt | -| -oD | Directory to write enumeration results to (optional) | subfinder -oD ~/outputs | -| -oI | Write output in Host,IP format | subfinder -oI | -| -oJ | Write output in JSON lines Format | subfinder -oJ | -| -r | Comma-separated list of resolvers to use | subfinder -r 1.1.1.1,1.0.0.1 | -| -rL | Text file containing list of resolvers to use | subfinder -rL resolvers.txt -| -silent | Show only subdomains in output | subfinder -silent | -| -sources | Comma separated list of sources to use | subfinder -sources shodan,censys | -| -t | Number of concurrent goroutines for resolving (default 10) | subfinder -t 100 | -| -timeout | Seconds to wait before timing out (default 30) | subfinder -timeout 30 | -| -v | Show Verbose output | subfinder -v | -| -version | Show current program version | subfinder -version | - - -# Installation Instructions - -### From Binary - -The installation is easy. You can download the pre-built binaries for different platforms from the [releases](https://github.com/projectdiscovery/subfinder/releases/) page. Extract them using tar, move it to your `$PATH` and you're ready to go. - -```bash -> tar -xzvf subfinder-linux-amd64.tar.gz -> mv subfinder /usr/local/local/bin/ -> subfinder -h -``` - -### From Source - -subfinder requires go1.13+ to install successfully. Run the following command to get the repo - - -```bash -GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/cmd/subfinder -``` - -### From Github - -```bash -git clone https://github.com/projectdiscovery/subfinder.git -cd subfinder/cmd/subfinder -go build . -mv subfinder /usr/local/bin/ -subfinder -h -``` - -### Upgrading -If you wish to upgrade the package you can use: - -```bash -GO111MODULE=on go get -u -v github.com/projectdiscovery/subfinder/cmd/subfinder +```yaml +Usage: + ./subfinder [flags] + +Flags: +INPUT: + -d, -domain string[] domains to find subdomains for + -dL, -list string file containing list of domains for subdomain discovery + +SOURCE: + -s, -sources string[] specific sources to use for discovery (-s crtsh,github). Use -ls to display all available sources. + -recursive use only sources that can handle subdomains recursively (e.g. subdomain.domain.tld vs domain.tld) + -all use all sources for enumeration (slow) + -es, -exclude-sources string[] sources to exclude from enumeration (-es alienvault,zoomeye) + +FILTER: + -m, -match string[] subdomain or list of subdomain to match (file or comma separated) + -f, -filter string[] subdomain or list of subdomain to filter (file or comma separated) + +RATE-LIMIT: + -rl, -rate-limit int maximum number of http requests to send per second + -t int number of concurrent goroutines for resolving (-active only) (default 10) + +OUTPUT: + -o, -output string file to write output to + -oJ, -json write output in JSONL(ines) format + -oD, -output-dir string directory to write output (-dL only) + -cs, -collect-sources include all sources in the output (-json only) + -oI, -ip include host IP in output (-active only) + +CONFIGURATION: + -config string flag config file (default "$HOME/.config/subfinder/config.yaml") + -pc, -provider-config string provider config file (default "$HOME/.config/subfinder/provider-config.yaml") + -r string[] comma separated list of resolvers to use + -rL, -rlist string file containing list of resolvers to use + -nW, -active display active subdomains only + -proxy string http proxy to use with subfinder + -ei, -exclude-ip exclude IPs from the list of domains + +DEBUG: + -silent show only subdomains in output + -version show version of subfinder + -v show verbose output + -nc, -no-color disable color in output + -ls, -list-sources list all available sources + +OPTIMIZATION: + -timeout int seconds to wait before timing out (default 30) + -max-time int minutes to wait for enumeration results (default 10) +``` + +# Installation + +`subfinder` requires **go1.18** to install successfully. Run the following command to install the latest version: + +```sh +go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest ``` ## Post Installation Instructions -Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key: - -- [Virustotal](https://www.virustotal.com) -- [Passivetotal](http://passivetotal.org) -- [SecurityTrails](http://securitytrails.com) -- [Censys](https://censys.io) -- [Binaryedge](https://binaryedge.io) -- [Shodan](https://shodan.io) -- [URLScan](https://urlscan.io) -- [Chaos](https://chaos.projectdiscovery.io) -- [Spyse](https://spyse.com) -- [DnsDB](https://api.dnsdb.info) -- [Zoomeye](https://www.zoomeye.org) -- [Github](https://github.com) -- [Intelx](https://intelx.io) - -Theses values are stored in the `$HOME/.config/subfinder/config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys can be specified for each of these services from which one of them will be used for enumeration. - -For sources that require multiple keys, namely `Censys`, `Passivetotal`, they can be added by separating them via a colon (:). - -An example config file - +`subfinder` can be used right after the installation, however the following services require configuring API keys to work: + +[BeVigil](https://bevigil.com/osint-api), [BinaryEdge](https://binaryedge.io), [BufferOver](https://tls.bufferover.run), [C99](https://api.c99.nl/), [Censys](https://censys.io), [CertSpotter](https://sslmate.com/certspotter/api/), [Chaos](https://chaos.projectdiscovery.io), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [FullHunt](https://fullhunt.io), [GitHub](https://github.com), [Intelx](https://intelx.io), [PassiveTotal](http://passivetotal.org), [quake](https://quake.360.cn), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [ThreatBook](https://x.threatbook.cn/en), [VirusTotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [ZoomEye](https://www.zoomeye.org), [ZoomEye API](https://api.zoomeye.org), [dnsrepo](https://dnsrepo.noc.org), [Hunter](https://hunter.qianxin.com/) + +You can also use the `subfinder -ls` command to display all the available sources. + +These values are stored in the `$HOME/.config/subfinder/provider-config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys +can be specified for each of these services from which one of them will be used for enumeration. + +Composite keys for sources like, `Censys`, `PassiveTotal`, `Fofa`, `Intellix` and `ZoomEye`, need to be separated with a colon (`:`). + +An example provider config file: ```yaml -resolvers: - - 1.1.1.1 - - 1.0.0.1 -sources: - - binaryedge - - bufferover - - censys - - passivetotal - - sitedossier binaryedge: - 0bf8919b-aab9-42e4-9574-d3b639324597 - ac244e2f-b635-4581-878a-33f4e79a2c13 censys: - ac244e2f-b635-4581-878a-33f4e79a2c13:dd510d6e-1b6e-4655-83f6-f347b363def9 certspotter: [] -passivetotal: +passivetotal: - sample-email@user.com:sample_password securitytrails: [] shodan: - AAAAClP1bJJSRMEYJazgwhJKrggRwKA github: - - d23a554bbc1aabb208c9acfbd2dd41ce7fc9db39 - - asdsd54bbc1aabb208c9acfbd2dd41ce7fc9db39 + - ghp_lkyJGU3jv1xmwk4SDXavrLDJ4dl2pSJMzj4X + - ghp_gkUuhkIYdQPj13ifH4KA3cXRn8JD2lqir2d4 +zoomeye: + - zoomeye_username:zoomeye_password ``` # Running Subfinder To run the tool on a target, just use the following command. -```bash -> subfinder -d freelancer.com -``` - -This will run the tool against freelancer.com. There are a number of configuration options that you can pass along with this command. The verbose switch (-v) can be used to display verbose information. - -```bash -[CERTSPOTTER] www.fi.freelancer.com -[DNSDUMPSTER] hosting.freelancer.com -[DNSDUMPSTER] support.freelancer.com -[DNSDUMPSTER] accounts.freelancer.com -[DNSDUMPSTER] phabricator.freelancer.com -[DNSDUMPSTER] cdn1.freelancer.com -[DNSDUMPSTER] t1.freelancer.com -[DNSDUMPSTER] wdc.t1.freelancer.com -[DNSDUMPSTER] dal.t1.freelancer.com -``` - -The `-silent` switch can be used to show only subdomains found without any other info. - - -The `-o` command can be used to specify an output file. - -```bash -> subfinder -d freelancer.com -o output.txt -``` - -To run the tool on a list of domains, `-dL` option can be used. This requires a directory to write the output files. Subdomains for each domain from the list are written in a text file in the directory specified by the `-oD` flag with their name being the domain name. - -```bash -> cat domains.txt + +```console +subfinder -d hackerone.com + + __ _____ __ + _______ __/ /_ / __(_)___ ____/ /__ _____ + / ___/ / / / __ \/ /_/ / __ \/ __ / _ \/ ___/ + (__ ) /_/ / /_/ / __/ / / / / /_/ / __/ / +/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.4.9 + + projectdiscovery.io + +Use with caution. You are responsible for your actions +Developers assume no liability and are not responsible for any misuse or damage. +By using subfinder, you also agree to the terms of the APIs used. + +[INF] Enumerating subdomains for hackerone.com + +www.hackerone.com +support.hackerone.com +links.hackerone.com +api.hackerone.com +o1.email.hackerone.com +go.hackerone.com +3d.hackerone.com +resources.hackerone.com +a.ns.hackerone.com +b.ns.hackerone.com +mta-sts.hackerone.com +docs.hackerone.com +mta-sts.forwarding.hackerone.com +gslink.hackerone.com hackerone.com -google.com - -> subfinder -dL domains.txt -oD ~/path/to/output -> ls ~/path/to/output - -hackerone.com.txt -google.com.txt -``` - -If you want to save results to a single file while using a domain list, specify the `-o` flag with the name of the output file. - - -```bash -> cat domains.txt -hackerone.com -google.com - -> subfinder -dL domains.txt -o ~/path/to/output.txt -> ls ~/path/to/ - -output.txt -``` - -If you want upload your data to chaos dataset, you can use `-cd` flag with your scan, chaos will resolve all the input and add valid subdomains to public dataset, which you can access on the go using [chaos-client](https://github.com/projectdiscovery/chaos-client) - -```bash -> subfinder -d hackerone.com -cd - -root@b0x:~# subfinder -d hackerone.com -cd - -www.hackerone.com -api.hackerone.com -go.hackerone.com -hackerone.com -staging.hackerone.com -[INF] Input processed successfully and subdomains with valid records will be updated to chaos dataset. -``` - -You can also get output in json format using `-oJ` switch. This switch saves the output in the JSON lines format. - -If you use the JSON format, or the `Host:IP` format, then it becomes mandatory for you to use the **-nW** format as resolving is essential for these output format. By default, resolving the found subdomains is disabled. - -```bash -> subfinder -d hackerone.com -o output.json -oJ -nW -> cat output.json - -{"host":"www.hackerone.com","ip":"104.16.99.52"} -{"host":"mta-sts.hackerone.com","ip":"185.199.108.153"} -{"host":"hackerone.com","ip":"104.16.100.52"} -{"host":"mta-sts.managed.hackerone.com","ip":"185.199.110.153"} -``` - -You can specify custom resolvers too. -```bash -> subfinder -d freelancer.com -o result.txt -nW -v -r 8.8.8.8,1.1.1.1 -> subfinder -d freelancer.com -o result.txt -nW -v -rL resolvers.txt -``` - -**The new highlight of this release is the addition of stdin/stdout features.** Now, domains can be piped to subfinder and enumeration can be ran on them. For example - - -```bash -> echo hackerone.com | subfinder -v -> cat targets.txt | subfinder -v -``` - -The subdomains discovered can be piped to other tools too. For example, you can pipe the subdomains discovered by subfinder to httpx [httpx](https://github.com/projectdiscovery/httpx) which will then find running http servers on the host. - -```bash -> echo hackerone.com | subfinder -silent | httpx -silent +info.hackerone.com +mta-sts.managed.hackerone.com +events.hackerone.com + +[INF] Found 18 subdomains for hackerone.com in 3 seconds 672 milliseconds +``` + +The subdomains discovered can be piped to other tools too. For example, you can pipe the discovered subdomains to [`httpx`](https://github.com/projectdiscovery/httpx) which will then find +running HTTP servers on the host. + +```console +echo hackerone.com | subfinder -silent | httpx -silent http://hackerone.com http://www.hackerone.com @@ -276,50 +203,92 @@ http://mta-sts.managed.hackerone.com ``` -## Running in a Docker Container - -You can use the official dockerhub image at [subfinder](https://hub.docker.com/r/projectdiscovery/subfinder). Simply run - - -```bash -> docker pull projectdiscovery/subfinder -``` - -The above command will pull the latest tagged release from the dockerhub repository. - -If you want to build the container yourself manually, git clone the repo, then build and run the following commands - -- Clone the repo using `git clone https://github.com/projectdiscovery/subfinder.git` -- Build your docker container -```bash -docker build -t projectdiscovery/subfinder . -``` - -- After building the container using either way, run the following - -```bash -docker run -it projectdiscovery/subfinder -``` -> The above command is the same as running `-h` - -If you are using docker, you need to first create your directory structure holding subfinder configuration file. After modifying the default config.yaml file, you can run: - -```bash -> mkdir -p $HOME/.config/subfinder -> cp config.yaml $HOME/.config/subfinder/config.yaml -> nano $HOME/.config/subfinder/config.yaml -``` - -After that, you can pass it as a volume using the following sample command. -```bash -> docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it projectdiscovery/subfinder -d freelancer.com -``` - -For example, this runs the tool against uber.com and output the results to your host file system: -```bash -docker run -v $HOME/.config/subfinder:/root/.config/subfinder -it projectdiscovery/subfinder -d uber.com > uber.com.txt -``` + + + + +
+ +## Subfinder with docker + +Pull the latest tagged [subfinder](https://hub.docker.com/r/projectdiscovery/subfinder) docker image: + +```sh +docker pull projectdiscovery/subfinder:latest +``` + +Running `subfinder` using the docker image: + +```sh +docker run projectdiscovery/subfinder:latest -d hackerone.com +``` + +Running `subfinder` using the docker image, with a local config file: + +```sh +docker run -v $HOME/.config/subfinder:/root/.config/subfinder -t projectdiscovery/subfinder -d hackerone.com +``` + +
+ + + + + +
+ +## Subfinder Go library + +Usage example: + +```go +package main + +import ( + "bytes" + "fmt" + "io" + "log" + + "github.com/projectdiscovery/subfinder/v2/pkg/resolve" + "github.com/projectdiscovery/subfinder/v2/pkg/runner" +) + +func main() { + runnerInstance, err := runner.NewRunner(&runner.Options{ + Threads: 10, // Thread controls the number of threads to use for active enumerations + Timeout: 30, // Timeout is the seconds to wait for sources to respond + MaxEnumerationTime: 10, // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration + Resolvers: resolve.DefaultResolvers, // Use the default list of resolvers by marshaling it to the config + ResultCallback: func(s *resolve.HostEntry) { // Callback function to execute for available host + log.Println(s.Host, s.Source) + }, + }) + + buf := bytes.Buffer{} + err = runnerInstance.EnumerateSingleDomain("projectdiscovery.io", []io.Writer{&buf}) + if err != nil { + log.Fatal(err) + } + + data, err := io.ReadAll(&buf) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s", data) +} +``` + +
+ +### Resources + +- [Recon with Me !!!](https://dhiyaneshgeek.github.io/bug/bounty/2020/02/06/recon-with-me/) # License -subfinder is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See the **[Thanks.md](https://github.com/projectdiscovery/subfinder/blob/master/THANKS.md)** file for more details. - -Read the disclaimer for usage at [DISCLAIMER.md](https://github.com/projectdiscovery/subfinder/blob/master/DISCLAIMER.md) and [contact us](mailto:contact@projectdiscovery.io) for any API removal. +`subfinder` is made with 🖤 by the [projectdiscovery](https://projectdiscovery.io) team. Community contributions have made the project what it is. See +the **[THANKS.md](https://github.com/projectdiscovery/subfinder/blob/master/THANKS.md)** file for more details. + +Read the usage disclaimer at [DISCLAIMER.md](https://github.com/projectdiscovery/subfinder/blob/master/DISCLAIMER.md) and [contact us](mailto:contact@projectdiscovery.io) for any API removal. diff --git a/THANKS.md b/THANKS.md index 56fa171..b853afc 100644 --- a/THANKS.md +++ b/THANKS.md @@ -4,8 +4,9 @@ - All the contributors at [CONTRIBUTORS](https://github.com/projectdiscovery/subfinder/graphs/contributors) who made subfinder what it is. -We'd like to thank some additional amazing people, wo contributed a lot in subfinder's journey - +We'd like to thank some additional amazing people, who contributed a lot in subfinder's journey - -- @infosec-au - Donating to the project -- @codingo - Initial work on the project, managing it, lot of work! -- @picatz - Improving the structure of the project a lot. New ideas! \ No newline at end of file +- [@vzamanillo](https://github.com/vzamanillo) - For adding multiple features and overall project improvements. +- [@infosec-au](https://github.com/infosec-au) - Donating to the project. +- [@codingo](https://github.com/codingo) - Initial work on the project, managing it, lot of work! +- [@picatz](https://github.com/picatz) - Improving the structure of the project a lot. New ideas! \ No newline at end of file diff --git a/cmd/subfinder/main.go b/cmd/subfinder/main.go deleted file mode 100644 index 7d84d89..0000000 --- a/cmd/subfinder/main.go +++ /dev/null @@ -1,21 +0,0 @@ -package main - -import ( - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/subfinder/pkg/runner" -) - -func main() { - // Parse the command line flags and read config files - options := runner.ParseOptions() - - runner, err := runner.NewRunner(options) - if err != nil { - gologger.Fatalf("Could not create runner: %s\n", err) - } - - err = runner.RunEnumeration() - if err != nil { - gologger.Fatalf("Could not run enumeration: %s\n", err) - } -} diff --git a/config.yaml b/config.yaml deleted file mode 100644 index 7155cd5..0000000 --- a/config.yaml +++ /dev/null @@ -1,68 +0,0 @@ -resolvers: - - 1.1.1.1 - - 1.0.0.1 - - 8.8.8.8 - - 8.8.4.4 - - 9.9.9.9 - - 9.9.9.10 - - 77.88.8.8 - - 77.88.8.1 - - 208.67.222.222 - - 208.67.220.220 -sources: - - alienvault - - archiveis - - binaryedge - - bufferover - - censys - - certspotter - - certspotterold - - commoncrawl - - crtsh - - dnsdumpster - - dnsdb - - entrust - - github - - googleter - - hackertarget - - intelx - - ipv4info - - passivetotal - - rapiddns - - securitytrails - - shodan - - sitedossier - - sublist3r - - spyse - - threatcrowd - - threatminer - - urlscan - - virustotal - - waybackarchive - - zoomeye -censys: - - -binaryedge: - - -certspotter: - - -github: - - -intelx: - - -passivetotal: - - -securitytrails: - - -virustotal: - - -urlscan: - - -chaos: - - -spyse: - - -shodan: - - -dnsdb: - - diff --git a/debian/changelog b/debian/changelog index afbb35f..ae161b3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +subfinder (2.5.5-0kali1) UNRELEASED; urgency=low + + * New upstream release. + + -- Kali Janitor Wed, 07 Dec 2022 03:04:26 -0000 + subfinder (2.3.8-0kali1) kali-dev; urgency=medium * Initial release (see 6496) diff --git a/go.mod b/go.mod deleted file mode 100644 index f1c0979..0000000 --- a/go.mod +++ /dev/null @@ -1,15 +0,0 @@ -module github.com/projectdiscovery/subfinder - -go 1.14 - -require ( - github.com/json-iterator/go v1.1.9 - github.com/lib/pq v1.6.0 - github.com/m-mizutani/urlscan-go v1.0.0 - github.com/miekg/dns v1.1.29 - github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/gologger v1.0.0 - github.com/rs/xid v1.2.1 - github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 - gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c -) diff --git a/go.sum b/go.sum deleted file mode 100644 index 957da47..0000000 --- a/go.sum +++ /dev/null @@ -1,87 +0,0 @@ -github.com/alexbrainman/sspi v0.0.0-20180613141037-e580b900e9f5/go.mod h1:976q2ETgjT2snVCf2ZaBnyBbVoPERGjUz+0sofzEfro= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= -github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= -github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= -github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= -github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= -github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= -github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= -github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= -github.com/jcmturner/gokrb5/v8 v8.2.0 h1:lzPl/30ZLkTveYsYZPKMcgXc8MbnE6RsTd4F9KgiLtk= -github.com/jcmturner/gokrb5/v8 v8.2.0/go.mod h1:T1hnNppQsBtxW0tCHMHTkAt8n/sABdzZgZdoFrZaZNM= -github.com/jcmturner/rpc/v2 v2.0.2 h1:gMB4IwRXYsWw4Bc6o/az2HJgFUA1ffSh90i26ZJ6Xl0= -github.com/jcmturner/rpc/v2 v2.0.2/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= -github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/lib/pq v1.6.0 h1:I5DPxhYJChW9KYc66se+oKFFQX6VuQrKiprsX6ivRZc= -github.com/lib/pq v1.6.0/go.mod h1:4vXEAYvW1fRQ2/FhZ78H73A60MHw1geSm145z2mdY1g= -github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs= -github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/m-mizutani/urlscan-go v1.0.0 h1:+fTiSRCQXdy3EM1BgO5gmAHFWbccTDdoEKy9Fa7m9xo= -github.com/m-mizutani/urlscan-go v1.0.0/go.mod h1:ppEBT0e/xv0bPcVWKev4cYG7Ey8933JsOzEzovxGMjI= -github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg= -github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/projectdiscovery/gologger v1.0.0 h1:XAQ8kHeVKXMjY4rLGh7eT5+oHU077BNEvs7X6n+vu1s= -github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= -github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= -github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= -github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= -github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4 h1:QmwruyY+bKbDDL0BaglrbZABEali68eoMFhTZpCjYVA= -golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= -gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= -gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/passive/doc.go b/pkg/passive/doc.go deleted file mode 100644 index 0d7ea64..0000000 --- a/pkg/passive/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Package passive provides capability for doing passive subdomain -// enumeration on targets. -package passive - diff --git a/pkg/passive/passive.go b/pkg/passive/passive.go deleted file mode 100644 index 6881ab1..0000000 --- a/pkg/passive/passive.go +++ /dev/null @@ -1,58 +0,0 @@ -package passive - -import ( - "context" - "fmt" - "sync" - "time" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -// EnumerateSubdomains enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomains(domain string, keys subscraping.Keys, timeout int, maxEnumTime time.Duration) chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - session, err := subscraping.NewSession(domain, keys, timeout) - if err != nil { - results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} - } - - ctx, cancel := context.WithTimeout(context.Background(), maxEnumTime) - - timeTaken := make(map[string]string) - timeTakenMutex := &sync.Mutex{} - - wg := &sync.WaitGroup{} - // Run each source in parallel on the target domain - for source, runner := range a.sources { - wg.Add(1) - - now := time.Now() - go func(source string, runner subscraping.Source) { - for resp := range runner.Run(ctx, domain, session) { - results <- resp - } - - duration := time.Now().Sub(now) - timeTakenMutex.Lock() - timeTaken[source] = fmt.Sprintf("Source took %s for enumeration\n", duration) - timeTakenMutex.Unlock() - - wg.Done() - }(source, runner) - } - wg.Wait() - - for source, data := range timeTaken { - gologger.Verbosef(data, source) - } - - close(results) - cancel() - }() - - return results -} diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go deleted file mode 100644 index 30b0890..0000000 --- a/pkg/passive/sources.go +++ /dev/null @@ -1,158 +0,0 @@ -package passive - -import ( - "github.com/projectdiscovery/subfinder/pkg/subscraping" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/alienvault" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/archiveis" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/binaryedge" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/bufferover" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/censys" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/certspotter" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/certspotterold" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/commoncrawl" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/crtsh" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/dnsdb" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/dnsdumpster" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/entrust" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/github" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/hackertarget" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/intelx" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/ipv4info" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/passivetotal" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/rapiddns" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/securitytrails" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/shodan" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/sitedossier" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/spyse" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/sublist3r" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/threatcrowd" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/threatminer" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/urlscan" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/virustotal" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/waybackarchive" - "github.com/projectdiscovery/subfinder/pkg/subscraping/sources/zoomeye" -) - -// DefaultSources contains the list of sources used by default -var DefaultSources = []string{ - "alienvault", - "archiveis", - "binaryedge", - "bufferover", - "censys", - "certspotter", - "certspotterold", - "commoncrawl", - "crtsh", - "dnsdumpster", - "dnsdb", - "entrust", - "github", - "hackertarget", - "ipv4info", - "intelx", - "passivetotal", - "rapiddns", - "securitytrails", - "shodan", - "sitedossier", - "spyse", - "sublist3r", - "threatcrowd", - "threatminer", - "urlscan", - "virustotal", - "waybackarchive", - "zoomeye", -} - -// Agent is a struct for running passive subdomain enumeration -// against a given host. It wraps subscraping package and provides -// a layer to build upon. -type Agent struct { - sources map[string]subscraping.Source -} - -// New creates a new agent for passive subdomain discovery -func New(sources []string, exclusions []string) *Agent { - // Create the agent, insert the sources and remove the excluded sources - agent := &Agent{sources: make(map[string]subscraping.Source)} - - agent.addSources(sources) - agent.removeSources(exclusions) - - return agent -} - -// addSources adds the given list of sources to the source array -func (a *Agent) addSources(sources []string) { - for _, source := range sources { - switch source { - case "alienvault": - a.sources[source] = &alienvault.Source{} - case "archiveis": - a.sources[source] = &archiveis.Source{} - case "binaryedge": - a.sources[source] = &binaryedge.Source{} - case "bufferover": - a.sources[source] = &bufferover.Source{} - case "censys": - a.sources[source] = &censys.Source{} - case "certspotter": - a.sources[source] = &certspotter.Source{} - case "certspotterold": - a.sources[source] = &certspotterold.Source{} - case "commoncrawl": - a.sources[source] = &commoncrawl.Source{} - case "crtsh": - a.sources[source] = &crtsh.Source{} - case "dnsdumpster": - a.sources[source] = &dnsdumpster.Source{} - case "dnsdb": - a.sources[source] = &dnsdb.Source{} - case "entrust": - a.sources[source] = &entrust.Source{} - case "github": - a.sources[source] = &github.Source{} - case "hackertarget": - a.sources[source] = &hackertarget.Source{} - case "ipv4info": - a.sources[source] = &ipv4info.Source{} - case "intelx": - a.sources[source] = &intelx.Source{} - case "passivetotal": - a.sources[source] = &passivetotal.Source{} - case "rapiddns": - a.sources[source] = &rapiddns.Source{} - case "securitytrails": - a.sources[source] = &securitytrails.Source{} - case "shodan": - a.sources[source] = &shodan.Source{} - case "sitedossier": - a.sources[source] = &sitedossier.Source{} - case "spyse": - a.sources[source] = &spyse.Source{} - case "sublist3r": - a.sources[source] = &sublist3r.Source{} - case "threatcrowd": - a.sources[source] = &threatcrowd.Source{} - case "threatminer": - a.sources[source] = &threatminer.Source{} - case "urlscan": - a.sources[source] = &urlscan.Source{} - case "virustotal": - a.sources[source] = &virustotal.Source{} - case "waybackarchive": - a.sources[source] = &waybackarchive.Source{} - case "zoomeye": - a.sources[source] = &zoomeye.Source{} - } - } -} - -// removeSources deletes the given sources from the source map -func (a *Agent) removeSources(sources []string) { - for _, source := range sources { - delete(a.sources, source) - } -} diff --git a/pkg/resolve/client.go b/pkg/resolve/client.go deleted file mode 100644 index 26aec2c..0000000 --- a/pkg/resolve/client.go +++ /dev/null @@ -1,59 +0,0 @@ -package resolve - -import ( - "bufio" - "math/rand" - "os" - "time" -) - -// DefaultResolvers contains the default list of resolvers known to be good -var DefaultResolvers = []string{ - "1.1.1.1", // Cloudflare primary - "1.0.0.1", // Cloudlfare secondary - "8.8.8.8", // Google primary - "8.8.4.4", // Google secondary - "9.9.9.9", // Quad9 Primary - "9.9.9.10", // Quad9 Secondary - "77.88.8.8", // Yandex Primary - "77.88.8.1", // Yandex Secondary - "208.67.222.222", // OpenDNS Primary - "208.67.220.220", // OpenDNS Secondary -} - -// Resolver is a struct for resolving DNS names -type Resolver struct { - resolvers []string - rand *rand.Rand -} - -// New creates a new resolver struct with the default resolvers -func New() *Resolver { - return &Resolver{ - resolvers: []string{}, - rand: rand.New(rand.NewSource(time.Now().UnixNano())), - } -} - -// AppendResolversFromFile appends the resolvers read from a file to the list of resolvers -func (r *Resolver) AppendResolversFromFile(file string) error { - f, err := os.Open(file) - if err != nil { - return err - } - scanner := bufio.NewScanner(f) - for scanner.Scan() { - text := scanner.Text() - if text == "" { - continue - } - r.resolvers = append(r.resolvers, text) - } - f.Close() - return scanner.Err() -} - -// AppendResolversFromSlice appends the slice to the list of resolvers -func (r *Resolver) AppendResolversFromSlice(list []string) { - r.resolvers = append(r.resolvers, list...) -} diff --git a/pkg/resolve/doc.go b/pkg/resolve/doc.go deleted file mode 100644 index e14f359..0000000 --- a/pkg/resolve/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package resolve is used to handle resolving records -// It also handles wildcard subdomains and rotating resolvers. -package resolve diff --git a/pkg/resolve/resolve.go b/pkg/resolve/resolve.go deleted file mode 100644 index b935202..0000000 --- a/pkg/resolve/resolve.go +++ /dev/null @@ -1,150 +0,0 @@ -package resolve - -import ( - "sync" - - "github.com/miekg/dns" - "github.com/rs/xid" -) - -const ( - maxResolveRetries = 5 - maxWildcardChecks = 3 -) - -// ResolutionPool is a pool of resolvers created for resolving subdomains -// for a given host. -type ResolutionPool struct { - *Resolver - Tasks chan string - Results chan Result - wg *sync.WaitGroup - removeWildcard bool - - wildcardIPs map[string]struct{} -} - -// Result contains the result for a host resolution -type Result struct { - Type ResultType - Host string - IP string - Error error -} - -// ResultType is the type of result found -type ResultType int - -// Types of data result can return -const ( - Subdomain ResultType = iota - Error -) - -// NewResolutionPool creates a pool of resolvers for resolving subdomains of a given domain -func (r *Resolver) NewResolutionPool(workers int, removeWildcard bool) *ResolutionPool { - resolutionPool := &ResolutionPool{ - Resolver: r, - Tasks: make(chan string), - Results: make(chan Result), - wg: &sync.WaitGroup{}, - removeWildcard: removeWildcard, - wildcardIPs: make(map[string]struct{}), - } - - go func() { - for i := 0; i < workers; i++ { - resolutionPool.wg.Add(1) - go resolutionPool.resolveWorker() - } - resolutionPool.wg.Wait() - close(resolutionPool.Results) - }() - - return resolutionPool -} - -// InitWildcards inits the wildcard ips array -func (r *ResolutionPool) InitWildcards(domain string) error { - for i := 0; i < maxWildcardChecks; i++ { - uid := xid.New().String() - - hosts, err := r.getARecords(uid + "." + domain) - if err != nil { - return err - } - - // Append all wildcard ips found for domains - for _, host := range hosts { - r.wildcardIPs[host] = struct{}{} - } - } - return nil -} - -func (r *ResolutionPool) resolveWorker() { - for task := range r.Tasks { - if !r.removeWildcard { - r.Results <- Result{Type: Subdomain, Host: task, IP: ""} - continue - } - - hosts, err := r.getARecords(task) - if err != nil { - r.Results <- Result{Type: Error, Error: err} - continue - } - - if len(hosts) == 0 { - continue - } - - for _, host := range hosts { - // Ignore the host if it exists in wildcard ips map - if _, ok := r.wildcardIPs[host]; ok { - continue - } - } - - r.Results <- Result{Type: Subdomain, Host: task, IP: hosts[0]} - } - r.wg.Done() -} - -// getARecords gets all the A records for a given host -func (r *ResolutionPool) getARecords(host string) ([]string, error) { - var iteration int - - m := new(dns.Msg) - m.Id = dns.Id() - m.RecursionDesired = true - m.Question = make([]dns.Question, 1) - m.Question[0] = dns.Question{ - Name: dns.Fqdn(host), - Qtype: dns.TypeA, - Qclass: dns.ClassINET, - } -exchange: - iteration++ - in, err := dns.Exchange(m, r.resolvers[r.rand.Intn(len(r.resolvers))]+":53") - if err != nil { - // Retry in case of I/O error - if iteration <= maxResolveRetries { - goto exchange - } - return nil, err - } - // Ignore the error in case we have bad result - if in != nil && in.Rcode != dns.RcodeSuccess { - return nil, nil - } - - var hosts []string - for _, record := range in.Answer { - if t, ok := record.(*dns.A); ok { - hosts = append(hosts, t.A.String()) - } - } - - return hosts, nil -} diff --git a/pkg/runner/banners.go b/pkg/runner/banners.go deleted file mode 100644 index fffc8f0..0000000 --- a/pkg/runner/banners.go +++ /dev/null @@ -1,57 +0,0 @@ -package runner - -import ( - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/subfinder/pkg/passive" - "github.com/projectdiscovery/subfinder/pkg/resolve" -) - -const banner = ` - _ __ _ _ -____ _| |__ / _(_)_ _ __| |___ _ _ -(_-< || | '_ \ _| | ' \/ _ / -_) '_| -/__/\_,_|_.__/_| |_|_||_\__,_\___|_| v2 -` - -// Version is the current version of subfinder -const Version = `2.3.8` - -// showBanner is used to show the banner to the user -func showBanner() { - gologger.Printf("%s\n", banner) - gologger.Printf("\t\tprojectdiscovery.io\n\n") - - gologger.Labelf("Use with caution. You are responsible for your actions\n") - gologger.Labelf("Developers assume no liability and are not responsible for any misuse or damage.\n") - gologger.Labelf("By using subfinder, you also agree to the terms of the APIs used.\n\n") -} - -// normalRunTasks runs the normal startup tasks -func (options *Options) normalRunTasks() { - configFile, err := UnmarshalRead(options.ConfigFile) - if err != nil { - gologger.Fatalf("Could not read configuration file %s: %s\n", options.ConfigFile, err) - } - options.YAMLConfig = configFile -} - -// firstRunTasks runs some housekeeping tasks done -// when the program is ran for the first time -func (options *Options) firstRunTasks() { - // Create the configuration file and display information - // about it to the user. - config := ConfigFile{ - // Use the default list of resolvers by marshalling it to the config - Resolvers: resolve.DefaultResolvers, - // Use the default list of passive sources - Sources: passive.DefaultSources, - } - - err := config.MarshalWrite(options.ConfigFile) - if err != nil { - gologger.Fatalf("Could not write configuration file to %s: %s\n", options.ConfigFile, err) - } - options.YAMLConfig = config - - gologger.Infof("Configuration file saved to %s\n", options.ConfigFile) -} diff --git a/pkg/runner/config.go b/pkg/runner/config.go deleted file mode 100644 index fa693ff..0000000 --- a/pkg/runner/config.go +++ /dev/null @@ -1,169 +0,0 @@ -package runner - -import ( - "math/rand" - "os" - "strings" - "time" - - "github.com/projectdiscovery/subfinder/pkg/subscraping" - "gopkg.in/yaml.v3" -) - -// ConfigFile contains the fields stored in the configuration file -type ConfigFile struct { - // Resolvers contains the list of resolvers to use while resolving - Resolvers []string `yaml:"resolvers,omitempty"` - // Sources contains a list of sources to use for enumeration - Sources []string `yaml:"sources,omitempty"` - // ExcludeSources contains the sources to not include in the enumeration process - ExcludeSources []string `yaml:"exclude-sources,omitempty"` - // API keys for different sources - Binaryedge []string `yaml:"binaryedge"` - Censys []string `yaml:"censys"` - Certspotter []string `yaml:"certspotter"` - Chaos []string `yaml:"chaos"` - DNSDB []string `yaml:"dnsdb"` - GitHub []string `yaml:"github"` - IntelX []string `yaml:"intelx"` - PassiveTotal []string `yaml:"passivetotal"` - SecurityTrails []string `yaml:"securitytrails"` - Shodan []string `yaml:"shodan"` - Spyse []string `yaml:"spyse"` - URLScan []string `yaml:"urlscan"` - Virustotal []string `yaml:"virustotal"` - ZoomEye []string `yaml:"zoomeye"` -} - -// GetConfigDirectory gets the subfinder config directory for a user -func GetConfigDirectory() (string, error) { - // Seed the random number generator - rand.Seed(time.Now().UnixNano()) - - var config string - - directory, err := os.UserHomeDir() - if err != nil { - return config, err - } - config = directory + "/.config/subfinder" - // Create All directory for subfinder even if they exist - os.MkdirAll(config, os.ModePerm) - - return config, nil -} - -// CheckConfigExists checks if the config file exists in the given path -func CheckConfigExists(configPath string) bool { - if _, err := os.Stat(configPath); err == nil { - return true - } else if os.IsNotExist(err) { - return false - } - return false -} - -// MarshalWrite writes the marshalled yaml config to disk -func (c ConfigFile) MarshalWrite(file string) error { - f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE, 0755) - if err != nil { - return err - } - - // Indent the spaces too - enc := yaml.NewEncoder(f) - enc.SetIndent(4) - err = enc.Encode(&c) - f.Close() - return err -} - -// UnmarshalRead reads the unmarshalled config yaml file from disk -func UnmarshalRead(file string) (ConfigFile, error) { - config := ConfigFile{} - - f, err := os.Open(file) - if err != nil { - return config, err - } - err = yaml.NewDecoder(f).Decode(&config) - f.Close() - return config, err -} - -// GetKeys gets the API keys from config file and creates a Keys struct -// We use random selection of api keys from the list of keys supplied. -// Keys that require 2 options are separated by colon (:). -func (c ConfigFile) GetKeys() subscraping.Keys { - keys := subscraping.Keys{} - - if len(c.Binaryedge) > 0 { - keys.Binaryedge = c.Binaryedge[rand.Intn(len(c.Binaryedge))] - } - - if len(c.Censys) > 0 { - censysKeys := c.Censys[rand.Intn(len(c.Censys))] - parts := strings.Split(censysKeys, ":") - if len(parts) == 2 { - keys.CensysToken = parts[0] - keys.CensysSecret = parts[1] - } - } - - if len(c.Certspotter) > 0 { - keys.Certspotter = c.Certspotter[rand.Intn(len(c.Certspotter))] - } - if len(c.Chaos) > 0 { - keys.Chaos = c.Chaos[rand.Intn(len(c.Chaos))] - } - if (len(c.DNSDB)) > 0 { - keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))] - } - if (len(c.GitHub)) > 0 { - keys.GitHub = c.GitHub - } - - if len(c.IntelX) > 0 { - intelxKeys := c.IntelX[rand.Intn(len(c.IntelX))] - parts := strings.Split(intelxKeys, ":") - if len(parts) == 2 { - keys.IntelXHost = parts[0] - keys.IntelXKey = parts[1] - } - } - - if len(c.PassiveTotal) > 0 { - passiveTotalKeys := c.PassiveTotal[rand.Intn(len(c.PassiveTotal))] - parts := strings.Split(passiveTotalKeys, ":") - if len(parts) == 2 { - keys.PassiveTotalUsername = parts[0] - keys.PassiveTotalPassword = parts[1] - } - } - - if len(c.SecurityTrails) > 0 { - keys.Securitytrails = c.SecurityTrails[rand.Intn(len(c.SecurityTrails))] - } - if len(c.Shodan) > 0 { - keys.Shodan = c.Shodan[rand.Intn(len(c.Shodan))] - } - if len(c.Spyse) > 0 { - keys.Spyse = c.Spyse[rand.Intn(len(c.Spyse))] - } - if len(c.URLScan) > 0 { - keys.URLScan = c.URLScan[rand.Intn(len(c.URLScan))] - } - if len(c.Virustotal) > 0 { - keys.Virustotal = c.Virustotal[rand.Intn(len(c.Virustotal))] - } - if len(c.ZoomEye) > 0 { - zoomEyeKeys := c.ZoomEye[rand.Intn(len(c.ZoomEye))] - parts := strings.Split(zoomEyeKeys, ":") - if len(parts) == 2 { - keys.ZoomEyeUsername = parts[0] - keys.ZoomEyePassword = parts[1] - } - } - - return keys -} diff --git a/pkg/runner/config_test.go b/pkg/runner/config_test.go deleted file mode 100644 index d086174..0000000 --- a/pkg/runner/config_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package runner - -import ( - "os" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestConfigGetDirectory(t *testing.T) { - directory, err := GetConfigDirectory() - if err != nil { - t.Fatalf("Expected nil got %v while getting home\n", err) - } - home, err := os.UserHomeDir() - if err != nil { - t.Fatalf("Expected nil got %v while getting dir\n", err) - } - config := home + "/.config/subfinder" - - assert.Equal(t, directory, config, "Directory and config should be equal") -} diff --git a/pkg/runner/doc.go b/pkg/runner/doc.go deleted file mode 100644 index 744872f..0000000 --- a/pkg/runner/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package runner implements the mechanism to drive the -// subdomain enumeration process -package runner diff --git a/pkg/runner/enumerate.go b/pkg/runner/enumerate.go deleted file mode 100644 index 91794c6..0000000 --- a/pkg/runner/enumerate.go +++ /dev/null @@ -1,180 +0,0 @@ -package runner - -import ( - "bytes" - "os" - "strings" - "sync" - "time" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/subfinder/pkg/resolve" - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -// EnumerateSingleDomain performs subdomain enumeration against a single domain -func (r *Runner) EnumerateSingleDomain(domain, output string, append bool) error { - gologger.Infof("Enumerating subdomains for %s\n", domain) - - // Get the API keys for sources from the configuration - // and also create the active resolving engine for the domain. - keys := r.options.YAMLConfig.GetKeys() - - // Check if the user has asked to remove wildcards explicitly. - // If yes, create the resolution pool and get the wildcards for the current domain - var resolutionPool *resolve.ResolutionPool - if r.options.RemoveWildcard { - resolutionPool = r.resolverClient.NewResolutionPool(r.options.Threads, r.options.RemoveWildcard) - err := resolutionPool.InitWildcards(domain) - if err != nil { - // Log the error but don't quit. - gologger.Warningf("Could not get wildcards for domain %s: %s\n", domain, err) - } - } - - // Run the passive subdomain enumeration - passiveResults := r.passiveAgent.EnumerateSubdomains(domain, keys, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) - - wg := &sync.WaitGroup{} - wg.Add(1) - // Create a unique map for filtering duplicate subdomains out - uniqueMap := make(map[string]struct{}) - // Process the results in a separate goroutine - go func() { - for result := range passiveResults { - switch result.Type { - case subscraping.Error: - gologger.Warningf("Could not run source %s: %s\n", result.Source, result.Error) - case subscraping.Subdomain: - // Validate the subdomain found and remove wildcards from - if !strings.HasSuffix(result.Value, "."+domain) { - continue - } - subdomain := strings.ReplaceAll(strings.ToLower(result.Value), "*.", "") - - // Check if the subdomain is a duplicate. If not, - // send the subdomain for resolution. - if _, ok := uniqueMap[subdomain]; ok { - continue - } - uniqueMap[subdomain] = struct{}{} - - // Log the verbose message about the found subdomain and send the - // host for resolution to the resolution pool - gologger.Verbosef("%s\n", result.Source, subdomain) - - // If the user asked to remove wildcard then send on the resolve - // queue. Otherwise, if mode is not verbose print the results on - // the screen as they are discovered. - if r.options.RemoveWildcard { - resolutionPool.Tasks <- subdomain - } - - if !r.options.Verbose { - gologger.Silentf("%s\n", subdomain) - } - } - } - // Close the task channel only if wildcards are asked to be removed - if r.options.RemoveWildcard { - close(resolutionPool.Tasks) - } - wg.Done() - }() - - // If the user asked to remove wildcards, listen from the results - // queue and write to the map. At the end, print the found results to the screen - foundResults := make(map[string]string) - if r.options.RemoveWildcard { - // Process the results coming from the resolutions pool - for result := range resolutionPool.Results { - switch result.Type { - case resolve.Error: - gologger.Warningf("Could not resolve host: %s\n", result.Error) - case resolve.Subdomain: - // Add the found subdomain to a map. - if _, ok := foundResults[result.Host]; !ok { - foundResults[result.Host] = result.IP - } - } - } - } - wg.Wait() - - // If verbose mode was used, then now print all the - // found subdomains on the screen together. - if r.options.Verbose { - if r.options.RemoveWildcard { - for result := range foundResults { - gologger.Silentf("%s\n", result) - } - } else { - for result := range uniqueMap { - gologger.Silentf("%s\n", result) - } - } - } - // In case the user has specified to upload to chaos, write everything to a temporary buffer and upload - if r.options.ChaosUpload { - var buf = &bytes.Buffer{} - err := WriteHostOutput(uniqueMap, buf) - // If an error occurs, do not interrupt, continue to check if user specifed an output file - if err != nil { - gologger.Errorf("Could not prepare results for chaos %s\n", err) - } else { - // no error in writing host output, upload to chaos - err = r.UploadToChaos(buf) - if err != nil { - gologger.Errorf("Could not upload results to chaos %s\n", err) - } else { - gologger.Infof("Input processed successfully and subdomains with valid records will be updated to chaos dataset.\n") - } - // clear buffer - buf = nil - } - } - // In case the user has given an output file, write all the found - // subdomains to the output file. - if output != "" { - // If the output format is json, append .json - // else append .txt - if r.options.OutputDirectory != "" { - if r.options.JSON { - output = output + ".json" - } else { - output = output + ".txt" - } - } - - var file *os.File - var err error - if append { - file, err = os.OpenFile(output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - } else { - file, err = os.Create(output) - } - if err != nil { - gologger.Errorf("Could not create file %s for %s: %s\n", output, domain, err) - return err - } - - // Write the output to the file depending upon user requirement - if r.options.HostIP { - err = WriteHostIPOutput(foundResults, file) - } else if r.options.JSON { - err = WriteJSONOutput(foundResults, file) - } else { - if r.options.RemoveWildcard { - err = WriteHostOutputNoWildcard(foundResults, file) - } else { - err = WriteHostOutput(uniqueMap, file) - } - } - if err != nil { - gologger.Errorf("Could not write results to file %s for %s: %s\n", output, domain, err) - } - file.Close() - return err - } - return nil -} diff --git a/pkg/runner/initialize.go b/pkg/runner/initialize.go deleted file mode 100644 index fbd693a..0000000 --- a/pkg/runner/initialize.go +++ /dev/null @@ -1,52 +0,0 @@ -package runner - -import ( - "strings" - - "github.com/projectdiscovery/subfinder/pkg/passive" - "github.com/projectdiscovery/subfinder/pkg/resolve" -) - -// initializePassiveEngine creates the passive engine and loads sources etc -func (r *Runner) initializePassiveEngine() { - var sources, exclusions []string - - // If there are any sources from CLI, only use them - // Otherwise, use the yaml file sources - if r.options.Sources != "" { - sources = append(sources, strings.Split(r.options.Sources, ",")...) - } else { - sources = append(sources, r.options.YAMLConfig.Sources...) - } - - if r.options.ExcludeSources != "" { - exclusions = append(exclusions, strings.Split(r.options.ExcludeSources, ",")...) - } else { - exclusions = append(exclusions, r.options.YAMLConfig.ExcludeSources...) - } - - r.passiveAgent = passive.New(sources, exclusions) -} - -// initializeActiveEngine creates the resolver used to resolve the found subdomains -func (r *Runner) initializeActiveEngine() error { - r.resolverClient = resolve.New() - - // If the file has been provided, read resolvers from the file - if r.options.ResolverList != "" { - err := r.resolverClient.AppendResolversFromFile(r.options.ResolverList) - if err != nil { - return err - } - } - - var resolvers []string - - if r.options.Resolvers != "" { - resolvers = append(resolvers, strings.Split(r.options.Resolvers, ",")...) - } else { - resolvers = append(resolvers, r.options.YAMLConfig.Resolvers...) - } - r.resolverClient.AppendResolversFromSlice(resolvers) - return nil -} diff --git a/pkg/runner/options.go b/pkg/runner/options.go deleted file mode 100644 index 8443008..0000000 --- a/pkg/runner/options.go +++ /dev/null @@ -1,143 +0,0 @@ -package runner - -import ( - "flag" - "os" - "path" - "reflect" - "strings" - - "github.com/projectdiscovery/gologger" -) - -// Options contains the configuration options for tuning -// the subdomain enumeration process. -type Options struct { - Verbose bool // Verbose flag indicates whether to show verbose output or not - NoColor bool // No-Color disables the colored output - Threads int // Thread controls the number of threads to use for active enumerations - Timeout int // Timeout is the seconds to wait for sources to respond - MaxEnumerationTime int // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration - Domain string // Domain is the domain to find subdomains for - DomainsFile string // DomainsFile is the file containing list of domains to find subdomains for - ChaosUpload bool // ChaosUpload indicates whether to upload results to the Chaos API - Output string // Output is the file to write found subdomains to. - OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given - JSON bool // JSON specifies whether to use json for output format or text file - HostIP bool // HostIP specifies whether to write subdomains in host:ip format - Silent bool // Silent suppresses any extra text and only writes subdomains to screen - Sources string // Sources contains a comma-separated list of sources to use for enumeration - ListSources bool // ListSources specifies whether to list all available sources - ExcludeSources string // ExcludeSources contains the comma-separated sources to not include in the enumeration process - Resolvers string // Resolvers is the comma-separated resolvers to use for enumeration - ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration - RemoveWildcard bool // RemoveWildcard specifies whether to remove potential wildcard or dead subdomains from the results. - ConfigFile string // ConfigFile contains the location of the config file - Stdin bool // Stdin specifies whether stdin input was given to the process - Version bool // Version specifies if we should just show version and exit - - YAMLConfig ConfigFile // YAMLConfig contains the unmarshalled yaml config file -} - -// ParseOptions parses the command line flags provided by a user -func ParseOptions() *Options { - options := &Options{} - - config, err := GetConfigDirectory() - if err != nil { - // This should never be reached - gologger.Fatalf("Could not get user home: %s\n", err) - } - - flag.BoolVar(&options.Verbose, "v", false, "Show Verbose output") - flag.BoolVar(&options.NoColor, "nC", false, "Don't Use colors in output") - flag.IntVar(&options.Threads, "t", 10, "Number of concurrent goroutines for resolving") - flag.IntVar(&options.Timeout, "timeout", 30, "Seconds to wait before timing out") - flag.IntVar(&options.MaxEnumerationTime, "max-time", 10, "Minutes to wait for enumeration results") - flag.StringVar(&options.Domain, "d", "", "Domain to find subdomains for") - flag.StringVar(&options.DomainsFile, "dL", "", "File containing list of domains to enumerate") - flag.BoolVar(&options.ChaosUpload, "cd", false, "Upload results to the Chaos API (api-key required)") - flag.StringVar(&options.Output, "o", "", "File to write output to (optional)") - flag.StringVar(&options.OutputDirectory, "oD", "", "Directory to write enumeration results to (optional)") - flag.BoolVar(&options.JSON, "oJ", false, "Write output in JSON lines Format") - flag.BoolVar(&options.HostIP, "oI", false, "Write output in Host,IP format") - flag.BoolVar(&options.Silent, "silent", false, "Show only subdomains in output") - flag.StringVar(&options.Sources, "sources", "", "Comma separated list of sources to use") - flag.BoolVar(&options.ListSources, "ls", false, "List all available sources") - flag.StringVar(&options.ExcludeSources, "exclude-sources", "", "List of sources to exclude from enumeration") - flag.StringVar(&options.Resolvers, "r", "", "Comma-separated list of resolvers to use") - flag.StringVar(&options.ResolverList, "rL", "", "Text file containing list of resolvers to use") - flag.BoolVar(&options.RemoveWildcard, "nW", false, "Remove Wildcard & Dead Subdomains from output") - flag.StringVar(&options.ConfigFile, "config", path.Join(config, "config.yaml"), "Configuration file for API Keys, etc") - flag.BoolVar(&options.Version, "version", false, "Show version of subfinder") - flag.Parse() - - // Check if stdin pipe was given - options.Stdin = hasStdin() - - // Read the inputs and configure the logging - options.configureOutput() - - // Show the user the banner - showBanner() - - if options.Version { - gologger.Infof("Current Version: %s\n", Version) - os.Exit(0) - } - - // Check if the config file exists. If not, it means this is the - // first run of the program. Show the first run notices and initialize the config file. - // Else show the normal banners and read the yaml fiile to the config - if !CheckConfigExists(options.ConfigFile) { - options.firstRunTasks() - } else { - options.normalRunTasks() - } - - if options.ListSources { - listSources(options) - os.Exit(0) - } - - // Validate the options passed by the user and if any - // invalid options have been used, exit. - err = options.validateOptions() - if err != nil { - gologger.Fatalf("Program exiting: %s\n", err) - } - - return options -} - -func hasStdin() bool { - fi, err := os.Stdin.Stat() - if err != nil { - return false - } - if fi.Mode()&os.ModeNamedPipe == 0 { - return false - } - return true -} - -func listSources(options *Options) { - gologger.Infof("Current list of available sources. [%d]\n", len(options.YAMLConfig.Sources)) - gologger.Infof("Sources marked with an * needs key or token in order to work.\n") - gologger.Infof("You can modify %s to configure your keys / tokens.\n\n", options.ConfigFile) - - keys := options.YAMLConfig.GetKeys() - needsKey := make(map[string]interface{}) - keysElem := reflect.ValueOf(&keys).Elem() - for i := 0; i < keysElem.NumField(); i++ { - needsKey[strings.ToLower(keysElem.Type().Field(i).Name)] = keysElem.Field(i).Interface() - } - - for _, source := range options.YAMLConfig.Sources { - message := "%s\n" - if _, ok := needsKey[source]; ok { - message = "%s *\n" - } - gologger.Silentf(message, source) - } -} diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go deleted file mode 100644 index 6b37277..0000000 --- a/pkg/runner/runner.go +++ /dev/null @@ -1,91 +0,0 @@ -package runner - -import ( - "bufio" - "io" - "os" - "path" - - "github.com/projectdiscovery/subfinder/pkg/passive" - "github.com/projectdiscovery/subfinder/pkg/resolve" -) - -// Runner is an instance of the subdomain enumeration -// client used to orchestrate the whole process. -type Runner struct { - options *Options - passiveAgent *passive.Agent - resolverClient *resolve.Resolver -} - -// NewRunner creates a new runner struct instance by parsing -// the configuration options, configuring sources, reading lists -// and setting up loggers, etc. -func NewRunner(options *Options) (*Runner, error) { - runner := &Runner{options: options} - - // Initialize the passive subdomain enumeration engine - runner.initializePassiveEngine() - - // Initialize the active subdomain enumeration engine - err := runner.initializeActiveEngine() - if err != nil { - return nil, err - } - - return runner, nil -} - -// RunEnumeration runs the subdomain enumeration flow on the targets specified -func (r *Runner) RunEnumeration() error { - // Check if only a single domain is sent as input. Process the domain now. - if r.options.Domain != "" { - return r.EnumerateSingleDomain(r.options.Domain, r.options.Output, false) - } - - // If we have multiple domains as input, - if r.options.DomainsFile != "" { - f, err := os.Open(r.options.DomainsFile) - if err != nil { - return err - } - err = r.EnumerateMultipleDomains(f) - f.Close() - return err - } - - // If we have STDIN input, treat it as multiple domains - if r.options.Stdin { - return r.EnumerateMultipleDomains(os.Stdin) - } - return nil -} - -// EnumerateMultipleDomains enumerates subdomains for multiple domains -// We keep enumerating subdomains for a given domain until we reach an error -func (r *Runner) EnumerateMultipleDomains(reader io.Reader) error { - scanner := bufio.NewScanner(reader) - for scanner.Scan() { - domain := scanner.Text() - if domain == "" { - continue - } - - var err error - // If the user has specifed an output file, use that output file instead - // of creating a new output file for each domain. Else create a new file - // for each domain in the directory. - if r.options.Output != "" { - err = r.EnumerateSingleDomain(domain, r.options.Output, true) - } else if r.options.OutputDirectory != "" { - outputFile := path.Join(r.options.OutputDirectory, domain) - err = r.EnumerateSingleDomain(domain, outputFile, false) - } else { - err = r.EnumerateSingleDomain(domain, "", true) - } - if err != nil { - return err - } - } - return nil -} diff --git a/pkg/runner/utils.go b/pkg/runner/utils.go deleted file mode 100644 index 0009ad3..0000000 --- a/pkg/runner/utils.go +++ /dev/null @@ -1,131 +0,0 @@ -package runner - -import ( - "bufio" - "crypto/tls" - "fmt" - "io" - "io/ioutil" - "net/http" - "strings" - "time" - - jsoniter "github.com/json-iterator/go" - "github.com/pkg/errors" -) - -// JSONResult contains the result for a host in JSON format -type JSONResult struct { - Host string `json:"host"` - IP string `json:"ip"` -} - -func (r *Runner) UploadToChaos(reader io.Reader) error { - httpClient := &http.Client{ - Transport: &http.Transport{ - MaxIdleConnsPerHost: 100, - MaxIdleConns: 100, - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, - Timeout: time.Duration(600) * time.Second, // 10 minutes - uploads may take long - } - - request, err := http.NewRequest("POST", "https://dns.projectdiscovery.io/dns/add", reader) - if err != nil { - return errors.Wrap(err, "could not create request") - } - request.Header.Set("Authorization", r.options.YAMLConfig.GetKeys().Chaos) - - resp, err := httpClient.Do(request) - if err != nil { - return errors.Wrap(err, "could not make request") - } - defer func() { - io.Copy(ioutil.Discard, resp.Body) - resp.Body.Close() - }() - - if resp.StatusCode != 200 { - return fmt.Errorf("invalid status code received: %d", resp.StatusCode) - } - return nil -} - -// WriteHostOutput writes the output list of subdomain to an io.Writer -func WriteHostOutput(results map[string]struct{}, writer io.Writer) error { - bufwriter := bufio.NewWriter(writer) - sb := &strings.Builder{} - - for host := range results { - sb.WriteString(host) - sb.WriteString("\n") - - _, err := bufwriter.WriteString(sb.String()) - if err != nil { - bufwriter.Flush() - return err - } - sb.Reset() - } - return bufwriter.Flush() -} - -// WriteHostOutputNoWildcard writes the output list of subdomain with nW flag to an io.Writer -func WriteHostOutputNoWildcard(results map[string]string, writer io.Writer) error { - bufwriter := bufio.NewWriter(writer) - sb := &strings.Builder{} - - for host := range results { - sb.WriteString(host) - sb.WriteString("\n") - - _, err := bufwriter.WriteString(sb.String()) - if err != nil { - bufwriter.Flush() - return err - } - sb.Reset() - } - return bufwriter.Flush() -} - -// WriteJSONOutput writes the output list of subdomain in JSON to an io.Writer -func WriteJSONOutput(results map[string]string, writer io.Writer) error { - encoder := jsoniter.NewEncoder(writer) - - data := JSONResult{} - - for host, ip := range results { - data.Host = host - data.IP = ip - - err := encoder.Encode(&data) - if err != nil { - return err - } - } - return nil -} - -// WriteHostIPOutput writes the output list of subdomain to an io.Writer -func WriteHostIPOutput(results map[string]string, writer io.Writer) error { - bufwriter := bufio.NewWriter(writer) - sb := &strings.Builder{} - - for host, ip := range results { - sb.WriteString(host) - sb.WriteString(",") - sb.WriteString(ip) - sb.WriteString("\n") - - _, err := bufwriter.WriteString(sb.String()) - if err != nil { - bufwriter.Flush() - return err - } - sb.Reset() - } - return bufwriter.Flush() -} diff --git a/pkg/runner/validate.go b/pkg/runner/validate.go deleted file mode 100644 index 95bc230..0000000 --- a/pkg/runner/validate.go +++ /dev/null @@ -1,58 +0,0 @@ -package runner - -import ( - "errors" - - "github.com/projectdiscovery/gologger" -) - -// validateOptions validates the configuration options passed -func (options *Options) validateOptions() error { - // Check if domain, list of domains, or stdin info was provided. - // If none was provided, then return. - if options.Domain == "" && options.DomainsFile == "" && !options.Stdin { - return errors.New("no input list provided") - } - - // Both verbose and silent flags were used - if options.Verbose && options.Silent { - return errors.New("both verbose and silent mode specified") - } - - // Validate threads and options - if options.Threads == 0 { - return errors.New("threads cannot be zero") - } - if options.Timeout == 0 { - return errors.New("timeout cannot be zero") - } - - // JSON cannot be used with hostIP - if options.JSON && options.HostIP { - return errors.New("hostip flag cannot be used with json flag") - } - - // Always remove wildcard with hostip and json - if options.HostIP && !options.RemoveWildcard { - return errors.New("hostip flag must be used with RemoveWildcard option") - } - if options.JSON && !options.RemoveWildcard { - return errors.New("JSON flag must be used with RemoveWildcard option") - } - - return nil -} - -// configureOutput configures the output on the screen -func (options *Options) configureOutput() { - // If the user desires verbose output, show verbose output - if options.Verbose { - gologger.MaxLevel = gologger.Verbose - } - if options.NoColor { - gologger.UseColors = false - } - if options.Silent { - gologger.MaxLevel = gologger.Silent - } -} diff --git a/pkg/subscraping/agent.go b/pkg/subscraping/agent.go deleted file mode 100755 index 18e248f..0000000 --- a/pkg/subscraping/agent.go +++ /dev/null @@ -1,96 +0,0 @@ -package subscraping - -import ( - "context" - "crypto/tls" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "time" -) - -// NewSession creates a new session object for a domain -func NewSession(domain string, keys Keys, timeout int) (*Session, error) { - client := &http.Client{ - Transport: &http.Transport{ - MaxIdleConns: 100, - MaxIdleConnsPerHost: 100, - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, - Timeout: time.Duration(timeout) * time.Second, - } - - session := &Session{ - Client: client, - Keys: keys, - } - - // Create a new extractor object for the current domain - extractor, err := NewSubdomainExtractor(domain) - session.Extractor = extractor - - return session, err -} - -// NormalGetWithContext makes a normal GET request to a URL with context -func (s *Session) NormalGetWithContext(ctx context.Context, url string) (*http.Response, error) { - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, err - } - - // Don't randomize user agents, as they cause issues sometimes - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36") - req.Header.Set("Accept", "*/*") - req.Header.Set("Accept-Language", "en") - - return httpRequestWrapper(s.Client, req) -} - -// Get makes a GET request to a URL -func (s *Session) Get(ctx context.Context, url string, cookies string, headers map[string]string) (*http.Response, error) { - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, err - } - - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36") - req.Header.Set("Accept", "*/*") - req.Header.Set("Accept-Language", "en") - - if cookies != "" { - req.Header.Set("Cookie", cookies) - } - - if headers != nil { - for key, value := range headers { - req.Header.Set(key, value) - } - } - - return httpRequestWrapper(s.Client, req) -} - -func (s *Session) DiscardHttpResponse(response *http.Response) { - if response != nil { - io.Copy(ioutil.Discard, response.Body) - response.Body.Close() - } -} - -func httpRequestWrapper(client *http.Client, request *http.Request) (*http.Response, error) { - resp, err := client.Do(request) - if err != nil { - return nil, err - } - - if resp.StatusCode != http.StatusOK { - requestUrl, _ := url.QueryUnescape(request.URL.String()) - return resp, fmt.Errorf("Unexpected status code %d received from %s", resp.StatusCode, requestUrl) - } - return resp, nil -} diff --git a/pkg/subscraping/sources/alienvault/alienvault.go b/pkg/subscraping/sources/alienvault/alienvault.go deleted file mode 100644 index 74dcb20..0000000 --- a/pkg/subscraping/sources/alienvault/alienvault.go +++ /dev/null @@ -1,55 +0,0 @@ -package alienvault - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -type alienvaultResponse struct { - PassiveDNS []struct { - Hostname string `json:"hostname"` - } `json:"passive_dns"` -} - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - close(results) - return - } - - otxResp := &alienvaultResponse{} - // Get the response body and decode - err = json.NewDecoder(resp.Body).Decode(&otxResp) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - for _, record := range otxResp.PassiveDNS { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "alienvault" -} diff --git a/pkg/subscraping/sources/archiveis/archiveis.go b/pkg/subscraping/sources/archiveis/archiveis.go deleted file mode 100755 index be7c749..0000000 --- a/pkg/subscraping/sources/archiveis/archiveis.go +++ /dev/null @@ -1,77 +0,0 @@ -// Package archiveis is a Archiveis Scraping Engine in Golang -package archiveis - -import ( - "context" - "io/ioutil" - "regexp" - - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -// ArchiveIs is a struct for archiveurlsagent -type ArchiveIs struct { - Results chan subscraping.Result - Session *subscraping.Session -} - -var reNext = regexp.MustCompile("") - -func (a *ArchiveIs) enumerate(ctx context.Context, baseURL string) { - select { - case <-ctx.Done(): - return - default: - } - - resp, err := a.Session.NormalGetWithContext(ctx, baseURL) - if err != nil { - a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} - a.Session.DiscardHttpResponse(resp) - return - } - - // Get the response body - body, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} - return - } - - src := string(body) - - for _, subdomain := range a.Session.Extractor.FindAllString(src, -1) { - a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Subdomain, Value: subdomain} - } - - match1 := reNext.FindStringSubmatch(src) - if len(match1) > 0 { - a.enumerate(ctx, match1[1]) - } -} - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - aInstance := ArchiveIs{ - Session: session, - Results: results, - } - - go func() { - aInstance.enumerate(ctx, "http://archive.is/*."+domain) - close(aInstance.Results) - }() - - return aInstance.Results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "archiveis" -} diff --git a/pkg/subscraping/sources/binaryedge/binaryedge.go b/pkg/subscraping/sources/binaryedge/binaryedge.go deleted file mode 100755 index 64630d8..0000000 --- a/pkg/subscraping/sources/binaryedge/binaryedge.go +++ /dev/null @@ -1,104 +0,0 @@ -package binaryedge - -import ( - "context" - "fmt" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -type binaryedgeResponse struct { - Subdomains []string `json:"events"` - Total int `json:"total"` -} - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - if session.Keys.Binaryedge == "" { - close(results) - return - } - - resp, err := session.Get(ctx, fmt.Sprintf("https://api.binaryedge.io/v2/query/domains/subdomain/%s", domain), "", map[string]string{"X-Key": session.Keys.Binaryedge}) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - close(results) - return - } - - response := new(binaryedgeResponse) - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - - for _, subdomain := range response.Subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - } - - remaining := response.Total - 100 - currentPage := 2 - - for { - further := s.getSubdomains(ctx, domain, &remaining, ¤tPage, session, results) - if !further { - break - } - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "binaryedge" -} - -func (s *Source) getSubdomains(ctx context.Context, domain string, remaining, currentPage *int, session *subscraping.Session, results chan subscraping.Result) bool { - for { - select { - case <-ctx.Done(): - return false - default: - resp, err := session.Get(ctx, fmt.Sprintf("https://api.binaryedge.io/v2/query/domains/subdomain/%s?page=%d", domain, *currentPage), "", map[string]string{"X-Key": session.Keys.Binaryedge}) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return false - } - - response := binaryedgeResponse{} - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - return false - } - resp.Body.Close() - - for _, subdomain := range response.Subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - } - - *remaining = *remaining - 100 - if *remaining <= 0 { - return false - } - *currentPage++ - return true - } - } -} diff --git a/pkg/subscraping/sources/bufferover/bufferover.go b/pkg/subscraping/sources/bufferover/bufferover.go deleted file mode 100755 index 88e8bda..0000000 --- a/pkg/subscraping/sources/bufferover/bufferover.go +++ /dev/null @@ -1,57 +0,0 @@ -// Package bufferover is a bufferover Scraping Engine in Golang -package bufferover - -import ( - "context" - "fmt" - "io/ioutil" - - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - // Run enumeration on subdomain dataset for historical SONAR datasets - s.getData(ctx, fmt.Sprintf("https://dns.bufferover.run/dns?q=.%s", domain), session, results) - s.getData(ctx, fmt.Sprintf("https://tls.bufferover.run/dns?q=.%s", domain), session, results) - - close(results) - }() - - return results -} - -func (s *Source) getData(ctx context.Context, URL string, session *subscraping.Session, results chan subscraping.Result) { - resp, err := session.NormalGetWithContext(ctx, URL) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - return - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - return - } - resp.Body.Close() - - src := string(body) - - for _, subdomain := range session.Extractor.FindAllString(src, -1) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - } - return -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "bufferover" -} diff --git a/pkg/subscraping/sources/censys/censys.go b/pkg/subscraping/sources/censys/censys.go deleted file mode 100644 index d54c79a..0000000 --- a/pkg/subscraping/sources/censys/censys.go +++ /dev/null @@ -1,96 +0,0 @@ -package censys - -import ( - "bytes" - "context" - "net/http" - "strconv" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -const maxCensysPages = 10 - -type resultsq struct { - Data []string `json:"parsed.extensions.subject_alt_name.dns_names"` - Data1 []string `json:"parsed.names"` -} - -type response struct { - Results []resultsq `json:"results"` - Metadata struct { - Pages int `json:"pages"` - } `json:"metadata"` -} - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - if session.Keys.CensysToken == "" || session.Keys.CensysSecret == "" { - close(results) - return - } - var response response - - currentPage := 1 - for { - var request = []byte(`{"query":"` + domain + `", "page":` + strconv.Itoa(currentPage) + `, "fields":["parsed.names","parsed.extensions.subject_alt_name.dns_names"], "flatten":true}`) - - req, err := http.NewRequestWithContext(ctx, "POST", "https://www.censys.io/api/v1/search/certificates", bytes.NewReader(request)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - close(results) - return - } - req.SetBasicAuth(session.Keys.CensysToken, session.Keys.CensysSecret) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - - resp, err := session.Client.Do(req) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - close(results) - return - } - - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - - // Exit the censys enumeration if max pages is reached - if currentPage >= response.Metadata.Pages || currentPage >= maxCensysPages { - break - } - - for _, res := range response.Results { - for _, part := range res.Data { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: part} - } - for _, part := range res.Data1 { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: part} - } - } - - currentPage++ - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "censys" -} diff --git a/pkg/subscraping/sources/certspotter/certspotter.go b/pkg/subscraping/sources/certspotter/certspotter.go deleted file mode 100755 index 0f6affc..0000000 --- a/pkg/subscraping/sources/certspotter/certspotter.go +++ /dev/null @@ -1,101 +0,0 @@ -package certspotter - -import ( - "context" - "fmt" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -type certspotterObject struct { - ID string `json:"id"` - DNSNames []string `json:"dns_names"` -} - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - if session.Keys.Certspotter == "" { - close(results) - return - } - - resp, err := session.Get(ctx, fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain), "", map[string]string{"Authorization": "Bearer " + session.Keys.Certspotter}) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - close(results) - return - } - - response := []certspotterObject{} - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - - for _, cert := range response { - for _, subdomain := range cert.DNSNames { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - } - } - - // if the number of responses is zero, close the channel and return. - if len(response) == 0 { - close(results) - return - } - - id := response[len(response)-1].ID - for { - reqURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, id) - - resp, err := session.Get(ctx, reqURL, "", map[string]string{"Authorization": "Bearer " + session.Keys.Certspotter}) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - close(results) - return - } - - response := []certspotterObject{} - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - - if len(response) == 0 { - break - } - - for _, cert := range response { - for _, subdomain := range cert.DNSNames { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - } - } - - id = response[len(response)-1].ID - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "certspotter" -} diff --git a/pkg/subscraping/sources/certspotterold/certspotterold.go b/pkg/subscraping/sources/certspotterold/certspotterold.go deleted file mode 100755 index b7f8bfd..0000000 --- a/pkg/subscraping/sources/certspotterold/certspotterold.go +++ /dev/null @@ -1,50 +0,0 @@ -package certspotterold - -import ( - "context" - "fmt" - "io/ioutil" - - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://certspotter.com/api/v0/certs?domain=%s", domain)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - close(results) - return - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - - src := string(body) - - for _, subdomain := range session.Extractor.FindAllString(src, -1) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "certspotterold" -} diff --git a/pkg/subscraping/sources/commoncrawl/commoncrawl.go b/pkg/subscraping/sources/commoncrawl/commoncrawl.go deleted file mode 100755 index 3b021f5..0000000 --- a/pkg/subscraping/sources/commoncrawl/commoncrawl.go +++ /dev/null @@ -1,108 +0,0 @@ -package commoncrawl - -import ( - "context" - "fmt" - "io/ioutil" - "net/url" - "strings" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -const indexURL = "https://index.commoncrawl.org/collinfo.json" - -type indexResponse struct { - ID string `json:"id"` - APIURL string `json:"cdx-api"` -} - -// Source is the passive scraping agent -type Source struct{} - -var years = [...]string{"2020", "2019", "2018", "2017"} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - resp, err := session.NormalGetWithContext(ctx, indexURL) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - close(results) - return - } - - indexes := []indexResponse{} - err = jsoniter.NewDecoder(resp.Body).Decode(&indexes) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - - searchIndexes := make(map[string]string) - for _, year := range years { - for _, index := range indexes { - if strings.Contains(index.ID, year) { - if _, ok := searchIndexes[year]; !ok { - searchIndexes[year] = index.APIURL - break - } - } - } - } - - for _, apiURL := range searchIndexes { - further := s.getSubdomains(ctx, apiURL, domain, session, results) - if !further { - break - } - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "commoncrawl" -} - -func (s *Source) getSubdomains(ctx context.Context, searchURL string, domain string, session *subscraping.Session, results chan subscraping.Result) bool { - for { - select { - case <-ctx.Done(): - return false - default: - resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("%s?url=*.%s&output=json", searchURL, domain)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return false - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - return false - } - resp.Body.Close() - - src, _ := url.QueryUnescape(string(body)) - - for _, subdomain := range session.Extractor.FindAllString(src, -1) { - subdomain = strings.TrimPrefix(subdomain, "25") - - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - } - return true - } - } -} diff --git a/pkg/subscraping/sources/crtsh/crtsh.go b/pkg/subscraping/sources/crtsh/crtsh.go deleted file mode 100755 index 981a178..0000000 --- a/pkg/subscraping/sources/crtsh/crtsh.go +++ /dev/null @@ -1,93 +0,0 @@ -package crtsh - -import ( - "context" - "database/sql" - "fmt" - "io/ioutil" - "strings" - - // postgres driver - _ "github.com/lib/pq" - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - found := s.getSubdomainsFromSQL(ctx, domain, session, results) - if found { - close(results) - return - } - _ = s.getSubdomainsFromHTTP(ctx, domain, session, results) - close(results) - }() - - return results -} - -func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) bool { - db, err := sql.Open("postgres", "host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes") - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return false - } - - pattern := "%." + domain - rows, err := db.Query(`SELECT DISTINCT ci.NAME_VALUE as domain - FROM certificate_identity ci - WHERE reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower($1)) - ORDER BY ci.NAME_VALUE`, pattern) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return false - } - - var data string - // Parse all the rows getting subdomains - for rows.Next() { - err := rows.Scan(&data) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return false - } - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data} - } - return true -} - -func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) bool { - resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - return false - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - return false - } - resp.Body.Close() - - // Also replace all newlines - src := strings.Replace(string(body), "\\n", " ", -1) - - for _, subdomain := range session.Extractor.FindAllString(src, -1) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - } - return true -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "crtsh" -} diff --git a/pkg/subscraping/sources/dnsdb/dnsdb.go b/pkg/subscraping/sources/dnsdb/dnsdb.go deleted file mode 100644 index a59624d..0000000 --- a/pkg/subscraping/sources/dnsdb/dnsdb.go +++ /dev/null @@ -1,70 +0,0 @@ -package dnsdb - -import ( - "bufio" - "context" - "encoding/json" - "fmt" - "strings" - - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -type dnsdbResponse struct { - Name string `json:"rrname"` -} - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - if session.Keys.DNSDB == "" { - close(results) - } else { - headers := map[string]string{ - "X-API-KEY": session.Keys.DNSDB, - "Accept": "application/json", - "Content-Type": "application/json", - } - - go func() { - resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdb.info/lookup/rrset/name/*.%s?limit=1000000000000", domain), "", headers) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - close(results) - return - } - - defer resp.Body.Close() - // Get the response body - scanner := bufio.NewScanner(resp.Body) - for scanner.Scan() { - line := scanner.Text() - if line == "" { - continue - } - out := &dnsdbResponse{} - err := json.Unmarshal([]byte(line), out) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(out.Name, ".")} - out = nil - } - close(results) - }() - } - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "DNSDB" -} diff --git a/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go deleted file mode 100755 index 170d4f8..0000000 --- a/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go +++ /dev/null @@ -1,113 +0,0 @@ -package dnsdumpster - -import ( - "context" - "io/ioutil" - "net" - "net/http" - "net/url" - "regexp" - "strings" - "time" - - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -var re = regexp.MustCompile("") - -// getCSRFToken gets the CSRF Token from the page -func getCSRFToken(page string) string { - if subs := re.FindStringSubmatch(page); len(subs) == 2 { - return strings.TrimSpace(subs[1]) - } - return "" -} - -// postForm posts a form for a domain and returns the response -func postForm(token, domain string) (string, error) { - dial := net.Dialer{} - client := &http.Client{ - Transport: &http.Transport{ - DialContext: dial.DialContext, - TLSHandshakeTimeout: 10 * time.Second, - }, - } - params := url.Values{ - "csrfmiddlewaretoken": {token}, - "targetip": {domain}, - } - - req, err := http.NewRequest("POST", "https://dnsdumpster.com/", strings.NewReader(params.Encode())) - if err != nil { - return "", err - } - - // The CSRF token needs to be sent as a cookie - cookie := &http.Cookie{ - Name: "csrftoken", - Domain: "dnsdumpster.com", - Value: token, - } - req.AddCookie(cookie) - - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36") - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("Referer", "https://dnsdumpster.com") - req.Header.Set("X-CSRF-Token", token) - - resp, err := client.Do(req) - if err != nil { - return "", err - } - // Now, grab the entire page - in, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - return string(in), err -} - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - resp, err := session.NormalGetWithContext(ctx, "https://dnsdumpster.com/") - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - close(results) - return - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - csrfToken := getCSRFToken(string(body)) - - data, err := postForm(csrfToken, domain) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - close(results) - return - } - - for _, subdomain := range session.Extractor.FindAllString(data, -1) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "dnsdumpster" -} diff --git a/pkg/subscraping/sources/entrust/entrust.go b/pkg/subscraping/sources/entrust/entrust.go deleted file mode 100755 index 0141868..0000000 --- a/pkg/subscraping/sources/entrust/entrust.go +++ /dev/null @@ -1,53 +0,0 @@ -package entrust - -import ( - "context" - "fmt" - "io/ioutil" - "strings" - - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://ctsearch.entrust.com/api/v1/certificates?fields=issuerCN,subjectO,issuerDN,issuerO,subjectDN,signAlg,san,publicKeyType,publicKeySize,validFrom,validTo,sn,ev,logEntries.logName,subjectCNReversed,cert&domain=%s&includeExpired=true&exactMatch=false&limit=5000", domain)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - close(results) - return - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - - src := string(body) - - for _, subdomain := range session.Extractor.FindAllString(src, -1) { - subdomain = strings.TrimPrefix(subdomain, "u003d") - - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "entrust" -} diff --git a/pkg/subscraping/sources/github/github.go b/pkg/subscraping/sources/github/github.go deleted file mode 100644 index f4331f0..0000000 --- a/pkg/subscraping/sources/github/github.go +++ /dev/null @@ -1,211 +0,0 @@ -// GitHub search package, based on gwen001's https://github.com/gwen001/github-search github-subdomains -package github - -import ( - "context" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "regexp" - "strconv" - "strings" - "time" - - jsoniter "github.com/json-iterator/go" - - "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/subfinder/pkg/subscraping" - "github.com/tomnomnom/linkheader" -) - -type textMatch struct { - Fragment string `json:"fragment"` -} - -type item struct { - Name string `json:"name"` - HtmlUrl string `json:"html_url"` - TextMatches []textMatch `json:"text_matches"` -} - -type response struct { - TotalCount int `json:"total_count"` - Items []item `json:"items"` -} - -// Source is the passive scraping agent -type Source struct{} - -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - if len(session.Keys.GitHub) == 0 { - close(results) - return - } - - tokens := NewTokenManager(session.Keys.GitHub) - - // search on GitHub with exact match - searchURL := fmt.Sprintf("https://api.github.com/search/code?per_page=100&q=\"%s\"", domain) - s.enumerate(ctx, searchURL, s.DomainRegexp(domain), tokens, session, results) - close(results) - }() - - return results -} - -func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *regexp.Regexp, tokens *Tokens, session *subscraping.Session, results chan subscraping.Result) { - select { - case <-ctx.Done(): - return - default: - } - - token := tokens.Get() - - if token.RetryAfter > 0 { - if len(tokens.pool) == 1 { - gologger.Verbosef("GitHub Search request rate limit exceeded, waiting for %d seconds before retry... \n", s.Name(), token.RetryAfter) - time.Sleep(time.Duration(token.RetryAfter) * time.Second) - } else { - token = tokens.Get() - } - } - - headers := map[string]string{ - "Accept": "application/vnd.github.v3.text-match+json", - "Authorization": "token " + token.Hash, - } - - // Initial request to GitHub search - resp, err := session.Get(ctx, searchURL, "", headers) - isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden - - if err != nil && !isForbidden { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - return - } else { - // Retry enumerarion after Retry-After seconds on rate limit abuse detected - ratelimitRemaining, _ := strconv.ParseInt(resp.Header.Get("X-Ratelimit-Remaining"), 10, 64) - if isForbidden && ratelimitRemaining == 0 { - retryAfterSeconds, _ := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64) - tokens.setCurrentTokenExceeded(retryAfterSeconds) - - s.enumerate(ctx, searchURL, domainRegexp, tokens, session, results) - } else { - // Links header, first, next, last... - linksHeader := linkheader.Parse(resp.Header.Get("Link")) - - data := response{} - - // Marshall json reponse - err = jsoniter.NewDecoder(resp.Body).Decode(&data) - resp.Body.Close() - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - - // Response items iteration - for _, item := range data.Items { - resp, err := session.NormalGetWithContext(ctx, rawUrl(item.HtmlUrl)) - if err != nil { - if resp != nil && resp.StatusCode != http.StatusNotFound { - session.DiscardHttpResponse(resp) - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - } - - var subdomains []string - - if resp.StatusCode == http.StatusOK { - // Get the item code from the raw file url - code, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - // Search for domain matches in the code - subdomains = append(subdomains, matches(domainRegexp, normalizeContent(string(code)))...) - } - - // Text matches iteration per item - for _, textMatch := range item.TextMatches { - // Search for domain matches in the text fragment - subdomains = append(subdomains, matches(domainRegexp, normalizeContent(textMatch.Fragment))...) - } - - for _, subdomain := range unique(subdomains) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - } - } - - // Proccess the next link recursively - for _, link := range linksHeader { - if link.Rel == "next" { - nextUrl, err := url.QueryUnescape(link.URL) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - s.enumerate(ctx, nextUrl, domainRegexp, tokens, session, results) - } - } - } - } - -} - -// Normalize content before matching, query unescape, remove tabs and new line chars -func normalizeContent(content string) string { - normalizedContent, _ := url.QueryUnescape(content) - normalizedContent = strings.Replace(normalizedContent, "\\t", "", -1) - normalizedContent = strings.Replace(normalizedContent, "\\n", "", -1) - return normalizedContent -} - -// Remove duplicates from string array -func unique(arr []string) []string { - occured := map[string]bool{} - result := []string{} - for e := range arr { - if occured[arr[e]] != true { - occured[arr[e]] = true - result = append(result, arr[e]) - } - } - return result -} - -// Find matches by regular expression in any content -func matches(regexp *regexp.Regexp, content string) []string { - var matches []string - match := regexp.FindAllString(content, -1) - if len(match) > 0 { - matches = unique(match) - } - return matches -} - -// Raw URL to get the files code and match for subdomains -func rawUrl(htmlUrl string) string { - domain := strings.Replace(htmlUrl, "https://github.com/", "https://raw.githubusercontent.com/", -1) - return strings.Replace(domain, "/blob/", "/", -1) -} - -// Domain regular expression to match subdomains in github files code -func (s *Source) DomainRegexp(domain string) *regexp.Regexp { - rdomain := strings.Replace(domain, ".", "\\.", -1) - return regexp.MustCompile("(\\w+[.])*" + rdomain) -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "github" -} diff --git a/pkg/subscraping/sources/github/tokenmanager.go b/pkg/subscraping/sources/github/tokenmanager.go deleted file mode 100644 index 298ea81..0000000 --- a/pkg/subscraping/sources/github/tokenmanager.go +++ /dev/null @@ -1,61 +0,0 @@ -package github - -import "time" - -type token struct { - Hash string - RetryAfter int64 - ExceededTime time.Time -} - -type Tokens struct { - current int - pool []token -} - -func NewTokenManager(keys []string) *Tokens { - pool := []token{} - for _, key := range keys { - t := token{Hash: key, ExceededTime: time.Time{}, RetryAfter: 0} - pool = append(pool, t) - } - - return &Tokens{ - current: 0, - pool: pool, - } -} - -func (r *Tokens) setCurrentTokenExceeded(retryAfter int64) { - if r.current >= len(r.pool) { - r.current = r.current % len(r.pool) - } - if r.pool[r.current].RetryAfter == 0 { - r.pool[r.current].ExceededTime = time.Now() - r.pool[r.current].RetryAfter = retryAfter - } -} - -func (r *Tokens) Get() token { - resetExceededTokens(r) - - if r.current >= len(r.pool) { - r.current = r.current % len(r.pool) - } - - result := r.pool[r.current] - r.current++ - - return result -} - -func resetExceededTokens(r *Tokens) { - for i, token := range r.pool { - if token.RetryAfter > 0 { - if int64(time.Since(token.ExceededTime)/time.Second) > token.RetryAfter { - r.pool[i].ExceededTime = time.Time{} - r.pool[i].RetryAfter = 0 - } - } - } -} diff --git a/pkg/subscraping/sources/hackertarget/hackertarget.go b/pkg/subscraping/sources/hackertarget/hackertarget.go deleted file mode 100755 index 5c0669a..0000000 --- a/pkg/subscraping/sources/hackertarget/hackertarget.go +++ /dev/null @@ -1,50 +0,0 @@ -package hackertarget - -import ( - "context" - "fmt" - "io/ioutil" - - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("http://api.hackertarget.com/hostsearch/?q=%s", domain)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - close(results) - return - } - - // Get the response body - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - src := string(body) - - for _, match := range session.Extractor.FindAllString(src, -1) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match} - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "hackertarget" -} diff --git a/pkg/subscraping/sources/intelx/intelx.go b/pkg/subscraping/sources/intelx/intelx.go deleted file mode 100644 index c07bbef..0000000 --- a/pkg/subscraping/sources/intelx/intelx.go +++ /dev/null @@ -1,114 +0,0 @@ -package intelx - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -type searchResponseType struct { - Id string `json:"id"` - Status int `json:"status"` -} - -type selectorType struct { - Selectvalue string `json:"selectorvalue"` -} - -type searchResultType struct { - Selectors []selectorType `json:"selectors"` - Status int `json:"status"` -} - -type requestBody struct { - Term string - Maxresults int - Media int - Target int - Terminate []int - Timeout int -} - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - defer close(results) - if session.Keys.IntelXKey == "" || session.Keys.IntelXHost == "" { - return - } - - searchURL := fmt.Sprintf("https://%s/phonebook/search?k=%s", session.Keys.IntelXHost, session.Keys.IntelXKey) - reqBody := requestBody{ - Term: domain, - Maxresults: 100000, - Media: 0, - Target: 1, - Timeout: 20, - } - - body, err := json.Marshal(reqBody) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - - resp, err := http.Post(searchURL, "application/json", bytes.NewBuffer(body)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - return - } - - var response searchResponseType - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - close(results) - return - } - - resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", session.Keys.IntelXHost, session.Keys.IntelXKey, response.Id) - status := 0 - for status == 0 || status == 3 { - resp, err = session.Get(ctx, resultsURL, "", nil) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - var response searchResultType - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - body, err = ioutil.ReadAll(resp.Body) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - resp.Body.Close() - status = response.Status - for _, hostname := range response.Selectors { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue} - } - } - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "intelx" -} diff --git a/pkg/subscraping/sources/ipv4info/ipv4info.go b/pkg/subscraping/sources/ipv4info/ipv4info.go deleted file mode 100755 index ac94c2b..0000000 --- a/pkg/subscraping/sources/ipv4info/ipv4info.go +++ /dev/null @@ -1,169 +0,0 @@ -package ipv4info - -import ( - "context" - "io/ioutil" - "regexp" - "strconv" - - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - resp, err := session.NormalGetWithContext(ctx, "http://ipv4info.com/search/"+domain) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - close(results) - return - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - src := string(body) - - regxTokens := regexp.MustCompile("/ip-address/(.*)/" + domain) - matchTokens := regxTokens.FindAllString(src, -1) - - if len(matchTokens) <= 0 { - close(results) - return - } - token := matchTokens[0] - - resp, err = session.NormalGetWithContext(ctx, "http://ipv4info.com"+token) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - close(results) - return - } - - body, err = ioutil.ReadAll(resp.Body) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - src = string(body) - - regxTokens = regexp.MustCompile("/dns/(.*?)/" + domain) - matchTokens = regxTokens.FindAllString(src, -1) - if len(matchTokens) <= 0 { - close(results) - return - } - token = matchTokens[0] - - resp, err = session.NormalGetWithContext(ctx, "http://ipv4info.com"+token) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - close(results) - return - } - - body, err = ioutil.ReadAll(resp.Body) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - src = string(body) - - regxTokens = regexp.MustCompile("/subdomains/(.*?)/" + domain) - matchTokens = regxTokens.FindAllString(src, -1) - if len(matchTokens) <= 0 { - close(results) - return - } - token = matchTokens[0] - - resp, err = session.NormalGetWithContext(ctx, "http://ipv4info.com"+token) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - close(results) - return - } - - body, err = ioutil.ReadAll(resp.Body) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - src = string(body) - - for _, match := range session.Extractor.FindAllString(src, -1) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match} - } - nextPage := 1 - - for { - further := s.getSubdomains(ctx, domain, &nextPage, src, session, results) - if !further { - break - } - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "ipv4info" -} - -func (s *Source) getSubdomains(ctx context.Context, domain string, nextPage *int, src string, session *subscraping.Session, results chan subscraping.Result) bool { - for { - select { - case <-ctx.Done(): - return false - default: - regxTokens := regexp.MustCompile("/subdomains/.*/page" + strconv.Itoa(*nextPage) + "/" + domain + ".html") - matchTokens := regxTokens.FindAllString(src, -1) - if len(matchTokens) == 0 { - return false - } - token := matchTokens[0] - - resp, err := session.NormalGetWithContext(ctx, "http://ipv4info.com"+token) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return false - } - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - return false - } - resp.Body.Close() - src = string(body) - for _, match := range session.Extractor.FindAllString(src, -1) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match} - } - *nextPage++ - return true - } - } -} diff --git a/pkg/subscraping/sources/passivetotal/passivetotal.go b/pkg/subscraping/sources/passivetotal/passivetotal.go deleted file mode 100755 index 03b5ff9..0000000 --- a/pkg/subscraping/sources/passivetotal/passivetotal.go +++ /dev/null @@ -1,72 +0,0 @@ -package passivetotal - -import ( - "bytes" - "context" - "net/http" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -type response struct { - Subdomains []string `json:"subdomains"` -} - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - if session.Keys.PassiveTotalUsername == "" || session.Keys.PassiveTotalPassword == "" { - close(results) - return - } - - // Create JSON Get body - var request = []byte(`{"query":"` + domain + `"}`) - - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.passivetotal.org/v2/enrichment/subdomains", bytes.NewBuffer(request)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - close(results) - return - } - - req.SetBasicAuth(session.Keys.PassiveTotalUsername, session.Keys.PassiveTotalPassword) - req.Header.Set("Content-Type", "application/json") - - resp, err := session.Client.Do(req) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - close(results) - return - } - - data := response{} - err = jsoniter.NewDecoder(resp.Body).Decode(&data) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - - for _, subdomain := range data.Subdomains { - finalSubdomain := subdomain + "." + domain - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: finalSubdomain} - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "passivetotal" -} diff --git a/pkg/subscraping/sources/rapiddns/rapiddns.go b/pkg/subscraping/sources/rapiddns/rapiddns.go deleted file mode 100644 index abb2828..0000000 --- a/pkg/subscraping/sources/rapiddns/rapiddns.go +++ /dev/null @@ -1,46 +0,0 @@ -// Package rapiddns is a RapidDNS Scraping Engine in Golang -package rapiddns - -import ( - "context" - "io/ioutil" - - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - defer close(results) - resp, err := session.NormalGetWithContext(ctx, "https://rapiddns.io/subdomain/"+domain+"?full=1") - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - return - } - - body, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - return - } - - src := string(body) - for _, subdomain := range session.Extractor.FindAllString(src, -1) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - } - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "rapiddns" -} diff --git a/pkg/subscraping/sources/securitytrails/securitytrails.go b/pkg/subscraping/sources/securitytrails/securitytrails.go deleted file mode 100755 index 71f2276..0000000 --- a/pkg/subscraping/sources/securitytrails/securitytrails.go +++ /dev/null @@ -1,65 +0,0 @@ -package securitytrails - -import ( - "context" - "fmt" - "strings" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -type response struct { - Subdomains []string `json:"subdomains"` -} - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - if session.Keys.Securitytrails == "" { - close(results) - return - } - - resp, err := session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", map[string]string{"APIKEY": session.Keys.Securitytrails}) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - close(results) - return - } - - response := response{} - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - - for _, subdomain := range response.Subdomains { - if strings.HasSuffix(subdomain, ".") { - subdomain = subdomain + domain - } else { - subdomain = subdomain + "." + domain - } - - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "securitytrails" -} diff --git a/pkg/subscraping/sources/shodan/shodan.go b/pkg/subscraping/sources/shodan/shodan.go deleted file mode 100644 index b258abb..0000000 --- a/pkg/subscraping/sources/shodan/shodan.go +++ /dev/null @@ -1,73 +0,0 @@ -package shodan - -import ( - "context" - "strconv" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -type shodanResult struct { - Matches []shodanObject `json:"matches"` - Result int `json:"result"` - Error string `json:"error"` -} - -type shodanObject struct { - Hostnames []string `json:"hostnames"` -} - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - if session.Keys.Shodan == "" { - close(results) - return - } - - for currentPage := 0; currentPage <= 10; currentPage++ { - resp, err := session.NormalGetWithContext(ctx, "https://api.shodan.io/shodan/host/search?query=hostname:"+domain+"&page="+strconv.Itoa(currentPage)+"&key="+session.Keys.Shodan) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - close(results) - return - } - - var response shodanResult - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - - if response.Error != "" || len(response.Matches) == 0 { - close(results) - return - } - - for _, block := range response.Matches { - for _, hostname := range block.Hostnames { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname} - } - } - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "shodan" -} diff --git a/pkg/subscraping/sources/sitedossier/sitedossier.go b/pkg/subscraping/sources/sitedossier/sitedossier.go deleted file mode 100755 index 99f61db..0000000 --- a/pkg/subscraping/sources/sitedossier/sitedossier.go +++ /dev/null @@ -1,84 +0,0 @@ -package sitedossier - -import ( - "context" - "fmt" - "io/ioutil" - "math/rand" - "regexp" - "time" - - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -var reNext = regexp.MustCompile("") - -type agent struct { - results chan subscraping.Result - session *subscraping.Session -} - -func (a *agent) enumerate(ctx context.Context, baseURL string) error { - for { - select { - case <-ctx.Done(): - return nil - default: - resp, err := a.session.NormalGetWithContext(ctx, baseURL) - if err != nil { - a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} - a.session.DiscardHttpResponse(resp) - close(a.results) - return err - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} - resp.Body.Close() - close(a.results) - return err - } - resp.Body.Close() - src := string(body) - - for _, match := range a.session.Extractor.FindAllString(src, -1) { - a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: match} - } - - match1 := reNext.FindStringSubmatch(src) - time.Sleep(time.Duration((3 + rand.Intn(5))) * time.Second) - - if len(match1) > 0 { - a.enumerate(ctx, "http://www.sitedossier.com"+match1[1]) - } - return nil - } - } -} - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - a := agent{ - session: session, - results: results, - } - - go func() { - err := a.enumerate(ctx, fmt.Sprintf("http://www.sitedossier.com/parentdomain/%s", domain)) - if err == nil { - close(a.results) - } - }() - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "sitedossier" -} diff --git a/pkg/subscraping/sources/spyse/spyse.go b/pkg/subscraping/sources/spyse/spyse.go deleted file mode 100644 index 8dc3983..0000000 --- a/pkg/subscraping/sources/spyse/spyse.go +++ /dev/null @@ -1,90 +0,0 @@ -package spyse - -import ( - "context" - "strconv" - "fmt" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - - -type resultObject struct { - Name string `json:"name"` -} - -type dataObject struct { - Items []resultObject `json:"items"` - Total_Count int `json:"total_count"` -} - -type errorObject struct { - Code string `json:"code"` - Message string `json:"message"` -} - - -type spyseResult struct { - Data dataObject `json:"data"` - Error []errorObject `json:"error"` -} - - -type Source struct{} - -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - if session.Keys.Spyse == "" { - close(results) - return - } - - maxCount := 100; - - for offSet := 0; offSet <= maxCount; offSet += 100 { - resp, err := session.Get(ctx, fmt.Sprintf("https://api.spyse.com/v3/data/domain/subdomain?domain=%s&limit=100&offset=%s", domain, strconv.Itoa(offSet)), "", map[string]string{"Authorization": "Bearer " + session.Keys.Spyse}) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - close(results) - return - } - - - var response spyseResult; - - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - - if response.Data.Total_Count == 0 { - close(results) - return - } - - maxCount = response.Data.Total_Count; - - for _, hostname := range response.Data.Items { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Name} - } - } - close(results) - }() - - return results -} - - -// Name returns the name of the source -func (s *Source) Name() string { - return "spyse" -} diff --git a/pkg/subscraping/sources/sublist3r/subllist3r.go b/pkg/subscraping/sources/sublist3r/subllist3r.go deleted file mode 100644 index 2ea9c74..0000000 --- a/pkg/subscraping/sources/sublist3r/subllist3r.go +++ /dev/null @@ -1,49 +0,0 @@ -package sublist3r - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://api.sublist3r.com/search.php?domain=%s", domain)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - close(results) - return - } - defer resp.Body.Close() - var subdomains []string - // Get the response body and unmarshal - err = json.NewDecoder(resp.Body).Decode(&subdomains) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - - for _, subdomain := range subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "sublist3r" -} diff --git a/pkg/subscraping/sources/threatcrowd/threatcrowd.go b/pkg/subscraping/sources/threatcrowd/threatcrowd.go deleted file mode 100755 index a27ada3..0000000 --- a/pkg/subscraping/sources/threatcrowd/threatcrowd.go +++ /dev/null @@ -1,51 +0,0 @@ -package threatcrowd - -import ( - "context" - "fmt" - "io/ioutil" - - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://www.threatcrowd.org/searchApi/v2/domain/report/?domain=%s", domain)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - close(results) - return - } - - // Get the response body - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - - src := string(body) - - for _, match := range session.Extractor.FindAllString(src, -1) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match} - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "threatcrowd" -} diff --git a/pkg/subscraping/sources/threatminer/threatminer.go b/pkg/subscraping/sources/threatminer/threatminer.go deleted file mode 100755 index 755e0c1..0000000 --- a/pkg/subscraping/sources/threatminer/threatminer.go +++ /dev/null @@ -1,51 +0,0 @@ -package threatminer - -import ( - "context" - "fmt" - "io/ioutil" - - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://api.threatminer.org/v2/domain.php?q=%s&rt=5", domain)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - close(results) - return - } - - // Get the response body - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - - src := string(body) - - for _, match := range session.Extractor.FindAllString(src, -1) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: match} - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "threatminer" -} diff --git a/pkg/subscraping/sources/urlscan/urlscan.go b/pkg/subscraping/sources/urlscan/urlscan.go deleted file mode 100755 index ddb61d9..0000000 --- a/pkg/subscraping/sources/urlscan/urlscan.go +++ /dev/null @@ -1,60 +0,0 @@ -package urlscan - -import ( - "context" - "fmt" - - jsoniter "github.com/json-iterator/go" - "github.com/m-mizutani/urlscan-go/urlscan" - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - if session.Keys.URLScan == "" { - close(results) - return - } - - client := urlscan.NewClient(session.Keys.URLScan) - task, err := client.Submit(urlscan.SubmitArguments{URL: fmt.Sprintf("https://%s", domain)}) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - close(results) - return - } - - err = task.Wait() - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - close(results) - return - } - - data, err := jsoniter.Marshal(task.Result.Data) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - close(results) - return - } - - match := session.Extractor.FindAllString(string(data), -1) - for _, m := range match { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: m} - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "urlscan" -} diff --git a/pkg/subscraping/sources/virustotal/virustotal.go b/pkg/subscraping/sources/virustotal/virustotal.go deleted file mode 100755 index 6442e44..0000000 --- a/pkg/subscraping/sources/virustotal/virustotal.go +++ /dev/null @@ -1,58 +0,0 @@ -package virustotal - -import ( - "context" - "fmt" - - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -type response struct { - Subdomains []string `json:"subdomains"` -} - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - if session.Keys.Virustotal == "" { - close(results) - return - } - - resp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("https://www.virustotal.com/vtapi/v2/domain/report?apikey=%s&domain=%s", session.Keys.Virustotal, domain)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - close(results) - return - } - - data := response{} - err = jsoniter.NewDecoder(resp.Body).Decode(&data) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - - for _, subdomain := range data.Subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "virustotal" -} diff --git a/pkg/subscraping/sources/waybackarchive/waybackarchive.go b/pkg/subscraping/sources/waybackarchive/waybackarchive.go deleted file mode 100755 index 64be137..0000000 --- a/pkg/subscraping/sources/waybackarchive/waybackarchive.go +++ /dev/null @@ -1,53 +0,0 @@ -package waybackarchive - -import ( - "context" - "fmt" - "io/ioutil" - "strings" - - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - pagesResp, err := session.NormalGetWithContext(ctx, fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=json&fl=original&collapse=urlkey", domain)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(pagesResp) - close(results) - return - } - - body, err := ioutil.ReadAll(pagesResp.Body) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - pagesResp.Body.Close() - close(results) - return - } - pagesResp.Body.Close() - - match := session.Extractor.FindAllString(string(body), -1) - for _, subdomain := range match { - subdomain = strings.TrimPrefix(subdomain, "25") - subdomain = strings.TrimPrefix(subdomain, "2F") - - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - } - close(results) - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "waybackarchive" -} diff --git a/pkg/subscraping/sources/zoomeye/zoomeye.go b/pkg/subscraping/sources/zoomeye/zoomeye.go deleted file mode 100644 index 9a2c92c..0000000 --- a/pkg/subscraping/sources/zoomeye/zoomeye.go +++ /dev/null @@ -1,138 +0,0 @@ -package zoomeye - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "net/http" - - "github.com/projectdiscovery/subfinder/pkg/subscraping" -) - -// zoomAuth holds the ZoomEye credentials -type zoomAuth struct { - User string `json:"username"` - Pass string `json:"password"` -} - -type loginResp struct { - JWT string `json:"access_token"` -} - -// search results -type zoomeyeResults struct { - Matches []struct { - Site string `json:"site"` - Domains []string `json:"domains"` - } `json:"matches"` -} - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - if session.Keys.ZoomEyeUsername == "" || session.Keys.ZoomEyePassword == "" { - close(results) - return - } - jwt, err := doLogin(session) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - close(results) - return - } - // check if jwt is null - if jwt == "" { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errors.New("could not log into zoomeye")} - close(results) - return - } - headers := map[string]string{ - "Authorization": fmt.Sprintf("JWT %s", jwt), - "Accept": "application/json", - "Content-Type": "application/json", - } - for currentPage := 0; currentPage <= 100; currentPage++ { - api := fmt.Sprintf("https://api.zoomeye.org/web/search?query=hostname:%s&page=%d", domain, currentPage) - resp, err := session.Get(ctx, api, "", headers) - isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden - if err != nil { - if !isForbidden && currentPage == 0 { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHttpResponse(resp) - } - close(results) - return - } - - defer resp.Body.Close() - res := &zoomeyeResults{} - err = json.NewDecoder(resp.Body).Decode(res) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - close(results) - return - } - resp.Body.Close() - for _, r := range res.Matches { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Site} - for _, domain := range r.Domains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domain} - } - } - currentPage++ - } - close(results) - }() - - return results -} - -// doLogin performs authentication on the ZoomEye API -func doLogin(session *subscraping.Session) (string, error) { - creds := &zoomAuth{ - User: session.Keys.ZoomEyeUsername, - Pass: session.Keys.ZoomEyePassword, - } - body, err := json.Marshal(&creds) - if err != nil { - return "", err - } - req, err := http.NewRequest("POST", "https://api.zoomeye.org/user/login", bytes.NewBuffer(body)) - if err != nil { - return "", err - } - req.Header.Add("Content-Type", "application/json") - resp, err := session.Client.Do(req) - if err != nil { - return "", err - } - // if not 200, bad credentials - if resp.StatusCode != 200 { - io.Copy(ioutil.Discard, resp.Body) - resp.Body.Close() - return "", fmt.Errorf("login failed, non-200 response from zoomeye") - } - - defer resp.Body.Close() - login := &loginResp{} - err = json.NewDecoder(resp.Body).Decode(login) - if err != nil { - return "", err - } - return login.JWT, nil -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "zoomeye" -} diff --git a/pkg/subscraping/types.go b/pkg/subscraping/types.go deleted file mode 100755 index 4b3b9a9..0000000 --- a/pkg/subscraping/types.go +++ /dev/null @@ -1,67 +0,0 @@ -package subscraping - -import ( - "context" - "net/http" - "regexp" -) - -// Source is an interface inherited by each passive source -type Source interface { - // Run takes a domain as argument and a session object - // which contains the extractor for subdomains, http client - // and other stuff. - Run(context.Context, string, *Session) <-chan Result - // Name returns the name of the source - Name() string -} - -// Session is the option passed to the source, an option is created -// uniquely for eac source. -type Session struct { - // Extractor is the regex for subdomains created for each domain - Extractor *regexp.Regexp - // Keys is the API keys for the application - Keys Keys - // Client is the current http client - Client *http.Client -} - -// Keys contains the current API Keys we have in store -type Keys struct { - Binaryedge string `json:"binaryedge"` - CensysToken string `json:"censysUsername"` - CensysSecret string `json:"censysPassword"` - Certspotter string `json:"certspotter"` - Chaos string `json:"chaos"` - DNSDB string `json:"dnsdb"` - GitHub []string `json:"github"` - IntelXHost string `json:"intelXHost"` - IntelXKey string `json:"intelXKey"` - PassiveTotalUsername string `json:"passivetotal_username"` - PassiveTotalPassword string `json:"passivetotal_password"` - Securitytrails string `json:"securitytrails"` - Shodan string `json:"shodan"` - Spyse string `json:"spyse"` - URLScan string `json:"urlscan"` - Virustotal string `json:"virustotal"` - ZoomEyeUsername string `json:"zoomeye_username"` - ZoomEyePassword string `json:"zoomeye_password"` -} - -// Result is a result structure returned by a source -type Result struct { - Type ResultType - Source string - Value string - Error error -} - -// ResultType is the type of result returned by the source -type ResultType int - -// Types of results returned by the source -const ( - Subdomain ResultType = iota - Error -) diff --git a/pkg/subscraping/utils.go b/pkg/subscraping/utils.go deleted file mode 100755 index a65d0ea..0000000 --- a/pkg/subscraping/utils.go +++ /dev/null @@ -1,30 +0,0 @@ -package subscraping - -import ( - "regexp" - "sync" -) - -var subdomainExtractorMutex = &sync.Mutex{} - -// NewSubdomainExtractor creates a new regular expression to extract -// subdomains from text based on the given domain. -func NewSubdomainExtractor(domain string) (*regexp.Regexp, error) { - subdomainExtractorMutex.Lock() - defer subdomainExtractorMutex.Unlock() - extractor, err := regexp.Compile(`[a-zA-Z0-9\*_.-]+\.` + domain) - if err != nil { - return nil, err - } - return extractor, nil -} - -// Exists check if a key exist in a slice -func Exists(values []string, key string) bool { - for _, v := range values { - if v == key { - return true - } - } - return false -} diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..d5a6217 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,16 @@ +sonar.projectKey=projectdiscovery_subfinder +sonar.organization=projectdiscovery + +# This is the name and version displayed in the SonarCloud UI. +#sonar.projectName=dnsx +#sonar.projectVersion=1.0 + +# Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows. +sonar.sources=v2/ +sonar.tests=v2/ +sonar.test.inclusions=**/*_test.go +sonar.go.coverage.reportPaths=v2/cov.out +sonar.externalIssuesReportPaths=v2/report.json + +# Encoding of the source code. Default is default system encoding +#sonar.sourceEncoding=UTF-8 \ No newline at end of file diff --git a/v2/.goreleaser.yml b/v2/.goreleaser.yml new file mode 100644 index 0000000..4d0ea97 --- /dev/null +++ b/v2/.goreleaser.yml @@ -0,0 +1,40 @@ +before: + hooks: + - go mod tidy + +builds: +- env: + - CGO_ENABLED=0 + goos: + - windows + - linux + - darwin + goarch: + - amd64 + - 386 + - arm + - arm64 + + ignore: + - goos: darwin + goarch: '386' + - goos: windows + goarch: 'arm' + + binary: '{{ .ProjectName }}' + main: cmd/subfinder/main.go + +archives: +- format: zip + replacements: + darwin: macOS + +checksum: + algorithm: sha256 + +announce: + slack: + enabled: true + channel: '#release' + username: GoReleaser + message_template: '{{ .ProjectName }} {{ .Tag }} is out! Check it out at {{ .ReleaseURL }}' \ No newline at end of file diff --git a/v2/Makefile b/v2/Makefile new file mode 100644 index 0000000..0c708b8 --- /dev/null +++ b/v2/Makefile @@ -0,0 +1,19 @@ +# Go parameters +GOCMD=go +GOBUILD=$(GOCMD) build +GOMOD=$(GOCMD) mod +GOTEST=$(GOCMD) test +GOFLAGS := -v +LDFLAGS := -s -w + +ifneq ($(shell go env GOOS),darwin) +LDFLAGS := -extldflags "-static" +endif + +all: build +build: + $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "subfinder" cmd/subfinder/main.go +test: + $(GOTEST) $(GOFLAGS) ./... +tidy: + $(GOMOD) tidy diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go new file mode 100644 index 0000000..608ecea --- /dev/null +++ b/v2/cmd/integration-test/integration-test.go @@ -0,0 +1,86 @@ +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/logrusorgru/aurora" + + "github.com/projectdiscovery/subfinder/v2/pkg/testutils" +) + +var ( + debug = os.Getenv("DEBUG") == "true" + githubAction = os.Getenv("GH_ACTION") == "true" + customTests = os.Getenv("TESTS") + + success = aurora.Green("[✓]").String() + failed = aurora.Red("[✘]").String() + + sourceTests = map[string]testutils.TestCase{ + "dnsrepo": dnsrepoTestcases{}, + } +) + +func main() { + failedTestCases := runTests(toMap(toSlice(customTests))) + + if len(failedTestCases) > 0 { + if githubAction { + debug = true + fmt.Println("::group::Failed integration tests in debug mode") + _ = runTests(failedTestCases) + fmt.Println("::endgroup::") + } + os.Exit(1) + } +} + +func runTests(customTestCases map[string]struct{}) map[string]struct{} { + failedTestCases := map[string]struct{}{} + + for source, testCase := range sourceTests { + if len(customTestCases) == 0 { + fmt.Printf("Running test cases for %q source\n", aurora.Blue(source)) + } + if err, failedTemplatePath := execute(source, testCase); err != nil { + failedTestCases[failedTemplatePath] = struct{}{} + } + } + return failedTestCases +} + +func execute(source string, testCase testutils.TestCase) (error, string) { + if err := testCase.Execute(); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, source, err) + return err, source + } + + fmt.Printf("%s Test \"%s\" passed!\n", success, source) + return nil, "" +} + +func expectResultsGreaterThanCount(results []string, expectedNumber int) error { + if len(results) > expectedNumber { + return nil + } + return fmt.Errorf("incorrect number of results: expected a result greater than %d,but got %d", expectedNumber, len(results)) +} +func toSlice(value string) []string { + if strings.TrimSpace(value) == "" { + return []string{} + } + + return strings.Split(value, ",") +} + +func toMap(slice []string) map[string]struct{} { + result := make(map[string]struct{}, len(slice)) + for _, value := range slice { + if _, ok := result[value]; !ok { + result[value] = struct{}{} + } + } + return result +} diff --git a/v2/cmd/integration-test/run.sh b/v2/cmd/integration-test/run.sh new file mode 100755 index 0000000..2d826de --- /dev/null +++ b/v2/cmd/integration-test/run.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +echo "::task~> Clean up & Build binaries files" +rm integration-test subfinder 2>/dev/null +cd ../subfinder +go build +mv subfinder ../integration-test/subfinder +cd ../integration-test +go build +echo "::done::" +echo "::task~> Run integration test" +./integration-test +echo "::done::" +if [ $? -eq 0 ] +then + exit 0 +else + exit 1 +fi diff --git a/v2/cmd/integration-test/source-test.go b/v2/cmd/integration-test/source-test.go new file mode 100644 index 0000000..3b5e4ca --- /dev/null +++ b/v2/cmd/integration-test/source-test.go @@ -0,0 +1,33 @@ +package main + +import ( + "errors" + "fmt" + "os" + + "github.com/projectdiscovery/subfinder/v2/pkg/testutils" +) + +type dnsrepoTestcases struct{} + +func (h dnsrepoTestcases) Execute() error { + token := os.Getenv("DNSREPO_API_KEY") + if token == "" { + return errors.New("missing dns repo api key") + } + dnsToken := fmt.Sprintf(`dnsrepo: [%s]`, token) + file, err := os.CreateTemp("", "provider.yaml") + if err != nil { + return err + } + defer os.RemoveAll(file.Name()) + _, err = file.WriteString(dnsToken) + if err != nil { + return err + } + results, err := testutils.RunSubfinderAndGetResults(debug, "hackerone.com", "-s", "dnsrepo", "-provider-config", file.Name()) + if err != nil { + return err + } + return expectResultsGreaterThanCount(results, 0) +} diff --git a/v2/cmd/subfinder/main.go b/v2/cmd/subfinder/main.go new file mode 100644 index 0000000..8a285ea --- /dev/null +++ b/v2/cmd/subfinder/main.go @@ -0,0 +1,23 @@ +package main + +import ( + // Attempts to increase the OS file descriptors - Fail silently + _ "github.com/projectdiscovery/fdmax/autofdmax" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/subfinder/v2/pkg/runner" +) + +func main() { + // Parse the command line flags and read config files + options := runner.ParseOptions() + + newRunner, err := runner.NewRunner(options) + if err != nil { + gologger.Fatal().Msgf("Could not create runner: %s\n", err) + } + + err = newRunner.RunEnumeration() + if err != nil { + gologger.Fatal().Msgf("Could not run enumeration: %s\n", err) + } +} diff --git a/v2/go.mod b/v2/go.mod new file mode 100644 index 0000000..f20b5c5 --- /dev/null +++ b/v2/go.mod @@ -0,0 +1,66 @@ +module github.com/projectdiscovery/subfinder/v2 + +go 1.18 + +require ( + github.com/corpix/uarand v0.2.0 + github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd + github.com/json-iterator/go v1.1.12 + github.com/lib/pq v1.10.7 + github.com/projectdiscovery/chaos-client v0.3.0 + github.com/projectdiscovery/dnsx v1.1.1 + github.com/projectdiscovery/fdmax v0.0.4 + github.com/projectdiscovery/fileutil v0.0.3 + github.com/projectdiscovery/gologger v1.1.5-0.20220817095646-8663411b1b0b + github.com/projectdiscovery/ratelimit v0.0.1 + github.com/rs/xid v1.4.0 + github.com/stretchr/testify v1.8.1 + github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect + github.com/aymerick/douceur v0.2.0 // indirect + github.com/dsnet/compress v0.0.1 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/gorilla/css v1.0.0 // indirect + github.com/mholt/archiver v3.1.1+incompatible // indirect + github.com/microcosm-cc/bluemonday v1.0.21 // indirect + github.com/nwaples/rardecode v1.1.0 // indirect + github.com/pierrec/lz4 v2.6.0+incompatible // indirect + github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc // indirect + github.com/projectdiscovery/cdncheck v0.0.3 // indirect + github.com/projectdiscovery/httputil v0.0.0-20210906072657-f3a099cb20bc // indirect + github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8 // indirect + github.com/projectdiscovery/mapcidr v1.0.1 // indirect + github.com/projectdiscovery/retryablehttp-go v1.0.2 // indirect + github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 // indirect + github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect + github.com/ulikunitz/xz v0.5.7 // indirect + github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect + github.com/yl2chen/cidranger v1.0.2 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.8.0 // indirect + golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect + golang.org/x/text v0.4.0 // indirect + golang.org/x/tools v0.1.12 // indirect + gopkg.in/djherbis/times.v1 v1.2.0 // indirect +) + +require ( + github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/logrusorgru/aurora v2.0.3+incompatible + github.com/miekg/dns v1.1.50 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pkg/errors v0.9.1 + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/projectdiscovery/goflags v0.1.3 + github.com/projectdiscovery/retryabledns v1.0.15 // indirect + github.com/projectdiscovery/stringsutil v0.0.2 // indirect + golang.org/x/net v0.1.0 // indirect + golang.org/x/sys v0.2.0 // indirect +) diff --git a/v2/go.sum b/v2/go.sum new file mode 100644 index 0000000..fface0d --- /dev/null +++ b/v2/go.sum @@ -0,0 +1,232 @@ +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= +github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= +github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= +github.com/corpix/uarand v0.2.0/go.mod h1:/3Z1QIqWkDIhf6XWn/08/uMHoQ8JUoTIKc2iPchBOmM= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= +github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd h1:FsX+T6wA8spPe4c1K9vi7T0LvNCO1TTqiL8u7Wok2hw= +github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd/go.mod h1:VzxiSdG6j1pi7rwGm/xYI5RbtpBgM8sARDXlvEvxlu0= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= +github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= +github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= +github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= +github.com/microcosm-cc/bluemonday v1.0.19/go.mod h1:QNzV2UbLK2/53oIIwTOyLUSABMkjZ4tqiyC1g/DyqxE= +github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg= +github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= +github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= +github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= +github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A= +github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= +github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc h1:jqZK68yPOnNNRmwuXqytl+T9EbwneEUCvMDRjLe0J04= +github.com/projectdiscovery/blackrock v0.0.0-20220628111055-35616c71b2dc/go.mod h1:5tNGQP9kOfW+X5+40pZP8aqPYLHs45nJkFaSHLxdeH8= +github.com/projectdiscovery/cdncheck v0.0.3 h1:li2/rUJmhVXSqRFyhJMqi6pdBX6ZxMnwzBfE0Kifj/g= +github.com/projectdiscovery/cdncheck v0.0.3/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= +github.com/projectdiscovery/chaos-client v0.3.0 h1:A4NgOYRCrlsSZUBTCT2HAT/uTEJly17+nWcXHRXR+Ko= +github.com/projectdiscovery/chaos-client v0.3.0/go.mod h1:AWx/KZgtBE5SULpsgyQLLfb+SQgVtCih83gvRtTpwl4= +github.com/projectdiscovery/dnsx v1.1.1 h1:yGYEH1vfVN7YdvdOtPzPvrc6JuHCi8wBTAkStP/f2QI= +github.com/projectdiscovery/dnsx v1.1.1/go.mod h1:DVvc+ePRCknahLpz4Y8nMppYOGUZhkEmYsTPuYx1a5w= +github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= +github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= +github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= +github.com/projectdiscovery/fileutil v0.0.0-20220609150212-453ac591c36c/go.mod h1:g8wsrb0S5NtEN0JgVyyPeb3FQdArx+UMESmFX94bcGY= +github.com/projectdiscovery/fileutil v0.0.0-20220705195237-01becc2a8963/go.mod h1:DaY7wmLPMleyHDCD/14YApPCDtrARY4J8Eny2ZGsG/g= +github.com/projectdiscovery/fileutil v0.0.3 h1:GSsoey4p8ZHIRxWF2VXh4mhLr+wfEkpJwvF0Dxpn/gg= +github.com/projectdiscovery/fileutil v0.0.3/go.mod h1:GLejWd3YerG3RNYD/Hk2pJlytlYRgHdkWfWUAdCH2YQ= +github.com/projectdiscovery/goflags v0.0.8/go.mod h1:GDSkWyXa6kfQjpJu10SO64DN8lXuKXVENlBMk8N7H80= +github.com/projectdiscovery/goflags v0.1.3 h1:dnJlg19VkDp1iYkpAod4Tv+OAngr7Mq61LMMpBQlO0M= +github.com/projectdiscovery/goflags v0.1.3/go.mod h1:/7ZAoY1SVfUcGobTP5QDvGQmrpPDDlBUDIMr7c+r94Q= +github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= +github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= +github.com/projectdiscovery/gologger v1.1.5-0.20220817095646-8663411b1b0b h1:sncWNStu8+oT3vDvKKFncr5FxEui5Bs0ET2Qkj0AVBo= +github.com/projectdiscovery/gologger v1.1.5-0.20220817095646-8663411b1b0b/go.mod h1:6fC5JFfw/DPbkaNFb13402F4eha0Yntc2F87gHtIdkA= +github.com/projectdiscovery/hmap v0.0.1/go.mod h1:VDEfgzkKQdq7iGTKz8Ooul0NuYHQ8qiDs6r8bPD1Sb0= +github.com/projectdiscovery/httputil v0.0.0-20210906072657-f3a099cb20bc h1:C0L6pUvVI+sPJSBaPQJEG/HjPtg8Mgs2vEpsdrl064A= +github.com/projectdiscovery/httputil v0.0.0-20210906072657-f3a099cb20bc/go.mod h1:BueJPSPWAX11IFS6bdAqTkekiIz5Fgco5LVc1kqO9L4= +github.com/projectdiscovery/ipranger v0.0.2/go.mod h1:kcAIk/lo5rW+IzUrFkeYyXnFJ+dKwYooEOHGVPP/RWE= +github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8 h1:HRqev12wKvcwK1fe4pSlMfQdPHo9LfTxuFeRN4f3tS4= +github.com/projectdiscovery/iputil v0.0.0-20220712175312-b9406f31cdd8/go.mod h1:vHRC+9exsfSbEngMKDl0xiWqkxlLk3lHQZpbS2yFT8U= +github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= +github.com/projectdiscovery/mapcidr v1.0.1 h1:eaLBRrImwlYXv8vbXTwR4sxoQqIxR3Y5k/Sd7HhTIII= +github.com/projectdiscovery/mapcidr v1.0.1/go.mod h1:/qxlpxXZQFFjHynSc9u5O0kUPzH46VskECiwLiz7/vw= +github.com/projectdiscovery/ratelimit v0.0.1 h1:GnCfbKmkLdDLXT3QS4KS0zCsuDGkoRQE0YDbTqzQmS8= +github.com/projectdiscovery/ratelimit v0.0.1/go.mod h1:zenrIElIcKg0Y9h7pMfTlw5vaI/kCl8uxXm+PfgbBSw= +github.com/projectdiscovery/retryabledns v1.0.15 h1:3Nn119UwYsfUPC3g0q57ftz0Wb5Zl5ppvw8R0Xu0DEI= +github.com/projectdiscovery/retryabledns v1.0.15/go.mod h1:3YbsQVqP7jbQ3CDmarhyVtkJaJ8XcB7S19vMeyMxZxk= +github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjBYRwXlNEq0PvrezMV0U= +github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= +github.com/projectdiscovery/sliceutil v0.0.0-20220617151003-15892688e1d6/go.mod h1:9YZb6LRjLYAvSOm65v787dwauurixSyjlqXyYa4rTTA= +github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4 h1:C04j5gVVMXqFyBIetAz92SyPRYCpkFgIwZw0L/pps9Q= +github.com/projectdiscovery/sliceutil v0.0.0-20220625085859-c3a4ecb669f4/go.mod h1:RxDaccMjPzIuF7F8XbdGl1yOcqxN4YPiHr9xHpfCkGI= +github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= +github.com/projectdiscovery/stringsutil v0.0.0-20220422150559-b54fb5dc6833/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= +github.com/projectdiscovery/stringsutil v0.0.0-20220612082425-0037ce9f89f3/go.mod h1:mF5sh4jTghoGWwgUb9qWi5waTFklClDbtrqtJU93awc= +github.com/projectdiscovery/stringsutil v0.0.0-20220731064040-4b67f194751e/go.mod h1:32NYmKyHkKsmisAOAaWrR15lz2ysz2M8x3KMeeoRHoU= +github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= +github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= +github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= +github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= +github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.7.3/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= +github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4= +github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= +github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= +github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= +go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/djherbis/times.v1 v1.2.0 h1:UCvDKl1L/fmBygl2Y7hubXCnY7t4Yj46ZrBFNUipFbM= +gopkg.in/djherbis/times.v1 v1.2.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/v2/pkg/passive/doc.go b/v2/pkg/passive/doc.go new file mode 100644 index 0000000..022a55a --- /dev/null +++ b/v2/pkg/passive/doc.go @@ -0,0 +1,3 @@ +// Package passive provides capability for doing passive subdomain +// enumeration on targets. +package passive diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go new file mode 100644 index 0000000..71b27e7 --- /dev/null +++ b/v2/pkg/passive/passive.go @@ -0,0 +1,58 @@ +package passive + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// EnumerateSubdomains enumerates all the subdomains for a given domain +func (a *Agent) EnumerateSubdomains(domain string, proxy string, rateLimit, timeout int, maxEnumTime time.Duration) chan subscraping.Result { + results := make(chan subscraping.Result) + go func() { + defer close(results) + + session, err := subscraping.NewSession(domain, proxy, rateLimit, timeout) + if err != nil { + results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} + return + } + + ctx, cancel := context.WithTimeout(context.Background(), maxEnumTime) + + timeTaken := make(map[string]string) + timeTakenMutex := &sync.Mutex{} + + wg := &sync.WaitGroup{} + // Run each source in parallel on the target domain + for _, runner := range a.sources { + wg.Add(1) + + now := time.Now() + go func(source subscraping.Source) { + for resp := range source.Run(ctx, domain, session) { + results <- resp + } + + duration := time.Since(now) + timeTakenMutex.Lock() + timeTaken[source.Name()] = fmt.Sprintf("Source took %s for enumeration\n", duration) + timeTakenMutex.Unlock() + + wg.Done() + }(runner) + } + wg.Wait() + + for source, data := range timeTaken { + gologger.Verbose().Label(source).Msg(data) + } + + cancel() + }() + return results +} diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go new file mode 100644 index 0000000..1271077 --- /dev/null +++ b/v2/pkg/passive/sources.go @@ -0,0 +1,150 @@ +package passive + +import ( + "fmt" + "strings" + + "golang.org/x/exp/maps" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/alienvault" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/anubis" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bevigil" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/binaryedge" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/c99" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chinaz" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsrepo" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fullhunt" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hunter" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/passivetotal" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/reconcloud" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/riddler" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/robtex" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatbook" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/threatminer" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeye" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi" +) + +var AllSources = [...]subscraping.Source{ + &alienvault.Source{}, + &anubis.Source{}, + &bevigil.Source{}, + &binaryedge.Source{}, + &bufferover.Source{}, + &c99.Source{}, + &censys.Source{}, + &certspotter.Source{}, + &chaos.Source{}, + &chinaz.Source{}, + &commoncrawl.Source{}, + &crtsh.Source{}, + &dnsdb.Source{}, + &dnsdumpster.Source{}, + &fofa.Source{}, + &fullhunt.Source{}, + &github.Source{}, + &hackertarget.Source{}, + &intelx.Source{}, + &passivetotal.Source{}, + &quake.Source{}, + &rapiddns.Source{}, + &riddler.Source{}, + &robtex.Source{}, + &securitytrails.Source{}, + &shodan.Source{}, + &sitedossier.Source{}, + &threatbook.Source{}, + &threatminer.Source{}, + &virustotal.Source{}, + &waybackarchive.Source{}, + &whoisxmlapi.Source{}, + &zoomeye.Source{}, + &zoomeyeapi.Source{}, + &dnsrepo.Source{}, + &hunter.Source{}, + &reconcloud.Source{}, + +} + +var NameSourceMap = make(map[string]subscraping.Source, len(AllSources)) + +func init() { + for _, currentSource := range AllSources { + NameSourceMap[strings.ToLower(currentSource.Name())] = currentSource + } +} + +// Agent is a struct for running passive subdomain enumeration +// against a given host. It wraps subscraping package and provides +// a layer to build upon. +type Agent struct { + sources []subscraping.Source +} + +// New creates a new agent for passive subdomain discovery +func New(sourceNames, excludedSourceNames []string, useAllSources, useSourcesSupportingRecurse bool) *Agent { + sources := make(map[string]subscraping.Source, len(AllSources)) + + if useAllSources { + maps.Copy(sources, NameSourceMap) + } else { + if len(sourceNames) > 0 { + for _, source := range sourceNames { + if NameSourceMap[source] == nil { + gologger.Warning().Msgf("There is no source with the name: '%s'", source) + } else { + sources[source] = NameSourceMap[source] + } + } + } else { + for _, currentSource := range AllSources { + if currentSource.IsDefault() { + sources[currentSource.Name()] = currentSource + } + } + } + } + + if len(excludedSourceNames) > 0 { + for _, sourceName := range excludedSourceNames { + delete(sources, sourceName) + } + } + + if useSourcesSupportingRecurse { + for sourceName, source := range sources { + if !source.HasRecursiveSupport() { + delete(sources, sourceName) + } + } + } + + gologger.Debug().Msgf(fmt.Sprintf("Selected source(s) for this search: %s", strings.Join(maps.Keys(sources), ", "))) + + // Create the agent, insert the sources and remove the excluded sources + agent := &Agent{sources: maps.Values(sources)} + + return agent +} diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go new file mode 100644 index 0000000..367be7c --- /dev/null +++ b/v2/pkg/passive/sources_test.go @@ -0,0 +1,162 @@ +package passive + +import ( + "fmt" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "golang.org/x/exp/maps" +) + +var ( + expectedAllSources = []string{ + "alienvault", + "anubis", + "bevigil", + "binaryedge", + "bufferover", + "c99", + "censys", + "certspotter", + "chaos", + "chinaz", + "commoncrawl", + "crtsh", + "dnsdumpster", + "dnsdb", + "dnsrepo", + "fofa", + "fullhunt", + "github", + "hackertarget", + "intelx", + "passivetotal", + "quake", + "rapiddns", + "riddler", + "robtex", + "securitytrails", + "shodan", + "sitedossier", + "threatbook", + "threatminer", + "virustotal", + "waybackarchive", + "whoisxmlapi", + "zoomeye", + "zoomeyeapi", + "hunter", + "reconcloud", + } + + expectedDefaultSources = []string{ + "alienvault", + "anubis", + "bevigil", + "bufferover", + "c99", + "certspotter", + "censys", + "chaos", + "chinaz", + "crtsh", + "dnsdumpster", + "dnsrepo", + "fofa", + "fullhunt", + "hackertarget", + "intelx", + "passivetotal", + "quake", + "robtex", + "riddler", + "securitytrails", + "shodan", + "threatminer", + "virustotal", + "whoisxmlapi", + "hunter", + "reconcloud", + } + + expectedDefaultRecursiveSources = []string{ + "alienvault", + "binaryedge", + "bufferover", + "certspotter", + "crtsh", + "dnsdumpster", + "hackertarget", + "passivetotal", + "securitytrails", + "virustotal", + "reconcloud", + } +) + +func TestSourceCategorization(t *testing.T) { + defaultSources := make([]string, 0, len(AllSources)) + recursiveSources := make([]string, 0, len(AllSources)) + for _, source := range AllSources { + sourceName := source.Name() + if source.IsDefault() { + defaultSources = append(defaultSources, sourceName) + } + + if source.HasRecursiveSupport() { + recursiveSources = append(recursiveSources, sourceName) + } + } + + assert.ElementsMatch(t, expectedDefaultSources, defaultSources) + assert.ElementsMatch(t, expectedDefaultRecursiveSources, recursiveSources) + assert.ElementsMatch(t, expectedAllSources, maps.Keys(NameSourceMap)) +} + +func TestSourceFiltering(t *testing.T) { + someSources := []string{ + "alienvault", + "chaos", + "crtsh", + "virustotal", + } + + someExclusions := []string{ + "alienvault", + "virustotal", + } + + tests := []struct { + sources []string + exclusions []string + withAllSources bool + withRecursion bool + expectedLength int + }{ + {someSources, someExclusions, false, false, len(someSources) - len(someExclusions)}, + {someSources, someExclusions, false, true, 1}, + {someSources, someExclusions, true, false, len(AllSources) - len(someExclusions)}, + {someSources, someExclusions, true, true, 9}, + + {someSources, []string{}, false, false, len(someSources)}, + {someSources, []string{}, true, false, len(AllSources)}, + + {[]string{}, []string{}, false, false, len(expectedDefaultSources)}, + {[]string{}, []string{}, false, true, 10}, + {[]string{}, []string{}, true, false, len(AllSources)}, + {[]string{}, []string{}, true, true, len(expectedDefaultRecursiveSources)}, + } + for index, test := range tests { + t.Run(strconv.Itoa(index+1), func(t *testing.T) { + agent := New(test.sources, test.exclusions, test.withAllSources, test.withRecursion) + + for _, v := range agent.sources { + fmt.Println(v.Name()) + } + + assert.Equal(t, test.expectedLength, len(agent.sources)) + agent = nil + }) + } +} diff --git a/v2/pkg/passive/sources_wo_auth_test.go b/v2/pkg/passive/sources_wo_auth_test.go new file mode 100644 index 0000000..6aa70b5 --- /dev/null +++ b/v2/pkg/passive/sources_wo_auth_test.go @@ -0,0 +1,50 @@ +package passive + +import ( + "context" + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +func TestSourcesWithoutKeys(t *testing.T) { + domain := "hackerone.com" + timeout := 60 + + gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) + + ctx := context.Background() + session, err := subscraping.NewSession(domain, "", 0, timeout) + assert.Nil(t, err) + + var expected = subscraping.Result{Type: subscraping.Subdomain, Value: domain, Error: nil} + + for _, source := range AllSources { + if source.NeedsKey() { + continue + } + + t.Run(source.Name(), func(t *testing.T) { + var results []subscraping.Result + + for result := range source.Run(ctx, domain, session) { + results = append(results, result) + + assert.Equal(t, source.Name(), result.Source) + + assert.Equal(t, expected.Type, result.Type) + assert.Equal(t, reflect.TypeOf(expected.Error), reflect.TypeOf(result.Error), result.Error) + + assert.True(t, strings.HasSuffix(strings.ToLower(result.Value), strings.ToLower(expected.Value))) + } + + assert.GreaterOrEqual(t, len(results), 1) + }) + } +} diff --git a/v2/pkg/resolve/client.go b/v2/pkg/resolve/client.go new file mode 100644 index 0000000..feaa9bc --- /dev/null +++ b/v2/pkg/resolve/client.go @@ -0,0 +1,32 @@ +package resolve + +import ( + "github.com/projectdiscovery/dnsx/libs/dnsx" +) + +// DefaultResolvers contains the default list of resolvers known to be good +var DefaultResolvers = []string{ + "1.1.1.1:53", // Cloudflare primary + "1.0.0.1:53", // Cloudflare secondary + "8.8.8.8:53", // Google primary + "8.8.4.4:53", // Google secondary + "9.9.9.9:53", // Quad9 Primary + "9.9.9.10:53", // Quad9 Secondary + "77.88.8.8:53", // Yandex Primary + "77.88.8.1:53", // Yandex Secondary + "208.67.222.222:53", // OpenDNS Primary + "208.67.220.220:53", // OpenDNS Secondary +} + +// Resolver is a struct for resolving DNS names +type Resolver struct { + DNSClient *dnsx.DNSX + Resolvers []string +} + +// New creates a new resolver struct with the default resolvers +func New() *Resolver { + return &Resolver{ + Resolvers: []string{}, + } +} diff --git a/v2/pkg/resolve/doc.go b/v2/pkg/resolve/doc.go new file mode 100644 index 0000000..e14f359 --- /dev/null +++ b/v2/pkg/resolve/doc.go @@ -0,0 +1,3 @@ +// Package resolve is used to handle resolving records +// It also handles wildcard subdomains and rotating resolvers. +package resolve diff --git a/v2/pkg/resolve/resolve.go b/v2/pkg/resolve/resolve.go new file mode 100644 index 0000000..65c8973 --- /dev/null +++ b/v2/pkg/resolve/resolve.go @@ -0,0 +1,122 @@ +package resolve + +import ( + "fmt" + "sync" + + "github.com/rs/xid" +) + +const ( + maxWildcardChecks = 3 +) + +// ResolutionPool is a pool of resolvers created for resolving subdomains +// for a given host. +type ResolutionPool struct { + *Resolver + Tasks chan HostEntry + Results chan Result + wg *sync.WaitGroup + removeWildcard bool + + wildcardIPs map[string]struct{} +} + +// HostEntry defines a host with the source +type HostEntry struct { + Host string + Source string +} + +// Result contains the result for a host resolution +type Result struct { + Type ResultType + Host string + IP string + Error error + Source string +} + +// ResultType is the type of result found +type ResultType int + +// Types of data result can return +const ( + Subdomain ResultType = iota + Error +) + +// NewResolutionPool creates a pool of resolvers for resolving subdomains of a given domain +func (r *Resolver) NewResolutionPool(workers int, removeWildcard bool) *ResolutionPool { + resolutionPool := &ResolutionPool{ + Resolver: r, + Tasks: make(chan HostEntry), + Results: make(chan Result), + wg: &sync.WaitGroup{}, + removeWildcard: removeWildcard, + wildcardIPs: make(map[string]struct{}), + } + + go func() { + for i := 0; i < workers; i++ { + resolutionPool.wg.Add(1) + go resolutionPool.resolveWorker() + } + resolutionPool.wg.Wait() + close(resolutionPool.Results) + }() + + return resolutionPool +} + +// InitWildcards inits the wildcard ips array +func (r *ResolutionPool) InitWildcards(domain string) error { + for i := 0; i < maxWildcardChecks; i++ { + uid := xid.New().String() + + hosts, _ := r.DNSClient.Lookup(uid + "." + domain) + if len(hosts) == 0 { + return fmt.Errorf("%s is not a wildcard domain", domain) + } + + // Append all wildcard ips found for domains + for _, host := range hosts { + r.wildcardIPs[host] = struct{}{} + } + } + return nil +} + +func (r *ResolutionPool) resolveWorker() { + for task := range r.Tasks { + if !r.removeWildcard { + r.Results <- Result{Type: Subdomain, Host: task.Host, IP: "", Source: task.Source} + continue + } + + hosts, err := r.DNSClient.Lookup(task.Host) + if err != nil { + r.Results <- Result{Type: Error, Host: task.Host, Source: task.Source, Error: err} + continue + } + + if len(hosts) == 0 { + continue + } + + var skip bool + for _, host := range hosts { + // Ignore the host if it exists in wildcard ips map + if _, ok := r.wildcardIPs[host]; ok { + skip = true + break + } + } + + if !skip { + r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source} + } + } + r.wg.Done() +} diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go new file mode 100644 index 0000000..991a1ef --- /dev/null +++ b/v2/pkg/runner/banners.go @@ -0,0 +1,26 @@ +package runner + +import ( + "github.com/projectdiscovery/gologger" +) + +const banner = ` + __ _____ __ + _______ __/ /_ / __(_)___ ____/ /__ _____ + / ___/ / / / __ \/ /_/ / __ \/ __ / _ \/ ___/ + (__ ) /_/ / /_/ / __/ / / / / /_/ / __/ / +/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.5.5 +` + +// Version is the current version of subfinder +const Version = `v2.5.5` + +// showBanner is used to show the banner to the user +func showBanner() { + gologger.Print().Msgf("%s\n", banner) + gologger.Print().Msgf("\t\tprojectdiscovery.io\n\n") + + gologger.Print().Msgf("Use with caution. You are responsible for your actions\n") + gologger.Print().Msgf("Developers assume no liability and are not responsible for any misuse or damage.\n") + gologger.Print().Msgf("By using subfinder, you also agree to the terms of the APIs used.\n\n") +} diff --git a/v2/pkg/runner/config.go b/v2/pkg/runner/config.go new file mode 100644 index 0000000..6aa1490 --- /dev/null +++ b/v2/pkg/runner/config.go @@ -0,0 +1,62 @@ +package runner + +import ( + "os" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/subfinder/v2/pkg/passive" +) + +// GetConfigDirectory gets the subfinder config directory for a user +func GetConfigDirectory() (string, error) { + var config string + + directory, err := os.UserHomeDir() + if err != nil { + return config, err + } + config = directory + "/.config/subfinder" + + // Create All directory for subfinder even if they exist + err = os.MkdirAll(config, os.ModePerm) + if err != nil { + return config, err + } + + return config, nil +} + +// CreateProviderConfigYAML marshals the input map to the given location on the disk +func CreateProviderConfigYAML(configFilePath string, sourcesRequiringApiKeysMap map[string][]string) error { + configFile, err := os.Create(configFilePath) + if err != nil { + return err + } + defer configFile.Close() + + return yaml.NewEncoder(configFile).Encode(sourcesRequiringApiKeysMap) +} + +// UnmarshalFrom writes the marshaled yaml config to disk +func UnmarshalFrom(file string) error { + f, err := os.Open(file) + if err != nil { + return err + } + defer f.Close() + + sourceApiKeysMap := map[string][]string{} + err = yaml.NewDecoder(f).Decode(sourceApiKeysMap) + for _, source := range passive.AllSources { + sourceName := strings.ToLower(source.Name()) + apiKeys := sourceApiKeysMap[sourceName] + if source.NeedsKey() && apiKeys != nil && len(apiKeys) > 0 { + gologger.Debug().Msgf("API key(s) found for %s.", sourceName) + source.AddApiKeys(apiKeys) + } + } + return err +} diff --git a/v2/pkg/runner/config_test.go b/v2/pkg/runner/config_test.go new file mode 100644 index 0000000..b0c3654 --- /dev/null +++ b/v2/pkg/runner/config_test.go @@ -0,0 +1,21 @@ +package runner + +import ( + "github.com/stretchr/testify/require" + "os" + "testing" +) + +func TestConfigGetDirectory(t *testing.T) { + directory, err := GetConfigDirectory() + if err != nil { + t.Fatalf("Expected nil got %v while getting home\n", err) + } + home, err := os.UserHomeDir() + if err != nil { + t.Fatalf("Expected nil got %v while getting dir\n", err) + } + config := home + "/.config/subfinder" + + require.Equal(t, directory, config, "Directory and config should be equal") +} diff --git a/v2/pkg/runner/doc.go b/v2/pkg/runner/doc.go new file mode 100644 index 0000000..744872f --- /dev/null +++ b/v2/pkg/runner/doc.go @@ -0,0 +1,3 @@ +// Package runner implements the mechanism to drive the +// subdomain enumeration process +package runner diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go new file mode 100644 index 0000000..9c15c70 --- /dev/null +++ b/v2/pkg/runner/enumerate.go @@ -0,0 +1,171 @@ +package runner + +import ( + "io" + "strings" + "sync" + "time" + + "github.com/hako/durafmt" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/subfinder/v2/pkg/resolve" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +const maxNumCount = 2 + +// EnumerateSingleDomain performs subdomain enumeration against a single domain +func (r *Runner) EnumerateSingleDomain(domain string, writers []io.Writer) error { + gologger.Info().Msgf("Enumerating subdomains for '%s'\n", domain) + + // Check if the user has asked to remove wildcards explicitly. + // If yes, create the resolution pool and get the wildcards for the current domain + var resolutionPool *resolve.ResolutionPool + if r.options.RemoveWildcard { + resolutionPool = r.resolverClient.NewResolutionPool(r.options.Threads, r.options.RemoveWildcard) + err := resolutionPool.InitWildcards(domain) + if err != nil { + // Log the error but don't quit. + gologger.Warning().Msgf("Could not get wildcards for domain '%s': %s\n", domain, err) + } + } + + // Run the passive subdomain enumeration + now := time.Now() + passiveResults := r.passiveAgent.EnumerateSubdomains(domain, r.options.Proxy, r.options.RateLimit, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + + wg := &sync.WaitGroup{} + wg.Add(1) + // Create a unique map for filtering duplicate subdomains out + uniqueMap := make(map[string]resolve.HostEntry) + // Create a map to track sources for each host + sourceMap := make(map[string]map[string]struct{}) + // Process the results in a separate goroutine + go func() { + for result := range passiveResults { + switch result.Type { + case subscraping.Error: + gologger.Warning().Msgf("Could not run source '%s': %s\n", result.Source, result.Error) + case subscraping.Subdomain: + // Validate the subdomain found and remove wildcards from + if !strings.HasSuffix(result.Value, "."+domain) { + continue + } + subdomain := strings.ReplaceAll(strings.ToLower(result.Value), "*.", "") + + if matchSubdomain := r.filterAndMatchSubdomain(subdomain); matchSubdomain { + if _, ok := uniqueMap[subdomain]; !ok { + sourceMap[subdomain] = make(map[string]struct{}) + } + + // Log the verbose message about the found subdomain per source + if _, ok := sourceMap[subdomain][result.Source]; !ok { + gologger.Verbose().Label(result.Source).Msg(subdomain) + } + + sourceMap[subdomain][result.Source] = struct{}{} + + // Check if the subdomain is a duplicate. If not, + // send the subdomain for resolution. + if _, ok := uniqueMap[subdomain]; ok { + continue + } + + hostEntry := resolve.HostEntry{Host: subdomain, Source: result.Source} + + uniqueMap[subdomain] = hostEntry + // If the user asked to remove wildcard then send on the resolve + // queue. Otherwise, if mode is not verbose print the results on + // the screen as they are discovered. + if r.options.RemoveWildcard { + resolutionPool.Tasks <- hostEntry + } + } + } + } + // Close the task channel only if wildcards are asked to be removed + if r.options.RemoveWildcard { + close(resolutionPool.Tasks) + } + wg.Done() + }() + + // If the user asked to remove wildcards, listen from the results + // queue and write to the map. At the end, print the found results to the screen + foundResults := make(map[string]resolve.Result) + if r.options.RemoveWildcard { + // Process the results coming from the resolutions pool + for result := range resolutionPool.Results { + switch result.Type { + case resolve.Error: + gologger.Warning().Msgf("Could not resolve host: '%s'\n", result.Error) + case resolve.Subdomain: + // Add the found subdomain to a map. + if _, ok := foundResults[result.Host]; !ok { + foundResults[result.Host] = result + } + } + } + } + wg.Wait() + outputWriter := NewOutputWriter(r.options.JSON) + // Now output all results in output writers + var err error + for _, writer := range writers { + if r.options.HostIP { + err = outputWriter.WriteHostIP(domain, foundResults, writer) + } else { + if r.options.RemoveWildcard { + err = outputWriter.WriteHostNoWildcard(domain, foundResults, writer) + } else { + if r.options.CaptureSources { + err = outputWriter.WriteSourceHost(domain, sourceMap, writer) + } else { + err = outputWriter.WriteHost(domain, uniqueMap, writer) + } + } + } + if err != nil { + gologger.Error().Msgf("Could not write results for '%s': %s\n", domain, err) + return err + } + } + + // Show found subdomain count in any case. + duration := durafmt.Parse(time.Since(now)).LimitFirstN(maxNumCount).String() + var numberOfSubDomains int + if r.options.RemoveWildcard { + numberOfSubDomains = len(foundResults) + } else { + numberOfSubDomains = len(uniqueMap) + } + + if r.options.ResultCallback != nil { + for _, v := range uniqueMap { + r.options.ResultCallback(&v) + } + } + gologger.Info().Msgf("Found %d subdomains for '%s' in %s\n", numberOfSubDomains, domain, duration) + + return nil +} + +func (r *Runner) filterAndMatchSubdomain(subdomain string) bool { + if r.options.filterRegexes != nil { + for _, filter := range r.options.filterRegexes { + if m := filter.MatchString(subdomain); m { + return false + } + } + } + if r.options.matchRegexes != nil { + for _, match := range r.options.matchRegexes { + if m := match.MatchString(subdomain); m { + return true + } + } + return false + } + return true +} diff --git a/v2/pkg/runner/enumerate_test.go b/v2/pkg/runner/enumerate_test.go new file mode 100644 index 0000000..20d3929 --- /dev/null +++ b/v2/pkg/runner/enumerate_test.go @@ -0,0 +1,145 @@ +package runner + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFilterAndMatchSubdomain(t *testing.T) { + options := &Options{} + options.Domain = []string{"example.com"} + options.Threads = 10 + options.Timeout = 10 + options.Output = os.Stdout + t.Run("Literal Match", func(t *testing.T) { + options.Match = []string{"req.example.com"} + err := options.validateOptions() + if err != nil { + t.Fatalf("Expected nil got %v while validation\n", err) + } + runner, err := NewRunner(options) + if err != nil { + t.Fatalf("Expected nil got %v while creating runner\n", err) + } + match := runner.filterAndMatchSubdomain("req.example.com") + require.True(t, match, "Expecting a boolean True value ") + }) + t.Run("Multiple Wildcards Match", func(t *testing.T) { + options.Match = []string{"*.ns.*.com"} + err := options.validateOptions() + if err != nil { + t.Fatalf("Expected nil got %v while validation\n", err) + } + runner, err := NewRunner(options) + if err != nil { + t.Fatalf("Expected nil got %v while creating runner\n", err) + } + subdomain := []string{"a.ns.example.com", "b.ns.hackerone.com"} + for _, sub := range subdomain { + match := runner.filterAndMatchSubdomain(sub) + require.True(t, match, "Expecting a boolean True value ") + } + }) + t.Run("Sequential Match", func(t *testing.T) { + options.Match = []string{"*.ns.example.com", "*.hackerone.com"} + err := options.validateOptions() + if err != nil { + t.Fatalf("Expected nil got %v while validation\n", err) + } + runner, err := NewRunner(options) + if err != nil { + t.Fatalf("Expected nil got %v while creating runner\n", err) + } + subdomain := []string{"a.ns.example.com", "b.hackerone.com"} + for _, sub := range subdomain { + match := runner.filterAndMatchSubdomain(sub) + require.True(t, match, "Expecting a boolean True value ") + } + }) + t.Run("Literal Filter", func(t *testing.T) { + options.Filter = []string{"req.example.com"} + err := options.validateOptions() + if err != nil { + t.Fatalf("Expected nil got %v while validation\n", err) + } + runner, err := NewRunner(options) + if err != nil { + t.Fatalf("Expected nil got %v while creating runner\n", err) + } + match := runner.filterAndMatchSubdomain("req.example.com") + require.False(t, match, "Expecting a boolean False value ") + }) + t.Run("Multiple Wildcards Filter", func(t *testing.T) { + options.Filter = []string{"*.ns.*.com"} + err := options.validateOptions() + if err != nil { + t.Fatalf("Expected nil got %v while validation\n", err) + } + runner, err := NewRunner(options) + if err != nil { + t.Fatalf("Expected nil got %v while creating runner\n", err) + } + subdomain := []string{"a.ns.example.com", "b.ns.hackerone.com"} + for _, sub := range subdomain { + match := runner.filterAndMatchSubdomain(sub) + require.False(t, match, "Expecting a boolean False value ") + } + }) + t.Run("Sequential Filter", func(t *testing.T) { + options.Filter = []string{"*.ns.example.com", "*.hackerone.com"} + err := options.validateOptions() + if err != nil { + t.Fatalf("Expected nil got %v while validation\n", err) + } + runner, err := NewRunner(options) + if err != nil { + t.Fatalf("Expected nil got %v while creating runner\n", err) + } + subdomain := []string{"a.ns.example.com", "b.hackerone.com"} + for _, sub := range subdomain { + match := runner.filterAndMatchSubdomain(sub) + require.False(t, match, "Expecting a boolean False value ") + } + }) + t.Run("Filter and Match", func(t *testing.T) { + options.Filter = []string{"example.com"} + options.Match = []string{"hackerone.com"} + err := options.validateOptions() + if err != nil { + t.Fatalf("Expected nil got %v while validation\n", err) + } + runner, err := NewRunner(options) + if err != nil { + t.Fatalf("Expected nil got %v while creating runner\n", err) + } + subdomain := []string{"example.com", "example.com"} + for _, sub := range subdomain { + match := runner.filterAndMatchSubdomain(sub) + require.False(t, match, "Expecting a boolean False value ") + } + }) + + t.Run("Filter and Match - Same Root Domain", func(t *testing.T) { + options.Filter = []string{"example.com"} + options.Match = []string{"www.example.com"} + err := options.validateOptions() + if err != nil { + t.Fatalf("Expected nil got %v while validation\n", err) + } + runner, err := NewRunner(options) + if err != nil { + t.Fatalf("Expected nil got %v while creating runner\n", err) + } + subdomain := map[string]string{"filter": "example.com", "match": "www.example.com"} + for key, sub := range subdomain { + result := runner.filterAndMatchSubdomain(sub) + if key == "filter" { + require.False(t, result, "Expecting a boolean False value ") + } else { + require.True(t, result, "Expecting a boolean True value ") + } + } + }) +} diff --git a/v2/pkg/runner/initialize.go b/v2/pkg/runner/initialize.go new file mode 100644 index 0000000..afc65b1 --- /dev/null +++ b/v2/pkg/runner/initialize.go @@ -0,0 +1,51 @@ +package runner + +import ( + "net" + "strings" + + "github.com/projectdiscovery/dnsx/libs/dnsx" + "github.com/projectdiscovery/subfinder/v2/pkg/passive" + "github.com/projectdiscovery/subfinder/v2/pkg/resolve" +) + +// initializePassiveEngine creates the passive engine and loads sources etc +func (r *Runner) initializePassiveEngine() { + r.passiveAgent = passive.New(r.options.Sources, r.options.ExcludeSources, r.options.All, r.options.OnlyRecursive) +} + +// initializeResolver creates the resolver used to resolve the found subdomains +func (r *Runner) initializeResolver() error { + var resolvers []string + + // If the file has been provided, read resolvers from the file + if r.options.ResolverList != "" { + var err error + resolvers, err = loadFromFile(r.options.ResolverList) + if err != nil { + return err + } + } + + if len(r.options.Resolvers) > 0 { + resolvers = append(resolvers, r.options.Resolvers...) + } else { + resolvers = append(resolvers, resolve.DefaultResolvers...) + } + + // Add default 53 UDP port if missing + for i, resolver := range resolvers { + if !strings.Contains(resolver, ":") { + resolvers[i] = net.JoinHostPort(resolver, "53") + } + } + + r.resolverClient = resolve.New() + var err error + r.resolverClient.DNSClient, err = dnsx.New(dnsx.Options{BaseResolvers: resolvers, MaxRetries: 5}) + if err != nil { + return nil + } + + return nil +} diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go new file mode 100644 index 0000000..e38df41 --- /dev/null +++ b/v2/pkg/runner/options.go @@ -0,0 +1,306 @@ +package runner + +import ( + "errors" + "fmt" + "io" + "math/rand" + "os" + "os/user" + "path/filepath" + "regexp" + "strings" + "time" + + "gopkg.in/yaml.v3" + + "github.com/projectdiscovery/fileutil" + "github.com/projectdiscovery/goflags" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/subfinder/v2/pkg/passive" + "github.com/projectdiscovery/subfinder/v2/pkg/resolve" +) + +var ( + defaultConfigLocation = filepath.Join(userHomeDir(), ".config/subfinder/config.yaml") + defaultProviderConfigLocation = filepath.Join(userHomeDir(), ".config/subfinder/provider-config.yaml") +) + +// Options contains the configuration options for tuning +// the subdomain enumeration process. +type Options struct { + Verbose bool // Verbose flag indicates whether to show verbose output or not + NoColor bool // NoColor disables the colored output + JSON bool // JSON specifies whether to use json for output format or text file + HostIP bool // HostIP specifies whether to write subdomains in host:ip format + Silent bool // Silent suppresses any extra text and only writes subdomains to screen + ListSources bool // ListSources specifies whether to list all available sources + RemoveWildcard bool // RemoveWildcard specifies whether to remove potential wildcard or dead subdomains from the results. + CaptureSources bool // CaptureSources specifies whether to save all sources that returned a specific domains or just the first source + Stdin bool // Stdin specifies whether stdin input was given to the process + Version bool // Version specifies if we should just show version and exit + OnlyRecursive bool // Recursive specifies whether to use only recursive subdomain enumeration sources + All bool // All specifies whether to use all (slow) sources. + Threads int // Threads controls the number of threads to use for active enumerations + Timeout int // Timeout is the seconds to wait for sources to respond + MaxEnumerationTime int // MaxEnumerationTime is the maximum amount of time in minutes to wait for enumeration + Domain goflags.StringSlice // Domain is the domain to find subdomains for + DomainsFile string // DomainsFile is the file containing list of domains to find subdomains for + Output io.Writer + OutputFile string // Output is the file to write found subdomains to. + OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given + Sources goflags.StringSlice `yaml:"sources,omitempty"` // Sources contains a comma-separated list of sources to use for enumeration + ExcludeSources goflags.StringSlice `yaml:"exclude-sources,omitempty"` // ExcludeSources contains the comma-separated sources to not include in the enumeration process + Resolvers goflags.StringSlice `yaml:"resolvers,omitempty"` // Resolvers is the comma-separated resolvers to use for enumeration + ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration + Config string // Config contains the location of the config file + ProviderConfig string // ProviderConfig contains the location of the provider config file + Proxy string // HTTP proxy + RateLimit int // Maximum number of HTTP requests to send per second + ExcludeIps bool + Match goflags.StringSlice + Filter goflags.StringSlice + matchRegexes []*regexp.Regexp + filterRegexes []*regexp.Regexp + ResultCallback OnResultCallback // OnResult callback +} + +// OnResultCallback (hostResult) +type OnResultCallback func(result *resolve.HostEntry) + +// ParseOptions parses the command line flags provided by a user +func ParseOptions() *Options { + // Seed default random number generator + rand.Seed(time.Now().UnixNano()) + + // Migrate config to provider config + if fileutil.FileExists(defaultConfigLocation) && !fileutil.FileExists(defaultProviderConfigLocation) { + gologger.Info().Msgf("Detected old '%s' config file, trying to migrate providers to '%s'\n", defaultConfigLocation, defaultProviderConfigLocation) + if err := migrateToProviderConfig(defaultConfigLocation, defaultProviderConfigLocation); err != nil { + gologger.Warning().Msgf("Could not migrate providers from existing config '%s' to provider config '%s': %s\n", defaultConfigLocation, defaultProviderConfigLocation, err) + } else { + // cleanup the existing config file post migration + _ = os.Remove(defaultConfigLocation) + gologger.Info().Msgf("Migration successful from '%s' to '%s'.\n", defaultConfigLocation, defaultProviderConfigLocation) + } + } + + options := &Options{} + + var err error + flagSet := goflags.NewFlagSet() + flagSet.SetDescription(`Subfinder is a subdomain discovery tool that discovers subdomains for websites by using passive online sources.`) + + createGroup(flagSet, "input", "Input", + flagSet.StringSliceVarP(&options.Domain, "domain", "d", []string{}, "domains to find subdomains for", goflags.NormalizedStringSliceOptions), + flagSet.StringVarP(&options.DomainsFile, "list", "dL", "", "file containing list of domains for subdomain discovery"), + ) + + createGroup(flagSet, "source", "Source", + flagSet.StringSliceVarP(&options.Sources, "sources", "s", []string{}, "specific sources to use for discovery (-s crtsh,github). Use -ls to display all available sources.", goflags.NormalizedStringSliceOptions), + flagSet.BoolVar(&options.OnlyRecursive, "recursive", false, "use only sources that can handle subdomains recursively (e.g. subdomain.domain.tld vs domain.tld)"), + flagSet.BoolVar(&options.All, "all", false, "use all sources for enumeration (slow)"), + flagSet.StringSliceVarP(&options.ExcludeSources, "exclude-sources", "es", []string{}, "sources to exclude from enumeration (-es alienvault,zoomeye)", goflags.NormalizedStringSliceOptions), + ) + + createGroup(flagSet, "filter", "Filter", + flagSet.StringSliceVarP(&options.Match, "match", "m", []string{}, "subdomain or list of subdomain to match (file or comma separated)", goflags.FileNormalizedStringSliceOptions), + flagSet.StringSliceVarP(&options.Filter, "filter", "f", []string{}, " subdomain or list of subdomain to filter (file or comma separated)", goflags.FileNormalizedStringSliceOptions), + ) + + createGroup(flagSet, "rate-limit", "Rate-limit", + flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 0, "maximum number of http requests to send per second"), + flagSet.IntVar(&options.Threads, "t", 10, "number of concurrent goroutines for resolving (-active only)"), + ) + + createGroup(flagSet, "output", "Output", + flagSet.StringVarP(&options.OutputFile, "output", "o", "", "file to write output to"), + flagSet.BoolVarP(&options.JSON, "json", "oJ", false, "write output in JSONL(ines) format"), + flagSet.StringVarP(&options.OutputDirectory, "output-dir", "oD", "", "directory to write output (-dL only)"), + flagSet.BoolVarP(&options.CaptureSources, "collect-sources", "cs", false, "include all sources in the output (-json only)"), + flagSet.BoolVarP(&options.HostIP, "ip", "oI", false, "include host IP in output (-active only)"), + ) + + createGroup(flagSet, "configuration", "Configuration", + flagSet.StringVar(&options.Config, "config", defaultConfigLocation, "flag config file"), + flagSet.StringVarP(&options.ProviderConfig, "provider-config", "pc", defaultProviderConfigLocation, "provider config file"), + flagSet.StringSliceVar(&options.Resolvers, "r", []string{}, "comma separated list of resolvers to use", goflags.NormalizedStringSliceOptions), + flagSet.StringVarP(&options.ResolverList, "rlist", "rL", "", "file containing list of resolvers to use"), + flagSet.BoolVarP(&options.RemoveWildcard, "active", "nW", false, "display active subdomains only"), + flagSet.StringVar(&options.Proxy, "proxy", "", "http proxy to use with subfinder"), + flagSet.BoolVarP(&options.ExcludeIps, "exclude-ip", "ei", false, "exclude IPs from the list of domains"), + ) + + createGroup(flagSet, "debug", "Debug", + flagSet.BoolVar(&options.Silent, "silent", false, "show only subdomains in output"), + flagSet.BoolVar(&options.Version, "version", false, "show version of subfinder"), + flagSet.BoolVar(&options.Verbose, "v", false, "show verbose output"), + flagSet.BoolVarP(&options.NoColor, "no-color", "nc", false, "disable color in output"), + flagSet.BoolVarP(&options.ListSources, "list-sources", "ls", false, "list all available sources"), + ) + + createGroup(flagSet, "optimization", "Optimization", + flagSet.IntVar(&options.Timeout, "timeout", 30, "seconds to wait before timing out"), + flagSet.IntVar(&options.MaxEnumerationTime, "max-time", 10, "minutes to wait for enumeration results"), + ) + + if err := flagSet.Parse(); err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + if options.Config != defaultConfigLocation { + // An empty source file is not a fatal error + if err := flagSet.MergeConfigFile(options.Config); err != nil && !errors.Is(err, io.EOF) { + gologger.Fatal().Msgf("Could not read config: %s\n", err) + } + } + + // Default output is stdout + options.Output = os.Stdout + + // Check if stdin pipe was given + options.Stdin = hasStdin() + + // Read the inputs and configure the logging + options.configureOutput() + + if options.Version { + gologger.Info().Msgf("Current Version: %s\n", Version) + os.Exit(0) + } + + options.preProcessOptions() + + if !options.Silent { + showBanner() + } + + // Check if the application loading with any provider configuration, then take it + // Otherwise load the default provider config + if fileutil.FileExists(options.ProviderConfig) { + gologger.Info().Msgf("Loading provider config from '%s'", options.ProviderConfig) + options.loadProvidersFrom(options.ProviderConfig) + } else { + gologger.Info().Msgf("Loading provider config from the default location: '%s'", defaultProviderConfigLocation) + options.loadProvidersFrom(defaultProviderConfigLocation) + } + if options.ListSources { + listSources(options) + os.Exit(0) + } + + // Validate the options passed by the user and if any + // invalid options have been used, exit. + err = options.validateOptions() + if err != nil { + gologger.Fatal().Msgf("Program exiting: %s\n", err) + } + + return options +} + +// loadProvidersFrom runs the app with source config +func (options *Options) loadProvidersFrom(location string) { + // todo: move elsewhere + if len(options.Resolvers) == 0 { + options.Resolvers = resolve.DefaultResolvers + } + + // We skip bailing out if file doesn't exist because we'll create it + // at the end of options parsing from default via goflags. + if err := UnmarshalFrom(location); isFatalErr(err) && !errors.Is(err, os.ErrNotExist) { + gologger.Fatal().Msgf("Could not read providers from '%s': %s\n", location, err) + } +} + +func migrateToProviderConfig(defaultConfigLocation, defaultProviderLocation string) error { + configs, err := unMarshalToLowerCaseMap(defaultConfigLocation) + if err != nil { + return err + } + + sourcesRequiringApiKeysMap := make(map[string][]string) + for _, source := range passive.AllSources { + if source.NeedsKey() { + sourceName := strings.ToLower(source.Name()) + if sourceKeys, ok := configs[sourceName]; ok { + sourcesRequiringApiKeysMap[sourceName] = sourceKeys + } else { + sourcesRequiringApiKeysMap[sourceName] = []string{} + } + } + } + + return CreateProviderConfigYAML(defaultProviderLocation, sourcesRequiringApiKeysMap) +} + +func unMarshalToLowerCaseMap(defaultConfigLocation string) (map[string][]string, error) { + defaultConfigFile, err := os.Open(defaultConfigLocation) + if err != nil { + return nil, err + } + defer defaultConfigFile.Close() + + configs := map[string][]string{} + if err := yaml.NewDecoder(defaultConfigFile).Decode(configs); isFatalErr(err) { + return nil, err + } + + for k, v := range configs { + configs[strings.ToLower(k)] = v + } + return configs, nil +} + +func isFatalErr(err error) bool { + return err != nil && !errors.Is(err, io.EOF) +} + +func hasStdin() bool { + stat, err := os.Stdin.Stat() + if err != nil { + return false + } + + isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0 + isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0 + + return isPipedFromChrDev || isPipedFromFIFO +} + +func listSources(options *Options) { + gologger.Info().Msgf("Current list of available sources. [%d]\n", len(passive.AllSources)) + gologger.Info().Msgf("Sources marked with an * need key(s) or token(s) to work.\n") + gologger.Info().Msgf("You can modify '%s' to configure your keys/tokens.\n\n", options.ProviderConfig) + + for _, source := range passive.AllSources { + message := "%s\n" + sourceName := source.Name() + if source.NeedsKey() { + message = "%s *\n" + } + gologger.Silent().Msgf(message, sourceName) + } +} + +func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) { + flagSet.SetGroup(groupName, description) + for _, currentFlag := range flags { + currentFlag.Group(groupName) + } +} + +func (options *Options) preProcessOptions() { + for i, domain := range options.Domain { + options.Domain[i], _ = sanitize(domain) + } +} + +func userHomeDir() string { + usr, err := user.Current() + if err != nil { + gologger.Fatal().Msgf("Could not get user home directory: %s\n", err) + } + return usr.HomeDir +} diff --git a/v2/pkg/runner/outputter.go b/v2/pkg/runner/outputter.go new file mode 100644 index 0000000..25c1d1a --- /dev/null +++ b/v2/pkg/runner/outputter.go @@ -0,0 +1,237 @@ +package runner + +import ( + "bufio" + "errors" + "io" + "os" + "path/filepath" + "strings" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/resolve" +) + +// OutputWriter outputs content to writers. +type OutputWriter struct { + JSON bool +} + +type jsonSourceResult struct { + Host string `json:"host"` + Input string `json:"input"` + Source string `json:"source"` +} + +type jsonSourceIPResult struct { + Host string `json:"host"` + IP string `json:"ip"` + Input string `json:"input"` + Source string `json:"source"` +} + +type jsonSourcesResult struct { + Host string `json:"host"` + Input string `json:"input"` + Sources []string `json:"sources"` +} + +// NewOutputWriter creates a new OutputWriter +func NewOutputWriter(json bool) *OutputWriter { + return &OutputWriter{JSON: json} +} + +func (o *OutputWriter) createFile(filename string, appendToFile bool) (*os.File, error) { + if filename == "" { + return nil, errors.New("empty filename") + } + + dir := filepath.Dir(filename) + + if dir != "" { + if _, err := os.Stat(dir); os.IsNotExist(err) { + err := os.MkdirAll(dir, os.ModePerm) + if err != nil { + return nil, err + } + } + } + + var file *os.File + var err error + if appendToFile { + file, err = os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + } else { + file, err = os.Create(filename) + } + if err != nil { + return nil, err + } + + return file, nil +} + +// WriteHostIP writes the output list of subdomain to an io.Writer +func (o *OutputWriter) WriteHostIP(input string, results map[string]resolve.Result, writer io.Writer) error { + var err error + if o.JSON { + err = writeJSONHostIP(input, results, writer) + } else { + err = writePlainHostIP(input, results, writer) + } + return err +} + +func writePlainHostIP(_ string, results map[string]resolve.Result, writer io.Writer) error { + bufwriter := bufio.NewWriter(writer) + sb := &strings.Builder{} + + for _, result := range results { + sb.WriteString(result.Host) + sb.WriteString(",") + sb.WriteString(result.IP) + sb.WriteString(",") + sb.WriteString(result.Source) + sb.WriteString("\n") + + _, err := bufwriter.WriteString(sb.String()) + if err != nil { + bufwriter.Flush() + return err + } + sb.Reset() + } + return bufwriter.Flush() +} + +func writeJSONHostIP(input string, results map[string]resolve.Result, writer io.Writer) error { + encoder := jsoniter.NewEncoder(writer) + + var data jsonSourceIPResult + + for _, result := range results { + data.Host = result.Host + data.IP = result.IP + data.Input = input + data.Source = result.Source + + err := encoder.Encode(&data) + if err != nil { + return err + } + } + return nil +} + +// WriteHostNoWildcard writes the output list of subdomain with nW flag to an io.Writer +func (o *OutputWriter) WriteHostNoWildcard(input string, results map[string]resolve.Result, writer io.Writer) error { + hosts := make(map[string]resolve.HostEntry) + for host, result := range results { + hosts[host] = resolve.HostEntry{Host: result.Host, Source: result.Source} + } + + return o.WriteHost(input, hosts, writer) +} + +// WriteHost writes the output list of subdomain to an io.Writer +func (o *OutputWriter) WriteHost(input string, results map[string]resolve.HostEntry, writer io.Writer) error { + var err error + if o.JSON { + err = writeJSONHost(input, results, writer) + } else { + err = writePlainHost(input, results, writer) + } + return err +} + +func writePlainHost(_ string, results map[string]resolve.HostEntry, writer io.Writer) error { + bufwriter := bufio.NewWriter(writer) + sb := &strings.Builder{} + + for _, result := range results { + sb.WriteString(result.Host) + sb.WriteString("\n") + + _, err := bufwriter.WriteString(sb.String()) + if err != nil { + bufwriter.Flush() + return err + } + sb.Reset() + } + return bufwriter.Flush() +} + +func writeJSONHost(input string, results map[string]resolve.HostEntry, writer io.Writer) error { + encoder := jsoniter.NewEncoder(writer) + + var data jsonSourceResult + for _, result := range results { + data.Host = result.Host + data.Input = input + data.Source = result.Source + err := encoder.Encode(data) + if err != nil { + return err + } + } + return nil +} + +// WriteSourceHost writes the output list of subdomain to an io.Writer +func (o *OutputWriter) WriteSourceHost(input string, sourceMap map[string]map[string]struct{}, writer io.Writer) error { + var err error + if o.JSON { + err = writeSourceJSONHost(input, sourceMap, writer) + } else { + err = writeSourcePlainHost(input, sourceMap, writer) + } + return err +} + +func writeSourceJSONHost(input string, sourceMap map[string]map[string]struct{}, writer io.Writer) error { + encoder := jsoniter.NewEncoder(writer) + + var data jsonSourcesResult + + for host, sources := range sourceMap { + data.Host = host + data.Input = input + keys := make([]string, 0, len(sources)) + for source := range sources { + keys = append(keys, source) + } + data.Sources = keys + + err := encoder.Encode(&data) + if err != nil { + return err + } + } + return nil +} + +func writeSourcePlainHost(_ string, sourceMap map[string]map[string]struct{}, writer io.Writer) error { + bufwriter := bufio.NewWriter(writer) + sb := &strings.Builder{} + + for host, sources := range sourceMap { + sb.WriteString(host) + sb.WriteString(",[") + sourcesString := "" + for source := range sources { + sourcesString += source + "," + } + sb.WriteString(strings.Trim(sourcesString, ", ")) + sb.WriteString("]\n") + + _, err := bufwriter.WriteString(sb.String()) + if err != nil { + bufwriter.Flush() + return err + } + sb.Reset() + } + return bufwriter.Flush() +} diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go new file mode 100644 index 0000000..07d5cc8 --- /dev/null +++ b/v2/pkg/runner/runner.go @@ -0,0 +1,124 @@ +package runner + +import ( + "bufio" + "io" + "os" + "path" + "regexp" + "strings" + + "github.com/pkg/errors" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/subfinder/v2/pkg/passive" + "github.com/projectdiscovery/subfinder/v2/pkg/resolve" +) + +// Runner is an instance of the subdomain enumeration +// client used to orchestrate the whole process. +type Runner struct { + options *Options + passiveAgent *passive.Agent + resolverClient *resolve.Resolver +} + +// NewRunner creates a new runner struct instance by parsing +// the configuration options, configuring sources, reading lists +// and setting up loggers, etc. +func NewRunner(options *Options) (*Runner, error) { + runner := &Runner{options: options} + + // Initialize the passive subdomain enumeration engine + runner.initializePassiveEngine() + + // Initialize the subdomain resolver + err := runner.initializeResolver() + if err != nil { + return nil, err + } + + return runner, nil +} + +// RunEnumeration runs the subdomain enumeration flow on the targets specified +func (r *Runner) RunEnumeration() error { + outputs := []io.Writer{r.options.Output} + + if len(r.options.Domain) > 0 { + domainsReader := strings.NewReader(strings.Join(r.options.Domain, "\n")) + return r.EnumerateMultipleDomains(domainsReader, outputs) + } + + // If we have multiple domains as input, + if r.options.DomainsFile != "" { + f, err := os.Open(r.options.DomainsFile) + if err != nil { + return err + } + err = r.EnumerateMultipleDomains(f, outputs) + f.Close() + return err + } + + // If we have STDIN input, treat it as multiple domains + if r.options.Stdin { + return r.EnumerateMultipleDomains(os.Stdin, outputs) + } + return nil +} + +// EnumerateMultipleDomains enumerates subdomains for multiple domains +// We keep enumerating subdomains for a given domain until we reach an error +func (r *Runner) EnumerateMultipleDomains(reader io.Reader, writers []io.Writer) error { + scanner := bufio.NewScanner(reader) + ip, _ := regexp.Compile(`^([0-9\.]+$)`) + for scanner.Scan() { + domain, err := sanitize(scanner.Text()) + isIp := ip.MatchString(domain) + if errors.Is(err, ErrEmptyInput) || (r.options.ExcludeIps && isIp) { + continue + } + + var file *os.File + // If the user has specified an output file, use that output file instead + // of creating a new output file for each domain. Else create a new file + // for each domain in the directory. + if r.options.OutputFile != "" { + outputWriter := NewOutputWriter(r.options.JSON) + file, err = outputWriter.createFile(r.options.OutputFile, true) + if err != nil { + gologger.Error().Msgf("Could not create file %s for %s: %s\n", r.options.OutputFile, r.options.Domain, err) + return err + } + + err = r.EnumerateSingleDomain(domain, append(writers, file)) + + file.Close() + } else if r.options.OutputDirectory != "" { + outputFile := path.Join(r.options.OutputDirectory, domain) + if r.options.JSON { + outputFile += ".json" + } else { + outputFile += ".txt" + } + + outputWriter := NewOutputWriter(r.options.JSON) + file, err = outputWriter.createFile(outputFile, false) + if err != nil { + gologger.Error().Msgf("Could not create file %s for %s: %s\n", r.options.OutputFile, r.options.Domain, err) + return err + } + + err = r.EnumerateSingleDomain(domain, append(writers, file)) + + file.Close() + } else { + err = r.EnumerateSingleDomain(domain, writers) + } + if err != nil { + return err + } + } + return nil +} diff --git a/v2/pkg/runner/util.go b/v2/pkg/runner/util.go new file mode 100644 index 0000000..c9ce3bf --- /dev/null +++ b/v2/pkg/runner/util.go @@ -0,0 +1,38 @@ +package runner + +import ( + "strings" + + "github.com/pkg/errors" + + "github.com/projectdiscovery/fileutil" +) + +var ( + ErrEmptyInput = errors.New("empty data") +) + +func loadFromFile(file string) ([]string, error) { + chanItems, err := fileutil.ReadFile(file) + if err != nil { + return nil, err + } + var items []string + for item := range chanItems { + var err error + item, err = sanitize(item) + if errors.Is(err, ErrEmptyInput) { + continue + } + items = append(items, item) + } + return items, nil +} + +func sanitize(data string) (string, error) { + data = strings.Trim(data, "\n\t\"' ") + if data == "" { + return "", ErrEmptyInput + } + return data, nil +} diff --git a/v2/pkg/runner/validate.go b/v2/pkg/runner/validate.go new file mode 100644 index 0000000..e8010cc --- /dev/null +++ b/v2/pkg/runner/validate.go @@ -0,0 +1,78 @@ +package runner + +import ( + "errors" + "fmt" + "regexp" + "strings" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/formatter" + "github.com/projectdiscovery/gologger/levels" +) + +// validateOptions validates the configuration options passed +func (options *Options) validateOptions() error { + // Check if domain, list of domains, or stdin info was provided. + // If none was provided, then return. + if len(options.Domain) == 0 && options.DomainsFile == "" && !options.Stdin { + return errors.New("no input list provided") + } + + // Both verbose and silent flags were used + if options.Verbose && options.Silent { + return errors.New("both verbose and silent mode specified") + } + + // Validate threads and options + if options.Threads == 0 { + return errors.New("threads cannot be zero") + } + if options.Timeout == 0 { + return errors.New("timeout cannot be zero") + } + + // Always remove wildcard with hostip + if options.HostIP && !options.RemoveWildcard { + return errors.New("hostip flag must be used with RemoveWildcard option") + } + + if options.Match != nil { + options.matchRegexes = make([]*regexp.Regexp, len(options.Match)) + var err error + for i, re := range options.Match { + if options.matchRegexes[i], err = regexp.Compile(stripRegexString(re)); err != nil { + return errors.New("invalid value for match regex option") + } + } + } + if options.Filter != nil { + options.filterRegexes = make([]*regexp.Regexp, len(options.Filter)) + var err error + for i, re := range options.Filter { + if options.filterRegexes[i], err = regexp.Compile(stripRegexString(re)); err != nil { + return errors.New("invalid value for filter regex option") + } + } + } + return nil +} +func stripRegexString(val string) string { + val = strings.ReplaceAll(val, ".", "\\.") + val = strings.ReplaceAll(val, "*", ".*") + return fmt.Sprint("^", val, "$") +} + +// configureOutput configures the output on the screen +func (options *Options) configureOutput() { + // If the user desires verbose output, show verbose output + if options.Verbose { + gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) + } + if options.NoColor { + gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true)) + } + if options.Silent { + gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) + } +} diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go new file mode 100644 index 0000000..914942b --- /dev/null +++ b/v2/pkg/subscraping/agent.go @@ -0,0 +1,139 @@ +package subscraping + +import ( + "bytes" + "context" + "crypto/tls" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "github.com/corpix/uarand" + "github.com/projectdiscovery/ratelimit" + + "github.com/projectdiscovery/gologger" +) + +// NewSession creates a new session object for a domain +func NewSession(domain string, proxy string, rateLimit, timeout int) (*Session, error) { + Transport := &http.Transport{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + + // Add proxy + if proxy != "" { + proxyURL, _ := url.Parse(proxy) + if proxyURL == nil { + // Log warning but continue anyway + gologger.Warning().Msgf("Invalid proxy provided: '%s'", proxy) + } else { + Transport.Proxy = http.ProxyURL(proxyURL) + } + } + + client := &http.Client{ + Transport: Transport, + Timeout: time.Duration(timeout) * time.Second, + } + + session := &Session{Client: client} + + // Initiate rate limit instance + if rateLimit > 0 { + session.RateLimiter = ratelimit.New(context.Background(), int64(rateLimit), time.Second) + } else { + session.RateLimiter = ratelimit.NewUnlimited(context.Background()) + } + + // Create a new extractor object for the current domain + extractor, err := NewSubdomainExtractor(domain) + session.Extractor = extractor + + return session, err +} + +// Get makes a GET request to a URL with extended parameters +func (s *Session) Get(ctx context.Context, getURL, cookies string, headers map[string]string) (*http.Response, error) { + return s.HTTPRequest(ctx, http.MethodGet, getURL, cookies, headers, nil, BasicAuth{}) +} + +// SimpleGet makes a simple GET request to a URL +func (s *Session) SimpleGet(ctx context.Context, getURL string) (*http.Response, error) { + return s.HTTPRequest(ctx, http.MethodGet, getURL, "", map[string]string{}, nil, BasicAuth{}) +} + +// Post makes a POST request to a URL with extended parameters +func (s *Session) Post(ctx context.Context, postURL, cookies string, headers map[string]string, body io.Reader) (*http.Response, error) { + return s.HTTPRequest(ctx, http.MethodPost, postURL, cookies, headers, body, BasicAuth{}) +} + +// SimplePost makes a simple POST request to a URL +func (s *Session) SimplePost(ctx context.Context, postURL, contentType string, body io.Reader) (*http.Response, error) { + return s.HTTPRequest(ctx, http.MethodPost, postURL, "", map[string]string{"Content-Type": contentType}, body, BasicAuth{}) +} + +// HTTPRequest makes any HTTP request to a URL with extended parameters +func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies string, headers map[string]string, body io.Reader, basicAuth BasicAuth) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, method, requestURL, body) + if err != nil { + return nil, err + } + + req.Header.Set("User-Agent", uarand.GetRandom()) + req.Header.Set("Accept", "*/*") + req.Header.Set("Accept-Language", "en") + req.Header.Set("Connection", "close") + + if basicAuth.Username != "" || basicAuth.Password != "" { + req.SetBasicAuth(basicAuth.Username, basicAuth.Password) + } + + if cookies != "" { + req.Header.Set("Cookie", cookies) + } + + for key, value := range headers { + req.Header.Set(key, value) + } + + s.RateLimiter.Take() + + return httpRequestWrapper(s.Client, req) +} + +// DiscardHTTPResponse discards the response content by demand +func (s *Session) DiscardHTTPResponse(response *http.Response) { + if response != nil { + _, err := io.Copy(io.Discard, response.Body) + if err != nil { + gologger.Warning().Msgf("Could not discard response body: %s\n", err) + return + } + response.Body.Close() + } +} + +func httpRequestWrapper(client *http.Client, request *http.Request) (*http.Response, error) { + response, err := client.Do(request) + if err != nil { + return nil, err + } + + if response.StatusCode != http.StatusOK { + requestURL, _ := url.QueryUnescape(request.URL.String()) + + gologger.Debug().MsgFunc(func() string { + buffer := new(bytes.Buffer) + _, _ = buffer.ReadFrom(response.Body) + return fmt.Sprintf("Response for failed request against '%s':\n%s", requestURL, buffer.String()) + }) + return response, fmt.Errorf("unexpected status code %d received from '%s'", response.StatusCode, requestURL) + } + return response, nil +} diff --git a/v2/pkg/subscraping/doc.go b/v2/pkg/subscraping/doc.go new file mode 100644 index 0000000..3f8e54c --- /dev/null +++ b/v2/pkg/subscraping/doc.go @@ -0,0 +1,2 @@ +// Package subscraping contains the logic of scraping agents +package subscraping diff --git a/v2/pkg/subscraping/sources/alienvault/alienvault.go b/v2/pkg/subscraping/sources/alienvault/alienvault.go new file mode 100644 index 0000000..8bde7f0 --- /dev/null +++ b/v2/pkg/subscraping/sources/alienvault/alienvault.go @@ -0,0 +1,79 @@ +// Package alienvault logic +package alienvault + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type alienvaultResponse struct { + Detail string `json:"detail"` + Error string `json:"error"` + PassiveDNS []struct { + Hostname string `json:"hostname"` + } `json:"passive_dns"` +} + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain)) + if err != nil && resp == nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var response alienvaultResponse + // Get the response body and decode + err = json.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + if response.Error != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s, %s", response.Detail, response.Error)} + return + } + + for _, record := range response.PassiveDNS { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "alienvault" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) NeedsKey() bool { + return false +} + +func (s *Source) AddApiKeys(_ []string) { + // no key needed +} diff --git a/v2/pkg/subscraping/sources/anubis/anubis.go b/v2/pkg/subscraping/sources/anubis/anubis.go new file mode 100644 index 0000000..f315db0 --- /dev/null +++ b/v2/pkg/subscraping/sources/anubis/anubis.go @@ -0,0 +1,67 @@ +// Package anubis logic +package anubis + +import ( + "context" + "fmt" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://jonlu.ca/anubis/subdomains/%s", domain)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var subdomains []string + err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + + resp.Body.Close() + + for _, record := range subdomains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record} + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "anubis" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return false +} + +func (s *Source) AddApiKeys(_ []string) { + // no key needed +} diff --git a/v2/pkg/subscraping/sources/bevigil/bevigil.go b/v2/pkg/subscraping/sources/bevigil/bevigil.go new file mode 100644 index 0000000..26ea4ab --- /dev/null +++ b/v2/pkg/subscraping/sources/bevigil/bevigil.go @@ -0,0 +1,82 @@ +// Package bevigil logic +package bevigil + +import ( + "context" + "fmt" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type Response struct { + Domain string `json:"domain"` + Subdomains []string `json:"subdomains"` +} + +type Source struct { + apiKeys []string +} + +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + getUrl := fmt.Sprintf("https://osint.bevigil.com/api/%s/subdomains/", domain) + + resp, err := session.Get(ctx, getUrl, "", map[string]string{"X-Access-Token": randomApiKey, "User-Agent": "subfinder"}) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var subdomains []string + var response Response + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + + resp.Body.Close() + + if len(response.Subdomains) > 0 { + subdomains = response.Subdomains + } + + for _, subdomain := range subdomains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + }() + + return results +} + +func (s *Source) Name() string { + return "bevigil" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/sources/binaryedge/binaryedge.go b/v2/pkg/subscraping/sources/binaryedge/binaryedge.go new file mode 100644 index 0000000..548663b --- /dev/null +++ b/v2/pkg/subscraping/sources/binaryedge/binaryedge.go @@ -0,0 +1,164 @@ +// Package binaryedge logic +package binaryedge + +import ( + "context" + "fmt" + "math" + "net/url" + "strconv" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +const ( + v1 = "v1" + v2 = "v2" + baseAPIURLFmt = "https://api.binaryedge.io/%s/query/domains/subdomain/%s" + v2SubscriptionURL = "https://api.binaryedge.io/v2/user/subscription" + v1PageSizeParam = "pagesize" + pageParam = "page" + firstPage = 1 + maxV1PageSize = 10000 +) + +type subdomainsResponse struct { + Message string `json:"message"` + Title string `json:"title"` + Status interface{} `json:"status"` // string for v1, int for v2 + Subdomains []string `json:"events"` + Page int `json:"page"` + PageSize int `json:"pagesize"` + Total int `json:"total"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + var baseURL string + + authHeader := map[string]string{"X-Key": randomApiKey} + + if isV2(ctx, session, authHeader) { + baseURL = fmt.Sprintf(baseAPIURLFmt, v2, domain) + } else { + authHeader = map[string]string{"X-Token": randomApiKey} + v1URLWithPageSize, err := addURLParam(fmt.Sprintf(baseAPIURLFmt, v1, domain), v1PageSizeParam, strconv.Itoa(maxV1PageSize)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + baseURL = v1URLWithPageSize.String() + } + + if baseURL == "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("can't get API URL")} + return + } + + s.enumerate(ctx, session, baseURL, firstPage, authHeader, results) + }() + + return results +} + +func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, baseURL string, page int, authHeader map[string]string, results chan subscraping.Result) { + pageURL, err := addURLParam(baseURL, pageParam, strconv.Itoa(page)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + resp, err := session.Get(ctx, pageURL.String(), "", authHeader) + if err != nil && resp == nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var response subdomainsResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + + // Check error messages + if response.Message != "" && response.Status != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf(response.Message)} + } + + resp.Body.Close() + + for _, subdomain := range response.Subdomains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + + totalPages := int(math.Ceil(float64(response.Total) / float64(response.PageSize))) + nextPage := response.Page + 1 + for currentPage := nextPage; currentPage <= totalPages; currentPage++ { + s.enumerate(ctx, session, baseURL, currentPage, authHeader, results) + } +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "binaryedge" +} + +func (s *Source) IsDefault() bool { + return false +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func isV2(ctx context.Context, session *subscraping.Session, authHeader map[string]string) bool { + resp, err := session.Get(ctx, v2SubscriptionURL, "", authHeader) + if err != nil { + session.DiscardHTTPResponse(resp) + return false + } + + resp.Body.Close() + + return true +} + +func addURLParam(targetURL, name, value string) (*url.URL, error) { + u, err := url.Parse(targetURL) + if err != nil { + return u, err + } + q, _ := url.ParseQuery(u.RawQuery) + q.Add(name, value) + u.RawQuery = q.Encode() + + return u, nil +} diff --git a/v2/pkg/subscraping/sources/bufferover/bufferover.go b/v2/pkg/subscraping/sources/bufferover/bufferover.go new file mode 100644 index 0000000..d309990 --- /dev/null +++ b/v2/pkg/subscraping/sources/bufferover/bufferover.go @@ -0,0 +1,107 @@ +// Package bufferover is a bufferover Scraping Engine in Golang +package bufferover + +import ( + "context" + "fmt" + "strings" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type response struct { + Meta struct { + Errors []string `json:"Errors"` + } `json:"Meta"` + FDNSA []string `json:"FDNS_A"` + RDNS []string `json:"RDNS"` + Results []string `json:"Results"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + s.getData(ctx, fmt.Sprintf("https://tls.bufferover.run/dns?q=.%s", domain), randomApiKey, session, results) + }() + + return results +} + +func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, session *subscraping.Session, results chan subscraping.Result) { + resp, err := session.Get(ctx, sourceURL, "", map[string]string{"x-api-key": apiKey}) + + if err != nil && resp == nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var bufforesponse response + err = jsoniter.NewDecoder(resp.Body).Decode(&bufforesponse) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + + resp.Body.Close() + + metaErrors := bufforesponse.Meta.Errors + + if len(metaErrors) > 0 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", strings.Join(metaErrors, ", "))} + return + } + + var subdomains []string + + if len(bufforesponse.FDNSA) > 0 { + subdomains = bufforesponse.FDNSA + subdomains = append(subdomains, bufforesponse.RDNS...) + } else if len(bufforesponse.Results) > 0 { + subdomains = bufforesponse.Results + } + + for _, subdomain := range subdomains { + for _, value := range session.Extractor.FindAllString(subdomain, -1) { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} + } + } +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "bufferover" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/sources/c99/c99.go b/v2/pkg/subscraping/sources/c99/c99.go new file mode 100644 index 0000000..f41a483 --- /dev/null +++ b/v2/pkg/subscraping/sources/c99/c99.go @@ -0,0 +1,91 @@ +// Package c99 logic +package c99 + +import ( + "context" + "fmt" + "strings" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +type dnsdbLookupResponse struct { + Success bool `json:"success"` + Subdomains []struct { + Subdomain string `json:"subdomain"` + IP string `json:"ip"` + Cloudflare bool `json:"cloudflare"` + } `json:"subdomains"` + Error string `json:"error"` +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", randomApiKey, domain) + resp, err := session.SimpleGet(ctx, searchURL) + if err != nil { + session.DiscardHTTPResponse(resp) + return + } + + defer resp.Body.Close() + + var response dnsdbLookupResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + if response.Error != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error)} + return + } + + for _, data := range response.Subdomains { + if !strings.HasPrefix(data.Subdomain, ".") { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain} + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "c99" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/sources/censys/censys.go b/v2/pkg/subscraping/sources/censys/censys.go new file mode 100644 index 0000000..c913aaa --- /dev/null +++ b/v2/pkg/subscraping/sources/censys/censys.go @@ -0,0 +1,122 @@ +// Package censys logic +package censys + +import ( + "bytes" + "context" + "strconv" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +const maxCensysPages = 10 + +type resultsq struct { + Data []string `json:"parsed.extensions.subject_alt_name.dns_names"` + Data1 []string `json:"parsed.names"` +} + +type response struct { + Results []resultsq `json:"results"` + Metadata struct { + Pages int `json:"pages"` + } `json:"metadata"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []apiKey +} + +type apiKey struct { + token string + secret string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey.token == "" || randomApiKey.secret == "" { + return + } + + currentPage := 1 + for { + var request = []byte(`{"query":"` + domain + `", "page":` + strconv.Itoa(currentPage) + `, "fields":["parsed.names","parsed.extensions.subject_alt_name.dns_names"], "flatten":true}`) + + resp, err := session.HTTPRequest( + ctx, + "POST", + "https://search.censys.io/api/v1/search/certificates", + "", + map[string]string{"Content-Type": "application/json", "Accept": "application/json"}, + bytes.NewReader(request), + subscraping.BasicAuth{Username: randomApiKey.token, Password: randomApiKey.secret}, + ) + + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var censysResponse response + err = jsoniter.NewDecoder(resp.Body).Decode(&censysResponse) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + + resp.Body.Close() + + for _, res := range censysResponse.Results { + for _, part := range res.Data { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: part} + } + for _, part := range res.Data1 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: part} + } + } + + // Exit the censys enumeration if max pages is reached + if currentPage >= censysResponse.Metadata.Pages || currentPage >= maxCensysPages { + break + } + + currentPage++ + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "censys" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { + return apiKey{k, v} + }) +} diff --git a/v2/pkg/subscraping/sources/certspotter/certspotter.go b/v2/pkg/subscraping/sources/certspotter/certspotter.go new file mode 100644 index 0000000..d94ff2a --- /dev/null +++ b/v2/pkg/subscraping/sources/certspotter/certspotter.go @@ -0,0 +1,120 @@ +// Package certspotter logic +package certspotter + +import ( + "context" + "fmt" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type certspotterObject struct { + ID string `json:"id"` + DNSNames []string `json:"dns_names"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + headers := map[string]string{"Authorization": "Bearer " + randomApiKey} + cookies := "" + + resp, err := session.Get(ctx, fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain), cookies, headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var response []certspotterObject + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + for _, cert := range response { + for _, subdomain := range cert.DNSNames { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } + + // if the number of responses is zero, close the channel and return. + if len(response) == 0 { + return + } + + id := response[len(response)-1].ID + for { + reqURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, id) + + resp, err := session.Get(ctx, reqURL, cookies, headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + var response []certspotterObject + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + if len(response) == 0 { + break + } + + for _, cert := range response { + for _, subdomain := range cert.DNSNames { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } + + id = response[len(response)-1].ID + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "certspotter" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/sources/chaos/chaos.go b/v2/pkg/subscraping/sources/chaos/chaos.go new file mode 100644 index 0000000..53a392c --- /dev/null +++ b/v2/pkg/subscraping/sources/chaos/chaos.go @@ -0,0 +1,63 @@ +// Package chaos logic +package chaos + +import ( + "context" + "fmt" + + "github.com/projectdiscovery/chaos-client/pkg/chaos" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(_ context.Context, domain string, _ *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + chaosClient := chaos.New(randomApiKey) + for result := range chaosClient.GetSubdomains(&chaos.SubdomainsRequest{ + Domain: domain, + }) { + if result.Error != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: result.Error} + break + } + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", result.Subdomain, domain)} + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "chaos" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/sources/chinaz/chinaz.go b/v2/pkg/subscraping/sources/chinaz/chinaz.go new file mode 100644 index 0000000..d7893ee --- /dev/null +++ b/v2/pkg/subscraping/sources/chinaz/chinaz.go @@ -0,0 +1,78 @@ +package chinaz + +// chinaz http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi +import ( + "context" + "fmt" + "io" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://apidatav2.chinaz.com/single/alexa?key=%s&domain=%s", randomApiKey, domain)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + body, err := io.ReadAll(resp.Body) + + resp.Body.Close() + + SubdomainList := jsoniter.Get(body, "Result").Get("ContributingSubdomainList") + + if SubdomainList.ToBool() { + _data := []byte(SubdomainList.ToString()) + for i := 0; i < SubdomainList.Size(); i++ { + subdomain := jsoniter.Get(_data, i, "DataUrl").ToString() + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } else { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "chinaz" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go b/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go new file mode 100644 index 0000000..09a16f3 --- /dev/null +++ b/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go @@ -0,0 +1,140 @@ +// Package commoncrawl logic +package commoncrawl + +import ( + "bufio" + "context" + "fmt" + "net/url" + "strconv" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +const ( + indexURL = "https://index.commoncrawl.org/collinfo.json" + maxYearsBack = 5 +) + +var year = time.Now().Year() + +type indexResponse struct { + ID string `json:"id"` + APIURL string `json:"cdx-api"` +} + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + resp, err := session.SimpleGet(ctx, indexURL) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var indexes []indexResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&indexes) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + years := make([]string, 0) + for i := 0; i < maxYearsBack; i++ { + years = append(years, strconv.Itoa(year-i)) + } + + searchIndexes := make(map[string]string) + for _, year := range years { + for _, index := range indexes { + if strings.Contains(index.ID, year) { + if _, ok := searchIndexes[year]; !ok { + searchIndexes[year] = index.APIURL + break + } + } + } + } + + for _, apiURL := range searchIndexes { + further := s.getSubdomains(ctx, apiURL, domain, session, results) + if !further { + break + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "commoncrawl" +} + +func (s *Source) IsDefault() bool { + return false +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return false +} + +func (s *Source) AddApiKeys(_ []string) { + // no key needed +} + +func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, session *subscraping.Session, results chan subscraping.Result) bool { + for { + select { + case <-ctx.Done(): + return false + default: + var headers = map[string]string{"Host": "index.commoncrawl.org"} + resp, err := session.Get(ctx, fmt.Sprintf("%s?url=*.%s", searchURL, domain), "", headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return false + } + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + line, _ = url.QueryUnescape(line) + subdomain := session.Extractor.FindString(line) + if subdomain != "" { + // fix for triple encoded URL + subdomain = strings.ToLower(subdomain) + subdomain = strings.TrimPrefix(subdomain, "25") + subdomain = strings.TrimPrefix(subdomain, "2f") + + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } + resp.Body.Close() + return true + } + } +} diff --git a/v2/pkg/subscraping/sources/crtsh/crtsh.go b/v2/pkg/subscraping/sources/crtsh/crtsh.go new file mode 100644 index 0000000..41f8904 --- /dev/null +++ b/v2/pkg/subscraping/sources/crtsh/crtsh.go @@ -0,0 +1,127 @@ +// Package crtsh logic +package crtsh + +import ( + "context" + "database/sql" + "fmt" + "strings" + + jsoniter "github.com/json-iterator/go" + + // postgres driver + _ "github.com/lib/pq" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type subdomain struct { + ID int `json:"id"` + NameValue string `json:"name_value"` +} + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + count := s.getSubdomainsFromSQL(domain, session, results) + if count > 0 { + return + } + _ = s.getSubdomainsFromHTTP(ctx, domain, session, results) + }() + + return results +} + +func (s *Source) getSubdomainsFromSQL(domain string, session *subscraping.Session, results chan subscraping.Result) int { + db, err := sql.Open("postgres", "host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes") + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return 0 + } + + defer db.Close() + + pattern := "%." + domain + query := `SELECT DISTINCT ci.NAME_VALUE as domain FROM certificate_identity ci + WHERE reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower($1)) + ORDER BY ci.NAME_VALUE` + rows, err := db.Query(query, pattern) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return 0 + } + if err := rows.Err(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return 0 + } + + var count int + var data string + // Parse all the rows getting subdomains + for rows.Next() { + err := rows.Scan(&data) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return count + } + count++ + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: session.Extractor.FindString(data)} + } + return count +} + +func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) bool { + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return false + } + + var subdomains []subdomain + err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return false + } + + resp.Body.Close() + + for _, subdomain := range subdomains { + for _, sub := range strings.Split(subdomain.NameValue, "\n") { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: session.Extractor.FindString(sub)} + } + } + + return true +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "crtsh" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) NeedsKey() bool { + return false +} + +func (s *Source) AddApiKeys(_ []string) { + // no key needed +} diff --git a/v2/pkg/subscraping/sources/dnsdb/dnsdb.go b/v2/pkg/subscraping/sources/dnsdb/dnsdb.go new file mode 100644 index 0000000..ce72901 --- /dev/null +++ b/v2/pkg/subscraping/sources/dnsdb/dnsdb.go @@ -0,0 +1,88 @@ +// Package dnsdb logic +package dnsdb + +import ( + "bufio" + "bytes" + "context" + "fmt" + "strings" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type dnsdbResponse struct { + Name string `json:"rrname"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + headers := map[string]string{ + "X-API-KEY": randomApiKey, + "Accept": "application/json", + "Content-Type": "application/json", + } + + resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdb.info/lookup/rrset/name/*.%s?limit=1000000000000", domain), "", headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + var response dnsdbResponse + err = jsoniter.NewDecoder(bytes.NewBufferString(line)).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(response.Name, ".")} + } + resp.Body.Close() + }() + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "dnsdb" +} + +func (s *Source) IsDefault() bool { + return false +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go new file mode 100644 index 0000000..c22cb90 --- /dev/null +++ b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go @@ -0,0 +1,120 @@ +// Package dnsdumpster logic +package dnsdumpster + +import ( + "context" + "fmt" + "io" + "net/url" + "regexp" + "strings" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// CSRFSubMatchLength CSRF regex submatch length +const CSRFSubMatchLength = 2 + +var re = regexp.MustCompile("") + +// getCSRFToken gets the CSRF Token from the page +func getCSRFToken(page string) string { + if subs := re.FindStringSubmatch(page); len(subs) == CSRFSubMatchLength { + return strings.TrimSpace(subs[1]) + } + return "" +} + +// postForm posts a form for a domain and returns the response +func postForm(ctx context.Context, session *subscraping.Session, token, domain string) (string, error) { + params := url.Values{ + "csrfmiddlewaretoken": {token}, + "targetip": {domain}, + "user": {"free"}, + } + + resp, err := session.HTTPRequest( + ctx, + "POST", + "https://dnsdumpster.com/", + fmt.Sprintf("csrftoken=%s; Domain=dnsdumpster.com", token), + map[string]string{ + "Content-Type": "application/x-www-form-urlencoded", + "Referer": "https://dnsdumpster.com", + "X-CSRF-Token": token, + }, + strings.NewReader(params.Encode()), + subscraping.BasicAuth{}, + ) + + if err != nil { + session.DiscardHTTPResponse(resp) + return "", err + } + + // Now, grab the entire page + in, err := io.ReadAll(resp.Body) + resp.Body.Close() + return string(in), err +} + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + resp, err := session.SimpleGet(ctx, "https://dnsdumpster.com/") + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + csrfToken := getCSRFToken(string(body)) + data, err := postForm(ctx, session, csrfToken, domain) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + for _, subdomain := range session.Extractor.FindAllString(data, -1) { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "dnsdumpster" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) NeedsKey() bool { + return false +} + +func (s *Source) AddApiKeys(_ []string) { + // no key needed +} diff --git a/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go b/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go new file mode 100644 index 0000000..262519f --- /dev/null +++ b/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go @@ -0,0 +1,79 @@ +package dnsrepo + +import ( + "context" + "encoding/json" + "fmt" + "io" + "strings" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +type DnsRepoResponse []struct { + Domain string +} + +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://dnsrepo.noc.org/api/?apikey=%s&search=%s", randomApiKey, domain)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + responseData, err := io.ReadAll(resp.Body) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + resp.Body.Close() + var result DnsRepoResponse + err = json.Unmarshal(responseData, &result) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + for _, sub := range result { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(sub.Domain, ".")} + } + + }() + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "dnsrepo" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/sources/fofa/fofa.go b/v2/pkg/subscraping/sources/fofa/fofa.go new file mode 100644 index 0000000..3cbdf59 --- /dev/null +++ b/v2/pkg/subscraping/sources/fofa/fofa.go @@ -0,0 +1,101 @@ +// Package fofa logic +package fofa + +import ( + "context" + "encoding/base64" + "fmt" + "strings" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type fofaResponse struct { + Error bool `json:"error"` + ErrMsg string `json:"errmsg"` + Size int `json:"size"` + Results []string `json:"results"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []apiKey +} + +type apiKey struct { + username string + secret string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey.username == "" || randomApiKey.secret == "" { + return + } + + // fofa api doc https://fofa.info/static_pages/api_help + qbase64 := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain))) + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://fofa.info/api/v1/search/all?full=true&fields=host&page=1&size=10000&email=%s&key=%s&qbase64=%s", randomApiKey.username, randomApiKey.secret, qbase64)) + if err != nil && resp == nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var response fofaResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + if response.Error { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.ErrMsg)} + return + } + + if response.Size > 0 { + for _, subdomain := range response.Results { + if strings.HasPrefix(strings.ToLower(subdomain), "http://") || strings.HasPrefix(strings.ToLower(subdomain), "https://") { + subdomain = subdomain[strings.Index(subdomain, "//")+2:] + } + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "fofa" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { + return apiKey{k, v} + }) +} diff --git a/v2/pkg/subscraping/sources/fullhunt/fullhunt.go b/v2/pkg/subscraping/sources/fullhunt/fullhunt.go new file mode 100644 index 0000000..72d6701 --- /dev/null +++ b/v2/pkg/subscraping/sources/fullhunt/fullhunt.go @@ -0,0 +1,76 @@ +package fullhunt + +import ( + "context" + "fmt" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +//fullhunt response +type fullHuntResponse struct { + Hosts []string `json:"hosts"` + Message string `json:"message"` + Status int `json:"status"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + resp, err := session.Get(ctx, fmt.Sprintf("https://fullhunt.io/api/v1/domain/%s/subdomains", domain), "", map[string]string{"X-API-KEY": randomApiKey}) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var response fullHuntResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + for _, record := range response.Hosts { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record} + } + }() + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "fullhunt" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/sources/github/github.go b/v2/pkg/subscraping/sources/github/github.go new file mode 100644 index 0000000..3c7f649 --- /dev/null +++ b/v2/pkg/subscraping/sources/github/github.go @@ -0,0 +1,212 @@ +// Package github GitHub search package +// Based on gwen001's https://github.com/gwen001/github-search github-subdomains +package github + +import ( + "bufio" + "context" + "fmt" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/tomnomnom/linkheader" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type textMatch struct { + Fragment string `json:"fragment"` +} + +type item struct { + Name string `json:"name"` + HTMLURL string `json:"html_url"` + TextMatches []textMatch `json:"text_matches"` +} + +type response struct { + TotalCount int `json:"total_count"` + Items []item `json:"items"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + if len(s.apiKeys) == 0 { + gologger.Debug().Msgf("Cannot use the '%s' source because there was no key defined for it.", s.Name()) + return + } + + tokens := NewTokenManager(s.apiKeys) + + searchURL := fmt.Sprintf("https://api.github.com/search/code?per_page=100&q=%s&sort=created&order=asc", domain) + s.enumerate(ctx, searchURL, domainRegexp(domain), tokens, session, results) + }() + + return results +} + +func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *regexp.Regexp, tokens *Tokens, session *subscraping.Session, results chan subscraping.Result) { + select { + case <-ctx.Done(): + return + default: + } + + token := tokens.Get() + + if token.RetryAfter > 0 { + if len(tokens.pool) == 1 { + gologger.Verbose().Label(s.Name()).Msgf("GitHub Search request rate limit exceeded, waiting for %d seconds before retry... \n", token.RetryAfter) + time.Sleep(time.Duration(token.RetryAfter) * time.Second) + } else { + token = tokens.Get() + } + } + + headers := map[string]string{"Accept": "application/vnd.github.v3.text-match+json", "Authorization": "token " + token.Hash} + + // Initial request to GitHub search + resp, err := session.Get(ctx, searchURL, "", headers) + isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden + if err != nil && !isForbidden { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + // Retry enumerarion after Retry-After seconds on rate limit abuse detected + ratelimitRemaining, _ := strconv.ParseInt(resp.Header.Get("X-Ratelimit-Remaining"), 10, 64) + if isForbidden && ratelimitRemaining == 0 { + retryAfterSeconds, _ := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64) + tokens.setCurrentTokenExceeded(retryAfterSeconds) + resp.Body.Close() + + s.enumerate(ctx, searchURL, domainRegexp, tokens, session, results) + } + + var data response + + // Marshall json response + err = jsoniter.NewDecoder(resp.Body).Decode(&data) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + + resp.Body.Close() + + err = proccesItems(ctx, data.Items, domainRegexp, s.Name(), session, results) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + // Links header, first, next, last... + linksHeader := linkheader.Parse(resp.Header.Get("Link")) + // Process the next link recursively + for _, link := range linksHeader { + if link.Rel == "next" { + nextURL, err := url.QueryUnescape(link.URL) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + s.enumerate(ctx, nextURL, domainRegexp, tokens, session, results) + } + } +} + +// proccesItems procceses github response items +func proccesItems(ctx context.Context, items []item, domainRegexp *regexp.Regexp, name string, session *subscraping.Session, results chan subscraping.Result) error { + for _, item := range items { + // find subdomains in code + resp, err := session.SimpleGet(ctx, rawURL(item.HTMLURL)) + if err != nil { + if resp != nil && resp.StatusCode != http.StatusNotFound { + session.DiscardHTTPResponse(resp) + } + return err + } + + if resp.StatusCode == http.StatusOK { + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + for _, subdomain := range domainRegexp.FindAllString(normalizeContent(line), -1) { + results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain} + } + } + resp.Body.Close() + } + + // find subdomains in text matches + for _, textMatch := range item.TextMatches { + for _, subdomain := range domainRegexp.FindAllString(normalizeContent(textMatch.Fragment), -1) { + results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain} + } + } + } + return nil +} + +// Normalize content before matching, query unescape, remove tabs and new line chars +func normalizeContent(content string) string { + normalizedContent, _ := url.QueryUnescape(content) + normalizedContent = strings.ReplaceAll(normalizedContent, "\\t", "") + normalizedContent = strings.ReplaceAll(normalizedContent, "\\n", "") + return normalizedContent +} + +// Raw URL to get the files code and match for subdomains +func rawURL(htmlURL string) string { + domain := strings.ReplaceAll(htmlURL, "https://github.com/", "https://raw.githubusercontent.com/") + return strings.ReplaceAll(domain, "/blob/", "/") +} + +// DomainRegexp regular expression to match subdomains in github files code +func domainRegexp(domain string) *regexp.Regexp { + rdomain := strings.ReplaceAll(domain, ".", "\\.") + return regexp.MustCompile("(\\w[a-zA-Z0-9][a-zA-Z0-9-\\.]*)" + rdomain) +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "github" +} + +func (s *Source) IsDefault() bool { + return false +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/sources/github/tokenmanager.go b/v2/pkg/subscraping/sources/github/tokenmanager.go new file mode 100644 index 0000000..effdfd7 --- /dev/null +++ b/v2/pkg/subscraping/sources/github/tokenmanager.go @@ -0,0 +1,66 @@ +package github + +import "time" + +// Token struct +type Token struct { + Hash string + RetryAfter int64 + ExceededTime time.Time +} + +// Tokens is the internal struct to manage the current token +// and the pool +type Tokens struct { + current int + pool []Token +} + +// NewTokenManager initialize the tokens pool +func NewTokenManager(keys []string) *Tokens { + pool := []Token{} + for _, key := range keys { + t := Token{Hash: key, ExceededTime: time.Time{}, RetryAfter: 0} + pool = append(pool, t) + } + + return &Tokens{ + current: 0, + pool: pool, + } +} + +func (r *Tokens) setCurrentTokenExceeded(retryAfter int64) { + if r.current >= len(r.pool) { + r.current %= len(r.pool) + } + if r.pool[r.current].RetryAfter == 0 { + r.pool[r.current].ExceededTime = time.Now() + r.pool[r.current].RetryAfter = retryAfter + } +} + +// Get returns a new token from the token pool +func (r *Tokens) Get() *Token { + resetExceededTokens(r) + + if r.current >= len(r.pool) { + r.current %= len(r.pool) + } + + result := &r.pool[r.current] + r.current++ + + return result +} + +func resetExceededTokens(r *Tokens) { + for i, token := range r.pool { + if token.RetryAfter > 0 { + if int64(time.Since(token.ExceededTime)/time.Second) > token.RetryAfter { + r.pool[i].ExceededTime = time.Time{} + r.pool[i].RetryAfter = 0 + } + } + } +} diff --git a/v2/pkg/subscraping/sources/hackertarget/hackertarget.go b/v2/pkg/subscraping/sources/hackertarget/hackertarget.go new file mode 100644 index 0000000..a2b82b6 --- /dev/null +++ b/v2/pkg/subscraping/sources/hackertarget/hackertarget.go @@ -0,0 +1,66 @@ +// Package hackertarget logic +package hackertarget + +import ( + "bufio" + "context" + "fmt" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://api.hackertarget.com/hostsearch/?q=%s", domain)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + defer resp.Body.Close() + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + match := session.Extractor.FindAllString(line, -1) + for _, subdomain := range match { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "hackertarget" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) NeedsKey() bool { + return false +} + +func (s *Source) AddApiKeys(_ []string) { + // no key needed +} diff --git a/v2/pkg/subscraping/sources/hunter/hunter.go b/v2/pkg/subscraping/sources/hunter/hunter.go new file mode 100644 index 0000000..ff960e2 --- /dev/null +++ b/v2/pkg/subscraping/sources/hunter/hunter.go @@ -0,0 +1,105 @@ +package hunter + +import ( + "context" + "encoding/base64" + "fmt" + + jsoniter "github.com/json-iterator/go" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type hunterResp struct { + Code int `json:"code"` + Data hunterData `json:"data"` + Message string `json:"message"` +} + +type infoArr struct { + URL string `json:"url"` + IP string `json:"ip"` + Port int `json:"port"` + Domain string `json:"domain"` + Protocol string `json:"protocol"` +} + +type hunterData struct { + InfoArr []infoArr `json:"arr"` + Total int `json:"total"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + var pages = 1 + for currentPage := 1; currentPage <= pages; currentPage++ { + // hunter api doc https://hunter.qianxin.com/home/helpCenter?r=5-1-2 + qbase64 := base64.URLEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain))) + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=1&page_size=100&is_web=3", randomApiKey, qbase64)) + if err != nil && resp == nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var response hunterResp + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + if response.Code == 401 || response.Code == 400 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message)} + return + } + + if response.Data.Total > 0 { + for _, hunterInfo := range response.Data.InfoArr { + subdomain := hunterInfo.Domain + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } + pages = int(response.Data.Total/1000) + 1 + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "hunter" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/sources/intelx/intelx.go b/v2/pkg/subscraping/sources/intelx/intelx.go new file mode 100644 index 0000000..b3fc942 --- /dev/null +++ b/v2/pkg/subscraping/sources/intelx/intelx.go @@ -0,0 +1,149 @@ +// Package intelx logic +package intelx + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type searchResponseType struct { + ID string `json:"id"` + Status int `json:"status"` +} + +type selectorType struct { + Selectvalue string `json:"selectorvalue"` +} + +type searchResultType struct { + Selectors []selectorType `json:"selectors"` + Status int `json:"status"` +} + +type requestBody struct { + Term string + Maxresults int + Media int + Target int + Terminate []int + Timeout int +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []apiKey +} + +type apiKey struct { + host string + key string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey.host == "" || randomApiKey.key == "" { + return + } + + searchURL := fmt.Sprintf("https://%s/phonebook/search?k=%s", randomApiKey.host, randomApiKey.key) + reqBody := requestBody{ + Term: domain, + Maxresults: 100000, + Media: 0, + Target: 1, + Timeout: 20, + } + + body, err := json.Marshal(reqBody) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + resp, err := session.SimplePost(ctx, searchURL, "application/json", bytes.NewBuffer(body)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var response searchResponseType + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + + resp.Body.Close() + + resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", randomApiKey.host, randomApiKey.key, response.ID) + status := 0 + for status == 0 || status == 3 { + resp, err = session.Get(ctx, resultsURL, "", nil) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + var response searchResultType + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + + _, err = io.ReadAll(resp.Body) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + status = response.Status + for _, hostname := range response.Selectors { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue} + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "intelx" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { + return apiKey{k, v} + }) +} diff --git a/v2/pkg/subscraping/sources/passivetotal/passivetotal.go b/v2/pkg/subscraping/sources/passivetotal/passivetotal.go new file mode 100644 index 0000000..8bb02fb --- /dev/null +++ b/v2/pkg/subscraping/sources/passivetotal/passivetotal.go @@ -0,0 +1,103 @@ +// Package passivetotal logic +package passivetotal + +import ( + "bytes" + "context" + "regexp" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +var passiveTotalFilterRegex = regexp.MustCompile(`^(?:\d{1,3}\.){3}\d{1,3}\\032`) + +type response struct { + Subdomains []string `json:"subdomains"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []apiKey +} + +type apiKey struct { + username string + password string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey.username == "" || randomApiKey.password == "" { + return + } + + // Create JSON Get body + var request = []byte(`{"query":"` + domain + `"}`) + + resp, err := session.HTTPRequest( + ctx, + "GET", + "https://api.passivetotal.org/v2/enrichment/subdomains", + "", + map[string]string{"Content-Type": "application/json"}, + bytes.NewBuffer(request), + subscraping.BasicAuth{Username: randomApiKey.username, Password: randomApiKey.password}, + ) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var data response + err = jsoniter.NewDecoder(resp.Body).Decode(&data) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + for _, subdomain := range data.Subdomains { + // skip entries like xxx.xxx.xxx.xxx\032domain.tld + if passiveTotalFilterRegex.MatchString(subdomain) { + continue + } + finalSubdomain := subdomain + "." + domain + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: finalSubdomain} + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "passivetotal" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { + return apiKey{k, v} + }) +} diff --git a/v2/pkg/subscraping/sources/quake/quake.go b/v2/pkg/subscraping/sources/quake/quake.go new file mode 100644 index 0000000..8058568 --- /dev/null +++ b/v2/pkg/subscraping/sources/quake/quake.go @@ -0,0 +1,104 @@ +// Package quake logic +package quake + +import ( + "bytes" + "context" + "fmt" + "strings" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type quakeResults struct { + Code int `json:"code"` + Message string `json:"message"` + Data []struct { + Service struct { + HTTP struct { + Host string `json:"host"` + } `json:"http"` + } + } `json:"data"` + Meta struct { + Pagination struct { + Total int `json:"total"` + } `json:"pagination"` + } `json:"meta"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + // quake api doc https://quake.360.cn/quake/#/help + var requestBody = []byte(fmt.Sprintf(`{"query":"domain: *.%s", "start":0, "size":500}`, domain)) + resp, err := session.Post(ctx, "https://quake.360.cn/api/v3/search/quake_service", "", map[string]string{"Content-Type": "application/json", "X-QuakeToken": randomApiKey}, bytes.NewReader(requestBody)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var response quakeResults + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + if response.Code != 0 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message)} + return + } + + if response.Meta.Pagination.Total > 0 { + for _, quakeDomain := range response.Data { + subdomain := quakeDomain.Service.HTTP.Host + if strings.ContainsAny(subdomain, "暂无权限") { + subdomain = "" + } + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "quake" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/sources/rapiddns/rapiddns.go b/v2/pkg/subscraping/sources/rapiddns/rapiddns.go new file mode 100644 index 0000000..d9690a8 --- /dev/null +++ b/v2/pkg/subscraping/sources/rapiddns/rapiddns.go @@ -0,0 +1,65 @@ +// Package rapiddns is a RapidDNS Scraping Engine in Golang +package rapiddns + +import ( + "context" + "io" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + resp, err := session.SimpleGet(ctx, "https://rapiddns.io/subdomain/"+domain+"?full=1") + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + + resp.Body.Close() + + src := string(body) + for _, subdomain := range session.Extractor.FindAllString(src, -1) { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "rapiddns" +} + +func (s *Source) IsDefault() bool { + return false +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return false +} + +func (s *Source) AddApiKeys(_ []string) { + // no key needed +} diff --git a/v2/pkg/subscraping/sources/reconcloud/reconcloud.go b/v2/pkg/subscraping/sources/reconcloud/reconcloud.go new file mode 100644 index 0000000..670ebfd --- /dev/null +++ b/v2/pkg/subscraping/sources/reconcloud/reconcloud.go @@ -0,0 +1,81 @@ +// Package reconcloud logic +package reconcloud + +import ( + "context" + "fmt" + + jsoniter "github.com/json-iterator/go" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type reconCloudResponse struct { + MsgType string `json:"msg_type"` + RequestID string `json:"request_id"` + OnCache bool `json:"on_cache"` + Step string `json:"step"` + CloudAssetsList []cloudAssetsList `json:"cloud_assets_list"` +} + +type cloudAssetsList struct { + Key string `json:"key"` + Domain string `json:"domain"` + CloudProvider string `json:"cloud_provider"` +} + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://recon.cloud/api/search?domain=%s", domain)) + if err != nil && resp == nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var response reconCloudResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + if len(response.CloudAssetsList) > 0 { + for _, cloudAsset := range response.CloudAssetsList { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: cloudAsset.Domain} + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "reconcloud" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) NeedsKey() bool { + return false +} + +func (s *Source) AddApiKeys(_ []string) { + // no key needed +} diff --git a/v2/pkg/subscraping/sources/riddler/riddler.go b/v2/pkg/subscraping/sources/riddler/riddler.go new file mode 100644 index 0000000..879c742 --- /dev/null +++ b/v2/pkg/subscraping/sources/riddler/riddler.go @@ -0,0 +1,65 @@ +// Package riddler logic +package riddler + +import ( + "bufio" + "context" + "fmt" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://riddler.io/search?q=pld:%s&view_type=data_table", domain)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + subdomain := session.Extractor.FindString(line) + if subdomain != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } + resp.Body.Close() + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "riddler" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return false +} + +func (s *Source) AddApiKeys(_ []string) { + // no key needed +} diff --git a/v2/pkg/subscraping/sources/robtex/robtext.go b/v2/pkg/subscraping/sources/robtex/robtext.go new file mode 100644 index 0000000..55f4819 --- /dev/null +++ b/v2/pkg/subscraping/sources/robtex/robtext.go @@ -0,0 +1,116 @@ +// Package robtex logic +package robtex + +import ( + "bufio" + "bytes" + "context" + "fmt" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +const ( + addrRecord = "A" + iPv6AddrRecord = "AAAA" + baseURL = "https://proapi.robtex.com/pdns" +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +type result struct { + Rrname string `json:"rrname"` + Rrdata string `json:"rrdata"` + Rrtype string `json:"rrtype"` +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + headers := map[string]string{"Content-Type": "application/x-ndjson"} + + ips, err := enumerate(ctx, session, fmt.Sprintf("%s/forward/%s?key=%s", baseURL, domain, randomApiKey), headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + for _, result := range ips { + if result.Rrtype == addrRecord || result.Rrtype == iPv6AddrRecord { + domains, err := enumerate(ctx, session, fmt.Sprintf("%s/reverse/%s?key=%s", baseURL, result.Rrdata, randomApiKey), headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + for _, result := range domains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Rrdata} + } + } + } + }() + return results +} + +func enumerate(ctx context.Context, session *subscraping.Session, targetURL string, headers map[string]string) ([]result, error) { + var results []result + + resp, err := session.Get(ctx, targetURL, "", headers) + if err != nil { + session.DiscardHTTPResponse(resp) + return results, err + } + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + var response result + err = jsoniter.NewDecoder(bytes.NewBufferString(line)).Decode(&response) + if err != nil { + return results, err + } + + results = append(results, response) + } + + resp.Body.Close() + + return results, nil +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "robtex" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/sources/securitytrails/securitytrails.go b/v2/pkg/subscraping/sources/securitytrails/securitytrails.go new file mode 100644 index 0000000..2c3e7fc --- /dev/null +++ b/v2/pkg/subscraping/sources/securitytrails/securitytrails.go @@ -0,0 +1,85 @@ +// Package securitytrails logic +package securitytrails + +import ( + "context" + "fmt" + "strings" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type response struct { + Subdomains []string `json:"subdomains"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + resp, err := session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", map[string]string{"APIKEY": randomApiKey}) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var securityTrailsResponse response + err = jsoniter.NewDecoder(resp.Body).Decode(&securityTrailsResponse) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + + resp.Body.Close() + + for _, subdomain := range securityTrailsResponse.Subdomains { + if strings.HasSuffix(subdomain, ".") { + subdomain += domain + } else { + subdomain = subdomain + "." + domain + } + + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "securitytrails" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/sources/shodan/shodan.go b/v2/pkg/subscraping/sources/shodan/shodan.go new file mode 100644 index 0000000..bc4416a --- /dev/null +++ b/v2/pkg/subscraping/sources/shodan/shodan.go @@ -0,0 +1,85 @@ +// Package shodan logic +package shodan + +import ( + "context" + "fmt" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +type dnsdbLookupResponse struct { + Domain string `json:"domain"` + Subdomains []string `json:"subdomains"` + Result int `json:"result"` + Error string `json:"error"` +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + searchURL := fmt.Sprintf("https://api.shodan.io/dns/domain/%s?key=%s", domain, randomApiKey) + resp, err := session.SimpleGet(ctx, searchURL) + if err != nil { + session.DiscardHTTPResponse(resp) + return + } + + defer resp.Body.Close() + + var response dnsdbLookupResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + if response.Error != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error)} + return + } + + for _, data := range response.Subdomains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", data, domain)} + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "shodan" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/sources/sitedossier/sitedossier.go b/v2/pkg/subscraping/sources/sitedossier/sitedossier.go new file mode 100644 index 0000000..47ade5f --- /dev/null +++ b/v2/pkg/subscraping/sources/sitedossier/sitedossier.go @@ -0,0 +1,102 @@ +// Package sitedossier logic +package sitedossier + +import ( + "context" + "fmt" + "io" + "math/rand" + "net/http" + "regexp" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// SleepRandIntn is the integer value to get the pseudo-random number +// to sleep before find the next match +const SleepRandIntn = 5 + +var reNext = regexp.MustCompile(``) + +type agent struct { + results chan subscraping.Result + session *subscraping.Session +} + +func (a *agent) enumerate(ctx context.Context, baseURL string) { + select { + case <-ctx.Done(): + return + default: + } + + resp, err := a.session.SimpleGet(ctx, baseURL) + isnotfound := resp != nil && resp.StatusCode == http.StatusNotFound + if err != nil && !isnotfound { + a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} + a.session.DiscardHTTPResponse(resp) + return + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + src := string(body) + for _, match := range a.session.Extractor.FindAllString(src, -1) { + a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: match} + } + + match1 := reNext.FindStringSubmatch(src) + time.Sleep(time.Duration((3 + rand.Intn(SleepRandIntn))) * time.Second) + + if len(match1) > 0 { + a.enumerate(ctx, "http://www.sitedossier.com"+match1[1]) + } +} + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + a := agent{ + session: session, + results: results, + } + + go func() { + a.enumerate(ctx, fmt.Sprintf("http://www.sitedossier.com/parentdomain/%s", domain)) + close(a.results) + }() + + return a.results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "sitedossier" +} + +func (s *Source) IsDefault() bool { + return false +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return false +} + +func (s *Source) AddApiKeys(_ []string) { + // no key needed +} diff --git a/v2/pkg/subscraping/sources/threatbook/threatbook.go b/v2/pkg/subscraping/sources/threatbook/threatbook.go new file mode 100644 index 0000000..65cfd7b --- /dev/null +++ b/v2/pkg/subscraping/sources/threatbook/threatbook.go @@ -0,0 +1,99 @@ +// Package threatbook logic +package threatbook + +import ( + "context" + "fmt" + "strconv" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type threatBookResponse struct { + ResponseCode int64 `json:"response_code"` + VerboseMsg string `json:"verbose_msg"` + Data struct { + Domain string `json:"domain"` + SubDomains struct { + Total string `json:"total"` + Data []string `json:"data"` + } `json:"sub_domains"` + } `json:"data"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatbook.cn/v3/domain/sub_domains?apikey=%s&resource=%s", randomApiKey, domain)) + if err != nil && resp == nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var response threatBookResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + if response.ResponseCode != 0 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("code %d, %s", response.ResponseCode, response.VerboseMsg)} + return + } + + total, err := strconv.ParseInt(response.Data.SubDomains.Total, 10, 64) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + if total > 0 { + for _, subdomain := range response.Data.SubDomains.Data { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "threatbook" +} + +func (s *Source) IsDefault() bool { + return false +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/sources/threatminer/threatminer.go b/v2/pkg/subscraping/sources/threatminer/threatminer.go new file mode 100644 index 0000000..f615f4f --- /dev/null +++ b/v2/pkg/subscraping/sources/threatminer/threatminer.go @@ -0,0 +1,72 @@ +// Package threatminer logic +package threatminer + +import ( + "context" + "fmt" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type response struct { + StatusCode string `json:"status_code"` + StatusMessage string `json:"status_message"` + Results []string `json:"results"` +} + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatminer.org/v2/domain.php?q=%s&rt=5", domain)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + defer resp.Body.Close() + + var data response + err = jsoniter.NewDecoder(resp.Body).Decode(&data) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + for _, subdomain := range data.Results { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "threatminer" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return false +} + +func (s *Source) AddApiKeys(_ []string) { + // no key needed +} diff --git a/v2/pkg/subscraping/sources/virustotal/virustotal.go b/v2/pkg/subscraping/sources/virustotal/virustotal.go new file mode 100644 index 0000000..062e780 --- /dev/null +++ b/v2/pkg/subscraping/sources/virustotal/virustotal.go @@ -0,0 +1,78 @@ +// Package virustotal logic +package virustotal + +import ( + "context" + "fmt" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type response struct { + Subdomains []string `json:"subdomains"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://www.virustotal.com/vtapi/v2/domain/report?apikey=%s&domain=%s", randomApiKey, domain)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var data response + err = jsoniter.NewDecoder(resp.Body).Decode(&data) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + + resp.Body.Close() + + for _, subdomain := range data.Subdomains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "virustotal" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go b/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go new file mode 100644 index 0000000..4263cb5 --- /dev/null +++ b/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go @@ -0,0 +1,74 @@ +// Package waybackarchive logic +package waybackarchive + +import ( + "bufio" + "context" + "fmt" + "net/url" + "strings" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey", domain)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + defer resp.Body.Close() + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + line, _ = url.QueryUnescape(line) + subdomain := session.Extractor.FindString(line) + if subdomain != "" { + // fix for triple encoded URL + subdomain = strings.ToLower(subdomain) + subdomain = strings.TrimPrefix(subdomain, "25") + subdomain = strings.TrimPrefix(subdomain, "2f") + + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "waybackarchive" +} + +func (s *Source) IsDefault() bool { + return false +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return false +} + +func (s *Source) AddApiKeys(_ []string) { + // no key needed +} diff --git a/v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go b/v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go new file mode 100644 index 0000000..ab1ae63 --- /dev/null +++ b/v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go @@ -0,0 +1,90 @@ +// Package whoisxmlapi logic +package whoisxmlapi + +import ( + "context" + "fmt" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type response struct { + Search string `json:"search"` + Result Result `json:"result"` +} + +type Result struct { + Count int `json:"count"` + Records []Record `json:"records"` +} + +type Record struct { + Domain string `json:"domain"` + FirstSeen int `json:"firstSeen"` + LastSeen int `json:"lastSeen"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://subdomains.whoisxmlapi.com/api/v1?apiKey=%s&domainName=%s", randomApiKey, domain)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var data response + err = jsoniter.NewDecoder(resp.Body).Decode(&data) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + + resp.Body.Close() + + for _, record := range data.Result.Records { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain} + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "whoisxmlapi" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/sources/zoomeye/zoomeye.go b/v2/pkg/subscraping/sources/zoomeye/zoomeye.go new file mode 100644 index 0000000..ab0f278 --- /dev/null +++ b/v2/pkg/subscraping/sources/zoomeye/zoomeye.go @@ -0,0 +1,151 @@ +// Package zoomeye logic +package zoomeye + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// zoomAuth holds the ZoomEye credentials +type zoomAuth struct { + User string `json:"username"` + Pass string `json:"password"` +} + +type loginResp struct { + JWT string `json:"access_token"` +} + +// search results +type zoomeyeResults struct { + Matches []struct { + Site string `json:"site"` + Domains []string `json:"domains"` + } `json:"matches"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []apiKey +} + +type apiKey struct { + username string + password string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey.username == "" || randomApiKey.password == "" { + return + } + + jwt, err := doLogin(ctx, session, randomApiKey) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + // check if jwt is null + if jwt == "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errors.New("could not log into zoomeye")} + return + } + + headers := map[string]string{ + "Authorization": fmt.Sprintf("JWT %s", jwt), + "Accept": "application/json", + "Content-Type": "application/json", + } + for currentPage := 0; currentPage <= 100; currentPage++ { + api := fmt.Sprintf("https://api.zoomeye.org/web/search?query=hostname:%s&page=%d", domain, currentPage) + resp, err := session.Get(ctx, api, "", headers) + isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden + if err != nil { + if !isForbidden && currentPage == 0 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + } + return + } + + var res zoomeyeResults + err = json.NewDecoder(resp.Body).Decode(&res) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + for _, r := range res.Matches { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Site} + for _, domain := range r.Domains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domain} + } + } + } + }() + + return results +} + +// doLogin performs authentication on the ZoomEye API +func doLogin(ctx context.Context, session *subscraping.Session, randomApiKey apiKey) (string, error) { + creds := &zoomAuth{ + User: randomApiKey.username, + Pass: randomApiKey.password, + } + body, err := json.Marshal(&creds) + if err != nil { + return "", err + } + resp, err := session.SimplePost(ctx, "https://api.zoomeye.org/user/login", "application/json", bytes.NewBuffer(body)) + if err != nil { + session.DiscardHTTPResponse(resp) + return "", err + } + + defer resp.Body.Close() + + var login loginResp + err = json.NewDecoder(resp.Body).Decode(&login) + if err != nil { + return "", err + } + return login.JWT, nil +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "zoomeye" +} + +func (s *Source) IsDefault() bool { + return false +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = subscraping.CreateApiKeys(keys, func(k, v string) apiKey { + return apiKey{k, v} + }) +} diff --git a/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go b/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go new file mode 100644 index 0000000..f82dd69 --- /dev/null +++ b/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go @@ -0,0 +1,95 @@ +package zoomeyeapi + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// search results +type zoomeyeResults struct { + Status int `json:"status"` + Total int `json:"total"` + List []struct { + Name string `json:"name"` + Ip []string `json:"ip"` + } `json:"list"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + headers := map[string]string{ + "API-KEY": randomApiKey, + "Accept": "application/json", + "Content-Type": "application/json", + } + var pages = 1 + for currentPage := 1; currentPage <= pages; currentPage++ { + api := fmt.Sprintf("https://api.zoomeye.org/domain/search?q=%s&type=1&s=1000&page=%d", domain, currentPage) + resp, err := session.Get(ctx, api, "", headers) + isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden + if err != nil { + if !isForbidden { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + } + return + } + + var res zoomeyeResults + err = json.NewDecoder(resp.Body).Decode(&res) + + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + _ = resp.Body.Close() + return + } + _ = resp.Body.Close() + pages = int(res.Total/1000) + 1 + for _, r := range res.List { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name} + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "zoomeyeapi" +} + +func (s *Source) IsDefault() bool { + return false +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go new file mode 100644 index 0000000..13ce5c2 --- /dev/null +++ b/v2/pkg/subscraping/types.go @@ -0,0 +1,67 @@ +package subscraping + +import ( + "context" + "net/http" + "regexp" + + "github.com/projectdiscovery/ratelimit" +) + +// BasicAuth request's Authorization header +type BasicAuth struct { + Username string + Password string +} + +// Source is an interface inherited by each passive source +type Source interface { + // Run takes a domain as argument and a session object + // which contains the extractor for subdomains, http client + // and other stuff. + Run(context.Context, string, *Session) <-chan Result + // Name returns the name of the source. It is preferred to use lower case names. + Name() string + + // IsDefault returns true if the current source should be + // used as part of the default execution. + IsDefault() bool + + // HasRecursiveSupport returns true if the current source + // accepts subdomains (e.g. subdomain.domain.tld), + // not just root domains. + HasRecursiveSupport() bool + + // NeedsKey returns true if the source requires an API key + NeedsKey() bool + + AddApiKeys([]string) +} + +// Session is the option passed to the source, an option is created +// uniquely for each source. +type Session struct { + // Extractor is the regex for subdomains created for each domain + Extractor *regexp.Regexp + // Client is the current http client + Client *http.Client + // Rate limit instance + RateLimiter *ratelimit.Limiter +} + +// Result is a result structure returned by a source +type Result struct { + Type ResultType + Source string + Value string + Error error +} + +// ResultType is the type of result returned by the source +type ResultType int + +// Types of results returned by the source +const ( + Subdomain ResultType = iota + Error +) diff --git a/v2/pkg/subscraping/utils.go b/v2/pkg/subscraping/utils.go new file mode 100644 index 0000000..8ede83c --- /dev/null +++ b/v2/pkg/subscraping/utils.go @@ -0,0 +1,63 @@ +package subscraping + +import ( + "math/rand" + "regexp" + "strings" + "sync" + "time" + + "github.com/projectdiscovery/gologger" +) + +const MultipleKeyPartsLength = 2 + +var subdomainExtractorMutex = &sync.Mutex{} + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +// NewSubdomainExtractor creates a new regular expression to extract +// subdomains from text based on the given domain. +func NewSubdomainExtractor(domain string) (*regexp.Regexp, error) { + subdomainExtractorMutex.Lock() + defer subdomainExtractorMutex.Unlock() + extractor, err := regexp.Compile(`[a-zA-Z0-9\*_.-]+\.` + domain) + if err != nil { + return nil, err + } + return extractor, nil +} + +func PickRandom[T any](v []T, sourceName string) T { + var result T + length := len(v) + if length == 0 { + gologger.Debug().Msgf("Cannot use the '%s' source because there was no API key/secret defined for it.", sourceName) + return result + } + return v[rand.Intn(length)] +} + +func CreateApiKeys[T any](keys []string, provider func(k, v string) T) []T { + var result []T + for _, key := range keys { + if keyPartA, keyPartB, ok := createMultiPartKey(key); ok { + result = append(result, provider(keyPartA, keyPartB)) + } + } + return result +} + +func createMultiPartKey(key string) (keyPartA, keyPartB string, ok bool) { + parts := strings.Split(key, ":") + ok = len(parts) == MultipleKeyPartsLength + + if ok { + keyPartA = parts[0] + keyPartB = parts[1] + } + + return +} diff --git a/v2/pkg/testutils/integration.go b/v2/pkg/testutils/integration.go new file mode 100644 index 0000000..2358cad --- /dev/null +++ b/v2/pkg/testutils/integration.go @@ -0,0 +1,42 @@ +package testutils + +import ( + "fmt" + "os" + "os/exec" + "strings" +) + +func RunSubfinderAndGetResults(debug bool, domain string, extra ...string) ([]string, error) { + cmd := exec.Command("bash", "-c") + cmdLine := fmt.Sprintf("echo %s | %s", domain, "./subfinder ") + cmdLine += strings.Join(extra, " ") + cmd.Args = append(cmd.Args, cmdLine) + if debug { + cmd.Args = append(cmd.Args, "-v") + cmd.Stderr = os.Stderr + fmt.Println(cmd.String()) + } else { + cmd.Args = append(cmd.Args, "-silent") + } + data, err := cmd.Output() + if debug { + fmt.Println(string(data)) + } + if err != nil { + return nil, err + } + var parts []string + items := strings.Split(string(data), "\n") + for _, i := range items { + if i != "" { + parts = append(parts, i) + } + } + return parts, nil +} + +// TestCase is a single integration test case +type TestCase interface { + Execute() error +}