diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..91e27c5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +Dockerfile +certgraph +build/ +*.json \ No newline at end of file diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 0000000..151ae27 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,67 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ master ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ master ] + schedule: + - cron: '31 22 * * 3' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..853b10e --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,22 @@ +name: Go + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.16 + + - name: Build + run: go build -v ./... diff --git a/.gitignore b/.gitignore index c167f4f..f353dbc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -go.sum certgraph build/ *.json diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8735826 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM golang:alpine + +RUN apk add --update git make + +WORKDIR /src/certgraph +ADD . . + +ENV CGO_ENABLED=0 +RUN make install + +ENTRYPOINT [ "certgraph" ] \ No newline at end of file diff --git a/Makefile b/Makefile index 178312e..f61a773 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,18 @@ GIT_DATE := $(shell git log -1 --date=short --pretty='%cd' | tr -d -) GIT_HASH := $(shell git rev-parse HEAD) -BUILD_FLAGS := -ldflags "-X main.gitDate=$(GIT_DATE) -X main.gitHash=$(GIT_HASH)" +BUILD_FLAGS := -trimpath -ldflags "-w -s -X main.gitDate=$(GIT_DATE) -X main.gitHash=$(GIT_HASH)" -PLATFORMS := linux/amd64 linux/386 linux/arm darwin/amd64 windows/amd64 windows/386 openbsd/amd64 +PLATFORMS := linux/amd64 linux/386 linux/arm linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/386 openbsd/amd64 SOURCES := $(shell find . -maxdepth 1 -type f -name "*.go") -ALL_SOURCES = $(shell find . -type f -name '*.go') +ALL_SOURCES = $(shell find . -type f -name '*.go') go.mod docs/* temp = $(subst /, ,$@) os = $(word 1, $(temp)) arch = $(word 2, $(temp)) ext = $(shell if [ "$(os)" = "windows" ]; then echo ".exe"; fi) -.PHONY: all release fmt clean serv $(PLATFORMS) +.PHONY: all release fmt clean serv $(PLATFORMS) docker check deps update-deps all: certgraph @@ -26,6 +26,13 @@ CGO_ENABLED=0 GOOS=$(os) GOARCH=$(arch) go build $(BUILD_FLAGS) -o 'build/bin/$(os)/$(arch)/certgraph$(ext)' $(SOURCES) mkdir -p build/$(GIT_DATE)/; cd build/bin/$(os)/$(arch)/; zip -r ../../../$(GIT_DATE)/certgraph-$(os)-$(arch)-$(GIT_DATE).zip .; cd ../../../ +docker: Dockerfile $(ALL_SOURCES) + docker build -t lanrat/certgraph . + +deps: go.mod + GOPROXY=direct go mod download + GOPROXY=direct go get -u all + fmt: gofmt -s -w -l . @@ -33,7 +40,25 @@ go install $(BUILD_FLAGS) clean: - rm -r certgraph build/ + rm -rf certgraph build/ -serv: - (cd docs; python -m SimpleHTTPServer) +check: | lint check1 check2 + +check1: + golangci-lint run + +check2: + staticcheck -f stylish -checks all ./... + +lint: + golint ./... + +serv: certgraph + ./certgraph --serve 127.0.0.1:8080 + +update-deps: + go get -u + go mod tidy + +test: + go test -v ./... | grep -v "\[no test files\]" \ No newline at end of file diff --git a/README.md b/README.md index e83f643..1635c0f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # CertGraph -### A tool to crawl the graph of certificate Alternate Names + +## A tool to crawl the graph of certificate Alternate Names CertGraph crawls SSL certificates creating a directed graph where each domain is a node and the certificate alternative names for that domain's certificate are the edges to other domain nodes. New domains are printed as they are found. In Detailed mode upon completion the Graph's adjacency list is printed. @@ -10,57 +11,66 @@ [Blog post with more information](https://lanrat.com/certgraph/) ## Usage -``` + +```console Usage of ./certgraph: [OPTION]... HOST... - https://github.com/lanrat/certgraph + https://github.com/lanrat/certgraph OPTIONS: + -apex + for every domain found, add the apex domain of the domain's parent -cdn - include certificates from CDNs + include certificates from CDNs -ct-expired - include expired certificates in certificate transparency search + include expired certificates in certificate transparency search -ct-subdomains - include sub-domains in certificate transparency search + include sub-domains in certificate transparency search -depth uint - maximum BFS depth to go (default 5) + maximum BFS depth to go (default 5) -details - print details about the domains crawled + print details about the domains crawled + -dns + check for DNS records to determine if domain is registered -driver string - driver to use [crtsh, google, http, smtp] (default "http") + driver(s) to use [censys, crtsh, google, http, smtp] (default "http") -json - print the graph as json, can be used for graph in web UI - -ns - check for NS records to determine if domain is registered + print the graph as json, can be used for graph in web UI -parallel uint - number of certificates to retrieve in parallel (default 10) + number of certificates to retrieve in parallel (default 10) + -regex string + regex domains must match to be part of the graph -sanscap int - maximum number of uniq TLD+1 domains in certificate to include, 0 has no limit (default 80) + maximum number of uniq apex domains in certificate to include, 0 has no limit (default 80) -save string - save certs to folder in PEM format + save certs to folder in PEM format + -serve string + address:port to serve html UI on -timeout uint - tcp timeout in seconds (default 10) - -tldplus1 - for every domain found, add tldPlus1 of the domain's parent + tcp timeout in seconds (default 10) + -updatepsl + Update the default Public Suffix List -verbose - verbose logging + verbose logging -version - print version and exit + print version and exit ``` ## Drivers CertGraph has multiple options for querying SSL certificates. The driver is responsible for retrieving the certificates for a given domain. Currently there are the following drivers: - * **http** this is the default driver which works by connecting to the hosts over HTTPS and retrieving the certificates from the SSL connection - - * **smtp** like the *http* driver, but connects over port 25 and issues the *starttls* command to retrieve the certificates from the SSL connection +* **http** this is the default driver which works by connecting to the hosts over HTTPS and retrieving the certificates from the SSL connection - * **crtsh** this driver searches Certificate Transparency logs via [crt.sh](https://crt.sh/). No packets are sent to any of the domains when using this driver +* **smtp** like the *http* driver, but connects over port 25 and issues the *starttls* command to retrieve the certificates from the SSL connection - * **google** this is another Certificate Transparency driver that behaves like *crtsh* but uses the [Google Certificate Transparency Lookup Tool](https://transparencyreport.google.com/https/certificates) +* **censys** this driver searches Certificate Transparency logs via [censys.io](https://search.censys.io/certificates). No packets are sent to any of the domains when using this driver. Requires Censys API keys +* **crtsh** this driver searches Certificate Transparency logs via [crt.sh](https://crt.sh/). No packets are sent to any of the domains when using this driver + +* **google** this is another Certificate Transparency driver that behaves like *crtsh* but uses the [Google Certificate Transparency Lookup Tool](https://transparencyreport.google.com/https/certificates) ## Example -``` + +```console $ ./certgraph -details eff.org eff.org 0 Good 42E3E4605D8BB4608EB64936E2176A98B97EBF2E0F8F93A64A6640713C7D4325 maps.eff.org 1 Good 42E3E4605D8BB4608EB64936E2176A98B97EBF2E0F8F93A64A6640713C7D4325 @@ -69,32 +79,54 @@ atlas.eff.org 1 Good 42E3E4605D8BB4608EB64936E2176A98B97EBF2E0F8F93A64A6640713C7D4325 kittens.eff.org 1 Good 42E3E4605D8BB4608EB64936E2176A98B97EBF2E0F8F93A64A6640713C7D4325 ``` + The above output represents the adjacency list for the graph for the root domain `eff.org`. The adjacency list is in the form: `Node Depth Status Cert-Fingerprint` ## [Releases](https://github.com/lanrat/certgraph/releases) -Precompiled releases will occasionally be uploaded to the [releases github page](https://github.com/lanrat/certgraph/releases). https://github.com/lanrat/certgraph/releases +Pre-compiled releases will occasionally be uploaded to the [releases github page](https://github.com/lanrat/certgraph/releases). [https://github.com/lanrat/certgraph/releases](https://github.com/lanrat/certgraph/releases) -Also available in [BlackArch](https://blackarch.org). +### [Docker](https://hub.docker.com/r/lanrat/certgraph/) + +CertGraph is an automated build on the Docker Hub! + +```console +$ docker run --rm -it lanrat/certgraph example.com +example.com +www.example.net +www.example.org +www.example.com +example.org +example.net +example.edu +www.example.edu +``` + +### Linux Distributions + +* [BlackArch](https://blackarch.org) +* [Kali Linux](https://www.kali.org/) ## Compiling -To compile certgraph you must have a working go 1.11 or newer compiler on your system, as certgraph makes use of go's modules for dependencies. +To compile certgraph you must have a working go 1.16 or newer compiler on your system. To compile for the running system compilation is as easy as running make -``` + +```console certgraph$ make go build -o certgraph certgraph.go ``` Alternatively you can use `go get` to install with this one-liner: -``` + +```console go get -u github.com/lanrat/certgraph ``` ## [Web UI](https://lanrat.github.io/certgraph/) -A web UI is provided in the docs folder and is accessible at the github pages url [https://lanrat.github.io/certgraph/](https://lanrat.github.io/certgraph/). +A web UI is provided in the docs folder and is accessible at the github pages url [https://lanrat.github.io/certgraph/](https://lanrat.github.io/certgraph/), or can be run from the embedded web server by calling `certgraph --serve 127.0.0.1:8080`. The web UI takes the output provided with the `-json` flag. The JSON graph can be sent to the web interface as an uploaded file, remote URL, or as the query string using the data variable. @@ -111,3 +143,23 @@ [![whitehouse.gov graph](https://cloud.githubusercontent.com/assets/164192/20861407/4775ff26-b944-11e6-888c-4d93e3333494.png)](https://lanrat.github.io/certgraph/?data=https://gist.githubusercontent.com/lanrat/96c47dfee0faaaad633cc830b7e3b997/raw/3c79fed837cb3202e220de21d2a8eb128f4bbd9f/whitehouse.json) +## BygoneSSL detection + +### Self Detection + +CertGraph can be used to detect [BygoneSSL](https://insecure.design) DoS with the following options. CT-DRIVER can be any Certificate Transparency capable driver. +Provide all known input domains you own. If any domains you do not own are printed, then you are vulnerable. + +```console +certgraph -depth 1 -driver CT-DRIVER -ct-subdomains -cdn -apex [DOMAIN]... +``` + +### Bug Bounty + +If you want to find a vulnerable site that has a bug bounty, certgraph can be used with the following options and any driver. But you will have better luck with a non Certificate Transparency driver to ensure that the certificates in question are actually in use + +```console +certgraph -cdn -dns -apex [DOMAIN]... +``` + +And domains that print `* Missing DNS for` have vulnerable certificates that should be rotated. diff --git a/certgraph.go b/certgraph.go index f45d0ce..58219f1 100644 --- a/certgraph.go +++ b/certgraph.go @@ -1,33 +1,50 @@ package main import ( + "embed" "encoding/json" "flag" "fmt" "net/url" "os" + "regexp" "strings" "sync" "time" "github.com/lanrat/certgraph/dns" "github.com/lanrat/certgraph/driver" + "github.com/lanrat/certgraph/driver/censys" "github.com/lanrat/certgraph/driver/crtsh" "github.com/lanrat/certgraph/driver/google" "github.com/lanrat/certgraph/driver/http" + "github.com/lanrat/certgraph/driver/multi" "github.com/lanrat/certgraph/driver/smtp" "github.com/lanrat/certgraph/graph" + "github.com/lanrat/certgraph/web" ) +// version vars var ( gitDate = "none" gitHash = "master" certGraph = graph.NewCertGraph() ) +// temp flag vars +var ( + timeoutSeconds uint + regexString string +) + +// webContent holds our static web server content. +//go:embed docs/* +var webContent embed.FS + var certDriver driver.Driver // config & flags +// TODO move driver options to own struct var config struct { timeout time.Duration verbose bool @@ -41,42 +58,63 @@ includeCTExpired bool cdn bool maxSANsSize int - tldPlus1 bool + apex bool updatePSL bool checkDNS bool printVersion bool + serve string + regex *regexp.Regexp } func init() { - var timeoutSeconds uint flag.BoolVar(&config.printVersion, "version", false, "print version and exit") flag.UintVar(&timeoutSeconds, "timeout", 10, "tcp timeout in seconds") flag.BoolVar(&config.verbose, "verbose", false, "verbose logging") - flag.StringVar(&config.driver, "driver", "http", fmt.Sprintf("driver to use [%s]", strings.Join(driver.Drivers, ", "))) + flag.StringVar(&config.driver, "driver", "http", fmt.Sprintf("driver(s) to use [%s]", strings.Join(driver.Drivers, ", "))) flag.BoolVar(&config.includeCTSubdomains, "ct-subdomains", false, "include sub-domains in certificate transparency search") flag.BoolVar(&config.includeCTExpired, "ct-expired", false, "include expired certificates in certificate transparency search") - flag.IntVar(&config.maxSANsSize, "sanscap", 80, "maximum number of uniq TLD+1 domains in certificate to include, 0 has no limit") + flag.IntVar(&config.maxSANsSize, "sanscap", 80, "maximum number of uniq apex domains in certificate to include, 0 has no limit") flag.BoolVar(&config.cdn, "cdn", false, "include certificates from CDNs") flag.BoolVar(&config.checkDNS, "dns", false, "check for DNS records to determine if domain is registered") - flag.BoolVar(&config.tldPlus1, "tldplus1", false, "for every domain found, add tldPlus1 of the domain's parent") + flag.BoolVar(&config.apex, "apex", false, "for every domain found, add the apex domain of the domain's parent") flag.BoolVar(&config.updatePSL, "updatepsl", false, "Update the default Public Suffix List") flag.UintVar(&config.maxDepth, "depth", 5, "maximum BFS depth to go") flag.UintVar(&config.parallel, "parallel", 10, "number of certificates to retrieve in parallel") flag.BoolVar(&config.details, "details", false, "print details about the domains crawled") flag.BoolVar(&config.printJSON, "json", false, "print the graph as json, can be used for graph in web UI") flag.StringVar(&config.savePath, "save", "", "save certs to folder in PEM format") + flag.StringVar(&config.serve, "serve", "", "address:port to serve html UI on") + flag.StringVar(®exString, "regex", "", "regex domains must match to be part of the graph") + flag.Usage = func() { fmt.Fprintf(os.Stderr, "Usage of %s: [OPTION]... HOST...\n\thttps://github.com/lanrat/certgraph\nOPTIONS:\n", os.Args[0]) flag.PrintDefaults() } +} + +func main() { flag.Parse() config.timeout = time.Duration(timeoutSeconds) * time.Second -} - -func main() { + var err error + // check for version flag if config.printVersion { fmt.Println(version()) + return + } + + // check for regex + if len(regexString) > 0 { + config.regex, err = regexp.Compile(regexString) + if err != nil { + e(err) + return + } + } + + if len(config.serve) > 0 { + err = web.Serve(config.serve, webContent) + e(err) return } @@ -95,7 +133,7 @@ // update the public suffix list if required if config.updatePSL { - err := dns.UpdatePublicSuffixList(config.timeout) + err = dns.UpdatePublicSuffixList(config.timeout) if err != nil { e(err) return @@ -108,18 +146,18 @@ d := strings.ToLower(domain) if len(d) > 0 { startDomains = append(startDomains, cleanInput(d)) - if config.tldPlus1 { - tldPlus1, err := dns.TLDPlus1(domain) + if config.apex { + apexDomain, err := dns.ApexDomain(domain) if err != nil { continue } - startDomains = append(startDomains, tldPlus1) + startDomains = append(startDomains, apexDomain) } } } // set driver - err := setDriver(config.driver) + certDriver, err = setDriver(config.driver) if err != nil { fmt.Fprintln(os.Stderr, err) return @@ -146,23 +184,42 @@ v("Graph Depth:", certGraph.DomainDepth()) } -// setDriver sets the driver variable for the provided driver string and does any necessary driver prep work +func setDriver(name string) (driver.Driver, error) { + if strings.Contains(name, ",") { + names := strings.Split(name, ",") + drivers := make([]driver.Driver, 0, len(names)) + for _, driverName := range names { + d, err := getDriverSingle(driverName) + if err != nil { + return nil, err + } + drivers = append(drivers, d) + } + return multi.Driver(drivers), nil + } + return getDriverSingle(name) +} + +// getDriverSingle sets the driver variable for the provided driver string and does any necessary driver prep work // TODO make config generic and move this to driver module -func setDriver(driver string) error { +func getDriverSingle(name string) (driver.Driver, error) { var err error - switch driver { + var d driver.Driver + switch name { case "google": - certDriver, err = google.Driver(50, config.savePath, config.includeCTSubdomains, config.includeCTExpired) + d, err = google.Driver(50, config.savePath, config.includeCTSubdomains, config.includeCTExpired) case "crtsh": - certDriver, err = crtsh.Driver(1000, config.timeout, config.savePath, config.includeCTSubdomains, config.includeCTExpired) + d, err = crtsh.Driver(1000, config.timeout, config.savePath, config.includeCTSubdomains, config.includeCTExpired) case "http": - certDriver, err = http.Driver(config.timeout, config.savePath) + d, err = http.Driver(config.timeout, config.savePath) case "smtp": - certDriver, err = smtp.Driver(config.timeout, config.savePath) + d, err = smtp.Driver(config.timeout, config.savePath) + case "censys": + d, err = censys.Driver(config.savePath, config.includeCTSubdomains, config.includeCTExpired) default: - return fmt.Errorf("Unknown driver name: %s", config.driver) - } - return err + return nil, fmt.Errorf("unknown driver name: %s", config.driver) + } + return d, err } // verbose logging @@ -237,6 +294,13 @@ <-threadPass defer func() { threadPass <- true }() + // regex match check + if config.regex != nil && !config.regex.MatchString(domainNode.Domain) { + // skip domain that does not match regex + v("domain does not match regex, skipping :", domainNode.Domain) + return + } + // operate on the node v("Visiting", domainNode.Depth, domainNode.Domain) visit(domainNode) @@ -244,13 +308,13 @@ for _, neighbor := range certGraph.GetDomainNeighbors(domainNode.Domain, config.cdn, config.maxSANsSize) { wg.Add(1) domainNodeInputChan <- graph.NewDomainNode(neighbor, domainNode.Depth+1) - if config.tldPlus1 { - tldPlus1, err := dns.TLDPlus1(neighbor) + if config.apex { + apexDomain, err := dns.ApexDomain(neighbor) if err != nil { continue } wg.Add(1) - domainNodeInputChan <- graph.NewDomainNode(tldPlus1, domainNode.Depth+1) + domainNodeInputChan <- graph.NewDomainNode(apexDomain, domainNode.Depth+1) } } }(domainNode) @@ -341,7 +405,7 @@ domainNode.AddCertFingerprint(certNode.Fingerprint, certDriver.GetName()) } - // we dont process any other certificates returned, they will be collected + // we don't process any other certificates returned, they will be collected // when we process the related domains } @@ -354,7 +418,7 @@ if config.checkDNS && !domainNode.HasDNS { // TODO print this in a better way // TODO for debugging - realDomain, _ := dns.TLDPlus1(domainNode.Domain) + realDomain, _ := dns.ApexDomain(domainNode.Domain) fmt.Fprintf(os.Stdout, "* Missing DNS for: %s\n", realDomain) } @@ -385,6 +449,7 @@ options["sanscap"] = config.maxSANsSize options["cdn"] = config.cdn options["timeout"] = config.timeout + options["regex"] = regexString data["options"] = options return data } diff --git a/dns/ns.go b/dns/ns.go index 5cd2635..0214684 100644 --- a/dns/ns.go +++ b/dns/ns.go @@ -1,3 +1,4 @@ +// Package dns adds utility functions for performing dns queries package dns import ( @@ -26,7 +27,7 @@ } // HasRecords does NS, CNAME, A, and AAAA lookups with a timeout -// returns error when no NS found, does not use TLDPlus1 +// returns error when no NS found, does not use alexDomain func HasRecords(domain string, timeout time.Duration) (bool, error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -68,10 +69,10 @@ return false, nil } -// HasRecordsCache returns true if the domain has no DNS records (at the tldplus1 level) +// HasRecordsCache returns true if the domain has no DNS records (at the apex domain level) // uses a cache to store results to prevent lots of DNS lookups func HasRecordsCache(domain string, timeout time.Duration) (bool, error) { - domain, err := TLDPlus1(domain) + domain, err := ApexDomain(domain) if err != nil { return false, err } diff --git a/dns/publicsuffix.go b/dns/publicsuffix.go index 86c9ef3..6183a4e 100644 --- a/dns/publicsuffix.go +++ b/dns/publicsuffix.go @@ -14,7 +14,6 @@ } suffixListURL = "https://publicsuffix.org/list/public_suffix_list.dat" suffixList = publicsuffix.DefaultList - nsCache = make(map[string]bool) ) // UpdatePublicSuffixList gets a new copy of the public suffix list from the internat and updates the built in copy with the new rules @@ -31,12 +30,12 @@ } defer resp.Body.Close() newSuffixList := publicsuffix.NewList() - newSuffixList.Load(resp.Body, suffixListParseOptions) + _, err = newSuffixList.Load(resp.Body, suffixListParseOptions) suffixList = newSuffixList return err } -// TLDPlus1 returns TLD+1 of domain -func TLDPlus1(domain string) (string, error) { +// ApexDomain returns TLD+1 of domain +func ApexDomain(domain string) (string, error) { return publicsuffix.DomainFromListWithOptions(suffixList, domain, suffixListFindOptions) } diff --git a/docs/index.html b/docs/index.html index c227b54..e63ac51 100644 --- a/docs/index.html +++ b/docs/index.html @@ -17,7 +17,6 @@ stroke: #333; stroke-width: 1.5px; } - .upload-drop-zone { height: 200px; border-width: 2px; diff --git a/driver/censys/censys.go b/driver/censys/censys.go new file mode 100644 index 0000000..41ea071 --- /dev/null +++ b/driver/censys/censys.go @@ -0,0 +1,242 @@ +package censys + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "path" + "time" + + "github.com/lanrat/certgraph/driver" + "github.com/lanrat/certgraph/fingerprint" + "github.com/lanrat/certgraph/status" +) + +const driverName = "censys" + +var debug = false + +// TODO support rate limits & pagnation + +var ( + defaultHTTPClient = &http.Client{} + + appID = flag.String("censys-appid", "", "censys API AppID") + secret = flag.String("censys-secret", "", "censys API Secret") +) + +func init() { + driver.AddDriver(driverName) +} + +type censys struct { + appID string + secret string + save bool + savePath string + includeSubdomains bool + includeExpired bool +} + +type censysCertDriver struct { + host string + fingerprints driver.FingerprintMap + driver *censys +} + +func (c *censysCertDriver) GetFingerprints() (driver.FingerprintMap, error) { + return c.fingerprints, nil +} + +func (c *censysCertDriver) GetStatus() status.Map { + return status.NewMap(c.host, status.New(status.CT)) +} + +func (c *censysCertDriver) GetRelated() ([]string, error) { + return make([]string, 0), nil +} + +func (c *censysCertDriver) QueryCert(fp fingerprint.Fingerprint) (*driver.CertResult, error) { + return c.driver.QueryCert(fp) +} + +// TODO support pagnation +func domainSearchParam(domain string, includeExpired, includeSubdomain bool) certSearchParam { + var s certSearchParam + if includeSubdomain { + s.Query = fmt.Sprintf("(parsed.names: %s )", domain) + } else { + s.Query = fmt.Sprintf("(parsed.names.raw: %s)", domain) + } + if !includeExpired { + dateStr := time.Now().Format("2006-01-02") // YYYY-MM-DD + expQuery := fmt.Sprintf(" AND ((parsed.validity.end: [%s TO *]) AND (parsed.validity.start: [* TO %s]))", dateStr, dateStr) + s.Query = s.Query + expQuery + } + s.Page = 1 + s.Flatten = true + s.Fields = []string{"parsed.fingerprint_sha256", "parsed.names"} + return s +} + +// Driver creates a new CT driver for censys +func Driver(savePath string, includeSubdomains, includeExpired bool) (driver.Driver, error) { + if *appID == "" || *secret == "" { + return nil, fmt.Errorf("censys requires an appID and secret to run") + } + d := new(censys) + d.appID = *appID + d.secret = *secret + d.savePath = savePath + d.includeSubdomains = includeSubdomains + d.includeExpired = includeExpired + return d, nil +} + +func (d *censys) GetName() string { + return driverName +} + +func (d *censys) request(method, url string, request io.Reader) (*http.Response, error) { + totalTrys := 3 + var err error + var req *http.Request + var resp *http.Response + for try := 1; try <= totalTrys; try++ { + req, err = http.NewRequest(method, url, request) + if err != nil { + return nil, err + } + if request != nil { + req.Header.Add("Content-Type", "application/json") + } + req.Header.Add("Accept", "application/json") + req.SetBasicAuth(d.appID, d.secret) + + resp, err = defaultHTTPClient.Do(req) + if err != nil { + err = fmt.Errorf("error on request [%d/%d] %s, got error %w: %+v", try, totalTrys, url, err, resp) + } else { + return resp, nil + } + + // sleep only if we will try again + if try < totalTrys { + time.Sleep(time.Second * 10) + } + } + return resp, err +} + +// jsonRequest performes a request to the API endpoint sending and receiving JSON objects +func (d *censys) jsonRequest(method, url string, request, response interface{}) error { + var payloadReader io.Reader + if request != nil { + jsonPayload, err := json.Marshal(request) + if err != nil { + return err + } + payloadReader = bytes.NewReader(jsonPayload) + } + + if debug { + log.Printf("DEBUG: request to %s %s", method, url) + if request != nil { + prettyJSONBytes, _ := json.MarshalIndent(request, "", "\t") + log.Printf("request payload:\n%s\n", string(prettyJSONBytes)) + } + } + + resp, err := d.request(method, url, payloadReader) + if err != nil { + return err + } + defer resp.Body.Close() + + // got an error, decode it + if resp.StatusCode != http.StatusOK { + var errorResp errorResponse + err := fmt.Errorf("error on request %s, got Status %s %s", url, resp.Status, http.StatusText(resp.StatusCode)) + jsonError := json.NewDecoder(resp.Body).Decode(&errorResp) + if jsonError != nil { + return fmt.Errorf("error decoding json %w on errord request: %s", jsonError, err.Error()) + } + return fmt.Errorf("%w, HTTPStatus: %d Message: %q", err, errorResp.ErrorCode, errorResp.Error) + } + + if response != nil { + err = json.NewDecoder(resp.Body).Decode(&response) + if err != nil { + return err + } + if debug { + prettyJSONBytes, _ := json.MarshalIndent(response, "", "\t") + log.Printf("response payload:\n%s\n", string(prettyJSONBytes)) + } + } + + return nil +} + +func (d *censys) QueryDomain(domain string) (driver.Result, error) { + results := &censysCertDriver{ + host: domain, + fingerprints: make(driver.FingerprintMap), + driver: d, + } + params := domainSearchParam(domain, d.includeExpired, d.includeSubdomains) + url := "https://search.censys.io/api/v1/search/certificates" + var resp certSearchResponse + err := d.jsonRequest(http.MethodPost, url, params, &resp) + if err != nil { + return results, err + } + + for _, r := range resp.Results { + fp := fingerprint.FromHexHash(r.Fingerprint) + results.fingerprints.Add(domain, fp) + } + + if debug { + log.Printf("censys: got %d results for %s.", len(resp.Results), domain) + } + + return results, nil +} + +func (d *censys) QueryCert(fp fingerprint.Fingerprint) (*driver.CertResult, error) { + certNode := new(driver.CertResult) + certNode.Fingerprint = fp + certNode.Domains = make([]string, 0, 5) + + url := fmt.Sprintf("https://search.censys.io/api/v1/view/certificates/%s", fp.HexString()) + var resp certViewResponse + err := d.jsonRequest(http.MethodGet, url, nil, &resp) + if err != nil { + return certNode, err + } + + if debug { + log.Printf("DEBUG QueryCert(%s): %v", fp.HexString(), resp.Parsed.Names) + } + + certNode.Domains = append(certNode.Domains, resp.Parsed.Names...) + + if d.save { + rawCert, err := base64.StdEncoding.DecodeString(resp.Raw) + if err != nil { + return certNode, err + } + err = driver.RawCertToPEMFile(rawCert, path.Join(d.savePath, fp.HexString())+".pem") + if err != nil { + return certNode, err + } + } + + return certNode, nil +} diff --git a/driver/censys/schemas.go b/driver/censys/schemas.go new file mode 100644 index 0000000..ecd23d3 --- /dev/null +++ b/driver/censys/schemas.go @@ -0,0 +1,215 @@ +package censys + +import "time" + +type certSearchParam struct { + Query string `json:"query"` + Page uint `json:"page"` + Fields []string `json:"fields"` + Flatten bool `json:"flatten"` +} + +type certSearchResponse struct { + Status string `json:"status"` + Metadata struct { + Query string `json:"query"` + Count uint `json:"count"` + BackendTime uint `json:"backend_time"` + Page uint `json:"page"` + Pages uint `json:"pages"` + } `json:"metadata"` + Results []struct { + Names []string `json:"parsed.names"` + Fingerprint string `json:"parsed.fingerprint_sha256"` + } `json:"results"` +} + +type certViewResponse struct { + Raw string `json:"raw"` + FingerprintSha256 string `json:"fingerprint_sha256"` + ParentSpkiSubjectFingerprint string `json:"parent_spki_subject_fingerprint"` + Metadata struct { + PostProcessedAt time.Time `json:"post_processed_at"` + PostProcessed bool `json:"post_processed"` + Source string `json:"source"` + ParseVersion int `json:"parse_version"` + ParseStatus string `json:"parse_status"` + AddedAt time.Time `json:"added_at"` + UpdatedAt time.Time `json:"updated_at"` + SeenInScan bool `json:"seen_in_scan"` + } `json:"metadata"` + Ct struct { + GoogleXenon2022 struct { + Index int `json:"index"` + CtToCensysAt time.Time `json:"ct_to_censys_at"` + AddedToCtAt time.Time `json:"added_to_ct_at"` + } `json:"google_xenon_2022"` + } `json:"ct"` + Parsed struct { + Version int `json:"version"` + SerialNumber string `json:"serial_number"` + SignatureAlgorithm struct { + Name string `json:"name"` + Oid string `json:"oid"` + } `json:"signature_algorithm"` + Issuer struct { + CommonName []string `json:"common_name"` + Country []string `json:"country"` + Organization []string `json:"organization"` + } `json:"issuer"` + IssuerDn string `json:"issuer_dn"` + Validity struct { + Start time.Time `json:"start"` + End time.Time `json:"end"` + Length int `json:"length"` + } `json:"validity"` + Subject struct { + CommonName []string `json:"common_name"` + } `json:"subject"` + SubjectDn string `json:"subject_dn"` + SubjectKeyInfo struct { + KeyAlgorithm struct { + Name string `json:"name"` + } `json:"key_algorithm"` + EcdsaPublicKey struct { + B string `json:"b"` + Curve string `json:"curve"` + Gx string `json:"gx"` + Gy string `json:"gy"` + Length int `json:"length"` + N string `json:"n"` + P string `json:"p"` + Pub string `json:"pub"` + X string `json:"x"` + Y string `json:"y"` + } `json:"ecdsa_public_key"` + FingerprintSha256 string `json:"fingerprint_sha256"` + } `json:"subject_key_info"` + Extensions struct { + KeyUsage struct { + DigitalSignature bool `json:"digital_signature"` + Value int `json:"value"` + } `json:"key_usage"` + BasicConstraints struct { + IsCa bool `json:"is_ca"` + } `json:"basic_constraints"` + SubjectAltName struct { + DNSNames []string `json:"dns_names"` + } `json:"subject_alt_name"` + AuthorityKeyID string `json:"authority_key_id"` + SubjectKeyID string `json:"subject_key_id"` + ExtendedKeyUsage struct { + ServerAuth bool `json:"server_auth"` + ClientAuth bool `json:"client_auth"` + } `json:"extended_key_usage"` + CertificatePolicies []struct { + ID string `json:"id"` + Cps []string `json:"cps,omitempty"` + } `json:"certificate_policies"` + AuthorityInfoAccess struct { + OcspUrls []string `json:"ocsp_urls"` + IssuerUrls []string `json:"issuer_urls"` + } `json:"authority_info_access"` + SignedCertificateTimestamps []struct { + Version int `json:"version"` + LogID string `json:"log_id"` + Timestamp int `json:"timestamp"` + Signature string `json:"signature"` + } `json:"signed_certificate_timestamps"` + } `json:"extensions"` + Signature struct { + SignatureAlgorithm struct { + Name string `json:"name"` + Oid string `json:"oid"` + } `json:"signature_algorithm"` + Value string `json:"value"` + Valid bool `json:"valid"` + SelfSigned bool `json:"self_signed"` + } `json:"signature"` + FingerprintMd5 string `json:"fingerprint_md5"` + FingerprintSha1 string `json:"fingerprint_sha1"` + FingerprintSha256 string `json:"fingerprint_sha256"` + TbsNoctFingerprint string `json:"tbs_noct_fingerprint"` + SpkiSubjectFingerprint string `json:"spki_subject_fingerprint"` + TbsFingerprint string `json:"tbs_fingerprint"` + ValidationLevel string `json:"validation_level"` + Names []string `json:"names"` + Redacted bool `json:"redacted"` + } `json:"parsed"` + Tags []string `json:"tags"` + Validation struct { + Nss struct { + Blacklisted bool `json:"blacklisted"` + HadTrustedPath bool `json:"had_trusted_path"` + InRevocationSet bool `json:"in_revocation_set"` + TrustedPath bool `json:"trusted_path"` + WasValid bool `json:"was_valid"` + Whitelisted bool `json:"whitelisted"` + Paths [][]string `json:"paths"` + Parents []string `json:"parents"` + Type string `json:"type"` + Valid bool `json:"valid"` + } `json:"nss"` + Microsoft struct { + Blacklisted bool `json:"blacklisted"` + HadTrustedPath bool `json:"had_trusted_path"` + InRevocationSet bool `json:"in_revocation_set"` + TrustedPath bool `json:"trusted_path"` + WasValid bool `json:"was_valid"` + Whitelisted bool `json:"whitelisted"` + Paths [][]string `json:"paths"` + Parents []string `json:"parents"` + Type string `json:"type"` + Valid bool `json:"valid"` + } `json:"microsoft"` + Apple struct { + Blacklisted bool `json:"blacklisted"` + HadTrustedPath bool `json:"had_trusted_path"` + InRevocationSet bool `json:"in_revocation_set"` + TrustedPath bool `json:"trusted_path"` + WasValid bool `json:"was_valid"` + Whitelisted bool `json:"whitelisted"` + Paths [][]string `json:"paths"` + Parents []string `json:"parents"` + Type string `json:"type"` + Valid bool `json:"valid"` + } `json:"apple"` + Revoked bool `json:"revoked"` + GoogleCtPrimary struct { + Blacklisted bool `json:"blacklisted"` + HadTrustedPath bool `json:"had_trusted_path"` + InRevocationSet bool `json:"in_revocation_set"` + TrustedPath bool `json:"trusted_path"` + WasValid bool `json:"was_valid"` + Whitelisted bool `json:"whitelisted"` + Paths [][]string `json:"paths"` + Parents []string `json:"parents"` + Type string `json:"type"` + Valid bool `json:"valid"` + } `json:"google_ct_primary"` + OcspRevocation struct { + NextUpdate time.Time `json:"next_update"` + Revoked bool `json:"revoked"` + } `json:"ocsp_revocation"` + CrlRevocation struct { + Revoked bool `json:"revoked"` + } `json:"crl_revocation"` + CrlError string `json:"crl_error"` + } `json:"validation"` + Zlint struct { + NoticesPresent bool `json:"notices_present"` + WarningsPresent bool `json:"warnings_present"` + ErrorsPresent bool `json:"errors_present"` + FatalsPresent bool `json:"fatals_present"` + Lints struct { + NSubjectCommonNameIncluded bool `json:"n_subject_common_name_included"` + } `json:"lints"` + Version int `json:"version"` + } `json:"zlint"` + Precert bool `json:"precert"` +} + +type errorResponse struct { + Error string `json:"error"` + ErrorCode int `json:"error_code"` +} diff --git a/driver/crtsh/crtsh.go b/driver/crtsh/crtsh.go index b53c038..9774cf1 100644 --- a/driver/crtsh/crtsh.go +++ b/driver/crtsh/crtsh.go @@ -1,19 +1,17 @@ +// Package crtsh implements an unofficial API client for Comodo's +// Certificate Transparency search +// https://crt.sh/ +// +// As the API is unofficial and has been reverse engineered it may stop working +// at any time and comes with no guarantees. +// view SQL excample: https://crt.sh/?showSQL=Y&exclude=expired&q= +// package crtsh - -/* - * This file implements an unofficial API client for Comodo's - * Certificate Transparency search - * https://crt.sh/ - * - * As the API is unofficial and has been reverse engineered it may stop working - * at any time and comes with no guarantees. - */ - -// TODO running in verbose gives error: pq: unnamed prepared statement does not exist import ( "database/sql" "fmt" + "log" "path" "time" @@ -23,8 +21,10 @@ _ "github.com/lib/pq" // portgresql ) -const connStr = "postgresql://guest@crt.sh/certwatch?sslmode=disable" +const connStr = "postgresql://guest@crt.sh/certwatch?sslmode=disable&fallback_application_name=certgraph&binary_parameters=yes" const driverName = "crtsh" + +const debug = false func init() { driver.AddDriver(driverName) @@ -76,8 +76,11 @@ } d.db, err = sql.Open("postgres", connStr) - - d.setSQLTimeout(d.timeout.Seconds()) + if err != nil { + return nil, err + } + + err = d.setSQLTimeout(d.timeout.Seconds()) return d, err } @@ -98,45 +101,43 @@ driver: d, } - queryStr := "" - - if d.includeSubdomains { - if d.includeExpired { - queryStr = `SELECT digest(certificate.certificate, 'sha256') sha256 - FROM certificate_identity, certificate - WHERE certificate.id = certificate_identity.certificate_id - AND (reverse(lower(certificate_identity.name_value)) LIKE reverse(lower('%%.'||$1)) - OR reverse(lower(certificate_identity.name_value)) LIKE reverse(lower($1))) - LIMIT $2` - } else { - queryStr = `SELECT digest(certificate.certificate, 'sha256') sha256 - FROM certificate_identity, certificate - WHERE certificate.id = certificate_identity.certificate_id - AND x509_notAfter(certificate.certificate) > statement_timestamp() - AND (reverse(lower(certificate_identity.name_value)) LIKE reverse(lower('%%.'||$1)) - OR reverse(lower(certificate_identity.name_value)) LIKE reverse(lower($1))) - LIMIT $2` - } - } else { - if d.includeExpired { - queryStr = `SELECT digest(certificate.certificate, 'sha256') sha256 - FROM certificate_identity, certificate - WHERE certificate.id = certificate_identity.certificate_id - AND reverse(lower(certificate_identity.name_value)) LIKE reverse(lower($1)) - LIMIT $2` - } else { - queryStr = `SELECT digest(certificate.certificate, 'sha256') sha256 - FROM certificate_identity, certificate - WHERE certificate.id = certificate_identity.certificate_id - AND x509_notAfter(certificate.certificate) > statement_timestamp() - AND reverse(lower(certificate_identity.name_value)) LIKE reverse(lower($1)) - LIMIT $2` - } - } - - if d.includeSubdomains { - domain = fmt.Sprintf("%%.%s", domain) - } + queryStr := `WITH myconstants (include_expired, include_subdomains) as ( + values ($1::bool, $2::bool) + ), + ci AS ( + SELECT digest(sub.CERTIFICATE, 'sha256') sha256, -- added + min(sub.CERTIFICATE_ID) ID, + min(sub.ISSUER_CA_ID) ISSUER_CA_ID, + array_agg(DISTINCT sub.NAME_VALUE) NAME_VALUES + FROM (SELECT * + FROM certificate_and_identities cai, myconstants + WHERE plainto_tsquery('certwatch', $4) @@ identities(cai.CERTIFICATE) + AND ( + -- domain only + (NOT myconstants.include_subdomains AND cai.NAME_VALUE ILIKE ($4)) + OR + -- include sub-domains + (myconstants.include_subdomains AND (cai.NAME_VALUE ILIKE ($4) OR cai.NAME_VALUE ILIKE ('%.' || $4))) + ) + AND ( + -- added + cai.NAME_TYPE = '2.5.4.3' -- commonName + OR + cai.NAME_TYPE = 'san:dNSName' -- dNSName + ) + AND + -- include expired? + (myconstants.include_expired OR (coalesce(x509_notAfter(cai.CERTIFICATE), 'infinity'::timestamp) >= date_trunc('year', now() AT TIME ZONE 'UTC') + AND x509_notAfter(cai.CERTIFICATE) >= now() AT TIME ZONE 'UTC')) + LIMIT $3 + ) sub + GROUP BY sub.CERTIFICATE + ) + SELECT + ci.sha256 -- added + --array_to_string(ci.name_values, chr(10)) name_value, + --ci.id id + FROM ci;` try := 0 var err error @@ -144,9 +145,15 @@ for try < 5 { // this is a hack while crt.sh gets there stuff togeather try++ - rows, err = d.db.Query(queryStr, domain, d.queryLimit) + if debug { + log.Printf("QueryDomain try %d: %s", try, queryStr) + } + rows, err = d.db.Query(queryStr, d.includeExpired, d.includeSubdomains, d.queryLimit, domain) if err == nil { break + } + if debug { + log.Printf("crtsh pq error on domain %q: %s", domain, err.Error()) } } /*if try > 1 { @@ -165,6 +172,10 @@ results.fingerprints.Add(domain, fingerprint.FromHashBytes(hash)) } + if debug { + log.Printf("crtsh: got %d results for %s.", len(results.fingerprints[domain]), domain) + } + return results, nil } @@ -173,11 +184,7 @@ certNode.Fingerprint = fp certNode.Domains = make([]string, 0, 5) - queryStr := `SELECT DISTINCT certificate_identity.name_value - FROM certificate, certificate_identity - WHERE certificate.id = certificate_identity.certificate_id - AND certificate_identity.name_type in ('dNSName', 'commonName') - AND digest(certificate.certificate, 'sha256') = $1` + queryStr := `SELECT DISTINCT name_value FROM certificate_and_identities WHERE digest(certificate, 'sha256') = $1;` try := 0 var err error @@ -199,22 +206,26 @@ for rows.Next() { var domain string - rows.Scan(&domain) + err = rows.Scan(&domain) + if err != nil { + return nil, err + } certNode.Domains = append(certNode.Domains, domain) } if d.save { var rawCert []byte - queryStr = `SELECT certificate.certificate - FROM certificate - WHERE digest(certificate.certificate, 'sha256') = $1` + queryStr = `SELECT certificate FORM certificate_and_identities WHERE digest(certificate, 'sha256') = $1;` row := d.db.QueryRow(queryStr, fp[:]) err = row.Scan(&rawCert) if err != nil { return certNode, err } - driver.RawCertToPEMFile(rawCert, path.Join(d.savePath, fp.HexString())+".pem") + err = driver.RawCertToPEMFile(rawCert, path.Join(d.savePath, fp.HexString())+".pem") + if err != nil { + return certNode, err + } } return certNode, nil diff --git a/driver/driver.go b/driver/driver.go index 5b4c819..95a198d 100644 --- a/driver/driver.go +++ b/driver/driver.go @@ -1,3 +1,4 @@ +// Package driver exposes interfaces and types certgraph drivers must implement package driver import ( @@ -8,6 +9,8 @@ "github.com/lanrat/certgraph/fingerprint" "github.com/lanrat/certgraph/status" ) + +// TODO add context instead of timeout on all requests // Drivers contains all the drivers that have been registered var Drivers []string @@ -66,7 +69,7 @@ certResult := new(CertResult) // generate Fingerprint - certResult.Fingerprint = fingerprint.FromBytes(cert.Raw) + certResult.Fingerprint = fingerprint.FromRawCertBytes(cert.Raw) // domains // used to ensure uniq entries in domains array diff --git a/driver/google/google.go b/driver/google/google.go index 321e421..7af1505 100644 --- a/driver/google/google.go +++ b/driver/google/google.go @@ -1,13 +1,11 @@ +// Package google file implements an unofficial API client for Google's +// Certificate Transparency search +// https://transparencyreport.google.com/https/certificates +// +// As the API is unofficial and has been reverse engineered it may stop working +// at any time and comes with no guarantees. +// package google - -/* - * This file implements an unofficial API client for Google's - * Certificate Transparency search - * https://transparencyreport.google.com/https/certificates - * - * As the API is unofficial and has been reverse engineered it may stop working - * at any time and comes with no guarantees. - */ import ( "encoding/json" @@ -30,9 +28,13 @@ } // Base URLs for Google's CT API -const searchURL1 = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch?include_expired=false&include_subdomains=false&domain=example.com" -const searchURL2 = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch/page?p=DEADBEEF" -const certURL = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certbyhash?hash=DEADBEEF" +const ( + searchURL1 = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch?include_expired=false&include_subdomains=false&domain=example.com" + searchURL2 = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch/page?p=DEADBEEF" + certURL = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certbyhash?hash=DEADBEEF" + //summaryURL is not currently used + //summaryURL = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/summary" +) type googleCT struct { maxPages float64 // this is a float because that is the type automatically decoded from the JSON response @@ -158,7 +160,7 @@ foundCerts := raw[0][1].([]interface{}) for _, foundCert := range foundCerts { certHash := foundCert.([]interface{})[5].(string) - certFP := fingerprint.FromB64(certHash) + certFP := fingerprint.FromB64Hash(certHash) results.fingerprints.Add(domain, certFP) } //fmt.Println("Page:", pageInfo[3]) diff --git a/driver/http/http.go b/driver/http/http.go index 591f566..376a3fc 100644 --- a/driver/http/http.go +++ b/driver/http/http.go @@ -1,3 +1,4 @@ +// Package http implements a certgraph driver for obtaining SSL certificates over https package http import ( @@ -53,7 +54,7 @@ if found { return cert, nil } - return nil, fmt.Errorf("Certificate with Fingerprint %s not found", fp.HexString()) + return nil, fmt.Errorf("certificate with Fingerprint %s not found", fp.HexString()) } // Driver creates a new SSL driver for HTTP Connections @@ -151,7 +152,7 @@ // save if c.parent.save && len(connState.PeerCertificates) > 0 { - driver.CertsToPEMFile(connState.PeerCertificates, path.Join(c.parent.savePath, certResult.Fingerprint.HexString())+".pem") + err = driver.CertsToPEMFile(connState.PeerCertificates, path.Join(c.parent.savePath, certResult.Fingerprint.HexString())+".pem") } return conn, err diff --git a/driver/multi/multi_driver.go b/driver/multi/multi_driver.go new file mode 100644 index 0000000..8dacdeb --- /dev/null +++ b/driver/multi/multi_driver.go @@ -0,0 +1,131 @@ +// Package multi exposes a generic driver interface allowing you to merge the results of multiple other drivers +package multi + +import ( + "errors" + "fmt" + "strings" + "sync" + + "github.com/lanrat/certgraph/driver" + "github.com/lanrat/certgraph/fingerprint" + "github.com/lanrat/certgraph/status" + "golang.org/x/sync/errgroup" +) + +type multiDriver struct { + drivers []driver.Driver +} + +// Driver returns a new instance of multi driver for the provided drivers +func Driver(drivers []driver.Driver) driver.Driver { + md := new(multiDriver) + md.drivers = drivers + return md +} + +func (d *multiDriver) GetName() string { + names := make([]string, 0, len(d.drivers)) + for _, driver := range d.drivers { + names = append(names, driver.GetName()) + } + return fmt.Sprintf("multi[%s]", strings.Join(names, ",")) +} + +func (d *multiDriver) QueryDomain(domain string) (driver.Result, error) { + r := newResult(domain) + var group errgroup.Group + for _, d := range d.drivers { + goFunc := func(localDriver driver.Driver) func() error { + return func() error { + return func(localDriver driver.Driver) error { + result, err := localDriver.QueryDomain(domain) + if err != nil { + return err + } + return r.add(result) + }(localDriver) + } + } + + group.Go(goFunc(d)) + } + err := group.Wait() + if err != nil { + return nil, err + } + return r, nil +} + +func newResult(host string) *multiResult { + r := new(multiResult) + r.host = host + r.results = make([]driver.Result, 0, 2) + r.fingerprints = make(driver.FingerprintMap) + return r +} + +type multiResult struct { + host string + results []driver.Result + resultLock sync.Mutex // also protects fingerprints + fingerprints driver.FingerprintMap +} + +func (c *multiResult) add(r driver.Result) error { + c.resultLock.Lock() + defer c.resultLock.Unlock() + fpm, err := r.GetFingerprints() + if err != nil { + return err + } + for domain := range fpm { + for _, fp := range fpm[domain] { + // TODO does not dedupe across drivers + c.fingerprints.Add(domain, fp) + } + } + + c.results = append(c.results, r) + return nil +} + +func (c *multiResult) QueryCert(fp fingerprint.Fingerprint) (*driver.CertResult, error) { + for _, result := range c.results { + cr, err := result.QueryCert(fp) + if err != nil { + return nil, err + } + if cr != nil { + return cr, nil + } + } + return nil, errors.New("unable to find working driver with QueryCert()") +} + +func (c *multiResult) GetFingerprints() (driver.FingerprintMap, error) { + return c.fingerprints, nil +} + +func (c *multiResult) GetStatus() status.Map { + // TODO nest other status inside + return status.NewMap(c.host, status.New(status.MULTI)) +} + +func (c *multiResult) GetRelated() ([]string, error) { + relatedMap := make(map[string]bool) + for _, result := range c.results { + related, err := result.GetRelated() + if err != nil { + return nil, err + } + for _, r := range related { + relatedMap[r] = true + } + } + related := make([]string, 0, len(relatedMap)) + for r := range relatedMap { + related = append(related, r) + } + return related, nil +} diff --git a/driver/save.go b/driver/save.go index e968aca..279a491 100644 --- a/driver/save.go +++ b/driver/save.go @@ -17,7 +17,10 @@ } defer f.Close() for _, cert := range certs { - pem.Encode(f, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}) + err = pem.Encode(f, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}) + if err != nil { + return err + } } return nil } @@ -32,8 +35,8 @@ return err } defer f.Close() - pem.Encode(f, &pem.Block{Type: "CERTIFICATE", Bytes: cert}) - return nil + err = pem.Encode(f, &pem.Block{Type: "CERTIFICATE", Bytes: cert}) + return err } func fileExists(f string) bool { diff --git a/driver/smtp/smtp.go b/driver/smtp/smtp.go index 3e6ed32..cb9d908 100644 --- a/driver/smtp/smtp.go +++ b/driver/smtp/smtp.go @@ -1,3 +1,4 @@ +// Package smtp implements a certgraph driver for obtaining SSL certificates over smtp with STARTTLS package smtp import ( @@ -55,7 +56,7 @@ if found { return cert, nil } - return nil, fmt.Errorf("Certificate with Fingerprint %s not found", fp.HexString()) + return nil, fmt.Errorf("certificate with Fingerprint %s not found", fp.HexString()) } // Driver creates a new SSL driver for SMTP Connections @@ -134,10 +135,10 @@ // save if d.save && len(certs) > 0 { - driver.CertsToPEMFile(certs, path.Join(d.savePath, certResult.Fingerprint.HexString())+".pem") + err = driver.CertsToPEMFile(certs, path.Join(d.savePath, certResult.Fingerprint.HexString())+".pem") } - return results, nil + return results, err } // getMX returns the MX records for the provided domain diff --git a/fingerprint/fingerprint.go b/fingerprint/fingerprint.go index 3bab154..6244f43 100644 --- a/fingerprint/fingerprint.go +++ b/fingerprint/fingerprint.go @@ -1,15 +1,18 @@ +// Package fingerprint defines types to define a certificate fingerprint for certgraph package fingerprint import ( "crypto/sha256" "encoding/base64" + "encoding/hex" "fmt" + "log" ) // Fingerprint sha256 of certificate bytes type Fingerprint [sha256.Size]byte -// HexString print Fingerprint as hex +// HexString print Fingerprint as uppercase hex func (fp *Fingerprint) HexString() string { return fmt.Sprintf("%X", *fp) } @@ -17,29 +20,38 @@ // FromHashBytes returns a Fingerprint generated by the first len(Fingerprint) bytes func FromHashBytes(data []byte) Fingerprint { var fp Fingerprint - /*if len(data) != sha256.Size { - v("Data is not correct SHA256 size", data) - }*/ + if len(data) != len(fp) { + log.Printf("len(data) %d\tlen(fp): %d", len(data), len(fp)) + // this should error.... + } for i := 0; i < len(data) && i < len(fp); i++ { fp[i] = data[i] } return fp } -// FromBytes returns a Fingerprint generated by the provided bytes -func FromBytes(data []byte) Fingerprint { - var fp Fingerprint - fp = sha256.Sum256(data) +// FromRawCertBytes returns a Fingerprint generated by the provided bytes +func FromRawCertBytes(data []byte) Fingerprint { + fp := sha256.Sum256(data) return fp } -// FromB64 returns a Fingerprint from a base64 encoded hash string -func FromB64(hash string) Fingerprint { - data, _ := base64.StdEncoding.DecodeString(hash) - /*if err != nil { - fmt.Println(err) - }*/ +// FromB64Hash returns a Fingerprint from a base64 encoded hash string +func FromB64Hash(hash string) Fingerprint { + data, err := base64.StdEncoding.DecodeString(hash) + if err != nil { + panic(err) + } return FromHashBytes(data) +} + +// FromHexHash returns a Fingerprint from a hex encoded hash string +func FromHexHash(hash string) Fingerprint { + decoded, err := hex.DecodeString(hash) + if err != nil { + panic(err) + } + return FromHashBytes(decoded) } // B64Encode returns the b64 string of a Fingerprint diff --git a/fingerprint/fp_test.go b/fingerprint/fp_test.go new file mode 100644 index 0000000..016b6cb --- /dev/null +++ b/fingerprint/fp_test.go @@ -0,0 +1,89 @@ +package fingerprint_test + +import ( + "crypto/sha256" + "encoding/base64" + "strings" + "testing" + + "github.com/lanrat/certgraph/fingerprint" +) + +const rawCert = "MIID/TCCA4KgAwIBAgIQBV74EmrgijxarGYRe4auizAKBggqhkjOPQQDAzBWMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMTAwLgYDVQQDEydEaWdpQ2VydCBUTFMgSHlicmlkIEVDQyBTSEEzODQgMjAyMCBDQTEwHhcNMjIwNDIwMDAwMDAwWhcNMjMwNDIwMjM1OTU5WjBmMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEVMBMGA1UEChMMR2l0SHViLCBJbmMuMRMwEQYDVQQDEwpnaXRodWIuY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEjkAQ9bMD0dVDhUlfevxOChhxQME0Sb7kZr7+3T/gW6CW4eduvDZxsQUwa37mhUXMzF88gh+FsUy9TieoqZhasKOCAiAwggIcMB8GA1UdIwQYMBaAFAq8CCkXjKU5bXoOzjPHLrPt+8N6MB0GA1UdDgQWBBQJJ/08CmhEtgPojKO+W3TVwfJnaTAlBgNVHREEHjAcggpnaXRodWIuY29tgg53d3cuZ2l0aHViLmNvbTAOBgNVHQ8BAf8EBAMCB4AwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMIGbBgNVHR8EgZMwgZAwRqBEoEKGQGh0dHA6Ly9jcmwzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRMU0h5YnJpZEVDQ1NIQTM4NDIwMjBDQTEtMS5jcmwwRqBEoEKGQGh0dHA6Ly9jcmw0LmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRMU0h5YnJpZEVDQ1NIQTM4NDIwMjBDQTEtMS5jcmwwPgYDVR0gBDcwNTAzBgZngQwBAgIwKTAnBggrBgEFBQcCARYbaHR0cDovL3d3dy5kaWdpY2VydC5jb20vQ1BTMIGFBggrBgEFBQcBAQR5MHcwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0LmNvbTBPBggrBgEFBQcwAoZDaHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0VExTSHlicmlkRUNDU0hBMzg0MjAyMENBMS0xLmNydDAJBgNVHRMEAjAAMBMGCisGAQQB1nkCBAMBAf8EAgUAMAoGCCqGSM49BAMDA2kAMGYCMQC/16+UbmbTo4bcahQGLY2+SrWtge+DC2dcEY2pv1Cwn3YUi51uqEE+v7U6PUvWWbQCMQCslhV/IibG55Uoa6F/hNpa21ZqEhp38u7CHTFb+6HGbLi0CtbSjgc1mn/yEt5pFu0=" +const fpHashHex = "46a1fe1780fd9a05a5529906ed08a5fea2cfe63567c9fdeb62c18ba74fae35d5" +const fpHashB64 = "RqH+F4D9mgWlUpkG7Qil/qLP5jVnyf3rYsGLp0+uNdU=" + +func TestFromHashBytes(t *testing.T) { + + data, err := base64.StdEncoding.DecodeString(rawCert) + if err != nil { + t.Errorf("error on b64 decode: %s", err.Error()) + } + + dataHashBytes := sha256.Sum256(data) + + fp := fingerprint.FromHashBytes(dataHashBytes[:]) + uppercaseHexHash := strings.ToUpper(fpHashHex) + hashHex := fp.HexString() + + if hashHex != uppercaseHexHash { + t.Errorf("fingerprint error, expected hex hash [%s] got [%s]", uppercaseHexHash, hashHex) + } + + hashB64 := fp.B64Encode() + + if hashB64 != fpHashB64 { + t.Errorf("fingerprint error, expected b64 hash [%s] got [%s]", hashB64, fpHashB64) + } +} + +func TestFromRawCertBytes(t *testing.T) { + + data, err := base64.StdEncoding.DecodeString(rawCert) + if err != nil { + t.Errorf("error on b64 decode: %s", err.Error()) + } + + fp := fingerprint.FromRawCertBytes(data) + uppercaseHash := strings.ToUpper(fpHashHex) + hashHex := fp.HexString() + + if hashHex != uppercaseHash { + t.Errorf("fingerprint error, expected hex hash [%s] got [%s]", uppercaseHash, hashHex) + } + + hashB64 := fp.B64Encode() + + if hashB64 != fpHashB64 { + t.Errorf("fingerprint error, expected b64 hash [%s] got [%s]", hashB64, fpHashB64) + } +} + +func TestFromB64Hash(t *testing.T) { + + fp := fingerprint.FromB64Hash(fpHashB64) + + uppercaseHash := strings.ToUpper(fpHashHex) + hashHex := fp.HexString() + + if hashHex != uppercaseHash { + t.Errorf("fingerprint error, expected hex hash [%s] got [%s]", uppercaseHash, hashHex) + } + + hashB64 := fp.B64Encode() + + if hashB64 != fpHashB64 { + t.Errorf("fingerprint error, expected b64 hash [%s] got [%s]", hashB64, fpHashB64) + } +} + +func TestFromHexHash(t *testing.T) { + + fp := fingerprint.FromHexHash(fpHashHex) + + hashB64 := fp.B64Encode() + + if fpHashB64 != hashB64 { + t.Errorf("fingerprint error, expected b64 hash [%s] got [%s]", fpHashHex, hashB64) + } +} diff --git a/go.mod b/go.mod index fff6d4e..c57e9c5 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,10 @@ module github.com/lanrat/certgraph require ( - github.com/lib/pq v1.0.0 - github.com/weppos/publicsuffix-go v0.4.0 - golang.org/x/net v0.0.0-20180906233101-161cd47e91fd // indirect - golang.org/x/text v0.3.0 // indirect + github.com/lib/pq v1.10.5 + github.com/weppos/publicsuffix-go v0.15.0 + golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c ) + +go 1.16 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..4a1dc55 --- /dev/null +++ b/go.sum @@ -0,0 +1,18 @@ +github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ= +github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/weppos/publicsuffix-go v0.15.0 h1:2uQCwDczZ8YZe5uD0mM3sXRoZYA74xxPuiKK8LdPcGQ= +github.com/weppos/publicsuffix-go v0.15.0/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/graph/cert_node.go b/graph/cert_node.go index 13c145e..1f40985 100644 --- a/graph/cert_node.go +++ b/graph/cert_node.go @@ -3,6 +3,7 @@ import ( "fmt" "strings" + "sync" "github.com/lanrat/certgraph/dns" "github.com/lanrat/certgraph/fingerprint" @@ -10,9 +11,10 @@ // CertNode graph node to store certificate information type CertNode struct { - Fingerprint fingerprint.Fingerprint - Domains []string - foundMap map[string]bool + Fingerprint fingerprint.Fingerprint + Domains []string + foundMap map[string]bool + foundMapLock sync.Mutex } func (c *CertNode) String() string { @@ -30,6 +32,8 @@ // AddFound adds a driver name to the source of the certificate func (c *CertNode) AddFound(driver string) { + c.foundMapLock.Lock() + defer c.foundMapLock.Unlock() if c.foundMap == nil { c.foundMap = make(map[string]bool) } @@ -57,17 +61,17 @@ return false } -// TLDPlus1Count the number of tld+1 domains in the certificate -func (c *CertNode) TLDPlus1Count() int { - tldPlus1Domains := make(map[string]bool) +// ApexCount the number of tld+1 domains in the certificate +func (c *CertNode) ApexCount() int { + apexDomains := make(map[string]bool) for _, domain := range c.Domains { - tldPlus1, err := dns.TLDPlus1(domain) + apexDomain, err := dns.ApexDomain(domain) if err != nil { continue } - tldPlus1Domains[tldPlus1] = true + apexDomains[apexDomain] = true } - return len(tldPlus1Domains) + return len(apexDomains) } // ToMap returns a map of the CertNode's fields (weak serialization) diff --git a/graph/domain_node.go b/graph/domain_node.go index b18d589..c47393f 100644 --- a/graph/domain_node.go +++ b/graph/domain_node.go @@ -22,10 +22,10 @@ HasDNS bool } -// NewDomainNode constructor for DomainNode, converts domain to nonWildcard +// NewDomainNode constructor for DomainNode, converts domain to lower nonWildcard func NewDomainNode(domain string, depth uint) *DomainNode { domainNode := new(DomainNode) - domainNode.Domain = nonWildcard(domain) + domainNode.Domain = nonWildcard(strings.ToLower(domain)) domainNode.Depth = depth domainNode.Certs = make(map[fingerprint.Fingerprint][]string) domainNode.RelatedDomains = make(status.Map) @@ -37,6 +37,7 @@ // in the map func (d *DomainNode) AddRelatedDomains(domains []string) { for _, domain := range domains { + domain = strings.ToLower(domain) if _, ok := d.RelatedDomains[domain]; ok { continue } @@ -44,7 +45,7 @@ } } -// CheckForDNS checks for the existence of DNS records for the domain's tld+1 +// CheckForDNS checks for the existence of DNS records for the domain's apex // sets the value to the node and returns the result as well func (d *DomainNode) CheckForDNS(timeout time.Duration) (bool, error) { hasDNS, err := dns.HasRecordsCache(d.Domain, timeout) @@ -75,7 +76,7 @@ return fingerprints } -// get the string representation of a node +// String returns the string representation of a node func (d *DomainNode) String() string { certString := "" // Certs diff --git a/graph/graph.go b/graph/graph.go index c519040..931464c 100644 --- a/graph/graph.go +++ b/graph/graph.go @@ -1,3 +1,4 @@ +// Package graph implements the graph data structures used by certgraph to build the certificate graph package graph import ( @@ -19,13 +20,6 @@ func NewCertGraph() *CertGraph { graph := new(CertGraph) return graph -} - -// LoadOrStoreCert will return the CertNode in the graph with the provided node's fingerprint, or store the node if it did not already exist -// returned bool is true if the CertNode was found, false if stored -func (graph *CertGraph) LoadOrStoreCert(certNode *CertNode) (*CertNode, bool) { - foundCertNode, ok := graph.certs.LoadOrStore(certNode.Fingerprint, certNode) - return foundCertNode.(*CertNode), ok } // AddCert add a CertNode to the graph @@ -97,7 +91,7 @@ certNode := node.(*CertNode) if !cdn && certNode.CDNCert() { //v(domain, "-> CDN CERT") - } else if maxSANsSize > 0 && certNode.TLDPlus1Count() > maxSANsSize { + } else if maxSANsSize > 0 && certNode.ApexCount() > maxSANsSize { //v(domain, "-> Large CERT") } else { for _, neighbor := range certNode.Domains { diff --git a/status/status.go b/status/status.go index 12ec034..5e00850 100644 --- a/status/status.go +++ b/status/status.go @@ -1,3 +1,4 @@ +// Package status defines the various status certgraph discovered hosts/certificates may have package status import ( @@ -62,9 +63,10 @@ ERROR = iota REDIRECT = iota CT = iota + MULTI = iota ) -// return domain status for printing +// String returns the domain status for printing func (status DomainStatus) String() string { switch status { case UNKNOWN: @@ -83,6 +85,8 @@ return "Redirect" case CT: return "CT" + case MULTI: + return "MULTI" } return "?" } diff --git a/web/web.go b/web/web.go new file mode 100644 index 0000000..a73f1ef --- /dev/null +++ b/web/web.go @@ -0,0 +1,26 @@ +// Package web defines a minimal web server for serving the web UI +package web + +import ( + "io/fs" + "log" + "net/http" +) + +// Serve starts a very basic webserver serving the embed web UI +func Serve(addr string, data fs.FS) error { + data, err := fs.Sub(data, "docs") + if err != nil { + return err + } + http.Handle("/", http.FileServer(http.FS(data))) + return http.ListenAndServe(addr, logRequest(http.DefaultServeMux)) +} + +// very minimal request logger +func logRequest(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + log.Printf("%s %s %s\n", r.RemoteAddr, r.Method, r.URL) + handler.ServeHTTP(w, r) + }) +}