Commit d4d4d7485c7f5c0122f9af25407784019930f147 - certgraph

+4

-0

.dockerignore less more

	0	Dockerfile
	1	certgraph
	2	build/
	3	*.json⏎

+67

-0

.github/workflows/codeql-analysis.yml less more

	0	# For most projects, this workflow file will not need changing; you simply need
	1	# to commit it to your repository.
	2	#
	3	# You may wish to alter this file to override the set of languages analyzed,
	4	# or to provide custom queries or build logic.
	5	#
	6	# ****** NOTE ******
	7	# We have attempted to detect the languages in your repository. Please check
	8	# the `language` matrix defined below to confirm you have the correct set of
	9	# supported CodeQL languages.
	10	#
	11	name: "CodeQL"
	12
	13	on:
	14	push:
	15	branches: [ master ]
	16	pull_request:
	17	# The branches below must be a subset of the branches above
	18	branches: [ master ]
	19	schedule:
	20	- cron: '31 22 * * 3'
	21
	22	jobs:
	23	analyze:
	24	name: Analyze
	25	runs-on: ubuntu-latest
	26
	27	strategy:
	28	fail-fast: false
	29	matrix:
	30	language: [ 'go' ]
	31	# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
	32	# Learn more:
	33	# https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
	34
	35	steps:
	36	- name: Checkout repository
	37	uses: actions/checkout@v2
	38
	39	# Initializes the CodeQL tools for scanning.
	40	- name: Initialize CodeQL
	41	uses: github/codeql-action/init@v1
	42	with:
	43	languages: ${{ matrix.language }}
	44	# If you wish to specify custom queries, you can do so here or in a config file.
	45	# By default, queries listed here will override any specified in a config file.
	46	# Prefix the list here with "+" to use these queries and those in the config file.
	47	# queries: ./path/to/local/query, your-org/your-repo/queries@main
	48
	49	# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
	50	# If this step fails, then you should remove it and run the build manually (see below)
	51	- name: Autobuild
	52	uses: github/codeql-action/autobuild@v1
	53
	54	# ℹ️ Command-line programs to run using the OS shell.
	55	# 📚 https://git.io/JvXDl
	56
	57	# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
	58	# and modify them (or add more) to build your code if your project
	59	# uses a compiled language
	60
	61	#- run: \|
	62	# make bootstrap
	63	# make release
	64
	65	- name: Perform CodeQL Analysis
	66	uses: github/codeql-action/analyze@v1

+22

-0

.github/workflows/go.yml less more

	0	name: Go
	1
	2	on:
	3	push:
	4	branches: [ master ]
	5	pull_request:
	6	branches: [ master ]
	7
	8	jobs:
	9
	10	build:
	11	runs-on: ubuntu-latest
	12	steps:
	13	- uses: actions/checkout@v2
	14
	15	- name: Set up Go
	16	uses: actions/setup-go@v2
	17	with:
	18	go-version: 1.16
	19
	20	- name: Build
	21	run: go build -v ./...

+0

-1

.gitignore less more

0		go.sum
1	0	certgraph
2	1	build/
3	2	*.json

+11

-0

Dockerfile less more

	0	FROM golang:alpine
	1
	2	RUN apk add --update git make
	3
	4	WORKDIR /src/certgraph
	5	ADD . .
	6
	7	ENV CGO_ENABLED=0
	8	RUN make install
	9
	10	ENTRYPOINT [ "certgraph" ]⏎

+32

-7

Makefile less more

0	0	GIT_DATE := $(shell git log -1 --date=short --pretty='%cd' \| tr -d -)
1	1	GIT_HASH := $(shell git rev-parse HEAD)
2	2
3		BUILD_FLAGS := -ldflags "-X main.gitDate=$(GIT_DATE) -X main.gitHash=$(GIT_HASH)"
	3	BUILD_FLAGS := -trimpath -ldflags "-w -s -X main.gitDate=$(GIT_DATE) -X main.gitHash=$(GIT_HASH)"
4	4
5		PLATFORMS := linux/amd64 linux/386 linux/arm darwin/amd64 windows/amd64 windows/386 openbsd/amd64
	5	PLATFORMS := linux/amd64 linux/386 linux/arm linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/386 openbsd/amd64
6	6	SOURCES := $(shell find . -maxdepth 1 -type f -name "*.go")
7		ALL_SOURCES = $(shell find . -type f -name '*.go')
	7	ALL_SOURCES = $(shell find . -type f -name '.go') go.mod docs/
8	8
9	9	temp = $(subst /, ,$@)
10	10	os = $(word 1, $(temp))
11	11	arch = $(word 2, $(temp))
12	12	ext = $(shell if [ "$(os)" = "windows" ]; then echo ".exe"; fi)
13	13
14		.PHONY: all release fmt clean serv $(PLATFORMS)
	14	.PHONY: all release fmt clean serv $(PLATFORMS) docker check deps update-deps
15	15
16	16	all: certgraph
17	17

25	25	CGO_ENABLED=0 GOOS=$(os) GOARCH=$(arch) go build $(BUILD_FLAGS) -o 'build/bin/$(os)/$(arch)/certgraph$(ext)' $(SOURCES)
26	26	mkdir -p build/$(GIT_DATE)/; cd build/bin/$(os)/$(arch)/; zip -r ../../../$(GIT_DATE)/certgraph-$(os)-$(arch)-$(GIT_DATE).zip .; cd ../../../
27	27
	28	docker: Dockerfile $(ALL_SOURCES)
	29	docker build -t lanrat/certgraph .
	30
	31	deps: go.mod
	32	GOPROXY=direct go mod download
	33	GOPROXY=direct go get -u all
	34
28	35	fmt:
29	36	gofmt -s -w -l .
30	37

32	39	go install $(BUILD_FLAGS)
33	40
34	41	clean:
35		rm -r certgraph build/
	42	rm -rf certgraph build/
36	43
37		serv:
38		(cd docs; python -m SimpleHTTPServer)
	44	check: \| lint check1 check2
	45
	46	check1:
	47	golangci-lint run
	48
	49	check2:
	50	staticcheck -f stylish -checks all ./...
	51
	52	lint:
	53	golint ./...
	54
	55	serv: certgraph
	56	./certgraph --serve 127.0.0.1:8080
	57
	58	update-deps:
	59	go get -u
	60	go mod tidy
	61
	62	test:
	63	go test -v ./... \| grep -v "\[no test files\]"⏎

+84

-32

README.md less more

0	0	# CertGraph
1		### A tool to crawl the graph of certificate Alternate Names
	1
	2	## A tool to crawl the graph of certificate Alternate Names
2	3
3	4	CertGraph crawls SSL certificates creating a directed graph where each domain is a node and the certificate alternative names for that domain's certificate are the edges to other domain nodes. New domains are printed as they are found. In Detailed mode upon completion the Graph's adjacency list is printed.
4	5

9	10	[Blog post with more information](https://lanrat.com/certgraph/)
10	11
11	12	## Usage
12		```
	13
	14	```console
13	15	Usage of ./certgraph: [OPTION]... HOST...
14		https://github.com/lanrat/certgraph
	16	https://github.com/lanrat/certgraph
15	17	OPTIONS:
	18	-apex
	19	for every domain found, add the apex domain of the domain's parent
16	20	-cdn
17		include certificates from CDNs
	21	include certificates from CDNs
18	22	-ct-expired
19		include expired certificates in certificate transparency search
	23	include expired certificates in certificate transparency search
20	24	-ct-subdomains
21		include sub-domains in certificate transparency search
	25	include sub-domains in certificate transparency search
22	26	-depth uint
23		maximum BFS depth to go (default 5)
	27	maximum BFS depth to go (default 5)
24	28	-details
25		print details about the domains crawled
	29	print details about the domains crawled
	30	-dns
	31	check for DNS records to determine if domain is registered
26	32	-driver string
27		driver to use [crtsh, google, http, smtp] (default "http")
	33	driver(s) to use [censys, crtsh, google, http, smtp] (default "http")
28	34	-json
29		print the graph as json, can be used for graph in web UI
30		-ns
31		check for NS records to determine if domain is registered
	35	print the graph as json, can be used for graph in web UI
32	36	-parallel uint
33		number of certificates to retrieve in parallel (default 10)
	37	number of certificates to retrieve in parallel (default 10)
	38	-regex string
	39	regex domains must match to be part of the graph
34	40	-sanscap int
35		maximum number of uniq TLD+1 domains in certificate to include, 0 has no limit (default 80)
	41	maximum number of uniq apex domains in certificate to include, 0 has no limit (default 80)
36	42	-save string
37		save certs to folder in PEM format
	43	save certs to folder in PEM format
	44	-serve string
	45	address:port to serve html UI on
38	46	-timeout uint
39		tcp timeout in seconds (default 10)
40		-tldplus1
41		for every domain found, add tldPlus1 of the domain's parent
	47	tcp timeout in seconds (default 10)
	48	-updatepsl
	49	Update the default Public Suffix List
42	50	-verbose
43		verbose logging
	51	verbose logging
44	52	-version
45		print version and exit
	53	print version and exit
46	54	```
47	55
48	56	## Drivers
49	57
50	58	CertGraph has multiple options for querying SSL certificates. The driver is responsible for retrieving the certificates for a given domain. Currently there are the following drivers:
51	59
52		* http this is the default driver which works by connecting to the hosts over HTTPS and retrieving the certificates from the SSL connection
53
54		* smtp like the http driver, but connects over port 25 and issues the starttls command to retrieve the certificates from the SSL connection
	60	* http this is the default driver which works by connecting to the hosts over HTTPS and retrieving the certificates from the SSL connection
55	61
56		* crtsh this driver searches Certificate Transparency logs via [crt.sh](https://crt.sh/). No packets are sent to any of the domains when using this driver
	62	* smtp like the http driver, but connects over port 25 and issues the starttls command to retrieve the certificates from the SSL connection
57	63
58		* google this is another Certificate Transparency driver that behaves like crtsh but uses the [Google Certificate Transparency Lookup Tool](https://transparencyreport.google.com/https/certificates)
	64	* censys this driver searches Certificate Transparency logs via [censys.io](https://search.censys.io/certificates). No packets are sent to any of the domains when using this driver. Requires Censys API keys
59	65
	66	* crtsh this driver searches Certificate Transparency logs via [crt.sh](https://crt.sh/). No packets are sent to any of the domains when using this driver
	67
	68	* google this is another Certificate Transparency driver that behaves like crtsh but uses the [Google Certificate Transparency Lookup Tool](https://transparencyreport.google.com/https/certificates)
60	69
61	70	## Example
62		```
	71
	72	```console
63	73	$ ./certgraph -details eff.org
64	74	eff.org 0 Good 42E3E4605D8BB4608EB64936E2176A98B97EBF2E0F8F93A64A6640713C7D4325
65	75	maps.eff.org 1 Good 42E3E4605D8BB4608EB64936E2176A98B97EBF2E0F8F93A64A6640713C7D4325

68	78	atlas.eff.org 1 Good 42E3E4605D8BB4608EB64936E2176A98B97EBF2E0F8F93A64A6640713C7D4325
69	79	kittens.eff.org 1 Good 42E3E4605D8BB4608EB64936E2176A98B97EBF2E0F8F93A64A6640713C7D4325
70	80	```
	81
71	82	The above output represents the adjacency list for the graph for the root domain `eff.org`. The adjacency list is in the form:
72	83	`Node Depth Status Cert-Fingerprint`
73	84
74	85	## [Releases](https://github.com/lanrat/certgraph/releases)
75	86
76		Precompiled releases will occasionally be uploaded to the [releases github page](https://github.com/lanrat/certgraph/releases). https://github.com/lanrat/certgraph/releases
	87	Pre-compiled releases will occasionally be uploaded to the [releases github page](https://github.com/lanrat/certgraph/releases). [https://github.com/lanrat/certgraph/releases](https://github.com/lanrat/certgraph/releases)
77	88
78		Also available in [BlackArch](https://blackarch.org).
	89	### [Docker](https://hub.docker.com/r/lanrat/certgraph/)
	90
	91	CertGraph is an automated build on the Docker Hub!
	92
	93	```console
	94	$ docker run --rm -it lanrat/certgraph example.com
	95	example.com
	96	www.example.net
	97	www.example.org
	98	www.example.com
	99	example.org
	100	example.net
	101	example.edu
	102	www.example.edu
	103	```
	104
	105	### Linux Distributions
	106
	107	* [BlackArch](https://blackarch.org)
	108	* [Kali Linux](https://www.kali.org/)
79	109
80	110	## Compiling
81	111
82		To compile certgraph you must have a working go 1.11 or newer compiler on your system, as certgraph makes use of go's modules for dependencies.
	112	To compile certgraph you must have a working go 1.16 or newer compiler on your system.
83	113	To compile for the running system compilation is as easy as running make
84		```
	114
	115	```console
85	116	certgraph$ make
86	117	go build -o certgraph certgraph.go
87	118	```
88	119
89	120	Alternatively you can use `go get` to install with this one-liner:
90		```
	121
	122	```console
91	123	go get -u github.com/lanrat/certgraph
92	124	```
93	125
94	126	## [Web UI](https://lanrat.github.io/certgraph/)
95	127
96		A web UI is provided in the docs folder and is accessible at the github pages url [https://lanrat.github.io/certgraph/](https://lanrat.github.io/certgraph/).
	128	A web UI is provided in the docs folder and is accessible at the github pages url [https://lanrat.github.io/certgraph/](https://lanrat.github.io/certgraph/), or can be run from the embedded web server by calling `certgraph --serve 127.0.0.1:8080`.
97	129
98	130	The web UI takes the output provided with the `-json` flag.
99	131	The JSON graph can be sent to the web interface as an uploaded file, remote URL, or as the query string using the data variable.

110	142
111	143	[![whitehouse.gov graph](https://cloud.githubusercontent.com/assets/164192/20861407/4775ff26-b944-11e6-888c-4d93e3333494.png)](https://lanrat.github.io/certgraph/?data=https://gist.githubusercontent.com/lanrat/96c47dfee0faaaad633cc830b7e3b997/raw/3c79fed837cb3202e220de21d2a8eb128f4bbd9f/whitehouse.json)
112	144
	145	## BygoneSSL detection
	146
	147	### Self Detection
	148
	149	CertGraph can be used to detect [BygoneSSL](https://insecure.design) DoS with the following options. CT-DRIVER can be any Certificate Transparency capable driver.
	150	Provide all known input domains you own. If any domains you do not own are printed, then you are vulnerable.
	151
	152	```console
	153	certgraph -depth 1 -driver CT-DRIVER -ct-subdomains -cdn -apex [DOMAIN]...
	154	```
	155
	156	### Bug Bounty
	157
	158	If you want to find a vulnerable site that has a bug bounty, certgraph can be used with the following options and any driver. But you will have better luck with a non Certificate Transparency driver to ensure that the certificates in question are actually in use
	159
	160	```console
	161	certgraph -cdn -dns -apex [DOMAIN]...
	162	```
	163
	164	And domains that print `* Missing DNS for` have vulnerable certificates that should be rotated.

+93

-28

certgraph.go less more

0	0	package main
1	1
2	2	import (
	3	"embed"
3	4	"encoding/json"
4	5	"flag"
5	6	"fmt"
6	7	"net/url"
7	8	"os"
	9	"regexp"
8	10	"strings"
9	11	"sync"
10	12	"time"
11	13
12	14	"github.com/lanrat/certgraph/dns"
13	15	"github.com/lanrat/certgraph/driver"
	16	"github.com/lanrat/certgraph/driver/censys"
14	17	"github.com/lanrat/certgraph/driver/crtsh"
15	18	"github.com/lanrat/certgraph/driver/google"
16	19	"github.com/lanrat/certgraph/driver/http"
	20	"github.com/lanrat/certgraph/driver/multi"
17	21	"github.com/lanrat/certgraph/driver/smtp"
18	22	"github.com/lanrat/certgraph/graph"
	23	"github.com/lanrat/certgraph/web"
19	24	)
20	25
	26	// version vars
21	27	var (
22	28	gitDate = "none"
23	29	gitHash = "master"
24	30	certGraph = graph.NewCertGraph()
25	31	)
26	32
	33	// temp flag vars
	34	var (
	35	timeoutSeconds uint
	36	regexString string
	37	)
	38
	39	// webContent holds our static web server content.
	40	//go:embed docs/*
	41	var webContent embed.FS
	42
27	43	var certDriver driver.Driver
28	44
29	45	// config & flags
	46	// TODO move driver options to own struct
30	47	var config struct {
31	48	timeout time.Duration
32	49	verbose bool

40	57	includeCTExpired bool
41	58	cdn bool
42	59	maxSANsSize int
43		tldPlus1 bool
	60	apex bool
44	61	updatePSL bool
45	62	checkDNS bool
46	63	printVersion bool
	64	serve string
	65	regex *regexp.Regexp
47	66	}
48	67
49	68	func init() {
50		var timeoutSeconds uint
51	69	flag.BoolVar(&config.printVersion, "version", false, "print version and exit")
52	70	flag.UintVar(&timeoutSeconds, "timeout", 10, "tcp timeout in seconds")
53	71	flag.BoolVar(&config.verbose, "verbose", false, "verbose logging")
54		flag.StringVar(&config.driver, "driver", "http", fmt.Sprintf("driver to use [%s]", strings.Join(driver.Drivers, ", ")))
	72	flag.StringVar(&config.driver, "driver", "http", fmt.Sprintf("driver(s) to use [%s]", strings.Join(driver.Drivers, ", ")))
55	73	flag.BoolVar(&config.includeCTSubdomains, "ct-subdomains", false, "include sub-domains in certificate transparency search")
56	74	flag.BoolVar(&config.includeCTExpired, "ct-expired", false, "include expired certificates in certificate transparency search")
57		flag.IntVar(&config.maxSANsSize, "sanscap", 80, "maximum number of uniq TLD+1 domains in certificate to include, 0 has no limit")
	75	flag.IntVar(&config.maxSANsSize, "sanscap", 80, "maximum number of uniq apex domains in certificate to include, 0 has no limit")
58	76	flag.BoolVar(&config.cdn, "cdn", false, "include certificates from CDNs")
59	77	flag.BoolVar(&config.checkDNS, "dns", false, "check for DNS records to determine if domain is registered")
60		flag.BoolVar(&config.tldPlus1, "tldplus1", false, "for every domain found, add tldPlus1 of the domain's parent")
	78	flag.BoolVar(&config.apex, "apex", false, "for every domain found, add the apex domain of the domain's parent")
61	79	flag.BoolVar(&config.updatePSL, "updatepsl", false, "Update the default Public Suffix List")
62	80	flag.UintVar(&config.maxDepth, "depth", 5, "maximum BFS depth to go")
63	81	flag.UintVar(&config.parallel, "parallel", 10, "number of certificates to retrieve in parallel")
64	82	flag.BoolVar(&config.details, "details", false, "print details about the domains crawled")
65	83	flag.BoolVar(&config.printJSON, "json", false, "print the graph as json, can be used for graph in web UI")
66	84	flag.StringVar(&config.savePath, "save", "", "save certs to folder in PEM format")
	85	flag.StringVar(&config.serve, "serve", "", "address:port to serve html UI on")
	86	flag.StringVar(&regexString, "regex", "", "regex domains must match to be part of the graph")
	87
67	88	flag.Usage = func() {
68	89	fmt.Fprintf(os.Stderr, "Usage of %s: [OPTION]... HOST...\n\thttps://github.com/lanrat/certgraph\nOPTIONS:\n", os.Args[0])
69	90	flag.PrintDefaults()
70	91	}
	92	}
	93
	94	func main() {
71	95	flag.Parse()
72	96	config.timeout = time.Duration(timeoutSeconds) * time.Second
73		}
74
75		func main() {
	97	var err error
	98
76	99	// check for version flag
77	100	if config.printVersion {
78	101	fmt.Println(version())
	102	return
	103	}
	104
	105	// check for regex
	106	if len(regexString) > 0 {
	107	config.regex, err = regexp.Compile(regexString)
	108	if err != nil {
	109	e(err)
	110	return
	111	}
	112	}
	113
	114	if len(config.serve) > 0 {
	115	err = web.Serve(config.serve, webContent)
	116	e(err)
79	117	return
80	118	}
81	119

94	132
95	133	// update the public suffix list if required
96	134	if config.updatePSL {
97		err := dns.UpdatePublicSuffixList(config.timeout)
	135	err = dns.UpdatePublicSuffixList(config.timeout)
98	136	if err != nil {
99	137	e(err)
100	138	return

107	145	d := strings.ToLower(domain)
108	146	if len(d) > 0 {
109	147	startDomains = append(startDomains, cleanInput(d))
110		if config.tldPlus1 {
111		tldPlus1, err := dns.TLDPlus1(domain)
	148	if config.apex {
	149	apexDomain, err := dns.ApexDomain(domain)
112	150	if err != nil {
113	151	continue
114	152	}
115		startDomains = append(startDomains, tldPlus1)
	153	startDomains = append(startDomains, apexDomain)
116	154	}
117	155	}
118	156	}
119	157
120	158	// set driver
121		err := setDriver(config.driver)
	159	certDriver, err = setDriver(config.driver)
122	160	if err != nil {
123	161	fmt.Fprintln(os.Stderr, err)
124	162	return

145	183	v("Graph Depth:", certGraph.DomainDepth())
146	184	}
147	185
148		// setDriver sets the driver variable for the provided driver string and does any necessary driver prep work
	186	func setDriver(name string) (driver.Driver, error) {
	187	if strings.Contains(name, ",") {
	188	names := strings.Split(name, ",")
	189	drivers := make([]driver.Driver, 0, len(names))
	190	for _, driverName := range names {
	191	d, err := getDriverSingle(driverName)
	192	if err != nil {
	193	return nil, err
	194	}
	195	drivers = append(drivers, d)
	196	}
	197	return multi.Driver(drivers), nil
	198	}
	199	return getDriverSingle(name)
	200	}
	201
	202	// getDriverSingle sets the driver variable for the provided driver string and does any necessary driver prep work
149	203	// TODO make config generic and move this to driver module
150		func setDriver(driver string) error {
	204	func getDriverSingle(name string) (driver.Driver, error) {
151	205	var err error
152		switch driver {
	206	var d driver.Driver
	207	switch name {
153	208	case "google":
154		certDriver, err = google.Driver(50, config.savePath, config.includeCTSubdomains, config.includeCTExpired)
	209	d, err = google.Driver(50, config.savePath, config.includeCTSubdomains, config.includeCTExpired)
155	210	case "crtsh":
156		certDriver, err = crtsh.Driver(1000, config.timeout, config.savePath, config.includeCTSubdomains, config.includeCTExpired)
	211	d, err = crtsh.Driver(1000, config.timeout, config.savePath, config.includeCTSubdomains, config.includeCTExpired)
157	212	case "http":
158		certDriver, err = http.Driver(config.timeout, config.savePath)
	213	d, err = http.Driver(config.timeout, config.savePath)
159	214	case "smtp":
160		certDriver, err = smtp.Driver(config.timeout, config.savePath)
	215	d, err = smtp.Driver(config.timeout, config.savePath)
	216	case "censys":
	217	d, err = censys.Driver(config.savePath, config.includeCTSubdomains, config.includeCTExpired)
161	218	default:
162		return fmt.Errorf("Unknown driver name: %s", config.driver)
163		}
164		return err
	219	return nil, fmt.Errorf("unknown driver name: %s", config.driver)
	220	}
	221	return d, err
165	222	}
166	223
167	224	// verbose logging

236	293	<-threadPass
237	294	defer func() { threadPass <- true }()
238	295
	296	// regex match check
	297	if config.regex != nil && !config.regex.MatchString(domainNode.Domain) {
	298	// skip domain that does not match regex
	299	v("domain does not match regex, skipping :", domainNode.Domain)
	300	return
	301	}
	302
239	303	// operate on the node
240	304	v("Visiting", domainNode.Depth, domainNode.Domain)
241	305	visit(domainNode)

243	307	for _, neighbor := range certGraph.GetDomainNeighbors(domainNode.Domain, config.cdn, config.maxSANsSize) {
244	308	wg.Add(1)
245	309	domainNodeInputChan <- graph.NewDomainNode(neighbor, domainNode.Depth+1)
246		if config.tldPlus1 {
247		tldPlus1, err := dns.TLDPlus1(neighbor)
	310	if config.apex {
	311	apexDomain, err := dns.ApexDomain(neighbor)
248	312	if err != nil {
249	313	continue
250	314	}
251	315	wg.Add(1)
252		domainNodeInputChan <- graph.NewDomainNode(tldPlus1, domainNode.Depth+1)
	316	domainNodeInputChan <- graph.NewDomainNode(apexDomain, domainNode.Depth+1)
253	317	}
254	318	}
255	319	}(domainNode)

340	404	domainNode.AddCertFingerprint(certNode.Fingerprint, certDriver.GetName())
341	405	}
342	406
343		// we dont process any other certificates returned, they will be collected
	407	// we don't process any other certificates returned, they will be collected
344	408	// when we process the related domains
345	409	}
346	410

353	417	if config.checkDNS && !domainNode.HasDNS {
354	418	// TODO print this in a better way
355	419	// TODO for debugging
356		realDomain, _ := dns.TLDPlus1(domainNode.Domain)
	420	realDomain, _ := dns.ApexDomain(domainNode.Domain)
357	421	fmt.Fprintf(os.Stdout, "* Missing DNS for: %s\n", realDomain)
358	422
359	423	}

384	448	options["sanscap"] = config.maxSANsSize
385	449	options["cdn"] = config.cdn
386	450	options["timeout"] = config.timeout
	451	options["regex"] = regexString
387	452	data["options"] = options
388	453	return data
389	454	}

+4

-3

dns/ns.go less more

	0	// Package dns adds utility functions for performing dns queries
0	1	package dns
1	2
2	3	import (

25	26	}
26	27
27	28	// HasRecords does NS, CNAME, A, and AAAA lookups with a timeout
28		// returns error when no NS found, does not use TLDPlus1
	29	// returns error when no NS found, does not use alexDomain
29	30	func HasRecords(domain string, timeout time.Duration) (bool, error) {
30	31	ctx, cancel := context.WithTimeout(context.Background(), timeout)
31	32	defer cancel()

67	68	return false, nil
68	69	}
69	70
70		// HasRecordsCache returns true if the domain has no DNS records (at the tldplus1 level)
	71	// HasRecordsCache returns true if the domain has no DNS records (at the apex domain level)
71	72	// uses a cache to store results to prevent lots of DNS lookups
72	73	func HasRecordsCache(domain string, timeout time.Duration) (bool, error) {
73		domain, err := TLDPlus1(domain)
	74	domain, err := ApexDomain(domain)
74	75	if err != nil {
75	76	return false, err
76	77	}

+3

-4

dns/publicsuffix.go less more

13	13	}
14	14	suffixListURL = "https://publicsuffix.org/list/public_suffix_list.dat"
15	15	suffixList = publicsuffix.DefaultList
16		nsCache = make(map[string]bool)
17	16	)
18	17
19	18	// UpdatePublicSuffixList gets a new copy of the public suffix list from the internat and updates the built in copy with the new rules

30	29	}
31	30	defer resp.Body.Close()
32	31	newSuffixList := publicsuffix.NewList()
33		newSuffixList.Load(resp.Body, suffixListParseOptions)
	32	_, err = newSuffixList.Load(resp.Body, suffixListParseOptions)
34	33	suffixList = newSuffixList
35	34	return err
36	35	}
37	36
38		// TLDPlus1 returns TLD+1 of domain
39		func TLDPlus1(domain string) (string, error) {
	37	// ApexDomain returns TLD+1 of domain
	38	func ApexDomain(domain string) (string, error) {
40	39	return publicsuffix.DomainFromListWithOptions(suffixList, domain, suffixListFindOptions)
41	40	}

+0

-1

docs/index.html less more

16	16	stroke: #333;
17	17	stroke-width: 1.5px;
18	18	}
19
20	19	.upload-drop-zone {
21	20	height: 200px;
22	21	border-width: 2px;

+242

-0

driver/censys/censys.go less more

	0	package censys
	1
	2	import (
	3	"bytes"
	4	"encoding/base64"
	5	"encoding/json"
	6	"flag"
	7	"fmt"
	8	"io"
	9	"log"
	10	"net/http"
	11	"path"
	12	"time"
	13
	14	"github.com/lanrat/certgraph/driver"
	15	"github.com/lanrat/certgraph/fingerprint"
	16	"github.com/lanrat/certgraph/status"
	17	)
	18
	19	const driverName = "censys"
	20
	21	var debug = false
	22
	23	// TODO support rate limits & pagnation
	24
	25	var (
	26	defaultHTTPClient = &http.Client{}
	27
	28	appID = flag.String("censys-appid", "", "censys API AppID")
	29	secret = flag.String("censys-secret", "", "censys API Secret")
	30	)
	31
	32	func init() {
	33	driver.AddDriver(driverName)
	34	}
	35
	36	type censys struct {
	37	appID string
	38	secret string
	39	save bool
	40	savePath string
	41	includeSubdomains bool
	42	includeExpired bool
	43	}
	44
	45	type censysCertDriver struct {
	46	host string
	47	fingerprints driver.FingerprintMap
	48	driver *censys
	49	}
	50
	51	func (c *censysCertDriver) GetFingerprints() (driver.FingerprintMap, error) {
	52	return c.fingerprints, nil
	53	}
	54
	55	func (c *censysCertDriver) GetStatus() status.Map {
	56	return status.NewMap(c.host, status.New(status.CT))
	57	}
	58
	59	func (c *censysCertDriver) GetRelated() ([]string, error) {
	60	return make([]string, 0), nil
	61	}
	62
	63	func (c censysCertDriver) QueryCert(fp fingerprint.Fingerprint) (driver.CertResult, error) {
	64	return c.driver.QueryCert(fp)
	65	}
	66
	67	// TODO support pagnation
	68	func domainSearchParam(domain string, includeExpired, includeSubdomain bool) certSearchParam {
	69	var s certSearchParam
	70	if includeSubdomain {
	71	s.Query = fmt.Sprintf("(parsed.names: %s )", domain)
	72	} else {
	73	s.Query = fmt.Sprintf("(parsed.names.raw: %s)", domain)
	74	}
	75	if !includeExpired {
	76	dateStr := time.Now().Format("2006-01-02") // YYYY-MM-DD
	77	expQuery := fmt.Sprintf(" AND ((parsed.validity.end: [%s TO ]) AND (parsed.validity.start: [ TO %s]))", dateStr, dateStr)
	78	s.Query = s.Query + expQuery
	79	}
	80	s.Page = 1
	81	s.Flatten = true
	82	s.Fields = []string{"parsed.fingerprint_sha256", "parsed.names"}
	83	return s
	84	}
	85
	86	// Driver creates a new CT driver for censys
	87	func Driver(savePath string, includeSubdomains, includeExpired bool) (driver.Driver, error) {
	88	if appID == "" \|\| secret == "" {
	89	return nil, fmt.Errorf("censys requires an appID and secret to run")
	90	}
	91	d := new(censys)
	92	d.appID = *appID
	93	d.secret = *secret
	94	d.savePath = savePath
	95	d.includeSubdomains = includeSubdomains
	96	d.includeExpired = includeExpired
	97	return d, nil
	98	}
	99
	100	func (d *censys) GetName() string {
	101	return driverName
	102	}
	103
	104	func (d censys) request(method, url string, request io.Reader) (http.Response, error) {
	105	totalTrys := 3
	106	var err error
	107	var req *http.Request
	108	var resp *http.Response
	109	for try := 1; try <= totalTrys; try++ {
	110	req, err = http.NewRequest(method, url, request)
	111	if err != nil {
	112	return nil, err
	113	}
	114	if request != nil {
	115	req.Header.Add("Content-Type", "application/json")
	116	}
	117	req.Header.Add("Accept", "application/json")
	118	req.SetBasicAuth(d.appID, d.secret)
	119
	120	resp, err = defaultHTTPClient.Do(req)
	121	if err != nil {
	122	err = fmt.Errorf("error on request [%d/%d] %s, got error %w: %+v", try, totalTrys, url, err, resp)
	123	} else {
	124	return resp, nil
	125	}
	126
	127	// sleep only if we will try again
	128	if try < totalTrys {
	129	time.Sleep(time.Second * 10)
	130	}
	131	}
	132	return resp, err
	133	}
	134
	135	// jsonRequest performes a request to the API endpoint sending and receiving JSON objects
	136	func (d *censys) jsonRequest(method, url string, request, response interface{}) error {
	137	var payloadReader io.Reader
	138	if request != nil {
	139	jsonPayload, err := json.Marshal(request)
	140	if err != nil {
	141	return err
	142	}
	143	payloadReader = bytes.NewReader(jsonPayload)
	144	}
	145
	146	if debug {
	147	log.Printf("DEBUG: request to %s %s", method, url)
	148	if request != nil {
	149	prettyJSONBytes, _ := json.MarshalIndent(request, "", "\t")
	150	log.Printf("request payload:\n%s\n", string(prettyJSONBytes))
	151	}
	152	}
	153
	154	resp, err := d.request(method, url, payloadReader)
	155	if err != nil {
	156	return err
	157	}
	158	defer resp.Body.Close()
	159
	160	// got an error, decode it
	161	if resp.StatusCode != http.StatusOK {
	162	var errorResp errorResponse
	163	err := fmt.Errorf("error on request %s, got Status %s %s", url, resp.Status, http.StatusText(resp.StatusCode))
	164	jsonError := json.NewDecoder(resp.Body).Decode(&errorResp)
	165	if jsonError != nil {
	166	return fmt.Errorf("error decoding json %w on errord request: %s", jsonError, err.Error())
	167	}
	168	return fmt.Errorf("%w, HTTPStatus: %d Message: %q", err, errorResp.ErrorCode, errorResp.Error)
	169	}
	170
	171	if response != nil {
	172	err = json.NewDecoder(resp.Body).Decode(&response)
	173	if err != nil {
	174	return err
	175	}
	176	if debug {
	177	prettyJSONBytes, _ := json.MarshalIndent(response, "", "\t")
	178	log.Printf("response payload:\n%s\n", string(prettyJSONBytes))
	179	}
	180	}
	181
	182	return nil
	183	}
	184
	185	func (d *censys) QueryDomain(domain string) (driver.Result, error) {
	186	results := &censysCertDriver{
	187	host: domain,
	188	fingerprints: make(driver.FingerprintMap),
	189	driver: d,
	190	}
	191	params := domainSearchParam(domain, d.includeExpired, d.includeSubdomains)
	192	url := "https://search.censys.io/api/v1/search/certificates"
	193	var resp certSearchResponse
	194	err := d.jsonRequest(http.MethodPost, url, params, &resp)
	195	if err != nil {
	196	return results, err
	197	}
	198
	199	for _, r := range resp.Results {
	200	fp := fingerprint.FromHexHash(r.Fingerprint)
	201	results.fingerprints.Add(domain, fp)
	202	}
	203
	204	if debug {
	205	log.Printf("censys: got %d results for %s.", len(resp.Results), domain)
	206	}
	207
	208	return results, nil
	209	}
	210
	211	func (d censys) QueryCert(fp fingerprint.Fingerprint) (driver.CertResult, error) {
	212	certNode := new(driver.CertResult)
	213	certNode.Fingerprint = fp
	214	certNode.Domains = make([]string, 0, 5)
	215
	216	url := fmt.Sprintf("https://search.censys.io/api/v1/view/certificates/%s", fp.HexString())
	217	var resp certViewResponse
	218	err := d.jsonRequest(http.MethodGet, url, nil, &resp)
	219	if err != nil {
	220	return certNode, err
	221	}
	222
	223	if debug {
	224	log.Printf("DEBUG QueryCert(%s): %v", fp.HexString(), resp.Parsed.Names)
	225	}
	226
	227	certNode.Domains = append(certNode.Domains, resp.Parsed.Names...)
	228
	229	if d.save {
	230	rawCert, err := base64.StdEncoding.DecodeString(resp.Raw)
	231	if err != nil {
	232	return certNode, err
	233	}
	234	err = driver.RawCertToPEMFile(rawCert, path.Join(d.savePath, fp.HexString())+".pem")
	235	if err != nil {
	236	return certNode, err
	237	}
	238	}
	239
	240	return certNode, nil
	241	}

+215

-0

driver/censys/schemas.go less more

	0	package censys
	1
	2	import "time"
	3
	4	type certSearchParam struct {
	5	Query string `json:"query"`
	6	Page uint `json:"page"`
	7	Fields []string `json:"fields"`
	8	Flatten bool `json:"flatten"`
	9	}
	10
	11	type certSearchResponse struct {
	12	Status string `json:"status"`
	13	Metadata struct {
	14	Query string `json:"query"`
	15	Count uint `json:"count"`
	16	BackendTime uint `json:"backend_time"`
	17	Page uint `json:"page"`
	18	Pages uint `json:"pages"`
	19	} `json:"metadata"`
	20	Results []struct {
	21	Names []string `json:"parsed.names"`
	22	Fingerprint string `json:"parsed.fingerprint_sha256"`
	23	} `json:"results"`
	24	}
	25
	26	type certViewResponse struct {
	27	Raw string `json:"raw"`
	28	FingerprintSha256 string `json:"fingerprint_sha256"`
	29	ParentSpkiSubjectFingerprint string `json:"parent_spki_subject_fingerprint"`
	30	Metadata struct {
	31	PostProcessedAt time.Time `json:"post_processed_at"`
	32	PostProcessed bool `json:"post_processed"`
	33	Source string `json:"source"`
	34	ParseVersion int `json:"parse_version"`
	35	ParseStatus string `json:"parse_status"`
	36	AddedAt time.Time `json:"added_at"`
	37	UpdatedAt time.Time `json:"updated_at"`
	38	SeenInScan bool `json:"seen_in_scan"`
	39	} `json:"metadata"`
	40	Ct struct {
	41	GoogleXenon2022 struct {
	42	Index int `json:"index"`
	43	CtToCensysAt time.Time `json:"ct_to_censys_at"`
	44	AddedToCtAt time.Time `json:"added_to_ct_at"`
	45	} `json:"google_xenon_2022"`
	46	} `json:"ct"`
	47	Parsed struct {
	48	Version int `json:"version"`
	49	SerialNumber string `json:"serial_number"`
	50	SignatureAlgorithm struct {
	51	Name string `json:"name"`
	52	Oid string `json:"oid"`
	53	} `json:"signature_algorithm"`
	54	Issuer struct {
	55	CommonName []string `json:"common_name"`
	56	Country []string `json:"country"`
	57	Organization []string `json:"organization"`
	58	} `json:"issuer"`
	59	IssuerDn string `json:"issuer_dn"`
	60	Validity struct {
	61	Start time.Time `json:"start"`
	62	End time.Time `json:"end"`
	63	Length int `json:"length"`
	64	} `json:"validity"`
	65	Subject struct {
	66	CommonName []string `json:"common_name"`
	67	} `json:"subject"`
	68	SubjectDn string `json:"subject_dn"`
	69	SubjectKeyInfo struct {
	70	KeyAlgorithm struct {
	71	Name string `json:"name"`
	72	} `json:"key_algorithm"`
	73	EcdsaPublicKey struct {
	74	B string `json:"b"`
	75	Curve string `json:"curve"`
	76	Gx string `json:"gx"`
	77	Gy string `json:"gy"`
	78	Length int `json:"length"`
	79	N string `json:"n"`
	80	P string `json:"p"`
	81	Pub string `json:"pub"`
	82	X string `json:"x"`
	83	Y string `json:"y"`
	84	} `json:"ecdsa_public_key"`
	85	FingerprintSha256 string `json:"fingerprint_sha256"`
	86	} `json:"subject_key_info"`
	87	Extensions struct {
	88	KeyUsage struct {
	89	DigitalSignature bool `json:"digital_signature"`
	90	Value int `json:"value"`
	91	} `json:"key_usage"`
	92	BasicConstraints struct {
	93	IsCa bool `json:"is_ca"`
	94	} `json:"basic_constraints"`
	95	SubjectAltName struct {
	96	DNSNames []string `json:"dns_names"`
	97	} `json:"subject_alt_name"`
	98	AuthorityKeyID string `json:"authority_key_id"`
	99	SubjectKeyID string `json:"subject_key_id"`
	100	ExtendedKeyUsage struct {
	101	ServerAuth bool `json:"server_auth"`
	102	ClientAuth bool `json:"client_auth"`
	103	} `json:"extended_key_usage"`
	104	CertificatePolicies []struct {
	105	ID string `json:"id"`
	106	Cps []string `json:"cps,omitempty"`
	107	} `json:"certificate_policies"`
	108	AuthorityInfoAccess struct {
	109	OcspUrls []string `json:"ocsp_urls"`
	110	IssuerUrls []string `json:"issuer_urls"`
	111	} `json:"authority_info_access"`
	112	SignedCertificateTimestamps []struct {
	113	Version int `json:"version"`
	114	LogID string `json:"log_id"`
	115	Timestamp int `json:"timestamp"`
	116	Signature string `json:"signature"`
	117	} `json:"signed_certificate_timestamps"`
	118	} `json:"extensions"`
	119	Signature struct {
	120	SignatureAlgorithm struct {
	121	Name string `json:"name"`
	122	Oid string `json:"oid"`
	123	} `json:"signature_algorithm"`
	124	Value string `json:"value"`
	125	Valid bool `json:"valid"`
	126	SelfSigned bool `json:"self_signed"`
	127	} `json:"signature"`
	128	FingerprintMd5 string `json:"fingerprint_md5"`
	129	FingerprintSha1 string `json:"fingerprint_sha1"`
	130	FingerprintSha256 string `json:"fingerprint_sha256"`
	131	TbsNoctFingerprint string `json:"tbs_noct_fingerprint"`
	132	SpkiSubjectFingerprint string `json:"spki_subject_fingerprint"`
	133	TbsFingerprint string `json:"tbs_fingerprint"`
	134	ValidationLevel string `json:"validation_level"`
	135	Names []string `json:"names"`
	136	Redacted bool `json:"redacted"`
	137	} `json:"parsed"`
	138	Tags []string `json:"tags"`
	139	Validation struct {
	140	Nss struct {
	141	Blacklisted bool `json:"blacklisted"`
	142	HadTrustedPath bool `json:"had_trusted_path"`
	143	InRevocationSet bool `json:"in_revocation_set"`
	144	TrustedPath bool `json:"trusted_path"`
	145	WasValid bool `json:"was_valid"`
	146	Whitelisted bool `json:"whitelisted"`
	147	Paths [][]string `json:"paths"`
	148	Parents []string `json:"parents"`
	149	Type string `json:"type"`
	150	Valid bool `json:"valid"`
	151	} `json:"nss"`
	152	Microsoft struct {
	153	Blacklisted bool `json:"blacklisted"`
	154	HadTrustedPath bool `json:"had_trusted_path"`
	155	InRevocationSet bool `json:"in_revocation_set"`
	156	TrustedPath bool `json:"trusted_path"`
	157	WasValid bool `json:"was_valid"`
	158	Whitelisted bool `json:"whitelisted"`
	159	Paths [][]string `json:"paths"`
	160	Parents []string `json:"parents"`
	161	Type string `json:"type"`
	162	Valid bool `json:"valid"`
	163	} `json:"microsoft"`
	164	Apple struct {
	165	Blacklisted bool `json:"blacklisted"`
	166	HadTrustedPath bool `json:"had_trusted_path"`
	167	InRevocationSet bool `json:"in_revocation_set"`
	168	TrustedPath bool `json:"trusted_path"`
	169	WasValid bool `json:"was_valid"`
	170	Whitelisted bool `json:"whitelisted"`
	171	Paths [][]string `json:"paths"`
	172	Parents []string `json:"parents"`
	173	Type string `json:"type"`
	174	Valid bool `json:"valid"`
	175	} `json:"apple"`
	176	Revoked bool `json:"revoked"`
	177	GoogleCtPrimary struct {
	178	Blacklisted bool `json:"blacklisted"`
	179	HadTrustedPath bool `json:"had_trusted_path"`
	180	InRevocationSet bool `json:"in_revocation_set"`
	181	TrustedPath bool `json:"trusted_path"`
	182	WasValid bool `json:"was_valid"`
	183	Whitelisted bool `json:"whitelisted"`
	184	Paths [][]string `json:"paths"`
	185	Parents []string `json:"parents"`
	186	Type string `json:"type"`
	187	Valid bool `json:"valid"`
	188	} `json:"google_ct_primary"`
	189	OcspRevocation struct {
	190	NextUpdate time.Time `json:"next_update"`
	191	Revoked bool `json:"revoked"`
	192	} `json:"ocsp_revocation"`
	193	CrlRevocation struct {
	194	Revoked bool `json:"revoked"`
	195	} `json:"crl_revocation"`
	196	CrlError string `json:"crl_error"`
	197	} `json:"validation"`
	198	Zlint struct {
	199	NoticesPresent bool `json:"notices_present"`
	200	WarningsPresent bool `json:"warnings_present"`
	201	ErrorsPresent bool `json:"errors_present"`
	202	FatalsPresent bool `json:"fatals_present"`
	203	Lints struct {
	204	NSubjectCommonNameIncluded bool `json:"n_subject_common_name_included"`
	205	} `json:"lints"`
	206	Version int `json:"version"`
	207	} `json:"zlint"`
	208	Precert bool `json:"precert"`
	209	}
	210
	211	type errorResponse struct {
	212	Error string `json:"error"`
	213	ErrorCode int `json:"error_code"`
	214	}

+75

-64

driver/crtsh/crtsh.go less more

	0	// Package crtsh implements an unofficial API client for Comodo's
	1	// Certificate Transparency search
	2	// https://crt.sh/
	3	//
	4	// As the API is unofficial and has been reverse engineered it may stop working
	5	// at any time and comes with no guarantees.
	6	// view SQL excample: https://crt.sh/?showSQL=Y&exclude=expired&q=
	7	//
0	8	package crtsh
1
2		/*
3		* This file implements an unofficial API client for Comodo's
4		* Certificate Transparency search
5		* https://crt.sh/
6		*
7		* As the API is unofficial and has been reverse engineered it may stop working
8		* at any time and comes with no guarantees.
9		*/
10
11		// TODO running in verbose gives error: pq: unnamed prepared statement does not exist
12	9
13	10	import (
14	11	"database/sql"
15	12	"fmt"
	13	"log"
16	14	"path"
17	15	"time"
18	16

22	20	_ "github.com/lib/pq" // portgresql
23	21	)
24	22
25		const connStr = "postgresql://[email protected]/certwatch?sslmode=disable"
	23	const connStr = "postgresql://[email protected]/certwatch?sslmode=disable&fallback_application_name=certgraph&binary_parameters=yes"
26	24	const driverName = "crtsh"
	25
	26	const debug = false
27	27
28	28	func init() {
29	29	driver.AddDriver(driverName)

75	75	}
76	76
77	77	d.db, err = sql.Open("postgres", connStr)
78
79		d.setSQLTimeout(d.timeout.Seconds())
	78	if err != nil {
	79	return nil, err
	80	}
	81
	82	err = d.setSQLTimeout(d.timeout.Seconds())
80	83
81	84	return d, err
82	85	}

97	100	driver: d,
98	101	}
99	102
100		queryStr := ""
101
102		if d.includeSubdomains {
103		if d.includeExpired {
104		queryStr = `SELECT digest(certificate.certificate, 'sha256') sha256
105		FROM certificate_identity, certificate
106		WHERE certificate.id = certificate_identity.certificate_id
107		AND (reverse(lower(certificate_identity.name_value)) LIKE reverse(lower('%%.'\|\|$1))
108		OR reverse(lower(certificate_identity.name_value)) LIKE reverse(lower($1)))
109		LIMIT $2`
110		} else {
111		queryStr = `SELECT digest(certificate.certificate, 'sha256') sha256
112		FROM certificate_identity, certificate
113		WHERE certificate.id = certificate_identity.certificate_id
114		AND x509_notAfter(certificate.certificate) > statement_timestamp()
115		AND (reverse(lower(certificate_identity.name_value)) LIKE reverse(lower('%%.'\|\|$1))
116		OR reverse(lower(certificate_identity.name_value)) LIKE reverse(lower($1)))
117		LIMIT $2`
118		}
119		} else {
120		if d.includeExpired {
121		queryStr = `SELECT digest(certificate.certificate, 'sha256') sha256
122		FROM certificate_identity, certificate
123		WHERE certificate.id = certificate_identity.certificate_id
124		AND reverse(lower(certificate_identity.name_value)) LIKE reverse(lower($1))
125		LIMIT $2`
126		} else {
127		queryStr = `SELECT digest(certificate.certificate, 'sha256') sha256
128		FROM certificate_identity, certificate
129		WHERE certificate.id = certificate_identity.certificate_id
130		AND x509_notAfter(certificate.certificate) > statement_timestamp()
131		AND reverse(lower(certificate_identity.name_value)) LIKE reverse(lower($1))
132		LIMIT $2`
133		}
134		}
135
136		if d.includeSubdomains {
137		domain = fmt.Sprintf("%%.%s", domain)
138		}
	103	queryStr := `WITH myconstants (include_expired, include_subdomains) as (
	104	values ($1::bool, $2::bool)
	105	),
	106	ci AS (
	107	SELECT digest(sub.CERTIFICATE, 'sha256') sha256, -- added
	108	min(sub.CERTIFICATE_ID) ID,
	109	min(sub.ISSUER_CA_ID) ISSUER_CA_ID,
	110	array_agg(DISTINCT sub.NAME_VALUE) NAME_VALUES
	111	FROM (SELECT *
	112	FROM certificate_and_identities cai, myconstants
	113	WHERE plainto_tsquery('certwatch', $4) @@ identities(cai.CERTIFICATE)
	114	AND (
	115	-- domain only
	116	(NOT myconstants.include_subdomains AND cai.NAME_VALUE ILIKE ($4))
	117	OR
	118	-- include sub-domains
	119	(myconstants.include_subdomains AND (cai.NAME_VALUE ILIKE ($4) OR cai.NAME_VALUE ILIKE ('%.' \|\| $4)))
	120	)
	121	AND (
	122	-- added
	123	cai.NAME_TYPE = '2.5.4.3' -- commonName
	124	OR
	125	cai.NAME_TYPE = 'san:dNSName' -- dNSName
	126	)
	127	AND
	128	-- include expired?
	129	(myconstants.include_expired OR (coalesce(x509_notAfter(cai.CERTIFICATE), 'infinity'::timestamp) >= date_trunc('year', now() AT TIME ZONE 'UTC')
	130	AND x509_notAfter(cai.CERTIFICATE) >= now() AT TIME ZONE 'UTC'))
	131	LIMIT $3
	132	) sub
	133	GROUP BY sub.CERTIFICATE
	134	)
	135	SELECT
	136	ci.sha256 -- added
	137	--array_to_string(ci.name_values, chr(10)) name_value,
	138	--ci.id id
	139	FROM ci;`
139	140
140	141	try := 0
141	142	var err error

143	144	for try < 5 {
144	145	// this is a hack while crt.sh gets there stuff togeather
145	146	try++
146		rows, err = d.db.Query(queryStr, domain, d.queryLimit)
	147	if debug {
	148	log.Printf("QueryDomain try %d: %s", try, queryStr)
	149	}
	150	rows, err = d.db.Query(queryStr, d.includeExpired, d.includeSubdomains, d.queryLimit, domain)
147	151	if err == nil {
148	152	break
	153	}
	154	if debug {
	155	log.Printf("crtsh pq error on domain %q: %s", domain, err.Error())
149	156	}
150	157	}
151	158	/*if try > 1 {

164	171	results.fingerprints.Add(domain, fingerprint.FromHashBytes(hash))
165	172	}
166	173
	174	if debug {
	175	log.Printf("crtsh: got %d results for %s.", len(results.fingerprints[domain]), domain)
	176	}
	177
167	178	return results, nil
168	179	}
169	180

172	183	certNode.Fingerprint = fp
173	184	certNode.Domains = make([]string, 0, 5)
174	185
175		queryStr := `SELECT DISTINCT certificate_identity.name_value
176		FROM certificate, certificate_identity
177		WHERE certificate.id = certificate_identity.certificate_id
178		AND certificate_identity.name_type in ('dNSName', 'commonName')
179		AND digest(certificate.certificate, 'sha256') = $1`
	186	queryStr := `SELECT DISTINCT name_value FROM certificate_and_identities WHERE digest(certificate, 'sha256') = $1;`
180	187
181	188	try := 0
182	189	var err error

198	205
199	206	for rows.Next() {
200	207	var domain string
201		rows.Scan(&domain)
	208	err = rows.Scan(&domain)
	209	if err != nil {
	210	return nil, err
	211	}
202	212	certNode.Domains = append(certNode.Domains, domain)
203	213	}
204	214
205	215	if d.save {
206	216	var rawCert []byte
207		queryStr = `SELECT certificate.certificate
208		FROM certificate
209		WHERE digest(certificate.certificate, 'sha256') = $1`
	217	queryStr = `SELECT certificate FORM certificate_and_identities WHERE digest(certificate, 'sha256') = $1;`
210	218	row := d.db.QueryRow(queryStr, fp[:])
211	219	err = row.Scan(&rawCert)
212	220	if err != nil {
213	221	return certNode, err
214	222	}
215	223
216		driver.RawCertToPEMFile(rawCert, path.Join(d.savePath, fp.HexString())+".pem")
	224	err = driver.RawCertToPEMFile(rawCert, path.Join(d.savePath, fp.HexString())+".pem")
	225	if err != nil {
	226	return certNode, err
	227	}
217	228	}
218	229
219	230	return certNode, nil

+4

-1

driver/driver.go less more

	0	// Package driver exposes interfaces and types certgraph drivers must implement
0	1	package driver
1	2
2	3	import (

7	8	"github.com/lanrat/certgraph/fingerprint"
8	9	"github.com/lanrat/certgraph/status"
9	10	)
	11
	12	// TODO add context instead of timeout on all requests
10	13
11	14	// Drivers contains all the drivers that have been registered
12	15	var Drivers []string

65	68	certResult := new(CertResult)
66	69
67	70	// generate Fingerprint
68		certResult.Fingerprint = fingerprint.FromBytes(cert.Raw)
	71	certResult.Fingerprint = fingerprint.FromRawCertBytes(cert.Raw)
69	72
70	73	// domains
71	74	// used to ensure uniq entries in domains array

+15

-13

driver/google/google.go less more

	0	// Package google file implements an unofficial API client for Google's
	1	// Certificate Transparency search
	2	// https://transparencyreport.google.com/https/certificates
	3	//
	4	// As the API is unofficial and has been reverse engineered it may stop working
	5	// at any time and comes with no guarantees.
	6	//
0	7	package google
1
2		/*
3		* This file implements an unofficial API client for Google's
4		* Certificate Transparency search
5		* https://transparencyreport.google.com/https/certificates
6		*
7		* As the API is unofficial and has been reverse engineered it may stop working
8		* at any time and comes with no guarantees.
9		*/
10	8
11	9	import (
12	10	"encoding/json"

29	27	}
30	28
31	29	// Base URLs for Google's CT API
32		const searchURL1 = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch?include_expired=false&include_subdomains=false&domain=example.com"
33		const searchURL2 = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch/page?p=DEADBEEF"
34		const certURL = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certbyhash?hash=DEADBEEF"
	30	const (
	31	searchURL1 = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch?include_expired=false&include_subdomains=false&domain=example.com"
	32	searchURL2 = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch/page?p=DEADBEEF"
	33	certURL = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certbyhash?hash=DEADBEEF"
	34	//summaryURL is not currently used
	35	//summaryURL = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/summary"
	36	)
35	37
36	38	type googleCT struct {
37	39	maxPages float64 // this is a float because that is the type automatically decoded from the JSON response

157	159	foundCerts := raw[0][1].([]interface{})
158	160	for _, foundCert := range foundCerts {
159	161	certHash := foundCert.([]interface{})[5].(string)
160		certFP := fingerprint.FromB64(certHash)
	162	certFP := fingerprint.FromB64Hash(certHash)
161	163	results.fingerprints.Add(domain, certFP)
162	164	}
163	165	//fmt.Println("Page:", pageInfo[3])

+3

-2

driver/http/http.go less more

	0	// Package http implements a certgraph driver for obtaining SSL certificates over https
0	1	package http
1	2
2	3	import (

52	53	if found {
53	54	return cert, nil
54	55	}
55		return nil, fmt.Errorf("Certificate with Fingerprint %s not found", fp.HexString())
	56	return nil, fmt.Errorf("certificate with Fingerprint %s not found", fp.HexString())
56	57	}
57	58
58	59	// Driver creates a new SSL driver for HTTP Connections

150	151
151	152	// save
152	153	if c.parent.save && len(connState.PeerCertificates) > 0 {
153		driver.CertsToPEMFile(connState.PeerCertificates, path.Join(c.parent.savePath, certResult.Fingerprint.HexString())+".pem")
	154	err = driver.CertsToPEMFile(connState.PeerCertificates, path.Join(c.parent.savePath, certResult.Fingerprint.HexString())+".pem")
154	155	}
155	156
156	157	return conn, err

+131

-0

driver/multi/multi_driver.go less more

	0	// Package multi exposes a generic driver interface allowing you to merge the results of multiple other drivers
	1	package multi
	2
	3	import (
	4	"errors"
	5	"fmt"
	6	"strings"
	7	"sync"
	8
	9	"github.com/lanrat/certgraph/driver"
	10	"github.com/lanrat/certgraph/fingerprint"
	11	"github.com/lanrat/certgraph/status"
	12	"golang.org/x/sync/errgroup"
	13	)
	14
	15	type multiDriver struct {
	16	drivers []driver.Driver
	17	}
	18
	19	// Driver returns a new instance of multi driver for the provided drivers
	20	func Driver(drivers []driver.Driver) driver.Driver {
	21	md := new(multiDriver)
	22	md.drivers = drivers
	23	return md
	24	}
	25
	26	func (d *multiDriver) GetName() string {
	27	names := make([]string, 0, len(d.drivers))
	28	for _, driver := range d.drivers {
	29	names = append(names, driver.GetName())
	30	}
	31	return fmt.Sprintf("multi[%s]", strings.Join(names, ","))
	32	}
	33
	34	func (d *multiDriver) QueryDomain(domain string) (driver.Result, error) {
	35	r := newResult(domain)
	36	var group errgroup.Group
	37	for _, d := range d.drivers {
	38	goFunc := func(localDriver driver.Driver) func() error {
	39	return func() error {
	40	return func(localDriver driver.Driver) error {
	41	result, err := localDriver.QueryDomain(domain)
	42	if err != nil {
	43	return err
	44	}
	45	return r.add(result)
	46	}(localDriver)
	47	}
	48	}
	49
	50	group.Go(goFunc(d))
	51	}
	52	err := group.Wait()
	53	if err != nil {
	54	return nil, err
	55	}
	56	return r, nil
	57	}
	58
	59	func newResult(host string) *multiResult {
	60	r := new(multiResult)
	61	r.host = host
	62	r.results = make([]driver.Result, 0, 2)
	63	r.fingerprints = make(driver.FingerprintMap)
	64	return r
	65	}
	66
	67	type multiResult struct {
	68	host string
	69	results []driver.Result
	70	resultLock sync.Mutex // also protects fingerprints
	71	fingerprints driver.FingerprintMap
	72	}
	73
	74	func (c *multiResult) add(r driver.Result) error {
	75	c.resultLock.Lock()
	76	defer c.resultLock.Unlock()
	77	fpm, err := r.GetFingerprints()
	78	if err != nil {
	79	return err
	80	}
	81	for domain := range fpm {
	82	for _, fp := range fpm[domain] {
	83	// TODO does not dedupe across drivers
	84	c.fingerprints.Add(domain, fp)
	85	}
	86	}
	87
	88	c.results = append(c.results, r)
	89	return nil
	90	}
	91
	92	func (c multiResult) QueryCert(fp fingerprint.Fingerprint) (driver.CertResult, error) {
	93	for _, result := range c.results {
	94	cr, err := result.QueryCert(fp)
	95	if err != nil {
	96	return nil, err
	97	}
	98	if cr != nil {
	99	return cr, nil
	100	}
	101	}
	102	return nil, errors.New("unable to find working driver with QueryCert()")
	103	}
	104
	105	func (c *multiResult) GetFingerprints() (driver.FingerprintMap, error) {
	106	return c.fingerprints, nil
	107	}
	108
	109	func (c *multiResult) GetStatus() status.Map {
	110	// TODO nest other status inside
	111	return status.NewMap(c.host, status.New(status.MULTI))
	112	}
	113
	114	func (c *multiResult) GetRelated() ([]string, error) {
	115	relatedMap := make(map[string]bool)
	116	for _, result := range c.results {
	117	related, err := result.GetRelated()
	118	if err != nil {
	119	return nil, err
	120	}
	121	for _, r := range related {
	122	relatedMap[r] = true
	123	}
	124	}
	125	related := make([]string, 0, len(relatedMap))
	126	for r := range relatedMap {
	127	related = append(related, r)
	128	}
	129	return related, nil
	130	}

+6

-3

driver/save.go less more

16	16	}
17	17	defer f.Close()
18	18	for _, cert := range certs {
19		pem.Encode(f, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})
	19	err = pem.Encode(f, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})
	20	if err != nil {
	21	return err
	22	}
20	23	}
21	24	return nil
22	25	}

31	34	return err
32	35	}
33	36	defer f.Close()
34		pem.Encode(f, &pem.Block{Type: "CERTIFICATE", Bytes: cert})
35		return nil
	37	err = pem.Encode(f, &pem.Block{Type: "CERTIFICATE", Bytes: cert})
	38	return err
36	39	}
37	40
38	41	func fileExists(f string) bool {

+4

-3

driver/smtp/smtp.go less more

	0	// Package smtp implements a certgraph driver for obtaining SSL certificates over smtp with STARTTLS
0	1	package smtp
1	2
2	3	import (

54	55	if found {
55	56	return cert, nil
56	57	}
57		return nil, fmt.Errorf("Certificate with Fingerprint %s not found", fp.HexString())
	58	return nil, fmt.Errorf("certificate with Fingerprint %s not found", fp.HexString())
58	59	}
59	60
60	61	// Driver creates a new SSL driver for SMTP Connections

133	134
134	135	// save
135	136	if d.save && len(certs) > 0 {
136		driver.CertsToPEMFile(certs, path.Join(d.savePath, certResult.Fingerprint.HexString())+".pem")
	137	err = driver.CertsToPEMFile(certs, path.Join(d.savePath, certResult.Fingerprint.HexString())+".pem")
137	138	}
138	139
139		return results, nil
	140	return results, err
140	141	}
141	142
142	143	// getMX returns the MX records for the provided domain

+26

-14

fingerprint/fingerprint.go less more

	0	// Package fingerprint defines types to define a certificate fingerprint for certgraph
0	1	package fingerprint
1	2
2	3	import (
3	4	"crypto/sha256"
4	5	"encoding/base64"
	6	"encoding/hex"
5	7	"fmt"
	8	"log"
6	9	)
7	10
8	11	// Fingerprint sha256 of certificate bytes
9	12	type Fingerprint [sha256.Size]byte
10	13
11		// HexString print Fingerprint as hex
	14	// HexString print Fingerprint as uppercase hex
12	15	func (fp *Fingerprint) HexString() string {
13	16	return fmt.Sprintf("%X", *fp)
14	17	}

16	19	// FromHashBytes returns a Fingerprint generated by the first len(Fingerprint) bytes
17	20	func FromHashBytes(data []byte) Fingerprint {
18	21	var fp Fingerprint
19		/*if len(data) != sha256.Size {
20		v("Data is not correct SHA256 size", data)
21		}*/
	22	if len(data) != len(fp) {
	23	log.Printf("len(data) %d\tlen(fp): %d", len(data), len(fp))
	24	// this should error....
	25	}
22	26	for i := 0; i < len(data) && i < len(fp); i++ {
23	27	fp[i] = data[i]
24	28	}
25	29	return fp
26	30	}
27	31
28		// FromBytes returns a Fingerprint generated by the provided bytes
29		func FromBytes(data []byte) Fingerprint {
30		var fp Fingerprint
31		fp = sha256.Sum256(data)
	32	// FromRawCertBytes returns a Fingerprint generated by the provided bytes
	33	func FromRawCertBytes(data []byte) Fingerprint {
	34	fp := sha256.Sum256(data)
32	35	return fp
33	36	}
34	37
35		// FromB64 returns a Fingerprint from a base64 encoded hash string
36		func FromB64(hash string) Fingerprint {
37		data, _ := base64.StdEncoding.DecodeString(hash)
38		/*if err != nil {
39		fmt.Println(err)
40		}*/
	38	// FromB64Hash returns a Fingerprint from a base64 encoded hash string
	39	func FromB64Hash(hash string) Fingerprint {
	40	data, err := base64.StdEncoding.DecodeString(hash)
	41	if err != nil {
	42	panic(err)
	43	}
41	44	return FromHashBytes(data)
	45	}
	46
	47	// FromHexHash returns a Fingerprint from a hex encoded hash string
	48	func FromHexHash(hash string) Fingerprint {
	49	decoded, err := hex.DecodeString(hash)
	50	if err != nil {
	51	panic(err)
	52	}
	53	return FromHashBytes(decoded)
42	54	}
43	55
44	56	// B64Encode returns the b64 string of a Fingerprint

+89

-0

fingerprint/fp_test.go less more

	0	package fingerprint_test
	1
	2	import (
	3	"crypto/sha256"
	4	"encoding/base64"
	5	"strings"
	6	"testing"
	7
	8	"github.com/lanrat/certgraph/fingerprint"
	9	)
	10
	11	const rawCert = "MIID/TCCA4KgAwIBAgIQBV74EmrgijxarGYRe4auizAKBggqhkjOPQQDAzBWMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMTAwLgYDVQQDEydEaWdpQ2VydCBUTFMgSHlicmlkIEVDQyBTSEEzODQgMjAyMCBDQTEwHhcNMjIwNDIwMDAwMDAwWhcNMjMwNDIwMjM1OTU5WjBmMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEVMBMGA1UEChMMR2l0SHViLCBJbmMuMRMwEQYDVQQDEwpnaXRodWIuY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEjkAQ9bMD0dVDhUlfevxOChhxQME0Sb7kZr7+3T/gW6CW4eduvDZxsQUwa37mhUXMzF88gh+FsUy9TieoqZhasKOCAiAwggIcMB8GA1UdIwQYMBaAFAq8CCkXjKU5bXoOzjPHLrPt+8N6MB0GA1UdDgQWBBQJJ/08CmhEtgPojKO+W3TVwfJnaTAlBgNVHREEHjAcggpnaXRodWIuY29tgg53d3cuZ2l0aHViLmNvbTAOBgNVHQ8BAf8EBAMCB4AwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMIGbBgNVHR8EgZMwgZAwRqBEoEKGQGh0dHA6Ly9jcmwzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRMU0h5YnJpZEVDQ1NIQTM4NDIwMjBDQTEtMS5jcmwwRqBEoEKGQGh0dHA6Ly9jcmw0LmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRMU0h5YnJpZEVDQ1NIQTM4NDIwMjBDQTEtMS5jcmwwPgYDVR0gBDcwNTAzBgZngQwBAgIwKTAnBggrBgEFBQcCARYbaHR0cDovL3d3dy5kaWdpY2VydC5jb20vQ1BTMIGFBggrBgEFBQcBAQR5MHcwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0LmNvbTBPBggrBgEFBQcwAoZDaHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0VExTSHlicmlkRUNDU0hBMzg0MjAyMENBMS0xLmNydDAJBgNVHRMEAjAAMBMGCisGAQQB1nkCBAMBAf8EAgUAMAoGCCqGSM49BAMDA2kAMGYCMQC/16+UbmbTo4bcahQGLY2+SrWtge+DC2dcEY2pv1Cwn3YUi51uqEE+v7U6PUvWWbQCMQCslhV/IibG55Uoa6F/hNpa21ZqEhp38u7CHTFb+6HGbLi0CtbSjgc1mn/yEt5pFu0="
	12	const fpHashHex = "46a1fe1780fd9a05a5529906ed08a5fea2cfe63567c9fdeb62c18ba74fae35d5"
	13	const fpHashB64 = "RqH+F4D9mgWlUpkG7Qil/qLP5jVnyf3rYsGLp0+uNdU="
	14
	15	func TestFromHashBytes(t *testing.T) {
	16
	17	data, err := base64.StdEncoding.DecodeString(rawCert)
	18	if err != nil {
	19	t.Errorf("error on b64 decode: %s", err.Error())
	20	}
	21
	22	dataHashBytes := sha256.Sum256(data)
	23
	24	fp := fingerprint.FromHashBytes(dataHashBytes[:])
	25	uppercaseHexHash := strings.ToUpper(fpHashHex)
	26	hashHex := fp.HexString()
	27
	28	if hashHex != uppercaseHexHash {
	29	t.Errorf("fingerprint error, expected hex hash [%s] got [%s]", uppercaseHexHash, hashHex)
	30	}
	31
	32	hashB64 := fp.B64Encode()
	33
	34	if hashB64 != fpHashB64 {
	35	t.Errorf("fingerprint error, expected b64 hash [%s] got [%s]", hashB64, fpHashB64)
	36	}
	37	}
	38
	39	func TestFromRawCertBytes(t *testing.T) {
	40
	41	data, err := base64.StdEncoding.DecodeString(rawCert)
	42	if err != nil {
	43	t.Errorf("error on b64 decode: %s", err.Error())
	44	}
	45
	46	fp := fingerprint.FromRawCertBytes(data)
	47	uppercaseHash := strings.ToUpper(fpHashHex)
	48	hashHex := fp.HexString()
	49
	50	if hashHex != uppercaseHash {
	51	t.Errorf("fingerprint error, expected hex hash [%s] got [%s]", uppercaseHash, hashHex)
	52	}
	53
	54	hashB64 := fp.B64Encode()
	55
	56	if hashB64 != fpHashB64 {
	57	t.Errorf("fingerprint error, expected b64 hash [%s] got [%s]", hashB64, fpHashB64)
	58	}
	59	}
	60
	61	func TestFromB64Hash(t *testing.T) {
	62
	63	fp := fingerprint.FromB64Hash(fpHashB64)
	64
	65	uppercaseHash := strings.ToUpper(fpHashHex)
	66	hashHex := fp.HexString()
	67
	68	if hashHex != uppercaseHash {
	69	t.Errorf("fingerprint error, expected hex hash [%s] got [%s]", uppercaseHash, hashHex)
	70	}
	71
	72	hashB64 := fp.B64Encode()
	73
	74	if hashB64 != fpHashB64 {
	75	t.Errorf("fingerprint error, expected b64 hash [%s] got [%s]", hashB64, fpHashB64)
	76	}
	77	}
	78
	79	func TestFromHexHash(t *testing.T) {
	80
	81	fp := fingerprint.FromHexHash(fpHashHex)
	82
	83	hashB64 := fp.B64Encode()
	84
	85	if fpHashB64 != hashB64 {
	86	t.Errorf("fingerprint error, expected b64 hash [%s] got [%s]", fpHashHex, hashB64)
	87	}
	88	}

+6

-4

go.mod less more

0	0	module github.com/lanrat/certgraph
1	1
2	2	require (
3		github.com/lib/pq v1.0.0
4		github.com/weppos/publicsuffix-go v0.4.0
5		golang.org/x/net v0.0.0-20180906233101-161cd47e91fd // indirect
6		golang.org/x/text v0.3.0 // indirect
	3	github.com/lib/pq v1.10.5
	4	github.com/weppos/publicsuffix-go v0.15.0
	5	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
	6	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
7	7	)
	8
	9	go 1.16

+18

-0

go.sum less more

	0	github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ=
	1	github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
	2	github.com/weppos/publicsuffix-go v0.15.0 h1:2uQCwDczZ8YZe5uD0mM3sXRoZYA74xxPuiKK8LdPcGQ=
	3	github.com/weppos/publicsuffix-go v0.15.0/go.mod h1:HYux0V0Zi04bHNwOHy4cXJVz/TQjYonnF6aoYhj+3QE=
	4	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
	5	golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
	6	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
	7	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
	8	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
	9	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
	10	golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
	11	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
	12	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
	13	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
	14	golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
	15	golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
	16	golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
	17	golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

+13

-9

graph/cert_node.go less more

2	2	import (
3	3	"fmt"
4	4	"strings"
	5	"sync"
5	6
6	7	"github.com/lanrat/certgraph/dns"
7	8	"github.com/lanrat/certgraph/fingerprint"

9	10
10	11	// CertNode graph node to store certificate information
11	12	type CertNode struct {
12		Fingerprint fingerprint.Fingerprint
13		Domains []string
14		foundMap map[string]bool
	13	Fingerprint fingerprint.Fingerprint
	14	Domains []string
	15	foundMap map[string]bool
	16	foundMapLock sync.Mutex
15	17	}
16	18
17	19	func (c *CertNode) String() string {

29	31
30	32	// AddFound adds a driver name to the source of the certificate
31	33	func (c *CertNode) AddFound(driver string) {
	34	c.foundMapLock.Lock()
	35	defer c.foundMapLock.Unlock()
32	36	if c.foundMap == nil {
33	37	c.foundMap = make(map[string]bool)
34	38	}

56	60	return false
57	61	}
58	62
59		// TLDPlus1Count the number of tld+1 domains in the certificate
60		func (c *CertNode) TLDPlus1Count() int {
61		tldPlus1Domains := make(map[string]bool)
	63	// ApexCount the number of tld+1 domains in the certificate
	64	func (c *CertNode) ApexCount() int {
	65	apexDomains := make(map[string]bool)
62	66	for _, domain := range c.Domains {
63		tldPlus1, err := dns.TLDPlus1(domain)
	67	apexDomain, err := dns.ApexDomain(domain)
64	68	if err != nil {
65	69	continue
66	70	}
67		tldPlus1Domains[tldPlus1] = true
	71	apexDomains[apexDomain] = true
68	72	}
69		return len(tldPlus1Domains)
	73	return len(apexDomains)
70	74	}
71	75
72	76	// ToMap returns a map of the CertNode's fields (weak serialization)

+5

-4

graph/domain_node.go less more

21	21	HasDNS bool
22	22	}
23	23
24		// NewDomainNode constructor for DomainNode, converts domain to nonWildcard
	24	// NewDomainNode constructor for DomainNode, converts domain to lower nonWildcard
25	25	func NewDomainNode(domain string, depth uint) *DomainNode {
26	26	domainNode := new(DomainNode)
27		domainNode.Domain = nonWildcard(domain)
	27	domainNode.Domain = nonWildcard(strings.ToLower(domain))
28	28	domainNode.Depth = depth
29	29	domainNode.Certs = make(map[fingerprint.Fingerprint][]string)
30	30	domainNode.RelatedDomains = make(status.Map)

36	36	// in the map
37	37	func (d *DomainNode) AddRelatedDomains(domains []string) {
38	38	for _, domain := range domains {
	39	domain = strings.ToLower(domain)
39	40	if _, ok := d.RelatedDomains[domain]; ok {
40	41	continue
41	42	}

43	44	}
44	45	}
45	46
46		// CheckForDNS checks for the existence of DNS records for the domain's tld+1
	47	// CheckForDNS checks for the existence of DNS records for the domain's apex
47	48	// sets the value to the node and returns the result as well
48	49	func (d *DomainNode) CheckForDNS(timeout time.Duration) (bool, error) {
49	50	hasDNS, err := dns.HasRecordsCache(d.Domain, timeout)

74	75	return fingerprints
75	76	}
76	77
77		// get the string representation of a node
	78	// String returns the string representation of a node
78	79	func (d *DomainNode) String() string {
79	80	certString := ""
80	81	// Certs

+2

-8

graph/graph.go less more

	0	// Package graph implements the graph data structures used by certgraph to build the certificate graph
0	1	package graph
1	2
2	3	import (

18	19	func NewCertGraph() *CertGraph {
19	20	graph := new(CertGraph)
20	21	return graph
21		}
22
23		// LoadOrStoreCert will return the CertNode in the graph with the provided node's fingerprint, or store the node if it did not already exist
24		// returned bool is true if the CertNode was found, false if stored
25		func (graph CertGraph) LoadOrStoreCert(certNode CertNode) (*CertNode, bool) {
26		foundCertNode, ok := graph.certs.LoadOrStore(certNode.Fingerprint, certNode)
27		return foundCertNode.(*CertNode), ok
28	22	}
29	23
30	24	// AddCert add a CertNode to the graph

96	90	certNode := node.(*CertNode)
97	91	if !cdn && certNode.CDNCert() {
98	92	//v(domain, "-> CDN CERT")
99		} else if maxSANsSize > 0 && certNode.TLDPlus1Count() > maxSANsSize {
	93	} else if maxSANsSize > 0 && certNode.ApexCount() > maxSANsSize {
100	94	//v(domain, "-> Large CERT")
101	95	} else {
102	96	for _, neighbor := range certNode.Domains {

+5

-1

status/status.go less more

	0	// Package status defines the various status certgraph discovered hosts/certificates may have
0	1	package status
1	2
2	3	import (

61	62	ERROR = iota
62	63	REDIRECT = iota
63	64	CT = iota
	65	MULTI = iota
64	66	)
65	67
66		// return domain status for printing
	68	// String returns the domain status for printing
67	69	func (status DomainStatus) String() string {
68	70	switch status {
69	71	case UNKNOWN:

82	84	return "Redirect"
83	85	case CT:
84	86	return "CT"
	87	case MULTI:
	88	return "MULTI"
85	89	}
86	90	return "?"
87	91	}

+26

-0

web/web.go less more

	0	// Package web defines a minimal web server for serving the web UI
	1	package web
	2
	3	import (
	4	"io/fs"
	5	"log"
	6	"net/http"
	7	)
	8
	9	// Serve starts a very basic webserver serving the embed web UI
	10	func Serve(addr string, data fs.FS) error {
	11	data, err := fs.Sub(data, "docs")
	12	if err != nil {
	13	return err
	14	}
	15	http.Handle("/", http.FileServer(http.FS(data)))
	16	return http.ListenAndServe(addr, logRequest(http.DefaultServeMux))
	17	}
	18
	19	// very minimal request logger
	20	func logRequest(handler http.Handler) http.Handler {
	21	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	22	log.Printf("%s %s %s\n", r.RemoteAddr, r.Method, r.URL)
	23	handler.ServeHTTP(w, r)
	24	})
	25	}