Commit run/474901aa-1dc3-49e8-9258-b474cc72de66/main - certgraph

+34

-0

.cspell.json less more

	0	// cSpell Settings
	1	// https://cspell.org/configuration/
	2	{
	3	// Version of the setting file. Always 0.2
	4	"version": "0.2",
	5	// language - current active spelling language
	6	"language": "en",
	7	// words - list of words to be always considered correct
	8	"words": [
	9	"censys",
	10	"certgraph",
	11	"crtsh"
	12	],
	13	// flagWords - list of words to be always considered incorrect
	14	// This is useful for offensive words and common spelling errors.
	15	// For example "hte" should be "the"
	16	"flagWords": [],
	17	// ignoreWords - a list of words to be ignored (even if they are in the flagWords).
	18	"ignoreWords": [
	19	"sanscap",
	20	"updatepsl",
	21	"weppos",
	22	"publicsuffix",
	23	"STARTTLS",
	24	"dedupe",
	25	"whitehouse",
	26	"httpse",
	27	"errgroup"
	28	],
	29	// ignorePaths - a list of globs to specify which files are to be ignored.
	30	"ignorePaths": [
	31	"LICENSE"
	32	]
	33	}

+4

-0

.dockerignore less more

	0	Dockerfile
	1	certgraph
	2	build/
	3	*.json⏎

+0

-4

~~.gitignore~~ less more

0		go.sum
1		certgraph
2		build/
3		*.json

+11

-0

Dockerfile less more

	0	FROM golang:alpine
	1
	2	RUN apk add --update git make
	3
	4	WORKDIR /src/certgraph
	5	ADD . .
	6
	7	ENV CGO_ENABLED=0
	8	RUN make install
	9
	10	ENTRYPOINT [ "certgraph" ]⏎

+35

-7

Makefile less more

0	0	GIT_DATE := $(shell git log -1 --date=short --pretty='%cd' \| tr -d -)
1	1	GIT_HASH := $(shell git rev-parse HEAD)
2	2
3		BUILD_FLAGS := -ldflags "-X main.gitDate=$(GIT_DATE) -X main.gitHash=$(GIT_HASH)"
	3	BUILD_FLAGS := -trimpath -ldflags "-w -s -X main.gitDate=$(GIT_DATE) -X main.gitHash=$(GIT_HASH)"
4	4
5		PLATFORMS := linux/amd64 linux/386 linux/arm darwin/amd64 windows/amd64 windows/386 openbsd/amd64
	5	PLATFORMS := linux/amd64 linux/386 linux/arm linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/386 openbsd/amd64
6	6	SOURCES := $(shell find . -maxdepth 1 -type f -name "*.go")
7		ALL_SOURCES = $(shell find . -type f -name '*.go')
	7	ALL_SOURCES = $(shell find . -type f -name '.go') go.mod docs/
8	8
9	9	temp = $(subst /, ,$@)
10	10	os = $(word 1, $(temp))
11	11	arch = $(word 2, $(temp))
12	12	ext = $(shell if [ "$(os)" = "windows" ]; then echo ".exe"; fi)
13	13
14		.PHONY: all release fmt clean serv $(PLATFORMS)
	14	.PHONY: all release fmt clean serv $(PLATFORMS) docker check deps update-deps
15	15
16	16	all: certgraph
17	17

25	25	CGO_ENABLED=0 GOOS=$(os) GOARCH=$(arch) go build $(BUILD_FLAGS) -o 'build/bin/$(os)/$(arch)/certgraph$(ext)' $(SOURCES)
26	26	mkdir -p build/$(GIT_DATE)/; cd build/bin/$(os)/$(arch)/; zip -r ../../../$(GIT_DATE)/certgraph-$(os)-$(arch)-$(GIT_DATE).zip .; cd ../../../
27	27
	28	docker: Dockerfile $(ALL_SOURCES)
	29	docker build -t lanrat/certgraph .
	30
	31	deps: go.mod
	32	GOPROXY=direct go mod download
	33	GOPROXY=direct go get -u all
	34
28	35	fmt:
29	36	gofmt -s -w -l .
30	37

32	39	go install $(BUILD_FLAGS)
33	40
34	41	clean:
35		rm -r certgraph build/
	42	rm -rf certgraph build/
36	43
37		serv:
38		(cd docs; python -m SimpleHTTPServer)
	44	check: \| lint check1 check2 vulncheck
	45
	46	check1:
	47	golangci-lint run
	48
	49	check2:
	50	staticcheck -f stylish -checks all ./...
	51
	52	vulncheck:
	53	govulncheck ./...
	54
	55	lint:
	56	golint ./...
	57
	58	serv: certgraph
	59	./certgraph --serve 127.0.0.1:8080
	60
	61	update-deps:
	62	go get -u
	63	go mod tidy
	64
	65	test:
	66	go test -v ./... \| grep -v "\[no test files\]"

+89

-33

README.md less more

0	0	# CertGraph
1		### A tool to crawl the graph of certificate Alternate Names
	1
	2	## A tool to crawl the graph of certificate Alternate Names
2	3
3	4	CertGraph crawls SSL certificates creating a directed graph where each domain is a node and the certificate alternative names for that domain's certificate are the edges to other domain nodes. New domains are printed as they are found. In Detailed mode upon completion the Graph's adjacency list is printed.
4	5

9	10	[Blog post with more information](https://lanrat.com/certgraph/)
10	11
11	12	## Usage
12		```
	13
	14	```console
13	15	Usage of ./certgraph: [OPTION]... HOST...
14		https://github.com/lanrat/certgraph
	16	https://github.com/lanrat/certgraph
15	17	OPTIONS:
	18	-apex
	19	for every domain found, add the apex domain of the domain's parent
16	20	-cdn
17		include certificates from CDNs
	21	include certificates from CDNs
	22	-censys-appid string
	23	censys API AppID
	24	-censys-secret string
	25	censys API Secret
18	26	-ct-expired
19		include expired certificates in certificate transparency search
	27	include expired certificates in certificate transparency search
20	28	-ct-subdomains
21		include sub-domains in certificate transparency search
	29	include sub-domains in certificate transparency search
22	30	-depth uint
23		maximum BFS depth to go (default 5)
	31	maximum BFS depth to go (default 5)
24	32	-details
25		print details about the domains crawled
	33	print details about the domains crawled
	34	-dns
	35	check for DNS records to determine if domain is registered
26	36	-driver string
27		driver to use [crtsh, google, http, smtp] (default "http")
	37	driver(s) to use [censys, crtsh, google, http, smtp] (default "http")
28	38	-json
29		print the graph as json, can be used for graph in web UI
30		-ns
31		check for NS records to determine if domain is registered
	39	print the graph as json, can be used for graph in web UI
32	40	-parallel uint
33		number of certificates to retrieve in parallel (default 10)
	41	number of certificates to retrieve in parallel (default 10)
	42	-regex string
	43	regex domains must match to be part of the graph
34	44	-sanscap int
35		maximum number of uniq TLD+1 domains in certificate to include, 0 has no limit (default 80)
	45	maximum number of uniq apex domains in certificate to include, 0 has no limit (default 80)
36	46	-save string
37		save certs to folder in PEM format
	47	save certs to folder in PEM format
	48	-serve string
	49	address:port to serve html UI on
38	50	-timeout uint
39		tcp timeout in seconds (default 10)
40		-tldplus1
41		for every domain found, add tldPlus1 of the domain's parent
	51	tcp timeout in seconds (default 10)
	52	-updatepsl
	53	Update the default Public Suffix List
42	54	-verbose
43		verbose logging
	55	verbose logging
44	56	-version
45		print version and exit
	57	print version and exit
46	58	```
47	59
48	60	## Drivers
49	61
50	62	CertGraph has multiple options for querying SSL certificates. The driver is responsible for retrieving the certificates for a given domain. Currently there are the following drivers:
51	63
52		* http this is the default driver which works by connecting to the hosts over HTTPS and retrieving the certificates from the SSL connection
53
54		* smtp like the http driver, but connects over port 25 and issues the starttls command to retrieve the certificates from the SSL connection
	64	* http this is the default driver which works by connecting to the hosts over HTTPS and retrieving the certificates from the SSL connection
55	65
56		* crtsh this driver searches Certificate Transparency logs via [crt.sh](https://crt.sh/). No packets are sent to any of the domains when using this driver
	66	* smtp like the http driver, but connects over port 25 and issues the starttls command to retrieve the certificates from the SSL connection
57	67
58		* google this is another Certificate Transparency driver that behaves like crtsh but uses the [Google Certificate Transparency Lookup Tool](https://transparencyreport.google.com/https/certificates)
	68	* censys this driver searches Certificate Transparency logs via [censys.io](https://search.censys.io/certificates). No packets are sent to any of the domains when using this driver. Requires Censys API keys
59	69
	70	* crtsh this driver searches Certificate Transparency logs via [crt.sh](https://crt.sh/). No packets are sent to any of the domains when using this driver
	71
	72	* google this is another Certificate Transparency driver that behaves like crtsh but uses the [Google Certificate Transparency Lookup Tool](https://transparencyreport.google.com/https/certificates)
60	73
61	74	## Example
62		```
	75
	76	```console
63	77	$ ./certgraph -details eff.org
64	78	eff.org 0 Good 42E3E4605D8BB4608EB64936E2176A98B97EBF2E0F8F93A64A6640713C7D4325
65	79	maps.eff.org 1 Good 42E3E4605D8BB4608EB64936E2176A98B97EBF2E0F8F93A64A6640713C7D4325

68	82	atlas.eff.org 1 Good 42E3E4605D8BB4608EB64936E2176A98B97EBF2E0F8F93A64A6640713C7D4325
69	83	kittens.eff.org 1 Good 42E3E4605D8BB4608EB64936E2176A98B97EBF2E0F8F93A64A6640713C7D4325
70	84	```
	85
71	86	The above output represents the adjacency list for the graph for the root domain `eff.org`. The adjacency list is in the form:
72	87	`Node Depth Status Cert-Fingerprint`
73	88
74	89	## [Releases](https://github.com/lanrat/certgraph/releases)
75	90
76		Precompiled releases will occasionally be uploaded to the [releases github page](https://github.com/lanrat/certgraph/releases). https://github.com/lanrat/certgraph/releases
	91	Pre-compiled releases will occasionally be uploaded to the [releases github page](https://github.com/lanrat/certgraph/releases). [https://github.com/lanrat/certgraph/releases](https://github.com/lanrat/certgraph/releases)
77	92
78		Also available in [BlackArch](https://blackarch.org).
	93	### [Docker](https://hub.docker.com/r/lanrat/certgraph/)
	94
	95	CertGraph is an automated build on the Docker Hub!
	96
	97	```console
	98	$ docker run --rm -it lanrat/certgraph example.com
	99	example.com
	100	www.example.net
	101	www.example.org
	102	www.example.com
	103	example.org
	104	example.net
	105	example.edu
	106	www.example.edu
	107	```
	108
	109	### Linux Distributions
	110
	111	* [BlackArch](https://blackarch.org)
	112	* [Kali Linux](https://www.kali.org/)
79	113
80	114	## Compiling
81	115
82		To compile certgraph you must have a working go 1.11 or newer compiler on your system, as certgraph makes use of go's modules for dependencies.
	116	To compile certgraph you must have a working go 1.16 or newer compiler on your system.
83	117	To compile for the running system compilation is as easy as running make
84		```
	118
	119	```console
85	120	certgraph$ make
86	121	go build -o certgraph certgraph.go
87	122	```
88	123
89	124	Alternatively you can use `go get` to install with this one-liner:
90		```
91		go get -u github.com/lanrat/certgraph
	125
	126	```console
	127	go install github.com/lanrat/certgraph@latest
92	128	```
93	129
94	130	## [Web UI](https://lanrat.github.io/certgraph/)
95	131
96		A web UI is provided in the docs folder and is accessible at the github pages url [https://lanrat.github.io/certgraph/](https://lanrat.github.io/certgraph/).
	132	A web UI is provided in the docs folder and is accessible at the github pages url [https://lanrat.github.io/certgraph/](https://lanrat.github.io/certgraph/), or can be run from the embedded web server by calling `certgraph --serve 127.0.0.1:8080`.
97	133
98	134	The web UI takes the output provided with the `-json` flag.
99	135	The JSON graph can be sent to the web interface as an uploaded file, remote URL, or as the query string using the data variable.

110	146
111	147	[![whitehouse.gov graph](https://cloud.githubusercontent.com/assets/164192/20861407/4775ff26-b944-11e6-888c-4d93e3333494.png)](https://lanrat.github.io/certgraph/?data=https://gist.githubusercontent.com/lanrat/96c47dfee0faaaad633cc830b7e3b997/raw/3c79fed837cb3202e220de21d2a8eb128f4bbd9f/whitehouse.json)
112	148
	149	## BygoneSSL detection
	150
	151	### Self Detection
	152
	153	CertGraph can be used to detect [BygoneSSL](https://insecure.design) DoS with the following options. CT-DRIVER can be any Certificate Transparency capable driver.
	154	Provide all known input domains you own. If any domains you do not own are printed, then you are vulnerable.
	155
	156	```console
	157	certgraph -depth 1 -driver CT-DRIVER -ct-subdomains -cdn -apex [DOMAIN]...
	158	```
	159
	160	### Bug Bounty
	161
	162	If you want to find a vulnerable site that has a bug bounty, certgraph can be used with the following options and any driver. But you will have better luck with a non Certificate Transparency driver to ensure that the certificates in question are actually in use
	163
	164	```console
	165	certgraph -cdn -dns -apex [DOMAIN]...
	166	```
	167
	168	And domains that print `* Missing DNS for` have vulnerable certificates that should be rotated.

+97

-29

certgraph.go less more

0	0	package main
1	1
	2	// cSpell:words certgraph crtsh
	3
2	4	import (
	5	"embed"
3	6	"encoding/json"
4	7	"flag"
5	8	"fmt"
6	9	"net/url"
7	10	"os"
	11	"regexp"
8	12	"strings"
9	13	"sync"
10	14	"time"
11	15
12	16	"github.com/lanrat/certgraph/dns"
13	17	"github.com/lanrat/certgraph/driver"
	18	"github.com/lanrat/certgraph/driver/censys"
14	19	"github.com/lanrat/certgraph/driver/crtsh"
15	20	"github.com/lanrat/certgraph/driver/google"
16	21	"github.com/lanrat/certgraph/driver/http"
	22	"github.com/lanrat/certgraph/driver/multi"
17	23	"github.com/lanrat/certgraph/driver/smtp"
18	24	"github.com/lanrat/certgraph/graph"
	25	"github.com/lanrat/certgraph/web"
19	26	)
20	27
	28	// version vars
21	29	var (
22	30	gitDate = "none"
23	31	gitHash = "master"
24	32	certGraph = graph.NewCertGraph()
25	33	)
26	34
	35	// temp flag vars
	36	var (
	37	timeoutSeconds uint
	38	regexString string
	39	)
	40
	41	// webContent holds our static web server content.
	42	//
	43	//go:embed docs/*
	44	var webContent embed.FS
	45
27	46	var certDriver driver.Driver
28	47
29	48	// config & flags
	49	// TODO move driver options to own struct
30	50	var config struct {
31	51	timeout time.Duration
32	52	verbose bool

40	60	includeCTExpired bool
41	61	cdn bool
42	62	maxSANsSize int
43		tldPlus1 bool
	63	apex bool
44	64	updatePSL bool
45	65	checkDNS bool
46	66	printVersion bool
	67	serve string
	68	regex *regexp.Regexp
47	69	}
48	70
49	71	func init() {
50		var timeoutSeconds uint
51	72	flag.BoolVar(&config.printVersion, "version", false, "print version and exit")
52	73	flag.UintVar(&timeoutSeconds, "timeout", 10, "tcp timeout in seconds")
53	74	flag.BoolVar(&config.verbose, "verbose", false, "verbose logging")
54		flag.StringVar(&config.driver, "driver", "http", fmt.Sprintf("driver to use [%s]", strings.Join(driver.Drivers, ", ")))
	75	flag.StringVar(&config.driver, "driver", "http", fmt.Sprintf("driver(s) to use [%s]", strings.Join(driver.Drivers, ", ")))
55	76	flag.BoolVar(&config.includeCTSubdomains, "ct-subdomains", false, "include sub-domains in certificate transparency search")
56	77	flag.BoolVar(&config.includeCTExpired, "ct-expired", false, "include expired certificates in certificate transparency search")
57		flag.IntVar(&config.maxSANsSize, "sanscap", 80, "maximum number of uniq TLD+1 domains in certificate to include, 0 has no limit")
	78	flag.IntVar(&config.maxSANsSize, "sanscap", 80, "maximum number of uniq apex domains in certificate to include, 0 has no limit")
58	79	flag.BoolVar(&config.cdn, "cdn", false, "include certificates from CDNs")
59	80	flag.BoolVar(&config.checkDNS, "dns", false, "check for DNS records to determine if domain is registered")
60		flag.BoolVar(&config.tldPlus1, "tldplus1", false, "for every domain found, add tldPlus1 of the domain's parent")
	81	flag.BoolVar(&config.apex, "apex", false, "for every domain found, add the apex domain of the domain's parent")
61	82	flag.BoolVar(&config.updatePSL, "updatepsl", false, "Update the default Public Suffix List")
62	83	flag.UintVar(&config.maxDepth, "depth", 5, "maximum BFS depth to go")
63	84	flag.UintVar(&config.parallel, "parallel", 10, "number of certificates to retrieve in parallel")
64	85	flag.BoolVar(&config.details, "details", false, "print details about the domains crawled")
65	86	flag.BoolVar(&config.printJSON, "json", false, "print the graph as json, can be used for graph in web UI")
66	87	flag.StringVar(&config.savePath, "save", "", "save certs to folder in PEM format")
	88	flag.StringVar(&config.serve, "serve", "", "address:port to serve html UI on")
	89	flag.StringVar(&regexString, "regex", "", "regex domains must match to be part of the graph")
	90
67	91	flag.Usage = func() {
68	92	fmt.Fprintf(os.Stderr, "Usage of %s: [OPTION]... HOST...\n\thttps://github.com/lanrat/certgraph\nOPTIONS:\n", os.Args[0])
69	93	flag.PrintDefaults()
70	94	}
	95	}
	96
	97	func main() {
71	98	flag.Parse()
72	99	config.timeout = time.Duration(timeoutSeconds) * time.Second
73		}
74
75		func main() {
	100	var err error
	101
76	102	// check for version flag
77	103	if config.printVersion {
78	104	fmt.Println(version())
	105	return
	106	}
	107
	108	// check for regex
	109	if len(regexString) > 0 {
	110	config.regex, err = regexp.Compile(regexString)
	111	if err != nil {
	112	e(err)
	113	return
	114	}
	115	}
	116
	117	if len(config.serve) > 0 {
	118	err = web.Serve(config.serve, webContent)
	119	e(err)
79	120	return
80	121	}
81	122

94	135
95	136	// update the public suffix list if required
96	137	if config.updatePSL {
97		err := dns.UpdatePublicSuffixList(config.timeout)
	138	err = dns.UpdatePublicSuffixList(config.timeout)
98	139	if err != nil {
99	140	e(err)
100	141	return

107	148	d := strings.ToLower(domain)
108	149	if len(d) > 0 {
109	150	startDomains = append(startDomains, cleanInput(d))
110		if config.tldPlus1 {
111		tldPlus1, err := dns.TLDPlus1(domain)
	151	if config.apex {
	152	apexDomain, err := dns.ApexDomain(domain)
112	153	if err != nil {
113	154	continue
114	155	}
115		startDomains = append(startDomains, tldPlus1)
	156	startDomains = append(startDomains, apexDomain)
116	157	}
117	158	}
118	159	}
119	160
120	161	// set driver
121		err := setDriver(config.driver)
	162	certDriver, err = setDriver(config.driver)
122	163	if err != nil {
123	164	fmt.Fprintln(os.Stderr, err)
124	165	return

145	186	v("Graph Depth:", certGraph.DomainDepth())
146	187	}
147	188
148		// setDriver sets the driver variable for the provided driver string and does any necessary driver prep work
	189	func setDriver(name string) (driver.Driver, error) {
	190	if strings.Contains(name, ",") {
	191	names := strings.Split(name, ",")
	192	drivers := make([]driver.Driver, 0, len(names))
	193	for _, driverName := range names {
	194	d, err := getDriverSingle(driverName)
	195	if err != nil {
	196	return nil, err
	197	}
	198	drivers = append(drivers, d)
	199	}
	200	return multi.Driver(drivers), nil
	201	}
	202	return getDriverSingle(name)
	203	}
	204
	205	// getDriverSingle sets the driver variable for the provided driver string and does any necessary driver prep work
149	206	// TODO make config generic and move this to driver module
150		func setDriver(driver string) error {
	207	func getDriverSingle(name string) (driver.Driver, error) {
151	208	var err error
152		switch driver {
	209	var d driver.Driver
	210	switch name {
153	211	case "google":
154		certDriver, err = google.Driver(50, config.savePath, config.includeCTSubdomains, config.includeCTExpired)
	212	d, err = google.Driver(50, config.savePath, config.includeCTSubdomains, config.includeCTExpired)
155	213	case "crtsh":
156		certDriver, err = crtsh.Driver(1000, config.timeout, config.savePath, config.includeCTSubdomains, config.includeCTExpired)
	214	d, err = crtsh.Driver(1000, config.timeout, config.savePath, config.includeCTSubdomains, config.includeCTExpired)
157	215	case "http":
158		certDriver, err = http.Driver(config.timeout, config.savePath)
	216	d, err = http.Driver(config.timeout, config.savePath)
159	217	case "smtp":
160		certDriver, err = smtp.Driver(config.timeout, config.savePath)
	218	d, err = smtp.Driver(config.timeout, config.savePath)
	219	case "censys":
	220	d, err = censys.Driver(config.savePath, config.includeCTSubdomains, config.includeCTExpired)
161	221	default:
162		return fmt.Errorf("Unknown driver name: %s", config.driver)
163		}
164		return err
	222	return nil, fmt.Errorf("unknown driver name: %s", config.driver)
	223	}
	224	return d, err
165	225	}
166	226
167	227	// verbose logging

236	296	<-threadPass
237	297	defer func() { threadPass <- true }()
238	298
	299	// regex match check
	300	if config.regex != nil && !config.regex.MatchString(domainNode.Domain) {
	301	// skip domain that does not match regex
	302	v("domain does not match regex, skipping :", domainNode.Domain)
	303	return
	304	}
	305
239	306	// operate on the node
240	307	v("Visiting", domainNode.Depth, domainNode.Domain)
241	308	visit(domainNode)

243	310	for _, neighbor := range certGraph.GetDomainNeighbors(domainNode.Domain, config.cdn, config.maxSANsSize) {
244	311	wg.Add(1)
245	312	domainNodeInputChan <- graph.NewDomainNode(neighbor, domainNode.Depth+1)
246		if config.tldPlus1 {
247		tldPlus1, err := dns.TLDPlus1(neighbor)
	313	if config.apex {
	314	apexDomain, err := dns.ApexDomain(neighbor)
248	315	if err != nil {
249	316	continue
250	317	}
251	318	wg.Add(1)
252		domainNodeInputChan <- graph.NewDomainNode(tldPlus1, domainNode.Depth+1)
	319	domainNodeInputChan <- graph.NewDomainNode(apexDomain, domainNode.Depth+1)
253	320	}
254	321	}
255	322	}(domainNode)

322	389	// fingerprints for the domain queried
323	390	fingerprints := fingerprintMap[domainNode.Domain]
324	391	for _, fp := range fingerprints {
325		// add certnode to graph
	392	// add certNode to graph
326	393	certNode, exists := certGraph.GetCert(fp)
327	394	if !exists {
328	395	// get cert details

340	407	domainNode.AddCertFingerprint(certNode.Fingerprint, certDriver.GetName())
341	408	}
342	409
343		// we dont process any other certificates returned, they will be collected
	410	// we don't process any other certificates returned, they will be collected
344	411	// when we process the related domains
345	412	}
346	413

353	420	if config.checkDNS && !domainNode.HasDNS {
354	421	// TODO print this in a better way
355	422	// TODO for debugging
356		realDomain, _ := dns.TLDPlus1(domainNode.Domain)
	423	realDomain, _ := dns.ApexDomain(domainNode.Domain)
357	424	fmt.Fprintf(os.Stdout, "* Missing DNS for: %s\n", realDomain)
358	425
359	426	}

384	451	options["sanscap"] = config.maxSANsSize
385	452	options["cdn"] = config.cdn
386	453	options["timeout"] = config.timeout
	454	options["regex"] = regexString
387	455	data["options"] = options
388	456	return data
389	457	}

+3

-2

debian/changelog less more

0		certgraph (20180911-0kali2) UNRELEASED; urgency=low
	0	certgraph (20220513+git20221025.1.26bc05e-0kali1) UNRELEASED; urgency=low
1	1
2	2	* Set upstream metadata fields: Bug-Database, Bug-Submit, Repository,
3	3	Repository-Browse.
	4	* New upstream snapshot.
4	5
5		-- Kali Janitor <[email protected]> Fri, 31 Jul 2020 00:51:32 -0000
	6	-- Kali Janitor <[email protected]> Fri, 25 Nov 2022 02:32:23 -0000
6	7
7	8	certgraph (20180911-0kali1) kali-dev; urgency=medium
8	9

+4

-3

dns/ns.go less more

	0	// Package dns adds utility functions for performing dns queries
0	1	package dns
1	2
2	3	import (

25	26	}
26	27
27	28	// HasRecords does NS, CNAME, A, and AAAA lookups with a timeout
28		// returns error when no NS found, does not use TLDPlus1
	29	// returns error when no NS found, does not use alexDomain
29	30	func HasRecords(domain string, timeout time.Duration) (bool, error) {
30	31	ctx, cancel := context.WithTimeout(context.Background(), timeout)
31	32	defer cancel()

67	68	return false, nil
68	69	}
69	70
70		// HasRecordsCache returns true if the domain has no DNS records (at the tldplus1 level)
	71	// HasRecordsCache returns true if the domain has no DNS records (at the apex domain level)
71	72	// uses a cache to store results to prevent lots of DNS lookups
72	73	func HasRecordsCache(domain string, timeout time.Duration) (bool, error) {
73		domain, err := TLDPlus1(domain)
	74	domain, err := ApexDomain(domain)
74	75	if err != nil {
75	76	return false, err
76	77	}

+3

-4

dns/publicsuffix.go less more

13	13	}
14	14	suffixListURL = "https://publicsuffix.org/list/public_suffix_list.dat"
15	15	suffixList = publicsuffix.DefaultList
16		nsCache = make(map[string]bool)
17	16	)
18	17
19	18	// UpdatePublicSuffixList gets a new copy of the public suffix list from the internat and updates the built in copy with the new rules

30	29	}
31	30	defer resp.Body.Close()
32	31	newSuffixList := publicsuffix.NewList()
33		newSuffixList.Load(resp.Body, suffixListParseOptions)
	32	_, err = newSuffixList.Load(resp.Body, suffixListParseOptions)
34	33	suffixList = newSuffixList
35	34	return err
36	35	}
37	36
38		// TLDPlus1 returns TLD+1 of domain
39		func TLDPlus1(domain string) (string, error) {
	37	// ApexDomain returns TLD+1 of domain
	38	func ApexDomain(domain string) (string, error) {
40	39	return publicsuffix.DomainFromListWithOptions(suffixList, domain, suffixListFindOptions)
41	40	}

+0

-1

docs/index.html less more

16	16	stroke: #333;
17	17	stroke-width: 1.5px;
18	18	}
19
20	19	.upload-drop-zone {
21	20	height: 200px;
22	21	border-width: 2px;

+248

-0

driver/censys/censys.go less more

	0	// Package censys file implements a client to search Censys's CT database
	1	// Certificate Transparency search
	2	// https://transparencyreport.google.com/https/certificates
	3	//
	4	// As the API is unofficial and has been reverse engineered it may stop working
	5	// at any time and comes with no guarantees.
	6	package censys
	7
	8	import (
	9	"bytes"
	10	"encoding/base64"
	11	"encoding/json"
	12	"flag"
	13	"fmt"
	14	"io"
	15	"log"
	16	"net/http"
	17	"path"
	18	"time"
	19
	20	"github.com/lanrat/certgraph/driver"
	21	"github.com/lanrat/certgraph/fingerprint"
	22	"github.com/lanrat/certgraph/status"
	23	)
	24
	25	const driverName = "censys"
	26
	27	var debug = false
	28
	29	// TODO support rate limits & pagination
	30
	31	var (
	32	defaultHTTPClient = &http.Client{}
	33
	34	appID = flag.String("censys-appid", "", "censys API AppID")
	35	secret = flag.String("censys-secret", "", "censys API Secret")
	36	)
	37
	38	func init() {
	39	driver.AddDriver(driverName)
	40	}
	41
	42	type censys struct {
	43	appID string
	44	secret string
	45	save bool
	46	savePath string
	47	includeSubdomains bool
	48	includeExpired bool
	49	}
	50
	51	type censysCertDriver struct {
	52	host string
	53	fingerprints driver.FingerprintMap
	54	driver *censys
	55	}
	56
	57	func (c *censysCertDriver) GetFingerprints() (driver.FingerprintMap, error) {
	58	return c.fingerprints, nil
	59	}
	60
	61	func (c *censysCertDriver) GetStatus() status.Map {
	62	return status.NewMap(c.host, status.New(status.CT))
	63	}
	64
	65	func (c *censysCertDriver) GetRelated() ([]string, error) {
	66	return make([]string, 0), nil
	67	}
	68
	69	func (c censysCertDriver) QueryCert(fp fingerprint.Fingerprint) (driver.CertResult, error) {
	70	return c.driver.QueryCert(fp)
	71	}
	72
	73	// TODO support pagination
	74	func domainSearchParam(domain string, includeExpired, includeSubdomain bool) certSearchParam {
	75	var s certSearchParam
	76	if includeSubdomain {
	77	s.Query = fmt.Sprintf("(parsed.names: %s )", domain)
	78	} else {
	79	s.Query = fmt.Sprintf("(parsed.names.raw: %s)", domain)
	80	}
	81	if !includeExpired {
	82	dateStr := time.Now().Format("2006-01-02") // YYYY-MM-DD
	83	expQuery := fmt.Sprintf(" AND ((parsed.validity.end: [%s TO ]) AND (parsed.validity.start: [ TO %s]))", dateStr, dateStr)
	84	s.Query = s.Query + expQuery
	85	}
	86	s.Page = 1
	87	s.Flatten = true
	88	s.Fields = []string{"parsed.fingerprint_sha256", "parsed.names"}
	89	return s
	90	}
	91
	92	// Driver creates a new CT driver for censys
	93	func Driver(savePath string, includeSubdomains, includeExpired bool) (driver.Driver, error) {
	94	if appID == "" \|\| secret == "" {
	95	return nil, fmt.Errorf("censys requires an appID and secret to run")
	96	}
	97	d := new(censys)
	98	d.appID = *appID
	99	d.secret = *secret
	100	d.savePath = savePath
	101	d.includeSubdomains = includeSubdomains
	102	d.includeExpired = includeExpired
	103	return d, nil
	104	}
	105
	106	func (d *censys) GetName() string {
	107	return driverName
	108	}
	109
	110	func (d censys) request(method, url string, request io.Reader) (http.Response, error) {
	111	totalTrys := 3
	112	var err error
	113	var req *http.Request
	114	var resp *http.Response
	115	for try := 1; try <= totalTrys; try++ {
	116	req, err = http.NewRequest(method, url, request)
	117	if err != nil {
	118	return nil, err
	119	}
	120	if request != nil {
	121	req.Header.Add("Content-Type", "application/json")
	122	}
	123	req.Header.Add("Accept", "application/json")
	124	req.SetBasicAuth(d.appID, d.secret)
	125
	126	resp, err = defaultHTTPClient.Do(req)
	127	if err != nil {
	128	err = fmt.Errorf("error on request [%d/%d] %s, got error %w: %+v", try, totalTrys, url, err, resp)
	129	} else {
	130	return resp, nil
	131	}
	132
	133	// sleep only if we will try again
	134	if try < totalTrys {
	135	time.Sleep(time.Second * 10)
	136	}
	137	}
	138	return resp, err
	139	}
	140
	141	// jsonRequest performs a request to the API endpoint sending and receiving JSON objects
	142	func (d *censys) jsonRequest(method, url string, request, response interface{}) error {
	143	var payloadReader io.Reader
	144	if request != nil {
	145	jsonPayload, err := json.Marshal(request)
	146	if err != nil {
	147	return err
	148	}
	149	payloadReader = bytes.NewReader(jsonPayload)
	150	}
	151
	152	if debug {
	153	log.Printf("DEBUG: request to %s %s", method, url)
	154	if request != nil {
	155	prettyJSONBytes, _ := json.MarshalIndent(request, "", "\t")
	156	log.Printf("request payload:\n%s\n", string(prettyJSONBytes))
	157	}
	158	}
	159
	160	resp, err := d.request(method, url, payloadReader)
	161	if err != nil {
	162	return err
	163	}
	164	defer resp.Body.Close()
	165
	166	// got an error, decode it
	167	if resp.StatusCode != http.StatusOK {
	168	var errorResp errorResponse
	169	err := fmt.Errorf("error on request %s, got Status %s %s", url, resp.Status, http.StatusText(resp.StatusCode))
	170	jsonError := json.NewDecoder(resp.Body).Decode(&errorResp)
	171	if jsonError != nil {
	172	return fmt.Errorf("error decoding json %w on error request: %s", jsonError, err.Error())
	173	}
	174	return fmt.Errorf("%w, HTTPStatus: %d Message: %q", err, errorResp.ErrorCode, errorResp.Error)
	175	}
	176
	177	if response != nil {
	178	err = json.NewDecoder(resp.Body).Decode(&response)
	179	if err != nil {
	180	return err
	181	}
	182	if debug {
	183	prettyJSONBytes, _ := json.MarshalIndent(response, "", "\t")
	184	log.Printf("response payload:\n%s\n", string(prettyJSONBytes))
	185	}
	186	}
	187
	188	return nil
	189	}
	190
	191	func (d *censys) QueryDomain(domain string) (driver.Result, error) {
	192	results := &censysCertDriver{
	193	host: domain,
	194	fingerprints: make(driver.FingerprintMap),
	195	driver: d,
	196	}
	197	params := domainSearchParam(domain, d.includeExpired, d.includeSubdomains)
	198	url := "https://search.censys.io/api/v1/search/certificates"
	199	var resp certSearchResponse
	200	err := d.jsonRequest(http.MethodPost, url, params, &resp)
	201	if err != nil {
	202	return results, err
	203	}
	204
	205	for _, r := range resp.Results {
	206	fp := fingerprint.FromHexHash(r.Fingerprint)
	207	results.fingerprints.Add(domain, fp)
	208	}
	209
	210	if debug {
	211	log.Printf("censys: got %d results for %s.", len(resp.Results), domain)
	212	}
	213
	214	return results, nil
	215	}
	216
	217	func (d censys) QueryCert(fp fingerprint.Fingerprint) (driver.CertResult, error) {
	218	certNode := new(driver.CertResult)
	219	certNode.Fingerprint = fp
	220	certNode.Domains = make([]string, 0, 5)
	221
	222	url := fmt.Sprintf("https://search.censys.io/api/v1/view/certificates/%s", fp.HexString())
	223	var resp certViewResponse
	224	err := d.jsonRequest(http.MethodGet, url, nil, &resp)
	225	if err != nil {
	226	return certNode, err
	227	}
	228
	229	if debug {
	230	log.Printf("DEBUG QueryCert(%s): %v", fp.HexString(), resp.Parsed.Names)
	231	}
	232
	233	certNode.Domains = append(certNode.Domains, resp.Parsed.Names...)
	234
	235	if d.save {
	236	rawCert, err := base64.StdEncoding.DecodeString(resp.Raw)
	237	if err != nil {
	238	return certNode, err
	239	}
	240	err = driver.RawCertToPEMFile(rawCert, path.Join(d.savePath, fp.HexString())+".pem")
	241	if err != nil {
	242	return certNode, err
	243	}
	244	}
	245
	246	return certNode, nil
	247	}

+217

-0

driver/censys/schemas.go less more

	0	package censys
	1
	2	import "time"
	3
	4	// cSpell:ignore spki noct zlint fatals precert
	5
	6	type certSearchParam struct {
	7	Query string `json:"query"`
	8	Page uint `json:"page"`
	9	Fields []string `json:"fields"`
	10	Flatten bool `json:"flatten"`
	11	}
	12
	13	type certSearchResponse struct {
	14	Status string `json:"status"`
	15	Metadata struct {
	16	Query string `json:"query"`
	17	Count uint `json:"count"`
	18	BackendTime uint `json:"backend_time"`
	19	Page uint `json:"page"`
	20	Pages uint `json:"pages"`
	21	} `json:"metadata"`
	22	Results []struct {
	23	Names []string `json:"parsed.names"`
	24	Fingerprint string `json:"parsed.fingerprint_sha256"`
	25	} `json:"results"`
	26	}
	27
	28	type certViewResponse struct {
	29	Raw string `json:"raw"`
	30	FingerprintSha256 string `json:"fingerprint_sha256"`
	31	ParentSpkiSubjectFingerprint string `json:"parent_spki_subject_fingerprint"`
	32	Metadata struct {
	33	PostProcessedAt time.Time `json:"post_processed_at"`
	34	PostProcessed bool `json:"post_processed"`
	35	Source string `json:"source"`
	36	ParseVersion int `json:"parse_version"`
	37	ParseStatus string `json:"parse_status"`
	38	AddedAt time.Time `json:"added_at"`
	39	UpdatedAt time.Time `json:"updated_at"`
	40	SeenInScan bool `json:"seen_in_scan"`
	41	} `json:"metadata"`
	42	Ct struct {
	43	GoogleXenon2022 struct {
	44	Index int `json:"index"`
	45	CtToCensysAt time.Time `json:"ct_to_censys_at"`
	46	AddedToCtAt time.Time `json:"added_to_ct_at"`
	47	} `json:"google_xenon_2022"`
	48	} `json:"ct"`
	49	Parsed struct {
	50	Version int `json:"version"`
	51	SerialNumber string `json:"serial_number"`
	52	SignatureAlgorithm struct {
	53	Name string `json:"name"`
	54	Oid string `json:"oid"`
	55	} `json:"signature_algorithm"`
	56	Issuer struct {
	57	CommonName []string `json:"common_name"`
	58	Country []string `json:"country"`
	59	Organization []string `json:"organization"`
	60	} `json:"issuer"`
	61	IssuerDn string `json:"issuer_dn"`
	62	Validity struct {
	63	Start time.Time `json:"start"`
	64	End time.Time `json:"end"`
	65	Length int `json:"length"`
	66	} `json:"validity"`
	67	Subject struct {
	68	CommonName []string `json:"common_name"`
	69	} `json:"subject"`
	70	SubjectDn string `json:"subject_dn"`
	71	SubjectKeyInfo struct {
	72	KeyAlgorithm struct {
	73	Name string `json:"name"`
	74	} `json:"key_algorithm"`
	75	EcdsaPublicKey struct {
	76	B string `json:"b"`
	77	Curve string `json:"curve"`
	78	Gx string `json:"gx"`
	79	Gy string `json:"gy"`
	80	Length int `json:"length"`
	81	N string `json:"n"`
	82	P string `json:"p"`
	83	Pub string `json:"pub"`
	84	X string `json:"x"`
	85	Y string `json:"y"`
	86	} `json:"ecdsa_public_key"`
	87	FingerprintSha256 string `json:"fingerprint_sha256"`
	88	} `json:"subject_key_info"`
	89	Extensions struct {
	90	KeyUsage struct {
	91	DigitalSignature bool `json:"digital_signature"`
	92	Value int `json:"value"`
	93	} `json:"key_usage"`
	94	BasicConstraints struct {
	95	IsCa bool `json:"is_ca"`
	96	} `json:"basic_constraints"`
	97	SubjectAltName struct {
	98	DNSNames []string `json:"dns_names"`
	99	} `json:"subject_alt_name"`
	100	AuthorityKeyID string `json:"authority_key_id"`
	101	SubjectKeyID string `json:"subject_key_id"`
	102	ExtendedKeyUsage struct {
	103	ServerAuth bool `json:"server_auth"`
	104	ClientAuth bool `json:"client_auth"`
	105	} `json:"extended_key_usage"`
	106	CertificatePolicies []struct {
	107	ID string `json:"id"`
	108	Cps []string `json:"cps,omitempty"`
	109	} `json:"certificate_policies"`
	110	AuthorityInfoAccess struct {
	111	OcspUrls []string `json:"ocsp_urls"`
	112	IssuerUrls []string `json:"issuer_urls"`
	113	} `json:"authority_info_access"`
	114	SignedCertificateTimestamps []struct {
	115	Version int `json:"version"`
	116	LogID string `json:"log_id"`
	117	Timestamp int `json:"timestamp"`
	118	Signature string `json:"signature"`
	119	} `json:"signed_certificate_timestamps"`
	120	} `json:"extensions"`
	121	Signature struct {
	122	SignatureAlgorithm struct {
	123	Name string `json:"name"`
	124	Oid string `json:"oid"`
	125	} `json:"signature_algorithm"`
	126	Value string `json:"value"`
	127	Valid bool `json:"valid"`
	128	SelfSigned bool `json:"self_signed"`
	129	} `json:"signature"`
	130	FingerprintMd5 string `json:"fingerprint_md5"`
	131	FingerprintSha1 string `json:"fingerprint_sha1"`
	132	FingerprintSha256 string `json:"fingerprint_sha256"`
	133	TbsNoctFingerprint string `json:"tbs_noct_fingerprint"`
	134	SpkiSubjectFingerprint string `json:"spki_subject_fingerprint"`
	135	TbsFingerprint string `json:"tbs_fingerprint"`
	136	ValidationLevel string `json:"validation_level"`
	137	Names []string `json:"names"`
	138	Redacted bool `json:"redacted"`
	139	} `json:"parsed"`
	140	Tags []string `json:"tags"`
	141	Validation struct {
	142	Nss struct {
	143	Blacklisted bool `json:"blacklisted"`
	144	HadTrustedPath bool `json:"had_trusted_path"`
	145	InRevocationSet bool `json:"in_revocation_set"`
	146	TrustedPath bool `json:"trusted_path"`
	147	WasValid bool `json:"was_valid"`
	148	Whitelisted bool `json:"whitelisted"`
	149	Paths [][]string `json:"paths"`
	150	Parents []string `json:"parents"`
	151	Type string `json:"type"`
	152	Valid bool `json:"valid"`
	153	} `json:"nss"`
	154	Microsoft struct {
	155	Blacklisted bool `json:"blacklisted"`
	156	HadTrustedPath bool `json:"had_trusted_path"`
	157	InRevocationSet bool `json:"in_revocation_set"`
	158	TrustedPath bool `json:"trusted_path"`
	159	WasValid bool `json:"was_valid"`
	160	Whitelisted bool `json:"whitelisted"`
	161	Paths [][]string `json:"paths"`
	162	Parents []string `json:"parents"`
	163	Type string `json:"type"`
	164	Valid bool `json:"valid"`
	165	} `json:"microsoft"`
	166	Apple struct {
	167	Blacklisted bool `json:"blacklisted"`
	168	HadTrustedPath bool `json:"had_trusted_path"`
	169	InRevocationSet bool `json:"in_revocation_set"`
	170	TrustedPath bool `json:"trusted_path"`
	171	WasValid bool `json:"was_valid"`
	172	Whitelisted bool `json:"whitelisted"`
	173	Paths [][]string `json:"paths"`
	174	Parents []string `json:"parents"`
	175	Type string `json:"type"`
	176	Valid bool `json:"valid"`
	177	} `json:"apple"`
	178	Revoked bool `json:"revoked"`
	179	GoogleCtPrimary struct {
	180	Blacklisted bool `json:"blacklisted"`
	181	HadTrustedPath bool `json:"had_trusted_path"`
	182	InRevocationSet bool `json:"in_revocation_set"`
	183	TrustedPath bool `json:"trusted_path"`
	184	WasValid bool `json:"was_valid"`
	185	Whitelisted bool `json:"whitelisted"`
	186	Paths [][]string `json:"paths"`
	187	Parents []string `json:"parents"`
	188	Type string `json:"type"`
	189	Valid bool `json:"valid"`
	190	} `json:"google_ct_primary"`
	191	OcspRevocation struct {
	192	NextUpdate time.Time `json:"next_update"`
	193	Revoked bool `json:"revoked"`
	194	} `json:"ocsp_revocation"`
	195	CrlRevocation struct {
	196	Revoked bool `json:"revoked"`
	197	} `json:"crl_revocation"`
	198	CrlError string `json:"crl_error"`
	199	} `json:"validation"`
	200	Zlint struct {
	201	NoticesPresent bool `json:"notices_present"`
	202	WarningsPresent bool `json:"warnings_present"`
	203	ErrorsPresent bool `json:"errors_present"`
	204	FatalsPresent bool `json:"fatals_present"`
	205	Lints struct {
	206	NSubjectCommonNameIncluded bool `json:"n_subject_common_name_included"`
	207	} `json:"lints"`
	208	Version int `json:"version"`
	209	} `json:"zlint"`
	210	Precert bool `json:"precert"`
	211	}
	212
	213	type errorResponse struct {
	214	Error string `json:"error"`
	215	ErrorCode int `json:"error_code"`
	216	}

+75

-64

driver/crtsh/crtsh.go less more

	0	// Package crtsh implements an unofficial API client for Comodo's
	1	// Certificate Transparency search
	2	// https://crt.sh/
	3	//
	4	// As the API is unofficial and has been reverse engineered it may stop working
	5	// at any time and comes with no guarantees.
	6	// view SQL excample: https://crt.sh/?showSQL=Y&exclude=expired&q=
	7	//
0	8	package crtsh
1
2		/*
3		* This file implements an unofficial API client for Comodo's
4		* Certificate Transparency search
5		* https://crt.sh/
6		*
7		* As the API is unofficial and has been reverse engineered it may stop working
8		* at any time and comes with no guarantees.
9		*/
10
11		// TODO running in verbose gives error: pq: unnamed prepared statement does not exist
12	9
13	10	import (
14	11	"database/sql"
15	12	"fmt"
	13	"log"
16	14	"path"
17	15	"time"
18	16

22	20	_ "github.com/lib/pq" // portgresql
23	21	)
24	22
25		const connStr = "postgresql://[email protected]/certwatch?sslmode=disable"
	23	const connStr = "postgresql://[email protected]/certwatch?sslmode=disable&fallback_application_name=certgraph&binary_parameters=yes"
26	24	const driverName = "crtsh"
	25
	26	const debug = false
27	27
28	28	func init() {
29	29	driver.AddDriver(driverName)

75	75	}
76	76
77	77	d.db, err = sql.Open("postgres", connStr)
78
79		d.setSQLTimeout(d.timeout.Seconds())
	78	if err != nil {
	79	return nil, err
	80	}
	81
	82	err = d.setSQLTimeout(d.timeout.Seconds())
80	83
81	84	return d, err
82	85	}

97	100	driver: d,
98	101	}
99	102
100		queryStr := ""
101
102		if d.includeSubdomains {
103		if d.includeExpired {
104		queryStr = `SELECT digest(certificate.certificate, 'sha256') sha256
105		FROM certificate_identity, certificate
106		WHERE certificate.id = certificate_identity.certificate_id
107		AND (reverse(lower(certificate_identity.name_value)) LIKE reverse(lower('%%.'\|\|$1))
108		OR reverse(lower(certificate_identity.name_value)) LIKE reverse(lower($1)))
109		LIMIT $2`
110		} else {
111		queryStr = `SELECT digest(certificate.certificate, 'sha256') sha256
112		FROM certificate_identity, certificate
113		WHERE certificate.id = certificate_identity.certificate_id
114		AND x509_notAfter(certificate.certificate) > statement_timestamp()
115		AND (reverse(lower(certificate_identity.name_value)) LIKE reverse(lower('%%.'\|\|$1))
116		OR reverse(lower(certificate_identity.name_value)) LIKE reverse(lower($1)))
117		LIMIT $2`
118		}
119		} else {
120		if d.includeExpired {
121		queryStr = `SELECT digest(certificate.certificate, 'sha256') sha256
122		FROM certificate_identity, certificate
123		WHERE certificate.id = certificate_identity.certificate_id
124		AND reverse(lower(certificate_identity.name_value)) LIKE reverse(lower($1))
125		LIMIT $2`
126		} else {
127		queryStr = `SELECT digest(certificate.certificate, 'sha256') sha256
128		FROM certificate_identity, certificate
129		WHERE certificate.id = certificate_identity.certificate_id
130		AND x509_notAfter(certificate.certificate) > statement_timestamp()
131		AND reverse(lower(certificate_identity.name_value)) LIKE reverse(lower($1))
132		LIMIT $2`
133		}
134		}
135
136		if d.includeSubdomains {
137		domain = fmt.Sprintf("%%.%s", domain)
138		}
	103	queryStr := `WITH myconstants (include_expired, include_subdomains) as (
	104	values ($1::bool, $2::bool)
	105	),
	106	ci AS (
	107	SELECT digest(sub.CERTIFICATE, 'sha256') sha256, -- added
	108	min(sub.CERTIFICATE_ID) ID,
	109	min(sub.ISSUER_CA_ID) ISSUER_CA_ID,
	110	array_agg(DISTINCT sub.NAME_VALUE) NAME_VALUES
	111	FROM (SELECT *
	112	FROM certificate_and_identities cai, myconstants
	113	WHERE plainto_tsquery('certwatch', $4) @@ identities(cai.CERTIFICATE)
	114	AND (
	115	-- domain only
	116	(NOT myconstants.include_subdomains AND cai.NAME_VALUE ILIKE ($4))
	117	OR
	118	-- include sub-domains
	119	(myconstants.include_subdomains AND (cai.NAME_VALUE ILIKE ($4) OR cai.NAME_VALUE ILIKE ('%.' \|\| $4)))
	120	)
	121	AND (
	122	-- added
	123	cai.NAME_TYPE = '2.5.4.3' -- commonName
	124	OR
	125	cai.NAME_TYPE = 'san:dNSName' -- dNSName
	126	)
	127	AND
	128	-- include expired?
	129	(myconstants.include_expired OR (coalesce(x509_notAfter(cai.CERTIFICATE), 'infinity'::timestamp) >= date_trunc('year', now() AT TIME ZONE 'UTC')
	130	AND x509_notAfter(cai.CERTIFICATE) >= now() AT TIME ZONE 'UTC'))
	131	LIMIT $3
	132	) sub
	133	GROUP BY sub.CERTIFICATE
	134	)
	135	SELECT
	136	ci.sha256 -- added
	137	--array_to_string(ci.name_values, chr(10)) name_value,
	138	--ci.id id
	139	FROM ci;`
139	140
140	141	try := 0
141	142	var err error

143	144	for try < 5 {
144	145	// this is a hack while crt.sh gets there stuff togeather
145	146	try++
146		rows, err = d.db.Query(queryStr, domain, d.queryLimit)
	147	if debug {
	148	log.Printf("QueryDomain try %d: %s", try, queryStr)
	149	}
	150	rows, err = d.db.Query(queryStr, d.includeExpired, d.includeSubdomains, d.queryLimit, domain)
147	151	if err == nil {
148	152	break
	153	}
	154	if debug {
	155	log.Printf("crtsh pq error on domain %q: %s", domain, err.Error())
149	156	}
150	157	}
151	158	/*if try > 1 {

164	171	results.fingerprints.Add(domain, fingerprint.FromHashBytes(hash))
165	172	}
166	173
	174	if debug {
	175	log.Printf("crtsh: got %d results for %s.", len(results.fingerprints[domain]), domain)
	176	}
	177
167	178	return results, nil
168	179	}
169	180

172	183	certNode.Fingerprint = fp
173	184	certNode.Domains = make([]string, 0, 5)
174	185
175		queryStr := `SELECT DISTINCT certificate_identity.name_value
176		FROM certificate, certificate_identity
177		WHERE certificate.id = certificate_identity.certificate_id
178		AND certificate_identity.name_type in ('dNSName', 'commonName')
179		AND digest(certificate.certificate, 'sha256') = $1`
	186	queryStr := `SELECT DISTINCT name_value FROM certificate_and_identities WHERE digest(certificate, 'sha256') = $1;`
180	187
181	188	try := 0
182	189	var err error

198	205
199	206	for rows.Next() {
200	207	var domain string
201		rows.Scan(&domain)
	208	err = rows.Scan(&domain)
	209	if err != nil {
	210	return nil, err
	211	}
202	212	certNode.Domains = append(certNode.Domains, domain)
203	213	}
204	214
205	215	if d.save {
206	216	var rawCert []byte
207		queryStr = `SELECT certificate.certificate
208		FROM certificate
209		WHERE digest(certificate.certificate, 'sha256') = $1`
	217	queryStr = `SELECT certificate FORM certificate_and_identities WHERE digest(certificate, 'sha256') = $1;`
210	218	row := d.db.QueryRow(queryStr, fp[:])
211	219	err = row.Scan(&rawCert)
212	220	if err != nil {
213	221	return certNode, err
214	222	}
215	223
216		driver.RawCertToPEMFile(rawCert, path.Join(d.savePath, fp.HexString())+".pem")
	224	err = driver.RawCertToPEMFile(rawCert, path.Join(d.savePath, fp.HexString())+".pem")
	225	if err != nil {
	226	return certNode, err
	227	}
217	228	}
218	229
219	230	return certNode, nil

+4

-1

driver/driver.go less more

	0	// Package driver exposes interfaces and types certgraph drivers must implement
0	1	package driver
1	2
2	3	import (

7	8	"github.com/lanrat/certgraph/fingerprint"
8	9	"github.com/lanrat/certgraph/status"
9	10	)
	11
	12	// TODO add context instead of timeout on all requests
10	13
11	14	// Drivers contains all the drivers that have been registered
12	15	var Drivers []string

65	68	certResult := new(CertResult)
66	69
67	70	// generate Fingerprint
68		certResult.Fingerprint = fingerprint.FromBytes(cert.Raw)
	71	certResult.Fingerprint = fingerprint.FromRawCertBytes(cert.Raw)
69	72
70	73	// domains
71	74	// used to ensure uniq entries in domains array

+21

-18

driver/google/google.go less more

	0	// Package google file implements an unofficial API client for Google's
	1	// Certificate Transparency search
	2	// https://transparencyreport.google.com/https/certificates
	3	//
	4	// As the API is unofficial and has been reverse engineered it may stop working
	5	// at any time and comes with no guarantees.
0	6	package google
1
2		/*
3		* This file implements an unofficial API client for Google's
4		* Certificate Transparency search
5		* https://transparencyreport.google.com/https/certificates
6		*
7		* As the API is unofficial and has been reverse engineered it may stop working
8		* at any time and comes with no guarantees.
9		*/
10	7
11	8	import (
12	9	"encoding/json"
13	10	"errors"
14		"io/ioutil"
	11	"io"
15	12	"net/http"
16	13	"net/url"
17	14	"strconv"

22	19	"github.com/lanrat/certgraph/status"
23	20	)
24	21
	22	// cSpell:ignore cdsr
	23
25	24	const driverName = "google"
26	25
27	26	func init() {

29	28	}
30	29
31	30	// Base URLs for Google's CT API
32		const searchURL1 = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch?include_expired=false&include_subdomains=false&domain=example.com"
33		const searchURL2 = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch/page?p=DEADBEEF"
34		const certURL = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certbyhash?hash=DEADBEEF"
	31	const (
	32	searchURL1 = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch?include_expired=false&include_subdomains=false&domain=example.com"
	33	searchURL2 = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certsearch/page?p=DEADBEEF"
	34	certURL = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/certbyhash?hash=DEADBEEF"
	35	//summaryURL is not currently used
	36	//summaryURL = "https://transparencyreport.google.com/transparencyreport/api/v3/httpsreport/ct/summary"
	37	)
35	38
36	39	type googleCT struct {
37	40	maxPages float64 // this is a float because that is the type automatically decoded from the JSON response

93	96	return errors.New("Got non OK HTTP status: '" + r.Status + "' on URL: " + url)
94	97	}
95	98
96		respData, err := ioutil.ReadAll(r.Body)
	99	respData, err := io.ReadAll(r.Body)
97	100	if err != nil {
98	101	return err
99	102	}

135	138	return results, err
136	139	}
137	140
138		// simple corectness checks
	141	// simple correctness checks
139	142	if raw[0][0] != "https.ct.cdsr" {
140	143	return results, errors.New("Got Unexpected Query output: " + raw[0][0].(string))
141	144	}

145	148	break
146	149	}
147	150	if len(raw[0][3].([]interface{})) != 5 {
148		// pageinfo result not correct length, likely no results
	151	// pageInfo result not correct length, likely no results
149	152	//fmt.Println(raw[0])
150	153	break
151	154	}

157	160	foundCerts := raw[0][1].([]interface{})
158	161	for _, foundCert := range foundCerts {
159	162	certHash := foundCert.([]interface{})[5].(string)
160		certFP := fingerprint.FromB64(certHash)
	163	certFP := fingerprint.FromB64Hash(certHash)
161	164	results.fingerprints.Add(domain, certFP)
162	165	}
163	166	//fmt.Println("Page:", pageInfo[3])

202	205	return certNode, err
203	206	}
204	207
205		// simple corectness checks
	208	// simple correctness checks
206	209	if raw[0][0] != "https.ct.chr" {
207	210	return certNode, errors.New("Got Unexpected Cert output: " + raw[0][0].(string))
208	211	}

+3

-2

driver/http/http.go less more

	0	// Package http implements a certgraph driver for obtaining SSL certificates over https
0	1	package http
1	2
2	3	import (

52	53	if found {
53	54	return cert, nil
54	55	}
55		return nil, fmt.Errorf("Certificate with Fingerprint %s not found", fp.HexString())
	56	return nil, fmt.Errorf("certificate with Fingerprint %s not found", fp.HexString())
56	57	}
57	58
58	59	// Driver creates a new SSL driver for HTTP Connections

150	151
151	152	// save
152	153	if c.parent.save && len(connState.PeerCertificates) > 0 {
153		driver.CertsToPEMFile(connState.PeerCertificates, path.Join(c.parent.savePath, certResult.Fingerprint.HexString())+".pem")
	154	err = driver.CertsToPEMFile(connState.PeerCertificates, path.Join(c.parent.savePath, certResult.Fingerprint.HexString())+".pem")
154	155	}
155	156
156	157	return conn, err

+131

-0

driver/multi/multi_driver.go less more

	0	// Package multi exposes a generic driver interface allowing you to merge the results of multiple other drivers
	1	package multi
	2
	3	import (
	4	"errors"
	5	"fmt"
	6	"strings"
	7	"sync"
	8
	9	"github.com/lanrat/certgraph/driver"
	10	"github.com/lanrat/certgraph/fingerprint"
	11	"github.com/lanrat/certgraph/status"
	12	"golang.org/x/sync/errgroup"
	13	)
	14
	15	type multiDriver struct {
	16	drivers []driver.Driver
	17	}
	18
	19	// Driver returns a new instance of multi driver for the provided drivers
	20	func Driver(drivers []driver.Driver) driver.Driver {
	21	md := new(multiDriver)
	22	md.drivers = drivers
	23	return md
	24	}
	25
	26	func (d *multiDriver) GetName() string {
	27	names := make([]string, 0, len(d.drivers))
	28	for _, driver := range d.drivers {
	29	names = append(names, driver.GetName())
	30	}
	31	return fmt.Sprintf("multi[%s]", strings.Join(names, ","))
	32	}
	33
	34	func (d *multiDriver) QueryDomain(domain string) (driver.Result, error) {
	35	r := newResult(domain)
	36	var group errgroup.Group
	37	for _, d := range d.drivers {
	38	goFunc := func(localDriver driver.Driver) func() error {
	39	return func() error {
	40	return func(localDriver driver.Driver) error {
	41	result, err := localDriver.QueryDomain(domain)
	42	if err != nil {
	43	return err
	44	}
	45	return r.add(result)
	46	}(localDriver)
	47	}
	48	}
	49
	50	group.Go(goFunc(d))
	51	}
	52	err := group.Wait()
	53	if err != nil {
	54	return nil, err
	55	}
	56	return r, nil
	57	}
	58
	59	func newResult(host string) *multiResult {
	60	r := new(multiResult)
	61	r.host = host
	62	r.results = make([]driver.Result, 0, 2)
	63	r.fingerprints = make(driver.FingerprintMap)
	64	return r
	65	}
	66
	67	type multiResult struct {
	68	host string
	69	results []driver.Result
	70	resultLock sync.Mutex // also protects fingerprints
	71	fingerprints driver.FingerprintMap
	72	}
	73
	74	func (c *multiResult) add(r driver.Result) error {
	75	c.resultLock.Lock()
	76	defer c.resultLock.Unlock()
	77	fpm, err := r.GetFingerprints()
	78	if err != nil {
	79	return err
	80	}
	81	for domain := range fpm {
	82	for _, fp := range fpm[domain] {
	83	// TODO does not dedupe across drivers
	84	c.fingerprints.Add(domain, fp)
	85	}
	86	}
	87
	88	c.results = append(c.results, r)
	89	return nil
	90	}
	91
	92	func (c multiResult) QueryCert(fp fingerprint.Fingerprint) (driver.CertResult, error) {
	93	for _, result := range c.results {
	94	cr, err := result.QueryCert(fp)
	95	if err != nil {
	96	return nil, err
	97	}
	98	if cr != nil {
	99	return cr, nil
	100	}
	101	}
	102	return nil, errors.New("unable to find working driver with QueryCert()")
	103	}
	104
	105	func (c *multiResult) GetFingerprints() (driver.FingerprintMap, error) {
	106	return c.fingerprints, nil
	107	}
	108
	109	func (c *multiResult) GetStatus() status.Map {
	110	// TODO nest other status inside
	111	return status.NewMap(c.host, status.New(status.MULTI))
	112	}
	113
	114	func (c *multiResult) GetRelated() ([]string, error) {
	115	relatedMap := make(map[string]bool)
	116	for _, result := range c.results {
	117	related, err := result.GetRelated()
	118	if err != nil {
	119	return nil, err
	120	}
	121	for _, r := range related {
	122	relatedMap[r] = true
	123	}
	124	}
	125	related := make([]string, 0, len(relatedMap))
	126	for r := range relatedMap {
	127	related = append(related, r)
	128	}
	129	return related, nil
	130	}

+6

-3

driver/save.go less more

16	16	}
17	17	defer f.Close()
18	18	for _, cert := range certs {
19		pem.Encode(f, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})
	19	err = pem.Encode(f, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})
	20	if err != nil {
	21	return err
	22	}
20	23	}
21	24	return nil
22	25	}

31	34	return err
32	35	}
33	36	defer f.Close()
34		pem.Encode(f, &pem.Block{Type: "CERTIFICATE", Bytes: cert})
35		return nil
	37	err = pem.Encode(f, &pem.Block{Type: "CERTIFICATE", Bytes: cert})
	38	return err
36	39	}
37	40
38	41	func fileExists(f string) bool {

+4

-3

driver/smtp/smtp.go less more

	0	// Package smtp implements a certgraph driver for obtaining SSL certificates over smtp with STARTTLS
0	1	package smtp
1	2
2	3	import (

54	55	if found {
55	56	return cert, nil
56	57	}
57		return nil, fmt.Errorf("Certificate with Fingerprint %s not found", fp.HexString())
	58	return nil, fmt.Errorf("certificate with Fingerprint %s not found", fp.HexString())
58	59	}
59	60
60	61	// Driver creates a new SSL driver for SMTP Connections

133	134
134	135	// save
135	136	if d.save && len(certs) > 0 {
136		driver.CertsToPEMFile(certs, path.Join(d.savePath, certResult.Fingerprint.HexString())+".pem")
	137	err = driver.CertsToPEMFile(certs, path.Join(d.savePath, certResult.Fingerprint.HexString())+".pem")
137	138	}
138	139
139		return results, nil
	140	return results, err
140	141	}
141	142
142	143	// getMX returns the MX records for the provided domain

+24

-14

fingerprint/fingerprint.go less more

	0	// Package fingerprint defines types to define a certificate fingerprint for certgraph
0	1	package fingerprint
1	2
2	3	import (
3	4	"crypto/sha256"
4	5	"encoding/base64"
	6	"encoding/hex"
5	7	"fmt"
6	8	)
7	9
8	10	// Fingerprint sha256 of certificate bytes
9	11	type Fingerprint [sha256.Size]byte
10	12
11		// HexString print Fingerprint as hex
	13	// HexString print Fingerprint as uppercase hex
12	14	func (fp *Fingerprint) HexString() string {
13	15	return fmt.Sprintf("%X", *fp)
14	16	}

16	18	// FromHashBytes returns a Fingerprint generated by the first len(Fingerprint) bytes
17	19	func FromHashBytes(data []byte) Fingerprint {
18	20	var fp Fingerprint
19		/*if len(data) != sha256.Size {
20		v("Data is not correct SHA256 size", data)
21		}*/
	21	// if len(data) != len(fp) {
	22	// // TODO this should error....
	23	// }
22	24	for i := 0; i < len(data) && i < len(fp); i++ {
23	25	fp[i] = data[i]
24	26	}
25	27	return fp
26	28	}
27	29
28		// FromBytes returns a Fingerprint generated by the provided bytes
29		func FromBytes(data []byte) Fingerprint {
30		var fp Fingerprint
31		fp = sha256.Sum256(data)
	30	// FromRawCertBytes returns a Fingerprint generated by the provided bytes
	31	func FromRawCertBytes(data []byte) Fingerprint {
	32	fp := sha256.Sum256(data)
32	33	return fp
33	34	}
34	35
35		// FromB64 returns a Fingerprint from a base64 encoded hash string
36		func FromB64(hash string) Fingerprint {
37		data, _ := base64.StdEncoding.DecodeString(hash)
38		/*if err != nil {
39		fmt.Println(err)
40		}*/
	36	// FromB64Hash returns a Fingerprint from a base64 encoded hash string
	37	func FromB64Hash(hash string) Fingerprint {
	38	data, err := base64.StdEncoding.DecodeString(hash)
	39	if err != nil {
	40	panic(err)
	41	}
41	42	return FromHashBytes(data)
	43	}
	44
	45	// FromHexHash returns a Fingerprint from a hex encoded hash string
	46	func FromHexHash(hash string) Fingerprint {
	47	decoded, err := hex.DecodeString(hash)
	48	if err != nil {
	49	panic(err)
	50	}
	51	return FromHashBytes(decoded)
42	52	}
43	53
44	54	// B64Encode returns the b64 string of a Fingerprint

+89

-0

fingerprint/fp_test.go less more

	0	package fingerprint_test
	1
	2	import (
	3	"crypto/sha256"
	4	"encoding/base64"
	5	"strings"
	6	"testing"
	7
	8	"github.com/lanrat/certgraph/fingerprint"
	9	)
	10
	11	const rawCert = "MIID/TCCA4KgAwIBAgIQBV74EmrgijxarGYRe4auizAKBggqhkjOPQQDAzBWMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMTAwLgYDVQQDEydEaWdpQ2VydCBUTFMgSHlicmlkIEVDQyBTSEEzODQgMjAyMCBDQTEwHhcNMjIwNDIwMDAwMDAwWhcNMjMwNDIwMjM1OTU5WjBmMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEVMBMGA1UEChMMR2l0SHViLCBJbmMuMRMwEQYDVQQDEwpnaXRodWIuY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEjkAQ9bMD0dVDhUlfevxOChhxQME0Sb7kZr7+3T/gW6CW4eduvDZxsQUwa37mhUXMzF88gh+FsUy9TieoqZhasKOCAiAwggIcMB8GA1UdIwQYMBaAFAq8CCkXjKU5bXoOzjPHLrPt+8N6MB0GA1UdDgQWBBQJJ/08CmhEtgPojKO+W3TVwfJnaTAlBgNVHREEHjAcggpnaXRodWIuY29tgg53d3cuZ2l0aHViLmNvbTAOBgNVHQ8BAf8EBAMCB4AwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMIGbBgNVHR8EgZMwgZAwRqBEoEKGQGh0dHA6Ly9jcmwzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRMU0h5YnJpZEVDQ1NIQTM4NDIwMjBDQTEtMS5jcmwwRqBEoEKGQGh0dHA6Ly9jcmw0LmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRMU0h5YnJpZEVDQ1NIQTM4NDIwMjBDQTEtMS5jcmwwPgYDVR0gBDcwNTAzBgZngQwBAgIwKTAnBggrBgEFBQcCARYbaHR0cDovL3d3dy5kaWdpY2VydC5jb20vQ1BTMIGFBggrBgEFBQcBAQR5MHcwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0LmNvbTBPBggrBgEFBQcwAoZDaHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0VExTSHlicmlkRUNDU0hBMzg0MjAyMENBMS0xLmNydDAJBgNVHRMEAjAAMBMGCisGAQQB1nkCBAMBAf8EAgUAMAoGCCqGSM49BAMDA2kAMGYCMQC/16+UbmbTo4bcahQGLY2+SrWtge+DC2dcEY2pv1Cwn3YUi51uqEE+v7U6PUvWWbQCMQCslhV/IibG55Uoa6F/hNpa21ZqEhp38u7CHTFb+6HGbLi0CtbSjgc1mn/yEt5pFu0="
	12	const fpHashHex = "46a1fe1780fd9a05a5529906ed08a5fea2cfe63567c9fdeb62c18ba74fae35d5"
	13	const fpHashB64 = "RqH+F4D9mgWlUpkG7Qil/qLP5jVnyf3rYsGLp0+uNdU="
	14
	15	func TestFromHashBytes(t *testing.T) {
	16
	17	data, err := base64.StdEncoding.DecodeString(rawCert)
	18	if err != nil {
	19	t.Errorf("error on b64 decode: %s", err.Error())
	20	}
	21
	22	dataHashBytes := sha256.Sum256(data)
	23
	24	fp := fingerprint.FromHashBytes(dataHashBytes[:])
	25	uppercaseHexHash := strings.ToUpper(fpHashHex)
	26	hashHex := fp.HexString()
	27
	28	if hashHex != uppercaseHexHash {
	29	t.Errorf("fingerprint error, expected hex hash [%s] got [%s]", uppercaseHexHash, hashHex)
	30	}
	31
	32	hashB64 := fp.B64Encode()
	33
	34	if hashB64 != fpHashB64 {
	35	t.Errorf("fingerprint error, expected b64 hash [%s] got [%s]", hashB64, fpHashB64)
	36	}
	37	}
	38
	39	func TestFromRawCertBytes(t *testing.T) {
	40
	41	data, err := base64.StdEncoding.DecodeString(rawCert)
	42	if err != nil {
	43	t.Errorf("error on b64 decode: %s", err.Error())
	44	}
	45
	46	fp := fingerprint.FromRawCertBytes(data)
	47	uppercaseHash := strings.ToUpper(fpHashHex)
	48	hashHex := fp.HexString()
	49
	50	if hashHex != uppercaseHash {
	51	t.Errorf("fingerprint error, expected hex hash [%s] got [%s]", uppercaseHash, hashHex)
	52	}
	53
	54	hashB64 := fp.B64Encode()
	55
	56	if hashB64 != fpHashB64 {
	57	t.Errorf("fingerprint error, expected b64 hash [%s] got [%s]", hashB64, fpHashB64)
	58	}
	59	}
	60
	61	func TestFromB64Hash(t *testing.T) {
	62
	63	fp := fingerprint.FromB64Hash(fpHashB64)
	64
	65	uppercaseHash := strings.ToUpper(fpHashHex)
	66	hashHex := fp.HexString()
	67
	68	if hashHex != uppercaseHash {
	69	t.Errorf("fingerprint error, expected hex hash [%s] got [%s]", uppercaseHash, hashHex)
	70	}
	71
	72	hashB64 := fp.B64Encode()
	73
	74	if hashB64 != fpHashB64 {
	75	t.Errorf("fingerprint error, expected b64 hash [%s] got [%s]", hashB64, fpHashB64)
	76	}
	77	}
	78
	79	func TestFromHexHash(t *testing.T) {
	80
	81	fp := fingerprint.FromHexHash(fpHashHex)
	82
	83	hashB64 := fp.B64Encode()
	84
	85	if fpHashB64 != hashB64 {
	86	t.Errorf("fingerprint error, expected b64 hash [%s] got [%s]", fpHashHex, hashB64)
	87	}
	88	}

+6

-4

go.mod less more

0	0	module github.com/lanrat/certgraph
1	1
2	2	require (
3		github.com/lib/pq v1.0.0
4		github.com/weppos/publicsuffix-go v0.4.0
5		golang.org/x/net v0.0.0-20180906233101-161cd47e91fd // indirect
6		golang.org/x/text v0.3.0 // indirect
	3	github.com/lib/pq v1.10.6
	4	github.com/weppos/publicsuffix-go v0.20.0
	5	golang.org/x/net v0.0.0-20220906165146-f3363e06e74c // indirect
	6	golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde
7	7	)
	8
	9	go 1.16

+22

-0

go.sum less more

	0	github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
	1	github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
	2	github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs=
	3	github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
	4	github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
	5	github.com/weppos/publicsuffix-go v0.20.0 h1:59ypvSUbW3Dunc6zVm+v+MmXf2Q6cGiNDkxgRIzEnaA=
	6	github.com/weppos/publicsuffix-go v0.20.0/go.mod h1:5ZC/Uv3fIEUE0eP6o9+Yg4+5+W8V0/BieMi05feGXVA=
	7	github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e0182326a282/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE=
	8	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
	9	golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
	10	golang.org/x/net v0.0.0-20220906165146-f3363e06e74c h1:yKufUcDwucU5urd+50/Opbt4AYpqthk7wHpHok8f1lo=
	11	golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
	12	golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde h1:ejfdSekXMDxDLbRrJMwUk6KnSLZ2McaUCVcIKM+N6jc=
	13	golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
	14	golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
	15	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
	16	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
	17	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
	18	golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
	19	golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
	20	golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
	21	golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

+13

-9

graph/cert_node.go less more

2	2	import (
3	3	"fmt"
4	4	"strings"
	5	"sync"
5	6
6	7	"github.com/lanrat/certgraph/dns"
7	8	"github.com/lanrat/certgraph/fingerprint"

9	10
10	11	// CertNode graph node to store certificate information
11	12	type CertNode struct {
12		Fingerprint fingerprint.Fingerprint
13		Domains []string
14		foundMap map[string]bool
	13	Fingerprint fingerprint.Fingerprint
	14	Domains []string
	15	foundMap map[string]bool
	16	foundMapLock sync.Mutex
15	17	}
16	18
17	19	func (c *CertNode) String() string {

29	31
30	32	// AddFound adds a driver name to the source of the certificate
31	33	func (c *CertNode) AddFound(driver string) {
	34	c.foundMapLock.Lock()
	35	defer c.foundMapLock.Unlock()
32	36	if c.foundMap == nil {
33	37	c.foundMap = make(map[string]bool)
34	38	}

56	60	return false
57	61	}
58	62
59		// TLDPlus1Count the number of tld+1 domains in the certificate
60		func (c *CertNode) TLDPlus1Count() int {
61		tldPlus1Domains := make(map[string]bool)
	63	// ApexCount the number of tld+1 domains in the certificate
	64	func (c *CertNode) ApexCount() int {
	65	apexDomains := make(map[string]bool)
62	66	for _, domain := range c.Domains {
63		tldPlus1, err := dns.TLDPlus1(domain)
	67	apexDomain, err := dns.ApexDomain(domain)
64	68	if err != nil {
65	69	continue
66	70	}
67		tldPlus1Domains[tldPlus1] = true
	71	apexDomains[apexDomain] = true
68	72	}
69		return len(tldPlus1Domains)
	73	return len(apexDomains)
70	74	}
71	75
72	76	// ToMap returns a map of the CertNode's fields (weak serialization)

+5

-4

graph/domain_node.go less more

21	21	HasDNS bool
22	22	}
23	23
24		// NewDomainNode constructor for DomainNode, converts domain to nonWildcard
	24	// NewDomainNode constructor for DomainNode, converts domain to lower nonWildcard
25	25	func NewDomainNode(domain string, depth uint) *DomainNode {
26	26	domainNode := new(DomainNode)
27		domainNode.Domain = nonWildcard(domain)
	27	domainNode.Domain = nonWildcard(strings.ToLower(domain))
28	28	domainNode.Depth = depth
29	29	domainNode.Certs = make(map[fingerprint.Fingerprint][]string)
30	30	domainNode.RelatedDomains = make(status.Map)

36	36	// in the map
37	37	func (d *DomainNode) AddRelatedDomains(domains []string) {
38	38	for _, domain := range domains {
	39	domain = strings.ToLower(domain)
39	40	if _, ok := d.RelatedDomains[domain]; ok {
40	41	continue
41	42	}

43	44	}
44	45	}
45	46
46		// CheckForDNS checks for the existence of DNS records for the domain's tld+1
	47	// CheckForDNS checks for the existence of DNS records for the domain's apex
47	48	// sets the value to the node and returns the result as well
48	49	func (d *DomainNode) CheckForDNS(timeout time.Duration) (bool, error) {
49	50	hasDNS, err := dns.HasRecordsCache(d.Domain, timeout)

74	75	return fingerprints
75	76	}
76	77
77		// get the string representation of a node
	78	// String returns the string representation of a node
78	79	func (d *DomainNode) String() string {
79	80	certString := ""
80	81	// Certs

+2

-8

graph/graph.go less more

	0	// Package graph implements the graph data structures used by certgraph to build the certificate graph
0	1	package graph
1	2
2	3	import (

18	19	func NewCertGraph() *CertGraph {
19	20	graph := new(CertGraph)
20	21	return graph
21		}
22
23		// LoadOrStoreCert will return the CertNode in the graph with the provided node's fingerprint, or store the node if it did not already exist
24		// returned bool is true if the CertNode was found, false if stored
25		func (graph CertGraph) LoadOrStoreCert(certNode CertNode) (*CertNode, bool) {
26		foundCertNode, ok := graph.certs.LoadOrStore(certNode.Fingerprint, certNode)
27		return foundCertNode.(*CertNode), ok
28	22	}
29	23
30	24	// AddCert add a CertNode to the graph

96	90	certNode := node.(*CertNode)
97	91	if !cdn && certNode.CDNCert() {
98	92	//v(domain, "-> CDN CERT")
99		} else if maxSANsSize > 0 && certNode.TLDPlus1Count() > maxSANsSize {
	93	} else if maxSANsSize > 0 && certNode.ApexCount() > maxSANsSize {
100	94	//v(domain, "-> Large CERT")
101	95	} else {
102	96	for _, neighbor := range certNode.Domains {

+5

-1

status/status.go less more

	0	// Package status defines the various status certgraph discovered hosts/certificates may have
0	1	package status
1	2
2	3	import (

61	62	ERROR = iota
62	63	REDIRECT = iota
63	64	CT = iota
	65	MULTI = iota
64	66	)
65	67
66		// return domain status for printing
	68	// String returns the domain status for printing
67	69	func (status DomainStatus) String() string {
68	70	switch status {
69	71	case UNKNOWN:

82	84	return "Redirect"
83	85	case CT:
84	86	return "CT"
	87	case MULTI:
	88	return "MULTI"
85	89	}
86	90	return "?"
87	91	}

+26

-0

web/web.go less more

	0	// Package web defines a minimal web server for serving the web UI
	1	package web
	2
	3	import (
	4	"io/fs"
	5	"log"
	6	"net/http"
	7	)
	8
	9	// Serve starts a very basic webserver serving the embed web UI
	10	func Serve(addr string, data fs.FS) error {
	11	data, err := fs.Sub(data, "docs")
	12	if err != nil {
	13	return err
	14	}
	15	http.Handle("/", http.FileServer(http.FS(data)))
	16	return http.ListenAndServe(addr, logRequest(http.DefaultServeMux))
	17	}
	18
	19	// very minimal request logger
	20	func logRequest(handler http.Handler) http.Handler {
	21	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	22	log.Printf("%s %s %s\n", r.RemoteAddr, r.Method, r.URL)
	23	handler.ServeHTTP(w, r)
	24	})
	25	}