Codebase list certgraph / 66fc50f driver / crtsh / crtsh.go
66fc50f

Tree @66fc50f (Download .tar.gz)

crtsh.go @66fc50fraw · history · blame

// Package crtsh implements an unofficial API client for Comodo's
// Certificate Transparency search
// https://crt.sh/
//
// As the API is unofficial and has been reverse engineered it may stop working
// at any time and comes with no guarantees.
//
package crtsh

// TODO running in verbose gives error: pq: unnamed prepared statement does not exist

import (
	"database/sql"
	"fmt"
	"path"
	"time"

	"github.com/lanrat/certgraph/driver"
	"github.com/lanrat/certgraph/fingerprint"
	"github.com/lanrat/certgraph/status"
	_ "github.com/lib/pq" // portgresql
)

const connStr = "postgresql://[email protected]/certwatch?sslmode=disable"
const driverName = "crtsh"

func init() {
	driver.AddDriver(driverName)
}

type crtsh struct {
	db                *sql.DB
	queryLimit        int
	timeout           time.Duration
	save              bool
	savePath          string
	includeSubdomains bool
	includeExpired    bool
}

type crtshCertDriver struct {
	host         string
	fingerprints driver.FingerprintMap
	driver       *crtsh
}

func (c *crtshCertDriver) GetFingerprints() (driver.FingerprintMap, error) {
	return c.fingerprints, nil
}

func (c *crtshCertDriver) GetStatus() status.Map {
	return status.NewMap(c.host, status.New(status.CT))
}

func (c *crtshCertDriver) GetRelated() ([]string, error) {
	return make([]string, 0), nil
}

func (c *crtshCertDriver) QueryCert(fp fingerprint.Fingerprint) (*driver.CertResult, error) {
	return c.driver.QueryCert(fp)
}

// Driver creates a new CT driver for crt.sh
func Driver(maxQueryResults int, timeout time.Duration, savePath string, includeSubdomains, includeExpired bool) (driver.Driver, error) {
	d := new(crtsh)
	d.queryLimit = maxQueryResults
	d.includeSubdomains = includeSubdomains
	d.includeExpired = includeExpired
	var err error

	if len(savePath) > 0 {
		d.save = true
		d.savePath = savePath
	}

	d.db, err = sql.Open("postgres", connStr)
	if err != nil {
		return nil, err
	}

	err = d.setSQLTimeout(d.timeout.Seconds())

	return d, err
}

func (d *crtsh) GetName() string {
	return driverName
}

func (d *crtsh) setSQLTimeout(sec float64) error {
	_, err := d.db.Exec(fmt.Sprintf("SET statement_timeout TO %f;", (1000 * sec)))
	return err
}

func (d *crtsh) QueryDomain(domain string) (driver.Result, error) {
	results := &crtshCertDriver{
		host:         domain,
		fingerprints: make(driver.FingerprintMap),
		driver:       d,
	}

	queryStr := ""

	if d.includeSubdomains {
		if d.includeExpired {
			queryStr = `SELECT digest(certificate.certificate, 'sha256') sha256
					FROM certificate_identity, certificate
					WHERE certificate.id = certificate_identity.certificate_id
					AND (reverse(lower(certificate_identity.name_value)) LIKE reverse(lower('%%.'||$1))
                	OR reverse(lower(certificate_identity.name_value)) LIKE reverse(lower($1)))
					LIMIT $2`
		} else {
			queryStr = `SELECT digest(certificate.certificate, 'sha256') sha256
					FROM certificate_identity, certificate
					WHERE certificate.id = certificate_identity.certificate_id
					AND x509_notAfter(certificate.certificate) > statement_timestamp()
					AND (reverse(lower(certificate_identity.name_value)) LIKE reverse(lower('%%.'||$1))
                	OR reverse(lower(certificate_identity.name_value)) LIKE reverse(lower($1)))
					LIMIT $2`
		}
	} else {
		if d.includeExpired {
			queryStr = `SELECT digest(certificate.certificate, 'sha256') sha256
					FROM certificate_identity, certificate
					WHERE certificate.id = certificate_identity.certificate_id
					AND reverse(lower(certificate_identity.name_value)) LIKE reverse(lower($1))
					LIMIT $2`
		} else {
			queryStr = `SELECT digest(certificate.certificate, 'sha256') sha256
					FROM certificate_identity, certificate
					WHERE certificate.id = certificate_identity.certificate_id
					AND x509_notAfter(certificate.certificate) > statement_timestamp()
					AND reverse(lower(certificate_identity.name_value)) LIKE reverse(lower($1))
					LIMIT $2`
		}
	}

	if d.includeSubdomains {
		domain = fmt.Sprintf("%%.%s", domain)
	}

	try := 0
	var err error
	var rows *sql.Rows
	for try < 5 {
		// this is a hack while crt.sh gets there stuff togeather
		try++
		rows, err = d.db.Query(queryStr, domain, d.queryLimit)
		if err == nil {
			break
		}
	}
	/*if try > 1 {
		fmt.Println("QueryDomain try ", try)
	}*/
	if err != nil {
		return results, err
	}

	for rows.Next() {
		var hash []byte
		err = rows.Scan(&hash)
		if err != nil {
			return results, err
		}
		results.fingerprints.Add(domain, fingerprint.FromHashBytes(hash))
	}

	return results, nil
}

func (d *crtsh) QueryCert(fp fingerprint.Fingerprint) (*driver.CertResult, error) {
	certNode := new(driver.CertResult)
	certNode.Fingerprint = fp
	certNode.Domains = make([]string, 0, 5)

	queryStr := `SELECT DISTINCT certificate_identity.name_value
				FROM certificate, certificate_identity
				WHERE certificate.id = certificate_identity.certificate_id
				AND certificate_identity.name_type in ('dNSName', 'commonName')
				AND digest(certificate.certificate, 'sha256') = $1`

	try := 0
	var err error
	var rows *sql.Rows
	for try < 5 {
		// this is a hack while crt.sh gets there stuff togeather
		try++
		rows, err = d.db.Query(queryStr, fp[:])
		if err == nil {
			break
		}
	}
	/*if try > 1 {
		fmt.Println("QueryCert try ", try)
	}*/
	if err != nil {
		return certNode, err
	}

	for rows.Next() {
		var domain string
		err = rows.Scan(&domain)
		if err != nil {
			return nil, err
		}
		certNode.Domains = append(certNode.Domains, domain)
	}

	if d.save {
		var rawCert []byte
		queryStr = `SELECT certificate.certificate
					FROM certificate
					WHERE digest(certificate.certificate, 'sha256') = $1`
		row := d.db.QueryRow(queryStr, fp[:])
		err = row.Scan(&rawCert)
		if err != nil {
			return certNode, err
		}

		err = driver.RawCertToPEMFile(rawCert, path.Join(d.savePath, fp.HexString())+".pem")
		if err != nil {
			return certNode, err
		}
	}

	return certNode, nil
}