perf: parallelize trust verification and default to mozilla root store

CHANGELOG.md

@@ -20,6 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- **Breaking:** Default `TrustStore` in `DefaultOptions()` changed from `"system"` to `"mozilla"` — pure-Go Mozilla root verification is used by default instead of macOS `SecTrustEvaluateWithError` syscalls, eliminating multi-minute hangs on large certificate stores
+- Parallelize trust verification in scan summary, dump-certs, and AIA resolution — mozilla checks run concurrently, system checks only run for certs mozilla didn't trust
+- Add `TrustStore` label to `VerifyChainTrustInput` and debug-log every trust verification call with subject, store, and result
 - Normalize all exported private key PEM output (`.key`, K8s `tls.key`, YAML `key`) to PKCS#8 (`PRIVATE KEY`) regardless of input format ([#167])
 - Bundle export warns when Kubernetes TLS secret contains an unencrypted private key alongside encrypted outputs ([#167])
 - Use browser Web Crypto API for PBKDF2 key derivation in WASM builds to avoid blocking the main thread during encrypted key export ([#167])

README.md

@@ -403,9 +403,9 @@ if certkit.CertExpiresWithin(cert, 30*24*time.Hour) {
     // cert expires within 30 days
 }
 
-// Build verified chains (library defaults to system trust store)
+// Build verified chains (library defaults to mozilla trust store)
 opts := certkit.DefaultOptions()
-opts.TrustStore = "mozilla" // override the default system trust store if needed
+opts.TrustStore = "system" // override the default mozilla trust store if needed
 bundle, _ := certkit.Bundle(ctx, certkit.BundleInput{Leaf: leaf, Options: opts})
 
 // Generate keys

bundle.go

@@ -13,6 +13,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"runtime"
 	"slices"
 	"strings"
 	"sync"
@@ -337,6 +338,8 @@ type VerifyChainTrustInput struct {
 	Cert          *x509.Certificate
 	Roots         *x509.CertPool
 	Intermediates *x509.CertPool
+	// TrustStore is an optional label for debug logging (e.g. "mozilla", "system").
+	TrustStore string
 }
 
 // CheckTrustAnchorsInput holds parameters for CheckTrustAnchors.
@@ -366,8 +369,18 @@ type CheckTrustAnchorsResult struct {
 // invalid at the leaf's issuance time. This is an uncommon edge case in
 // practice (intermediates outlive the leaves they sign).
 func VerifyChainTrust(input VerifyChainTrustInput) bool {
+	if input.Cert == nil {
+		return false
+	}
+	store := input.TrustStore
+	if store == "" {
+		store = "unknown"
+	}
+	slog.Debug("verifying chain trust", "subject", input.Cert.Subject.CommonName, "store", store)
 	chains, err := verifyChainTrustChains(input)
-	return err == nil && len(chains) > 0
+	trusted := err == nil && len(chains) > 0
+	slog.Debug("chain trust result", "subject", input.Cert.Subject.CommonName, "store", store, "trusted", trusted)
+	return trusted
 }
 
 func verifyChainTrustChains(input VerifyChainTrustInput) ([][]*x509.Certificate, error) {
@@ -413,6 +426,7 @@ func CheckTrustAnchors(input CheckTrustAnchorsInput) CheckTrustAnchorsResult {
 		Cert:          input.Cert,
 		Roots:         mozillaPool,
 		Intermediates: input.Intermediates,
+		TrustStore:    "mozilla",
 	}) {
 		result.Anchors = append(result.Anchors, "mozilla")
 	}
@@ -422,13 +436,15 @@ func CheckTrustAnchors(input CheckTrustAnchorsInput) CheckTrustAnchorsResult {
 		Cert:          input.Cert,
 		Roots:         systemPool,
 		Intermediates: input.Intermediates,
+		TrustStore:    "system",
 	}) {
 		result.Anchors = append(result.Anchors, "system")
 	}
 	if input.FileRoots != nil && VerifyChainTrust(VerifyChainTrustInput{
 		Cert:          input.Cert,
 		Roots:         input.FileRoots,
 		Intermediates: input.Intermediates,
+		TrustStore:    "file",
 	}) {
 		result.Anchors = append(result.Anchors, "file")
 	}
@@ -496,7 +512,7 @@ func DefaultOptions() BundleOptions {
 		AIATimeout:       2 * time.Second,
 		AIAMaxDepth:      5,
 		AIAMaxTotalCerts: defaultAIAMaxTotalCerts,
-		TrustStore:       "system",
+		TrustStore:       "mozilla",
 		Verify:           true,
 		MaxIntermediates: defaultBundleMaxIntermediates,
 	}
@@ -705,25 +721,64 @@ func countAIAUnresolvedIssuers(certs []*x509.Certificate, roots *x509.CertPool)
 		}
 	}
 
-	unresolved := 0
-	for _, cert := range certs {
+	// Identify candidates that need trust verification.
+	type candidate struct {
+		idx  int
+		cert *x509.Certificate
+	}
+	var candidates []candidate
+	skipFlags := make([]bool, len(certs))
+	for i, cert := range certs {
 		if cert == nil {
+			skipFlags[i] = true
 			continue
 		}
 		if len(cert.IssuingCertificateURL) == 0 {
+			skipFlags[i] = true
 			continue
 		}
 		if IsMozillaRoot(cert) {
+			skipFlags[i] = true
 			continue
 		}
 		if bytes.Equal(cert.RawSubject, cert.RawIssuer) {
+			skipFlags[i] = true
+			continue
+		}
+		if roots != nil {
+			candidates = append(candidates, candidate{idx: i, cert: cert})
+		}
+	}
+
+	// Verify trust for all candidates concurrently. Bounded to NumCPU
+	// because system trust checks on macOS block in SecTrustEvaluateWithError.
+	trusted := make([]bool, len(certs))
+	if len(candidates) > 0 {
+		var wg sync.WaitGroup
+		sem := make(chan struct{}, runtime.NumCPU())
+		for _, c := range candidates {
+			wg.Add(1)
+			sem <- struct{}{}
+			go func(idx int, cert *x509.Certificate) {
+				defer wg.Done()
+				defer func() { <-sem }()
+				trusted[idx] = VerifyChainTrust(VerifyChainTrustInput{
+					Cert:          cert,
+					Roots:         roots,
+					Intermediates: intermediates,
+					TrustStore:    "aia-resolve",
+				})
+			}(c.idx, c.cert)
+		}
+		wg.Wait()
+	}
+
+	unresolved := 0
+	for i, cert := range certs {
+		if skipFlags[i] {
 			continue
 		}
-		if roots != nil && VerifyChainTrust(VerifyChainTrustInput{
-			Cert:          cert,
-			Roots:         roots,
-			Intermediates: intermediates,
-		}) {
+		if trusted[i] {
 			continue
 		}
 		if hasIssuerInSet(cert, certs) {

bundle_test.go

@@ -159,8 +159,8 @@ func TestDefaultOptions(t *testing.T) {
 	if opts.AIAMaxTotalCerts != defaultAIAMaxTotalCerts {
 		t.Fatalf("AIAMaxTotalCerts = %d, want %d", opts.AIAMaxTotalCerts, defaultAIAMaxTotalCerts)
 	}
-	if opts.TrustStore != "system" {
-		t.Fatalf("TrustStore = %q, want system", opts.TrustStore)
+	if opts.TrustStore != "mozilla" {
+		t.Fatalf("TrustStore = %q, want mozilla", opts.TrustStore)
 	}
 	if !opts.Verify {
 		t.Fatal("Verify = false, want true")

cmd/certkit/scan.go

@@ -12,8 +12,10 @@ import (
 	"log/slog"
 	"net/http"
 	"os"
+	"runtime"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/sensiblebit/certkit"
@@ -279,10 +281,61 @@ func runScan(cmd *cobra.Command, args []string) error {
 			}
 			intermediatePool := store.IntermediatePool()
 
+			// Pre-compute trust status concurrently. Mozilla checks are pure Go
+			// and fast; system checks hit macOS SecTrust which is slow. Run
+			// mozilla first, then only check system for untrusted remainders.
+			type dumpTrust struct {
+				mozilla bool
+				system  bool
+			}
+			now := time.Now()
+			trustStatus := make([]dumpTrust, len(certs))
+			if !scanForceExport && (trustPools.Mozilla != nil || trustPools.System != nil) {
+				var wg sync.WaitGroup
+				if trustPools.Mozilla != nil {
+					for i, c := range certs {
+						if !allowExpired && now.After(c.Cert.NotAfter) {
+							continue
+						}
+						wg.Add(1)
+						go func(idx int, cert *x509.Certificate) {
+							defer wg.Done()
+							trustStatus[idx].mozilla = certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{
+								Cert:          cert,
+								Roots:         trustPools.Mozilla,
+								Intermediates: intermediatePool,
+								TrustStore:    "mozilla",
+							})
+						}(i, c.Cert)
+					}
+					wg.Wait()
+				}
+				if trustPools.System != nil {
+					sem := make(chan struct{}, runtime.NumCPU())
+					for i, c := range certs {
+						if (!allowExpired && now.After(c.Cert.NotAfter)) || trustStatus[i].mozilla {
+							continue
+						}
+						wg.Add(1)
+						sem <- struct{}{}
+						go func(idx int, cert *x509.Certificate) {
+							defer wg.Done()
+							defer func() { <-sem }()
+							trustStatus[idx].system = certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{
+								Cert:          cert,
+								Roots:         trustPools.System,
+								Intermediates: intermediatePool,
+								TrustStore:    "system",
+							})
+						}(i, c.Cert)
+					}
+					wg.Wait()
+				}
+			}
+
 			var data []byte
 			var count, skipped int
-			now := time.Now()
-			for _, c := range certs {
+			for i, c := range certs {
 				cert := c.Cert
 
 				// Skip expired certificates unless --allow-expired is set
@@ -294,17 +347,7 @@ func runScan(cmd *cobra.Command, args []string) error {
 
 				// Validate chain unless --force is set (uses same logic as summary)
 				if !scanForceExport {
-					mozillaTrusted := trustPools.Mozilla != nil && certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{
-						Cert:          cert,
-						Roots:         trustPools.Mozilla,
-						Intermediates: intermediatePool,
-					})
-					systemTrusted := trustPools.System != nil && certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{
-						Cert:          cert,
-						Roots:         trustPools.System,
-						Intermediates: intermediatePool,
-					})
-					if !mozillaTrusted && !systemTrusted && (trustPools.Mozilla != nil || trustPools.System != nil) {
+					if !trustStatus[i].mozilla && !trustStatus[i].system && (trustPools.Mozilla != nil || trustPools.System != nil) {
 						slog.Debug("skipping untrusted certificate", "subject", cert.Subject)
 						skipped++
 						continue

cmd/wasm/main.go

@@ -327,7 +327,7 @@ func getState(_ js.Value, _ []js.Value) any {
 		expired := now.After(rec.NotAfter)
 		trusted := false
 		if roots != nil {
-			trusted = certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{Cert: rec.Cert, Roots: roots, Intermediates: intermediatePool})
+			trusted = certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{Cert: rec.Cert, Roots: roots, Intermediates: intermediatePool, TrustStore: "mozilla"})
 		}
 
 		serial := ""

internal/certstore/memstore.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"log/slog"
 	"maps"
+	"runtime"
 	"slices"
 	"sort"
 	"sync"
@@ -398,19 +399,61 @@ func (s *MemStore) ScanSummary(input ScanSummaryInput) ScanSummary {
 		}
 	}
 
+	// Pre-compute trust status for all non-expired certs concurrently.
+	// Mozilla checks are pure Go and fast; system checks hit the macOS
+	// Security framework (SecTrustEvaluateWithError) which is slow.
+	// Run mozilla first, then only check system for certs mozilla didn't trust.
+	type trustStatus struct {
+		mozilla bool
+		system  bool
+	}
 	now := time.Now()
-	for _, rec := range certs {
-		expired := now.After(rec.NotAfter)
-		mozillaTrusted := false
-		systemTrusted := false
-		if !expired {
-			if input.MozillaPool != nil {
-				mozillaTrusted = certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{Cert: rec.Cert, Roots: input.MozillaPool, Intermediates: intermediatePool})
+	trustResults := make([]trustStatus, len(certs))
+	var wg sync.WaitGroup
+	if input.MozillaPool != nil {
+		for i, rec := range certs {
+			if now.After(rec.NotAfter) {
+				continue
 			}
-			if input.SystemPool != nil {
-				systemTrusted = certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{Cert: rec.Cert, Roots: input.SystemPool, Intermediates: intermediatePool})
+			wg.Add(1)
+			go func(idx int, cert *x509.Certificate) {
+				defer wg.Done()
+				trustResults[idx].mozilla = certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{
+					Cert:          cert,
+					Roots:         input.MozillaPool,
+					Intermediates: intermediatePool,
+					TrustStore:    "mozilla",
+				})
+			}(i, rec.Cert)
+		}
+		wg.Wait()
+	}
+	if input.SystemPool != nil {
+		sem := make(chan struct{}, runtime.NumCPU())
+		for i, rec := range certs {
+			if now.After(rec.NotAfter) || trustResults[i].mozilla {
+				continue
 			}
+			wg.Add(1)
+			sem <- struct{}{}
+			go func(idx int, cert *x509.Certificate) {
+				defer wg.Done()
+				defer func() { <-sem }()
+				trustResults[idx].system = certkit.VerifyChainTrust(certkit.VerifyChainTrustInput{
+					Cert:          cert,
+					Roots:         input.SystemPool,
+					Intermediates: intermediatePool,
+					TrustStore:    "system",
+				})
+			}(i, rec.Cert)
 		}
+		wg.Wait()
+	}
+
+	for i, rec := range certs {
+		expired := now.After(rec.NotAfter)
+		mozillaTrusted := trustResults[i].mozilla
+		systemTrusted := trustResults[i].system
 
 		switch rec.CertType {
 		case "root":