crypto/x509: defer directory scan when cert bundle provides roots

src/crypto/x509/cert_pool.go

@@ -33,6 +33,37 @@ type CertPool struct {
 	// verifications, one using the roots provided by the caller, and one using
 	// the system platform verifier.
 	systemPool bool
+
+	// lazyRoots holds deferred cert directories that are loaded on demand
+	// if primary verification fails. The pointer is shared across clones
+	// so the directory scan happens at most once across all pools derived
+	// from the same system roots.
+	// See https://go.dev/issue/38869.
+	lazyRoots *lazyRootState
+}
+
+// lazyRootState holds directories to be lazily scanned for certificates.
+// It is shared by pointer across CertPool clones.
+type lazyRootState struct {
+	once sync.Once
+	dirs []string
+	pool *CertPool
+}
+
+// load triggers the one-time directory scan and returns the resulting pool.
+// Returns nil if no directories were configured or no certificates were found.
+func (s *lazyRootState) load() *CertPool {
+	if s == nil {
+		return nil
+	}
+	s.once.Do(func() {
+		p := NewCertPool()
+		readCertsFromDirs(p, s.dirs)
+		if p.len() > 0 {
+			s.pool = p
+		}
+	})
+	return s.pool
 }
 
 // lazyCert is minimal metadata about a Cert and a func to retrieve it
@@ -90,6 +121,7 @@ func (s *CertPool) Clone() *CertPool {
 		lazyCerts:  make([]lazyCert, len(s.lazyCerts)),
 		haveSum:    make(map[sum224]bool, len(s.haveSum)),
 		systemPool: s.systemPool,
+		lazyRoots:  s.lazyRoots,
 	}
 	for k, v := range s.byName {
 		indexes := make([]int, len(v))
@@ -272,7 +304,7 @@ func (s *CertPool) Equal(other *CertPool) bool {
 	if s == nil || other == nil {
 		return s == other
 	}
-	if s.systemPool != other.systemPool || len(s.haveSum) != len(other.haveSum) {
+	if s.systemPool != other.systemPool || s.lazyRoots != other.lazyRoots || len(s.haveSum) != len(other.haveSum) {
 		return false
 	}
 	for h := range s.haveSum {

src/crypto/x509/root.go

@@ -9,6 +9,7 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"slices"
 	"runtime"
 	"strings"
 	"sync"
@@ -133,6 +134,7 @@ const (
 )
 
 var x509sslcertoverrideplatform = godebug.New("x509sslcertoverrideplatform")
+var x509lazydirscan = godebug.New("x509lazydirscan")
 
 func loadSystemRoots() (*CertPool, error) {
 	certFilePath, certDirPath := os.Getenv(certFileEnv), os.Getenv(certDirEnv)
@@ -150,6 +152,28 @@ func loadSystemRoots() (*CertPool, error) {
 	return loadOnDiskRoots(certFilePath, certDirPath)
 }
 
+// readCertsFromDirs reads PEM certificates from each directory and appends
+// them to pool. It returns the first non-IsNotExist error encountered, if any.
+func readCertsFromDirs(pool *CertPool, dirs []string) error {
+	var firstErr error
+	for _, directory := range dirs {
+		fis, err := readUniqueDirectoryEntries(directory)
+		if err != nil {
+			if firstErr == nil && !os.IsNotExist(err) {
+				firstErr = err
+			}
+			continue
+		}
+		for _, fi := range fis {
+			data, err := os.ReadFile(filepath.Join(directory, fi.Name()))
+			if err == nil {
+				pool.AppendCertsFromPEM(data)
+			}
+		}
+	}
+	return firstErr
+}
+
 func loadOnDiskRoots(certFilePath, certDirPath string) (*CertPool, error) {
 	roots := NewCertPool()
 
@@ -171,29 +195,33 @@ func loadOnDiskRoots(certFilePath, certDirPath string) (*CertPool, error) {
 	}
 
 	dirs := certDirectories
+	userProvidedDirs := false
 	if certDirPath != "" {
 		// OpenSSL and BoringSSL both use ":" as the SSL_CERT_DIR separator on
 		// Unix-like systems, and ";" on Windows.
 		// See:
 		//  * https://golang.org/issue/35325
 		//  * https://docs.openssl.org/4.0/man1/openssl-rehash/#environment
 		dirs = filepath.SplitList(certDirPath)
+		userProvidedDirs = true
 	}
 
-	for _, directory := range dirs {
-		fis, err := readUniqueDirectoryEntries(directory)
-		if err != nil {
-			if firstErr == nil && !os.IsNotExist(err) {
-				firstErr = err
-			}
-			continue
-		}
-		for _, fi := range fis {
-			data, err := os.ReadFile(filepath.Join(directory, fi.Name()))
-			if err == nil {
-				roots.AppendCertsFromPEM(data)
-			}
+	// If we already have roots from a cert file and the directories were not
+	// explicitly provided by the user, skip scanning them. On most systems
+	// the cert file is a bundling of the certificates in the directory, so
+	// scanning would just re-read the same certificates.
+	// The directories are stored for lazy loading in case chain construction
+	// using the bundle roots fails, ensuring backward compatibility.
+	// See https://go.dev/issue/38869.
+	if roots.len() > 0 && !userProvidedDirs && x509lazydirscan.Value() != "0" {
+		roots.lazyRoots = &lazyRootState{
+			dirs: slices.Clone(dirs),
 		}
+		return roots, nil
+	}
+
+	if err := readCertsFromDirs(roots, dirs); err != nil && firstErr == nil {
+		firstErr = err
 	}
 
 	if roots.len() > 0 || firstErr == nil {