Use slices.DeleteFunc and pre-compile regexes

Author: jporzucek

main.go

@@ -520,13 +520,14 @@ func run(state overseer.State) {
 	verificationCacheMetrics := verificationcache.InMemoryMetrics{}
 
 	// Load allowlisted secrets if specified
-	var allowlistedSecrets map[string]struct{}
+	var allowlistedSecrets *detectors.CompiledAllowlist
 	if *allowlistSecretsFile != "" {
 		allowlistedSecrets, err = detectors.LoadAllowlistedSecrets(*allowlistSecretsFile)
 		if err != nil {
 			logFatal(err, "failed to load allowlisted secrets")
 		}
-		logger.Info("loaded allowlisted secrets", "count", len(allowlistedSecrets), "file", *allowlistSecretsFile)
+		totalCount := len(allowlistedSecrets.ExactMatches) + len(allowlistedSecrets.CompiledRegexes)
+		logger.Info("loaded allowlisted secrets", "exact_matches", len(allowlistedSecrets.ExactMatches), "regex_patterns", len(allowlistedSecrets.CompiledRegexes), "total", totalCount, "file", *allowlistSecretsFile)
 	}
 
 	engConf := engine.Config{

pkg/detectors/falsepositives.go

@@ -7,6 +7,7 @@ import (
 	"math"
 	"os"
 	"regexp"
+	"slices"
 	"strings"
 	"unicode"
 	"unicode/utf8"
@@ -15,6 +16,7 @@ import (
 	"gopkg.in/yaml.v3"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/log"
 )
 
 var (
@@ -39,6 +41,13 @@ type AllowlistEntry struct {
 	Values      []string `yaml:"values"`                // List of secret patterns/regexes to allowlist
 }
 
+// CompiledAllowlist holds both exact string matches and compiled regex patterns for efficient matching
+type CompiledAllowlist struct {
+	ExactMatches    map[string]struct{} // For exact string matching (O(1) lookup)
+	CompiledRegexes []*regexp.Regexp    // Pre-compiled regex patterns
+	RegexPatterns   []string            // Original regex patterns (for logging/debugging)
+}
+
 var (
 	filter *ahocorasick.Trie
 
@@ -213,46 +222,41 @@ func FilterKnownFalsePositives(ctx context.Context, detector Detector, results [
 // FilterAllowlistedSecrets filters out results that match allowlisted secrets.
 // This allows users to specify known safe secrets that should not be reported.
 // Supports regex patterns.
-func FilterAllowlistedSecrets(ctx context.Context, results []Result, allowlistedSecrets map[string]struct{}) []Result {
-	if len(allowlistedSecrets) == 0 {
+func FilterAllowlistedSecrets(ctx context.Context, results []Result, allowlist *CompiledAllowlist) []Result {
+	if allowlist == nil || (len(allowlist.ExactMatches) == 0 && len(allowlist.CompiledRegexes) == 0) {
 		return results
 	}
 
-	var filteredResults []Result
-	for _, result := range results {
+	return slices.DeleteFunc(results, func(result Result) bool {
 		if len(result.Raw) == 0 {
-			filteredResults = append(filteredResults, result)
-			continue
+			return false // Keep results with empty Raw
 		}
 
-		isAllowlisted := false
-		var matchReason string
-
 		// Check if the raw secret matches any allowlisted secret
 		rawSecret := string(result.Raw)
-		if isAllowlisted, matchReason = isSecretAllowlisted(rawSecret, allowlistedSecrets); isAllowlisted {
-			ctx.Logger().V(4).Info("Skipping result: allowlisted secret", "result", maskSecret(rawSecret), "reason", matchReason)
-			continue
+		log.RedactGlobally(rawSecret)
+		if isAllowlisted, matchReason := isSecretAllowlisted(rawSecret, allowlist); isAllowlisted {
+			ctx.Logger().V(4).Info("Skipping result: allowlisted secret", "result", rawSecret, "reason", matchReason)
+			return true // Delete this result
 		}
 
 		// Also check RawV2 if present
 		if result.RawV2 != nil {
 			rawV2Secret := string(result.RawV2)
-			if isAllowlisted, matchReason = isSecretAllowlisted(rawV2Secret, allowlistedSecrets); isAllowlisted {
-				ctx.Logger().V(4).Info("Skipping result: allowlisted secret", "result", maskSecret(rawV2Secret), "reason", matchReason)
-				continue
+			if isAllowlisted, matchReason := isSecretAllowlisted(rawV2Secret, allowlist); isAllowlisted {
+				ctx.Logger().V(4).Info("Skipping result: allowlisted secret", "result", rawV2Secret, "reason", matchReason)
+				return true // Delete this result
 			}
 		}
 
-		filteredResults = append(filteredResults, result)
-	}
-
-	return filteredResults
+		return false // Keep this result
+	})
 }
 
 // LoadAllowlistedSecrets loads secrets from a YAML file that should be allowlisted.
 // The YAML format supports multiline secrets and includes optional descriptions.
-func LoadAllowlistedSecrets(yamlFile string) (map[string]struct{}, error) {
+// Returns a CompiledAllowlist with pre-compiled regex patterns for efficient matching.
+func LoadAllowlistedSecrets(yamlFile string) (*CompiledAllowlist, error) {
 	file, err := os.Open(yamlFile)
 	if err != nil {
 		return nil, fmt.Errorf("failed to open allowlist file: %w", err)
@@ -270,49 +274,62 @@ func LoadAllowlistedSecrets(yamlFile string) (map[string]struct{}, error) {
 		content = append(content, '\n')
 	}
 
-	var entries []AllowlistEntry
-	if err := yaml.Unmarshal(content, &entries); err != nil {
+	var allowList []AllowlistEntry
+	if err := yaml.Unmarshal(content, &allowList); err != nil {
 		return nil, fmt.Errorf("failed to parse YAML allowlist file: %w", err)
 	}
+	// Use the shared compilation function
+	allowlist := CompileAllowlistPatterns(allowList)
+	return allowlist, nil
+}
 
-	allowlistedSecrets := make(map[string]struct{})
-	for _, entry := range entries {
-		for _, value := range entry.Values {
-			if strings.TrimSpace(value) != "" { // Skip empty values
-				allowlistedSecrets[value] = struct{}{}
+// CompileAllowlistPatterns compiles a list of patterns into a CompiledAllowlist.
+// All patterns are first attempted to be compiled as regex. If compilation fails,
+// they are treated as exact string matches.
+func CompileAllowlistPatterns(allowList []AllowlistEntry) *CompiledAllowlist {
+	allowlist := &CompiledAllowlist{
+		ExactMatches: make(map[string]struct{}),
+	}
+
+	for _, entry := range allowList {
+		for _, pattern := range entry.Values {
+			pattern = strings.TrimSpace(pattern)
+			if pattern == "" {
+				continue // Skip empty patterns
+			}
+
+			// Always try to compile as regex first
+			if compiledRegex, err := regexp.Compile(pattern); err == nil {
+				// Successfully compiled as regex
+				allowlist.CompiledRegexes = append(allowlist.CompiledRegexes, compiledRegex)
+				allowlist.RegexPatterns = append(allowlist.RegexPatterns, pattern)
+			} else {
+				// Invalid regex, treat as exact string match
+				allowlist.ExactMatches[pattern] = struct{}{}
 			}
 		}
 	}
 
-	return allowlistedSecrets, nil
+	return allowlist
 }
 
 // isSecretAllowlisted checks if a secret matches any allowlisted pattern (exact string or regex)
-func isSecretAllowlisted(secret string, allowlistedSecrets map[string]struct{}) (bool, string) {
-	// First, try exact string matching for performance
-	if _, isAllowlisted := allowlistedSecrets[secret]; isAllowlisted {
+func isSecretAllowlisted(secret string, allowlist *CompiledAllowlist) (bool, string) {
+	if allowlist == nil {
+		return false, ""
+	}
+
+	// First, try exact string matching for performance (O(1) lookup)
+	if _, isAllowlisted := allowlist.ExactMatches[secret]; isAllowlisted {
 		return true, "exact match"
 	}
 
-	// Try regex matching
-	for pattern := range allowlistedSecrets {
-		if regex, err := regexp.Compile(pattern); err == nil {
-			if regex.MatchString(secret) {
-				return true, "regex match: " + pattern
-			}
+	// Try pre-compiled regex patterns
+	for i, compiledRegex := range allowlist.CompiledRegexes {
+		if compiledRegex.MatchString(secret) {
+			return true, "regex match: " + allowlist.RegexPatterns[i]
 		}
 	}
 
 	return false, ""
 }
-
-// maskSecret masks a secret for safe logging by showing only the first and last few characters
-func maskSecret(secret string) string {
-	if len(secret) <= 8 {
-		return "***"
-	}
-	if len(secret) <= 16 {
-		return secret[:2] + "***" + secret[len(secret)-2:]
-	}
-	return secret[:4] + "***" + secret[len(secret)-4:]
-}