diff --git a/CHANGELOG.md b/CHANGELOG.md index a4e50d6..6526ee6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to this project will be documented in this file. +## [1.40.0] +- Dropped the “prevalidated” flag from rule definitions and validation logic so every finding now flows through the standard active/inactive/unknown pipeline, simplifying rule configuration and preventing special‑case bypasses +- Improved Tailscale api key detectors + ## [1.39.0] - Added support for scanning Confluence pages via `--confluence-url` and `--cql` diff --git a/Cargo.toml b/Cargo.toml index 90ac4a4..15ad49a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ publish = false [package] name = "kingfisher" -version = "1.39.0" +version = "1.40.0" description = "MongoDB's blazingly fast secret scanning and validation tool" edition.workspace = true rust-version.workspace = true diff --git a/data/rules/generic.yml b/data/rules/generic.yml index 3f71d4a..73714b9 100644 --- a/data/rules/generic.yml +++ b/data/rules/generic.yml @@ -127,7 +127,7 @@ rules: 'policy_path': os.path.join(TEST_DIR, 'policy.json') }) - name: Weak Password Pattern - id: kingfisher.weak_password.1 + id: kingfisher.generic.6 pattern: | (?xi) \b diff --git a/data/rules/pem.yml b/data/rules/pem.yml index dc8e558..390171d 100644 --- a/data/rules/pem.yml +++ b/data/rules/pem.yml @@ -10,7 +10,6 @@ rules: -----END\ .{0,20}\ ?PRIVATE\ KEY\ ?.{0,20}----- min_entropy: 4.5 confidence: high - prevalidated: true examples: - | -----BEGIN RSA PRIVATE KEY----- @@ -62,7 +61,6 @@ rules: (?: [^a-zA-Z0-9+/=] | $ ) min_entropy: 4.5 confidence: high - prevalidated: true examples: - 'PRIVATE_KEY_B64=LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBb3kxWFh1VkFRcHFIYlFFMDVta2hyTmcvMTI0Ri8ySzlPYW5pelpUWlVVaEswOFU4CkxhaC9SbVVsWHFRMDEvU255aktGOWZqUDhFcU1OZ1dpamUzYmVwL3RPOVpTMEFUMi9PVlJXeS9TOG52RDQ5WTMKenMxMktSbERhR2lZc0RsYUZrbHJkeDQ4RWhRVmdHN3hmWE1jaC9OejJzc2FEby9kRkNBOW80TkZZQWUzM2UveApWNVo1UHNkWkl6dkNZQVlCNDRoUEtpN3JXRE1IbFdzM1kvVkVtQXMzSzVNK2QvL3QzRHB4WnBEbWJERGdYa2w2CjZUdDh3VXloUVZ3MkZpMStobTF1T2QwYjFkaW9aNko2OXNTT2JOZXpSR3YxYjdZaFltT0JKL1JBbHN5ZHoxTmgKVXpXT1lYV0Z1OGJrOU9JM3lQMEc0TE84QjhtbWRldE1RVVoyelFJREFRQUJBb0lCQUN2ckhUUHVVZ0JiSlE0QwpvQ0ZQdEgrWDZIN3NIdk1ndVR0VzdUTlYxN1BYMkVQdE53ZzI3S0tld0pNYmNSbWF3THBjSk5BU09xMDY4MGZxCjlsaHE1NEsybnB4WFVBeXErV3NSc1hid2hUODhibm5aQTBaRzZJR2hTaEpFN0t1cGxBU2htQ29FV2ppbmJTNFgKTGlvTW5HWSs4VFMzSzNrMTRWUDBaWUtuNXprMERHZnFBMEo0VTRXSmxUeGwrTWZxd0pJOTlrcTdHbFVlZkdncQpuK3Q1d2NrV3BPbTd5TUJjZTlTSXlmTm54bnU3TkZYQm50VTN5RGxSUThWUWZmNEtRMzJCaWNiYlJWemR1TThNCnNxMU5CZWNzL0EzUXRvdG1nWUc4d094ZXpNS3Iyays2QzB2NmlFc0h5T0lmR25GWktSZDJFd0dnWlo3aytURHUKUUYrcjd1VUNnWUVBMkRqNUJoYmpybDFRNTZya3BhTGFvVldRV1Y5YUYzUUJtNlNZM2VQYmlvY2JNR2k1ak1ESQpkSjdJVXlLYUljK3BNV1RQYlBmVUd2WmNENlczZDFBNUNUSnFuWHVuVlY3czRqaWJ6WDZUbjhNM3IrMHZTZnNZCmdPMHBtRFpndlNqaVZTRUNBQTZFOFUxQ1lFZU5KUDFDOW12cGJVNzJRTEpndWp3M3JMb2oyYmNDZ1lFQXdUSXYKOUNSeWNOQXRBbDcvUHdWZGh5eXRvVHBSRnZDSU1HSVk5SjMxZ3lva0ZlaFQvWjQ4WkF6anl6ZTBSUXYzdGUxTQoveVJMQkVETGkwbEtrZFVXckVkaVR3dm1KdkpwMDZ0OEdCbERsK25ycXVLWTFxVThDbTR5cis4QzZtRThkVnZrClNINXBhRXptOERFTE1wSjhGVTZFYnhmZHZjRzZmSGx6dnVnZmc1c0NnWUFFQ1BRa3QvS2h3MTRLSkxkRm5BZG0KY1ZsVFFhTkZ3c1Z3NlI1dExaNWdOR3MrZVFYVmFaZVVEWTZCZHFqWHJxOWltNVgvVzVTYXVEUTVtb2NVOCt0TQpqNk5Mc3c0SldzOGkzWm1TdVNUNkcwT0R4ZkpXK0JlWitGTUpZeUpsQlVsTCsyUzFLWkF6akpTTGhXcE40V2dKCmZ6UUk5U3RGUTg3b1NzMWpMTW9VZXdLQmdGOE9CMlFURHErTTdhaE4vejROc0wvU2JyZDJEdkcvZFBLQlFaQVIKcS90V0g1MGJ5ejlzdkgvcGk2YXdDS1UwUnpPZXh4UjkwZDhNMWxqNHZaVFZDQ3ZKajRnZTdhVlovbEdqL1JHSwpWS1NJOW1nRXgzaE1vaWJybzByR3lXTnlaaUhFRGFUUmRhRll2UU9PemRpYkZDd1RqcnR1UGE2Z2c5VzhtQU5sCkNDUmpBb0dBSTRIbnpyV3kzaU5kR2xqVnh4bW1DN1V0c0MvajJBUEZpcHc0ZHJ0U2NsMDFRZzF5WkowbDNBTk4KOU5lTmVSUUFzN3pFTng2T1B1SzlxYy83T1ROMTJKaHdoUTIzdXZwNjZjV0krdTRjcVpOZTJyZVFVVWVmM3psbQpMcXRmOU50VHp5M3pjMGZQcGoxQnBlRmxHSG9SVDhjVHpBWjFTeGwyZWChazlqS2RVeDQ9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0t' - ' "privateKey": "LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBbUhKOEJHdTFYZUZ4aENVQXBrNHNSTVI4RnRTdGtyMEx0OWtWTGNSUjRFWitiOWhHCmR0blJpOFhqV3d5MU5zMHliMkJMdHBpVHZKSFVKTUphWXluZ2ZkZnZhcWhocm1yYm5vV0pLQkxmeUxwTXFNS1EKQ3RialFxbnVrQURJUWVQd2ZGeTNpVHkxd1JkRC9zTUs1U0VtV0Fxb0pZQk50eTFZZzA2UzVkYVlPM2xjY3hrYQpQWjRjcm9McWF6Ny9tU3dDVTR5VWRSb3h4WVF4VG1MZXg5M2tqU09TTmdpK0FXc0lCbjV3UHI0VHNuVHFSeWpIClN2aEdMdk9YREpRYWZRdk56WjFSL1FYMzlOQk9xOEVKZW5pWXdaUm9uNVcvNVhMYW94MFFyUGhrY1BES3A5SVUKeHpJakUwWlNmMStUK1FFbTQ3TkFtSnhvZjFhdGRFVzZDTCtheHdJREFRQUJBb0lCQUQ3enI4REhsWnFSK1NWZgpmbGd1bWRzLzVCb3Rjd3ZRWXlGbFZIaVV4RmEvNVlCY0tDVDJKN0QzWTc1NmplNTJaK2hVTkkvUGk5cG53ZG40CkpBa2xCdDRRcUg0NzBES05UK216TFFOT1gvanM3YkVXdnhLcTBDZjhNbFptN0V0QlRGS2VtdS9pRVJBT2duYVcKcGs0ZUZVNXdBQ1dVU1FObWgxR1p4ZEdCZjFXM1VjUnQxcFRvOEtQTDluZm4vSGJiRFNsQkNVL3VIcWd2TSt2cApmTE03bzRIVDZ1K1ZzU00rWGZqeDhpeE5ZRHdoalNuKzQyZm13d1d3ZzJISHUrdUozZ1pUSWQwRUI1VW9hdUNjCjZUTlVtcEJscjU5UGFmVkZRWUY1S3VxaHJXKzVQaWpHcHBZcXg4Ynl6aFpOQzkwZnl5V0NXcXg2eGFZVm5OdzgKNkJmUXM2a0NnWUVBeVlyRVg1NU1RTzJnWDY2TGwxaGJDMzNzWk1OZzloVG1SK1doSTFjNksvbFZ1TFoyL0RPdwpsYTZ6eHdBU204Z0ZyVUFYbUljV2h2b3FwWGVzNWZzOVZKeDlNT0ZVYVBrckRPQllnY1laMUR6VVNVOHc3SSttCnlyV3hRUkRNajhvSGpRbHVpM0s2MzZucm5RajhxOGkvQ2dranVPcHJGZnliMzVEMFlDdjVXZzBDZ1lFQXdhT3cKRWFhN0l1MjFGa08vbmFjdVhjSnBhNkVlUTNqZFNlNlRQaXZ6bVVXU0haeGJuUy9XSnJaRjQwSExzUWxOZHl0ZgpNTTBKZFU0VmMyR0NVc1pMYjdQSmJwdVRqRERSSHJXV1pCMnhiemF0K3A3N2RzNWlOcXFRcTZ6M0syUVh4Y3ZTCis5am5VZXpDU2Y0N1R1OWNTTW96V3hTMW82b1BPSFdHVFRvdHR5TUNnWUFQdWc1Y3o4TnZoWnR3Ry9TMG1LWnkKSFI5bk5YL0pkQlFNSkRVUXh1dTVKcm16c2psU3NNM2t3RDh6RmlSZGw1d3B5c2lNbEc0RGxsM2hqNWNrVXhpVQpFNm9KT0d3WHpPbTVGWUNTajl6UUhQY0x5V3d0NlgvQWJiRXBQS0JaMEJBS3gyT2k2ZzcvQ1FsanRhSFIzZFphCmVDQWJlOTlqVmRUcit5bTJuM2ZUdVFLQmdBMm5TZ25rbEx0Z3dXMEJkK2hZMm1jWUJ6RGttbXF0Z2dUdGdvcFcKdFFWd3AxM1pJWWlTeituSTNtS295QUVDbytpc01Ua1NyQUVPY1dyQ1RGc2p5anZsRkdYdEtGa3hNLzJUVmpoVwo4NlRnMlNHYnhpVlpaZ2x1dTJhdmVub2Z3NkZadnRXdE5KcE5OR0hkUURkUG4xVXVsTEp1WW1SWTRGdmR4WXQ2CmQ3QzdBb0dBRUsvalFiZ0l3OXFLQUNOZ0JySnB1cU5Ham9JajFoQTRlb29DMXp1bFEyZUpnZ2J5OTBpSDg2VzEKM0xyOVZMVFkyc2JKTzlqekZVR0lOL01BOEhYQTE1a2grZHRibkRsdFRFZGNnenBCRzhCQUZRQ3hQWnBGWHhtZgpDUmhXN1l6RW1IeWJ4R0toR3NOK2M3NUhKTHZFSWwrRTh6eitXRk9xT240dkJXU1ZwSnc9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==",' \ No newline at end of file diff --git a/data/rules/privkey.yml b/data/rules/privkey.yml index 58dce65..7b5e551 100644 --- a/data/rules/privkey.yml +++ b/data/rules/privkey.yml @@ -24,7 +24,6 @@ rules: ----- min_entropy: 4.5 confidence: high - prevalidated: true examples: - |- -----BEGIN RSA PRIVATE KEY----- @@ -77,7 +76,6 @@ rules: ) min_entropy: 4.5 confidence: high - prevalidated: true examples: - | -----BEGIN PRIVATE KEY----- diff --git a/data/rules/tailscale.yml b/data/rules/tailscale.yml index f393da7..3544b1f 100644 --- a/data/rules/tailscale.yml +++ b/data/rules/tailscale.yml @@ -5,14 +5,14 @@ rules: (?xi) \b ( - tskey-[a-z]+-[A-Z0-9_-]{20,24} + tskey-[a-z]{3,10}-[A-Z0-9_-]{20,36} ) \b min_entropy: 3.0 confidence: medium examples: - - tskey-secret-12345678-abcdefghijkl - - tskey-api-abcdefg-1234567890123 + - tskey-secret-weRTWSfoeFKI-3480754342kDSFelW3 + - tskey-api-weRTWSfoeFKI-3480754342kDSFelW3 references: - https://tailscale.com/kb/1215/oauth-clients validation: diff --git a/src/matcher.rs b/src/matcher.rs index 33e2425..21f7dec 100644 --- a/src/matcher.rs +++ b/src/matcher.rs @@ -930,7 +930,6 @@ mod test { visible: true, examples: vec![], negative_examples: vec![], - prevalidated: false, references: vec![], validation: None::, // no HTTP validation needed depends_on_rule: vec![], @@ -972,7 +971,6 @@ mod test { visible: true, examples: vec![], negative_examples: vec![], - prevalidated: false, references: vec![], validation: Some(Validation::Http(HttpValidation { request: HttpRequest { @@ -1089,7 +1087,6 @@ mod test { visible: true, examples: vec![], negative_examples: vec![], - prevalidated: false, references: vec![], validation: None::, depends_on_rule: vec![], diff --git a/src/rules/rule.rs b/src/rules/rule.rs index a47b172..1947a0a 100644 --- a/src/rules/rule.rs +++ b/src/rules/rule.rs @@ -282,9 +282,6 @@ pub struct RuleSyntax { /// Optional dependencies on other rules. #[serde(default)] pub depends_on_rule: Vec>, - /// Whether matches should always be considered validated. - #[serde(default)] - pub prevalidated: bool, } lazy_static! { diff --git a/src/safe_list.rs b/src/safe_list.rs index 468ac2c..bbea073 100644 --- a/src/safe_list.rs +++ b/src/safe_list.rs @@ -1,46 +1,96 @@ use once_cell::sync::Lazy; use regex::bytes::Regex; use tracing::debug; + +/// Case-insensitive patterns that indicate a *benign* match (placeholders, examples, redactions, etc.). +/// `is_safe_match()` returns true if any of these are present. static SAFE_LIST_FILTER_REGEX: Lazy>> = Lazy::new(|| { vec![ + // Assignment-like value that ends with "EXAMPLEKEY" (common placeholder) + // e.g., "KEY=ABC_EXAMPLEKEY" or "key: fooEXAMPLEKEY" compile_regex(r"(?i)[:=][^:=]{0,64}EXAMPLEKEY"), + + // AWS-style AKIA keys explicitly marked as example/fake/test/sample + // e.g., "AKIA...EXAMPLE", "AKIA...FAKE", "AKIA...SAMPLE" compile_regex(r"(?i)\b(AKIA(?:.*?EXAMPLE|.*?FAKE|TEST|.*?SAMPLE))\b"), + + // Secret-y key name followed by short value and then "&&" / "||" or a run of asterisks + // e.g., "password=foo &&", "secret: *****" (redacted/masked) compile_regex( r"(?i)(password|pass|pwd|passwd|secret|cred|key|auth|authorization)[^=:?]{0,8}[=:?][^=:?]{0,8}\s(&&|\|\||\*{5,50})", ), + + // Secret-y key name with short value, then *another* short assignment on the same line + // Typical of docs/examples rather than hardcoded secrets compile_regex( r"(?i)(password|pass|pwd|passwd|secret|cred|key|auth|authorization)[^=:?]{0,8}[=:?][^=:?]{0,8}\b\w{4,12}\s{0,6}=\s{0,6}\D{0,3}\w{1,12}", ), + + // Secret-y key assigned to a shell variable reference (e.g., "$FOO") — not a literal secret compile_regex( r"(?i)(password|pass|pwd|passwd|secret|cred|key|auth|authorization)[^=:?]{0,8}[=:?][^=:?]{0,8}\$\w{4,30}", ), + + // Secret-y key set via command that *generates* randomness, not a literal value + // e.g., "password = openssl rand -base64 32" compile_regex( r"(?i)(password|pass|pwd|passwd|secret|cred|key|auth|authorization)[^=:?]{0,16}[=:?][^=:?]{0,8}\bopenssl\s{0,4}rand\b", ), + + // Secret-y key assigned a value containing "encrypted" (marker/metadata, not a secret) compile_regex( r"(?i)(password|pass|pwd|passwd|secret|cred|key|auth|authorization)[^=:?]{0,8}[=:?][^=:?]{0,8}encrypted", ), + + // Secret-y key assigned boolean literals — not secrets + // e.g., "auth=false" compile_regex( r"(?i)(password|pass|pwd|passwd|secret|cred|key|auth|authorization)[^=:?]{0,8}[=:?][^=:?]{0,8}\b(?:false|true)\b", ), + + // Secret-y key assigned to null-ish or self-referential placeholders — not secrets + // e.g., "password: null", "secret = none" compile_regex( r"(?i)(password|pass|pwd|passwd|secret|cred|key|auth|authorization)[^=:?]{0,8}[=:?][^=:?]{0,8}\b(null|nil|none|password|pass|pwd|passwd|secret|cred|key|auth|authorization).{1,6}$", ), + + // The classic xkcd "hunter2" fake password compile_regex( r"(?i)(password|pass|pwd|passwd|secret|cred|key|auth|authorization)[^=:?]{0,8}[=:?][^=:?]{0,8}hunter2", ), + + // Obvious placeholder sequences + // (Consider grouping like (?i)(?:123456789|abcdefghij) for clarity.) compile_regex(r"(?i)123456789|abcdefghij"), + + // Literal placeholder tag often used in docs/config compile_regex(r"(?i)"), + + // OpenAPI schema references in assignment/query contexts — not secrets + // e.g., "password?ref=#/components/schemas/Credential" compile_regex(r"(?i)[=:?][^=:?]{0,8}#/components/schemas/"), + + // Example MongoDB URIs with placeholder user/pass like "user:pass" or "foo:bar" compile_regex( r"(?i)\b(mongodb(?:\+srv)?://(?:user|foo)[^:@]+:(?:pass|bar)[^@]+@[-\w.%+/:]{3,64}(?:/\w+)?)", ), + + // "classpath://" URIs — configuration references, not secrets compile_regex(r"(?i)\b(classpath://)"), + + // Assignment where the value dereferences a placeholder/property like ${env_var} + // e.g., "password=${db_password}" compile_regex(r"(?i)(\b[^\s\t]{0,16}[=:][^$]*\$\{[a-z_-]{5,30}\})"), + + // URLs with basic auth to hosts ending in "example" or "test" — placeholders + // e.g., "https://user:pass@example" compile_regex(r"(?i)\b((?:https?:)?//[^:@]{3,50}:[^:@]{3,50}@[\w.]{0,16}(?:example|test))"), + + // Assignment ending with "SECRETMANAGER" — explicit placeholder compile_regex(r"(?i)[:=][^:=]{0,32}\bSECRETMANAGER"), ] }); + fn compile_regex(pattern: &str) -> Option { match Regex::new(pattern) { Ok(regex) => Some(regex), @@ -50,6 +100,8 @@ fn compile_regex(pattern: &str) -> Option { } } } + +/// Returns true if the input likely contains *benign* placeholder/test strings. pub fn is_safe_match(input: &[u8]) -> bool { SAFE_LIST_FILTER_REGEX .iter() diff --git a/src/scanner/validation.rs b/src/scanner/validation.rs index 1ba02e1..05865b2 100644 --- a/src/scanner/validation.rs +++ b/src/scanner/validation.rs @@ -368,15 +368,6 @@ async fn validate_single( fail_count: &AtomicUsize, cache2: &Arc>, ) { - // Bypass validation if the rule is prevalidated (eg a Private Key) - if om.rule.syntax().prevalidated { - om.validation_success = true; - om.validation_response_status = http::StatusCode::OK; - om.validation_response_body = "Prevalidated".to_string(); - success_count.fetch_add(1, Ordering::Relaxed); - return; - } - // Build key let dep_vars_str = dep_vars .get(om.rule.id()) diff --git a/tests/int_rules_no_validated_findings.rs b/tests/int_rules_no_validated_findings.rs index 01a6ad9..e69e9bf 100644 --- a/tests/int_rules_no_validated_findings.rs +++ b/tests/int_rules_no_validated_findings.rs @@ -45,23 +45,18 @@ fn scan_rules_has_no_validated_findings() -> Result<()> { for finding in findings { let rule_id = finding["rule"]["id"].as_str().unwrap_or("unknown"); - let rule_prevalidated = finding["rule"]["prevalidated"].as_bool().unwrap_or(false); - let status = - finding["finding"]["validation"]["status"].as_str().unwrap_or("").to_ascii_lowercase(); - - let response = finding["finding"]["validation"]["response"] + let status = finding["finding"]["validation"]["status"] .as_str() .unwrap_or("") .to_ascii_lowercase(); - // Skip anything intentionally marked as prevalidated - if rule_prevalidated || status == "prevalidated" || response == "prevalidated" { - continue; - } - // Fail only on genuinely validated secrets - assert_ne!(&status, "active credential", "Validated finding detected in rule {rule_id}"); + assert_ne!( + &status, + "active credential", + "Validated finding detected in rule {rule_id}" + ); } Ok(()) diff --git a/tests/smoke_docker.rs b/tests/smoke_docker.rs index 40cc420..3caef75 100644 --- a/tests/smoke_docker.rs +++ b/tests/smoke_docker.rs @@ -14,7 +14,7 @@ fn smoke_scan_docker_image() -> anyhow::Result<()> { "--no-update-check", ]) .assert() - .code(205) - .stdout(predicate::str::contains("Active Credential")); + .code(200) + .stdout(predicate::str::contains("Not Attempted")); Ok(()) }