Merge pull request #87 from mongodb/development

v1.40.0
This commit is contained in:
Mick Grove 2025-08-13 10:19:36 -07:00 committed by GitHub
commit 3c090ec980
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 69 additions and 37 deletions

View file

@ -2,6 +2,10 @@
All notable changes to this project will be documented in this file.
## [1.40.0]
- Dropped the “prevalidated” flag from rule definitions and validation logic so every finding now flows through the standard active/inactive/unknown pipeline, simplifying rule configuration and preventing specialcase bypasses
- Improved Tailscale api key detectors
## [1.39.0]
- Added support for scanning Confluence pages via `--confluence-url` and `--cql`

View file

@ -10,7 +10,7 @@ publish = false
[package]
name = "kingfisher"
version = "1.39.0"
version = "1.40.0"
description = "MongoDB's blazingly fast secret scanning and validation tool"
edition.workspace = true
rust-version.workspace = true

View file

@ -127,7 +127,7 @@ rules:
'policy_path': os.path.join(TEST_DIR, 'policy.json')
})
- name: Weak Password Pattern
id: kingfisher.weak_password.1
id: kingfisher.generic.6
pattern: |
(?xi)
\b

View file

@ -10,7 +10,6 @@ rules:
-----END\ .{0,20}\ ?PRIVATE\ KEY\ ?.{0,20}-----
min_entropy: 4.5
confidence: high
prevalidated: true
examples:
- |
-----BEGIN RSA PRIVATE KEY-----
@ -62,7 +61,6 @@ rules:
(?: [^a-zA-Z0-9+/=] | $ )
min_entropy: 4.5
confidence: high
prevalidated: true
examples:
- 'PRIVATE_KEY_B64=LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBb3kxWFh1VkFRcHFIYlFFMDVta2hyTmcvMTI0Ri8ySzlPYW5pelpUWlVVaEswOFU4CkxhaC9SbVVsWHFRMDEvU255aktGOWZqUDhFcU1OZ1dpamUzYmVwL3RPOVpTMEFUMi9PVlJXeS9TOG52RDQ5WTMKenMxMktSbERhR2lZc0RsYUZrbHJkeDQ4RWhRVmdHN3hmWE1jaC9OejJzc2FEby9kRkNBOW80TkZZQWUzM2UveApWNVo1UHNkWkl6dkNZQVlCNDRoUEtpN3JXRE1IbFdzM1kvVkVtQXMzSzVNK2QvL3QzRHB4WnBEbWJERGdYa2w2CjZUdDh3VXloUVZ3MkZpMStobTF1T2QwYjFkaW9aNko2OXNTT2JOZXpSR3YxYjdZaFltT0JKL1JBbHN5ZHoxTmgKVXpXT1lYV0Z1OGJrOU9JM3lQMEc0TE84QjhtbWRldE1RVVoyelFJREFRQUJBb0lCQUN2ckhUUHVVZ0JiSlE0QwpvQ0ZQdEgrWDZIN3NIdk1ndVR0VzdUTlYxN1BYMkVQdE53ZzI3S0tld0pNYmNSbWF3THBjSk5BU09xMDY4MGZxCjlsaHE1NEsybnB4WFVBeXErV3NSc1hid2hUODhibm5aQTBaRzZJR2hTaEpFN0t1cGxBU2htQ29FV2ppbmJTNFgKTGlvTW5HWSs4VFMzSzNrMTRWUDBaWUtuNXprMERHZnFBMEo0VTRXSmxUeGwrTWZxd0pJOTlrcTdHbFVlZkdncQpuK3Q1d2NrV3BPbTd5TUJjZTlTSXlmTm54bnU3TkZYQm50VTN5RGxSUThWUWZmNEtRMzJCaWNiYlJWemR1TThNCnNxMU5CZWNzL0EzUXRvdG1nWUc4d094ZXpNS3Iyays2QzB2NmlFc0h5T0lmR25GWktSZDJFd0dnWlo3aytURHUKUUYrcjd1VUNnWUVBMkRqNUJoYmpybDFRNTZya3BhTGFvVldRV1Y5YUYzUUJtNlNZM2VQYmlvY2JNR2k1ak1ESQpkSjdJVXlLYUljK3BNV1RQYlBmVUd2WmNENlczZDFBNUNUSnFuWHVuVlY3czRqaWJ6WDZUbjhNM3IrMHZTZnNZCmdPMHBtRFpndlNqaVZTRUNBQTZFOFUxQ1lFZU5KUDFDOW12cGJVNzJRTEpndWp3M3JMb2oyYmNDZ1lFQXdUSXYKOUNSeWNOQXRBbDcvUHdWZGh5eXRvVHBSRnZDSU1HSVk5SjMxZ3lva0ZlaFQvWjQ4WkF6anl6ZTBSUXYzdGUxTQoveVJMQkVETGkwbEtrZFVXckVkaVR3dm1KdkpwMDZ0OEdCbERsK25ycXVLWTFxVThDbTR5cis4QzZtRThkVnZrClNINXBhRXptOERFTE1wSjhGVTZFYnhmZHZjRzZmSGx6dnVnZmc1c0NnWUFFQ1BRa3QvS2h3MTRLSkxkRm5BZG0KY1ZsVFFhTkZ3c1Z3NlI1dExaNWdOR3MrZVFYVmFaZVVEWTZCZHFqWHJxOWltNVgvVzVTYXVEUTVtb2NVOCt0TQpqNk5Mc3c0SldzOGkzWm1TdVNUNkcwT0R4ZkpXK0JlWitGTUpZeUpsQlVsTCsyUzFLWkF6akpTTGhXcE40V2dKCmZ6UUk5U3RGUTg3b1NzMWpMTW9VZXdLQmdGOE9CMlFURHErTTdhaE4vejROc0wvU2JyZDJEdkcvZFBLQlFaQVIKcS90V0g1MGJ5ejlzdkgvcGk2YXdDS1UwUnpPZXh4UjkwZDhNMWxqNHZaVFZDQ3ZKajRnZTdhVlovbEdqL1JHSwpWS1NJOW1nRXgzaE1vaWJybzByR3lXTnlaaUhFRGFUUmRhRll2UU9PemRpYkZDd1RqcnR1UGE2Z2c5VzhtQU5sCkNDUmpBb0dBSTRIbnpyV3kzaU5kR2xqVnh4bW1DN1V0c0MvajJBUEZpcHc0ZHJ0U2NsMDFRZzF5WkowbDNBTk4KOU5lTmVSUUFzN3pFTng2T1B1SzlxYy83T1ROMTJKaHdoUTIzdXZwNjZjV0krdTRjcVpOZTJyZVFVVWVmM3psbQpMcXRmOU50VHp5M3pjMGZQcGoxQnBlRmxHSG9SVDhjVHpBWjFTeGwyZWChazlqS2RVeDQ9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0t'
- ' "privateKey": "LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBbUhKOEJHdTFYZUZ4aENVQXBrNHNSTVI4RnRTdGtyMEx0OWtWTGNSUjRFWitiOWhHCmR0blJpOFhqV3d5MU5zMHliMkJMdHBpVHZKSFVKTUphWXluZ2ZkZnZhcWhocm1yYm5vV0pLQkxmeUxwTXFNS1EKQ3RialFxbnVrQURJUWVQd2ZGeTNpVHkxd1JkRC9zTUs1U0VtV0Fxb0pZQk50eTFZZzA2UzVkYVlPM2xjY3hrYQpQWjRjcm9McWF6Ny9tU3dDVTR5VWRSb3h4WVF4VG1MZXg5M2tqU09TTmdpK0FXc0lCbjV3UHI0VHNuVHFSeWpIClN2aEdMdk9YREpRYWZRdk56WjFSL1FYMzlOQk9xOEVKZW5pWXdaUm9uNVcvNVhMYW94MFFyUGhrY1BES3A5SVUKeHpJakUwWlNmMStUK1FFbTQ3TkFtSnhvZjFhdGRFVzZDTCtheHdJREFRQUJBb0lCQUQ3enI4REhsWnFSK1NWZgpmbGd1bWRzLzVCb3Rjd3ZRWXlGbFZIaVV4RmEvNVlCY0tDVDJKN0QzWTc1NmplNTJaK2hVTkkvUGk5cG53ZG40CkpBa2xCdDRRcUg0NzBES05UK216TFFOT1gvanM3YkVXdnhLcTBDZjhNbFptN0V0QlRGS2VtdS9pRVJBT2duYVcKcGs0ZUZVNXdBQ1dVU1FObWgxR1p4ZEdCZjFXM1VjUnQxcFRvOEtQTDluZm4vSGJiRFNsQkNVL3VIcWd2TSt2cApmTE03bzRIVDZ1K1ZzU00rWGZqeDhpeE5ZRHdoalNuKzQyZm13d1d3ZzJISHUrdUozZ1pUSWQwRUI1VW9hdUNjCjZUTlVtcEJscjU5UGFmVkZRWUY1S3VxaHJXKzVQaWpHcHBZcXg4Ynl6aFpOQzkwZnl5V0NXcXg2eGFZVm5OdzgKNkJmUXM2a0NnWUVBeVlyRVg1NU1RTzJnWDY2TGwxaGJDMzNzWk1OZzloVG1SK1doSTFjNksvbFZ1TFoyL0RPdwpsYTZ6eHdBU204Z0ZyVUFYbUljV2h2b3FwWGVzNWZzOVZKeDlNT0ZVYVBrckRPQllnY1laMUR6VVNVOHc3SSttCnlyV3hRUkRNajhvSGpRbHVpM0s2MzZucm5RajhxOGkvQ2dranVPcHJGZnliMzVEMFlDdjVXZzBDZ1lFQXdhT3cKRWFhN0l1MjFGa08vbmFjdVhjSnBhNkVlUTNqZFNlNlRQaXZ6bVVXU0haeGJuUy9XSnJaRjQwSExzUWxOZHl0ZgpNTTBKZFU0VmMyR0NVc1pMYjdQSmJwdVRqRERSSHJXV1pCMnhiemF0K3A3N2RzNWlOcXFRcTZ6M0syUVh4Y3ZTCis5am5VZXpDU2Y0N1R1OWNTTW96V3hTMW82b1BPSFdHVFRvdHR5TUNnWUFQdWc1Y3o4TnZoWnR3Ry9TMG1LWnkKSFI5bk5YL0pkQlFNSkRVUXh1dTVKcm16c2psU3NNM2t3RDh6RmlSZGw1d3B5c2lNbEc0RGxsM2hqNWNrVXhpVQpFNm9KT0d3WHpPbTVGWUNTajl6UUhQY0x5V3d0NlgvQWJiRXBQS0JaMEJBS3gyT2k2ZzcvQ1FsanRhSFIzZFphCmVDQWJlOTlqVmRUcit5bTJuM2ZUdVFLQmdBMm5TZ25rbEx0Z3dXMEJkK2hZMm1jWUJ6RGttbXF0Z2dUdGdvcFcKdFFWd3AxM1pJWWlTeituSTNtS295QUVDbytpc01Ua1NyQUVPY1dyQ1RGc2p5anZsRkdYdEtGa3hNLzJUVmpoVwo4NlRnMlNHYnhpVlpaZ2x1dTJhdmVub2Z3NkZadnRXdE5KcE5OR0hkUURkUG4xVXVsTEp1WW1SWTRGdmR4WXQ2CmQ3QzdBb0dBRUsvalFiZ0l3OXFLQUNOZ0JySnB1cU5Ham9JajFoQTRlb29DMXp1bFEyZUpnZ2J5OTBpSDg2VzEKM0xyOVZMVFkyc2JKTzlqekZVR0lOL01BOEhYQTE1a2grZHRibkRsdFRFZGNnenBCRzhCQUZRQ3hQWnBGWHhtZgpDUmhXN1l6RW1IeWJ4R0toR3NOK2M3NUhKTHZFSWwrRTh6eitXRk9xT240dkJXU1ZwSnc9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==",'

View file

@ -24,7 +24,6 @@ rules:
-----
min_entropy: 4.5
confidence: high
prevalidated: true
examples:
- |-
-----BEGIN RSA PRIVATE KEY-----
@ -77,7 +76,6 @@ rules:
)
min_entropy: 4.5
confidence: high
prevalidated: true
examples:
- |
-----BEGIN PRIVATE KEY-----

View file

@ -5,14 +5,14 @@ rules:
(?xi)
\b
(
tskey-[a-z]+-[A-Z0-9_-]{20,24}
tskey-[a-z]{3,10}-[A-Z0-9_-]{20,36}
)
\b
min_entropy: 3.0
confidence: medium
examples:
- tskey-secret-12345678-abcdefghijkl
- tskey-api-abcdefg-1234567890123
- tskey-secret-weRTWSfoeFKI-3480754342kDSFelW3
- tskey-api-weRTWSfoeFKI-3480754342kDSFelW3
references:
- https://tailscale.com/kb/1215/oauth-clients
validation:

View file

@ -930,7 +930,6 @@ mod test {
visible: true,
examples: vec![],
negative_examples: vec![],
prevalidated: false,
references: vec![],
validation: None::<Validation>, // no HTTP validation needed
depends_on_rule: vec![],
@ -972,7 +971,6 @@ mod test {
visible: true,
examples: vec![],
negative_examples: vec![],
prevalidated: false,
references: vec![],
validation: Some(Validation::Http(HttpValidation {
request: HttpRequest {
@ -1089,7 +1087,6 @@ mod test {
visible: true,
examples: vec![],
negative_examples: vec![],
prevalidated: false,
references: vec![],
validation: None::<Validation>,
depends_on_rule: vec![],

View file

@ -282,9 +282,6 @@ pub struct RuleSyntax {
/// Optional dependencies on other rules.
#[serde(default)]
pub depends_on_rule: Vec<Option<DependsOnRule>>,
/// Whether matches should always be considered validated.
#[serde(default)]
pub prevalidated: bool,
}
lazy_static! {

View file

@ -1,46 +1,96 @@
use once_cell::sync::Lazy;
use regex::bytes::Regex;
use tracing::debug;
/// Case-insensitive patterns that indicate a *benign* match (placeholders, examples, redactions, etc.).
/// `is_safe_match()` returns true if any of these are present.
static SAFE_LIST_FILTER_REGEX: Lazy<Vec<Option<Regex>>> = Lazy::new(|| {
vec![
// Assignment-like value that ends with "EXAMPLEKEY" (common placeholder)
// e.g., "KEY=ABC_EXAMPLEKEY" or "key: fooEXAMPLEKEY"
compile_regex(r"(?i)[:=][^:=]{0,64}EXAMPLEKEY"),
// AWS-style AKIA keys explicitly marked as example/fake/test/sample
// e.g., "AKIA...EXAMPLE", "AKIA...FAKE", "AKIA...SAMPLE"
compile_regex(r"(?i)\b(AKIA(?:.*?EXAMPLE|.*?FAKE|TEST|.*?SAMPLE))\b"),
// Secret-y key name followed by short value and then "&&" / "||" or a run of asterisks
// e.g., "password=foo &&", "secret: *****" (redacted/masked)
compile_regex(
r"(?i)(password|pass|pwd|passwd|secret|cred|key|auth|authorization)[^=:?]{0,8}[=:?][^=:?]{0,8}\s(&&|\|\||\*{5,50})",
),
// Secret-y key name with short value, then *another* short assignment on the same line
// Typical of docs/examples rather than hardcoded secrets
compile_regex(
r"(?i)(password|pass|pwd|passwd|secret|cred|key|auth|authorization)[^=:?]{0,8}[=:?][^=:?]{0,8}\b\w{4,12}\s{0,6}=\s{0,6}\D{0,3}\w{1,12}",
),
// Secret-y key assigned to a shell variable reference (e.g., "$FOO") — not a literal secret
compile_regex(
r"(?i)(password|pass|pwd|passwd|secret|cred|key|auth|authorization)[^=:?]{0,8}[=:?][^=:?]{0,8}\$\w{4,30}",
),
// Secret-y key set via command that *generates* randomness, not a literal value
// e.g., "password = openssl rand -base64 32"
compile_regex(
r"(?i)(password|pass|pwd|passwd|secret|cred|key|auth|authorization)[^=:?]{0,16}[=:?][^=:?]{0,8}\bopenssl\s{0,4}rand\b",
),
// Secret-y key assigned a value containing "encrypted" (marker/metadata, not a secret)
compile_regex(
r"(?i)(password|pass|pwd|passwd|secret|cred|key|auth|authorization)[^=:?]{0,8}[=:?][^=:?]{0,8}encrypted",
),
// Secret-y key assigned boolean literals — not secrets
// e.g., "auth=false"
compile_regex(
r"(?i)(password|pass|pwd|passwd|secret|cred|key|auth|authorization)[^=:?]{0,8}[=:?][^=:?]{0,8}\b(?:false|true)\b",
),
// Secret-y key assigned to null-ish or self-referential placeholders — not secrets
// e.g., "password: null", "secret = none"
compile_regex(
r"(?i)(password|pass|pwd|passwd|secret|cred|key|auth|authorization)[^=:?]{0,8}[=:?][^=:?]{0,8}\b(null|nil|none|password|pass|pwd|passwd|secret|cred|key|auth|authorization).{1,6}$",
),
// The classic xkcd "hunter2" fake password
compile_regex(
r"(?i)(password|pass|pwd|passwd|secret|cred|key|auth|authorization)[^=:?]{0,8}[=:?][^=:?]{0,8}hunter2",
),
// Obvious placeholder sequences
// (Consider grouping like (?i)(?:123456789|abcdefghij) for clarity.)
compile_regex(r"(?i)123456789|abcdefghij"),
// Literal placeholder tag often used in docs/config
compile_regex(r"(?i)<secretmanager>"),
// OpenAPI schema references in assignment/query contexts — not secrets
// e.g., "password?ref=#/components/schemas/Credential"
compile_regex(r"(?i)[=:?][^=:?]{0,8}#/components/schemas/"),
// Example MongoDB URIs with placeholder user/pass like "user:pass" or "foo:bar"
compile_regex(
r"(?i)\b(mongodb(?:\+srv)?://(?:user|foo)[^:@]+:(?:pass|bar)[^@]+@[-\w.%+/:]{3,64}(?:/\w+)?)",
),
// "classpath://" URIs — configuration references, not secrets
compile_regex(r"(?i)\b(classpath://)"),
// Assignment where the value dereferences a placeholder/property like ${env_var}
// e.g., "password=${db_password}"
compile_regex(r"(?i)(\b[^\s\t]{0,16}[=:][^$]*\$\{[a-z_-]{5,30}\})"),
// URLs with basic auth to hosts ending in "example" or "test" — placeholders
// e.g., "https://user:pass@example"
compile_regex(r"(?i)\b((?:https?:)?//[^:@]{3,50}:[^:@]{3,50}@[\w.]{0,16}(?:example|test))"),
// Assignment ending with "SECRETMANAGER" — explicit placeholder
compile_regex(r"(?i)[:=][^:=]{0,32}\bSECRETMANAGER"),
]
});
fn compile_regex(pattern: &str) -> Option<Regex> {
match Regex::new(pattern) {
Ok(regex) => Some(regex),
@ -50,6 +100,8 @@ fn compile_regex(pattern: &str) -> Option<Regex> {
}
}
}
/// Returns true if the input likely contains *benign* placeholder/test strings.
pub fn is_safe_match(input: &[u8]) -> bool {
SAFE_LIST_FILTER_REGEX
.iter()

View file

@ -368,15 +368,6 @@ async fn validate_single(
fail_count: &AtomicUsize,
cache2: &Arc<SkipMap<String, CachedResponse>>,
) {
// Bypass validation if the rule is prevalidated (eg a Private Key)
if om.rule.syntax().prevalidated {
om.validation_success = true;
om.validation_response_status = http::StatusCode::OK;
om.validation_response_body = "Prevalidated".to_string();
success_count.fetch_add(1, Ordering::Relaxed);
return;
}
// Build key
let dep_vars_str = dep_vars
.get(om.rule.id())

View file

@ -45,23 +45,18 @@ fn scan_rules_has_no_validated_findings() -> Result<()> {
for finding in findings {
let rule_id = finding["rule"]["id"].as_str().unwrap_or("unknown");
let rule_prevalidated = finding["rule"]["prevalidated"].as_bool().unwrap_or(false);
let status =
finding["finding"]["validation"]["status"].as_str().unwrap_or("").to_ascii_lowercase();
let response = finding["finding"]["validation"]["response"]
let status = finding["finding"]["validation"]["status"]
.as_str()
.unwrap_or("")
.to_ascii_lowercase();
// Skip anything intentionally marked as prevalidated
if rule_prevalidated || status == "prevalidated" || response == "prevalidated" {
continue;
}
// Fail only on genuinely validated secrets
assert_ne!(&status, "active credential", "Validated finding detected in rule {rule_id}");
assert_ne!(
&status,
"active credential",
"Validated finding detected in rule {rule_id}"
);
}
Ok(())

View file

@ -14,7 +14,7 @@ fn smoke_scan_docker_image() -> anyhow::Result<()> {
"--no-update-check",
])
.assert()
.code(205)
.stdout(predicate::str::contains("Active Credential"));
.code(200)
.stdout(predicate::str::contains("Not Attempted"));
Ok(())
}