eblume/kingfisher
1
0
Fork 0
forked from mirrors/kingfisher
Code Pull requests Activity Actions

- Updating to support Bitbucket App Passwords

Browse source 
- Improved boundaries for several rules
- Added more rules
This commit is contained in:
Mick Grove 2025-11-20 16:33:28 -08:00
commit 17e0ca3594
11 changed files with 245 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

5
CHANGELOG.md
View file

@ -2,6 +2,11 @@
All notable changes to this project will be documented in this file.
## [v1.66.0]
- Updating to support Bitbucket App Passwords
- Improved boundaries for several rules
- Added more rules
## [v1.65.0]
- Skip reporting MongoDB and Postgres findings when their connection strings cannot be parsed, even when validation is disabled.
- Improve MySQL detection by broadening URI coverage and adding live validation that skips clearly invalid connection strings.

2
Cargo.toml
View file

@ -10,7 +10,7 @@ publish = false
[package]
name = "kingfisher"
version = "1.65.0"
version = "1.66.0"
description = "MongoDB's blazingly fast and accurate secret scanning and validation tool"
edition.workspace = true
rust-version.workspace = true

6
data/rules/anthropic.yml
View file

@ -2,16 +2,14 @@ rules:
- name: Anthropic API Key
id: kingfisher.anthropic.1
pattern: |
(?xi)
\b
(
(?xi)
(
sk-ant-api
\d{2,4}
-
[\w\-]{93}
AA
)
\b
pattern_requirements:
min_digits: 2
min_uppercase: 1

34
data/rules/eraserio.yml Normal file
View file

@ -0,0 +1,34 @@
rules:
- name: Eraser API Key
id: kingfisher.eraser.1
pattern: |
(?xi)
\b
eraser
(?:[^A-Za-z0-9]{0,16})?
(?:SECRET|PRIVATE|ACCESS|KEY|TOKEN)
(?:[^A-Za-z0-9]{0,16})?
\b
(
[A-Za-z0-9]{20}
)
\b
min_entropy: 3.5
confidence: medium
examples:
- eraser_token = Q7MD4J9L2X0B6R3T8W1P
references:
- https://eraser.io/docs/api/authentication
validation:
type: Http
content:
request:
method: GET
url: https://app.eraser.io/api/reports/usage?rangeDays=1
headers:
Authorization: "Bearer {{ TOKEN }}"
accept: application/json
response_matcher:
- report_response: true
- type: StatusMatch
status: [200, 403]

10
data/rules/github.yml
View file

@ -3,7 +3,7 @@ rules:
id: kingfisher.github.1
pattern: |
(?xi)
(
(
github_pat_
[A-Z0-9_+]{82,84}
)
@ -39,8 +39,7 @@ rules:
- name: GitHub Personal Access Token
id: kingfisher.github.2
pattern: |
(?xi)
\b
(?xi)
(
ghp_(?P<body>[A-Z0-9]{30})(?P<checksum>[A-Z0-9]{6})
)
@ -85,9 +84,8 @@ rules:
- name: GitHub OAuth Access Token
id: kingfisher.github.3
pattern: |
(?xi)
\b
(
(?xi)
(
gho_(?P<body>[A-Z0-9]{30})(?P<checksum>[A-Z0-9]{6})
)
pattern_requirements:

37
data/rules/monday.yml Normal file
View file

@ -0,0 +1,37 @@
rules:
- name: Monday.com API Key
id: kingfisher.monday.1
pattern: |
(?xi)
\b
monday
(?:.|[\n\r]){0,40}?
(?:SECRET|PRIVATE|ACCESS|KEY|TOKEN)
(?:.|[\n\r]){0,40}?
\b
(
eyJ[A-Za-z0-9-_]{10,200}\.eyJ[A-Za-z0-9-_]{50,1000}\.[A-Za-z0-9-_]{20,500}
)
\b
min_entropy: 3.3
confidence: medium
examples:
- monday SECRET_TOKEN=eyJhbGciOiJIUzI1TiJ9.eyJ0aWQiOjU7OTC4MzIwMywiYWFpIjoxMSwidWlkIjo5NjYwMzk5MCwiaWBkIjoiMjAyNS0xMS0yMVQwMDoyNjoxMy43OCVaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MzI2MDI5MTIsInJnbiI6InVzZTEifQ.wQtV6psL1JqFHdXgRB2J7-qslSyS2I4TYJHtkX9ofvk
validation:
type: Http
content:
request:
url: https://api.monday.com/v2
method: POST
headers:
Content-Type: application/json
Authorization: '{{ TOKEN }}'
body: |
{"query": "query { me { id name } }"}
response_matcher:
- report_response: true
- type: StatusMatch
status: [200]
- type: WordMatch
words: ["data", "me", "id"]
match_all_words: true

1
data/rules/nvidia.yml
View file

@ -3,7 +3,6 @@ rules:
id: kingfisher.nvidia.nim.1
pattern: |
(?xi)
\b
(
nvapi-[A-Z0-9_-]{60,70}
)

3
data/rules/openai.yml
View file

@ -3,11 +3,9 @@ rules:
id: kingfisher.openai.1
pattern: |
(?xi)
\b
(
sk-[A-Z0-9]{48}
)
\b
pattern_requirements:
min_digits: 2
min_entropy: 3.3
@ -35,7 +33,6 @@ rules:
id: kingfisher.openai.2
pattern: |
(?xi)
\b
(
(sk-(?:proj|svcacct|None)-[A-Z0-9_-]{100,})
)

6
data/rules/sentry.yml
View file

@ -3,14 +3,13 @@ rules:
id: kingfisher.sentry.1
pattern: |
(?xi)
\b
sentry
(?:.|[\n\r]){0,32}?
(?:SECRET|PRIVATE|ACCESS|KEY|TOKEN)
(?:.|[\n\r]){0,32}?
\b
(
[a-f0-9]{64}
[a-f0-9]{64}
)
\b
pattern_requirements:
@ -44,7 +43,6 @@ rules:
(
sntrys_eyJpYXQiO[a-zA-Z0-9+/]{10,200}(?:LCJyZWdpb25fdXJs|InJlZ2lvbl91cmwi|cmVnaW9uX3VybCI6)[a-zA-Z0-9+/]{10,200}={0,2}_[a-zA-Z0-9+/]{43}
)
\b
pattern_requirements:
min_digits: 2
min_entropy: 4.2
@ -73,11 +71,9 @@ rules:
id: kingfisher.sentry.3
pattern: |
(?xi)
\b
(
sntryu_[a-f0-9]{64}
)
\b
pattern_requirements:
min_digits: 2
min_entropy: 3.5

4
data/rules/supabase.yml
View file

@ -3,11 +3,9 @@ rules:
id: kingfisher.supabase.1
pattern: |
(?xi)
\b
(
sbp_[a-z0-9_-]{40}
)
\b
pattern_requirements:
min_digits: 2
min_entropy: 3.5
@ -34,11 +32,9 @@ rules:
id: kingfisher.supabase.2
pattern: |
(?xi)
\b
(
sb_secret_[a-z0-9_-]{31}
)
\b
pattern_requirements:
min_digits: 2
min_entropy: 4.0

174
src/git_binary.rs
View file

@ -4,6 +4,7 @@ use std::{
};
use tracing::{debug, debug_span};
use url::Url;
use crate::{bitbucket::is_bitbucket_access_token, git_url::GitUrl};
@ -101,6 +102,8 @@ pub struct Git {
credentials: Vec<String>,
ignore_certs: bool,
bitbucket_access_token: Option<String>,
bitbucket_env: Vec<(String, String)>,
bitbucket_basic_auth: Option<(String, String)>,
}
impl Git {
@ -110,23 +113,60 @@ impl Git {
pub fn new(ignore_certs: bool) -> Self {
let mut credentials = Vec::new();
fn normalized_env_var(name: &str) -> Option<String> {
std::env::var(name)
.ok()
.map(|value| value.trim().to_owned())
.filter(|value| !value.is_empty())
}
let bitbucket_username = normalized_env_var("KF_BITBUCKET_USERNAME");
let bitbucket_app_password = normalized_env_var("KF_BITBUCKET_APP_PASSWORD");
let bitbucket_token = normalized_env_var("KF_BITBUCKET_TOKEN");
let bitbucket_password = normalized_env_var("KF_BITBUCKET_PASSWORD");
let bitbucket_oauth_token = normalized_env_var("KF_BITBUCKET_OAUTH_TOKEN");
let mut bitbucket_env = Vec::new();
for (key, value) in [
("KF_BITBUCKET_USERNAME", bitbucket_username.as_ref()),
("KF_BITBUCKET_APP_PASSWORD", bitbucket_app_password.as_ref()),
("KF_BITBUCKET_TOKEN", bitbucket_token.as_ref()),
("KF_BITBUCKET_PASSWORD", bitbucket_password.as_ref()),
("KF_BITBUCKET_OAUTH_TOKEN", bitbucket_oauth_token.as_ref()),
] {
if let Some(value) = value {
bitbucket_env.push((key.to_string(), value.to_string()));
}
}
let has_github_token =
matches!(std::env::var("KF_GITHUB_TOKEN"), Ok(token) if !token.is_empty());
let has_gitlab_token =
matches!(std::env::var("KF_GITLAB_TOKEN"), Ok(token) if !token.is_empty());
let has_gitea_token =
matches!(std::env::var("KF_GITEA_TOKEN"), Ok(token) if !token.is_empty());
let has_bitbucket_username =
matches!(std::env::var("KF_BITBUCKET_USERNAME"), Ok(value) if !value.is_empty());
let bitbucket_access_token = std::env::var("KF_BITBUCKET_TOKEN")
.ok()
.filter(|value| !value.is_empty() && is_bitbucket_access_token(value));
let has_bitbucket_password =
["KF_BITBUCKET_APP_PASSWORD", "KF_BITBUCKET_TOKEN", "KF_BITBUCKET_PASSWORD"]
.iter()
.any(|key| matches!(std::env::var(key), Ok(value) if !value.is_empty()));
let has_bitbucket_oauth_token =
matches!(std::env::var("KF_BITBUCKET_OAUTH_TOKEN"), Ok(value) if !value.is_empty());
let bitbucket_access_token =
bitbucket_token.as_ref().filter(|token| is_bitbucket_access_token(token)).cloned();
let bitbucket_basic_password = bitbucket_app_password
.clone()
.or(bitbucket_token.clone())
.or(bitbucket_password.clone());
let bitbucket_basic_auth = if let Some(token) = bitbucket_oauth_token.clone() {
Some(("x-token-auth".to_string(), token))
} else if let Some(token) = bitbucket_access_token.clone() {
Some(("x-token-auth".to_string(), token))
} else if let (Some(username), Some(password)) =
(bitbucket_username.clone(), bitbucket_basic_password)
{
Some((username, password))
} else {
None
};
let has_bitbucket_username = bitbucket_username.is_some();
let has_bitbucket_password = bitbucket_app_password.is_some()
|| bitbucket_token.is_some()
|| bitbucket_password.is_some();
let has_bitbucket_oauth_token = bitbucket_oauth_token.is_some();
let has_bitbucket_credentials = has_bitbucket_oauth_token
|| bitbucket_access_token.is_some()
|| (has_bitbucket_username && has_bitbucket_password);
@ -186,7 +226,13 @@ impl Git {
credentials.push(HUGGINGFACE_CREDENTIAL_HELPER.into());
}
Self { credentials, ignore_certs, bitbucket_access_token }
Self {
credentials,
ignore_certs,
bitbucket_access_token,
bitbucket_env,
bitbucket_basic_auth,
}
}
/// Create a basic `git` `Command` with environment variables set to
@ -201,6 +247,9 @@ impl Git {
if self.ignore_certs {
cmd.env("GIT_SSL_NO_VERIFY", "1");
}
for (key, value) in &self.bitbucket_env {
cmd.env(key, value);
}
if let Some(token) = &self.bitbucket_access_token {
cmd.env("KF_BITBUCKET_ACCESS_TOKEN", token);
}
@ -268,11 +317,31 @@ impl Git {
cmd.arg("--quiet");
cmd.arg("-c");
cmd.arg("remote.origin.fetch=+refs/*:refs/remotes/origin/*");
cmd.arg(repo_url.as_str());
cmd.arg(self.repo_arg_for_clone(repo_url));
cmd.arg(output_dir);
debug!("{cmd:#?}");
self.run_cmd(cmd)
}
fn repo_arg_for_clone(&self, repo_url: &GitUrl) -> String {
if let Some((username, password)) = &self.bitbucket_basic_auth {
if let Ok(mut url) = Url::parse(repo_url.as_str()) {
if url
.host_str()
.map(|host| host.eq_ignore_ascii_case("bitbucket.org"))
.unwrap_or(false)
{
if url.set_username(username).is_ok()
&& url.set_password(Some(password)).is_ok()
{
return url.into();
}
}
}
}
repo_url.as_str().to_string()
}
}
impl Default for Git {
@ -349,6 +418,61 @@ mod tests {
);
}
#[test]
fn test_repo_arg_for_clone_includes_bitbucket_app_password() {
let url =
GitUrl::try_from(url::Url::parse("https://bitbucket.org/workspace/demo.git").unwrap())
.unwrap();
temp_env::with_vars(
&[
("KF_BITBUCKET_USERNAME", Some("user")),
("KF_BITBUCKET_APP_PASSWORD", Some("secret")),
],
|| {
let git = Git::new(false);
assert_eq!(
git.repo_arg_for_clone(&url),
"https://user:secret@bitbucket.org/workspace/demo.git"
);
},
);
}
#[test]
fn test_repo_arg_for_clone_uses_token_auth_when_available() {
let url =
GitUrl::try_from(url::Url::parse("https://bitbucket.org/workspace/demo.git").unwrap())
.unwrap();
temp_env::with_vars(&[("KF_BITBUCKET_OAUTH_TOKEN", Some("token123"))], || {
let git = Git::new(false);
assert_eq!(
git.repo_arg_for_clone(&url),
"https://x-token-auth:token123@bitbucket.org/workspace/demo.git"
);
});
}
#[test]
fn test_repo_arg_for_clone_leaves_non_bitbucket_urls_untouched() {
let url = GitUrl::try_from(
url::Url::parse("https://github.com/octocat/Hello-World.git").unwrap(),
)
.unwrap();
temp_env::with_vars(
&[
("KF_BITBUCKET_USERNAME", Some("user")),
("KF_BITBUCKET_APP_PASSWORD", Some("secret")),
],
|| {
let git = Git::new(false);
assert_eq!(git.repo_arg_for_clone(&url), url.as_str());
},
);
}
#[test]
fn test_git_new_bitbucket_access_token() {
let token = "AT1234567890_ACCESS_TOKEN_EXAMPLE_WITH_UNDERSCORE";
@ -360,6 +484,30 @@ mod tests {
});
}
#[test]
fn test_git_new_bitbucket_trims_whitespace() {
let trimmed_token = "AT1234567890_ACCESS_TOKEN_EXAMPLE_WITH_UNDERSCORE";
let token = format!(" {trimmed_token} \n");
temp_env::with_vars(
&[("KF_BITBUCKET_USERNAME", Some(" user\n")), ("KF_BITBUCKET_TOKEN", Some(&token))],
|| {
let git = Git::new(false);
assert_eq!(
git.bitbucket_env,
vec![
("KF_BITBUCKET_USERNAME".to_string(), "user".to_string()),
("KF_BITBUCKET_TOKEN".to_string(), trimmed_token.to_string(),),
],
);
assert_eq!(git.credentials.len(), 4);
assert!(git.credentials.iter().any(|value| value == BITBUCKET_CREDENTIAL_HELPER));
assert_eq!(git.bitbucket_access_token.as_deref(), Some(trimmed_token));
},
);
}
#[test]
fn test_clone_mode_arg() {
assert_eq!(CloneMode::Bare.arg(), Some("--bare"));