diff --git a/CHANGELOG.md b/CHANGELOG.md index fd69ef5..60adaf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. +## [v1.66.0] +- Updating to support Bitbucket App Passwords +- Improved boundaries for several rules +- Added more rules + ## [v1.65.0] - Skip reporting MongoDB and Postgres findings when their connection strings cannot be parsed, even when validation is disabled. - Improve MySQL detection by broadening URI coverage and adding live validation that skips clearly invalid connection strings. diff --git a/Cargo.toml b/Cargo.toml index d6b0a0e..2776297 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ publish = false [package] name = "kingfisher" -version = "1.65.0" +version = "1.66.0" description = "MongoDB's blazingly fast and accurate secret scanning and validation tool" edition.workspace = true rust-version.workspace = true diff --git a/data/rules/anthropic.yml b/data/rules/anthropic.yml index 8907056..860de92 100644 --- a/data/rules/anthropic.yml +++ b/data/rules/anthropic.yml @@ -2,16 +2,14 @@ rules: - name: Anthropic API Key id: kingfisher.anthropic.1 pattern: | - (?xi) - \b - ( + (?xi) + ( sk-ant-api \d{2,4} - [\w\-]{93} AA ) - \b pattern_requirements: min_digits: 2 min_uppercase: 1 diff --git a/data/rules/eraserio.yml b/data/rules/eraserio.yml new file mode 100644 index 0000000..f56612f --- /dev/null +++ b/data/rules/eraserio.yml @@ -0,0 +1,34 @@ +rules: + - name: Eraser API Key + id: kingfisher.eraser.1 + pattern: | + (?xi) + \b + eraser + (?:[^A-Za-z0-9]{0,16})? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:[^A-Za-z0-9]{0,16})? + \b + ( + [A-Za-z0-9]{20} + ) + \b + min_entropy: 3.5 + confidence: medium + examples: + - eraser_token = Q7MD4J9L2X0B6R3T8W1P + references: + - https://eraser.io/docs/api/authentication + validation: + type: Http + content: + request: + method: GET + url: https://app.eraser.io/api/reports/usage?rangeDays=1 + headers: + Authorization: "Bearer {{ TOKEN }}" + accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200, 403] \ No newline at end of file diff --git a/data/rules/github.yml b/data/rules/github.yml index 3aa8d7d..96ebdd8 100644 --- a/data/rules/github.yml +++ b/data/rules/github.yml @@ -3,7 +3,7 @@ rules: id: kingfisher.github.1 pattern: | (?xi) - ( + ( github_pat_ [A-Z0-9_+]{82,84} ) @@ -39,8 +39,7 @@ rules: - name: GitHub Personal Access Token id: kingfisher.github.2 pattern: | - (?xi) - \b + (?xi) ( ghp_(?P[A-Z0-9]{30})(?P[A-Z0-9]{6}) ) @@ -85,9 +84,8 @@ rules: - name: GitHub OAuth Access Token id: kingfisher.github.3 pattern: | - (?xi) - \b - ( + (?xi) + ( gho_(?P[A-Z0-9]{30})(?P[A-Z0-9]{6}) ) pattern_requirements: diff --git a/data/rules/monday.yml b/data/rules/monday.yml new file mode 100644 index 0000000..8459472 --- /dev/null +++ b/data/rules/monday.yml @@ -0,0 +1,37 @@ +rules: + - name: Monday.com API Key + id: kingfisher.monday.1 + pattern: | + (?xi) + \b + monday + (?:.|[\n\r]){0,40}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,40}? + \b + ( + eyJ[A-Za-z0-9-_]{10,200}\.eyJ[A-Za-z0-9-_]{50,1000}\.[A-Za-z0-9-_]{20,500} + ) + \b + min_entropy: 3.3 + confidence: medium + examples: + - monday SECRET_TOKEN=eyJhbGciOiJIUzI1TiJ9.eyJ0aWQiOjU7OTC4MzIwMywiYWFpIjoxMSwidWlkIjo5NjYwMzk5MCwiaWBkIjoiMjAyNS0xMS0yMVQwMDoyNjoxMy43OCVaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MzI2MDI5MTIsInJnbiI6InVzZTEifQ.wQtV6psL1JqFHdXgRB2J7-qslSyS2I4TYJHtkX9ofvk + validation: + type: Http + content: + request: + url: https://api.monday.com/v2 + method: POST + headers: + Content-Type: application/json + Authorization: '{{ TOKEN }}' + body: | + {"query": "query { me { id name } }"} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ["data", "me", "id"] + match_all_words: true \ No newline at end of file diff --git a/data/rules/nvidia.yml b/data/rules/nvidia.yml index 3143a80..1329314 100644 --- a/data/rules/nvidia.yml +++ b/data/rules/nvidia.yml @@ -3,7 +3,6 @@ rules: id: kingfisher.nvidia.nim.1 pattern: | (?xi) - \b ( nvapi-[A-Z0-9_-]{60,70} ) diff --git a/data/rules/openai.yml b/data/rules/openai.yml index b6f90e0..3e8d7d8 100644 --- a/data/rules/openai.yml +++ b/data/rules/openai.yml @@ -3,11 +3,9 @@ rules: id: kingfisher.openai.1 pattern: | (?xi) - \b ( sk-[A-Z0-9]{48} ) - \b pattern_requirements: min_digits: 2 min_entropy: 3.3 @@ -35,7 +33,6 @@ rules: id: kingfisher.openai.2 pattern: | (?xi) - \b ( (sk-(?:proj|svcacct|None)-[A-Z0-9_-]{100,}) ) diff --git a/data/rules/sentry.yml b/data/rules/sentry.yml index a53b647..a76f51a 100644 --- a/data/rules/sentry.yml +++ b/data/rules/sentry.yml @@ -3,14 +3,13 @@ rules: id: kingfisher.sentry.1 pattern: | (?xi) - \b sentry (?:.|[\n\r]){0,32}? (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) (?:.|[\n\r]){0,32}? \b ( - [a-f0-9]{64} + [a-f0-9]{64} ) \b pattern_requirements: @@ -44,7 +43,6 @@ rules: ( sntrys_eyJpYXQiO[a-zA-Z0-9+/]{10,200}(?:LCJyZWdpb25fdXJs|InJlZ2lvbl91cmwi|cmVnaW9uX3VybCI6)[a-zA-Z0-9+/]{10,200}={0,2}_[a-zA-Z0-9+/]{43} ) - \b pattern_requirements: min_digits: 2 min_entropy: 4.2 @@ -73,11 +71,9 @@ rules: id: kingfisher.sentry.3 pattern: | (?xi) - \b ( sntryu_[a-f0-9]{64} ) - \b pattern_requirements: min_digits: 2 min_entropy: 3.5 diff --git a/data/rules/supabase.yml b/data/rules/supabase.yml index b2d7c32..6232195 100644 --- a/data/rules/supabase.yml +++ b/data/rules/supabase.yml @@ -3,11 +3,9 @@ rules: id: kingfisher.supabase.1 pattern: | (?xi) - \b ( sbp_[a-z0-9_-]{40} ) - \b pattern_requirements: min_digits: 2 min_entropy: 3.5 @@ -34,11 +32,9 @@ rules: id: kingfisher.supabase.2 pattern: | (?xi) - \b ( sb_secret_[a-z0-9_-]{31} ) - \b pattern_requirements: min_digits: 2 min_entropy: 4.0 diff --git a/src/git_binary.rs b/src/git_binary.rs index 6472cfc..15728f6 100644 --- a/src/git_binary.rs +++ b/src/git_binary.rs @@ -4,6 +4,7 @@ use std::{ }; use tracing::{debug, debug_span}; +use url::Url; use crate::{bitbucket::is_bitbucket_access_token, git_url::GitUrl}; @@ -101,6 +102,8 @@ pub struct Git { credentials: Vec, ignore_certs: bool, bitbucket_access_token: Option, + bitbucket_env: Vec<(String, String)>, + bitbucket_basic_auth: Option<(String, String)>, } impl Git { @@ -110,23 +113,60 @@ impl Git { pub fn new(ignore_certs: bool) -> Self { let mut credentials = Vec::new(); + fn normalized_env_var(name: &str) -> Option { + std::env::var(name) + .ok() + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()) + } + + let bitbucket_username = normalized_env_var("KF_BITBUCKET_USERNAME"); + let bitbucket_app_password = normalized_env_var("KF_BITBUCKET_APP_PASSWORD"); + let bitbucket_token = normalized_env_var("KF_BITBUCKET_TOKEN"); + let bitbucket_password = normalized_env_var("KF_BITBUCKET_PASSWORD"); + let bitbucket_oauth_token = normalized_env_var("KF_BITBUCKET_OAUTH_TOKEN"); + + let mut bitbucket_env = Vec::new(); + for (key, value) in [ + ("KF_BITBUCKET_USERNAME", bitbucket_username.as_ref()), + ("KF_BITBUCKET_APP_PASSWORD", bitbucket_app_password.as_ref()), + ("KF_BITBUCKET_TOKEN", bitbucket_token.as_ref()), + ("KF_BITBUCKET_PASSWORD", bitbucket_password.as_ref()), + ("KF_BITBUCKET_OAUTH_TOKEN", bitbucket_oauth_token.as_ref()), + ] { + if let Some(value) = value { + bitbucket_env.push((key.to_string(), value.to_string())); + } + } + let has_github_token = matches!(std::env::var("KF_GITHUB_TOKEN"), Ok(token) if !token.is_empty()); let has_gitlab_token = matches!(std::env::var("KF_GITLAB_TOKEN"), Ok(token) if !token.is_empty()); let has_gitea_token = matches!(std::env::var("KF_GITEA_TOKEN"), Ok(token) if !token.is_empty()); - let has_bitbucket_username = - matches!(std::env::var("KF_BITBUCKET_USERNAME"), Ok(value) if !value.is_empty()); - let bitbucket_access_token = std::env::var("KF_BITBUCKET_TOKEN") - .ok() - .filter(|value| !value.is_empty() && is_bitbucket_access_token(value)); - let has_bitbucket_password = - ["KF_BITBUCKET_APP_PASSWORD", "KF_BITBUCKET_TOKEN", "KF_BITBUCKET_PASSWORD"] - .iter() - .any(|key| matches!(std::env::var(key), Ok(value) if !value.is_empty())); - let has_bitbucket_oauth_token = - matches!(std::env::var("KF_BITBUCKET_OAUTH_TOKEN"), Ok(value) if !value.is_empty()); + let bitbucket_access_token = + bitbucket_token.as_ref().filter(|token| is_bitbucket_access_token(token)).cloned(); + let bitbucket_basic_password = bitbucket_app_password + .clone() + .or(bitbucket_token.clone()) + .or(bitbucket_password.clone()); + let bitbucket_basic_auth = if let Some(token) = bitbucket_oauth_token.clone() { + Some(("x-token-auth".to_string(), token)) + } else if let Some(token) = bitbucket_access_token.clone() { + Some(("x-token-auth".to_string(), token)) + } else if let (Some(username), Some(password)) = + (bitbucket_username.clone(), bitbucket_basic_password) + { + Some((username, password)) + } else { + None + }; + let has_bitbucket_username = bitbucket_username.is_some(); + let has_bitbucket_password = bitbucket_app_password.is_some() + || bitbucket_token.is_some() + || bitbucket_password.is_some(); + let has_bitbucket_oauth_token = bitbucket_oauth_token.is_some(); let has_bitbucket_credentials = has_bitbucket_oauth_token || bitbucket_access_token.is_some() || (has_bitbucket_username && has_bitbucket_password); @@ -186,7 +226,13 @@ impl Git { credentials.push(HUGGINGFACE_CREDENTIAL_HELPER.into()); } - Self { credentials, ignore_certs, bitbucket_access_token } + Self { + credentials, + ignore_certs, + bitbucket_access_token, + bitbucket_env, + bitbucket_basic_auth, + } } /// Create a basic `git` `Command` with environment variables set to @@ -201,6 +247,9 @@ impl Git { if self.ignore_certs { cmd.env("GIT_SSL_NO_VERIFY", "1"); } + for (key, value) in &self.bitbucket_env { + cmd.env(key, value); + } if let Some(token) = &self.bitbucket_access_token { cmd.env("KF_BITBUCKET_ACCESS_TOKEN", token); } @@ -268,11 +317,31 @@ impl Git { cmd.arg("--quiet"); cmd.arg("-c"); cmd.arg("remote.origin.fetch=+refs/*:refs/remotes/origin/*"); - cmd.arg(repo_url.as_str()); + cmd.arg(self.repo_arg_for_clone(repo_url)); cmd.arg(output_dir); debug!("{cmd:#?}"); self.run_cmd(cmd) } + + fn repo_arg_for_clone(&self, repo_url: &GitUrl) -> String { + if let Some((username, password)) = &self.bitbucket_basic_auth { + if let Ok(mut url) = Url::parse(repo_url.as_str()) { + if url + .host_str() + .map(|host| host.eq_ignore_ascii_case("bitbucket.org")) + .unwrap_or(false) + { + if url.set_username(username).is_ok() + && url.set_password(Some(password)).is_ok() + { + return url.into(); + } + } + } + } + + repo_url.as_str().to_string() + } } impl Default for Git { @@ -349,6 +418,61 @@ mod tests { ); } + #[test] + fn test_repo_arg_for_clone_includes_bitbucket_app_password() { + let url = + GitUrl::try_from(url::Url::parse("https://bitbucket.org/workspace/demo.git").unwrap()) + .unwrap(); + + temp_env::with_vars( + &[ + ("KF_BITBUCKET_USERNAME", Some("user")), + ("KF_BITBUCKET_APP_PASSWORD", Some("secret")), + ], + || { + let git = Git::new(false); + assert_eq!( + git.repo_arg_for_clone(&url), + "https://user:secret@bitbucket.org/workspace/demo.git" + ); + }, + ); + } + + #[test] + fn test_repo_arg_for_clone_uses_token_auth_when_available() { + let url = + GitUrl::try_from(url::Url::parse("https://bitbucket.org/workspace/demo.git").unwrap()) + .unwrap(); + + temp_env::with_vars(&[("KF_BITBUCKET_OAUTH_TOKEN", Some("token123"))], || { + let git = Git::new(false); + assert_eq!( + git.repo_arg_for_clone(&url), + "https://x-token-auth:token123@bitbucket.org/workspace/demo.git" + ); + }); + } + + #[test] + fn test_repo_arg_for_clone_leaves_non_bitbucket_urls_untouched() { + let url = GitUrl::try_from( + url::Url::parse("https://github.com/octocat/Hello-World.git").unwrap(), + ) + .unwrap(); + + temp_env::with_vars( + &[ + ("KF_BITBUCKET_USERNAME", Some("user")), + ("KF_BITBUCKET_APP_PASSWORD", Some("secret")), + ], + || { + let git = Git::new(false); + assert_eq!(git.repo_arg_for_clone(&url), url.as_str()); + }, + ); + } + #[test] fn test_git_new_bitbucket_access_token() { let token = "AT1234567890_ACCESS_TOKEN_EXAMPLE_WITH_UNDERSCORE"; @@ -360,6 +484,30 @@ mod tests { }); } + #[test] + fn test_git_new_bitbucket_trims_whitespace() { + let trimmed_token = "AT1234567890_ACCESS_TOKEN_EXAMPLE_WITH_UNDERSCORE"; + let token = format!(" {trimmed_token} \n"); + + temp_env::with_vars( + &[("KF_BITBUCKET_USERNAME", Some(" user\n")), ("KF_BITBUCKET_TOKEN", Some(&token))], + || { + let git = Git::new(false); + + assert_eq!( + git.bitbucket_env, + vec![ + ("KF_BITBUCKET_USERNAME".to_string(), "user".to_string()), + ("KF_BITBUCKET_TOKEN".to_string(), trimmed_token.to_string(),), + ], + ); + assert_eq!(git.credentials.len(), 4); + assert!(git.credentials.iter().any(|value| value == BITBUCKET_CREDENTIAL_HELPER)); + assert_eq!(git.bitbucket_access_token.as_deref(), Some(trimmed_token)); + }, + ); + } + #[test] fn test_clone_mode_arg() { assert_eq!(CloneMode::Bare.arg(), Some("--bare"));