diff --git a/CHANGELOG.md b/CHANGELOG.md index 414ed97..db2e20b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. ## [1.45.0] - Added `--repo-artifacts` flag to scan repository issues, gists/snippets, and wikis when cloning via `--git-url` +- Added rules for sendbird, mattermost, langchain, notion +- JWT validation hardened to reject alg:none by default (only allowed if explicitly configured), require iss for OIDC/JWKS verification, ensuring "Active Credential" means cryptographically verified and time-valid, not just unexpired +- Updated the Git cloning logic to include all refs and minimize clone output, allowing Kingfisher to analyze pull request and deleted branch history ## [1.44.0] - Fixed issue with self-update on Linux diff --git a/Cargo.toml b/Cargo.toml index ba14c38..9ebd262 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ publish = false [package] name = "kingfisher" -version = "1.44.0" +version = "1.45.0" description = "MongoDB's blazingly fast secret scanning and validation tool" edition.workspace = true rust-version.workspace = true diff --git a/README.md b/README.md index 66c4555..56e5f18 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,8 @@ Kingfisher originated as a fork of Praetorian's Nosey Parker, and is built atop - **Performance**: multithreaded, Hyperscan‑powered scanning built for huge codebases - **Extensible rules**: hundreds of built-in detectors plus YAML-defined custom rules ([docs/RULES.md](/docs/RULES.md)) - **Multiple targets**: - - **Git history**: local repos or GitHub/GitLab orgs/users + - **Git history**: local repos or GitHub/GitLab orgs/users + - **Repository artifacts**: with `--repo-artifacts`, scan GitHub/GitLab repository artifacts such as issues, pull/merge requests, wikis, snippets, and owner gists in addition to code - **Docker images**: public or private via `--docker-image` - **Jira issues**: JQL‑driven scans with `--jira-url` and `--jql` - **Confluence pages**: CQL‑driven scans with `--confluence-url` and `--cql` diff --git a/data/rules/langchain.yml b/data/rules/langchain.yml new file mode 100644 index 0000000..cd853e2 --- /dev/null +++ b/data/rules/langchain.yml @@ -0,0 +1,52 @@ +rules: + - name: LangSmith Personal Access Token + id: kingfisher.langchain.1 + pattern: | + (?xi) + \b + ( + lsv2_(?:pt)_[0-9a-f]{32}_[0-9a-f]{10} + ) + \b + min_entropy: 4.0 + examples: + - "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" + validation: + type: Http + content: + request: + method: GET + url: "https://api.smith.langchain.com/api/v1/api-key/current" + headers: + X-API-Key: "{{ TOKEN }}" + Accept: "application/json" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - name: LangSmith Service Key + id: kingfisher.langchain.2 + pattern: | + (?xi) + \b + ( + lsv2_sk_[0-9a-f]{32}_[0-9a-f]{10} + ) + \b + min_entropy: 4.0 + examples: + - "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" + validation: + type: Http + content: + request: + method: GET + url: "https://api.smith.langchain.com/api/v1/orgs/current" + headers: + X-API-Key: "{{ TOKEN }}" + Accept: "application/json" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] \ No newline at end of file diff --git a/data/rules/mattermost.yml b/data/rules/mattermost.yml new file mode 100644 index 0000000..564adb6 --- /dev/null +++ b/data/rules/mattermost.yml @@ -0,0 +1,66 @@ +rules: + - name: Mattermost URL + id: kingfisher.mattermost.1 + pattern: | + (?xi) + \b + mattermost + (?:.|[\n\r]){0,32}? + ( + https?:\/\/[a-z0-9.-]+ + (?::\d{2,5})? + (?:\/[A-Za-z0-9._~\-\/]*)? + ) + \b + confidence: medium + visible: false + min_entropy: 2.0 + examples: + - mattermost_url = "https://community.mattermost.com" + - mattermost_url='http://localhost:8065' + - 'mattermost_url: https://intra.example.com/mattermost' + + - name: Mattermost Access Token + id: kingfisher.mattermost.2 + pattern: | + (?xi) + \b + mattermost + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9]{26} + ) + \b + confidence: medium + min_entropy: 4.0 + examples: + - "mattermost_token: abcde12345fghij67890klmno1" + validation: + type: Http + content: + request: + method: GET + # Normalize any captured base that already includes /api/v4 + url: > + {%- assign base = MATTERMOST_URL | replace: "/api/v4/", "/" | replace: "/api/v4", "" -%} + {{ base }}/api/v4/users/me + headers: + Authorization: "Bearer {{ TOKEN }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + words: ['"id"', '"username"'] + match_all_words: true + depends_on_rule: + - rule_id: "kingfisher.mattermost.1" + variable: MATTERMOST_URL + references: + - https://developers.mattermost.com/api-documentation/ + - https://developers.mattermost.com/integrate/faq/ diff --git a/data/rules/notion.yml b/data/rules/notion.yml new file mode 100644 index 0000000..b4a0b44 --- /dev/null +++ b/data/rules/notion.yml @@ -0,0 +1,83 @@ +rules: + - name: Notion Legacy Token + id: kingfisher.notion.1 + pattern: | + (?xi) + notion + (?:.|[\\n\r]){0,32}? + \b + ( + secret_[A-Z0-9]{43} + ) + \b + min_entropy: 4.0 + confidence: medium + examples: + - "notion secret_efky1RtWsI0CB1Sn4TRRBLpemW1V11XwPRX3lzUKc5Q" + validation: + type: Http + content: + request: + headers: + Notion-Version: "2022-06-28" + Authorization: "Bearer {{ TOKEN }}" + method: GET + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"object":"user"' + - '"type":"bot"' + match_all_words: true + url: https://api.notion.com/v1/users/me + - name: Notion Token + id: kingfisher.notion.2 + pattern: | + (?xi) + notion + (?:.|[\\n\r]){0,32}? + \b + ( + ntn_[A-Z0-9]{40,55} + ) + \b + min_entropy: 4.0 + confidence: medium + references: + - https://developers.notion.com/page/changelog#september-11-2024 + examples: + - "notion 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" + validation: + type: Http + content: + request: + headers: + Notion-Version: "2022-06-28" + Authorization: "Bearer {{ TOKEN }}" + method: GET + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"object":"user"' + - '"type":"bot"' + match_all_words: true + url: https://api.notion.com/v1/users/me + + - name: Notion OAuth Refresh Token + id: kingfisher.notion.2 + pattern: | + (?xi) + \b + ( + nrt_[A-Z0-9_]{40,55} + ) + \b + min_entropy: 3.5 + confidence: medium + examples: + - "nrt_4Y29zY29vbF9leGFtcGxlX3JlZnJlc2hfdG9rZW4xMjM0NQ" \ No newline at end of file diff --git a/data/rules/sendbird.yml b/data/rules/sendbird.yml new file mode 100644 index 0000000..9c36a46 --- /dev/null +++ b/data/rules/sendbird.yml @@ -0,0 +1,57 @@ +rules: + - name: Sendbird Application ID + id: kingfisher.sendbird.1 + pattern: | + (?xi) + sendbird + (?:.|[\\n\r]){0,32}? + (?:APPLICATION|APP_ID|APP|CLIENT|ID) + (?:.|[\n\r]){0,32}? + \b + ( + [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} + ) + \b + confidence: medium + visible: false + min_entropy: 3.0 + examples: + - "sendbird_app_id: 12345678-1234-1234-1234-1234567890ab" + + - name: Sendbird API Token + id: kingfisher.sendbird.2 + pattern: | + (?xi) + sendbird + (?:.|[\\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [a-f0-9]{40} + ) + \b + confidence: medium + min_entropy: 4.0 + examples: + - "sendbird_api_token: 1234567890abcdef1234567890abcdef12345678" + validation: + type: Http + content: + request: + method: GET + url: "https://api-{{SENDBIRD_APP_ID}}.sendbird.com/v3/users" + headers: + "Api-Token": "{{TOKEN}}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"users":' + depends_on_rule: + - rule_id: "kingfisher.sendbird.1" + variable: SENDBIRD_APP_ID + references: + - https://sendbird.com/docs/chat/platform-api/v3/prepare-to-use-api#2-authentication \ No newline at end of file diff --git a/data/rules/stackhawk.yml b/data/rules/stackhawk.yml index fc62628..b8fec69 100644 --- a/data/rules/stackhawk.yml +++ b/data/rules/stackhawk.yml @@ -1,7 +1,13 @@ rules: - name: StackHawk API Key id: kingfisher.stackhawk.1 - pattern: '\b(hawk\.[0-9A-Za-z_-]{20}\.[0-9A-Za-z_-]{20})\b' + pattern: | + (?xi) + \b + ( + hawk\.[0-9A-Z_-]{20}\.[0-9A-Z_-]{20} + ) + \b confidence: medium min_entropy: 3.0 examples: diff --git a/src/git_binary.rs b/src/git_binary.rs index 6e9bd8d..bf32f96 100644 --- a/src/git_binary.rs +++ b/src/git_binary.rs @@ -137,6 +137,9 @@ impl Git { if let Some(arg) = clone_mode.arg() { cmd.arg(arg); } + cmd.arg("--quiet"); + cmd.arg("-c"); + cmd.arg("remote.origin.fetch=+refs/*:refs/remotes/origin/*"); cmd.arg(repo_url.as_str()); cmd.arg(output_dir); debug!("{cmd:#?}"); diff --git a/src/update.rs b/src/update.rs index 8b9b138..8f66c59 100644 --- a/src/update.rs +++ b/src/update.rs @@ -56,7 +56,7 @@ pub fn check_for_update(global_args: &GlobalArgs, base_url: Option<&str>) -> Opt info!( "{}", styles.style_finding_active_heading.apply_to( - "Homebrew install detected – will notify about updates but not self‑update" + "Homebrew install detected - will notify about updates but not self-update" ) ); } @@ -158,9 +158,9 @@ pub fn check_for_update(global_args: &GlobalArgs, base_url: Option<&str>) -> Opt warn!( "{}", styles.style_finding_active_heading.apply_to( - "Cannot replace the current binary – permission denied.\n\ + "Cannot replace the current binary - permission denied.\n\ If you installed via a package manager, run its upgrade command.\n\ - Otherwise reinstall to a user‑writable directory or re‑run with sudo." + Otherwise reinstall to a user-writable directory or re-run with sudo." ) ); } diff --git a/src/validation/jwt.rs b/src/validation/jwt.rs index bf4fc5f..3fc35fd 100644 --- a/src/validation/jwt.rs +++ b/src/validation/jwt.rs @@ -46,8 +46,33 @@ struct Claims { aud: Option, } +/// Runtime options for JWT validation policy. +#[derive(Clone, Default)] +pub struct ValidateOptions { + /// If true, accept unsigned tokens (`alg: "none"`) as long as temporal checks pass. + /// Default is **false** (more secure). + pub allow_alg_none: bool, + + /// If provided and `iss` is absent, use this key to cryptographically verify the token. + /// Useful for non-OIDC flows where you already know the verification key. + pub fallback_decoding_key: Option, +} + +/// Backwards-compatible entry point with secure defaults: +/// - `alg: none` is **rejected** +/// - `iss` is **required** unless `fallback_decoding_key` is supplied (not supplied here) pub async fn validate_jwt(token: &str) -> Result<(bool, String)> { - // --- insecure payload decode ------------------------------------------------- + validate_jwt_with( + token, + &ValidateOptions { allow_alg_none: false, fallback_decoding_key: None }, + ) + .await +} + +/// Strict validator with policy control. +/// Returns (is_active_credential, explanation). +pub async fn validate_jwt_with(token: &str, opts: &ValidateOptions) -> Result<(bool, String)> { + // --- insecure payload decode to read claims -------------------------------- let claims: Claims = { let payload_b64 = token.split('.').nth(1).ok_or_else(|| anyhow!("invalid JWT format"))?; let payload_json = URL_SAFE_NO_PAD @@ -69,132 +94,146 @@ pub async fn validate_jwt(token: &str) -> Result<(bool, String)> { } } + // parse header (for alg, kid) let header_b64 = token.split('.').next().ok_or_else(|| anyhow!("invalid JWT format"))?; let header_json = URL_SAFE_NO_PAD.decode(header_b64).map_err(|e| anyhow!("invalid base64 in header: {e}"))?; let header_val: serde_json::Value = serde_json::from_slice(&header_json).map_err(|e| anyhow!("invalid header json: {e}"))?; let alg_str = header_val.get("alg").and_then(|v| v.as_str()).unwrap_or(""); + let header = decode_header(token).map_err(|e| anyhow!("decode header: {e}"))?; + let alg = header.alg; - // If alg is "none", skip signature/JWKS entirely + // --- Policy: reject `alg: none` unless explicitly allowed ------------------ if alg_str.eq_ignore_ascii_case("none") { - // still enforce your time/claims checks that already ran - return Ok(( - true, - format!( - "JWT valid (alg: none, iss: {}, aud: {:?})", - claims.iss.clone().unwrap_or_default(), - extract_aud_strings(&claims), - ), - )); + if opts.allow_alg_none { + // time-valid is enough if explicitly allowed + return Ok(( + true, + format!( + "JWT valid (alg: none, iss: {}, aud: {:?})", + claims.iss.clone().unwrap_or_default(), + extract_aud_strings(&claims), + ), + )); + } else { + return Ok((false, "unsigned JWT (alg: none) not allowed".into())); + } } - // --------------------------------------------------------------------------- let issuer = claims.iss.clone().unwrap_or_default(); let aud_strings = extract_aud_strings(&claims); - if issuer.trim().is_empty() && aud_strings.iter().all(|s| s.trim().is_empty()) { - return Ok((false, "JWT missing issuer and audience".into())); - } - if let Some(iss) = claims.iss.clone() { - // parse header now (kid, alg) - let header = decode_header(token).map_err(|e| anyhow!("decode header: {e}"))?; - let alg = header.alg; + // --- New rule: require `iss` OR use fallback key for crypto verification --- + if issuer.trim().is_empty() { + // No issuer — we may still accept if we can cryptographically verify with a fallback key + if let Some(decoding_key) = opts.fallback_decoding_key.as_ref() { + // Verify signature (aud checked if present) + let mut validation = JwtValidation::new(alg); + if !aud_strings.is_empty() { + validation.set_audience(&aud_strings); + } + // We already did exp/nbf manually. + validation.validate_exp = false; + validation.validate_nbf = false; - // build discovery URL and fetch it (redirects disabled) - let config_url = format!("{}/.well-known/openid-configuration", iss.trim_end_matches('/')); - let cfg_resp = NO_REDIRECT_CLIENT - .get(&config_url) - .send() - .await - .map_err(|e| anyhow!("issuer discovery failed: {e}"))?; + decode::(token, decoding_key, &validation) + .map_err(|e| anyhow!("signature verification (fallback key) failed: {e}"))?; - if !cfg_resp.status().is_success() { - return Ok((false, format!("issuer discovery failed: {}", cfg_resp.status()))); - } - - let cfg_json: serde_json::Value = - cfg_resp.json().await.map_err(|e| anyhow!("invalid discovery JSON: {e}"))?; - - // extract jwks_uri - let jwks_uri = cfg_json - .get("jwks_uri") - .and_then(|v| v.as_str()) - .ok_or_else(|| anyhow!("jwks_uri missing"))?; - - // must be HTTPS - let url = Url::parse(jwks_uri).map_err(|e| anyhow!("invalid jwks_uri: {e}"))?; - if url.scheme() != "https" { - return Ok((false, "jwks_uri must use https".to_string())); - } - - // host must match issuer host  —  prevents open redirects / SSRF-on-other-host - let iss_host = Url::parse(&iss) - .map_err(|e| anyhow!("invalid iss: {e}"))? - .host_str() - .unwrap_or_default() - .to_ascii_lowercase(); - let jwks_host = url.host_str().unwrap_or_default().to_ascii_lowercase(); - if jwks_host != iss_host { + return Ok(( + true, + format!("JWT valid via fallback key (alg: {:?}, aud: {:?})", alg, aud_strings), + )); + } else { return Ok(( false, - format!("jwks_uri host ({jwks_host}) must match issuer host ({iss_host})"), + "issuer (iss) required or a fallback verification key must be provided".into(), )); } + } - // ----------------------------------------------------------------------- - // DNS resolution + private-range block - for addr in lookup_host((jwks_host.as_str(), 443)).await? { - if is_blocked_ip(addr.ip()) { - return Ok((false, "jwks_uri resolves to private or link-local IP".to_string())); - } - } + // --- With `iss`: OIDC discovery + JWKS verification path ------------------- + // build discovery URL and fetch it (redirects disabled) + let config_url = format!("{}/.well-known/openid-configuration", issuer.trim_end_matches('/')); + let cfg_resp = NO_REDIRECT_CLIENT + .get(&config_url) + .send() + .await + .map_err(|e| anyhow!("issuer discovery failed: {e}"))?; - // reachability check (existing helper) - check_url_resolvable(&url).await.map_err(|e| anyhow!("jwks uri unresolvable: {e}"))?; + if !cfg_resp.status().is_success() { + return Ok((false, format!("issuer discovery failed: {}", cfg_resp.status()))); + } - // fetch JWKS with redirect-free client - let jwks_resp = NO_REDIRECT_CLIENT - .get(url) - .send() - .await - .map_err(|e| anyhow!("jwks fetch failed: {e}"))?; - if !jwks_resp.status().is_success() { - return Ok((false, format!("jwks fetch failed: {}", jwks_resp.status()))); - } + let cfg_json: serde_json::Value = + cfg_resp.json().await.map_err(|e| anyhow!("invalid discovery JSON: {e}"))?; - let jwk_set: JwkSet = - jwks_resp.json().await.map_err(|e| anyhow!("invalid jwks json: {e}"))?; + // extract jwks_uri + let jwks_uri = cfg_json + .get("jwks_uri") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow!("jwks_uri missing"))?; - // select key by kid - let kid = header.kid.ok_or_else(|| anyhow!("no kid in header"))?; - let jwk = jwk_set - .keys - .iter() - .find(|k| k.common.key_id.as_deref() == Some(&kid)) - .ok_or_else(|| anyhow!("kid not found in jwks"))?; - - // verify signature - let decoding_key = DecodingKey::from_jwk(jwk).map_err(|e| anyhow!("invalid jwk: {e}"))?; - let mut validation = JwtValidation::new(header.alg); - validation.set_audience(&extract_aud_strings(&claims)); - validation.validate_exp = false; - validation.validate_nbf = false; - - decode::(token, &decoding_key, &validation) - .map_err(|e| anyhow!("signature verification failed: {e}"))?; + // must be HTTPS + let url = Url::parse(jwks_uri).map_err(|e| anyhow!("invalid jwks_uri: {e}"))?; + if url.scheme() != "https" { + return Ok((false, "jwks_uri must use https".to_string())); + } + // host must match issuer host + let iss_host = Url::parse(&issuer) + .map_err(|e| anyhow!("invalid iss: {e}"))? + .host_str() + .unwrap_or_default() + .to_ascii_lowercase(); + let jwks_host = url.host_str().unwrap_or_default().to_ascii_lowercase(); + if jwks_host != iss_host { return Ok(( - true, - format!( - "JWT valid (alg: {:?}, iss: {issuer}, aud: {:?})", - alg, - extract_aud_strings(&claims) - ), + false, + format!("jwks_uri host ({jwks_host}) must match issuer host ({iss_host})"), )); } - Ok((true, format!("JWT not expired (iss: {issuer}, aud: {:?})", extract_aud_strings(&claims)))) + // DNS resolution + private-range block + for addr in lookup_host((jwks_host.as_str(), 443)).await? { + if is_blocked_ip(addr.ip()) { + return Ok((false, "jwks_uri resolves to private or link-local IP".to_string())); + } + } + + // reachability check (existing helper) + check_url_resolvable(&url).await.map_err(|e| anyhow!("jwks uri unresolvable: {e}"))?; + + // fetch JWKS with redirect-free client + let jwks_resp = + NO_REDIRECT_CLIENT.get(url).send().await.map_err(|e| anyhow!("jwks fetch failed: {e}"))?; + if !jwks_resp.status().is_success() { + return Ok((false, format!("jwks fetch failed: {}", jwks_resp.status()))); + } + + let jwk_set: JwkSet = jwks_resp.json().await.map_err(|e| anyhow!("invalid jwks json: {e}"))?; + + // select key by kid + let kid = header.kid.ok_or_else(|| anyhow!("no kid in header"))?; + let jwk = jwk_set + .keys + .iter() + .find(|k| k.common.key_id.as_deref() == Some(&kid)) + .ok_or_else(|| anyhow!("kid not found in jwks"))?; + + // verify signature + let decoding_key = DecodingKey::from_jwk(jwk).map_err(|e| anyhow!("invalid jwk: {e}"))?; + let mut validation = JwtValidation::new(header.alg); + if !aud_strings.is_empty() { + validation.set_audience(&aud_strings); + } + validation.validate_exp = false; + validation.validate_nbf = false; + + decode::(token, &decoding_key, &validation) + .map_err(|e| anyhow!("signature verification failed: {e}"))?; + + Ok((true, format!("JWT valid (alg: {:?}, iss: {issuer}, aud: {:?})", alg, aud_strings))) } /// Helper: normalize aud into a flat Vec @@ -212,12 +251,11 @@ fn is_blocked_ip(ip: std::net::IpAddr) -> bool { #[cfg(test)] mod tests { + use super::{validate_jwt, validate_jwt_with, ValidateOptions}; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; use chrono::{Duration as ChronoDuration, Utc}; - use super::validate_jwt; - - fn build_token(exp_offset: i64) -> String { + fn build_unsigned_token(exp_offset: i64) -> String { let header = URL_SAFE_NO_PAD.encode(r#"{"alg":"none"}"#); let exp = (Utc::now() + ChronoDuration::seconds(exp_offset)).timestamp(); let payload = URL_SAFE_NO_PAD.encode(format!( @@ -231,16 +269,35 @@ mod tests { } #[tokio::test] - async fn valid_token() { - let token = build_token(60); + async fn unsigned_token_rejected_by_default() { + let token = build_unsigned_token(60); let res = validate_jwt(&token).await.unwrap(); - assert!(res.0); + assert!(!res.0); + assert!(res.1.contains("unsigned JWT (alg: none) not allowed")); } #[tokio::test] - async fn expired_token() { - let token = build_token(-60); - let res = validate_jwt(&token).await.unwrap(); + async fn valid_token_allows_alg_none_when_opted_in() { + let token = build_unsigned_token(60); + let res = validate_jwt_with( + &token, + &ValidateOptions { allow_alg_none: true, fallback_decoding_key: None }, + ) + .await + .unwrap(); + assert!(res.0, "expected success when alg none is explicitly allowed"); + } + + #[tokio::test] + async fn expired_token_still_rejected() { + let token = build_unsigned_token(-60); + let res = validate_jwt_with( + &token, + &ValidateOptions { allow_alg_none: true, fallback_decoding_key: None }, + ) + .await + .unwrap(); assert!(!res.0); + assert!(res.1.contains("expired")); } }