- Added '--repo-artifacts' flag to scan repository issues, gists/snippets, and wikis when cloning via '--git-url'

- Added rules for sendbird, mattermost, langchain, notion
- JWT validation hardened to reject alg:none by default (only allowed if explicitly configured), require iss for OIDC/JWKS verification, ensuring Active Credential means cryptographically verified and time-valid, not just unexpired
- Updated the Git cloning logic to include all refs and minimize clone output, allowing Kingfisher to analyze pull request and deleted branch history
This commit is contained in:
Mick Grove 2025-08-21 15:39:04 -07:00
commit 81d2f47c67
11 changed files with 442 additions and 114 deletions

View file

@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file.
## [1.45.0]
- Added `--repo-artifacts` flag to scan repository issues, gists/snippets, and wikis when cloning via `--git-url`
- Added rules for sendbird, mattermost, langchain, notion
- JWT validation hardened to reject alg:none by default (only allowed if explicitly configured), require iss for OIDC/JWKS verification, ensuring "Active Credential" means cryptographically verified and time-valid, not just unexpired
- Updated the Git cloning logic to include all refs and minimize clone output, allowing Kingfisher to analyze pull request and deleted branch history
## [1.44.0]
- Fixed issue with self-update on Linux

View file

@ -10,7 +10,7 @@ publish = false
[package]
name = "kingfisher"
version = "1.44.0"
version = "1.45.0"
description = "MongoDB's blazingly fast secret scanning and validation tool"
edition.workspace = true
rust-version.workspace = true

View file

@ -24,7 +24,8 @@ Kingfisher originated as a fork of Praetorian's Nosey Parker, and is built atop
- **Performance**: multithreaded, Hyperscanpowered scanning built for huge codebases
- **Extensible rules**: hundreds of built-in detectors plus YAML-defined custom rules ([docs/RULES.md](/docs/RULES.md))
- **Multiple targets**:
- **Git history**: local repos or GitHub/GitLab orgs/users
- **Git history**: local repos or GitHub/GitLab orgs/users
- **Repository artifacts**: with `--repo-artifacts`, scan GitHub/GitLab repository artifacts such as issues, pull/merge requests, wikis, snippets, and owner gists in addition to code
- **Docker images**: public or private via `--docker-image`
- **Jira issues**: JQLdriven scans with `--jira-url` and `--jql`
- **Confluence pages**: CQLdriven scans with `--confluence-url` and `--cql`

52
data/rules/langchain.yml Normal file
View file

@ -0,0 +1,52 @@
rules:
- name: LangSmith Personal Access Token
id: kingfisher.langchain.1
pattern: |
(?xi)
\b
(
lsv2_(?:pt)_[0-9a-f]{32}_[0-9a-f]{10}
)
\b
min_entropy: 4.0
examples:
- "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"
validation:
type: Http
content:
request:
method: GET
url: "https://api.smith.langchain.com/api/v1/api-key/current"
headers:
X-API-Key: "{{ TOKEN }}"
Accept: "application/json"
response_matcher:
- report_response: true
- type: StatusMatch
status: [200]
- type: JsonValid
- name: LangSmith Service Key
id: kingfisher.langchain.2
pattern: |
(?xi)
\b
(
lsv2_sk_[0-9a-f]{32}_[0-9a-f]{10}
)
\b
min_entropy: 4.0
examples:
- "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"
validation:
type: Http
content:
request:
method: GET
url: "https://api.smith.langchain.com/api/v1/orgs/current"
headers:
X-API-Key: "{{ TOKEN }}"
Accept: "application/json"
response_matcher:
- report_response: true
- type: StatusMatch
status: [200]

66
data/rules/mattermost.yml Normal file
View file

@ -0,0 +1,66 @@
rules:
- name: Mattermost URL
id: kingfisher.mattermost.1
pattern: |
(?xi)
\b
mattermost
(?:.|[\n\r]){0,32}?
(
https?:\/\/[a-z0-9.-]+
(?::\d{2,5})?
(?:\/[A-Za-z0-9._~\-\/]*)?
)
\b
confidence: medium
visible: false
min_entropy: 2.0
examples:
- mattermost_url = "https://community.mattermost.com"
- mattermost_url='http://localhost:8065'
- 'mattermost_url: https://intra.example.com/mattermost'
- name: Mattermost Access Token
id: kingfisher.mattermost.2
pattern: |
(?xi)
\b
mattermost
(?:.|[\n\r]){0,32}?
(?:SECRET|PRIVATE|ACCESS|KEY|TOKEN)
(?:.|[\n\r]){0,32}?
\b
(
[A-Z0-9]{26}
)
\b
confidence: medium
min_entropy: 4.0
examples:
- "mattermost_token: abcde12345fghij67890klmno1"
validation:
type: Http
content:
request:
method: GET
# Normalize any captured base that already includes /api/v4
url: >
{%- assign base = MATTERMOST_URL | replace: "/api/v4/", "/" | replace: "/api/v4", "" -%}
{{ base }}/api/v4/users/me
headers:
Authorization: "Bearer {{ TOKEN }}"
Accept: application/json
response_matcher:
- report_response: true
- type: StatusMatch
status: [200]
- type: JsonValid
- type: WordMatch
words: ['"id"', '"username"']
match_all_words: true
depends_on_rule:
- rule_id: "kingfisher.mattermost.1"
variable: MATTERMOST_URL
references:
- https://developers.mattermost.com/api-documentation/
- https://developers.mattermost.com/integrate/faq/

83
data/rules/notion.yml Normal file
View file

@ -0,0 +1,83 @@
rules:
- name: Notion Legacy Token
id: kingfisher.notion.1
pattern: |
(?xi)
notion
(?:.|[\\n\r]){0,32}?
\b
(
secret_[A-Z0-9]{43}
)
\b
min_entropy: 4.0
confidence: medium
examples:
- "notion secret_efky1RtWsI0CB1Sn4TRRBLpemW1V11XwPRX3lzUKc5Q"
validation:
type: Http
content:
request:
headers:
Notion-Version: "2022-06-28"
Authorization: "Bearer {{ TOKEN }}"
method: GET
response_matcher:
- report_response: true
- type: StatusMatch
status: [200]
- type: WordMatch
words:
- '"object":"user"'
- '"type":"bot"'
match_all_words: true
url: https://api.notion.com/v1/users/me
- name: Notion Token
id: kingfisher.notion.2
pattern: |
(?xi)
notion
(?:.|[\\n\r]){0,32}?
\b
(
ntn_[A-Z0-9]{40,55}
)
\b
min_entropy: 4.0
confidence: medium
references:
- https://developers.notion.com/page/changelog#september-11-2024
examples:
- "notion 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"
validation:
type: Http
content:
request:
headers:
Notion-Version: "2022-06-28"
Authorization: "Bearer {{ TOKEN }}"
method: GET
response_matcher:
- report_response: true
- type: StatusMatch
status: [200]
- type: WordMatch
words:
- '"object":"user"'
- '"type":"bot"'
match_all_words: true
url: https://api.notion.com/v1/users/me
- name: Notion OAuth Refresh Token
id: kingfisher.notion.2
pattern: |
(?xi)
\b
(
nrt_[A-Z0-9_]{40,55}
)
\b
min_entropy: 3.5
confidence: medium
examples:
- "nrt_4Y29zY29vbF9leGFtcGxlX3JlZnJlc2hfdG9rZW4xMjM0NQ"

57
data/rules/sendbird.yml Normal file
View file

@ -0,0 +1,57 @@
rules:
- name: Sendbird Application ID
id: kingfisher.sendbird.1
pattern: |
(?xi)
sendbird
(?:.|[\\n\r]){0,32}?
(?:APPLICATION|APP_ID|APP|CLIENT|ID)
(?:.|[\n\r]){0,32}?
\b
(
[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}
)
\b
confidence: medium
visible: false
min_entropy: 3.0
examples:
- "sendbird_app_id: 12345678-1234-1234-1234-1234567890ab"
- name: Sendbird API Token
id: kingfisher.sendbird.2
pattern: |
(?xi)
sendbird
(?:.|[\\n\r]){0,32}?
(?:SECRET|PRIVATE|ACCESS|KEY|TOKEN)
(?:.|[\n\r]){0,32}?
\b
(
[a-f0-9]{40}
)
\b
confidence: medium
min_entropy: 4.0
examples:
- "sendbird_api_token: 1234567890abcdef1234567890abcdef12345678"
validation:
type: Http
content:
request:
method: GET
url: "https://api-{{SENDBIRD_APP_ID}}.sendbird.com/v3/users"
headers:
"Api-Token": "{{TOKEN}}"
response_matcher:
- report_response: true
- type: StatusMatch
status: [200]
- type: WordMatch
words:
- '"users":'
depends_on_rule:
- rule_id: "kingfisher.sendbird.1"
variable: SENDBIRD_APP_ID
references:
- https://sendbird.com/docs/chat/platform-api/v3/prepare-to-use-api#2-authentication

View file

@ -1,7 +1,13 @@
rules:
- name: StackHawk API Key
id: kingfisher.stackhawk.1
pattern: '\b(hawk\.[0-9A-Za-z_-]{20}\.[0-9A-Za-z_-]{20})\b'
pattern: |
(?xi)
\b
(
hawk\.[0-9A-Z_-]{20}\.[0-9A-Z_-]{20}
)
\b
confidence: medium
min_entropy: 3.0
examples:

View file

@ -137,6 +137,9 @@ impl Git {
if let Some(arg) = clone_mode.arg() {
cmd.arg(arg);
}
cmd.arg("--quiet");
cmd.arg("-c");
cmd.arg("remote.origin.fetch=+refs/*:refs/remotes/origin/*");
cmd.arg(repo_url.as_str());
cmd.arg(output_dir);
debug!("{cmd:#?}");

View file

@ -56,7 +56,7 @@ pub fn check_for_update(global_args: &GlobalArgs, base_url: Option<&str>) -> Opt
info!(
"{}",
styles.style_finding_active_heading.apply_to(
"Homebrew install detected will notify about updates but not selfupdate"
"Homebrew install detected - will notify about updates but not self-update"
)
);
}
@ -158,9 +158,9 @@ pub fn check_for_update(global_args: &GlobalArgs, base_url: Option<&str>) -> Opt
warn!(
"{}",
styles.style_finding_active_heading.apply_to(
"Cannot replace the current binary permission denied.\n\
"Cannot replace the current binary - permission denied.\n\
If you installed via a package manager, run its upgrade command.\n\
Otherwise reinstall to a userwritable directory or rerun with sudo."
Otherwise reinstall to a user-writable directory or re-run with sudo."
)
);
}

View file

@ -46,8 +46,33 @@ struct Claims {
aud: Option<Aud>,
}
/// Runtime options for JWT validation policy.
#[derive(Clone, Default)]
pub struct ValidateOptions {
/// If true, accept unsigned tokens (`alg: "none"`) as long as temporal checks pass.
/// Default is **false** (more secure).
pub allow_alg_none: bool,
/// If provided and `iss` is absent, use this key to cryptographically verify the token.
/// Useful for non-OIDC flows where you already know the verification key.
pub fallback_decoding_key: Option<DecodingKey>,
}
/// Backwards-compatible entry point with secure defaults:
/// - `alg: none` is **rejected**
/// - `iss` is **required** unless `fallback_decoding_key` is supplied (not supplied here)
pub async fn validate_jwt(token: &str) -> Result<(bool, String)> {
// --- insecure payload decode -------------------------------------------------
validate_jwt_with(
token,
&ValidateOptions { allow_alg_none: false, fallback_decoding_key: None },
)
.await
}
/// Strict validator with policy control.
/// Returns (is_active_credential, explanation).
pub async fn validate_jwt_with(token: &str, opts: &ValidateOptions) -> Result<(bool, String)> {
// --- insecure payload decode to read claims --------------------------------
let claims: Claims = {
let payload_b64 = token.split('.').nth(1).ok_or_else(|| anyhow!("invalid JWT format"))?;
let payload_json = URL_SAFE_NO_PAD
@ -69,132 +94,146 @@ pub async fn validate_jwt(token: &str) -> Result<(bool, String)> {
}
}
// parse header (for alg, kid)
let header_b64 = token.split('.').next().ok_or_else(|| anyhow!("invalid JWT format"))?;
let header_json =
URL_SAFE_NO_PAD.decode(header_b64).map_err(|e| anyhow!("invalid base64 in header: {e}"))?;
let header_val: serde_json::Value =
serde_json::from_slice(&header_json).map_err(|e| anyhow!("invalid header json: {e}"))?;
let alg_str = header_val.get("alg").and_then(|v| v.as_str()).unwrap_or("");
let header = decode_header(token).map_err(|e| anyhow!("decode header: {e}"))?;
let alg = header.alg;
// If alg is "none", skip signature/JWKS entirely
// --- Policy: reject `alg: none` unless explicitly allowed ------------------
if alg_str.eq_ignore_ascii_case("none") {
// still enforce your time/claims checks that already ran
return Ok((
true,
format!(
"JWT valid (alg: none, iss: {}, aud: {:?})",
claims.iss.clone().unwrap_or_default(),
extract_aud_strings(&claims),
),
));
if opts.allow_alg_none {
// time-valid is enough if explicitly allowed
return Ok((
true,
format!(
"JWT valid (alg: none, iss: {}, aud: {:?})",
claims.iss.clone().unwrap_or_default(),
extract_aud_strings(&claims),
),
));
} else {
return Ok((false, "unsigned JWT (alg: none) not allowed".into()));
}
}
// ---------------------------------------------------------------------------
let issuer = claims.iss.clone().unwrap_or_default();
let aud_strings = extract_aud_strings(&claims);
if issuer.trim().is_empty() && aud_strings.iter().all(|s| s.trim().is_empty()) {
return Ok((false, "JWT missing issuer and audience".into()));
}
if let Some(iss) = claims.iss.clone() {
// parse header now (kid, alg)
let header = decode_header(token).map_err(|e| anyhow!("decode header: {e}"))?;
let alg = header.alg;
// --- New rule: require `iss` OR use fallback key for crypto verification ---
if issuer.trim().is_empty() {
// No issuer — we may still accept if we can cryptographically verify with a fallback key
if let Some(decoding_key) = opts.fallback_decoding_key.as_ref() {
// Verify signature (aud checked if present)
let mut validation = JwtValidation::new(alg);
if !aud_strings.is_empty() {
validation.set_audience(&aud_strings);
}
// We already did exp/nbf manually.
validation.validate_exp = false;
validation.validate_nbf = false;
// build discovery URL and fetch it (redirects disabled)
let config_url = format!("{}/.well-known/openid-configuration", iss.trim_end_matches('/'));
let cfg_resp = NO_REDIRECT_CLIENT
.get(&config_url)
.send()
.await
.map_err(|e| anyhow!("issuer discovery failed: {e}"))?;
decode::<Claims>(token, decoding_key, &validation)
.map_err(|e| anyhow!("signature verification (fallback key) failed: {e}"))?;
if !cfg_resp.status().is_success() {
return Ok((false, format!("issuer discovery failed: {}", cfg_resp.status())));
}
let cfg_json: serde_json::Value =
cfg_resp.json().await.map_err(|e| anyhow!("invalid discovery JSON: {e}"))?;
// extract jwks_uri
let jwks_uri = cfg_json
.get("jwks_uri")
.and_then(|v| v.as_str())
.ok_or_else(|| anyhow!("jwks_uri missing"))?;
// must be HTTPS
let url = Url::parse(jwks_uri).map_err(|e| anyhow!("invalid jwks_uri: {e}"))?;
if url.scheme() != "https" {
return Ok((false, "jwks_uri must use https".to_string()));
}
// host must match issuer host  —  prevents open redirects / SSRF-on-other-host
let iss_host = Url::parse(&iss)
.map_err(|e| anyhow!("invalid iss: {e}"))?
.host_str()
.unwrap_or_default()
.to_ascii_lowercase();
let jwks_host = url.host_str().unwrap_or_default().to_ascii_lowercase();
if jwks_host != iss_host {
return Ok((
true,
format!("JWT valid via fallback key (alg: {:?}, aud: {:?})", alg, aud_strings),
));
} else {
return Ok((
false,
format!("jwks_uri host ({jwks_host}) must match issuer host ({iss_host})"),
"issuer (iss) required or a fallback verification key must be provided".into(),
));
}
}
// -----------------------------------------------------------------------
// DNS resolution + private-range block
for addr in lookup_host((jwks_host.as_str(), 443)).await? {
if is_blocked_ip(addr.ip()) {
return Ok((false, "jwks_uri resolves to private or link-local IP".to_string()));
}
}
// --- With `iss`: OIDC discovery + JWKS verification path -------------------
// build discovery URL and fetch it (redirects disabled)
let config_url = format!("{}/.well-known/openid-configuration", issuer.trim_end_matches('/'));
let cfg_resp = NO_REDIRECT_CLIENT
.get(&config_url)
.send()
.await
.map_err(|e| anyhow!("issuer discovery failed: {e}"))?;
// reachability check (existing helper)
check_url_resolvable(&url).await.map_err(|e| anyhow!("jwks uri unresolvable: {e}"))?;
if !cfg_resp.status().is_success() {
return Ok((false, format!("issuer discovery failed: {}", cfg_resp.status())));
}
// fetch JWKS with redirect-free client
let jwks_resp = NO_REDIRECT_CLIENT
.get(url)
.send()
.await
.map_err(|e| anyhow!("jwks fetch failed: {e}"))?;
if !jwks_resp.status().is_success() {
return Ok((false, format!("jwks fetch failed: {}", jwks_resp.status())));
}
let cfg_json: serde_json::Value =
cfg_resp.json().await.map_err(|e| anyhow!("invalid discovery JSON: {e}"))?;
let jwk_set: JwkSet =
jwks_resp.json().await.map_err(|e| anyhow!("invalid jwks json: {e}"))?;
// extract jwks_uri
let jwks_uri = cfg_json
.get("jwks_uri")
.and_then(|v| v.as_str())
.ok_or_else(|| anyhow!("jwks_uri missing"))?;
// select key by kid
let kid = header.kid.ok_or_else(|| anyhow!("no kid in header"))?;
let jwk = jwk_set
.keys
.iter()
.find(|k| k.common.key_id.as_deref() == Some(&kid))
.ok_or_else(|| anyhow!("kid not found in jwks"))?;
// verify signature
let decoding_key = DecodingKey::from_jwk(jwk).map_err(|e| anyhow!("invalid jwk: {e}"))?;
let mut validation = JwtValidation::new(header.alg);
validation.set_audience(&extract_aud_strings(&claims));
validation.validate_exp = false;
validation.validate_nbf = false;
decode::<Claims>(token, &decoding_key, &validation)
.map_err(|e| anyhow!("signature verification failed: {e}"))?;
// must be HTTPS
let url = Url::parse(jwks_uri).map_err(|e| anyhow!("invalid jwks_uri: {e}"))?;
if url.scheme() != "https" {
return Ok((false, "jwks_uri must use https".to_string()));
}
// host must match issuer host
let iss_host = Url::parse(&issuer)
.map_err(|e| anyhow!("invalid iss: {e}"))?
.host_str()
.unwrap_or_default()
.to_ascii_lowercase();
let jwks_host = url.host_str().unwrap_or_default().to_ascii_lowercase();
if jwks_host != iss_host {
return Ok((
true,
format!(
"JWT valid (alg: {:?}, iss: {issuer}, aud: {:?})",
alg,
extract_aud_strings(&claims)
),
false,
format!("jwks_uri host ({jwks_host}) must match issuer host ({iss_host})"),
));
}
Ok((true, format!("JWT not expired (iss: {issuer}, aud: {:?})", extract_aud_strings(&claims))))
// DNS resolution + private-range block
for addr in lookup_host((jwks_host.as_str(), 443)).await? {
if is_blocked_ip(addr.ip()) {
return Ok((false, "jwks_uri resolves to private or link-local IP".to_string()));
}
}
// reachability check (existing helper)
check_url_resolvable(&url).await.map_err(|e| anyhow!("jwks uri unresolvable: {e}"))?;
// fetch JWKS with redirect-free client
let jwks_resp =
NO_REDIRECT_CLIENT.get(url).send().await.map_err(|e| anyhow!("jwks fetch failed: {e}"))?;
if !jwks_resp.status().is_success() {
return Ok((false, format!("jwks fetch failed: {}", jwks_resp.status())));
}
let jwk_set: JwkSet = jwks_resp.json().await.map_err(|e| anyhow!("invalid jwks json: {e}"))?;
// select key by kid
let kid = header.kid.ok_or_else(|| anyhow!("no kid in header"))?;
let jwk = jwk_set
.keys
.iter()
.find(|k| k.common.key_id.as_deref() == Some(&kid))
.ok_or_else(|| anyhow!("kid not found in jwks"))?;
// verify signature
let decoding_key = DecodingKey::from_jwk(jwk).map_err(|e| anyhow!("invalid jwk: {e}"))?;
let mut validation = JwtValidation::new(header.alg);
if !aud_strings.is_empty() {
validation.set_audience(&aud_strings);
}
validation.validate_exp = false;
validation.validate_nbf = false;
decode::<Claims>(token, &decoding_key, &validation)
.map_err(|e| anyhow!("signature verification failed: {e}"))?;
Ok((true, format!("JWT valid (alg: {:?}, iss: {issuer}, aud: {:?})", alg, aud_strings)))
}
/// Helper: normalize aud into a flat Vec<String>
@ -212,12 +251,11 @@ fn is_blocked_ip(ip: std::net::IpAddr) -> bool {
#[cfg(test)]
mod tests {
use super::{validate_jwt, validate_jwt_with, ValidateOptions};
use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _};
use chrono::{Duration as ChronoDuration, Utc};
use super::validate_jwt;
fn build_token(exp_offset: i64) -> String {
fn build_unsigned_token(exp_offset: i64) -> String {
let header = URL_SAFE_NO_PAD.encode(r#"{"alg":"none"}"#);
let exp = (Utc::now() + ChronoDuration::seconds(exp_offset)).timestamp();
let payload = URL_SAFE_NO_PAD.encode(format!(
@ -231,16 +269,35 @@ mod tests {
}
#[tokio::test]
async fn valid_token() {
let token = build_token(60);
async fn unsigned_token_rejected_by_default() {
let token = build_unsigned_token(60);
let res = validate_jwt(&token).await.unwrap();
assert!(res.0);
assert!(!res.0);
assert!(res.1.contains("unsigned JWT (alg: none) not allowed"));
}
#[tokio::test]
async fn expired_token() {
let token = build_token(-60);
let res = validate_jwt(&token).await.unwrap();
async fn valid_token_allows_alg_none_when_opted_in() {
let token = build_unsigned_token(60);
let res = validate_jwt_with(
&token,
&ValidateOptions { allow_alg_none: true, fallback_decoding_key: None },
)
.await
.unwrap();
assert!(res.0, "expected success when alg none is explicitly allowed");
}
#[tokio::test]
async fn expired_token_still_rejected() {
let token = build_unsigned_token(-60);
let res = validate_jwt_with(
&token,
&ValidateOptions { allow_alg_none: true, fallback_decoding_key: None },
)
.await
.unwrap();
assert!(!res.0);
assert!(res.1.contains("expired"));
}
}