Merge pull request #260 from mongodb/development

v1.90.0
This commit is contained in:
Mick Grove 2026-03-18 19:50:07 -07:00 committed by GitHub
commit 8b83b2d87c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 222 additions and 17 deletions

View file

@ -2,6 +2,12 @@
All notable changes to this project will be documented in this file.
## [v1.90.0]
- Added `--max-validation-response-length <BYTES>` for `scan` to control validation response storage truncation (default: `2048`, `0` disables truncation).
- Updated `--full-validation-response` to bypass both validation storage truncation and reporter truncation, preserving complete response bodies end-to-end for parsing/reporting workflows.
- Added Testkube detection/validation coverage with `kingfisher.testkube.*` rules for API keys plus dependent organization/environment IDs used for live API validation.
- Improved TrueNAS rule
## [v1.89.0]
- Added TOON output for `scan`, `validate`, and `revoke`, optimized for LLM/agent workflows; prefer `--format toon` when calling Kingfisher from an LLM.
- Expanded built-in revocation support with new YAML revocation flows for Cloudflare, Confluent, Doppler, Mapbox, Particle.io, Twitch, and additional Vercel token formats.

View file

@ -48,7 +48,7 @@ http = "1.4"
[package]
name = "kingfisher"
version = "1.89.0"
version = "1.90.0"
description = "MongoDB's blazingly fast and accurate secret scanning and validation tool"
edition.workspace = true
rust-version.workspace = true

View file

@ -4,7 +4,7 @@
<img src="docs/kingfisher_logo.png" alt="Kingfisher Logo" width="126" height="173" style="vertical-align: right;" />
[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
[![Detection Rules](https://img.shields.io/badge/Detection%20Rules-540-2ea043.svg)](https://github.com/mongodb/kingfisher)<br>
[![Detection Rules](https://img.shields.io/badge/Detection%20Rules-546-2ea043.svg)](https://github.com/mongodb/kingfisher)<br>
[![ghcr downloads](https://ghcr-badge.elias.eu.org/shield/mongodb/kingfisher/kingfisher)](https://github.com/mongodb/kingfisher/pkgs/container/kingfisher)<br>
@ -25,7 +25,7 @@ Kingfisher is a high-performance, open source secret detection tool for source c
- Scan code, Git history, and integrated platforms (GitHub, GitLab, Azure Repos, Bitbucket, Gitea, Hugging Face, Jira, Confluence, Slack, Microsoft Teams, Docker, AWS S3, and Google Cloud Storage)
- Validate discovered credentials against provider APIs to reduce false positives
- Revoke supported secrets directly from the CLI
- Generate JSON, SARIF, and HTML outputs for security teams, compliance, and CI
- Generate JSON, SARIF, TOON, and HTML outputs for security teams, compliance, and CI
## Key Features
@ -464,7 +464,13 @@ kingfisher scan /path/to/repo \
--validation-rps-rule github=2 \
--validation-rps-rule pypi=0.5
# Include full validation response bodies (not truncated to 512 characters)
# Increase validation response storage limit (default: 2048 bytes)
kingfisher scan /path/to/repo --max-validation-response-length 8192
# Disable validation response storage truncation entirely (0 = unlimited)
kingfisher scan /path/to/repo --max-validation-response-length 0
# Include full validation response bodies end-to-end (no validation or reporter truncation)
# Useful for parsing complete validation responses (e.g., GitHub token metadata)
kingfisher scan /path/to/repo --full-validation-response

View file

@ -1,4 +1,28 @@
rules:
- name: TrueNAS Instance URL
id: kingfisher.truenas.3
visible: false
confidence: medium
min_entropy: 2.0
pattern: |
(?x)
\b
(
https?://[a-zA-Z0-9._:-]+
)
/api/v2\.0/
(?:system|pool|device|sharing|jail|vm|chart|app|zvol|dataset|replication|snapshot|boot|tunable|smb|nfs|iscsi|certificate|acme|filesystem|reporting|alert|update|core|initshutdownscript)
examples:
- http://192.168.0.30/api/v2.0/system/info
- https://truenas.example.com/api/v2.0/device/get_info
- https://nas.local:443/api/v2.0/pool/dataset
- http://10.0.0.1/api/v2.0/sharing/smb
- https://truenas.local/api/v2.0/jail/query
- http://192.168.1.50:80/api/v2.0/zvol/id
- https://nas:443/api/v2.0/boot/environment
references:
- https://www.truenas.com/docs/api/scale_rest_api.html
- name: TrueNAS API Key (WebSocket)
id: kingfisher.truenas.1
pattern: |
@ -30,12 +54,31 @@ rules:
- https://www.truenas.com/docs/scale/scaleclireference/auth/cliapikey/
- https://www.truenas.com/docs/scale/api/
- https://www.truenas.com/community/threads/api-examples-in-perl-python.108053/
depends_on_rule:
- rule_id: kingfisher.truenas.3
variable: TRUENAS_URL
validation:
type: Http
content:
request:
method: GET
url: "{{ TRUENAS_URL }}/api/v2.0/system/info"
headers:
Authorization: "Bearer {{ TOKEN }}"
Accept: application/json
response_matcher:
- report_response: true
- type: StatusMatch
status: [200]
- type: JsonValid
- name: TrueNAS API Key (REST API)
id: kingfisher.truenas.2
pattern: |
(?x)
Bearer\s*
/api/v2\.0
(?:.|[\n\r]){0,256}?
Bearer\s+
(\d+-[a-zA-Z0-9]{64})
\b
pattern_requirements:
@ -51,3 +94,67 @@ rules:
- https://www.truenas.com/docs/scale/scaleclireference/auth/cliapikey/
- https://www.truenas.com/docs/scale/api/
- https://www.truenas.com/community/threads/api-examples-in-perl-python.108053/
depends_on_rule:
- rule_id: kingfisher.truenas.3
variable: TRUENAS_URL
validation:
type: Http
content:
request:
method: GET
url: "{{ TRUENAS_URL }}/api/v2.0/system/info"
headers:
Authorization: "Bearer {{ TOKEN }}"
Accept: application/json
response_matcher:
- report_response: true
- type: StatusMatch
status: [200]
- type: JsonValid
- name: TrueNAS API Key (keyword proximity)
id: kingfisher.truenas.4
pattern: |
(?xi)
\b(?:truenas|true[_-]nas|tn[_-])
(?:.|[\n\r]){0,64}?
(?:api[_-]?key|api[_-]?token|key|token|secret|password|auth)
(?:.|[\n\r]){0,32}?
(
\d+-[a-zA-Z0-9]{64}
)
\b
pattern_requirements:
min_digits: 2
min_entropy: 3.3
confidence: medium
examples:
- 'TRUENAS_API_KEY=8-Lp22ov7halMBLUpG97Wg4y7fibQi3CW19VJiZcCu746zgCs0mdDdTCoOcpgEucgu'
- 'truenas_token: "10-6LZBVhNq8zze0rzXJptfSWDBoskWuThnQb3fUVw4sVNgJ7GKT3ITVIovhwPf34oL"'
- '"truenas_api_key": "9-hTSZDBPyg0PjRZvWb8omoxJ7X2gAjRGmiPKql9ENGIUP9OPtEAzz5f6g9YIMVbZT"'
- 'tn_api_key = "8-Lp22ov7halMBLUpG97Wg4y7fibQi3CW19VJiZcCu746zgCs0mdDdTCoOcpgEucgu"'
- 'true-nas_secret=9-hTSZDBPyg0PjRZvWb8omoxJ7X2gAjRGmiPKql9ENGIUP9OPtEAzz5f6g9YIMVbZT'
references:
- https://www.truenas.com/docs/api/core_websocket_api.html
- https://www.truenas.com/docs/api/scale_rest_api.html
- https://www.truenas.com/docs/scale/scaletutorials/toptoolbar/managingapikeys/
- https://www.truenas.com/docs/scale/scaleclireference/auth/cliapikey/
- https://www.truenas.com/docs/scale/api/
- https://www.truenas.com/community/threads/api-examples-in-perl-python.108053/
depends_on_rule:
- rule_id: kingfisher.truenas.3
variable: TRUENAS_URL
validation:
type: Http
content:
request:
method: GET
url: "{{ TRUENAS_URL }}/api/v2.0/system/info"
headers:
Authorization: "Bearer {{ TOKEN }}"
Accept: application/json
response_matcher:
- report_response: true
- type: StatusMatch
status: [200]
- type: JsonValid

View file

@ -171,19 +171,26 @@ kingfisher scan /path/to/code --validation-timeout 15
# Set number of retry attempts (default: 1, range: 0-5)
kingfisher scan /path/to/code --validation-retries 2
# Include full validation response bodies without truncation
# Increase validation response storage limit (default: 2048 bytes)
kingfisher scan /path/to/code --max-validation-response-length 8192
# Disable validation response storage truncation entirely (0 = unlimited)
kingfisher scan /path/to/code --max-validation-response-length 0
# Include full validation response bodies end-to-end (no validation or reporter truncation)
kingfisher scan /path/to/code --full-validation-response
# Combine options
kingfisher scan /path/to/code \
--validation-timeout 20 \
--validation-retries 3 \
--full-validation-response
--max-validation-response-length 8192
```
- `--validation-timeout SECONDS`: per-request and per-match timeout for validation (default: 10, range: 1-60).
- `--validation-retries N`: number of retry attempts for validation requests (default: 1, range: 0-5).
- `--full-validation-response`: include complete validation response bodies without truncation. By default, validation responses are truncated to 512 characters for readability. This flag is useful when you need to parse full validation responses (e.g., GitHub token validation responses that include user metadata beyond the first 512 characters).
- `--max-validation-response-length BYTES`: maximum bytes stored from validation response bodies (default: 2048; `0` disables truncation at storage time).
- `--full-validation-response`: include complete validation response bodies end-to-end. This bypasses both storage-time truncation and reporter display truncation, and takes precedence over `--max-validation-response-length`.
## Scanning in CI Pipelines
@ -358,6 +365,8 @@ kingfisher scan /path/to/repo --rule-stats
- `--no-ignore-if-contains`: Ignore the `ignore_if_contains` filter in rules so placeholder words still produce findings
- `--validation-timeout SECONDS`: per-request and per-match timeout for validation (default: 10, range: 1-60).
- `--validation-retries N`: number of retry attempts for validation requests (default: 1, range: 0-5).
- `--max-validation-response-length BYTES`: maximum bytes stored from validation response bodies (default: 2048; `0` disables truncation at storage time).
- `--full-validation-response`: include complete validation response bodies end-to-end (bypasses storage and reporter truncation).
### Exclude specific paths

View file

@ -119,6 +119,16 @@ pub struct ScanArgs {
#[arg(global = true, long, default_value_t = false)]
pub full_validation_response: bool,
/// Maximum bytes to store from validation response bodies (0 = unlimited).
/// Overridden by --full-validation-response which forces unlimited storage.
#[arg(
global = true,
long = "max-validation-response-length",
default_value_t = 2048,
value_name = "BYTES"
)]
pub max_validation_response_length: usize,
/// Map validated cloud credentials to their effective identities; use only when
/// authorized for the target account because this triggers additional network
/// requests to determine granted access

View file

@ -975,6 +975,7 @@ pub(crate) fn create_minimal_scan_args() -> crate::cli::commands::scan::ScanArgs
validation_rps: None,
validation_rps_rule: Vec::new(),
full_validation_response: false,
max_validation_response_length: 2048,
}
}

View file

@ -597,6 +597,7 @@ fn create_default_scan_args() -> cli::commands::scan::ScanArgs {
validation_rps: None,
validation_rps_rule: Vec::new(),
full_validation_response: false,
max_validation_response_length: 2048,
}
}
/// Run the rules check command

View file

@ -1822,6 +1822,7 @@ mod tests {
validation_rps: None,
validation_rps_rule: Vec::new(),
full_validation_response: false,
max_validation_response_length: 2048,
}
}

View file

@ -207,6 +207,7 @@ mod tests {
validation_rps: None,
validation_rps_rule: Vec::new(),
full_validation_response: false,
max_validation_response_length: 2048,
}
}

View file

@ -500,6 +500,14 @@ fn apply_baseline_if_configured(
Ok(())
}
fn effective_max_validation_body_len(args: &scan::ScanArgs) -> usize {
if args.full_validation_response {
0
} else {
args.max_validation_response_length
}
}
/// Runs the validation phase on matches in the datastore.
#[allow(clippy::too_many_arguments)]
async fn run_validation_phase(
@ -523,6 +531,7 @@ async fn run_validation_phase(
rate_limiter.clone(),
Duration::from_secs(args.validation_timeout),
args.validation_retries,
effective_max_validation_body_len(args),
)
.await?;
}
@ -668,6 +677,7 @@ async fn run_parallel_scan(
rate_limiter.clone(),
Duration::from_secs(args.validation_timeout),
args.validation_retries,
effective_max_validation_body_len(args),
)
.await?;
}
@ -756,6 +766,7 @@ async fn run_parallel_scan(
rate_limiter.clone(),
Duration::from_secs(args.validation_timeout),
args.validation_retries,
effective_max_validation_body_len(&args),
))?;
}
}

View file

@ -209,6 +209,7 @@ pub async fn run_secret_validation(
rate_limiter: Option<Arc<ValidationRateLimiter>>,
validation_timeout: Duration,
validation_retries: u32,
max_body_len: usize,
) -> Result<()> {
// ── 1. Concurrency & counters ───────────────────────────────────────────
let concurrency = if num_jobs > 0 { num_jobs } else { num_cpus::get() };
@ -348,6 +349,7 @@ pub async fn run_secret_validation(
rate_limiter.as_deref(),
validation_timeout,
validation_retries,
max_body_len,
)
.await;
@ -475,6 +477,7 @@ pub async fn run_secret_validation(
rate_limiter.as_deref(),
validation_timeout,
validation_retries,
max_body_len,
)
.await;
for d in &mut dups {
@ -563,6 +566,7 @@ async fn validate_single(
rate_limiter: Option<&ValidationRateLimiter>,
validation_timeout: Duration,
validation_retries: u32,
max_body_len: usize,
) {
// Build key
let dep_vars_str = dep_vars
@ -624,6 +628,7 @@ async fn validate_single(
validation_timeout,
validation_retries,
rate_limiter,
max_body_len,
)
.await
})

View file

@ -42,7 +42,6 @@ pub use kingfisher_scanner::validation::{
pub mod utils;
const VALIDATION_CACHE_SECONDS: u64 = 1200; // 20 minutes
const MAX_VALIDATION_BODY_LEN: usize = 2048;
fn truncate_to_char_boundary(s: &mut String, max_len: usize) {
if s.len() <= max_len {
@ -57,6 +56,19 @@ fn truncate_to_char_boundary(s: &mut String, max_len: usize) {
s.truncate(new_len);
}
/// Build a truncated preview from `body` without cloning the full string.
/// When `max_len` is 0, truncation is disabled and the full body is returned.
fn truncate_preview(body: &str, max_len: usize) -> String {
if max_len == 0 || body.len() <= max_len {
return body.to_string();
}
let mut end = max_len;
while end > 0 && !body.is_char_boundary(end) {
end -= 1;
}
body[..end].to_string()
}
static USER_AGENT_SUFFIX: OnceCell<String> = OnceCell::new();
const BROWSER_USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \
@ -333,6 +345,7 @@ pub async fn validate_single_match(
validation_timeout: Duration,
validation_retries: u32,
rate_limiter: Option<&crate::validation_rate_limit::ValidationRateLimiter>,
max_body_len: usize,
) {
let fp = validation_dedup_key(m);
let timeout_result = time::timeout(validation_timeout, async {
@ -346,6 +359,7 @@ pub async fn validate_single_match(
validation_timeout,
validation_retries,
rate_limiter,
max_body_len,
)
.await
})
@ -388,6 +402,7 @@ async fn timed_validate_single_match<'a>(
validation_timeout: Duration,
validation_retries: u32,
rate_limiter: Option<&crate::validation_rate_limit::ValidationRateLimiter>,
max_body_len: usize,
) {
// Select the appropriate HTTP client based on rule's TLS mode preference
let rule_tls_mode = m.rule.tls_mode();
@ -715,10 +730,7 @@ async fn timed_validate_single_match<'a>(
return;
}
};
// Validate against the full response body, but keep a truncated preview for
// reporting/storage to avoid huge outputs.
let mut display_body = body.clone();
truncate_to_char_boundary(&mut display_body, MAX_VALIDATION_BODY_LEN);
let display_body = truncate_preview(&body, max_body_len);
m.validation_response_status = status;
let body_opt = validation_body::from_string(display_body.clone());
@ -836,7 +848,9 @@ async fn timed_validate_single_match<'a>(
} else if body.as_bytes().contains(&0) {
body = format!("grpc-status={grpc_status} grpc-message={grpc_message}");
}
truncate_to_char_boundary(&mut body, MAX_VALIDATION_BODY_LEN);
if max_body_len > 0 {
truncate_to_char_boundary(&mut body, max_body_len);
}
m.validation_response_status = status;
m.validation_response_body = validation_body::from_string(body.clone());
@ -1445,16 +1459,36 @@ mod tests {
#[test]
fn truncate_to_char_boundary_handles_multibyte_characters() {
let mut body = "a".repeat(MAX_VALIDATION_BODY_LEN);
let max_len = 2048;
let mut body = "a".repeat(max_len);
body.push('é');
truncate_to_char_boundary(&mut body, MAX_VALIDATION_BODY_LEN);
truncate_to_char_boundary(&mut body, max_len);
assert_eq!(body.len(), MAX_VALIDATION_BODY_LEN);
assert_eq!(body.len(), max_len);
assert!(body.is_char_boundary(body.len()));
assert!(body.ends_with('a'));
}
#[test]
fn truncate_skipped_when_max_body_len_is_zero() {
let original_len = 4096;
let body = "x".repeat(original_len);
let preview = truncate_preview(&body, 0);
assert_eq!(preview.len(), original_len);
}
#[test]
fn truncate_applies_custom_max_body_len() {
let body = "y".repeat(5000);
let preview = truncate_preview(&body, 1024);
assert_eq!(preview.len(), 1024);
}
mod tls_mode_tests {
use super::*;

View file

@ -184,6 +184,7 @@ fn run_skiplist(skip_regex: Vec<String>, skip_skipword: Vec<String>) -> Result<u
validation_rps_rule: Vec::new(),
validation_timeout: 10,
full_validation_response: false,
max_validation_response_length: 2048,
};
let global_args = GlobalArgs {

View file

@ -169,6 +169,7 @@ fn test_bitbucket_remote_scan() -> Result<()> {
validation_rps_rule: Vec::new(),
validation_timeout: 10,
full_validation_response: false,
max_validation_response_length: 2048,
};
let global_args = GlobalArgs {

View file

@ -189,6 +189,7 @@ rules:
validation_rps_rule: Vec::new(),
validation_timeout: 10,
full_validation_response: false,
max_validation_response_length: 2048,
};
let global_args = GlobalArgs {

View file

@ -176,6 +176,7 @@ fn test_github_remote_scan() -> Result<()> {
validation_rps_rule: Vec::new(),
validation_timeout: 10,
full_validation_response: false,
max_validation_response_length: 2048,
};
// Create global arguments
let global_args = GlobalArgs {

View file

@ -174,6 +174,7 @@ fn test_gitlab_remote_scan() -> Result<()> {
validation_rps_rule: Vec::new(),
validation_timeout: 10,
full_validation_response: false,
max_validation_response_length: 2048,
};
let global_args = GlobalArgs {
@ -348,6 +349,7 @@ fn test_gitlab_remote_scan_no_history() -> Result<()> {
validation_rps_rule: Vec::new(),
validation_timeout: 10,
full_validation_response: false,
max_validation_response_length: 2048,
};
let global_args = GlobalArgs {

View file

@ -152,6 +152,7 @@ async fn test_redact_hashes_finding_values() -> Result<()> {
validation_rps_rule: Vec::new(),
validation_timeout: 10,
full_validation_response: false,
max_validation_response_length: 2048,
};
let global_args = GlobalArgs {

View file

@ -157,6 +157,7 @@ impl TestContext {
validation_rps_rule: Vec::new(),
validation_timeout: 10,
full_validation_response: false,
max_validation_response_length: 2048,
};
let loaded = RuleLoader::from_rule_specifiers(&scan_args.rules).load(&scan_args)?;
@ -317,6 +318,7 @@ async fn test_scan_slack_messages() -> Result<()> {
validation_rps_rule: Vec::new(),
validation_timeout: 10,
full_validation_response: false,
max_validation_response_length: 2048,
};
let global_args = GlobalArgs {

View file

@ -192,6 +192,7 @@ async fn test_scan_teams_messages() -> Result<()> {
validation_rps_rule: Vec::new(),
validation_timeout: 10,
full_validation_response: false,
max_validation_response_length: 2048,
};
let global_args = GlobalArgs {

View file

@ -232,6 +232,7 @@ async fn test_validation_cache_and_depvars() -> Result<()> {
validation_rps_rule: Vec::new(),
validation_timeout: 10,
full_validation_response: false,
max_validation_response_length: 2048,
};
/* --------------------------------------------------------- *

View file

@ -175,6 +175,7 @@ impl TestContext {
validation_rps_rule: Vec::new(),
validation_timeout: 10,
full_validation_response: false,
max_validation_response_length: 2048,
};
let loaded = RuleLoader::from_rule_specifiers(&scan_args.rules)
@ -322,6 +323,7 @@ impl TestContext {
validation_rps_rule: Vec::new(),
validation_timeout: 10,
full_validation_response: false,
max_validation_response_length: 2048,
};
let global_args = GlobalArgs {