From e53ad9f3094dc666208482863513f6f32ee44397 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sat, 9 Aug 2025 08:45:27 -0700 Subject: [PATCH 1/4] Added X Consumer key detection and validation --- data/rules/twitter.yml | 59 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/data/rules/twitter.yml b/data/rules/twitter.yml index 2722fc1..419c7c4 100644 --- a/data/rules/twitter.yml +++ b/data/rules/twitter.yml @@ -1,6 +1,6 @@ rules: - name: X / Twitter Bearer Token (App-only) - id: kingfisher.twitter.bearer.1 + id: kingfisher.twitter.1 pattern: | (?xi) \b @@ -36,3 +36,60 @@ rules: match_all_words: true references: - https://developer.x.com/en/docs/x-api/v1/developer-utilities/rate-limit-status/api-reference/get-application-rate_limit_status + - name: Twitter Consumer Key + id: kingfisher.twitter.2 + pattern: | + (?xi) + \b + twitter + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9]{25} + ) + \b + min_entropy: 3.5 + examples: + - "TWITTER_KEY=4RTBCyG2TbvL407A1lWxQFKCC" + - name: X / Twitter Consumer Secret + id: kingfisher.twitter.3 + pattern: | + (?xi) + \b + twitter + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9]{50} + ) + \b + min_entropy: 4.5 + examples: + - "TWITTER_SECRET=ZGwXeK2DNCqv49Z9ofwYdqlBgeoHDyh8uoAgHju6OeYC7wTQJq" + references: + - https://developer.x.com/en/docs/authentication/oauth-2-0/application-only + validation: + type: Http + content: + request: + method: POST + url: https://api.twitter.com/oauth2/token + headers: + Authorization: "Basic {{ TWITTER_KEY | append: ':' | append: TOKEN | b64enc }}" + Content-Type: "application/x-www-form-urlencoded;charset=UTF-8" + body: "grant_type=client_credentials" + response_matcher: + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"token_type":"bearer"' + - '"access_token":' + match_all_words: true + depends_on_rule: + - rule_id: "kingfisher.twitter.2" + variable: TWITTER_KEY From 229a66655c17369b2d62fa31b9a9717f99bf04ff Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sat, 9 Aug 2025 08:46:07 -0700 Subject: [PATCH 2/4] Added X Consumer key detection and validation --- CHANGELOG.md | 3 +++ Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 608d824..8ecd444 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All notable changes to this project will be documented in this file. +## [1.38.0] +- Added X Consumer key detection and validation + ## [1.37.0] - GitLab: Matched GitLab group repository listings to glab by only enumerating projects that belong directly to each group, without automatically traversing subgroups diff --git a/Cargo.toml b/Cargo.toml index aec70d4..5eb52fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ publish = false [package] name = "kingfisher" -version = "1.37.0" +version = "1.38.0" description = "MongoDB's blazingly fast secret scanning and validation tool" edition.workspace = true rust-version.workspace = true From c9c0aba6876ddec193a2c44f2c0aea4b27be4ee4 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sat, 9 Aug 2025 15:36:12 -0700 Subject: [PATCH 3/4] - --quiet now suppresses scan summaries and rule statistics unless --rule-stats is explicitly provided - Added X Consumer key detection and validation --- CHANGELOG.md | 1 + src/scanner/summary.rs | 78 +++++++++++++++++++++++------------------- tests/int_quiet.rs | 59 ++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 35 deletions(-) create mode 100644 tests/int_quiet.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ecd444..b8d7720 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ All notable changes to this project will be documented in this file. ## [1.38.0] +- `--quiet` now suppresses scan summaries and rule statistics unless `--rule-stats` is explicitly provided - Added X Consumer key detection and validation ## [1.37.0] diff --git a/src/scanner/summary.rs b/src/scanner/summary.rs index 7555d22..9d4f30e 100644 --- a/src/scanner/summary.rs +++ b/src/scanner/summary.rs @@ -38,14 +38,50 @@ macro_rules! safe_println { pub fn print_scan_summary( start_time: Instant, datastore: &Arc>, - _global_args: &global::GlobalArgs, + global_args: &global::GlobalArgs, args: &scan::ScanArgs, // inputs: &FilesystemEnumeratorResult, rules_db: &RulesDatabase, matcher_stats: &Mutex, profiler: Option<&ConcurrentRuleProfiler>, ) { - // let duration = start_time.elapsed(); + if global_args.quiet { + if args.rule_stats { + if let Some(prof) = profiler { + let stats = prof.generate_report(); + if !stats.is_empty() { + let name_w = stats.iter().map(|s| s.rule_name.len()).max().unwrap_or(4); + let id_w = stats.iter().map(|s| s.rule_id.len()).max().unwrap_or(2); + safe_println!("\n{:-^1$}", " Rule Performance Stats ", name_w + id_w + 47); + safe_println!( + "{: 8} {: >15} {: >15}", + "Rule", + "ID", + "Matches", + "Slowest", + "Average", + name_w = name_w, + id_w = id_w + ); + safe_println!("{:-8} {: >15?} {: >15?}", + rs.rule_name, + rs.rule_id, + rs.total_matches, + rs.slowest_match_time, + rs.average_match_time, + name_w = name_w, + id_w = id_w + ); + } + } + } + } + return; + } + let ds = datastore.lock().unwrap(); let num_rules = rules_db.num_rules(); @@ -53,17 +89,12 @@ pub fn print_scan_summary( let mut sorted_findings: Vec<_> = findings_by_rule.into_iter().collect(); sorted_findings.sort_by(|a, b| b.1.cmp(&a.1)); let duration = start_time.elapsed(); - // let ds = datastore.lock().unwrap(); - // Get all matches let all_matches = ds.get_matches(); - // Count total findings let total_findings = if args.no_dedup { - // When no_dedup is true, count each origin of validated matches as a separate finding all_matches.iter().fold(0, |count, msg| { let (origin_set, _, match_item) = &**msg; - // If this is a validated match, count each origin as a separate finding if match_item.validation_success { count + origin_set.len() } else { @@ -73,14 +104,13 @@ pub fn print_scan_summary( } else { ds.get_num_matches() }; - // Count successful and failed validations + let (successful_validations, failed_validations) = all_matches.iter().fold((0, 0), |(success, fail), msg| { let (origin_set, _, match_item) = &**msg; if match_item.validation_success { if match_item.validation_response_status != StatusCode::CONTINUE.as_u16() { if args.no_dedup { - // Count each origin of a successful validation as a separate success (success + origin_set.len(), fail) } else { (success + 1, fail) @@ -88,17 +118,14 @@ pub fn print_scan_summary( } else { (success, fail) } + } else if match_item.validation_response_status != StatusCode::CONTINUE.as_u16() { + (success, fail + 1) } else { - if match_item.validation_response_status != StatusCode::CONTINUE.as_u16() { - (success, fail + 1) - } else { - (success, fail) - } + (success, fail) } }); let matcher_stats = matcher_stats.lock().unwrap(); - // Generate JSON or JSONL output if args.output_args.format == ReportOutputFormat::Json || args.output_args.format == ReportOutputFormat::Jsonl { @@ -107,15 +134,11 @@ pub fn print_scan_summary( "successful_validations": successful_validations, "failed_validations": failed_validations, "rules_applied": num_rules, - // "git_repositories": num_git_repos, - // "commits": num_commits, "blobs_scanned": matcher_stats.blobs_scanned, - // "files_read": num_files, "bytes_scanned": matcher_stats.bytes_scanned, "scan_duration": duration.as_secs_f64(), "findings_by_rule": sorted_findings }); - // only printing to stdout, not to the file itself safe_println!("{}", summary.to_string()); } else if args.output_args.format == ReportOutputFormat::Pretty || args.output_args.output.is_some() @@ -133,37 +156,23 @@ pub fn print_scan_summary( failed_validations.separate_with_commas() ); safe_println!(" |Rules Applied...............: {}", num_rules.separate_with_commas()); - // safe_println!(" |Git Repositories............: {}", - // num_git_repos.separate_with_commas()); safe_println!( - // "|__Commits...................: {}", - // num_commits.separate_with_commas() - // ); safe_println!( " |__Blobs Scanned.............: {}", matcher_stats.blobs_scanned.separate_with_commas() ); - // safe_println!(" |Files Read..................: {}", - // num_files.separate_with_commas()); safe_println!( " |Bytes Scanned...............: {}", HumanBytes(matcher_stats.bytes_scanned) ); - safe_println!( - " |Scan Duration...............: {}", - // HumanDuration(duration), - humantime::format_duration(duration) - ); + safe_println!(" |Scan Duration...............: {}", humantime::format_duration(duration)); } if args.rule_stats { if let Some(prof) = profiler { let stats = prof.generate_report(); if !stats.is_empty() { - // Calculate dynamic column widths let name_w = stats.iter().map(|s| s.rule_name.len()).max().unwrap_or(4); let id_w = stats.iter().map(|s| s.rule_id.len()).max().unwrap_or(2); - - // Header safe_println!("\n{:-^1$}", " Rule Performance Stats ", name_w + id_w + 47); safe_println!( "{: 8} {: >15} {: >15}", @@ -177,7 +186,6 @@ pub fn print_scan_summary( ); safe_println!("{:-8} {: >15?} {: >15?}", diff --git a/tests/int_quiet.rs b/tests/int_quiet.rs new file mode 100644 index 0000000..598f103 --- /dev/null +++ b/tests/int_quiet.rs @@ -0,0 +1,59 @@ +use assert_cmd::Command; +use predicates::prelude::*; + +const FORMATS: [&str; 4] = ["pretty", "json", "jsonl", "bson"]; + +fn contains_bytes(haystack: &[u8], needle: &[u8]) -> bool { + haystack.windows(needle.len()).any(|window| window == needle) +} + +#[test] +fn scan_quiet_suppresses_summary() { + for format in FORMATS { + Command::cargo_bin("kingfisher") + .unwrap() + .env("NO_COLOR", "1") + .args([ + "scan", + "testdata/slack_tokens.properties", + "--confidence=low", + "--format", + format, + "--no-update-check", + "--no-validate", + "--quiet", + ]) + .assert() + .code(200) + .stdout(predicate::function(|out: &[u8]| !contains_bytes(out, b"Scan Summary"))) + .stdout(predicate::function(|out: &[u8]| { + !contains_bytes(out, b"Rule Performance Stats") + })); + } +} + +#[test] +fn scan_quiet_with_rule_stats_prints_rule_stats() { + for format in FORMATS { + Command::cargo_bin("kingfisher") + .unwrap() + .env("NO_COLOR", "1") + .args([ + "scan", + "testdata/slack_tokens.properties", + "--confidence=low", + "--format", + format, + "--no-update-check", + "--quiet", + "--no-validate", + "--rule-stats", + ]) + .assert() + .code(200) + .stdout(predicate::function(|out: &[u8]| !contains_bytes(out, b"Scan Summary"))) + .stdout(predicate::function(|out: &[u8]| { + contains_bytes(out, b"Rule Performance Stats") + })); + } +} \ No newline at end of file From 979bc469c56b09818f4187f2e6ef207196759f8b Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sat, 9 Aug 2025 15:52:00 -0700 Subject: [PATCH 4/4] - --quiet now suppresses scan summaries and rule statistics unless --rule-stats is explicitly provided - Added X Consumer key detection and validation --- data/rules/twitter.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/data/rules/twitter.yml b/data/rules/twitter.yml index 419c7c4..e75b48e 100644 --- a/data/rules/twitter.yml +++ b/data/rules/twitter.yml @@ -51,6 +51,7 @@ rules: ) \b min_entropy: 3.5 + visible: false examples: - "TWITTER_KEY=4RTBCyG2TbvL407A1lWxQFKCC" - name: X / Twitter Consumer Secret