From bcf2b60e0bf5816159d719b399591d0f44a84e8a Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Tue, 29 Jul 2025 19:00:49 -0700 Subject: [PATCH 1/5] Added support for Slack --- .github/workflows/release.yml | 1 - CHANGELOG.md | 3 + Cargo.toml | 2 +- README.md | 19 ++++-- src/cli/commands/inputs.rs | 14 +++- src/findings_store.rs | 10 +++ src/lib.rs | 1 + src/main.rs | 4 ++ src/reporter.rs | 16 +++-- src/reporter/json_format.rs | 8 +++ src/reporter/pretty_format.rs | 6 ++ src/reporter/sarif_format.rs | 4 ++ src/scanner/repos.rs | 34 +++++++++- src/scanner/runner.rs | 6 +- src/slack.rs | 118 ++++++++++++++++++++++++++++++++++ tests/int_dedup.rs | 2 + tests/int_github.rs | 2 + tests/int_gitlab.rs | 2 + tests/int_validation_cache.rs | 2 + tests/int_vulnerable_files.rs | 4 ++ 20 files changed, 240 insertions(+), 18 deletions(-) create mode 100644 src/slack.rs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 56fb028..956bc1a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -189,7 +189,6 @@ jobs: - name: Cache vcpkg artifacts uses: actions/cache@v3 with: - # Adjust these paths if your vcpkg root is somewhere else path: | C:\vcpkg\buildtrees C:\vcpkg\packages diff --git a/CHANGELOG.md b/CHANGELOG.md index 5556a43..5c1e055 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All notable changes to this project will be documented in this file. +## [1.28.0] +- Added support for scanning Slack + ## [1.27.0] - Added Buildkite rule - Added support for scanning Docker images via `--docker-image` diff --git a/Cargo.toml b/Cargo.toml index 79595e8..fa9cf8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ publish = false [package] name = "kingfisher" -version = "1.27.0" +version = "1.28.0" description = "MongoDB's blazingly fast secret scanning and validation tool" edition.workspace = true rust-version.workspace = true diff --git a/README.md b/README.md index ad65c78..d459b5b 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,11 @@ Kingfisher extends Nosey Parker by: 1. **Validating secrets** in real time via cloud-provider APIs 2. Enhancing regex-based detection with **source-code parsing** for improved accuracy 3. Adding **GitLab** repository scanning support -4. Adding support for scanning **Docker** images via `--docker-image` +4. Adding support for scanning **Docker** images 5. Providing **Jira** scanning capabilities -6. Introducing a baseline feature that suppresses known secrets and reports only newly introduced ones -7. Offering native **Windows** support +6. Adding **Slack** scanning capabilities +7. Introducing a baseline feature that suppresses known secrets and reports only newly introduced ones +8. Offering native **Windows** support **MongoDB Blog**: [Introducing Kingfisher: Real-Time Secret Detection and Validation](https://www.mongodb.com/blog/post/product-release-announcements/introducing-kingfisher-real-time-secret-detection-validation) @@ -29,6 +30,7 @@ Kingfisher extends Nosey Parker by: - **Built-In Validation**: Hundreds of built-in detection rules, many with live-credential validators that call the relevant service APIs (AWS, Azure, GCP, Stripe, etc.) to confirm a secret is active. You can extend or override the library by adding YAML-defined rules on the command line—see [docs/RULES.md](/docs/RULES.md) for details - **Git History Scanning**: Scan local repos, remote GitHub/GitLab orgs/users, or arbitrary GitHub/GitLab repos - **Jira Scanning**: Scan issues returned from a JQL search using `--jira-url` and `--jql` +- **Slack Scanning**: Scan messages returned from a Slack search query using `--slack-query` - **Docker Image Scanning**: Scan public or private docker images via `--docker-image` - **Baseline Support:** Generate and manage baseline files to ignore known secrets and report only newly introduced ones. See ([docs/BASELINE.md](docs/BASELINE.md)) for details. @@ -353,7 +355,16 @@ KF_JIRA_TOKEN="token" kingfisher scan \ --max-results 1000 ``` --- +## Scanning Slack +### Scan Slack messages matching a search query + +```bash +KF_SLACK_TOKEN="token" kingfisher scan \ + --slack-query "from:username has:link" \ + --max-results 1000 +``` +*The Slack token must be a user token with the `search:read` scope. Bot tokens (those beginning with `xoxb-`) cannot call the Slack search API.* ## Environment Variables for Tokens @@ -362,8 +373,8 @@ KF_JIRA_TOKEN="token" kingfisher scan \ | `KF_GITHUB_TOKEN` | GitHub Personal Access Token | | `KF_GITLAB_TOKEN` | GitLab Personal Access Token | | `KF_JIRA_TOKEN` | Jira API token | +| `KF_SLACK_TOKEN` | Slack API token | | `KF_DOCKER_TOKEN` | Docker registry token (`user:pass` or bearer token). If unset, credentials from the Docker keychain are used | - Set them temporarily per command: ```bash diff --git a/src/cli/commands/inputs.rs b/src/cli/commands/inputs.rs index f698d87..8a1c23d 100644 --- a/src/cli/commands/inputs.rs +++ b/src/cli/commands/inputs.rs @@ -27,7 +27,8 @@ pub struct InputSpecifierArgs { "all_github_organizations", "all_gitlab_groups", "jira_url", - "docker_image" + "docker_image", + "slack_query" ]), value_hint = ValueHint::AnyPath )] @@ -94,7 +95,15 @@ pub struct InputSpecifierArgs { #[arg(long, requires = "jira_url")] pub jql: Option, - /// Maximum number of Jira results to fetch + /// Slack search query + #[arg(long)] + pub slack_query: Option, + + /// Use the specified URL for Slack API access + #[arg(long, default_value = "https://slack.com/api/", value_hint = ValueHint::Url)] + pub slack_api_url: Url, + + /// Maximum number of Slack or Jira results to fetch #[arg(long, default_value_t = 100)] pub max_results: usize, @@ -102,7 +111,6 @@ pub struct InputSpecifierArgs { #[arg(long = "docker-image")] pub docker_image: Vec, - /// Select how to clone Git repositories #[arg(long, default_value_t=GitCloneMode::Bare, alias="git-clone-mode")] pub git_clone: GitCloneMode, diff --git a/src/findings_store.rs b/src/findings_store.rs index 5972490..93e9f1c 100644 --- a/src/findings_store.rs +++ b/src/findings_store.rs @@ -53,6 +53,7 @@ pub struct FindingsStore { blob_meta: FxHashMap>, origin_meta: FxHashMap>, docker_images: FxHashMap, + slack_links: FxHashMap, } impl FindingsStore { pub fn new(clone_dir: PathBuf) -> Self { @@ -71,6 +72,7 @@ impl FindingsStore { seen_bloom, bloom_items: 0, docker_images: FxHashMap::default(), + slack_links: FxHashMap::default(), } } @@ -296,6 +298,14 @@ impl FindingsStore { &self.docker_images } + pub fn register_slack_message(&mut self, path: PathBuf, permalink: String) { + self.slack_links.insert(path, permalink); + } + + pub fn slack_links(&self) -> &FxHashMap { + &self.slack_links + } + pub fn get_finding_data_iter( &self, ) -> impl Iterator + '_ { diff --git a/src/lib.rs b/src/lib.rs index af74e7b..85bc57c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,6 +33,7 @@ pub mod safe_list; pub mod scanner; pub mod scanner_pool; pub mod serde_utils; +pub mod slack; pub mod snippet; pub mod update; pub mod util; diff --git a/src/main.rs b/src/main.rs index a85fb48..56f1e15 100644 --- a/src/main.rs +++ b/src/main.rs @@ -282,6 +282,10 @@ fn create_default_scan_args() -> cli::commands::scan::ScanArgs { jira_url: None, jql: None, max_results: 100, + // Slack query + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), + // Docker image scanning docker_image: Vec::new(), diff --git a/src/reporter.rs b/src/reporter.rs index ca69c97..e6709dc 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -122,13 +122,8 @@ impl DetailsReporter { args: &cli::commands::scan::ScanArgs, ) -> Option { // drop any trailing slash so we don’t end up with “//browse/…” - let jira_url = args - .input_specifier_args - .jira_url - .as_ref()? - .as_str() - .trim_end_matches('/'); - + let jira_url = args.input_specifier_args.jira_url.as_ref()?.as_str().trim_end_matches('/'); + let ds = self.datastore.lock().ok()?; let root = ds.clone_root(); let jira_dir = root.join("jira_issues"); @@ -140,6 +135,13 @@ impl DetailsReporter { } } + /// If the given file path corresponds to a Slack message downloaded to disk, + /// return the permalink for that message. + fn slack_message_url(&self, path: &std::path::Path) -> Option { + let ds = self.datastore.lock().ok()?; + ds.slack_links().get(path).cloned() + } + fn docker_display_path(&self, path: &std::path::Path) -> Option { let ds = self.datastore.lock().ok()?; for (dir, image) in ds.docker_images().iter() { diff --git a/src/reporter/json_format.rs b/src/reporter/json_format.rs index 6916337..6bad0cb 100644 --- a/src/reporter/json_format.rs +++ b/src/reporter/json_format.rs @@ -103,6 +103,8 @@ impl DetailsReporter { if let Origin::File(e) = origin { if let Some(url) = self.jira_issue_url(&e.path, args) { Some(url) + } else if let Some(url) = self.slack_message_url(&e.path) { + Some(url) } else if let Some(mapped) = self.docker_display_path(&e.path) { Some(mapped) } else { @@ -254,6 +256,8 @@ impl DetailsReporter { if let Origin::File(e) = origin { if let Some(url) = self.jira_issue_url(&e.path, args) { Some(url) + } else if let Some(url) = self.slack_message_url(&e.path) { + Some(url) } else if let Some(mapped) = self.docker_display_path(&e.path) { Some(mapped) } else { @@ -434,6 +438,10 @@ mod tests { jql: None, max_results: 100, // Docker image scanning + // Slack options + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), + docker_image: Vec::new(), // clone / history options git_clone: GitCloneMode::Bare, diff --git a/src/reporter/pretty_format.rs b/src/reporter/pretty_format.rs index ffa7cf0..62dd354 100644 --- a/src/reporter/pretty_format.rs +++ b/src/reporter/pretty_format.rs @@ -216,6 +216,8 @@ impl<'a> Display for PrettyFinding<'a> { Origin::File(e) => { let display_path = if let Some(url) = reporter.jira_issue_url(&e.path, args) { url + } else if let Some(url) = reporter.slack_message_url(&e.path) { + url } else if let Some(mapped) = reporter.docker_display_path(&e.path) { mapped } else { @@ -347,6 +349,10 @@ fn test_pretty_format_with_nan_entropy_panics() { jira_url: None, jql: None, max_results: 100, + + // Slack options + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), // Docker image scanning docker_image: Vec::new(), // git clone / history options diff --git a/src/reporter/sarif_format.rs b/src/reporter/sarif_format.rs index f771c17..3db2d99 100644 --- a/src/reporter/sarif_format.rs +++ b/src/reporter/sarif_format.rs @@ -73,6 +73,8 @@ impl DetailsReporter { Origin::File(e) => { let uri = if let Some(url) = self.jira_issue_url(&e.path, args) { url + } else if let Some(url) = self.slack_message_url(&e.path) { + url } else { e.path.display().to_string() }; @@ -209,6 +211,8 @@ impl DetailsReporter { let uri = if let Some(url) = self.jira_issue_url(&e.path, args) { url + } else if let Some(url) = self.slack_message_url(&e.path) { + url } else { e.path.display().to_string() }; diff --git a/src/scanner/repos.rs b/src/scanner/repos.rs index f84b758..2a8044a 100644 --- a/src/scanner/repos.rs +++ b/src/scanner/repos.rs @@ -23,7 +23,7 @@ use crate::{ github, gitlab, jira, matcher::Match, origin::OriginSet, - PathBuf, + slack, PathBuf, }; pub type DatastoreMessage = (OriginSet, BlobMetadata, Vec<(Option, Match)>); @@ -252,4 +252,36 @@ pub async fn fetch_jira_issues( ) .await?; Ok(vec![output_dir]) +} + +pub async fn fetch_slack_messages( + args: &scan::ScanArgs, + global_args: &global::GlobalArgs, + datastore: &Arc>, +) -> Result> { + let Some(query) = args.input_specifier_args.slack_query.as_deref() else { + return Ok(Vec::new()); + }; + let api_url = args.input_specifier_args.slack_api_url.clone(); + let max_results = args.input_specifier_args.max_results; + let output_root = { + let ds = datastore.lock().unwrap(); + ds.clone_root() + }; + let output_dir = output_root.join("slack_messages"); + let paths = slack::download_messages_to_dir( + api_url, + query, + max_results, + global_args.ignore_certs, + &output_dir, + ) + .await?; + { + let mut ds = datastore.lock().unwrap(); + for (path, link) in &paths { + ds.register_slack_message(path.clone(), link.clone()); + } + } + Ok(vec![output_dir]) } \ No newline at end of file diff --git a/src/scanner/runner.rs b/src/scanner/runner.rs index f1271cf..b727718 100644 --- a/src/scanner/runner.rs +++ b/src/scanner/runner.rs @@ -18,7 +18,7 @@ use crate::{ rules_database::RulesDatabase, scanner::{ clone_or_update_git_repos, enumerate_filesystem_inputs, enumerate_github_repos, - repos::{enumerate_gitlab_repos, fetch_jira_issues}, + repos::{enumerate_gitlab_repos, fetch_jira_issues, fetch_slack_messages}, run_secret_validation, save_docker_images, summary::print_scan_summary, }, @@ -68,6 +68,10 @@ pub async fn run_async_scan( let jira_dirs = fetch_jira_issues(args, global_args, &datastore).await?; input_roots.extend(jira_dirs); + // Fetch Slack messages if requested + let slack_dirs = fetch_slack_messages(args, global_args, &datastore).await?; + input_roots.extend(slack_dirs); + // Save Docker images if specified if !args.input_specifier_args.docker_image.is_empty() { let clone_root = { diff --git a/src/slack.rs b/src/slack.rs new file mode 100644 index 0000000..69a02a0 --- /dev/null +++ b/src/slack.rs @@ -0,0 +1,118 @@ +use anyhow::{Context, Result}; +use reqwest::Client; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; +use url::Url; + +#[derive(Debug, Serialize, Deserialize)] +pub struct SlackMessage { + pub permalink: String, + pub text: Option, + pub ts: String, + pub channel: SlackChannel, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct SlackChannel { + pub id: String, + pub name: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +struct SlackPagination { + page: Option, + page_count: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +struct SlackMessages { + matches: Vec, + pagination: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +struct SlackSearchResponse { + ok: bool, + error: Option, + messages: Option, +} + +pub async fn search_messages( + api_url: Url, + query: &str, + max_results: usize, + ignore_certs: bool, +) -> Result> { + let token = std::env::var("KF_SLACK_TOKEN") + .context("KF_SLACK_TOKEN environment variable must be set")?; + + let client = Client::builder() + .danger_accept_invalid_certs(ignore_certs) + .build() + .context("Failed to build HTTP client")?; + + let mut page = 1u32; + let mut messages = Vec::new(); + + loop { + let url = api_url.join("search.messages").context("Failed to build Slack API URL")?; + + let resp = client + .get(url) + .bearer_auth(&token) + .query(&[("query", query), ("count", "100"), ("page", &page.to_string())]) + .send() + .await + .context("Failed to send Slack request")?; + + let body: SlackSearchResponse = + resp.json().await.context("Failed to parse Slack response")?; + + if !body.ok { + let err = body.error.unwrap_or_else(|| "unknown".to_string()); + if err == "not_allowed_token_type" { + return Err(anyhow::anyhow!( + "Slack API error: not_allowed_token_type - use a user token with the `search:read` scope" + )); + } + return Err(anyhow::anyhow!("Slack API error: {}", err)); + } + + let Some(msgs) = body.messages else { + break; + }; + for m in msgs.matches { + messages.push(m); + if messages.len() >= max_results { + return Ok(messages); + } + } + let next_page = msgs.pagination.as_ref().and_then(|p| p.page).unwrap_or(page); + let page_count = msgs.pagination.as_ref().and_then(|p| p.page_count).unwrap_or(next_page); + if next_page >= page_count { + break; + } + page += 1; + } + + Ok(messages) +} + +pub async fn download_messages_to_dir( + api_url: Url, + query: &str, + max_results: usize, + ignore_certs: bool, + output_dir: &PathBuf, +) -> Result> { + std::fs::create_dir_all(output_dir)?; + let messages = search_messages(api_url, query, max_results, ignore_certs).await?; + let mut paths = Vec::new(); + for msg in messages { + let ts = msg.ts.replace('.', "_"); + let file = output_dir.join(format!("{}_{}.json", msg.channel.id, ts)); + std::fs::write(&file, serde_json::to_vec(&msg)?)?; + paths.push((file, msg.permalink)); + } + Ok(paths) +} \ No newline at end of file diff --git a/tests/int_dedup.rs b/tests/int_dedup.rs index 2763ebd..c42967f 100644 --- a/tests/int_dedup.rs +++ b/tests/int_dedup.rs @@ -82,6 +82,8 @@ rules: jira_url: None, jql: None, max_results: 100, + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), // Docker image scanning docker_image: Vec::new(), // git clone / history options diff --git a/tests/int_github.rs b/tests/int_github.rs index d4f7f25..4bda269 100644 --- a/tests/int_github.rs +++ b/tests/int_github.rs @@ -69,6 +69,8 @@ fn test_github_remote_scan() -> Result<()> { jira_url: None, jql: None, max_results: 100, + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), // Docker image scanning docker_image: Vec::new(), // git clone / history options diff --git a/tests/int_gitlab.rs b/tests/int_gitlab.rs index 67b1bc3..4087618 100644 --- a/tests/int_gitlab.rs +++ b/tests/int_gitlab.rs @@ -68,6 +68,8 @@ fn test_gitlab_remote_scan() -> Result<()> { jira_url: None, jql: None, max_results: 100, + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), // Docker image scanning docker_image: Vec::new(), git_clone: GitCloneMode::Bare, diff --git a/tests/int_validation_cache.rs b/tests/int_validation_cache.rs index 3e21947..ae8dd50 100644 --- a/tests/int_validation_cache.rs +++ b/tests/int_validation_cache.rs @@ -125,6 +125,8 @@ async fn test_validation_cache_and_depvars() -> Result<()> { jira_url: None, jql: None, max_results: 100, + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), // Docker image scanning docker_image: Vec::new(), // git clone / history options diff --git a/tests/int_vulnerable_files.rs b/tests/int_vulnerable_files.rs index ad78192..187427e 100644 --- a/tests/int_vulnerable_files.rs +++ b/tests/int_vulnerable_files.rs @@ -68,6 +68,8 @@ impl TestContext { jira_url: None, jql: None, max_results: 100, + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), // Docker image scanning docker_image: Vec::new(), // git clone / history options @@ -138,6 +140,8 @@ impl TestContext { jira_url: None, jql: None, max_results: 100, + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), // Docker image scanning docker_image: Vec::new(), // git clone / history options From 0f6f7abf371864aa1700636c09a8b28a35f5680e Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Tue, 29 Jul 2025 19:51:02 -0700 Subject: [PATCH 2/5] Added support for Slack --- README.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index d459b5b..5ade809 100644 --- a/README.md +++ b/README.md @@ -6,33 +6,29 @@ [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) Kingfisher is a blazingly fast secret‑scanning and validation tool built in Rust. It combines Intel’s hardware‑accelerated Hyperscan regex engine with language‑aware parsing via Tree‑Sitter, and **ships with hundreds of built‑in rules** to detect, validate, and triage secrets before they ever reach production -

-Kingfisher originated as a fork of [Nosey Parker](https://github.com/praetorian-inc/noseyparker) by Praetorian Security, Inc, and is built atop their incredible work and the work contributed by the Nosey Parker community. +Kingfisher originated as a fork of Praetorian's [Nosey Parker](https://github.com/praetorian-inc/noseyparker), and is built atop their incredible work and the work contributed by the Nosey Parker community. -Kingfisher extends Nosey Parker by: -1. **Validating secrets** in real time via cloud-provider APIs -2. Enhancing regex-based detection with **source-code parsing** for improved accuracy -3. Adding **GitLab** repository scanning support -4. Adding support for scanning **Docker** images -5. Providing **Jira** scanning capabilities -6. Adding **Slack** scanning capabilities -7. Introducing a baseline feature that suppresses known secrets and reports only newly introduced ones -8. Offering native **Windows** support +## What Kingfisher Adds +- **Live validation** via cloud-provider APIs +- **Language-aware detection** (AST parsing) for ~20 languages +- **Extra targets**: GitLab repos, Docker images, Jira issues, and Slack messages +- **Baseline mode**: ignore known secrets, flag only new ones +- **Native Windows** binaries -**MongoDB Blog**: [Introducing Kingfisher: Real-Time Secret Detection and Validation](https://www.mongodb.com/blog/post/product-release-announcements/introducing-kingfisher-real-time-secret-detection-validation) ## Key Features +- **Performance**: multithreaded, Hyperscan‑powered scanning built for huge codebases +- **Extensible rules**: hundreds of built-in detectors plus YAML-defined custom rules ([docs/RULES.md](/docs/RULES.md)) +- **Multiple targets**: + - **Git history**: local repos or GitHub/GitLab orgs/users + - **Docker images**: public or private via `--docker-image` + - **Jira issues**: JQL‑driven scans with `--jira-url` and `--jql` + - **Slack messages**: query‑based scans with `--slack-query` +- **Baseline management**: generate and track baselines to suppress known secrets ([docs/BASELINE.md](/docs/BASELINE.md)) -- **Performance**: Multi‑threaded, Hyperscan‑powered scanning for massive codebases -- **Language‑Aware Accuracy**: AST parsing in 20+ languages via Tree‑Sitter reduces contextless regex matches. see [docs/PARSING.md](/docs/PARSING.md) -- **Built-In Validation**: Hundreds of built-in detection rules, many with live-credential validators that call the relevant service APIs (AWS, Azure, GCP, Stripe, etc.) to confirm a secret is active. You can extend or override the library by adding YAML-defined rules on the command line—see [docs/RULES.md](/docs/RULES.md) for details -- **Git History Scanning**: Scan local repos, remote GitHub/GitLab orgs/users, or arbitrary GitHub/GitLab repos -- **Jira Scanning**: Scan issues returned from a JQL search using `--jira-url` and `--jql` -- **Slack Scanning**: Scan messages returned from a Slack search query using `--slack-query` -- **Docker Image Scanning**: Scan public or private docker images via `--docker-image` -- **Baseline Support:** Generate and manage baseline files to ignore known secrets and report only newly introduced ones. See ([docs/BASELINE.md](docs/BASELINE.md)) for details. +**Learn more:** [Introducing Kingfisher: Real‑Time Secret Detection and Validation](https://www.mongodb.com/blog/post/product-release-announcements/introducing-kingfisher-real-time-secret-detection-validation) # Getting Started ## Installation @@ -360,9 +356,13 @@ KF_JIRA_TOKEN="token" kingfisher scan \ ### Scan Slack messages matching a search query ```bash -KF_SLACK_TOKEN="token" kingfisher scan \ +KF_SLACK_TOKEN="xoxp-1234..." kingfisher scan \ --slack-query "from:username has:link" \ --max-results 1000 + +KF_SLACK_TOKEN="xoxp-1234..." kingfisher scan \ + --slack-query "akia" \ + --max-results 1000 ``` *The Slack token must be a user token with the `search:read` scope. Bot tokens (those beginning with `xoxb-`) cannot call the Slack search API.* From 1b427d97ca7348148332045a6b31eaa64fc725c2 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Tue, 29 Jul 2025 20:20:33 -0700 Subject: [PATCH 3/5] Added support for Slack. Wrote a basic integration test --- README.md | 4 +- tests/int_slack.rs | 198 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 tests/int_slack.rs diff --git a/README.md b/README.md index 5ade809..09fabc1 100644 --- a/README.md +++ b/README.md @@ -12,10 +12,10 @@ Kingfisher originated as a fork of Praetorian's [Nosey Parker](https://github.co ## What Kingfisher Adds - **Live validation** via cloud-provider APIs -- **Language-aware detection** (AST parsing) for ~20 languages +- **Language-aware detection** (source-code parsing) for ~20 languages - **Extra targets**: GitLab repos, Docker images, Jira issues, and Slack messages - **Baseline mode**: ignore known secrets, flag only new ones -- **Native Windows** binaries +- **Native Windows** binary ## Key Features diff --git a/tests/int_slack.rs b/tests/int_slack.rs new file mode 100644 index 0000000..d238bce --- /dev/null +++ b/tests/int_slack.rs @@ -0,0 +1,198 @@ +use std::{ + env, + sync::{Arc, Mutex}, +}; + +use anyhow::Result; +use kingfisher::{ + cli::{ + commands::{ + github::{GitCloneMode, GitHistoryMode, GitHubRepoType}, + gitlab::GitLabRepoType, + inputs::{ContentFilteringArgs, InputSpecifierArgs}, + output::{OutputArgs, ReportOutputFormat}, + rules::RuleSpecifierArgs, + scan::{ConfidenceLevel, ScanArgs}, + }, + global::{AdvancedArgs, Mode}, + GlobalArgs, + }, + findings_store::FindingsStore, + rule_loader::RuleLoader, + rules_database::RulesDatabase, + scanner::run_async_scan, +}; +use tempfile::TempDir; +use url::Url; +use wiremock::{ + matchers::{method, path}, + Mock, MockServer, ResponseTemplate, +}; + +struct TestContext { + rules_db: Arc, +} + +impl TestContext { + fn new() -> Result { + let scan_args = ScanArgs { + num_jobs: 2, + rules: RuleSpecifierArgs { + rules_path: Vec::new(), + rule: vec!["all".into()], + load_builtins: true, + }, + input_specifier_args: InputSpecifierArgs { + path_inputs: Vec::new(), + git_url: Vec::new(), + github_user: Vec::new(), + github_organization: Vec::new(), + all_github_organizations: false, + github_api_url: Url::parse("https://api.github.com/").unwrap(), + github_repo_type: GitHubRepoType::Source, + gitlab_user: Vec::new(), + gitlab_group: Vec::new(), + all_gitlab_groups: false, + gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), + gitlab_repo_type: GitLabRepoType::Owner, + jira_url: None, + jql: None, + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), + max_results: 10, + docker_image: Vec::new(), + git_clone: GitCloneMode::Bare, + git_history: GitHistoryMode::Full, + scan_nested_repos: true, + commit_metadata: true, + }, + content_filtering_args: ContentFilteringArgs { + max_file_size_mb: 25.0, + extraction_depth: 2, + no_binary: true, + no_extract_archives: false, + exclude: Vec::new(), + }, + confidence: ConfidenceLevel::Low, + no_validate: true, + rule_stats: false, + only_valid: false, + min_entropy: Some(0.0), + redact: false, + git_repo_timeout: 1800, + output_args: OutputArgs { output: None, format: ReportOutputFormat::Pretty }, + no_dedup: true, + snippet_length: 128, + baseline_file: None, + manage_baseline: false, + }; + + let loaded = RuleLoader::from_rule_specifiers(&scan_args.rules).load(&scan_args)?; + let resolved = loaded.resolve_enabled_rules()?; + let rules_db = RulesDatabase::from_rules(resolved.into_iter().cloned().collect())?; + Ok(Self { rules_db: Arc::new(rules_db) }) + } +} + +#[tokio::test] +async fn test_scan_slack_messages() -> Result<()> { + let ctx = TestContext::new()?; + + let server = MockServer::start().await; + let response = serde_json::json!({ + "ok": true, + "messages": { + "matches": [{ + "permalink": "https://example.slack.com/archives/C123/p1234", + "text": "This contains a github token ghp_1wuHFikBKQtCcH3EB2FBUkyn8krXhP2qLqPa", + "ts": "1234.56", + "channel": {"id": "C123", "name": "general"} + }], + "pagination": {"page": 1, "page_count": 1} + } + }); + Mock::given(method("GET")) + .and(path("/search.messages")) + .respond_with(ResponseTemplate::new(200).set_body_json(response)) + .mount(&server) + .await; + + env::set_var("KF_SLACK_TOKEN", "xoxp-test"); + + let temp_dir = TempDir::new()?; + let clone_dir = temp_dir.path().to_path_buf(); + + let scan_args = ScanArgs { + num_jobs: 2, + rules: RuleSpecifierArgs { + rules_path: Vec::new(), + rule: vec!["all".into()], + load_builtins: true, + }, + input_specifier_args: InputSpecifierArgs { + path_inputs: Vec::new(), + git_url: Vec::new(), + github_user: Vec::new(), + github_organization: Vec::new(), + all_github_organizations: false, + github_api_url: Url::parse("https://api.github.com/").unwrap(), + github_repo_type: GitHubRepoType::Source, + gitlab_user: Vec::new(), + gitlab_group: Vec::new(), + all_gitlab_groups: false, + gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), + gitlab_repo_type: GitLabRepoType::Owner, + jira_url: None, + jql: None, + slack_query: Some("test".into()), + slack_api_url: Url::parse(&format!("{}/", server.uri()))?, + max_results: 10, + docker_image: Vec::new(), + git_clone: GitCloneMode::Bare, + git_history: GitHistoryMode::Full, + scan_nested_repos: true, + commit_metadata: true, + }, + content_filtering_args: ContentFilteringArgs { + max_file_size_mb: 25.0, + extraction_depth: 2, + no_binary: true, + no_extract_archives: false, + exclude: Vec::new(), + }, + confidence: ConfidenceLevel::Low, + no_validate: true, + rule_stats: false, + only_valid: false, + min_entropy: Some(0.0), + redact: false, + git_repo_timeout: 1800, + output_args: OutputArgs { output: None, format: ReportOutputFormat::Pretty }, + no_dedup: true, + snippet_length: 128, + baseline_file: None, + manage_baseline: false, + }; + + let global_args = GlobalArgs { + verbose: 0, + quiet: true, + color: Mode::Auto, + no_update_check: false, + self_update: false, + progress: Mode::Never, + ignore_certs: false, + advanced: AdvancedArgs { rlimit_nofile: 16384 }, + }; + + let datastore = Arc::new(Mutex::new(FindingsStore::new(clone_dir))); + + run_async_scan(&global_args, &scan_args, Arc::clone(&datastore), &ctx.rules_db).await?; + + let findings = { + let ds = datastore.lock().unwrap(); + ds.get_matches().len() + }; + assert!(findings > 0); + Ok(()) +} \ No newline at end of file From 1db01311412c4c739d4b7fcf2fbea0a28e0c5028 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Tue, 29 Jul 2025 20:54:22 -0700 Subject: [PATCH 4/5] Added support for Slack. Wrote a basic integration test --- README.md | 29 +++++++++------------------- install-prereceive-hook.sh | 39 -------------------------------------- 2 files changed, 9 insertions(+), 59 deletions(-) delete mode 100755 install-prereceive-hook.sh diff --git a/README.md b/README.md index 09fabc1..a7c668d 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,14 @@ Kingfisher originated as a fork of Praetorian's [Nosey Parker](https://github.co **Learn more:** [Introducing Kingfisher: Real‑Time Secret Detection and Validation](https://www.mongodb.com/blog/post/product-release-announcements/introducing-kingfisher-real-time-secret-detection-validation) +# Benchmark Results + +See ([docs/COMPARISON.md](docs/COMPARISON.md)) + +

+ Kingfisher Runtime Comparison +

+ # Getting Started ## Installation @@ -424,15 +432,6 @@ This creates `.git/hooks/pre-commit` that scans the files staged for commit with Installs a global pre-commit hook at `$HOME/.git/hooks/pre-commit`; for every Git repository you use, it runs `kingfisher scan --no-update-check` on the staged files and cancels the commit if any secrets are detected. -To check incoming pushes on a server-side repository, install the pre-receive hook: - -```bash -./install-prereceive-hook.sh -``` - -The resulting `.git/hooks/pre-receive` script scans the files in each pushed commit and rejects the push if any secrets are detected. - - ## Update Checks Kingfisher automatically queries GitHub for a newer release when it starts and tells you whether an update is available. @@ -558,20 +557,10 @@ Real breaches show how one exposed key can snowball into a full-scale incident: Leaked secrets fuel unauthorized access, lateral movement, regulatory fines, and brand-damaging incident-response costs. -# Benchmark Results - -See ([docs/COMPARISON.md](docs/COMPARISON.md)) - - -

- Kingfisher Runtime Comparison -

- - # Roadmap - More rules -- Packages for Linux (deb, rpm) +- More targets - Please file a [feature request](https://github.com/mongodb/kingfisher/issues) if you have specific features you'd like added # License diff --git a/install-prereceive-hook.sh b/install-prereceive-hook.sh deleted file mode 100755 index f7a4d5e..0000000 --- a/install-prereceive-hook.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -HOOK_DIR="$(git rev-parse --git-dir)/hooks" -HOOK_PATH="$HOOK_DIR/pre-receive" - -if [ -e "$HOOK_PATH" ]; then - echo "Error: $HOOK_PATH already exists. Move or remove the existing hook to continue." >&2 - exit 1 -fi - -cat > "$HOOK_PATH" <<'HOOK' -#!/usr/bin/env bash -# Pre-receive hook to scan pushed commits with Kingfisher -set -euo pipefail - -if ! command -v kingfisher >/dev/null 2>&1; then - echo "kingfisher not found in PATH" >&2 - exit 1 -fi - -while read -r oldrev newrev refname; do - if [ "$oldrev" = "0000000000000000000000000000000000000000" ]; then - git diff-tree --name-only -r "$newrev" -z | - xargs -0 --no-run-if-empty kingfisher scan --no-update-check - else - git diff-tree --no-commit-id --name-only -r "$oldrev" "$newrev" -z | - xargs -0 --no-run-if-empty kingfisher scan --no-update-check - fi - status=$? - if [ "$status" -ne 0 ]; then - echo "Kingfisher detected secrets in push. Push rejected." >&2 - exit "$status" - fi -done -HOOK - -chmod +x "$HOOK_PATH" -echo "Pre-receive hook installed to $HOOK_PATH" From aaabcbd4997c99c23308ea0e08ace1a3fd3d9fa2 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Tue, 29 Jul 2025 20:55:44 -0700 Subject: [PATCH 5/5] Added support for Slack. Wrote a basic integration test --- src/slack.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/slack.rs b/src/slack.rs index 69a02a0..a0cd1f5 100644 --- a/src/slack.rs +++ b/src/slack.rs @@ -4,33 +4,32 @@ use serde::{Deserialize, Serialize}; use std::path::PathBuf; use url::Url; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Deserialize, Serialize)] pub struct SlackMessage { pub permalink: String, pub text: Option, pub ts: String, pub channel: SlackChannel, } - -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Deserialize, Serialize)] pub struct SlackChannel { pub id: String, pub name: Option, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Deserialize)] struct SlackPagination { page: Option, page_count: Option, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Deserialize)] struct SlackMessages { matches: Vec, pagination: Option, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Deserialize)] struct SlackSearchResponse { ok: bool, error: Option, @@ -87,12 +86,13 @@ pub async fn search_messages( return Ok(messages); } } - let next_page = msgs.pagination.as_ref().and_then(|p| p.page).unwrap_or(page); + let next_page = + msgs.pagination.as_ref().and_then(|p| p.page).map(|p| p + 1).unwrap_or(page + 1); let page_count = msgs.pagination.as_ref().and_then(|p| p.page_count).unwrap_or(next_page); - if next_page >= page_count { + if next_page > page_count { break; } - page += 1; + page = next_page; } Ok(messages)