From bcf2b60e0bf5816159d719b399591d0f44a84e8a Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Tue, 29 Jul 2025 19:00:49 -0700 Subject: [PATCH] Added support for Slack --- .github/workflows/release.yml | 1 - CHANGELOG.md | 3 + Cargo.toml | 2 +- README.md | 19 ++++-- src/cli/commands/inputs.rs | 14 +++- src/findings_store.rs | 10 +++ src/lib.rs | 1 + src/main.rs | 4 ++ src/reporter.rs | 16 +++-- src/reporter/json_format.rs | 8 +++ src/reporter/pretty_format.rs | 6 ++ src/reporter/sarif_format.rs | 4 ++ src/scanner/repos.rs | 34 +++++++++- src/scanner/runner.rs | 6 +- src/slack.rs | 118 ++++++++++++++++++++++++++++++++++ tests/int_dedup.rs | 2 + tests/int_github.rs | 2 + tests/int_gitlab.rs | 2 + tests/int_validation_cache.rs | 2 + tests/int_vulnerable_files.rs | 4 ++ 20 files changed, 240 insertions(+), 18 deletions(-) create mode 100644 src/slack.rs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 56fb028..956bc1a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -189,7 +189,6 @@ jobs: - name: Cache vcpkg artifacts uses: actions/cache@v3 with: - # Adjust these paths if your vcpkg root is somewhere else path: | C:\vcpkg\buildtrees C:\vcpkg\packages diff --git a/CHANGELOG.md b/CHANGELOG.md index 5556a43..5c1e055 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All notable changes to this project will be documented in this file. +## [1.28.0] +- Added support for scanning Slack + ## [1.27.0] - Added Buildkite rule - Added support for scanning Docker images via `--docker-image` diff --git a/Cargo.toml b/Cargo.toml index 79595e8..fa9cf8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ publish = false [package] name = "kingfisher" -version = "1.27.0" +version = "1.28.0" description = "MongoDB's blazingly fast secret scanning and validation tool" edition.workspace = true rust-version.workspace = true diff --git a/README.md b/README.md index ad65c78..d459b5b 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,11 @@ Kingfisher extends Nosey Parker by: 1. **Validating secrets** in real time via cloud-provider APIs 2. Enhancing regex-based detection with **source-code parsing** for improved accuracy 3. Adding **GitLab** repository scanning support -4. Adding support for scanning **Docker** images via `--docker-image` +4. Adding support for scanning **Docker** images 5. Providing **Jira** scanning capabilities -6. Introducing a baseline feature that suppresses known secrets and reports only newly introduced ones -7. Offering native **Windows** support +6. Adding **Slack** scanning capabilities +7. Introducing a baseline feature that suppresses known secrets and reports only newly introduced ones +8. Offering native **Windows** support **MongoDB Blog**: [Introducing Kingfisher: Real-Time Secret Detection and Validation](https://www.mongodb.com/blog/post/product-release-announcements/introducing-kingfisher-real-time-secret-detection-validation) @@ -29,6 +30,7 @@ Kingfisher extends Nosey Parker by: - **Built-In Validation**: Hundreds of built-in detection rules, many with live-credential validators that call the relevant service APIs (AWS, Azure, GCP, Stripe, etc.) to confirm a secret is active. You can extend or override the library by adding YAML-defined rules on the command line—see [docs/RULES.md](/docs/RULES.md) for details - **Git History Scanning**: Scan local repos, remote GitHub/GitLab orgs/users, or arbitrary GitHub/GitLab repos - **Jira Scanning**: Scan issues returned from a JQL search using `--jira-url` and `--jql` +- **Slack Scanning**: Scan messages returned from a Slack search query using `--slack-query` - **Docker Image Scanning**: Scan public or private docker images via `--docker-image` - **Baseline Support:** Generate and manage baseline files to ignore known secrets and report only newly introduced ones. See ([docs/BASELINE.md](docs/BASELINE.md)) for details. @@ -353,7 +355,16 @@ KF_JIRA_TOKEN="token" kingfisher scan \ --max-results 1000 ``` --- +## Scanning Slack +### Scan Slack messages matching a search query + +```bash +KF_SLACK_TOKEN="token" kingfisher scan \ + --slack-query "from:username has:link" \ + --max-results 1000 +``` +*The Slack token must be a user token with the `search:read` scope. Bot tokens (those beginning with `xoxb-`) cannot call the Slack search API.* ## Environment Variables for Tokens @@ -362,8 +373,8 @@ KF_JIRA_TOKEN="token" kingfisher scan \ | `KF_GITHUB_TOKEN` | GitHub Personal Access Token | | `KF_GITLAB_TOKEN` | GitLab Personal Access Token | | `KF_JIRA_TOKEN` | Jira API token | +| `KF_SLACK_TOKEN` | Slack API token | | `KF_DOCKER_TOKEN` | Docker registry token (`user:pass` or bearer token). If unset, credentials from the Docker keychain are used | - Set them temporarily per command: ```bash diff --git a/src/cli/commands/inputs.rs b/src/cli/commands/inputs.rs index f698d87..8a1c23d 100644 --- a/src/cli/commands/inputs.rs +++ b/src/cli/commands/inputs.rs @@ -27,7 +27,8 @@ pub struct InputSpecifierArgs { "all_github_organizations", "all_gitlab_groups", "jira_url", - "docker_image" + "docker_image", + "slack_query" ]), value_hint = ValueHint::AnyPath )] @@ -94,7 +95,15 @@ pub struct InputSpecifierArgs { #[arg(long, requires = "jira_url")] pub jql: Option, - /// Maximum number of Jira results to fetch + /// Slack search query + #[arg(long)] + pub slack_query: Option, + + /// Use the specified URL for Slack API access + #[arg(long, default_value = "https://slack.com/api/", value_hint = ValueHint::Url)] + pub slack_api_url: Url, + + /// Maximum number of Slack or Jira results to fetch #[arg(long, default_value_t = 100)] pub max_results: usize, @@ -102,7 +111,6 @@ pub struct InputSpecifierArgs { #[arg(long = "docker-image")] pub docker_image: Vec, - /// Select how to clone Git repositories #[arg(long, default_value_t=GitCloneMode::Bare, alias="git-clone-mode")] pub git_clone: GitCloneMode, diff --git a/src/findings_store.rs b/src/findings_store.rs index 5972490..93e9f1c 100644 --- a/src/findings_store.rs +++ b/src/findings_store.rs @@ -53,6 +53,7 @@ pub struct FindingsStore { blob_meta: FxHashMap>, origin_meta: FxHashMap>, docker_images: FxHashMap, + slack_links: FxHashMap, } impl FindingsStore { pub fn new(clone_dir: PathBuf) -> Self { @@ -71,6 +72,7 @@ impl FindingsStore { seen_bloom, bloom_items: 0, docker_images: FxHashMap::default(), + slack_links: FxHashMap::default(), } } @@ -296,6 +298,14 @@ impl FindingsStore { &self.docker_images } + pub fn register_slack_message(&mut self, path: PathBuf, permalink: String) { + self.slack_links.insert(path, permalink); + } + + pub fn slack_links(&self) -> &FxHashMap { + &self.slack_links + } + pub fn get_finding_data_iter( &self, ) -> impl Iterator + '_ { diff --git a/src/lib.rs b/src/lib.rs index af74e7b..85bc57c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,6 +33,7 @@ pub mod safe_list; pub mod scanner; pub mod scanner_pool; pub mod serde_utils; +pub mod slack; pub mod snippet; pub mod update; pub mod util; diff --git a/src/main.rs b/src/main.rs index a85fb48..56f1e15 100644 --- a/src/main.rs +++ b/src/main.rs @@ -282,6 +282,10 @@ fn create_default_scan_args() -> cli::commands::scan::ScanArgs { jira_url: None, jql: None, max_results: 100, + // Slack query + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), + // Docker image scanning docker_image: Vec::new(), diff --git a/src/reporter.rs b/src/reporter.rs index ca69c97..e6709dc 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -122,13 +122,8 @@ impl DetailsReporter { args: &cli::commands::scan::ScanArgs, ) -> Option { // drop any trailing slash so we don’t end up with “//browse/…” - let jira_url = args - .input_specifier_args - .jira_url - .as_ref()? - .as_str() - .trim_end_matches('/'); - + let jira_url = args.input_specifier_args.jira_url.as_ref()?.as_str().trim_end_matches('/'); + let ds = self.datastore.lock().ok()?; let root = ds.clone_root(); let jira_dir = root.join("jira_issues"); @@ -140,6 +135,13 @@ impl DetailsReporter { } } + /// If the given file path corresponds to a Slack message downloaded to disk, + /// return the permalink for that message. + fn slack_message_url(&self, path: &std::path::Path) -> Option { + let ds = self.datastore.lock().ok()?; + ds.slack_links().get(path).cloned() + } + fn docker_display_path(&self, path: &std::path::Path) -> Option { let ds = self.datastore.lock().ok()?; for (dir, image) in ds.docker_images().iter() { diff --git a/src/reporter/json_format.rs b/src/reporter/json_format.rs index 6916337..6bad0cb 100644 --- a/src/reporter/json_format.rs +++ b/src/reporter/json_format.rs @@ -103,6 +103,8 @@ impl DetailsReporter { if let Origin::File(e) = origin { if let Some(url) = self.jira_issue_url(&e.path, args) { Some(url) + } else if let Some(url) = self.slack_message_url(&e.path) { + Some(url) } else if let Some(mapped) = self.docker_display_path(&e.path) { Some(mapped) } else { @@ -254,6 +256,8 @@ impl DetailsReporter { if let Origin::File(e) = origin { if let Some(url) = self.jira_issue_url(&e.path, args) { Some(url) + } else if let Some(url) = self.slack_message_url(&e.path) { + Some(url) } else if let Some(mapped) = self.docker_display_path(&e.path) { Some(mapped) } else { @@ -434,6 +438,10 @@ mod tests { jql: None, max_results: 100, // Docker image scanning + // Slack options + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), + docker_image: Vec::new(), // clone / history options git_clone: GitCloneMode::Bare, diff --git a/src/reporter/pretty_format.rs b/src/reporter/pretty_format.rs index ffa7cf0..62dd354 100644 --- a/src/reporter/pretty_format.rs +++ b/src/reporter/pretty_format.rs @@ -216,6 +216,8 @@ impl<'a> Display for PrettyFinding<'a> { Origin::File(e) => { let display_path = if let Some(url) = reporter.jira_issue_url(&e.path, args) { url + } else if let Some(url) = reporter.slack_message_url(&e.path) { + url } else if let Some(mapped) = reporter.docker_display_path(&e.path) { mapped } else { @@ -347,6 +349,10 @@ fn test_pretty_format_with_nan_entropy_panics() { jira_url: None, jql: None, max_results: 100, + + // Slack options + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), // Docker image scanning docker_image: Vec::new(), // git clone / history options diff --git a/src/reporter/sarif_format.rs b/src/reporter/sarif_format.rs index f771c17..3db2d99 100644 --- a/src/reporter/sarif_format.rs +++ b/src/reporter/sarif_format.rs @@ -73,6 +73,8 @@ impl DetailsReporter { Origin::File(e) => { let uri = if let Some(url) = self.jira_issue_url(&e.path, args) { url + } else if let Some(url) = self.slack_message_url(&e.path) { + url } else { e.path.display().to_string() }; @@ -209,6 +211,8 @@ impl DetailsReporter { let uri = if let Some(url) = self.jira_issue_url(&e.path, args) { url + } else if let Some(url) = self.slack_message_url(&e.path) { + url } else { e.path.display().to_string() }; diff --git a/src/scanner/repos.rs b/src/scanner/repos.rs index f84b758..2a8044a 100644 --- a/src/scanner/repos.rs +++ b/src/scanner/repos.rs @@ -23,7 +23,7 @@ use crate::{ github, gitlab, jira, matcher::Match, origin::OriginSet, - PathBuf, + slack, PathBuf, }; pub type DatastoreMessage = (OriginSet, BlobMetadata, Vec<(Option, Match)>); @@ -252,4 +252,36 @@ pub async fn fetch_jira_issues( ) .await?; Ok(vec![output_dir]) +} + +pub async fn fetch_slack_messages( + args: &scan::ScanArgs, + global_args: &global::GlobalArgs, + datastore: &Arc>, +) -> Result> { + let Some(query) = args.input_specifier_args.slack_query.as_deref() else { + return Ok(Vec::new()); + }; + let api_url = args.input_specifier_args.slack_api_url.clone(); + let max_results = args.input_specifier_args.max_results; + let output_root = { + let ds = datastore.lock().unwrap(); + ds.clone_root() + }; + let output_dir = output_root.join("slack_messages"); + let paths = slack::download_messages_to_dir( + api_url, + query, + max_results, + global_args.ignore_certs, + &output_dir, + ) + .await?; + { + let mut ds = datastore.lock().unwrap(); + for (path, link) in &paths { + ds.register_slack_message(path.clone(), link.clone()); + } + } + Ok(vec![output_dir]) } \ No newline at end of file diff --git a/src/scanner/runner.rs b/src/scanner/runner.rs index f1271cf..b727718 100644 --- a/src/scanner/runner.rs +++ b/src/scanner/runner.rs @@ -18,7 +18,7 @@ use crate::{ rules_database::RulesDatabase, scanner::{ clone_or_update_git_repos, enumerate_filesystem_inputs, enumerate_github_repos, - repos::{enumerate_gitlab_repos, fetch_jira_issues}, + repos::{enumerate_gitlab_repos, fetch_jira_issues, fetch_slack_messages}, run_secret_validation, save_docker_images, summary::print_scan_summary, }, @@ -68,6 +68,10 @@ pub async fn run_async_scan( let jira_dirs = fetch_jira_issues(args, global_args, &datastore).await?; input_roots.extend(jira_dirs); + // Fetch Slack messages if requested + let slack_dirs = fetch_slack_messages(args, global_args, &datastore).await?; + input_roots.extend(slack_dirs); + // Save Docker images if specified if !args.input_specifier_args.docker_image.is_empty() { let clone_root = { diff --git a/src/slack.rs b/src/slack.rs new file mode 100644 index 0000000..69a02a0 --- /dev/null +++ b/src/slack.rs @@ -0,0 +1,118 @@ +use anyhow::{Context, Result}; +use reqwest::Client; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; +use url::Url; + +#[derive(Debug, Serialize, Deserialize)] +pub struct SlackMessage { + pub permalink: String, + pub text: Option, + pub ts: String, + pub channel: SlackChannel, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct SlackChannel { + pub id: String, + pub name: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +struct SlackPagination { + page: Option, + page_count: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +struct SlackMessages { + matches: Vec, + pagination: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +struct SlackSearchResponse { + ok: bool, + error: Option, + messages: Option, +} + +pub async fn search_messages( + api_url: Url, + query: &str, + max_results: usize, + ignore_certs: bool, +) -> Result> { + let token = std::env::var("KF_SLACK_TOKEN") + .context("KF_SLACK_TOKEN environment variable must be set")?; + + let client = Client::builder() + .danger_accept_invalid_certs(ignore_certs) + .build() + .context("Failed to build HTTP client")?; + + let mut page = 1u32; + let mut messages = Vec::new(); + + loop { + let url = api_url.join("search.messages").context("Failed to build Slack API URL")?; + + let resp = client + .get(url) + .bearer_auth(&token) + .query(&[("query", query), ("count", "100"), ("page", &page.to_string())]) + .send() + .await + .context("Failed to send Slack request")?; + + let body: SlackSearchResponse = + resp.json().await.context("Failed to parse Slack response")?; + + if !body.ok { + let err = body.error.unwrap_or_else(|| "unknown".to_string()); + if err == "not_allowed_token_type" { + return Err(anyhow::anyhow!( + "Slack API error: not_allowed_token_type - use a user token with the `search:read` scope" + )); + } + return Err(anyhow::anyhow!("Slack API error: {}", err)); + } + + let Some(msgs) = body.messages else { + break; + }; + for m in msgs.matches { + messages.push(m); + if messages.len() >= max_results { + return Ok(messages); + } + } + let next_page = msgs.pagination.as_ref().and_then(|p| p.page).unwrap_or(page); + let page_count = msgs.pagination.as_ref().and_then(|p| p.page_count).unwrap_or(next_page); + if next_page >= page_count { + break; + } + page += 1; + } + + Ok(messages) +} + +pub async fn download_messages_to_dir( + api_url: Url, + query: &str, + max_results: usize, + ignore_certs: bool, + output_dir: &PathBuf, +) -> Result> { + std::fs::create_dir_all(output_dir)?; + let messages = search_messages(api_url, query, max_results, ignore_certs).await?; + let mut paths = Vec::new(); + for msg in messages { + let ts = msg.ts.replace('.', "_"); + let file = output_dir.join(format!("{}_{}.json", msg.channel.id, ts)); + std::fs::write(&file, serde_json::to_vec(&msg)?)?; + paths.push((file, msg.permalink)); + } + Ok(paths) +} \ No newline at end of file diff --git a/tests/int_dedup.rs b/tests/int_dedup.rs index 2763ebd..c42967f 100644 --- a/tests/int_dedup.rs +++ b/tests/int_dedup.rs @@ -82,6 +82,8 @@ rules: jira_url: None, jql: None, max_results: 100, + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), // Docker image scanning docker_image: Vec::new(), // git clone / history options diff --git a/tests/int_github.rs b/tests/int_github.rs index d4f7f25..4bda269 100644 --- a/tests/int_github.rs +++ b/tests/int_github.rs @@ -69,6 +69,8 @@ fn test_github_remote_scan() -> Result<()> { jira_url: None, jql: None, max_results: 100, + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), // Docker image scanning docker_image: Vec::new(), // git clone / history options diff --git a/tests/int_gitlab.rs b/tests/int_gitlab.rs index 67b1bc3..4087618 100644 --- a/tests/int_gitlab.rs +++ b/tests/int_gitlab.rs @@ -68,6 +68,8 @@ fn test_gitlab_remote_scan() -> Result<()> { jira_url: None, jql: None, max_results: 100, + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), // Docker image scanning docker_image: Vec::new(), git_clone: GitCloneMode::Bare, diff --git a/tests/int_validation_cache.rs b/tests/int_validation_cache.rs index 3e21947..ae8dd50 100644 --- a/tests/int_validation_cache.rs +++ b/tests/int_validation_cache.rs @@ -125,6 +125,8 @@ async fn test_validation_cache_and_depvars() -> Result<()> { jira_url: None, jql: None, max_results: 100, + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), // Docker image scanning docker_image: Vec::new(), // git clone / history options diff --git a/tests/int_vulnerable_files.rs b/tests/int_vulnerable_files.rs index ad78192..187427e 100644 --- a/tests/int_vulnerable_files.rs +++ b/tests/int_vulnerable_files.rs @@ -68,6 +68,8 @@ impl TestContext { jira_url: None, jql: None, max_results: 100, + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), // Docker image scanning docker_image: Vec::new(), // git clone / history options @@ -138,6 +140,8 @@ impl TestContext { jira_url: None, jql: None, max_results: 100, + slack_query: None, + slack_api_url: Url::parse("https://slack.com/api/").unwrap(), // Docker image scanning docker_image: Vec::new(), // git clone / history options