From 33592fbc65c6ba18d81a7a7491328533a38edfb3 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sat, 26 Jul 2025 22:00:05 -0700 Subject: [PATCH 01/13] added buildkite rule --- CHANGELOG.md | 3 +++ Cargo.toml | 2 +- data/rules/buildkit.yml | 28 ++++++++++++++++++++++++++++ data/rules/datadog.yml | 13 ++++++------- 4 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 data/rules/buildkit.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index f21fc28..a23a7ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All notable changes to this project will be documented in this file. +## [1.27.0] +- Added Buildkite rule + ## [1.26.0] - Added rule for ElevenLabs - Added support for scanning Jira issues via a given JQL (Jira Query Language) diff --git a/Cargo.toml b/Cargo.toml index eff7a26..28422bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ publish = false [package] name = "kingfisher" -version = "1.26.0" +version = "1.27.0" description = "MongoDB's blazingly fast secret scanning and validation tool" edition.workspace = true rust-version.workspace = true diff --git a/data/rules/buildkit.yml b/data/rules/buildkit.yml new file mode 100644 index 0000000..8add2f1 --- /dev/null +++ b/data/rules/buildkit.yml @@ -0,0 +1,28 @@ +rules: + - name: Buildkite API Key + id: kingfisher.buildkite.1 + pattern: | + (?xi) + \b + ( + bkua_[a-z0-9]{40} + ) + \b + min_entropy: 3.5 + confidence: medium + examples: + - bkua_3c7019c2e4b6e76fe2e8bdde7c154e3c1a211743 + validation: + type: Http + content: + request: + method: GET + url: https://api.buildkite.com/v2/access-token + headers: + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"uuid"', '"user"'] \ No newline at end of file diff --git a/data/rules/datadog.yml b/data/rules/datadog.yml index c851ed8..718b282 100644 --- a/data/rules/datadog.yml +++ b/data/rules/datadog.yml @@ -43,12 +43,11 @@ rules: id: kingfisher.datadog.2 pattern: | (?xi) - \b - (?: - dd[_-]?\w{0,8}[_-]?(?:key|secret) | - datadog - ) - (?:.|[\n\r]){0,64}? + \b + datadog + (?:.|[\n\r]){0,16}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,16}? \b ( [a-z0-9]{40} @@ -57,7 +56,7 @@ rules: min_entropy: 3.3 confidence: medium examples: - - dd_secret_key-3c0c3965368a6b10f7640dbda46abfdca981c2d3 + - datadog_secret_key-3c0c3965368a6b10f7640dbda46abfdca981c2d3 - datadog_token = BzHpkcs7LujMb3Q1vLRRjbpBNxxYV0ousumYoKJS references: - https://docs.datadoghq.com/account_management/api-app-keys/ \ No newline at end of file From e5942822c62de402d91bc521dd1bd05aa6236763 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sat, 26 Jul 2025 22:00:35 -0700 Subject: [PATCH 02/13] added buildkite rule --- data/rules/{buildkit.yml => buildkite.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename data/rules/{buildkit.yml => buildkite.yml} (100%) diff --git a/data/rules/buildkit.yml b/data/rules/buildkite.yml similarity index 100% rename from data/rules/buildkit.yml rename to data/rules/buildkite.yml From bb78d90067454de40e074d4d6524f7f56962ec29 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sat, 26 Jul 2025 22:01:49 -0700 Subject: [PATCH 03/13] added buildkite rule --- data/rules/buildkite.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data/rules/buildkite.yml b/data/rules/buildkite.yml index 8add2f1..5405c59 100644 --- a/data/rules/buildkite.yml +++ b/data/rules/buildkite.yml @@ -12,6 +12,8 @@ rules: confidence: medium examples: - bkua_3c7019c2e4b6e76fe2e8bdde7c154e3c1a211743 + references: + - https://buildkite.com/docs/apis/rest-api/access-token validation: type: Http content: From 9a3fabdbf27c78795acc2bb87aa3a4f0a5cd2732 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 27 Jul 2025 12:20:20 -0700 Subject: [PATCH 04/13] WIP: Adding support for scanning Docker images --- CHANGELOG.md | 1 + Cargo.toml | 1 + README.md | 8 ++++ data/rules/buildkite.yml | 3 +- src/cli/commands/inputs.rs | 8 +++- src/main.rs | 4 +- src/reporter/json_format.rs | 4 +- src/reporter/pretty_format.rs | 4 +- src/scanner/docker.rs | 77 +++++++++++++++++++++++++++++++++++ src/scanner/mod.rs | 2 + src/scanner/runner.rs | 13 +++++- tests/int_dedup.rs | 4 +- tests/int_github.rs | 4 +- tests/int_gitlab.rs | 4 +- tests/int_validation_cache.rs | 4 +- tests/int_vulnerable_files.rs | 8 +++- 16 files changed, 137 insertions(+), 12 deletions(-) create mode 100644 src/scanner/docker.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a23a7ae..5556a43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file. ## [1.27.0] - Added Buildkite rule +- Added support for scanning Docker images via `--docker-image` ## [1.26.0] - Added rule for ElevenLabs diff --git a/Cargo.toml b/Cargo.toml index 28422bd..14e4cfe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -186,6 +186,7 @@ globset = "0.4.16" jsonwebtoken = "9.3.1" ipnet = "2.11.0" jira_query = "1.6.0" +oci-distribution = "0.11.0" [dependencies.tikv-jemallocator] version = "0.6" diff --git a/README.md b/README.md index 8d398fa..b14ee12 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,14 @@ kingfisher scan /path/to/repo --format sarif --output findings.sarif cat /path/to/file.py | kingfisher scan - ``` +### Scan a Docker image (without Docker installed) + +```bash +kingfisher scan --docker-image ubuntu:latest +``` + +### Sc + ### Scan using a rule _family_ with one flag _(prefix matching: `--rule kingfisher.aws` loads `kingfisher.aws._`)\* diff --git a/data/rules/buildkite.yml b/data/rules/buildkite.yml index 5405c59..3728e98 100644 --- a/data/rules/buildkite.yml +++ b/data/rules/buildkite.yml @@ -27,4 +27,5 @@ rules: - type: StatusMatch status: [200] - type: WordMatch - words: ['"uuid"', '"user"'] \ No newline at end of file + words: ['"uuid"', '"user"'] + \ No newline at end of file diff --git a/src/cli/commands/inputs.rs b/src/cli/commands/inputs.rs index c7a59bc..f698d87 100644 --- a/src/cli/commands/inputs.rs +++ b/src/cli/commands/inputs.rs @@ -26,7 +26,8 @@ pub struct InputSpecifierArgs { "git_url", "all_github_organizations", "all_gitlab_groups", - "jira_url" + "jira_url", + "docker_image" ]), value_hint = ValueHint::AnyPath )] @@ -97,6 +98,11 @@ pub struct InputSpecifierArgs { #[arg(long, default_value_t = 100)] pub max_results: usize, + /// Docker/OCI images to scan (no local Docker required) + #[arg(long = "docker-image")] + pub docker_image: Vec, + + /// Select how to clone Git repositories #[arg(long, default_value_t=GitCloneMode::Bare, alias="git-clone-mode")] pub git_clone: GitCloneMode, diff --git a/src/main.rs b/src/main.rs index 3b0d444..a85fb48 100644 --- a/src/main.rs +++ b/src/main.rs @@ -281,7 +281,9 @@ fn create_default_scan_args() -> cli::commands::scan::ScanArgs { jira_url: None, jql: None, - max_results: 50, + max_results: 100, + // Docker image scanning + docker_image: Vec::new(), // git clone / history options git_clone: GitCloneMode::Bare, diff --git a/src/reporter/json_format.rs b/src/reporter/json_format.rs index 31123b6..4132def 100644 --- a/src/reporter/json_format.rs +++ b/src/reporter/json_format.rs @@ -428,7 +428,9 @@ mod tests { // Jira options jira_url: None, jql: None, - max_results: 50, + max_results: 100, + // Docker image scanning + docker_image: Vec::new(), // clone / history options git_clone: GitCloneMode::Bare, git_history: GitHistoryMode::Full, diff --git a/src/reporter/pretty_format.rs b/src/reporter/pretty_format.rs index 0b4a46c..5960538 100644 --- a/src/reporter/pretty_format.rs +++ b/src/reporter/pretty_format.rs @@ -344,7 +344,9 @@ fn test_pretty_format_with_nan_entropy_panics() { // Jira options jira_url: None, jql: None, - max_results: 50, + max_results: 100, + // Docker image scanning + docker_image: Vec::new(), // git clone / history options git_clone: GitCloneMode::Bare, git_history: GitHistoryMode::Full, diff --git a/src/scanner/docker.rs b/src/scanner/docker.rs new file mode 100644 index 0000000..f636e91 --- /dev/null +++ b/src/scanner/docker.rs @@ -0,0 +1,77 @@ +use std::io::Write; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use oci_distribution::client::{linux_amd64_resolver, Client, ClientConfig}; +use oci_distribution::{secrets::RegistryAuth, Reference}; +use tracing::debug; + +use crate::decompress::decompress_file; + +pub struct Docker; + +impl Docker { + pub fn new() -> Self { + Docker + } + + pub async fn save_image_to_dir(&self, image: &str, out_dir: &Path) -> Result<()> { + let reference: Reference = + image.parse().with_context(|| format!("invalid image reference {image}"))?; + debug!("Pulling {image}"); + let mut client = Client::new(ClientConfig { + platform_resolver: Some(Box::new(linux_amd64_resolver)), + ..Default::default() + }); + let auth = RegistryAuth::Anonymous; + let accepted = vec![ + oci_distribution::manifest::IMAGE_LAYER_MEDIA_TYPE, + oci_distribution::manifest::IMAGE_LAYER_GZIP_MEDIA_TYPE, + oci_distribution::manifest::IMAGE_DOCKER_LAYER_TAR_MEDIA_TYPE, + oci_distribution::manifest::IMAGE_DOCKER_LAYER_GZIP_MEDIA_TYPE, + ]; + let image = client.pull(&reference, &auth, accepted).await?; + + std::fs::create_dir_all(out_dir)?; + for (idx, layer) in image.layers.into_iter().enumerate() { + let ext = match layer.media_type.as_str() { + oci_distribution::manifest::IMAGE_LAYER_GZIP_MEDIA_TYPE + | oci_distribution::manifest::IMAGE_DOCKER_LAYER_GZIP_MEDIA_TYPE => "tar.gz", + oci_distribution::manifest::IMAGE_LAYER_MEDIA_TYPE + | oci_distribution::manifest::IMAGE_DOCKER_LAYER_TAR_MEDIA_TYPE => "tar", + _ => "bin", + }; + let file_name = format!("layer_{idx}.{ext}"); + let tmp_path = out_dir.join(file_name); + let mut tmp = std::fs::File::create(&tmp_path)?; + tmp.write_all(&layer.data)?; + decompress_file(&tmp_path, Some(out_dir))?; + } + Ok(()) + } +} + +pub async fn save_docker_images(images: &[String], clone_root: &Path) -> Result> { + let docker = Docker::new(); + let mut dirs = Vec::new(); + for image in images { + let dir_name = image.replace(['/', ':'], "_"); + let out_dir = clone_root.join(format!("docker_{dir_name}")); + docker + .save_image_to_dir(image, &out_dir) + .await + .with_context(|| format!("saving image {image}"))?; + dirs.push(out_dir); + } + Ok(dirs) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn docker_struct_new() { + let _ = Docker::new(); + } +} \ No newline at end of file diff --git a/src/scanner/mod.rs b/src/scanner/mod.rs index 0b4423f..fff9440 100644 --- a/src/scanner/mod.rs +++ b/src/scanner/mod.rs @@ -3,7 +3,9 @@ pub(crate) use enumerate::enumerate_filesystem_inputs; pub(crate) use repos::{clone_or_update_git_repos, enumerate_github_repos}; pub use runner::{load_and_record_rules, run_async_scan, run_scan}; pub(crate) use validation::run_secret_validation; +pub(crate) use docker::save_docker_images; +mod docker; mod enumerate; mod processing; mod repos; diff --git a/src/scanner/runner.rs b/src/scanner/runner.rs index 99ad466..568b498 100644 --- a/src/scanner/runner.rs +++ b/src/scanner/runner.rs @@ -19,7 +19,7 @@ use crate::{ scanner::{ clone_or_update_git_repos, enumerate_filesystem_inputs, enumerate_github_repos, repos::{enumerate_gitlab_repos, fetch_jira_issues}, - run_secret_validation, + run_secret_validation, save_docker_images, summary::print_scan_summary, }, }; @@ -68,6 +68,17 @@ pub async fn run_async_scan( let jira_dirs = fetch_jira_issues(args, global_args, &datastore).await?; input_roots.extend(jira_dirs); + // Save Docker images if specified + if !args.input_specifier_args.docker_image.is_empty() { + let clone_root = { + let ds = datastore.lock().unwrap(); + ds.clone_root() + }; + let docker_dirs = + save_docker_images(&args.input_specifier_args.docker_image, &clone_root).await?; + input_roots.extend(docker_dirs); + } + if input_roots.is_empty() { bail!("No inputs to scan"); } diff --git a/tests/int_dedup.rs b/tests/int_dedup.rs index 4c4975c..2763ebd 100644 --- a/tests/int_dedup.rs +++ b/tests/int_dedup.rs @@ -81,7 +81,9 @@ rules: jira_url: None, jql: None, - max_results: 50, + max_results: 100, + // Docker image scanning + docker_image: Vec::new(), // git clone / history options git_clone: GitCloneMode::Bare, git_history: GitHistoryMode::Full, diff --git a/tests/int_github.rs b/tests/int_github.rs index c8256c8..d4f7f25 100644 --- a/tests/int_github.rs +++ b/tests/int_github.rs @@ -68,7 +68,9 @@ fn test_github_remote_scan() -> Result<()> { jira_url: None, jql: None, - max_results: 50, + max_results: 100, + // Docker image scanning + docker_image: Vec::new(), // git clone / history options git_clone: GitCloneMode::Bare, git_history: GitHistoryMode::Full, diff --git a/tests/int_gitlab.rs b/tests/int_gitlab.rs index 6ec6e19..67b1bc3 100644 --- a/tests/int_gitlab.rs +++ b/tests/int_gitlab.rs @@ -67,7 +67,9 @@ fn test_gitlab_remote_scan() -> Result<()> { jira_url: None, jql: None, - max_results: 50, + max_results: 100, + // Docker image scanning + docker_image: Vec::new(), git_clone: GitCloneMode::Bare, git_history: GitHistoryMode::Full, scan_nested_repos: true, diff --git a/tests/int_validation_cache.rs b/tests/int_validation_cache.rs index 933c068..3e21947 100644 --- a/tests/int_validation_cache.rs +++ b/tests/int_validation_cache.rs @@ -124,7 +124,9 @@ async fn test_validation_cache_and_depvars() -> Result<()> { jira_url: None, jql: None, - max_results: 50, + max_results: 100, + // Docker image scanning + docker_image: Vec::new(), // git clone / history options git_clone: GitCloneMode::Bare, git_history: GitHistoryMode::Full, diff --git a/tests/int_vulnerable_files.rs b/tests/int_vulnerable_files.rs index 0da7868..ad78192 100644 --- a/tests/int_vulnerable_files.rs +++ b/tests/int_vulnerable_files.rs @@ -67,7 +67,9 @@ impl TestContext { jira_url: None, jql: None, - max_results: 50, + max_results: 100, + // Docker image scanning + docker_image: Vec::new(), // git clone / history options git_clone: GitCloneMode::Bare, git_history: GitHistoryMode::Full, @@ -135,7 +137,9 @@ impl TestContext { jira_url: None, jql: None, - max_results: 50, + max_results: 100, + // Docker image scanning + docker_image: Vec::new(), // git clone / history options git_clone: GitCloneMode::Bare, git_history: GitHistoryMode::Full, From 2d8550aa1544a4c604a8b97278bd999c0d71d383 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 27 Jul 2025 13:17:49 -0700 Subject: [PATCH 05/13] WIP: Adding support for scanning Docker images --- src/scanner/docker.rs | 38 ++++++++++++++++++++++++++++++++------ src/scanner/runner.rs | 8 ++++++-- tests/smoke_docker.rs | 20 ++++++++++++++++++++ 3 files changed, 58 insertions(+), 8 deletions(-) create mode 100644 tests/smoke_docker.rs diff --git a/src/scanner/docker.rs b/src/scanner/docker.rs index f636e91..dd70bbe 100644 --- a/src/scanner/docker.rs +++ b/src/scanner/docker.rs @@ -1,9 +1,11 @@ use std::io::Write; use std::path::{Path, PathBuf}; +use std::time::Duration; use anyhow::{Context, Result}; use oci_distribution::client::{linux_amd64_resolver, Client, ClientConfig}; use oci_distribution::{secrets::RegistryAuth, Reference}; +use indicatif::{ProgressBar, ProgressStyle}; use tracing::debug; use crate::decompress::decompress_file; @@ -15,14 +17,30 @@ impl Docker { Docker } - pub async fn save_image_to_dir(&self, image: &str, out_dir: &Path) -> Result<()> { + pub async fn save_image_to_dir( + &self, + image: &str, + out_dir: &Path, + use_progress: bool, + ) -> Result<()> { let reference: Reference = image.parse().with_context(|| format!("invalid image reference {image}"))?; debug!("Pulling {image}"); - let mut client = Client::new(ClientConfig { + let pb = if use_progress { + let style = ProgressStyle::with_template("{spinner} {msg}") + .expect("progress template"); + let pb = ProgressBar::new_spinner().with_style(style); + pb.enable_steady_tick(Duration::from_millis(100)); + pb.set_message(format!("pulling {image}")); + pb + } else { + ProgressBar::hidden() + }; + let client = Client::new(ClientConfig { platform_resolver: Some(Box::new(linux_amd64_resolver)), ..Default::default() }); + let mut client = client; let auth = RegistryAuth::Anonymous; let accepted = vec![ oci_distribution::manifest::IMAGE_LAYER_MEDIA_TYPE, @@ -30,10 +48,12 @@ impl Docker { oci_distribution::manifest::IMAGE_DOCKER_LAYER_TAR_MEDIA_TYPE, oci_distribution::manifest::IMAGE_DOCKER_LAYER_GZIP_MEDIA_TYPE, ]; - let image = client.pull(&reference, &auth, accepted).await?; + let pulled = client.pull(&reference, &auth, accepted).await?; + pb.set_length(pulled.layers.len() as u64); + pb.set_message("extracting layers"); std::fs::create_dir_all(out_dir)?; - for (idx, layer) in image.layers.into_iter().enumerate() { + for (idx, layer) in pulled.layers.into_iter().enumerate() { let ext = match layer.media_type.as_str() { oci_distribution::manifest::IMAGE_LAYER_GZIP_MEDIA_TYPE | oci_distribution::manifest::IMAGE_DOCKER_LAYER_GZIP_MEDIA_TYPE => "tar.gz", @@ -46,19 +66,25 @@ impl Docker { let mut tmp = std::fs::File::create(&tmp_path)?; tmp.write_all(&layer.data)?; decompress_file(&tmp_path, Some(out_dir))?; + pb.inc(1); } + pb.finish_with_message(format!("saved {image}")); Ok(()) } } -pub async fn save_docker_images(images: &[String], clone_root: &Path) -> Result> { +pub async fn save_docker_images( + images: &[String], + clone_root: &Path, + use_progress: bool, +) -> Result> { let docker = Docker::new(); let mut dirs = Vec::new(); for image in images { let dir_name = image.replace(['/', ':'], "_"); let out_dir = clone_root.join(format!("docker_{dir_name}")); docker - .save_image_to_dir(image, &out_dir) + .save_image_to_dir(image, &out_dir, use_progress) .await .with_context(|| format!("saving image {image}"))?; dirs.push(out_dir); diff --git a/src/scanner/runner.rs b/src/scanner/runner.rs index 568b498..4dec5e7 100644 --- a/src/scanner/runner.rs +++ b/src/scanner/runner.rs @@ -74,8 +74,12 @@ pub async fn run_async_scan( let ds = datastore.lock().unwrap(); ds.clone_root() }; - let docker_dirs = - save_docker_images(&args.input_specifier_args.docker_image, &clone_root).await?; + let docker_dirs = save_docker_images( + &args.input_specifier_args.docker_image, + &clone_root, + progress_enabled, + ) + .await?; input_roots.extend(docker_dirs); } diff --git a/tests/smoke_docker.rs b/tests/smoke_docker.rs new file mode 100644 index 0000000..2fd61bc --- /dev/null +++ b/tests/smoke_docker.rs @@ -0,0 +1,20 @@ +use assert_cmd::prelude::*; +use predicates::prelude::*; +use std::process::Command; + +#[test] +fn smoke_scan_docker_image() -> anyhow::Result<()> { + Command::cargo_bin("kingfisher")? + .args([ + "scan", + "--docker-image", + "ghcr.io/owasp/wrongsecrets/wrongsecrets-master:latest-master", + "--format", + "json", + "--no-update-check", + ]) + .assert() + .code(205) + .stdout(predicate::str::contains("Active Credential")); + Ok(()) +} \ No newline at end of file From 29b97b409194fa49f426a5756b9a4a1597a97f4e Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 27 Jul 2025 14:59:19 -0700 Subject: [PATCH 06/13] WIP: Adding support for scanning Docker images --- Cargo.toml | 1 + src/findings_store.rs | 9 ++++ src/reporter.rs | 15 +++++++ src/reporter/json_format.rs | 4 ++ src/reporter/pretty_format.rs | 2 + src/scanner/docker.rs | 85 ++++++++++++++++++++++++++++++----- src/scanner/runner.rs | 9 +++- 7 files changed, 114 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 14e4cfe..e80faa7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -187,6 +187,7 @@ jsonwebtoken = "9.3.1" ipnet = "2.11.0" jira_query = "1.6.0" oci-distribution = "0.11.0" +walkdir = "2.5.0" [dependencies.tikv-jemallocator] version = "0.6" diff --git a/src/findings_store.rs b/src/findings_store.rs index 7d3cd76..5972490 100644 --- a/src/findings_store.rs +++ b/src/findings_store.rs @@ -52,6 +52,7 @@ pub struct FindingsStore { bloom_items: usize, blob_meta: FxHashMap>, origin_meta: FxHashMap>, + docker_images: FxHashMap, } impl FindingsStore { pub fn new(clone_dir: PathBuf) -> Self { @@ -69,6 +70,7 @@ impl FindingsStore { clone_dir, seen_bloom, bloom_items: 0, + docker_images: FxHashMap::default(), } } @@ -286,6 +288,13 @@ impl FindingsStore { self.clone_dir.clone() } + pub fn register_docker_image(&mut self, dir: PathBuf, image: String) { + self.docker_images.insert(dir, image); + } + + pub fn docker_images(&self) -> &FxHashMap { + &self.docker_images + } pub fn get_finding_data_iter( &self, diff --git a/src/reporter.rs b/src/reporter.rs index f6a3331..91fcbd1 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -140,6 +140,21 @@ impl DetailsReporter { } } + fn docker_display_path(&self, path: &std::path::Path) -> Option { + let ds = self.datastore.lock().ok()?; + for (dir, image) in ds.docker_images().iter() { + if path.starts_with(dir) { + let rel = path.strip_prefix(dir).ok()?; + let mut rel_str = rel.display().to_string(); + rel_str = rel_str.replace(".decomp.tar!", ".tar.gz => "); + rel_str = rel_str.replace(".tar!", ".tar => "); + rel_str = rel_str.replace('!', " => "); + return Some(format!("{} => {}", image, rel_str)); + } + } + None + } + fn gather_findings(&self) -> Result> { let metadata_list = self.get_finding_data()?; let all_matches = self.get_filtered_matches()?; diff --git a/src/reporter/json_format.rs b/src/reporter/json_format.rs index 4132def..6916337 100644 --- a/src/reporter/json_format.rs +++ b/src/reporter/json_format.rs @@ -103,6 +103,8 @@ impl DetailsReporter { if let Origin::File(e) = origin { if let Some(url) = self.jira_issue_url(&e.path, args) { Some(url) + } else if let Some(mapped) = self.docker_display_path(&e.path) { + Some(mapped) } else { Some(e.path.display().to_string()) } @@ -252,6 +254,8 @@ impl DetailsReporter { if let Origin::File(e) = origin { if let Some(url) = self.jira_issue_url(&e.path, args) { Some(url) + } else if let Some(mapped) = self.docker_display_path(&e.path) { + Some(mapped) } else { Some(e.path.display().to_string()) } diff --git a/src/reporter/pretty_format.rs b/src/reporter/pretty_format.rs index 5960538..ffa7cf0 100644 --- a/src/reporter/pretty_format.rs +++ b/src/reporter/pretty_format.rs @@ -216,6 +216,8 @@ impl<'a> Display for PrettyFinding<'a> { Origin::File(e) => { let display_path = if let Some(url) = reporter.jira_issue_url(&e.path, args) { url + } else if let Some(mapped) = reporter.docker_display_path(&e.path) { + mapped } else { e.path.display().to_string() }; diff --git a/src/scanner/docker.rs b/src/scanner/docker.rs index dd70bbe..4a61ac3 100644 --- a/src/scanner/docker.rs +++ b/src/scanner/docker.rs @@ -1,12 +1,16 @@ -use std::io::Write; +use std::fs::File; +use std::io::{Read, Write}; use std::path::{Path, PathBuf}; +use std::process::Command; use std::time::Duration; -use anyhow::{Context, Result}; +use anyhow::{anyhow, Context, Result}; +use indicatif::{ProgressBar, ProgressStyle}; use oci_distribution::client::{linux_amd64_resolver, Client, ClientConfig}; use oci_distribution::{secrets::RegistryAuth, Reference}; -use indicatif::{ProgressBar, ProgressStyle}; +use sha2::{Digest, Sha256}; use tracing::debug; +use walkdir::WalkDir; use crate::decompress::decompress_file; @@ -17,19 +21,80 @@ impl Docker { Docker } + fn try_save_local_image(&self, image: &str, out_dir: &Path, use_progress: bool) -> Result<()> { + let docker = Command::new("docker") + .args(["image", "inspect", image]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status(); + + if !matches!(docker, Ok(s) if s.success()) { + return Err(anyhow!("image not local")); + } + + let pb = if use_progress { + let style = ProgressStyle::with_template("{spinner} {msg} {pos}/{len}") + .expect("progress template"); + let pb = ProgressBar::new(0).with_style(style); + pb.enable_steady_tick(Duration::from_millis(100)); + pb + } else { + ProgressBar::hidden() + }; + pb.set_message(format!("saving local {image}")); + + std::fs::create_dir_all(out_dir)?; + let tar_path = out_dir.join("local_image.tar"); + let status = Command::new("docker") + .args(["image", "save", image, "-o", tar_path.to_str().unwrap()]) + .status() + .with_context(|| "running docker save")?; + if !status.success() { + pb.finish_with_message("docker save failed"); + return Err(anyhow!("failed to save local image")); + } + + pb.set_message("extracting layers"); + decompress_file(&tar_path, Some(out_dir))?; + + let mut layer_paths = Vec::new(); + for entry in WalkDir::new(out_dir) { + let entry = entry?; + if entry.file_name() == "layer.tar" { + layer_paths.push(entry.path().to_path_buf()); + } + } + + pb.set_length(layer_paths.len() as u64); + for p in layer_paths { + let mut data = Vec::new(); + File::open(&p)?.read_to_end(&mut data)?; + let digest = format!("{:x}", Sha256::digest(&data)); + let new_path = out_dir.join(format!("layer_{digest}.tar")); + std::fs::rename(&p, &new_path)?; + pb.inc(1); + } + + pb.finish_with_message(format!("saved {image}")); + Ok(()) + } + pub async fn save_image_to_dir( &self, image: &str, out_dir: &Path, use_progress: bool, ) -> Result<()> { + if self.try_save_local_image(image, out_dir, use_progress).is_ok() { + return Ok(()); + } let reference: Reference = image.parse().with_context(|| format!("invalid image reference {image}"))?; debug!("Pulling {image}"); let pb = if use_progress { - let style = ProgressStyle::with_template("{spinner} {msg}") + let style = ProgressStyle::with_template("{spinner} {msg} {pos}/{len}") .expect("progress template"); - let pb = ProgressBar::new_spinner().with_style(style); + let pb = ProgressBar::new(0).with_style(style); pb.enable_steady_tick(Duration::from_millis(100)); pb.set_message(format!("pulling {image}")); pb @@ -53,7 +118,7 @@ impl Docker { pb.set_message("extracting layers"); std::fs::create_dir_all(out_dir)?; - for (idx, layer) in pulled.layers.into_iter().enumerate() { + for layer in pulled.layers.into_iter() { let ext = match layer.media_type.as_str() { oci_distribution::manifest::IMAGE_LAYER_GZIP_MEDIA_TYPE | oci_distribution::manifest::IMAGE_DOCKER_LAYER_GZIP_MEDIA_TYPE => "tar.gz", @@ -61,11 +126,11 @@ impl Docker { | oci_distribution::manifest::IMAGE_DOCKER_LAYER_TAR_MEDIA_TYPE => "tar", _ => "bin", }; - let file_name = format!("layer_{idx}.{ext}"); + let digest = layer.sha256_digest(); + let file_name = format!("layer_{digest}.{ext}"); let tmp_path = out_dir.join(file_name); let mut tmp = std::fs::File::create(&tmp_path)?; tmp.write_all(&layer.data)?; - decompress_file(&tmp_path, Some(out_dir))?; pb.inc(1); } pb.finish_with_message(format!("saved {image}")); @@ -77,7 +142,7 @@ pub async fn save_docker_images( images: &[String], clone_root: &Path, use_progress: bool, -) -> Result> { +) -> Result> { let docker = Docker::new(); let mut dirs = Vec::new(); for image in images { @@ -87,7 +152,7 @@ pub async fn save_docker_images( .save_image_to_dir(image, &out_dir, use_progress) .await .with_context(|| format!("saving image {image}"))?; - dirs.push(out_dir); + dirs.push((out_dir, image.clone())); } Ok(dirs) } diff --git a/src/scanner/runner.rs b/src/scanner/runner.rs index 4dec5e7..52c8004 100644 --- a/src/scanner/runner.rs +++ b/src/scanner/runner.rs @@ -80,9 +80,16 @@ pub async fn run_async_scan( progress_enabled, ) .await?; - input_roots.extend(docker_dirs); + for (dir, img) in docker_dirs { + { + let mut ds = datastore.lock().unwrap(); + ds.register_docker_image(dir.clone(), img); + } + input_roots.push(dir); + } } + if input_roots.is_empty() { bail!("No inputs to scan"); } From f48ea7ca0a00b2a3ebc50fada8f7a03ed57ed668 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 27 Jul 2025 17:54:26 -0700 Subject: [PATCH 07/13] Adding support for scanning Docker images --- src/decompress.rs | 33 ++++++++++++++++++++++++++++++--- src/scanner/docker.rs | 5 +++++ src/scanner/runner.rs | 1 - 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 88a11ae..8bc43aa 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -256,12 +256,28 @@ fn make_output_path(path: &Path, base: Option<&Path>, extension: &str) -> PathBu } } -/* ───────────────────────────────────────────────────────────── */ pub fn decompress_file_to_temp(path: &Path) -> Result<(CompressedContent, TempDir)> { let temp_dir = tempdir()?; - let content = decompress_file(path, Some(temp_dir.path()))?; + let mut content = decompress_file(path, Some(temp_dir.path()))?; - if let CompressedContent::Archive(ref files) = content { + // if let CompressedContent::Archive(ref files) = content { + let mut prefix_for_replace = None; + if let Some(stem) = path.file_stem() { + let candidate = temp_dir.path().join(stem).with_extension("decomp.tar"); + prefix_for_replace = Some(candidate); + } + + if let CompressedContent::Archive(ref mut files) = content { + if let Some(prefix) = &prefix_for_replace { + let prefix_str = prefix.display().to_string(); + for (name, _) in files.iter_mut() { + if let Some(rest) = name.strip_prefix(&prefix_str) { + if let Some((_, suffix)) = rest.split_once('!') { + *name = format!("{}!{}", path.display(), suffix); + } + } + } + } for (name, data) in files { let rel = name.split_once('!').map(|(_, sub)| sub).unwrap_or(name); let p = temp_dir.path().join(rel.replace('\\', "/")); @@ -270,6 +286,17 @@ pub fn decompress_file_to_temp(path: &Path) -> Result<(CompressedContent, TempDi } fs::write(p, data)?; } + } else if let CompressedContent::ArchiveFiles(ref mut entries) = content { + if let Some(prefix) = &prefix_for_replace { + let prefix_str = prefix.display().to_string(); + for (name, _) in entries.iter_mut() { + if let Some(rest) = name.strip_prefix(&prefix_str) { + if let Some((_, suffix)) = rest.split_once('!') { + *name = format!("{}!{}", path.display(), suffix); + } + } + } + } } Ok((content, temp_dir)) } diff --git a/src/scanner/docker.rs b/src/scanner/docker.rs index 4a61ac3..f6e16ed 100644 --- a/src/scanner/docker.rs +++ b/src/scanner/docker.rs @@ -72,6 +72,9 @@ impl Docker { let digest = format!("{:x}", Sha256::digest(&data)); let new_path = out_dir.join(format!("layer_{digest}.tar")); std::fs::rename(&p, &new_path)?; + // extract layer contents so inner filenames appear in scan results + decompress_file(&new_path, Some(out_dir))?; + std::fs::remove_file(&new_path)?; pb.inc(1); } @@ -131,6 +134,8 @@ impl Docker { let tmp_path = out_dir.join(file_name); let mut tmp = std::fs::File::create(&tmp_path)?; tmp.write_all(&layer.data)?; + decompress_file(&tmp_path, Some(out_dir))?; + std::fs::remove_file(&tmp_path)?; pb.inc(1); } pb.finish_with_message(format!("saved {image}")); diff --git a/src/scanner/runner.rs b/src/scanner/runner.rs index 52c8004..f1271cf 100644 --- a/src/scanner/runner.rs +++ b/src/scanner/runner.rs @@ -89,7 +89,6 @@ pub async fn run_async_scan( } } - if input_roots.is_empty() { bail!("No inputs to scan"); } From c4dd9564434a0c655d36f50b69377bef7ce92976 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 27 Jul 2025 19:03:43 -0700 Subject: [PATCH 08/13] Adding support for scanning Docker images --- src/reporter.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/reporter.rs b/src/reporter.rs index 91fcbd1..ca69c97 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -146,10 +146,10 @@ impl DetailsReporter { if path.starts_with(dir) { let rel = path.strip_prefix(dir).ok()?; let mut rel_str = rel.display().to_string(); - rel_str = rel_str.replace(".decomp.tar!", ".tar.gz => "); - rel_str = rel_str.replace(".tar!", ".tar => "); - rel_str = rel_str.replace('!', " => "); - return Some(format!("{} => {}", image, rel_str)); + rel_str = rel_str.replace(".decomp.tar!", ".tar.gz | "); + rel_str = rel_str.replace(".tar!", ".tar | "); + rel_str = rel_str.replace('!', " | "); + return Some(format!("{} | {}", image, rel_str)); } } None From 97dbda8f517c9573596d9189f331b192294efbc1 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 27 Jul 2025 20:25:45 -0700 Subject: [PATCH 09/13] Adding support for scanning Docker images --- README.md | 18 +++++++++++++----- src/scanner/docker.rs | 16 +++++++++++++++- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b14ee12..6138ea9 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,10 @@ Kingfisher extends Nosey Parker by: 1. Validating secrets in real time via cloud-provider APIs 2. Enhancing regex-based detection with source-code parsing for improved accuracy 3. Adding GitLab repository scanning support -4. Providing Jira scanning capabilities -5. Introducing a baseline feature that suppresses known secrets and reports only newly introduced ones -5. Offering native Windows environment support +4. Adding support for scanning Docker images via `--docker-image` +5. Providing Jira scanning capabilities +6. Introducing a baseline feature that suppresses known secrets and reports only newly introduced ones +7. Offering native Windows support **MongoDB Blog**: [Introducing Kingfisher: Real-Time Secret Detection and Validation](https://www.mongodb.com/blog/post/product-release-announcements/introducing-kingfisher-real-time-secret-detection-validation) @@ -195,14 +196,20 @@ kingfisher scan /path/to/repo --format sarif --output findings.sarif ```bash cat /path/to/file.py | kingfisher scan - -``` -### Scan a Docker image (without Docker installed) +``` +### Scan a Docker image ```bash +# pulls from a registry if not available locally kingfisher scan --docker-image ubuntu:latest +# layers are automatically extracted so reported paths include files inside +# the container image + +# set KF_DOCKER_TOKEN for private registries ("user:pass" or just the token) ``` + ### Sc ### Scan using a rule _family_ with one flag @@ -328,6 +335,7 @@ KF_JIRA_TOKEN="token" kingfisher scan \ | `KF_GITHUB_TOKEN` | GitHub Personal Access Token | | `KF_GITLAB_TOKEN` | GitLab Personal Access Token | | `KF_JIRA_TOKEN` | Jira API token | +| `KF_DOCKER_TOKEN` | Docker token | Set them temporarily per command: diff --git a/src/scanner/docker.rs b/src/scanner/docker.rs index f6e16ed..0588f56 100644 --- a/src/scanner/docker.rs +++ b/src/scanner/docker.rs @@ -1,5 +1,6 @@ use std::fs::File; use std::io::{Read, Write}; +use std::env; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::Duration; @@ -14,6 +15,19 @@ use walkdir::WalkDir; use crate::decompress::decompress_file; +fn registry_auth_from_env() -> RegistryAuth { + match env::var("KF_DOCKER_TOKEN") { + Ok(token) => { + if let Some((user, pass)) = token.split_once(':') { + RegistryAuth::Basic(user.to_string(), pass.to_string()) + } else { + RegistryAuth::Basic(String::new(), token) + } + } + Err(_) => RegistryAuth::Anonymous, + } +} + pub struct Docker; impl Docker { @@ -109,7 +123,7 @@ impl Docker { ..Default::default() }); let mut client = client; - let auth = RegistryAuth::Anonymous; + let auth = registry_auth_from_env(); let accepted = vec![ oci_distribution::manifest::IMAGE_LAYER_MEDIA_TYPE, oci_distribution::manifest::IMAGE_LAYER_GZIP_MEDIA_TYPE, From d9e3a61689b6f0b822424b03273270843f04cfa7 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Mon, 28 Jul 2025 09:55:48 -0700 Subject: [PATCH 10/13] changed from oci-distribution to newer oci-client --- Cargo.toml | 2 +- README.md | 2 +- src/scanner/docker.rs | 122 ++++++++++++++++++++++++++++++++++-------- 3 files changed, 103 insertions(+), 23 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e80faa7..79595e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -186,7 +186,7 @@ globset = "0.4.16" jsonwebtoken = "9.3.1" ipnet = "2.11.0" jira_query = "1.6.0" -oci-distribution = "0.11.0" +oci-client = { version = "0.15", default-features = false, features = ["rustls-tls"] } walkdir = "2.5.0" [dependencies.tikv-jemallocator] diff --git a/README.md b/README.md index 6138ea9..059453c 100644 --- a/README.md +++ b/README.md @@ -335,7 +335,7 @@ KF_JIRA_TOKEN="token" kingfisher scan \ | `KF_GITHUB_TOKEN` | GitHub Personal Access Token | | `KF_GITLAB_TOKEN` | GitLab Personal Access Token | | `KF_JIRA_TOKEN` | Jira API token | -| `KF_DOCKER_TOKEN` | Docker token | +| `KF_DOCKER_TOKEN` | Docker registry token (`user:pass` or bearer token). If unset, credentials from the Docker keychain are used | Set them temporarily per command: diff --git a/src/scanner/docker.rs b/src/scanner/docker.rs index 0588f56..c4915c2 100644 --- a/src/scanner/docker.rs +++ b/src/scanner/docker.rs @@ -1,30 +1,110 @@ +use std::env; use std::fs::File; use std::io::{Read, Write}; -use std::env; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::process::{Command, Stdio}; use std::time::Duration; use anyhow::{anyhow, Context, Result}; +use base64::Engine; use indicatif::{ProgressBar, ProgressStyle}; -use oci_distribution::client::{linux_amd64_resolver, Client, ClientConfig}; -use oci_distribution::{secrets::RegistryAuth, Reference}; +use oci_client::client::{linux_amd64_resolver, Client, ClientConfig}; +use oci_client::secrets::RegistryAuth; +use oci_client::Reference; +use serde_json::Value; use sha2::{Digest, Sha256}; use tracing::debug; use walkdir::WalkDir; use crate::decompress::decompress_file; -fn registry_auth_from_env() -> RegistryAuth { - match env::var("KF_DOCKER_TOKEN") { - Ok(token) => { - if let Some((user, pass)) = token.split_once(':') { - RegistryAuth::Basic(user.to_string(), pass.to_string()) - } else { - RegistryAuth::Basic(String::new(), token) +fn helper_get_creds(helper: &str, registry: &str) -> Option<(String, String)> { + fn run(bin: &str, registry: &str) -> Option<(String, String)> { + let mut child = Command::new(bin) + .arg("get") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .ok()?; + { + let stdin = child.stdin.as_mut()?; + let _ = stdin.write_all(format!("{registry}\n").as_bytes()); + } + let output = child.wait_with_output().ok()?; + if !output.status.success() { + return None; + } + let v: Value = serde_json::from_slice(&output.stdout).ok()?; + let user = v.get("Username")?.as_str()?.to_string(); + let secret = v.get("Secret")?.as_str()?.to_string(); + Some((user, secret)) + } + + let bin = format!("docker-credential-{helper}"); + if let Some(creds) = run(&bin, registry) { + return Some(creds); + } + if helper == "keychain" && bin != "docker-credential-osxkeychain" { + if let Some(creds) = run("docker-credential-osxkeychain", registry) { + return Some(creds); + } + } + None +} + +fn creds_from_docker_config(registry: &str) -> Option<(String, String)> { + let config_dir = env::var("DOCKER_CONFIG") + .map(PathBuf::from) + .or_else(|_| env::var("HOME").map(|h| PathBuf::from(h).join(".docker"))) + .ok()?; + let path = config_dir.join("config.json"); + let mut content = String::new(); + File::open(path).ok()?.read_to_string(&mut content).ok()?; + let json: Value = serde_json::from_str(&content).ok()?; + + if let Some(ch) = json.get("credHelpers").and_then(|v| v.get(registry)).and_then(|v| v.as_str()) + { + if let Some(creds) = helper_get_creds(ch, registry) { + return Some(creds); + } + } + if let Some(store) = json.get("credsStore").and_then(|v| v.as_str()) { + if let Some(creds) = helper_get_creds(store, registry) { + return Some(creds); + } + } + + if let Some(auths) = json.get("auths").and_then(|v| v.as_object()) { + if let Some(entry) = auths + .get(registry) + .or_else(|| auths.get(&format!("https://{registry}"))) + .or_else(|| auths.get(&format!("http://{registry}"))) + { + if let Some(auth) = entry.get("auth").and_then(|v| v.as_str()) { + let decoded = base64::engine::general_purpose::STANDARD.decode(auth).ok()?; + let cred = String::from_utf8(decoded).ok()?; + if let Some((u, p)) = cred.split_once(':') { + return Some((u.to_string(), p.to_string())); + } } } - Err(_) => RegistryAuth::Anonymous, + } + None +} + +fn registry_auth(reference: &Reference) -> RegistryAuth { + if let Ok(token) = env::var("KF_DOCKER_TOKEN") { + if let Some((user, pass)) = token.split_once(':') { + return RegistryAuth::Basic(user.to_string(), pass.to_string()); + } else { + return RegistryAuth::Bearer(token); + } + } + if let Some((user, pass)) = creds_from_docker_config(reference.registry()) { + RegistryAuth::Basic(user, pass) + } else { + RegistryAuth::Anonymous } } @@ -123,12 +203,12 @@ impl Docker { ..Default::default() }); let mut client = client; - let auth = registry_auth_from_env(); + let auth = registry_auth(&reference); let accepted = vec![ - oci_distribution::manifest::IMAGE_LAYER_MEDIA_TYPE, - oci_distribution::manifest::IMAGE_LAYER_GZIP_MEDIA_TYPE, - oci_distribution::manifest::IMAGE_DOCKER_LAYER_TAR_MEDIA_TYPE, - oci_distribution::manifest::IMAGE_DOCKER_LAYER_GZIP_MEDIA_TYPE, + oci_client::manifest::IMAGE_LAYER_MEDIA_TYPE, + oci_client::manifest::IMAGE_LAYER_GZIP_MEDIA_TYPE, + oci_client::manifest::IMAGE_DOCKER_LAYER_TAR_MEDIA_TYPE, + oci_client::manifest::IMAGE_DOCKER_LAYER_GZIP_MEDIA_TYPE, ]; let pulled = client.pull(&reference, &auth, accepted).await?; pb.set_length(pulled.layers.len() as u64); @@ -137,10 +217,10 @@ impl Docker { std::fs::create_dir_all(out_dir)?; for layer in pulled.layers.into_iter() { let ext = match layer.media_type.as_str() { - oci_distribution::manifest::IMAGE_LAYER_GZIP_MEDIA_TYPE - | oci_distribution::manifest::IMAGE_DOCKER_LAYER_GZIP_MEDIA_TYPE => "tar.gz", - oci_distribution::manifest::IMAGE_LAYER_MEDIA_TYPE - | oci_distribution::manifest::IMAGE_DOCKER_LAYER_TAR_MEDIA_TYPE => "tar", + oci_client::manifest::IMAGE_LAYER_GZIP_MEDIA_TYPE + | oci_client::manifest::IMAGE_DOCKER_LAYER_GZIP_MEDIA_TYPE => "tar.gz", + oci_client::manifest::IMAGE_LAYER_MEDIA_TYPE + | oci_client::manifest::IMAGE_DOCKER_LAYER_TAR_MEDIA_TYPE => "tar", _ => "bin", }; let digest = layer.sha256_digest(); From 24fc33d5b99c9f4767e26149c45615fd3fb270c1 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Mon, 28 Jul 2025 10:07:55 -0700 Subject: [PATCH 11/13] improved authentication options for Docker support --- README.md | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 059453c..e2b982f 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ Kingfisher extends Nosey Parker by: - **Built-In Validation**: Hundreds of built-in detection rules, many with live-credential validators that call the relevant service APIs (AWS, Azure, GCP, Stripe, etc.) to confirm a secret is active. You can extend or override the library by adding YAML-defined rules on the command line—see [docs/RULES.md](/docs/RULES.md) for details - **Git History Scanning**: Scan local repos, remote GitHub/GitLab orgs/users, or arbitrary GitHub/GitLab repos - **Jira Scanning**: Scan issues returned from a JQL search using `--jira-url` and `--jql` +- **Docker Image Scanning**: Scan public or private docker images via `--docker-image` - **Baseline Support:** Generate and manage baseline files to ignore known secrets and report only newly introduced ones. See ([docs/BASELINE.md](docs/BASELINE.md)) for details. # Getting Started @@ -198,19 +199,6 @@ kingfisher scan /path/to/repo --format sarif --output findings.sarif cat /path/to/file.py | kingfisher scan - ``` -### Scan a Docker image - -```bash -# pulls from a registry if not available locally -kingfisher scan --docker-image ubuntu:latest -# layers are automatically extracted so reported paths include files inside -# the container image - -# set KF_DOCKER_TOKEN for private registries ("user:pass" or just the token) -``` - - -### Sc ### Scan using a rule _family_ with one flag @@ -258,8 +246,35 @@ kingfisher scan ./my-project \ --exclude tests \ -v ``` +## Scanning Docker Images ---- +Kingfisher will first try to use any locally available image, then fall back to pulling via OCI. + +Authentication happens *in this order*: + +1. **`KF_DOCKER_TOKEN`** env var + - If it contains `user:pass`, it’s used as Basic auth + - Otherwise it’s sent as a Bearer token +2. **Docker CLI credentials** + - Checks `credHelpers` (per-registry) and `credsStore` in `~/.docker/config.json`. + - Falls back to the legacy `auths` → `auth` (base64) entries. +3. **Anonymous** (no credentials) + + +```bash +# 1) Scan public or already-pulled image +kingfisher scan --docker-image ghcr.io/owasp/wrongsecrets/wrongsecrets-master:latest-master + +# 2) For private registries, explicitly set KF_DOCKER_TOKEN: +# - Basic auth: "user:pass" +# - Bearer only: "TOKEN" +export KF_DOCKER_TOKEN="AWS:$(aws ecr get-login-password --region us-east-1)" +kingfisher scan --docker-image some-private-registry.dkr.ecr.us-east-1.amazonaws.com/base/amazonlinux2023:latest + +# 3) Or rely on your Docker CLI login/keychain: +# (e.g. aws ecr get-login-password … | docker login …) +kingfisher scan --docker-image private.registry.example.com/my-image:tag +``` ## Scanning GitHub From 4b032214f60fb1aaa20a9951a674eaa43874841a Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Mon, 28 Jul 2025 10:25:11 -0700 Subject: [PATCH 12/13] improved precommit hook, to allow global installation --- README.md | 21 ++++++++++----- install-precommit-hook.sh | 53 ++++++++++++++++++++++++++++++++------ install-prereceive-hook.sh | 0 src/scanner/docker.rs | 4 +-- 4 files changed, 61 insertions(+), 17 deletions(-) mode change 100644 => 100755 install-precommit-hook.sh mode change 100644 => 100755 install-prereceive-hook.sh diff --git a/README.md b/README.md index e2b982f..8dab5ad 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,13 @@ Kingfisher is a blazingly fast secret‑scanning and validation tool built in Ru Kingfisher originated as a fork of [Nosey Parker](https://github.com/praetorian-inc/noseyparker) by Praetorian Security, Inc, and is built atop their incredible work and the work contributed by the Nosey Parker community. Kingfisher extends Nosey Parker by: -1. Validating secrets in real time via cloud-provider APIs -2. Enhancing regex-based detection with source-code parsing for improved accuracy -3. Adding GitLab repository scanning support -4. Adding support for scanning Docker images via `--docker-image` -5. Providing Jira scanning capabilities +1. **Validating secrets** in real time via cloud-provider APIs +2. Enhancing regex-based detection with **source-code parsing** for improved accuracy +3. Adding **GitLab** repository scanning support +4. Adding support for scanning **Docker** images via `--docker-image` +5. Providing **Jira** scanning capabilities 6. Introducing a baseline feature that suppresses known secrets and reports only newly introduced ones -7. Offering native Windows support +7. Offering native **Windows** support **MongoDB Blog**: [Introducing Kingfisher: Real-Time Secret Detection and Validation](https://www.mongodb.com/blog/post/product-release-announcements/introducing-kingfisher-real-time-secret-detection-validation) @@ -387,12 +387,19 @@ _If no token is provided Kingfisher still works for public repositories._ Run the provided helper script to add a hook that scans staged files before each commit: ```bash -./install-precommit-hook.sh +# local (current repo only ─ default) +./install-kingfisher-hook.sh ``` This creates `.git/hooks/pre-commit` that scans the files staged for commit with `kingfisher scan --no-update-check` and blocks the commit if any secrets are found. +```bash +# global (every repo on this machine) +./install-kingfisher-hook.sh --global ### Install a Pre-Receive Hook +``` + +Installs a global pre-commit hook at `$HOME/.git/hooks/pre-commit`; for every Git repository you use, it runs `kingfisher scan --no-update-check` on the staged files and cancels the commit if any secrets are detected. To check incoming pushes on a server-side repository, install the pre-receive hook: diff --git a/install-precommit-hook.sh b/install-precommit-hook.sh old mode 100644 new mode 100755 index 6a6283f..7a81ba6 --- a/install-precommit-hook.sh +++ b/install-precommit-hook.sh @@ -1,17 +1,54 @@ #!/usr/bin/env bash +# +# Install a Git pre-commit hook that runs `kingfisher scan`. +# --global → install once for all repos using core.hooksPath +# --force → overwrite an existing pre-commit hook +# set -euo pipefail -HOOK_DIR="$(git rev-parse --git-dir)/hooks" +MODE="local" +FORCE=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + -g|--global) MODE="global" ;; + -f|--force) FORCE=1 ;; + -h|--help) + echo "Usage: $0 [--global] [--force]" && exit 0 + ;; + *) echo "Unknown flag: $1" >&2; exit 1 ;; + esac + shift +done + +if [[ "$MODE" == "local" ]]; then + # ensure we're inside a Git repo + REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null) \ + || { echo "Not inside a Git repository" >&2; exit 1; } + + HOOK_DIR="$(git rev-parse --git-dir)/hooks" +else + # global: honour existing core.hooksPath or default to ~/.git-hooks + HOOK_DIR=$(git config --global --get core.hooksPath || echo "$HOME/.git-hooks") + mkdir -p "$HOOK_DIR" + + # if the user hasn’t set core.hooksPath, do it now + if ! git config --global --get core.hooksPath >/dev/null; then + git config --global core.hooksPath "$HOOK_DIR" + echo "Set git config --global core.hooksPath to $HOOK_DIR" + fi +fi + HOOK_PATH="$HOOK_DIR/pre-commit" -if [ -e "$HOOK_PATH" ]; then - echo "Error: $HOOK_PATH already exists. Move or remove the existing hook to continue." >&2 +if [[ -e "$HOOK_PATH" && $FORCE -eq 0 ]]; then + echo "Error: $HOOK_PATH already exists. Use --force to overwrite." >&2 exit 1 fi -cat > "$HOOK_PATH" <<'HOOK' +cat >"$HOOK_PATH" <<'HOOK' #!/usr/bin/env bash -# Pre-commit hook to run Kingfisher scan on staged changes +# Git pre-commit hook to run Kingfisher on staged changes set -euo pipefail if ! command -v kingfisher >/dev/null 2>&1; then @@ -22,11 +59,11 @@ fi git diff --cached --name-only -z | \ xargs -0 --no-run-if-empty kingfisher scan --no-update-check status=$? -if [ "$status" -ne 0 ]; then +if [[ $status -ne 0 ]]; then echo "Kingfisher detected secrets in staged files. Commit aborted." >&2 - exit "$status" + exit $status fi HOOK chmod +x "$HOOK_PATH" -echo "Pre-commit hook installed to $HOOK_PATH" +echo "Pre-commit hook installed to $HOOK_PATH ($MODE mode)" diff --git a/install-prereceive-hook.sh b/install-prereceive-hook.sh old mode 100644 new mode 100755 diff --git a/src/scanner/docker.rs b/src/scanner/docker.rs index c4915c2..a18ca24 100644 --- a/src/scanner/docker.rs +++ b/src/scanner/docker.rs @@ -140,7 +140,7 @@ impl Docker { std::fs::create_dir_all(out_dir)?; let tar_path = out_dir.join("local_image.tar"); let status = Command::new("docker") - .args(["image", "save", image, "-o", tar_path.to_str().unwrap()]) + .args(["image", "save", image, "-o", &tar_path.to_string_lossy()]) .status() .with_context(|| "running docker save")?; if !status.success() { @@ -202,7 +202,7 @@ impl Docker { platform_resolver: Some(Box::new(linux_amd64_resolver)), ..Default::default() }); - let mut client = client; + let client = client; let auth = registry_auth(&reference); let accepted = vec![ oci_client::manifest::IMAGE_LAYER_MEDIA_TYPE, From a270d8ecb23c15c1f2fac3d1eacc6360cdca582f Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Mon, 28 Jul 2025 10:26:32 -0700 Subject: [PATCH 13/13] improved precommit hook, to allow global installation --- src/scanner/docker.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/scanner/docker.rs b/src/scanner/docker.rs index a18ca24..775e687 100644 --- a/src/scanner/docker.rs +++ b/src/scanner/docker.rs @@ -53,6 +53,20 @@ fn helper_get_creds(helper: &str, registry: &str) -> Option<(String, String)> { None } +/// Turn `registry.example.com/foo/bar:latest` into something like +/// `registry.example.com_foo_bar_latest_4d3c9e83` +fn image_dir_name(reference: &str) -> String { + // keep it readable + let mut name = reference.replace(['/', ':'], "_"); + + // add a truncated SHA-256 to guarantee uniqueness + let hash = Sha256::digest(reference.as_bytes()); + let short = &hex::encode(hash)[..8]; // 8-char prefix is plenty + name.push('_'); + name.push_str(short); + name +} + fn creds_from_docker_config(registry: &str) -> Option<(String, String)> { let config_dir = env::var("DOCKER_CONFIG") .map(PathBuf::from) @@ -244,8 +258,9 @@ pub async fn save_docker_images( ) -> Result> { let docker = Docker::new(); let mut dirs = Vec::new(); + for image in images { - let dir_name = image.replace(['/', ':'], "_"); + let dir_name = image_dir_name(image); let out_dir = clone_root.join(format!("docker_{dir_name}")); docker .save_image_to_dir(image, &out_dir, use_progress) @@ -253,6 +268,7 @@ pub async fn save_docker_images( .with_context(|| format!("saving image {image}"))?; dirs.push((out_dir, image.clone())); } + Ok(dirs) }