From 955118e48390d8c4a443c8447c960b3b9da0d14d Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Wed, 23 Jul 2025 18:14:43 -0700 Subject: [PATCH 1/4] Added precommit and prereceive hook installations. Fixing Gitlab support --- README.md | 22 +++++++++++++++++++++- install-precommit-hook.sh | 32 ++++++++++++++++++++++++++++++++ install-prereceive-hook.sh | 34 ++++++++++++++++++++++++++++++++++ src/git_binary.rs | 36 ++++++++++++++++++++++++++---------- 4 files changed, 113 insertions(+), 11 deletions(-) create mode 100644 install-precommit-hook.sh create mode 100644 install-prereceive-hook.sh diff --git a/README.md b/README.md index 9a7895e..f7e7c0b 100644 --- a/README.md +++ b/README.md @@ -318,7 +318,27 @@ _If no token is provided Kingfisher still works for public repositories._ | 200 | Findings discovered | | 205 | Validated findings discovered | ---- + +## Install a Pre-Commit Hook + +Run the provided helper script to add a hook that scans staged files before each commit: + +```bash +./install-precommit-hook.sh +``` + +This creates `.git/hooks/pre-commit` that scans the files staged for commit with `kingfisher scan --no-update-check` and blocks the commit if any secrets are found. + +### Install a Pre-Receive Hook + +To check incoming pushes on a server-side repository, install the pre-receive hook: + +```bash +./install-prereceive-hook.sh +``` + +The resulting `.git/hooks/pre-receive` script scans the files in each pushed commit and rejects the push if any secrets are detected. + ## Update Checks diff --git a/install-precommit-hook.sh b/install-precommit-hook.sh new file mode 100644 index 0000000..6a6283f --- /dev/null +++ b/install-precommit-hook.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +HOOK_DIR="$(git rev-parse --git-dir)/hooks" +HOOK_PATH="$HOOK_DIR/pre-commit" + +if [ -e "$HOOK_PATH" ]; then + echo "Error: $HOOK_PATH already exists. Move or remove the existing hook to continue." >&2 + exit 1 +fi + +cat > "$HOOK_PATH" <<'HOOK' +#!/usr/bin/env bash +# Pre-commit hook to run Kingfisher scan on staged changes +set -euo pipefail + +if ! command -v kingfisher >/dev/null 2>&1; then + echo "kingfisher not found in PATH" >&2 + exit 1 +fi + +git diff --cached --name-only -z | \ + xargs -0 --no-run-if-empty kingfisher scan --no-update-check +status=$? +if [ "$status" -ne 0 ]; then + echo "Kingfisher detected secrets in staged files. Commit aborted." >&2 + exit "$status" +fi +HOOK + +chmod +x "$HOOK_PATH" +echo "Pre-commit hook installed to $HOOK_PATH" diff --git a/install-prereceive-hook.sh b/install-prereceive-hook.sh new file mode 100644 index 0000000..3ca4081 --- /dev/null +++ b/install-prereceive-hook.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +HOOK_DIR="$(git rev-parse --git-dir)/hooks" +HOOK_PATH="$HOOK_DIR/pre-receive" + +if [ -e "$HOOK_PATH" ]; then + echo "Error: $HOOK_PATH already exists. Move or remove the existing hook to continue." >&2 + exit 1 +fi + +cat > "$HOOK_PATH" <<'HOOK' +#!/usr/bin/env bash +# Pre-receive hook to scan pushed commits with Kingfisher +set -euo pipefail + +if ! command -v kingfisher >/dev/null 2>&1; then + echo "kingfisher not found in PATH" >&2 + exit 1 +fi + +while read -r oldrev newrev refname; do + git diff-tree --no-commit-id --name-only -r "$oldrev" "$newrev" -z | + xargs -0 --no-run-if-empty kingfisher scan --no-update-check + status=$? + if [ "$status" -ne 0 ]; then + echo "Kingfisher detected secrets in push. Push rejected." >&2 + exit "$status" + fi +done +HOOK + +chmod +x "$HOOK_PATH" +echo "Pre-receive hook installed to $HOOK_PATH" diff --git a/src/git_binary.rs b/src/git_binary.rs index a0e7c2e..b2b6918 100644 --- a/src/git_binary.rs +++ b/src/git_binary.rs @@ -36,17 +36,33 @@ impl Git { /// Create a new `Git` instance. /// /// * `ignore_certs`: If `true`, disables SSL certificate verification for `git` operations. - pub fn new(ignore_certs: bool) -> Self { - let credentials = if std::env::var("KF_GITHUB_TOKEN").is_ok() { - vec![ - "-c".into(), - r#"credential.helper="#.into(), - "-c".into(), +pub fn new(ignore_certs: bool) -> Self { + let mut credentials = Vec::new(); + + // If either GitHub or GitLab token is set, first clear existing credential.helpers + if std::env::var("KF_GITHUB_TOKEN").is_ok() + || std::env::var("KF_GITLAB_TOKEN").is_ok() + { + credentials.push("-c".into()); + credentials.push(r#"credential.helper="#.into()); + } + + // Inject GitHub token helper + if std::env::var("KF_GITHUB_TOKEN").is_ok() { + credentials.push("-c".into()); + credentials.push( r#"credential.helper=!_ghcreds() { echo username="kingfisher"; echo password="$KF_GITHUB_TOKEN"; }; _ghcreds"#.into(), - ] - } else { - Vec::new() - }; + ); + } + + // Inject GitLab token helper + if std::env::var("KF_GITLAB_TOKEN").is_ok() { + credentials.push("-c".into()); + credentials.push( + r#"credential.helper=!_glcreds() { echo username="oauth2"; echo password="$KF_GITLAB_TOKEN"; }; _glcreds"#.into(), + ); + } + Self { credentials, ignore_certs } } From 9b4856d7d532b5122a696939c900a82c1b6dd2ce Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Wed, 23 Jul 2025 19:57:33 -0700 Subject: [PATCH 2/4] Fixed Gitlab support. Added pre-commit and pre-receive installation scripts. --- CHANGELOG.md | 9 +++++++++ Cargo.toml | 2 +- data/rules/baseten.yml | 11 +++++++---- data/rules/mongodb.yml | 4 +++- src/validation.rs | 16 ++++++++-------- src/validation/mongodb.rs | 23 +++++++++++++++++++---- 6 files changed, 47 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f609a2..3f3092e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to this project will be documented in this file. +## [1.26.0] +- Fixed GitLab authentication bug +- Improved Baseten rule to be less noisy +- Added pre-commit and pre-receive installation hooks + +## [1.25.0] +- MongoDB validator now skips `mongodb+srv://` URIs and returns a message that validation was skipped +- Fixed noisy Baseten rule + ## [1.24.0] - Now generating DEB and RPM packages - Now releasing Docker images, and updated README diff --git a/Cargo.toml b/Cargo.toml index 1361d7a..0146316 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ publish = false [package] name = "kingfisher" -version = "1.24.0" +version = "1.26.0" description = "MongoDB's blazingly fast secret scanning and validation tool" edition.workspace = true rust-version.workspace = true diff --git a/data/rules/baseten.yml b/data/rules/baseten.yml index 17d2285..8773d6c 100644 --- a/data/rules/baseten.yml +++ b/data/rules/baseten.yml @@ -4,6 +4,9 @@ rules: pattern: | (?x) \b + baseten + (?:.|[\n\r]){0,32}? + \b ( [A-Za-z0-9]{8} \. @@ -13,10 +16,10 @@ rules: min_entropy: 3.4 confidence: medium examples: - - WSsDXzCD.uOcxAp7k82IvCKyY36TnpVbP4ZszP1qw - - crXCQC3W.CgCGGY1b9IfJan5TppW0Z07C9oMN2DmR - - h2wFkhFC.3WFVwVcxGFr4Qup0gyhvIuONwQxEpL0A - - XqbIpj04.x73j1zLUOEgGIKROqVbxsmggPdL8JvAY + - baseten_key = WSsDXzCD.uOcxAp7k82IvCKyY36TnpVbP4ZszP1qw + - baseten_key = crXCQC3W.CgCGGY1b9IfJan5TppW0Z07C9oMN2DmR + - baseten_key = h2wFkhFC.3WFVwVcxGFr4Qup0gyhvIuONwQxEpL0A + - baseten_key = XqbIpj04.x73j1zLUOEgGIKROqVbxsmggPdL8JvAY references: - https://docs.baseten.co/examples/vllm - https://docs.baseten.co/reference/management-api/api-keys/lists-the-users-api-keys diff --git a/data/rules/mongodb.yml b/data/rules/mongodb.yml index 63e4775..46fbcf8 100644 --- a/data/rules/mongodb.yml +++ b/data/rules/mongodb.yml @@ -91,4 +91,6 @@ rules: \b min_entropy: 3.5 examples: - - mdb_sa_sk_BdIX_jLzut2WTgglKzKvSgWMDDj5hEoTqdwOyLOL \ No newline at end of file + - mdb_sa_sk_BdIX_jLzut2WTgglKzKvSgWMDDj5hEoTqdwOyLOL + validation: + type: MongoDB \ No newline at end of file diff --git a/src/validation.rs b/src/validation.rs index 6cb3711..59f5362 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -538,16 +538,16 @@ async fn timed_validate_single_match<'a>( } match mongodb::validate_mongodb(&uri).await { - Ok(ok) => { + Ok((ok, msg)) => { m.validation_success = ok; - m.validation_response_body = if ok { - "MongoDB connection is valid." + m.validation_response_body = msg; + m.validation_response_status = if uri.starts_with("mongodb+srv://") { + StatusCode::CONTINUE + } else if ok { + StatusCode::OK } else { - "MongoDB connection failed." - } - .to_string(); - m.validation_response_status = - if ok { StatusCode::OK } else { StatusCode::UNAUTHORIZED }; + StatusCode::UNAUTHORIZED + }; } Err(e) => { m.validation_success = false; diff --git a/src/validation/mongodb.rs b/src/validation/mongodb.rs index efac6fa..74a82d5 100644 --- a/src/validation/mongodb.rs +++ b/src/validation/mongodb.rs @@ -19,15 +19,24 @@ const FAST_SELECT_MS: u64 = 300; const SRV_CONNECT_MS: u64 = 15_000; // gives Atlas a fighting chance const SRV_SELECT_MS: u64 = 15_000; -/// Validates a MongoDB URI in ≤ 2 s. Returns `Ok(true)` on successful ping. -pub async fn validate_mongodb(uri: &str) -> Result { +/// Validates a MongoDB URI in ≤ 2 s. Returns `(bool, String)` where the +/// boolean indicates success and the string provides a status message. +pub async fn validate_mongodb(uri: &str) -> Result<(bool, String)> { // ---- quick reject without touching the network if !looks_like_mongodb_uri(uri) { - return Ok(false); + return Ok((false, "Invalid MongoDB URI".to_string())); } let is_srv = uri.starts_with("mongodb+srv://"); + if is_srv { + // Skip SRV URIs to avoid slow DNS lookups and topology discovery. + return Ok(( + false, + "Validation skipped for mongodb+srv:// URI (performance reasons)".to_string(), + )); + } + // ---- build client opts let mut opts = ClientOptions::parse(uri).await?; if !is_srv { @@ -46,7 +55,13 @@ pub async fn validate_mongodb(uri: &str) -> Result { // ---- dial and ping let client = Client::with_options(opts)?; - Ok(client.database("admin").run_command(doc! { "ping": 1 }).await.is_ok()) + let ok = client.database("admin").run_command(doc! { "ping": 1 }).await.is_ok(); + let msg = if ok { + "MongoDB connection is valid.".to_string() + } else { + "MongoDB connection failed.".to_string() + }; + Ok((ok, msg)) } // pub fn generate_mongodb_cache_key(mongodb_uri: &str) -> String { From 9a87e30171637e6ddae9875d26f56532284a25d4 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Wed, 23 Jul 2025 19:58:24 -0700 Subject: [PATCH 3/4] Fixed version number --- CHANGELOG.md | 7 ++----- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f3092e..d89083c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,9 @@ All notable changes to this project will be documented in this file. -## [1.26.0] -- Fixed GitLab authentication bug -- Improved Baseten rule to be less noisy -- Added pre-commit and pre-receive installation hooks - ## [1.25.0] +- Fixed GitLab authentication bug +- Added pre-commit and pre-receive installation hooks - MongoDB validator now skips `mongodb+srv://` URIs and returns a message that validation was skipped - Fixed noisy Baseten rule diff --git a/Cargo.toml b/Cargo.toml index 0146316..930056a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ publish = false [package] name = "kingfisher" -version = "1.26.0" +version = "1.25.0" description = "MongoDB's blazingly fast secret scanning and validation tool" edition.workspace = true rust-version.workspace = true From 425dcbf0e9b58802de9b498d829191937a6a7aac Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Wed, 23 Jul 2025 20:47:16 -0700 Subject: [PATCH 4/4] Update install-prereceive-hook.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- install-prereceive-hook.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/install-prereceive-hook.sh b/install-prereceive-hook.sh index 3ca4081..f7a4d5e 100644 --- a/install-prereceive-hook.sh +++ b/install-prereceive-hook.sh @@ -20,8 +20,13 @@ if ! command -v kingfisher >/dev/null 2>&1; then fi while read -r oldrev newrev refname; do - git diff-tree --no-commit-id --name-only -r "$oldrev" "$newrev" -z | - xargs -0 --no-run-if-empty kingfisher scan --no-update-check + if [ "$oldrev" = "0000000000000000000000000000000000000000" ]; then + git diff-tree --name-only -r "$newrev" -z | + xargs -0 --no-run-if-empty kingfisher scan --no-update-check + else + git diff-tree --no-commit-id --name-only -r "$oldrev" "$newrev" -z | + xargs -0 --no-run-if-empty kingfisher scan --no-update-check + fi status=$? if [ "$status" -ne 0 ]; then echo "Kingfisher detected secrets in push. Push rejected." >&2