From 611f19fd74fbe8a8dce5577b8058f85d450c0ad3 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Tue, 9 Sep 2025 16:45:02 -0700 Subject: [PATCH] - Enabled MongoDB URI validation - AWS + GCP validators now respect HTTPS_PROXY and share a consistent user agent across AWS, GCP, and HTTP validation --- CHANGELOG.md | 4 ++ Cargo.toml | 6 ++- data/rules/mongodb.yml | 2 + src/reporter.rs | 26 +++++++++++++ src/validation.rs | 23 +++++++----- src/validation/aws.rs | 63 ++++++++++++++++++++++++++++++-- src/validation/gcp.rs | 11 +++++- src/validation/httpvalidation.rs | 10 ++--- 8 files changed, 122 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb350a4..51f9ed0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to this project will be documented in this file. +## [1.49.0] +- Enabled MongoDB URI validation +- AWS + GCP validators now respect HTTPS_PROXY and share a consistent user agent across AWS, GCP, and HTTP validation + ## [1.48.0] - Improved error message when self-update cannot find the current binary - Optimized memory usage via string interning and extensive data sharing diff --git a/Cargo.toml b/Cargo.toml index 14cb21a..4d810df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ publish = false [package] name = "kingfisher" -version = "1.48.0" +version = "1.49.0" description = "MongoDB's blazingly fast secret scanning and validation tool" edition.workspace = true rust-version.workspace = true @@ -122,6 +122,10 @@ reqwest-middleware = "0.4.2" tracing-subscriber = {version = "0.3.19", features = ["env-filter"] } tracing-core = "0.1.34" tree-sitter = "0.25.8" +aws-smithy-runtime = "1.9.1" +aws-smithy-http-client = "1.1.1" +aws-smithy-runtime-api = "1.9.0" +aws-smithy-types = "1.3.2" tree-sitter-bash = "0.25.0" tree-sitter-c = "0.24.1" tree-sitter-c-sharp = "0.23.1" diff --git a/data/rules/mongodb.yml b/data/rules/mongodb.yml index 46fbcf8..dc02e94 100644 --- a/data/rules/mongodb.yml +++ b/data/rules/mongodb.yml @@ -80,6 +80,8 @@ rules: - client = mongoc_client_new ("mongodb+srv://someuser:hunter2@my-atlas-rd941.mongodb.net/test?retryWrites=true&w=majority"); - "mongodb+srv://user:passw0rd@cluster0.something.mongodb.net/" - "mongodb://mongoadmin:contoso@something.foo.mongodb.net/myFirstDatabase" + validation: + type: MongoDB - name: MongoDB Atlas Service Account Token id: kingfisher.mongodb.4 pattern: | diff --git a/src/reporter.rs b/src/reporter.rs index dacf75b..a4bf2fd 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -290,6 +290,32 @@ impl DetailsReporter { } matches = expanded; } + matches.sort_by(|a, b| { + let path_a = a + .origin + .first() + .full_path() + .map(|p| p.to_string_lossy().to_string()) + .unwrap_or_default(); + let path_b = b + .origin + .first() + .full_path() + .map(|p| p.to_string_lossy().to_string()) + .unwrap_or_default(); + path_a + .cmp(&path_b) + .then_with(|| { + a.m.location.source_span.start.line.cmp(&b.m.location.source_span.start.line) + }) + .then_with(|| { + a.m.location + .source_span + .start + .column + .cmp(&b.m.location.source_span.start.column) + }) + }); Ok(matches) } diff --git a/src/validation.rs b/src/validation.rs index a645ce7..8cf97ba 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -12,7 +12,7 @@ use dashmap::DashMap; use http::StatusCode; use liquid::Object; use liquid_core::{Value, ValueView}; -use once_cell::sync::OnceCell; +use once_cell::sync::{Lazy, OnceCell}; use reqwest::{header, header::HeaderValue, multipart, Client, Url}; use rustc_hash::FxHashMap; use tokio::{sync::Notify, time}; @@ -37,6 +37,17 @@ mod utils; const VALIDATION_CACHE_SECONDS: u64 = 1200; // 20 minutes const MAX_VALIDATION_BODY_LEN: usize = 2048; +pub static GLOBAL_USER_AGENT: Lazy = Lazy::new(|| { + format!( + "{} {}/{}", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \\ \ + AppleWebKit/537.36 (KHTML, like Gecko) \\ \ + Chrome/140.0.0.0 Safari/537.36", + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_VERSION") + ) +}); + // Use SkipMap-based cache instead of a mutex-wrapped FxHashMap. type Cache = Arc>; @@ -405,16 +416,8 @@ async fn timed_validate_single_match<'a>( &url, ) { // add realistic UA & accept headers - let ua = format!( - "{} {}/{}", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \ - AppleWebKit/537.36 (KHTML, like Gecko) \ - Chrome/132.0.0.0 Safari/537.36", - env!("CARGO_PKG_NAME"), - env!("CARGO_PKG_VERSION") - ); let std_headers = [ - (header::USER_AGENT, ua.as_str()), + (header::USER_AGENT, GLOBAL_USER_AGENT.as_str()), (header::ACCEPT , "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"), (header::ACCEPT_LANGUAGE, "en-US,en;q=0.5"), (header::ACCEPT_ENCODING, "gzip, deflate, br"), diff --git a/src/validation/aws.rs b/src/validation/aws.rs index 3f077c7..9b6079a 100644 --- a/src/validation/aws.rs +++ b/src/validation/aws.rs @@ -3,14 +3,55 @@ use std::time::Duration; use anyhow::{anyhow, Result}; use aws_config::BehaviorVersion; use aws_credential_types::Credentials; -use aws_sdk_sts::Client as StsClient; +use aws_sdk_sts::{config::Builder as StsConfigBuilder, Client as StsClient}; +use aws_smithy_http_client::{ + proxy::ProxyConfig, tls, Builder as HttpClientBuilder, ConnectorBuilder, +}; +use aws_smithy_runtime_api::{ + box_error::BoxError, + client::{ + http::SharedHttpClient, + interceptors::{context::BeforeTransmitInterceptorContextMut, Intercept}, + runtime_components::RuntimeComponents, + }, +}; +use aws_smithy_types::config_bag::ConfigBag; use aws_types::region::Region; use base32::Alphabet; use byteorder::{BigEndian, ByteOrder}; -use http::StatusCode; +use http::{ + header::{HeaderValue, USER_AGENT}, + StatusCode, +}; + +use crate::validation::GLOBAL_USER_AGENT; use crate::validation::{Cache, CachedResponse, VALIDATION_CACHE_SECONDS}; +#[derive(Debug)] +struct UaInterceptor; + +impl Intercept for UaInterceptor { + fn name(&self) -> &'static str { + "ua" + } + + fn modify_before_transmit( + &self, + context: &mut BeforeTransmitInterceptorContextMut<'_>, + _rc: &RuntimeComponents, + _cfg: &mut ConfigBag, + ) -> std::result::Result<(), BoxError> { + let req = context.request_mut(); + req.headers_mut().insert( + USER_AGENT, + HeaderValue::from_str(GLOBAL_USER_AGENT.as_str()) + .map_err(|e| format!("invalid USER_AGENT header: {e}"))?, + ); + Ok(()) + } +} + /// Generate a standardized cache key for AWS validation attempts pub fn generate_aws_cache_key(aws_access_key_id: &str, aws_secret_access_key: &str) -> String { use sha1::{Digest, Sha1}; @@ -62,14 +103,30 @@ pub async fn validate_aws_credentials( None, // expiry "static", // provider name ); + // Create HTTP client that respects proxy settings from the environment + let http_client: SharedHttpClient = + HttpClientBuilder::new().build_with_connector_fn(|settings, runtime_components| { + let mut conn_builder = ConnectorBuilder::default() + .tls_provider(tls::Provider::Rustls(tls::rustls_provider::CryptoMode::AwsLc)); + + conn_builder.set_connector_settings(settings.cloned()); + if let Some(components) = runtime_components { + conn_builder.set_sleep_impl(components.sleep_impl()); + } + conn_builder.set_proxy_config(Some(ProxyConfig::from_env())); + conn_builder.build() + }); + // Create AWS config let config = aws_config::defaults(BehaviorVersion::latest()) .region(Region::new("us-east-1")) .credentials_provider(credentials) + .http_client(http_client) .load() .await; // Create STS client - let sts_client = StsClient::new(&config); + let sts_config = StsConfigBuilder::from(&config).interceptor(UaInterceptor).build(); + let sts_client = StsClient::from_conf(sts_config); // Call get-caller-identity match sts_client.get_caller_identity().send().await { Ok(identity) => { diff --git a/src/validation/gcp.rs b/src/validation/gcp.rs index c03d77a..87499dc 100644 --- a/src/validation/gcp.rs +++ b/src/validation/gcp.rs @@ -1,11 +1,12 @@ use std::sync::Arc; +use crate::validation::GLOBAL_USER_AGENT; use anyhow::{anyhow, Result}; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; use chrono::{Duration as ChronoDuration, Utc}; use once_cell::sync::OnceCell; use pem::parse; -use reqwest::Client; +use reqwest::{Client, Proxy}; use ring::{rand, signature}; use serde_json::Value as JsonValue; use tokio::sync::Semaphore; @@ -36,7 +37,13 @@ impl GcpValidator { pub fn new() -> Result { const MAX_CONCURRENT_VALIDATIONS: usize = 500; let semaphore = Arc::new(Semaphore::new(MAX_CONCURRENT_VALIDATIONS)); - let client = Client::builder().build()?; + let mut builder = Client::builder(); + + if let Ok(proxy) = std::env::var("HTTPS_PROXY").or_else(|_| std::env::var("https_proxy")) { + builder = builder.proxy(Proxy::all(&proxy)?); + } + + let client = builder.user_agent(GLOBAL_USER_AGENT.as_str()).build()?; Ok(Self { semaphore, client }) } diff --git a/src/validation/httpvalidation.rs b/src/validation/httpvalidation.rs index cc866b3..ba2941b 100644 --- a/src/validation/httpvalidation.rs +++ b/src/validation/httpvalidation.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeMap, future::Future, str::FromStr, time::Duration}; +use crate::validation::GLOBAL_USER_AGENT; use anyhow::{anyhow, Error, Result}; use http::StatusCode; use liquid::Object; @@ -76,14 +77,9 @@ pub fn build_request_builder( .map_err(|e| format!("Error processing headers: {}", e))?; // Prepare a standard set of headers. - let user_agent = format!( - "{}/{}", - //"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36", - env!("CARGO_PKG_NAME"), - env!("CARGO_PKG_VERSION") - ); + let user_agent = GLOBAL_USER_AGENT.as_str(); let standard_headers = [ - (header::USER_AGENT, user_agent.as_str()), + (header::USER_AGENT, user_agent), ( header::ACCEPT, "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",