eblume/kingfisher
1
0
Fork 0
forked from mirrors/kingfisher
Code Pull requests Activity Actions

- Enabled MongoDB URI validation

Browse source 
- AWS + GCP validators now respect HTTPS_PROXY and share a consistent user agent across AWS, GCP, and HTTP validation
This commit is contained in:
Mick Grove 2025-09-09 16:45:02 -07:00
commit 611f19fd74
8 changed files with 122 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

4
CHANGELOG.md
View file

@ -2,6 +2,10 @@
All notable changes to this project will be documented in this file.
## [1.49.0]
- Enabled MongoDB URI validation
- AWS + GCP validators now respect HTTPS_PROXY and share a consistent user agent across AWS, GCP, and HTTP validation
## [1.48.0]
- Improved error message when self-update cannot find the current binary
- Optimized memory usage via string interning and extensive data sharing

6
Cargo.toml
View file

@ -10,7 +10,7 @@ publish = false
[package]
name = "kingfisher"
version = "1.48.0"
version = "1.49.0"
description = "MongoDB's blazingly fast secret scanning and validation tool"
edition.workspace = true
rust-version.workspace = true
@ -122,6 +122,10 @@ reqwest-middleware = "0.4.2"
tracing-subscriber = {version = "0.3.19", features = ["env-filter"] }
tracing-core = "0.1.34"
tree-sitter = "0.25.8"
aws-smithy-runtime = "1.9.1"
aws-smithy-http-client = "1.1.1"
aws-smithy-runtime-api = "1.9.0"
aws-smithy-types = "1.3.2"
tree-sitter-bash = "0.25.0"
tree-sitter-c = "0.24.1"
tree-sitter-c-sharp = "0.23.1"

2
data/rules/mongodb.yml
View file

@ -80,6 +80,8 @@ rules:
- client = mongoc_client_new ("mongodb+srv://someuser:hunter2@my-atlas-rd941.mongodb.net/test?retryWrites=true&w=majority");
- "mongodb+srv://user:passw0rd@cluster0.something.mongodb.net/"
- "mongodb://mongoadmin:contoso@something.foo.mongodb.net/myFirstDatabase"
validation:
type: MongoDB
- name: MongoDB Atlas Service Account Token
id: kingfisher.mongodb.4
pattern: |

26
src/reporter.rs
View file

@ -290,6 +290,32 @@ impl DetailsReporter {
}
matches = expanded;
}
matches.sort_by(|a, b| {
let path_a = a
.origin
.first()
.full_path()
.map(|p| p.to_string_lossy().to_string())
.unwrap_or_default();
let path_b = b
.origin
.first()
.full_path()
.map(|p| p.to_string_lossy().to_string())
.unwrap_or_default();
path_a
.cmp(&path_b)
.then_with(|| {
a.m.location.source_span.start.line.cmp(&b.m.location.source_span.start.line)
})
.then_with(|| {
a.m.location
.source_span
.start
.column
.cmp(&b.m.location.source_span.start.column)
})
});
Ok(matches)
}

23
src/validation.rs
View file

@ -12,7 +12,7 @@ use dashmap::DashMap;
use http::StatusCode;
use liquid::Object;
use liquid_core::{Value, ValueView};
use once_cell::sync::OnceCell;
use once_cell::sync::{Lazy, OnceCell};
use reqwest::{header, header::HeaderValue, multipart, Client, Url};
use rustc_hash::FxHashMap;
use tokio::{sync::Notify, time};
@ -37,6 +37,17 @@ mod utils;
const VALIDATION_CACHE_SECONDS: u64 = 1200; // 20 minutes
const MAX_VALIDATION_BODY_LEN: usize = 2048;
pub static GLOBAL_USER_AGENT: Lazy<String> = Lazy::new(|| {
format!(
"{} {}/{}",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) \\ \
AppleWebKit/537.36 (KHTML, like Gecko) \\ \
Chrome/140.0.0.0 Safari/537.36",
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_VERSION")
)
});
// Use SkipMap-based cache instead of a mutex-wrapped FxHashMap.
type Cache = Arc<SkipMap<String, CachedResponse>>;
@ -405,16 +416,8 @@ async fn timed_validate_single_match<'a>(
&url,
) {
// add realistic UA & accept headers
let ua = format!(
"{} {}/{}",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) \
AppleWebKit/537.36 (KHTML, like Gecko) \
Chrome/132.0.0.0 Safari/537.36",
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_VERSION")
);
let std_headers = [
(header::USER_AGENT, ua.as_str()),
(header::USER_AGENT, GLOBAL_USER_AGENT.as_str()),
(header::ACCEPT , "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"),
(header::ACCEPT_LANGUAGE, "en-US,en;q=0.5"),
(header::ACCEPT_ENCODING, "gzip, deflate, br"),

63
src/validation/aws.rs
View file

@ -3,14 +3,55 @@ use std::time::Duration;
use anyhow::{anyhow, Result};
use aws_config::BehaviorVersion;
use aws_credential_types::Credentials;
use aws_sdk_sts::Client as StsClient;
use aws_sdk_sts::{config::Builder as StsConfigBuilder, Client as StsClient};
use aws_smithy_http_client::{
proxy::ProxyConfig, tls, Builder as HttpClientBuilder, ConnectorBuilder,
};
use aws_smithy_runtime_api::{
box_error::BoxError,
client::{
http::SharedHttpClient,
interceptors::{context::BeforeTransmitInterceptorContextMut, Intercept},
runtime_components::RuntimeComponents,
},
};
use aws_smithy_types::config_bag::ConfigBag;
use aws_types::region::Region;
use base32::Alphabet;
use byteorder::{BigEndian, ByteOrder};
use http::StatusCode;
use http::{
header::{HeaderValue, USER_AGENT},
StatusCode,
};
use crate::validation::GLOBAL_USER_AGENT;
use crate::validation::{Cache, CachedResponse, VALIDATION_CACHE_SECONDS};
#[derive(Debug)]
struct UaInterceptor;
impl Intercept for UaInterceptor {
fn name(&self) -> &'static str {
"ua"
}
fn modify_before_transmit(
&self,
context: &mut BeforeTransmitInterceptorContextMut<'_>,
_rc: &RuntimeComponents,
_cfg: &mut ConfigBag,
) -> std::result::Result<(), BoxError> {
let req = context.request_mut();
req.headers_mut().insert(
USER_AGENT,
HeaderValue::from_str(GLOBAL_USER_AGENT.as_str())
.map_err(|e| format!("invalid USER_AGENT header: {e}"))?,
);
Ok(())
}
}
/// Generate a standardized cache key for AWS validation attempts
pub fn generate_aws_cache_key(aws_access_key_id: &str, aws_secret_access_key: &str) -> String {
use sha1::{Digest, Sha1};
@ -62,14 +103,30 @@ pub async fn validate_aws_credentials(
None, // expiry
"static", // provider name
);
// Create HTTP client that respects proxy settings from the environment
let http_client: SharedHttpClient =
HttpClientBuilder::new().build_with_connector_fn(|settings, runtime_components| {
let mut conn_builder = ConnectorBuilder::default()
.tls_provider(tls::Provider::Rustls(tls::rustls_provider::CryptoMode::AwsLc));
conn_builder.set_connector_settings(settings.cloned());
if let Some(components) = runtime_components {
conn_builder.set_sleep_impl(components.sleep_impl());
}
conn_builder.set_proxy_config(Some(ProxyConfig::from_env()));
conn_builder.build()
});
// Create AWS config
let config = aws_config::defaults(BehaviorVersion::latest())
.region(Region::new("us-east-1"))
.credentials_provider(credentials)
.http_client(http_client)
.load()
.await;
// Create STS client
let sts_client = StsClient::new(&config);
let sts_config = StsConfigBuilder::from(&config).interceptor(UaInterceptor).build();
let sts_client = StsClient::from_conf(sts_config);
// Call get-caller-identity
match sts_client.get_caller_identity().send().await {
Ok(identity) => {

11
src/validation/gcp.rs
View file

@ -1,11 +1,12 @@
use std::sync::Arc;
use crate::validation::GLOBAL_USER_AGENT;
use anyhow::{anyhow, Result};
use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _};
use chrono::{Duration as ChronoDuration, Utc};
use once_cell::sync::OnceCell;
use pem::parse;
use reqwest::Client;
use reqwest::{Client, Proxy};
use ring::{rand, signature};
use serde_json::Value as JsonValue;
use tokio::sync::Semaphore;
@ -36,7 +37,13 @@ impl GcpValidator {
pub fn new() -> Result<Self> {
const MAX_CONCURRENT_VALIDATIONS: usize = 500;
let semaphore = Arc::new(Semaphore::new(MAX_CONCURRENT_VALIDATIONS));
let client = Client::builder().build()?;
let mut builder = Client::builder();
if let Ok(proxy) = std::env::var("HTTPS_PROXY").or_else(|_| std::env::var("https_proxy")) {
builder = builder.proxy(Proxy::all(&proxy)?);
}
let client = builder.user_agent(GLOBAL_USER_AGENT.as_str()).build()?;
Ok(Self { semaphore, client })
}

10
src/validation/httpvalidation.rs
View file

@ -1,5 +1,6 @@
use std::{collections::BTreeMap, future::Future, str::FromStr, time::Duration};
use crate::validation::GLOBAL_USER_AGENT;
use anyhow::{anyhow, Error, Result};
use http::StatusCode;
use liquid::Object;
@ -76,14 +77,9 @@ pub fn build_request_builder(
.map_err(|e| format!("Error processing headers: {}", e))?;
// Prepare a standard set of headers.
let user_agent = format!(
"{}/{}",
//"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_VERSION")
);
let user_agent = GLOBAL_USER_AGENT.as_str();
let standard_headers = [
(header::USER_AGENT, user_agent.as_str()),
(header::USER_AGENT, user_agent),
(
header::ACCEPT,
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",