diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c1e055..79a3ac4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. +## [1.29.0] +- Fixed issue when more than 1 named capture group is used in a rule variable +- Added a new liquid template filters: `b64dec` +- Added custom validator for Coinbase, and a Coinbase rule that uses it + ## [1.28.0] - Added support for scanning Slack diff --git a/Cargo.toml b/Cargo.toml index fa9cf8c..e4576b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ publish = false [package] name = "kingfisher" -version = "1.28.0" +version = "1.29.0" description = "MongoDB's blazingly fast secret scanning and validation tool" edition.workspace = true rust-version.workspace = true @@ -188,6 +188,10 @@ ipnet = "2.11.0" jira_query = "1.6.0" oci-client = { version = "0.15", default-features = false, features = ["rustls-tls"] } walkdir = "2.5.0" +p256 = "0.13.2" +sec1 = "0.7.3" +rand_core = "0.9.3" +ed25519-dalek = { version = "2.2", features = ["pkcs8"] } [dependencies.tikv-jemallocator] version = "0.6" diff --git a/data/rules/coinbase.yml b/data/rules/coinbase.yml new file mode 100644 index 0000000..c5a5763 --- /dev/null +++ b/data/rules/coinbase.yml @@ -0,0 +1,80 @@ +rules: + - name: Coinbase Access Token + id: kingfisher.coinbase.1 + pattern: | + (?xi) + \b + coinbase + (?:.|[\n\r]){0,16}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,16}? + \b + ( + [a-zA-Z-0-9]{32} + ) + \b + min_entropy: 3.5 + examples: + - coinbase_token = 32iAkQCcHHYxXGx20VogBZoj27PC1ouI + references: + - https://docs.cloud.coinbase.com/wallet-sdk/docs/api-keys + validation: + type: Http + content: + request: + method: GET + url: https://api.coinbase.com/v2/user + headers: + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - name: Coinbase CDP API Key (ECDSA) + id: kingfisher.coinbase.2 + pattern: | + (?xims) + "name"\s*:\s*" + (?Porganizations/[0-9a-f-]{36}/apiKeys/[0-9a-f-]{36})" + .*"privateKey"\s*:\s*" + (?P + -----BEGIN\sEC\s{0,1} + PRIVATE\sKEY + (\sBLOCK)? + ----- + [a-z0-9 /+=\r\n\\n]{32,}? + -----END\s + (?: + RSA | + PGP | + DSA | + OPENSSH | + ENCRYPTED | + EC + )? + \s{0,1} + PRIVATE\sKEY + (\sBLOCK)? + ----- + ) + validation: + type: Coinbase + examples: + - | + { + "name": "organizations/243873d8-c14e-436d-9cea-10d530cbe201/apiKeys/d29bb143-ad4c-234f-9bd7-c705c16b6d19", + "privateKey": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIDs+vr9F40Mer+qYksK3QhkSMfUXOZsbRVSrelWGnMh3oAoGCCqGSM49\nAwEHoUQDQgAEOXj2qKzLYx21D3plbOa81ilURS/4K1jzLXBvgwfUe4hWDgBdKQvq\nIiet5qqZEwVlR/LqKQEUlP8YLrjLFU8Unw==\n-----END EC PRIVATE KEY-----\n" + } + - name: Coinbase CDP API Key (Ed25519) + id: kingfisher.coinbase.3 + pattern: | + (?xis) + "id"\s*:\s*"(?P[0-9a-f-]{36})"[^{]*?"privateKey"\s*:\s*"(?P[A-Za-z0-9+/=]{88})" + validation: + type: Coinbase + examples: + - | + { + "id": "413b23bf-4582-4e57-b33a-85d9527d9972", + "privateKey": "ygWq07YCO8UkmC9BE0PDBJNGhiu80yslsMUF9WnjPaIF5DBxb/wljjRuHhfuR/AMPC+kdgtL+mWKq/HOnq/YcQ==" + } \ No newline at end of file diff --git a/data/rules/privkey.yml b/data/rules/privkey.yml index b0e25b5..58dce65 100644 --- a/data/rules/privkey.yml +++ b/data/rules/privkey.yml @@ -45,8 +45,7 @@ rules: - name: Contains Private Key id: kingfisher.privkey.2 pattern: | - (?xi) - (?ims) + (?xims) ( -----BEGIN\s (?: @@ -68,7 +67,8 @@ rules: PGP | DSA | OPENSSH | - ENCRYPTED + ENCRYPTED | + EC )? \s{0,1} PRIVATE\sKEY diff --git a/docs/RULES.md b/docs/RULES.md index 01ae9ec..01cce17 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -107,6 +107,8 @@ Below is the complete list of Liquid filters available in Kingfisher, along with | --------------------- | -------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------- | | `b64enc` | – | Base64-encodes the input using the standard alphabet. | `{{ TOKEN \| b64enc }}` | | `b64url_enc` | – | URL-safe Base64 (no padding). Useful for JWT headers & payloads. | `{{ TOKEN \| b64url_enc }}` | +| `b64dec` | – | Decodes a Base64 string. | `{{ "aGVsbG8=" \| b64dec }}` | +| `es256_sign` | `key` (string) | Signs the input with an ECDSA P-256 private key and returns a Base64URL signature. | `{{ "data" \| es256_sign: PRIVKEY }}` | | `sha256` | – | Computes the SHA-256 hex digest of the input. | `{{ TOKEN \| sha256 }}` | | `hmac_sha1` | `key` (string) | Computes HMAC-SHA1 over the input, returns Base64-encoded result. | `{{ TOKEN \| hmac_sha1: "secret-key" }}` | | `hmac_sha256` | `key` (string) | Computes HMAC-SHA256 over the input, returns Base64-encoded result. | `{{ TOKEN \| hmac_sha256: "secret-key" }}` | diff --git a/src/liquid_filters.rs b/src/liquid_filters.rs index 89fb891..2142d83 100644 --- a/src/liquid_filters.rs +++ b/src/liquid_filters.rs @@ -6,6 +6,10 @@ use liquid_core::{ Display_filter, Error as LiquidError, Expression, Filter, FilterParameters, FilterReflection, FromFilterParameters, ParseFilter, Result, Runtime, Value, ValueView, }; + +use p256::ecdsa::{signature::Signer, SigningKey}; +use p256::pkcs8::DecodePrivateKey; +use sec1::DecodeEcPrivateKey; use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; use rand::{distr::Alphanumeric, Rng}; use sha1::Sha1; @@ -267,6 +271,31 @@ impl Filter for B64EncFilter { } } +#[derive(Debug, Clone, Default, FilterReflection, ParseFilter)] +#[filter(name = "b64dec", description = "Decodes a Base64 string", parsed(B64DecFilter))] +pub struct B64DecFilter; + +impl std::fmt::Display for B64DecFilter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "b64dec") + } +} + +impl Filter for B64DecFilter { + fn evaluate( + &self, + input: &dyn ValueView, + _runtime: &dyn Runtime, + ) -> Result { + let input_str = input.to_kstr(); + match general_purpose::STANDARD.decode(input_str.as_bytes()) { + Ok(bytes) => Ok(Value::scalar(String::from_utf8_lossy(&bytes).to_string())), + Err(e) => Err(LiquidError::with_msg(e.to_string())), + } + } +} + + // ----------------------------------------------------------------------------- // Authentication & Security // ----------------------------------------------------------------------------- @@ -388,6 +417,7 @@ pub fn register_all(builder: liquid::ParserBuilder) -> liquid::ParserBuilder { .filter(UuidFilter::default()) .filter(JwtHeaderFilter::default()) .filter(B64EncFilter::default()) + .filter(B64DecFilter::default()) .filter(RandomStringFilter::default()) .filter(HmacSha256::default()) .filter(HmacSha1::default()) @@ -424,6 +454,11 @@ mod tests { assert_eq!(render(r#"{{ "hello" | b64enc }}"#), "aGVsbG8="); } + #[test] + fn b64dec_filter() { + assert_eq!(render(r#"{{ "aGVsbG8=" | b64dec }}"#), "hello"); + } + #[test] fn sha256_filter() { let expect = format!("{:x}", Sha256::digest(b"hello")); diff --git a/src/main.rs b/src/main.rs index 56f1e15..06ef3cf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,6 +22,10 @@ use std::alloc::System; #[global_allocator] static GLOBAL: System = System; +// use std::alloc::System; +// #[global_allocator] +// static GLOBAL: System = System; + use std::{ io::Read, sync::{Arc, Mutex}, diff --git a/src/rules/rule.rs b/src/rules/rule.rs index a301a09..a47b172 100644 --- a/src/rules/rule.rs +++ b/src/rules/rule.rs @@ -35,6 +35,7 @@ fn default_true() -> bool { pub enum Validation { AWS, AzureStorage, + Coinbase, GCP, MongoDB, Postgres, diff --git a/src/validation.rs b/src/validation.rs index 59f5362..4823c27 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -25,6 +25,7 @@ use crate::{ mod aws; mod azure; +mod coinbase; mod gcp; mod httpvalidation; mod jwt; @@ -254,7 +255,7 @@ async fn timed_validate_single_match<'a>( if !missing.is_empty() { m.validation_success = false; m.validation_response_body = - format!("Validation skipped – missing dependent rules: {}", missing.join(", ")); + format!("Validation skipped - missing dependent rules: {}", missing.join(", ")); m.validation_response_status = StatusCode::PRECONDITION_REQUIRED; commit_and_return(m); return; @@ -828,7 +829,43 @@ async fn timed_validate_single_match<'a>( }, ); } + // ----------------------------------------------------- Coinbase validator + Some(Validation::Coinbase) => { + let cred_name = globals + .get("CRED_NAME") + .and_then(|v| v.as_scalar()) + .map(|s| s.into_owned().to_kstr().to_string()) + .unwrap_or_default(); + let private_key = globals + .get("PRIVATE_KEY") + .and_then(|v| v.as_scalar()) + .map(|s| s.into_owned().to_kstr().to_string()) + .unwrap_or_default(); + if cred_name.is_empty() || private_key.is_empty() { + m.validation_success = false; + m.validation_response_body = "Missing key name or private key.".to_string(); + m.validation_response_status = StatusCode::BAD_REQUEST; + commit_and_return(m); + return; + } + + match coinbase::validate_cdp_api_key(&cred_name, &private_key, client, parser, cache) + .await + { + Ok((ok, msg)) => { + m.validation_success = ok; + m.validation_response_body = msg; + m.validation_response_status = + if ok { StatusCode::OK } else { StatusCode::UNAUTHORIZED }; + } + Err(e) => { + m.validation_success = false; + m.validation_response_body = format!("Coinbase validation error: {}", e); + m.validation_response_status = StatusCode::BAD_GATEWAY; + } + } + } // --------------------------------------------------------- Raw / none Some(Validation::Raw(raw)) => { debug!("Raw validation not implemented: {}", raw); diff --git a/src/validation/coinbase.rs b/src/validation/coinbase.rs new file mode 100644 index 0000000..be6045f --- /dev/null +++ b/src/validation/coinbase.rs @@ -0,0 +1,199 @@ +use std::collections::BTreeMap; +use std::time::Duration; + +use anyhow::{anyhow, Result}; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; +use chrono::Utc; +use p256::{ + ecdsa::{signature::Signer as _, SigningKey}, + pkcs8::DecodePrivateKey, + SecretKey, +}; +use ed25519_dalek::{SigningKey as Ed25519Key, Signer as _}; +use rand::rngs::OsRng; +use reqwest::{Client, StatusCode, Url}; +use sha1::{Digest, Sha1}; +use rand::TryRngCore; + +use crate::validation::{httpvalidation, Cache, CachedResponse, VALIDATION_CACHE_SECONDS}; + +pub fn generate_coinbase_cache_key(cred_name: &str, private_key: &str) -> String { + let mut h = Sha1::new(); + h.update(cred_name.as_bytes()); + h.update(b"\0"); + h.update(private_key.as_bytes()); + format!("COINBASE:{:x}", h.finalize()) +} + +pub async fn validate_cdp_api_key( + cred_name: &str, + private_key_pem: &str, + client: &Client, + parser: &liquid::Parser, + cache: &Cache, +) -> Result<(bool, String)> { + let cache_key = generate_coinbase_cache_key(cred_name, private_key_pem); + if let Some(entry) = cache.get(&cache_key) { + let c = entry.value(); + if c.timestamp.elapsed() < Duration::from_secs(VALIDATION_CACHE_SECONDS) { + return Ok((c.is_valid, c.body.clone())); + } + } + + let jwt = build_jwt("GET", "api.coinbase.com", "/v2/user", cred_name, private_key_pem)?; + + let url = Url::parse("https://api.coinbase.com/v2/user")?; + let headers = BTreeMap::from([("Authorization".to_string(), format!("Bearer {}", jwt))]); + let rb = httpvalidation::build_request_builder( + client, + "GET", + &url, + &headers, + &None, + parser, + &liquid::Object::new(), + ) + .map_err(|e| anyhow!(e))?; + let resp = + httpvalidation::retry_request(rb, 1, Duration::from_millis(500), Duration::from_secs(2)) + .await?; + + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + let ok = status == StatusCode::OK; + let msg = body; + + cache.insert(cache_key.clone(), CachedResponse::new(msg.clone(), status, ok)); + + Ok((ok, msg)) +} + + +// fn build_jwt( +// method: &str, +// host: &str, +// endpoint: &str, +// cred_name: &str, +// pem: &str, +// ) -> Result { +// let pem = +// pem.replace("\r\n", "\n").replace("\\r\\n", "\n").replace("\\n", "\n").replace("\r", "\n"); +// let secret_key = SecretKey::from_sec1_pem(&pem) +// .or_else(|_| SecretKey::from_pkcs8_pem(&pem)) +// .map_err(|e| anyhow!("invalid EC key: {e}"))?; +// let signing_key = SigningKey::from(secret_key); + +// let mut rng = OsRng; +// let mut nonce = [0u8; 16]; + +// let _ = rng.try_fill_bytes(&mut nonce); + +// let header = serde_json::json!({ +// "typ": "JWT", +// "alg": "ES256", +// "kid": cred_name, +// "nonce": hex::encode(nonce), +// }); +// let header_b64 = URL_SAFE_NO_PAD.encode(header.to_string()); + +// let now = Utc::now().timestamp(); +// let claims = serde_json::json!({ +// "sub": cred_name, +// "iss": "cdp", +// "nbf": now, +// "exp": now + 60, +// "uri": format!("{} {}{}", method, host, endpoint), +// }); +// let claims_b64 = URL_SAFE_NO_PAD.encode(claims.to_string()); + +// let signing_input = format!("{header_b64}.{claims_b64}"); +// let sig: p256::ecdsa::Signature = signing_key.sign(signing_input.as_bytes()); +// let sig_b64 = URL_SAFE_NO_PAD.encode(sig.to_bytes()); + +// Ok(format!("{signing_input}.{sig_b64}")) +// } + +fn build_jwt( + method: &str, + host: &str, + endpoint: &str, + key_name: &str, + pem: &str, +) -> Result { + let pem = + pem.replace("\r\n", "\n").replace("\\r\\n", "\n").replace("\\n", "\n").replace("\r", "\n"); + + let mut rng = OsRng; + let mut nonce = [0u8; 16]; + + let _ = rng.try_fill_bytes(&mut nonce); + + // Try ECDSA (PEM encoded EC key). Fallback to raw Ed25519 base64 key. + if let Ok(secret_key) = SecretKey::from_sec1_pem(&pem) + .or_else(|_| SecretKey::from_pkcs8_pem(&pem)) + { + let signing_key = SigningKey::from(secret_key); + let header = serde_json::json!({ + "typ": "JWT", + "alg": "ES256", + "kid": key_name, + "nonce": hex::encode(nonce), + }); + let header_b64 = URL_SAFE_NO_PAD.encode(header.to_string()); + + let now = Utc::now().timestamp(); + let claims = serde_json::json!({ + "sub": key_name, + "iss": "cdp", + "nbf": now, + "exp": now + 120, + "uri": format!("{} {}{}", method, host, endpoint), + }); + let claims_b64 = URL_SAFE_NO_PAD.encode(claims.to_string()); + + let signing_input = format!("{header_b64}.{claims_b64}"); + let sig: p256::ecdsa::Signature = signing_key.sign(signing_input.as_bytes()); + let sig_b64 = URL_SAFE_NO_PAD.encode(sig.to_bytes()); + + return Ok(format!("{signing_input}.{sig_b64}")); + } else { + // Assume base64-encoded Ed25519 keypair + let key_bytes = base64::engine::general_purpose::STANDARD + .decode(pem.as_bytes()) + .map_err(|e| anyhow!("invalid base64 key: {e}"))?; + let signing_key = match key_bytes.len() { + 32 => { + let arr: [u8; 32] = key_bytes[..32].try_into().unwrap(); + Ed25519Key::from_bytes(&arr) + } + 64 => { + let arr: [u8; 64] = key_bytes[..64].try_into().unwrap(); + Ed25519Key::from_keypair_bytes(&arr).map_err(|e| anyhow!("invalid Ed25519 key: {e}"))? + } + _ => return Err(anyhow!("invalid Ed25519 key length")), + }; + + let header = serde_json::json!({ + "typ": "JWT", + "alg": "EdDSA", + "kid": key_name, + "nonce": hex::encode(nonce), + }); + let header_b64 = URL_SAFE_NO_PAD.encode(header.to_string()); + + let now = Utc::now().timestamp(); + let claims = serde_json::json!({ + "sub": key_name, + "iss": "cdp", + "nbf": now, + "exp": now + 120, + "uri": format!("{} {}{}", method, host, endpoint), + }); + let claims_b64 = URL_SAFE_NO_PAD.encode(claims.to_string()); + + let signing_input = format!("{header_b64}.{claims_b64}"); + let sig: ed25519_dalek::Signature = signing_key.sign(signing_input.as_bytes()); + let sig_b64 = URL_SAFE_NO_PAD.encode(sig.to_bytes()); + return Ok(format!("{signing_input}.{sig_b64}")); + } +} \ No newline at end of file diff --git a/src/validation/utils.rs b/src/validation/utils.rs index 1f2d8ee..9736aab 100644 --- a/src/validation/utils.rs +++ b/src/validation/utils.rs @@ -2,19 +2,33 @@ use reqwest::Url; use tokio::net::lookup_host; use crate::validation::SerializableCaptures; -pub fn process_captures(captures: &SerializableCaptures) -> Vec<(String, String, usize, usize)> { - let has_multiple_captures = captures.captures.len() > 1; + +/// Return (NAME, value, start, end) for every capture we care about. +/// +/// * If a capture has a name, use that (upper-cased) +/// * If it’s unnamed, fall back to `"TOKEN"` +/// * Skip the unnamed “whole-match” capture **only when** there are +/// additional captures to return. +pub fn process_captures( + captures: &SerializableCaptures, +) -> Vec<(String, String, usize, usize)> { + let multiple = captures.captures.len() > 1; + captures .captures .iter() - .enumerate() - .filter(|(idx, _)| !has_multiple_captures || *idx > 0) - .map(|(_, capture)| { - let name = capture.name.as_ref().map_or("TOKEN".to_string(), |n| n.to_uppercase()); - (name, capture.value.clone().into_owned(), capture.start, capture.end) + .filter(|cap| !multiple || cap.name.is_some()) + .map(|cap| { + let name = cap + .name + .as_ref() + .map(|n| n.to_uppercase()) + .unwrap_or_else(|| "TOKEN".to_string()); + (name, cap.value.clone().into_owned(), cap.start, cap.end) }) .collect() } + pub fn find_closest_variable( captures: &[(String, String, usize, usize)], target_value: &String, @@ -47,6 +61,7 @@ pub fn find_closest_variable( } closest_value } + pub async fn check_url_resolvable(url: &Url) -> Result<(), Box> { let host = url.host_str().ok_or("No host in URL")?; let port = url.port().unwrap_or(if url.scheme() == "https" { 443 } else { 80 });