From 51bc64339c771c043bbd6d0589bcfeab764f603c Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Thu, 31 Jul 2025 16:49:46 -0700 Subject: [PATCH] - Fixed issue when more than 1 named capture group is used in a rule variable - Added 2 new liquid template filters: 'b64dec' and 'es256_sign' - Added custom validator for Coinbase, and a Coinbase rule that uses it --- CHANGELOG.md | 5 ++ Cargo.toml | 3 + data/rules/coinbase.yml | 67 ++++++++++++++++++++++ data/rules/privkey.yml | 6 +- docs/RULES.md | 2 + src/liquid_filters.rs | 86 ++++++++++++++++++++++++++++ src/main.rs | 26 +++++---- src/rules/rule.rs | 1 + src/validation.rs | 39 ++++++++++++- src/validation/coinbase.rs | 113 +++++++++++++++++++++++++++++++++++++ src/validation/utils.rs | 43 +++++++++++--- 11 files changed, 369 insertions(+), 22 deletions(-) create mode 100644 data/rules/coinbase.yml create mode 100644 src/validation/coinbase.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c1e055..3c2d7eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. +## [1.29.0] +- Fixed issue when more than 1 named capture group is used in a rule variable +- Added 2 new liquid template filters: `b64dec` and `es256_sign` +- Added custom validator for Coinbase, and a Coinbase rule that uses it + ## [1.28.0] - Added support for scanning Slack diff --git a/Cargo.toml b/Cargo.toml index fa9cf8c..e10fc6b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -188,6 +188,9 @@ ipnet = "2.11.0" jira_query = "1.6.0" oci-client = { version = "0.15", default-features = false, features = ["rustls-tls"] } walkdir = "2.5.0" +p256 = "0.13.2" +sec1 = "0.7.3" +rand_core = "0.9.3" [dependencies.tikv-jemallocator] version = "0.6" diff --git a/data/rules/coinbase.yml b/data/rules/coinbase.yml new file mode 100644 index 0000000..4819820 --- /dev/null +++ b/data/rules/coinbase.yml @@ -0,0 +1,67 @@ +rules: + - name: Coinbase Access Token + id: kingfisher.coinbase.1 + pattern: | + (?xi) + \b + coinbase + (?:.|[\n\r]){0,16}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,16}? + \b + ( + [a-zA-Z-0-9]{32} + ) + \b + min_entropy: 3.5 + examples: + - coinbase_token = 32iAkQCcHHYxXGx20VogBZoj27PC1ouI + references: + - https://docs.cloud.coinbase.com/wallet-sdk/docs/api-keys + validation: + type: Http + content: + request: + method: GET + url: https://api.coinbase.com/v2/user + headers: + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - name: Coinbase CDP API Key + id: kingfisher.coinbase.2 + pattern: | + (?xims) + "name"\s*:\s*" + (?Porganizations/[0-9a-f-]{36}/apiKeys/[0-9a-f-]{36})" + .*"privateKey"\s*:\s*" + (?P + -----BEGIN\sEC\s{0,1} + PRIVATE\sKEY + (\sBLOCK)? + ----- + [a-z0-9 /+=\r\n\\n]{32,}? + -----END\s + (?: + RSA | + PGP | + DSA | + OPENSSH | + ENCRYPTED | + EC + )? + \s{0,1} + PRIVATE\sKEY + (\sBLOCK)? + ----- + ) + validation: + type: Coinbase + examples: + - | + { + "name": "organizations/243873d8-c14e-436d-9cea-10d530cbe201/apiKeys/d29bb143-ad4c-234f-9bd7-c705c16b6d19", + "privateKey": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIDs+vr9F40Mer+qYksK3QhkSMfUXOZsbRVSrelWGnMh3oAoGCCqGSM49\nAwEHoUQDQgAEOXj2qKzLYx21D3plbOa81ilURS/4K1jzLXBvgwfUe4hWDgBdKQvq\nIiet5qqZEwVlR/LqKQEUlP8YLrjLFU8Unw==\n-----END EC PRIVATE KEY-----\n" + } \ No newline at end of file diff --git a/data/rules/privkey.yml b/data/rules/privkey.yml index b0e25b5..58dce65 100644 --- a/data/rules/privkey.yml +++ b/data/rules/privkey.yml @@ -45,8 +45,7 @@ rules: - name: Contains Private Key id: kingfisher.privkey.2 pattern: | - (?xi) - (?ims) + (?xims) ( -----BEGIN\s (?: @@ -68,7 +67,8 @@ rules: PGP | DSA | OPENSSH | - ENCRYPTED + ENCRYPTED | + EC )? \s{0,1} PRIVATE\sKEY diff --git a/docs/RULES.md b/docs/RULES.md index 01ae9ec..01cce17 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -107,6 +107,8 @@ Below is the complete list of Liquid filters available in Kingfisher, along with | --------------------- | -------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------- | | `b64enc` | – | Base64-encodes the input using the standard alphabet. | `{{ TOKEN \| b64enc }}` | | `b64url_enc` | – | URL-safe Base64 (no padding). Useful for JWT headers & payloads. | `{{ TOKEN \| b64url_enc }}` | +| `b64dec` | – | Decodes a Base64 string. | `{{ "aGVsbG8=" \| b64dec }}` | +| `es256_sign` | `key` (string) | Signs the input with an ECDSA P-256 private key and returns a Base64URL signature. | `{{ "data" \| es256_sign: PRIVKEY }}` | | `sha256` | – | Computes the SHA-256 hex digest of the input. | `{{ TOKEN \| sha256 }}` | | `hmac_sha1` | `key` (string) | Computes HMAC-SHA1 over the input, returns Base64-encoded result. | `{{ TOKEN \| hmac_sha1: "secret-key" }}` | | `hmac_sha256` | `key` (string) | Computes HMAC-SHA256 over the input, returns Base64-encoded result. | `{{ TOKEN \| hmac_sha256: "secret-key" }}` | diff --git a/src/liquid_filters.rs b/src/liquid_filters.rs index 89fb891..87428fb 100644 --- a/src/liquid_filters.rs +++ b/src/liquid_filters.rs @@ -6,6 +6,10 @@ use liquid_core::{ Display_filter, Error as LiquidError, Expression, Filter, FilterParameters, FilterReflection, FromFilterParameters, ParseFilter, Result, Runtime, Value, ValueView, }; + +use p256::ecdsa::{signature::Signer, SigningKey}; +use p256::pkcs8::DecodePrivateKey; +use sec1::DecodeEcPrivateKey; use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; use rand::{distr::Alphanumeric, Rng}; use sha1::Sha1; @@ -267,6 +271,69 @@ impl Filter for B64EncFilter { } } +#[derive(Debug, Clone, Default, FilterReflection, ParseFilter)] +#[filter(name = "b64dec", description = "Decodes a Base64 string", parsed(B64DecFilter))] +pub struct B64DecFilter; + +impl std::fmt::Display for B64DecFilter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "b64dec") + } +} + +impl Filter for B64DecFilter { + fn evaluate( + &self, + input: &dyn ValueView, + _runtime: &dyn Runtime, + ) -> Result { + let input_str = input.to_kstr(); + match general_purpose::STANDARD.decode(input_str.as_bytes()) { + Ok(bytes) => Ok(Value::scalar(String::from_utf8_lossy(&bytes).to_string())), + Err(e) => Err(LiquidError::with_msg(e.to_string())), + } + } +} + +#[derive(Debug, FilterParameters)] +struct Es256Args { + #[parameter(description = "PEM EC private key", arg_type = "str")] + key: Expression, +} + +#[derive(Clone, ParseFilter, FilterReflection, Default)] +#[filter( + name = "es256_sign", + description = "ECDSA P-256 SHA-256 signature (Base64URL)", + parameters(Es256Args), + parsed(Es256SignFilter) +)] +pub struct Es256Sign; + +#[derive(Debug, FromFilterParameters, Display_filter)] +#[name = "es256_sign"] +struct Es256SignFilter { + #[parameters] + args: Es256Args, +} + +impl Filter for Es256SignFilter { + fn evaluate(&self, input: &dyn ValueView, runtime: &dyn Runtime) -> Result { + let args = self.args.evaluate(runtime)?; + let key_pem = args.key.to_kstr(); + let signing_key = SigningKey::from_sec1_pem(&key_pem) + .or_else(|_| SigningKey::from_pkcs8_pem(&key_pem)) + .map_err(|e| LiquidError::with_msg(e.to_string()))?; + let sig: p256::ecdsa::Signature = signing_key.sign(input.to_kstr().as_bytes()); + // turn the signature into raw bytes… + let raw = sig.to_bytes(); + // …then Base64-URL encode + let b64 = general_purpose::URL_SAFE_NO_PAD.encode(raw); + Ok(Value::scalar(b64)) + } +} + + // ----------------------------------------------------------------------------- // Authentication & Security // ----------------------------------------------------------------------------- @@ -388,6 +455,8 @@ pub fn register_all(builder: liquid::ParserBuilder) -> liquid::ParserBuilder { .filter(UuidFilter::default()) .filter(JwtHeaderFilter::default()) .filter(B64EncFilter::default()) + .filter(B64DecFilter::default()) + .filter(Es256Sign::default()) .filter(RandomStringFilter::default()) .filter(HmacSha256::default()) .filter(HmacSha1::default()) @@ -424,6 +493,11 @@ mod tests { assert_eq!(render(r#"{{ "hello" | b64enc }}"#), "aGVsbG8="); } + #[test] + fn b64dec_filter() { + assert_eq!(render(r#"{{ "aGVsbG8=" | b64dec }}"#), "hello"); + } + #[test] fn sha256_filter() { let expect = format!("{:x}", Sha256::digest(b"hello")); @@ -441,6 +515,18 @@ mod tests { assert_eq!(render(r#"{{ "data" | hmac_sha1: "key1" }}"#), expect); } + #[test] + fn es256_sign_filter() { + let key = "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIDs+vr9F40Mer+qYksK3QhkSMfUXOZsbRVSrelWGnMh3oAoGCCqGSM49\nAwEHoUQDQgAEOXj2qKzLYx21D3plbOa81ilURS/4K1jzLXBvgwfUe4hWDgBdKQvq\nIiet5qqZEwVlR/LqKQEUlP8YLrjLFU8Unw==\n-----END EC PRIVATE KEY-----"; + use p256::ecdsa::{signature::Signer, SigningKey}; + let sk = SigningKey::from_sec1_pem(key).unwrap(); + let sig: p256::ecdsa::Signature = sk.sign(b"hello"); + let expect = general_purpose::URL_SAFE_NO_PAD.encode(sig.to_bytes()); + let tmpl = format!(r#"{{ "hello" | es256_sign: "{}" }}"#, key.replace('\n', "\\n")); + assert_eq!(render(&tmpl), expect); + } + + #[test] fn b64url_enc_filter() { assert_eq!( diff --git a/src/main.rs b/src/main.rs index 56f1e15..688f5de 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,20 +5,24 @@ // * Fallback - system allocator (`system-alloc` feature) // ──────────────────────────────────────────────────────────── -// --- jemalloc (opt-in) --- -#[cfg(feature = "use-jemalloc")] -#[global_allocator] -static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; +// // --- jemalloc (opt-in) --- +// #[cfg(feature = "use-jemalloc")] +// #[global_allocator] +// static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; -// --- mimalloc (default) --- -#[cfg(all(not(feature = "use-jemalloc"), not(feature = "system-alloc")))] -#[global_allocator] -static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; +// // --- mimalloc (default) --- +// #[cfg(all(not(feature = "use-jemalloc"), not(feature = "system-alloc")))] +// #[global_allocator] +// static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + +// // --- system allocator (explicit opt-out) --- +// #[cfg(feature = "system-alloc")] +// use std::alloc::System; +// #[cfg(feature = "system-alloc")] +// #[global_allocator] +// static GLOBAL: System = System; -// --- system allocator (explicit opt-out) --- -#[cfg(feature = "system-alloc")] use std::alloc::System; -#[cfg(feature = "system-alloc")] #[global_allocator] static GLOBAL: System = System; diff --git a/src/rules/rule.rs b/src/rules/rule.rs index a301a09..a47b172 100644 --- a/src/rules/rule.rs +++ b/src/rules/rule.rs @@ -35,6 +35,7 @@ fn default_true() -> bool { pub enum Validation { AWS, AzureStorage, + Coinbase, GCP, MongoDB, Postgres, diff --git a/src/validation.rs b/src/validation.rs index 59f5362..4823c27 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -25,6 +25,7 @@ use crate::{ mod aws; mod azure; +mod coinbase; mod gcp; mod httpvalidation; mod jwt; @@ -254,7 +255,7 @@ async fn timed_validate_single_match<'a>( if !missing.is_empty() { m.validation_success = false; m.validation_response_body = - format!("Validation skipped – missing dependent rules: {}", missing.join(", ")); + format!("Validation skipped - missing dependent rules: {}", missing.join(", ")); m.validation_response_status = StatusCode::PRECONDITION_REQUIRED; commit_and_return(m); return; @@ -828,7 +829,43 @@ async fn timed_validate_single_match<'a>( }, ); } + // ----------------------------------------------------- Coinbase validator + Some(Validation::Coinbase) => { + let cred_name = globals + .get("CRED_NAME") + .and_then(|v| v.as_scalar()) + .map(|s| s.into_owned().to_kstr().to_string()) + .unwrap_or_default(); + let private_key = globals + .get("PRIVATE_KEY") + .and_then(|v| v.as_scalar()) + .map(|s| s.into_owned().to_kstr().to_string()) + .unwrap_or_default(); + if cred_name.is_empty() || private_key.is_empty() { + m.validation_success = false; + m.validation_response_body = "Missing key name or private key.".to_string(); + m.validation_response_status = StatusCode::BAD_REQUEST; + commit_and_return(m); + return; + } + + match coinbase::validate_cdp_api_key(&cred_name, &private_key, client, parser, cache) + .await + { + Ok((ok, msg)) => { + m.validation_success = ok; + m.validation_response_body = msg; + m.validation_response_status = + if ok { StatusCode::OK } else { StatusCode::UNAUTHORIZED }; + } + Err(e) => { + m.validation_success = false; + m.validation_response_body = format!("Coinbase validation error: {}", e); + m.validation_response_status = StatusCode::BAD_GATEWAY; + } + } + } // --------------------------------------------------------- Raw / none Some(Validation::Raw(raw)) => { debug!("Raw validation not implemented: {}", raw); diff --git a/src/validation/coinbase.rs b/src/validation/coinbase.rs new file mode 100644 index 0000000..6cf1f1e --- /dev/null +++ b/src/validation/coinbase.rs @@ -0,0 +1,113 @@ +use std::collections::BTreeMap; +use std::time::Duration; + +use anyhow::{anyhow, Result}; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; +use chrono::Utc; +use p256::{ + ecdsa::{signature::Signer, SigningKey}, + pkcs8::DecodePrivateKey, + SecretKey, +}; +use rand::TryRngCore; + +use rand::rngs::OsRng; +use reqwest::{Client, StatusCode, Url}; +use sha1::{Digest, Sha1}; + +use crate::validation::{httpvalidation, Cache, CachedResponse, VALIDATION_CACHE_SECONDS}; + +pub fn generate_coinbase_cache_key(cred_name: &str, private_key: &str) -> String { + let mut h = Sha1::new(); + h.update(cred_name.as_bytes()); + h.update(b"\0"); + h.update(private_key.as_bytes()); + format!("COINBASE:{:x}", h.finalize()) +} + +pub async fn validate_cdp_api_key( + cred_name: &str, + private_key_pem: &str, + client: &Client, + parser: &liquid::Parser, + cache: &Cache, +) -> Result<(bool, String)> { + let cache_key = generate_coinbase_cache_key(cred_name, private_key_pem); + if let Some(entry) = cache.get(&cache_key) { + let c = entry.value(); + if c.timestamp.elapsed() < Duration::from_secs(VALIDATION_CACHE_SECONDS) { + return Ok((c.is_valid, c.body.clone())); + } + } + + let jwt = build_jwt("GET", "api.coinbase.com", "/v2/user", cred_name, private_key_pem)?; + + let url = Url::parse("https://api.coinbase.com/v2/user")?; + let headers = BTreeMap::from([("Authorization".to_string(), format!("Bearer {}", jwt))]); + let rb = httpvalidation::build_request_builder( + client, + "GET", + &url, + &headers, + &None, + parser, + &liquid::Object::new(), + ) + .map_err(|e| anyhow!(e))?; + let resp = + httpvalidation::retry_request(rb, 1, Duration::from_millis(500), Duration::from_secs(2)) + .await?; + + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + let ok = status == StatusCode::OK; + let msg = format!("{body}"); + + cache.insert(cache_key.clone(), CachedResponse::new(msg.clone(), status, ok)); + + Ok((ok, msg)) +} + +fn build_jwt( + method: &str, + host: &str, + endpoint: &str, + cred_name: &str, + pem: &str, +) -> Result { + let pem = + pem.replace("\r\n", "\n").replace("\\r\\n", "\n").replace("\\n", "\n").replace("\r", "\n"); + let secret_key = SecretKey::from_sec1_pem(&pem) + .or_else(|_| SecretKey::from_pkcs8_pem(&pem)) + .map_err(|e| anyhow!("invalid EC key: {e}"))?; + let signing_key = SigningKey::from(secret_key); + + let mut rng = OsRng; + let mut nonce = [0u8; 16]; + + let _ = rng.try_fill_bytes(&mut nonce); + + let header = serde_json::json!({ + "typ": "JWT", + "alg": "ES256", + "kid": cred_name, + "nonce": hex::encode(nonce), + }); + let header_b64 = URL_SAFE_NO_PAD.encode(header.to_string()); + + let now = Utc::now().timestamp(); + let claims = serde_json::json!({ + "sub": cred_name, + "iss": "cdp", + "nbf": now, + "exp": now + 60, + "uri": format!("{} {}{}", method, host, endpoint), + }); + let claims_b64 = URL_SAFE_NO_PAD.encode(claims.to_string()); + + let signing_input = format!("{header_b64}.{claims_b64}"); + let sig: p256::ecdsa::Signature = signing_key.sign(signing_input.as_bytes()); + let sig_b64 = URL_SAFE_NO_PAD.encode(sig.to_bytes()); + + Ok(format!("{signing_input}.{sig_b64}")) +} diff --git a/src/validation/utils.rs b/src/validation/utils.rs index 1f2d8ee..87372d4 100644 --- a/src/validation/utils.rs +++ b/src/validation/utils.rs @@ -2,19 +2,47 @@ use reqwest::Url; use tokio::net::lookup_host; use crate::validation::SerializableCaptures; -pub fn process_captures(captures: &SerializableCaptures) -> Vec<(String, String, usize, usize)> { - let has_multiple_captures = captures.captures.len() > 1; + +// pub fn process_captures(captures: &SerializableCaptures) -> Vec<(String, String, usize, usize)> { +// let has_multiple_captures = captures.captures.len() > 1; +// captures +// .captures +// .iter() +// .enumerate() +// .filter(|(idx, _)| !has_multiple_captures || *idx > 0) +// .map(|(_, capture)| { +// let name = capture.name.as_ref().map_or("TOKEN".to_string(), |n| n.to_uppercase()); +// (name, capture.value.clone().into_owned(), capture.start, capture.end) +// }) +// .collect() +// } + +/// Return (NAME, value, start, end) for every capture we care about. +/// +/// * If a capture has a name, use that (upper-cased) +/// * If it’s unnamed, fall back to `"TOKEN"` +/// * Skip the unnamed “whole-match” capture **only when** there are +/// additional captures to return. +pub fn process_captures( + captures: &SerializableCaptures, +) -> Vec<(String, String, usize, usize)> { + let multiple = captures.captures.len() > 1; + captures .captures .iter() - .enumerate() - .filter(|(idx, _)| !has_multiple_captures || *idx > 0) - .map(|(_, capture)| { - let name = capture.name.as_ref().map_or("TOKEN".to_string(), |n| n.to_uppercase()); - (name, capture.value.clone().into_owned(), capture.start, capture.end) + .filter(|cap| multiple.then(|| cap.name.is_some()).unwrap_or(true)) + .map(|cap| { + let name = cap + .name + .as_ref() + .map(|n| n.to_uppercase()) + .unwrap_or_else(|| "TOKEN".to_string()); + (name, cap.value.clone().into_owned(), cap.start, cap.end) }) .collect() } + pub fn find_closest_variable( captures: &[(String, String, usize, usize)], target_value: &String, @@ -47,6 +75,7 @@ pub fn find_closest_variable( } closest_value } + pub async fn check_url_resolvable(url: &Url) -> Result<(), Box> { let host = url.host_str().ok_or("No host in URL")?; let port = url.port().unwrap_or(if url.scheme() == "https" { 443 } else { 80 });