From f0a99dcfcde604ba6ce974c97148af73123e7076 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Thu, 31 Jul 2025 18:29:21 -0700 Subject: [PATCH] bug fixes in response to code review. Also added support for ed25519 coinbase cdp api keys --- CHANGELOG.md | 2 +- Cargo.toml | 1 + data/rules/coinbase.yml | 15 +++- src/liquid_filters.rs | 51 ------------- src/main.rs | 30 ++++---- src/validation/coinbase.rs | 146 +++++++++++++++++++++++++++++-------- src/validation/utils.rs | 16 +--- 7 files changed, 148 insertions(+), 113 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c2d7eb..79a3ac4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. ## [1.29.0] - Fixed issue when more than 1 named capture group is used in a rule variable -- Added 2 new liquid template filters: `b64dec` and `es256_sign` +- Added a new liquid template filters: `b64dec` - Added custom validator for Coinbase, and a Coinbase rule that uses it ## [1.28.0] diff --git a/Cargo.toml b/Cargo.toml index 4cb3b18..e4576b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -191,6 +191,7 @@ walkdir = "2.5.0" p256 = "0.13.2" sec1 = "0.7.3" rand_core = "0.9.3" +ed25519-dalek = { version = "2.2", features = ["pkcs8"] } [dependencies.tikv-jemallocator] version = "0.6" diff --git a/data/rules/coinbase.yml b/data/rules/coinbase.yml index 4819820..c5a5763 100644 --- a/data/rules/coinbase.yml +++ b/data/rules/coinbase.yml @@ -30,7 +30,7 @@ rules: - report_response: true - type: StatusMatch status: [200] - - name: Coinbase CDP API Key + - name: Coinbase CDP API Key (ECDSA) id: kingfisher.coinbase.2 pattern: | (?xims) @@ -64,4 +64,17 @@ rules: { "name": "organizations/243873d8-c14e-436d-9cea-10d530cbe201/apiKeys/d29bb143-ad4c-234f-9bd7-c705c16b6d19", "privateKey": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIDs+vr9F40Mer+qYksK3QhkSMfUXOZsbRVSrelWGnMh3oAoGCCqGSM49\nAwEHoUQDQgAEOXj2qKzLYx21D3plbOa81ilURS/4K1jzLXBvgwfUe4hWDgBdKQvq\nIiet5qqZEwVlR/LqKQEUlP8YLrjLFU8Unw==\n-----END EC PRIVATE KEY-----\n" + } + - name: Coinbase CDP API Key (Ed25519) + id: kingfisher.coinbase.3 + pattern: | + (?xis) + "id"\s*:\s*"(?P[0-9a-f-]{36})"[^{]*?"privateKey"\s*:\s*"(?P[A-Za-z0-9+/=]{88})" + validation: + type: Coinbase + examples: + - | + { + "id": "413b23bf-4582-4e57-b33a-85d9527d9972", + "privateKey": "ygWq07YCO8UkmC9BE0PDBJNGhiu80yslsMUF9WnjPaIF5DBxb/wljjRuHhfuR/AMPC+kdgtL+mWKq/HOnq/YcQ==" } \ No newline at end of file diff --git a/src/liquid_filters.rs b/src/liquid_filters.rs index 87428fb..2142d83 100644 --- a/src/liquid_filters.rs +++ b/src/liquid_filters.rs @@ -295,44 +295,6 @@ impl Filter for B64DecFilter { } } -#[derive(Debug, FilterParameters)] -struct Es256Args { - #[parameter(description = "PEM EC private key", arg_type = "str")] - key: Expression, -} - -#[derive(Clone, ParseFilter, FilterReflection, Default)] -#[filter( - name = "es256_sign", - description = "ECDSA P-256 SHA-256 signature (Base64URL)", - parameters(Es256Args), - parsed(Es256SignFilter) -)] -pub struct Es256Sign; - -#[derive(Debug, FromFilterParameters, Display_filter)] -#[name = "es256_sign"] -struct Es256SignFilter { - #[parameters] - args: Es256Args, -} - -impl Filter for Es256SignFilter { - fn evaluate(&self, input: &dyn ValueView, runtime: &dyn Runtime) -> Result { - let args = self.args.evaluate(runtime)?; - let key_pem = args.key.to_kstr(); - let signing_key = SigningKey::from_sec1_pem(&key_pem) - .or_else(|_| SigningKey::from_pkcs8_pem(&key_pem)) - .map_err(|e| LiquidError::with_msg(e.to_string()))?; - let sig: p256::ecdsa::Signature = signing_key.sign(input.to_kstr().as_bytes()); - // turn the signature into raw bytes… - let raw = sig.to_bytes(); - // …then Base64-URL encode - let b64 = general_purpose::URL_SAFE_NO_PAD.encode(raw); - Ok(Value::scalar(b64)) - } -} - // ----------------------------------------------------------------------------- // Authentication & Security @@ -456,7 +418,6 @@ pub fn register_all(builder: liquid::ParserBuilder) -> liquid::ParserBuilder { .filter(JwtHeaderFilter::default()) .filter(B64EncFilter::default()) .filter(B64DecFilter::default()) - .filter(Es256Sign::default()) .filter(RandomStringFilter::default()) .filter(HmacSha256::default()) .filter(HmacSha1::default()) @@ -515,18 +476,6 @@ mod tests { assert_eq!(render(r#"{{ "data" | hmac_sha1: "key1" }}"#), expect); } - #[test] - fn es256_sign_filter() { - let key = "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIDs+vr9F40Mer+qYksK3QhkSMfUXOZsbRVSrelWGnMh3oAoGCCqGSM49\nAwEHoUQDQgAEOXj2qKzLYx21D3plbOa81ilURS/4K1jzLXBvgwfUe4hWDgBdKQvq\nIiet5qqZEwVlR/LqKQEUlP8YLrjLFU8Unw==\n-----END EC PRIVATE KEY-----"; - use p256::ecdsa::{signature::Signer, SigningKey}; - let sk = SigningKey::from_sec1_pem(key).unwrap(); - let sig: p256::ecdsa::Signature = sk.sign(b"hello"); - let expect = general_purpose::URL_SAFE_NO_PAD.encode(sig.to_bytes()); - let tmpl = format!(r#"{{ "hello" | es256_sign: "{}" }}"#, key.replace('\n', "\\n")); - assert_eq!(render(&tmpl), expect); - } - - #[test] fn b64url_enc_filter() { assert_eq!( diff --git a/src/main.rs b/src/main.rs index 688f5de..06ef3cf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,27 +5,27 @@ // * Fallback - system allocator (`system-alloc` feature) // ──────────────────────────────────────────────────────────── -// // --- jemalloc (opt-in) --- -// #[cfg(feature = "use-jemalloc")] -// #[global_allocator] -// static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; +// --- jemalloc (opt-in) --- +#[cfg(feature = "use-jemalloc")] +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; -// // --- mimalloc (default) --- -// #[cfg(all(not(feature = "use-jemalloc"), not(feature = "system-alloc")))] -// #[global_allocator] -// static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; - -// // --- system allocator (explicit opt-out) --- -// #[cfg(feature = "system-alloc")] -// use std::alloc::System; -// #[cfg(feature = "system-alloc")] -// #[global_allocator] -// static GLOBAL: System = System; +// --- mimalloc (default) --- +#[cfg(all(not(feature = "use-jemalloc"), not(feature = "system-alloc")))] +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; +// --- system allocator (explicit opt-out) --- +#[cfg(feature = "system-alloc")] use std::alloc::System; +#[cfg(feature = "system-alloc")] #[global_allocator] static GLOBAL: System = System; +// use std::alloc::System; +// #[global_allocator] +// static GLOBAL: System = System; + use std::{ io::Read, sync::{Arc, Mutex}, diff --git a/src/validation/coinbase.rs b/src/validation/coinbase.rs index 6cf1f1e..be6045f 100644 --- a/src/validation/coinbase.rs +++ b/src/validation/coinbase.rs @@ -5,15 +5,15 @@ use anyhow::{anyhow, Result}; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; use chrono::Utc; use p256::{ - ecdsa::{signature::Signer, SigningKey}, + ecdsa::{signature::Signer as _, SigningKey}, pkcs8::DecodePrivateKey, SecretKey, }; -use rand::TryRngCore; - +use ed25519_dalek::{SigningKey as Ed25519Key, Signer as _}; use rand::rngs::OsRng; use reqwest::{Client, StatusCode, Url}; use sha1::{Digest, Sha1}; +use rand::TryRngCore; use crate::validation::{httpvalidation, Cache, CachedResponse, VALIDATION_CACHE_SECONDS}; @@ -61,53 +61,139 @@ pub async fn validate_cdp_api_key( let status = resp.status(); let body = resp.text().await.unwrap_or_default(); let ok = status == StatusCode::OK; - let msg = format!("{body}"); + let msg = body; cache.insert(cache_key.clone(), CachedResponse::new(msg.clone(), status, ok)); Ok((ok, msg)) } + +// fn build_jwt( +// method: &str, +// host: &str, +// endpoint: &str, +// cred_name: &str, +// pem: &str, +// ) -> Result { +// let pem = +// pem.replace("\r\n", "\n").replace("\\r\\n", "\n").replace("\\n", "\n").replace("\r", "\n"); +// let secret_key = SecretKey::from_sec1_pem(&pem) +// .or_else(|_| SecretKey::from_pkcs8_pem(&pem)) +// .map_err(|e| anyhow!("invalid EC key: {e}"))?; +// let signing_key = SigningKey::from(secret_key); + +// let mut rng = OsRng; +// let mut nonce = [0u8; 16]; + +// let _ = rng.try_fill_bytes(&mut nonce); + +// let header = serde_json::json!({ +// "typ": "JWT", +// "alg": "ES256", +// "kid": cred_name, +// "nonce": hex::encode(nonce), +// }); +// let header_b64 = URL_SAFE_NO_PAD.encode(header.to_string()); + +// let now = Utc::now().timestamp(); +// let claims = serde_json::json!({ +// "sub": cred_name, +// "iss": "cdp", +// "nbf": now, +// "exp": now + 60, +// "uri": format!("{} {}{}", method, host, endpoint), +// }); +// let claims_b64 = URL_SAFE_NO_PAD.encode(claims.to_string()); + +// let signing_input = format!("{header_b64}.{claims_b64}"); +// let sig: p256::ecdsa::Signature = signing_key.sign(signing_input.as_bytes()); +// let sig_b64 = URL_SAFE_NO_PAD.encode(sig.to_bytes()); + +// Ok(format!("{signing_input}.{sig_b64}")) +// } + fn build_jwt( method: &str, host: &str, endpoint: &str, - cred_name: &str, + key_name: &str, pem: &str, ) -> Result { let pem = pem.replace("\r\n", "\n").replace("\\r\\n", "\n").replace("\\n", "\n").replace("\r", "\n"); - let secret_key = SecretKey::from_sec1_pem(&pem) - .or_else(|_| SecretKey::from_pkcs8_pem(&pem)) - .map_err(|e| anyhow!("invalid EC key: {e}"))?; - let signing_key = SigningKey::from(secret_key); let mut rng = OsRng; let mut nonce = [0u8; 16]; let _ = rng.try_fill_bytes(&mut nonce); - let header = serde_json::json!({ - "typ": "JWT", - "alg": "ES256", - "kid": cred_name, - "nonce": hex::encode(nonce), - }); - let header_b64 = URL_SAFE_NO_PAD.encode(header.to_string()); + // Try ECDSA (PEM encoded EC key). Fallback to raw Ed25519 base64 key. + if let Ok(secret_key) = SecretKey::from_sec1_pem(&pem) + .or_else(|_| SecretKey::from_pkcs8_pem(&pem)) + { + let signing_key = SigningKey::from(secret_key); + let header = serde_json::json!({ + "typ": "JWT", + "alg": "ES256", + "kid": key_name, + "nonce": hex::encode(nonce), + }); + let header_b64 = URL_SAFE_NO_PAD.encode(header.to_string()); - let now = Utc::now().timestamp(); - let claims = serde_json::json!({ - "sub": cred_name, - "iss": "cdp", - "nbf": now, - "exp": now + 60, - "uri": format!("{} {}{}", method, host, endpoint), - }); - let claims_b64 = URL_SAFE_NO_PAD.encode(claims.to_string()); + let now = Utc::now().timestamp(); + let claims = serde_json::json!({ + "sub": key_name, + "iss": "cdp", + "nbf": now, + "exp": now + 120, + "uri": format!("{} {}{}", method, host, endpoint), + }); + let claims_b64 = URL_SAFE_NO_PAD.encode(claims.to_string()); - let signing_input = format!("{header_b64}.{claims_b64}"); - let sig: p256::ecdsa::Signature = signing_key.sign(signing_input.as_bytes()); - let sig_b64 = URL_SAFE_NO_PAD.encode(sig.to_bytes()); + let signing_input = format!("{header_b64}.{claims_b64}"); + let sig: p256::ecdsa::Signature = signing_key.sign(signing_input.as_bytes()); + let sig_b64 = URL_SAFE_NO_PAD.encode(sig.to_bytes()); - Ok(format!("{signing_input}.{sig_b64}")) -} + return Ok(format!("{signing_input}.{sig_b64}")); + } else { + // Assume base64-encoded Ed25519 keypair + let key_bytes = base64::engine::general_purpose::STANDARD + .decode(pem.as_bytes()) + .map_err(|e| anyhow!("invalid base64 key: {e}"))?; + let signing_key = match key_bytes.len() { + 32 => { + let arr: [u8; 32] = key_bytes[..32].try_into().unwrap(); + Ed25519Key::from_bytes(&arr) + } + 64 => { + let arr: [u8; 64] = key_bytes[..64].try_into().unwrap(); + Ed25519Key::from_keypair_bytes(&arr).map_err(|e| anyhow!("invalid Ed25519 key: {e}"))? + } + _ => return Err(anyhow!("invalid Ed25519 key length")), + }; + + let header = serde_json::json!({ + "typ": "JWT", + "alg": "EdDSA", + "kid": key_name, + "nonce": hex::encode(nonce), + }); + let header_b64 = URL_SAFE_NO_PAD.encode(header.to_string()); + + let now = Utc::now().timestamp(); + let claims = serde_json::json!({ + "sub": key_name, + "iss": "cdp", + "nbf": now, + "exp": now + 120, + "uri": format!("{} {}{}", method, host, endpoint), + }); + let claims_b64 = URL_SAFE_NO_PAD.encode(claims.to_string()); + + let signing_input = format!("{header_b64}.{claims_b64}"); + let sig: ed25519_dalek::Signature = signing_key.sign(signing_input.as_bytes()); + let sig_b64 = URL_SAFE_NO_PAD.encode(sig.to_bytes()); + return Ok(format!("{signing_input}.{sig_b64}")); + } +} \ No newline at end of file diff --git a/src/validation/utils.rs b/src/validation/utils.rs index 87372d4..9736aab 100644 --- a/src/validation/utils.rs +++ b/src/validation/utils.rs @@ -3,20 +3,6 @@ use tokio::net::lookup_host; use crate::validation::SerializableCaptures; -// pub fn process_captures(captures: &SerializableCaptures) -> Vec<(String, String, usize, usize)> { -// let has_multiple_captures = captures.captures.len() > 1; -// captures -// .captures -// .iter() -// .enumerate() -// .filter(|(idx, _)| !has_multiple_captures || *idx > 0) -// .map(|(_, capture)| { -// let name = capture.name.as_ref().map_or("TOKEN".to_string(), |n| n.to_uppercase()); -// (name, capture.value.clone().into_owned(), capture.start, capture.end) -// }) -// .collect() -// } - /// Return (NAME, value, start, end) for every capture we care about. /// /// * If a capture has a name, use that (upper-cased) @@ -31,7 +17,7 @@ pub fn process_captures( captures .captures .iter() - .filter(|cap| multiple.then(|| cap.name.is_some()).unwrap_or(true)) + .filter(|cap| !multiple || cap.name.is_some()) .map(|cap| { let name = cap .name