Merge pull request #65 from mongodb/development

v1.29.0
2025-07-31 19:00:39 -07:00 · 2025-07-31 19:00:39 -07:00 · 8a05329f77
commit 8a05329f77
parent d6ad8e56f0 f0a99dcfcd
11 changed files with 394 additions and 12 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -2,6 +2,11 @@

 All notable changes to this project will be documented in this file.

+## [1.29.0]
+- Fixed issue when more than 1 named capture group is used in a rule variable
+- Added a new liquid template filters: `b64dec`
+- Added custom validator for Coinbase, and a Coinbase rule that uses it
+
 ## [1.28.0]
 - Added support for scanning Slack

--- a/Cargo.toml
+++ b/Cargo.toml
@ -10,7 +10,7 @@ publish = false

 [package]
 name = "kingfisher"
-version = "1.28.0"
+version = "1.29.0"
 description = "MongoDB's blazingly fast secret scanning and validation tool"
 edition.workspace = true
 rust-version.workspace = true
@ -188,6 +188,10 @@ ipnet = "2.11.0"
 jira_query = "1.6.0"
 oci-client = { version = "0.15", default-features = false, features = ["rustls-tls"] }
 walkdir = "2.5.0"
+p256 = "0.13.2"
+sec1 = "0.7.3"
+rand_core = "0.9.3"
+ed25519-dalek = { version = "2.2", features = ["pkcs8"] }

 [dependencies.tikv-jemallocator]
 version = "0.6"
--- a/data/rules/coinbase.yml
+++ b/data/rules/coinbase.yml
@ -0,0 +1,80 @@
+rules:
+  - name: Coinbase Access Token
+    id: kingfisher.coinbase.1
+    pattern: |
+      (?xi)
+      \b
+        coinbase
+        (?:.|[\n\r]){0,16}?
+        (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN)
+        (?:.|[\n\r]){0,16}?
+        \b
+        (
+          [a-zA-Z-0-9]{32}
+        )
+        \b
+    min_entropy: 3.5
+    examples:
+      - coinbase_token = 32iAkQCcHHYxXGx20VogBZoj27PC1ouI
+    references:
+      - https://docs.cloud.coinbase.com/wallet-sdk/docs/api-keys
+    validation:
+      type: Http
+      content:
+        request:
+          method: GET
+          url: https://api.coinbase.com/v2/user
+          headers:
+            Authorization: "Bearer {{ TOKEN }}"
+          response_matcher:
+            - report_response: true
+            - type: StatusMatch
+              status: [200]
+  - name: Coinbase CDP API Key (ECDSA)
+    id: kingfisher.coinbase.2
+    pattern: |
+      (?xims)
+        "name"\s*:\s*"
+        (?P<CRED_NAME>organizations/[0-9a-f-]{36}/apiKeys/[0-9a-f-]{36})"
+        .*"privateKey"\s*:\s*"
+        (?P<PRIVATE_KEY>
+          -----BEGIN\sEC\s{0,1}
+          PRIVATE\sKEY
+          (\sBLOCK)?
+          -----
+          [a-z0-9 /+=\r\n\\n]{32,}?
+          -----END\s
+          (?:
+            RSA |
+            PGP |
+            DSA |
+            OPENSSH |
+            ENCRYPTED |
+            EC
+          )?
+          \s{0,1}
+          PRIVATE\sKEY
+          (\sBLOCK)?
+          -----
+        )
+    validation:
+      type: Coinbase
+    examples:
+      - |
+        {
+          "name": "organizations/243873d8-c14e-436d-9cea-10d530cbe201/apiKeys/d29bb143-ad4c-234f-9bd7-c705c16b6d19",
+          "privateKey": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIDs+vr9F40Mer+qYksK3QhkSMfUXOZsbRVSrelWGnMh3oAoGCCqGSM49\nAwEHoUQDQgAEOXj2qKzLYx21D3plbOa81ilURS/4K1jzLXBvgwfUe4hWDgBdKQvq\nIiet5qqZEwVlR/LqKQEUlP8YLrjLFU8Unw==\n-----END EC PRIVATE KEY-----\n"
+        }
+  - name: Coinbase CDP API Key (Ed25519)
+    id: kingfisher.coinbase.3
+    pattern: |
+      (?xis)
+      "id"\s*:\s*"(?P<CRED_NAME>[0-9a-f-]{36})"[^{]*?"privateKey"\s*:\s*"(?P<PRIVATE_KEY>[A-Za-z0-9+/=]{88})"
+    validation:
+      type: Coinbase
+    examples:
+      - |
+        {
+          "id": "413b23bf-4582-4e57-b33a-85d9527d9972",
+          "privateKey": "ygWq07YCO8UkmC9BE0PDBJNGhiu80yslsMUF9WnjPaIF5DBxb/wljjRuHhfuR/AMPC+kdgtL+mWKq/HOnq/YcQ=="
+        }
--- a/data/rules/privkey.yml
+++ b/data/rules/privkey.yml
@ -45,8 +45,7 @@ rules:
  - name: Contains Private Key
    id: kingfisher.privkey.2
    pattern: |
-      (?xi)                    
-      (?ims)
+      (?xims)
      (                       
        -----BEGIN\s
        (?:
@ -68,7 +67,8 @@ rules:
          PGP |
          DSA |
          OPENSSH |
-          ENCRYPTED
+          ENCRYPTED |
+          EC
        )?
        \s{0,1}
        PRIVATE\sKEY
--- a/docs/RULES.md
+++ b/docs/RULES.md
@ -107,6 +107,8 @@ Below is the complete list of Liquid filters available in Kingfisher, along with
 | --------------------- | -------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------- |
 | `b64enc`              | –                                            | Base64-encodes the input using the standard alphabet.                                                          | `{{ TOKEN \| b64enc }}`                                              |
 | `b64url_enc`          | –                                            | URL-safe Base64 (no padding). Useful for JWT headers & payloads.                                               | `{{ TOKEN \| b64url_enc }}`                                          |
+| `b64dec`              | –                                            | Decodes a Base64 string.                                                                                       | `{{ "aGVsbG8=" \| b64dec }}`                                          |
+| `es256_sign`          | `key` (string)                               | Signs the input with an ECDSA P-256 private key and returns a Base64URL signature.                             | `{{ "data" \| es256_sign: PRIVKEY }}`                                |
 | `sha256`              | –                                            | Computes the SHA-256 hex digest of the input.                                                                  | `{{ TOKEN \| sha256 }}`                                              |
 | `hmac_sha1`           | `key` (string)                               | Computes HMAC-SHA1 over the input, returns Base64-encoded result.                                              | `{{ TOKEN \| hmac_sha1: "secret-key" }}`                             |
 | `hmac_sha256`         | `key` (string)                               | Computes HMAC-SHA256 over the input, returns Base64-encoded result.                                            | `{{ TOKEN \| hmac_sha256: "secret-key" }}`                           |
--- a/src/liquid_filters.rs
+++ b/src/liquid_filters.rs
@ -6,6 +6,10 @@ use liquid_core::{
    Display_filter, Error as LiquidError, Expression, Filter, FilterParameters, FilterReflection,
    FromFilterParameters, ParseFilter, Result, Runtime, Value, ValueView,
 };
+
+use p256::ecdsa::{signature::Signer, SigningKey};
+use p256::pkcs8::DecodePrivateKey;
+use sec1::DecodeEcPrivateKey;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use rand::{distr::Alphanumeric, Rng};
 use sha1::Sha1;
@ -267,6 +271,31 @@ impl Filter for B64EncFilter {
    }
 }

+#[derive(Debug, Clone, Default, FilterReflection, ParseFilter)]
+#[filter(name = "b64dec", description = "Decodes a Base64 string", parsed(B64DecFilter))]
+pub struct B64DecFilter;
+
+impl std::fmt::Display for B64DecFilter {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "b64dec")
+    }
+}
+
+impl Filter for B64DecFilter {
+    fn evaluate(
+        &self,
+        input: &dyn ValueView,
+        _runtime: &dyn Runtime,
+    ) -> Result<Value, LiquidError> {
+        let input_str = input.to_kstr();
+        match general_purpose::STANDARD.decode(input_str.as_bytes()) {
+            Ok(bytes) => Ok(Value::scalar(String::from_utf8_lossy(&bytes).to_string())),
+            Err(e) => Err(LiquidError::with_msg(e.to_string())),
+        }
+    }
+}
+
+
 // -----------------------------------------------------------------------------
 // Authentication & Security
 // -----------------------------------------------------------------------------
@ -388,6 +417,7 @@ pub fn register_all(builder: liquid::ParserBuilder) -> liquid::ParserBuilder {
        .filter(UuidFilter::default())
        .filter(JwtHeaderFilter::default())
        .filter(B64EncFilter::default())
+        .filter(B64DecFilter::default())
        .filter(RandomStringFilter::default())
        .filter(HmacSha256::default())
        .filter(HmacSha1::default())
@ -424,6 +454,11 @@ mod tests {
        assert_eq!(render(r#"{{ "hello" | b64enc }}"#), "aGVsbG8=");
    }

+    #[test]
+    fn b64dec_filter() {
+        assert_eq!(render(r#"{{ "aGVsbG8=" | b64dec }}"#), "hello");
+    }
+
    #[test]
    fn sha256_filter() {
        let expect = format!("{:x}", Sha256::digest(b"hello"));
--- a/src/main.rs
+++ b/src/main.rs
@ -22,6 +22,10 @@ use std::alloc::System;
 #[global_allocator]
 static GLOBAL: System = System;

+// use std::alloc::System;
+// #[global_allocator]
+// static GLOBAL: System = System;
+
 use std::{
    io::Read,
    sync::{Arc, Mutex},
--- a/src/rules/rule.rs
+++ b/src/rules/rule.rs
@ -35,6 +35,7 @@ fn default_true() -> bool {
 pub enum Validation {
    AWS,
    AzureStorage,
+    Coinbase,
    GCP,
    MongoDB,
    Postgres,
--- a/src/validation.rs
+++ b/src/validation.rs
@ -25,6 +25,7 @@ use crate::{

 mod aws;
 mod azure;
+mod coinbase;
 mod gcp;
 mod httpvalidation;
 mod jwt;
@ -254,7 +255,7 @@ async fn timed_validate_single_match<'a>(
        if !missing.is_empty() {
            m.validation_success = false;
            m.validation_response_body =
-                format!("Validation skipped – missing dependent rules: {}", missing.join(", "));
+                format!("Validation skipped - missing dependent rules: {}", missing.join(", "));
            m.validation_response_status = StatusCode::PRECONDITION_REQUIRED;
            commit_and_return(m);
            return;
@ -828,7 +829,43 @@ async fn timed_validate_single_match<'a>(
                },
            );
        }
+        // ----------------------------------------------------- Coinbase validator
+        Some(Validation::Coinbase) => {
+            let cred_name = globals
+                .get("CRED_NAME")
+                .and_then(|v| v.as_scalar())
+                .map(|s| s.into_owned().to_kstr().to_string())
+                .unwrap_or_default();
+            let private_key = globals
+                .get("PRIVATE_KEY")
+                .and_then(|v| v.as_scalar())
+                .map(|s| s.into_owned().to_kstr().to_string())
+                .unwrap_or_default();

+            if cred_name.is_empty() || private_key.is_empty() {
+                m.validation_success = false;
+                m.validation_response_body = "Missing key name or private key.".to_string();
+                m.validation_response_status = StatusCode::BAD_REQUEST;
+                commit_and_return(m);
+                return;
+            }
+
+            match coinbase::validate_cdp_api_key(&cred_name, &private_key, client, parser, cache)
+                .await
+            {
+                Ok((ok, msg)) => {
+                    m.validation_success = ok;
+                    m.validation_response_body = msg;
+                    m.validation_response_status =
+                        if ok { StatusCode::OK } else { StatusCode::UNAUTHORIZED };
+                }
+                Err(e) => {
+                    m.validation_success = false;
+                    m.validation_response_body = format!("Coinbase validation error: {}", e);
+                    m.validation_response_status = StatusCode::BAD_GATEWAY;
+                }
+            }
+        }
        // --------------------------------------------------------- Raw / none
        Some(Validation::Raw(raw)) => {
            debug!("Raw validation not implemented: {}", raw);
--- a/src/validation/coinbase.rs
+++ b/src/validation/coinbase.rs
@ -0,0 +1,199 @@
+use std::collections::BTreeMap;
+use std::time::Duration;
+
+use anyhow::{anyhow, Result};
+use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _};
+use chrono::Utc;
+use p256::{
+    ecdsa::{signature::Signer as _, SigningKey},
+    pkcs8::DecodePrivateKey,
+    SecretKey,
+};
+use ed25519_dalek::{SigningKey as Ed25519Key, Signer as _};
+use rand::rngs::OsRng;
+use reqwest::{Client, StatusCode, Url};
+use sha1::{Digest, Sha1};
+use rand::TryRngCore;
+
+use crate::validation::{httpvalidation, Cache, CachedResponse, VALIDATION_CACHE_SECONDS};
+
+pub fn generate_coinbase_cache_key(cred_name: &str, private_key: &str) -> String {
+    let mut h = Sha1::new();
+    h.update(cred_name.as_bytes());
+    h.update(b"\0");
+    h.update(private_key.as_bytes());
+    format!("COINBASE:{:x}", h.finalize())
+}
+
+pub async fn validate_cdp_api_key(
+    cred_name: &str,
+    private_key_pem: &str,
+    client: &Client,
+    parser: &liquid::Parser,
+    cache: &Cache,
+) -> Result<(bool, String)> {
+    let cache_key = generate_coinbase_cache_key(cred_name, private_key_pem);
+    if let Some(entry) = cache.get(&cache_key) {
+        let c = entry.value();
+        if c.timestamp.elapsed() < Duration::from_secs(VALIDATION_CACHE_SECONDS) {
+            return Ok((c.is_valid, c.body.clone()));
+        }
+    }
+
+    let jwt = build_jwt("GET", "api.coinbase.com", "/v2/user", cred_name, private_key_pem)?;
+
+    let url = Url::parse("https://api.coinbase.com/v2/user")?;
+    let headers = BTreeMap::from([("Authorization".to_string(), format!("Bearer {}", jwt))]);
+    let rb = httpvalidation::build_request_builder(
+        client,
+        "GET",
+        &url,
+        &headers,
+        &None,
+        parser,
+        &liquid::Object::new(),
+    )
+    .map_err(|e| anyhow!(e))?;
+    let resp =
+        httpvalidation::retry_request(rb, 1, Duration::from_millis(500), Duration::from_secs(2))
+            .await?;
+
+    let status = resp.status();
+    let body = resp.text().await.unwrap_or_default();
+    let ok = status == StatusCode::OK;
+    let msg = body;
+
+    cache.insert(cache_key.clone(), CachedResponse::new(msg.clone(), status, ok));
+
+    Ok((ok, msg))
+}
+
+
+// fn build_jwt(
+//     method: &str,
+//     host: &str,
+//     endpoint: &str,
+//     cred_name: &str,
+//     pem: &str,
+// ) -> Result<String> {
+//     let pem =
+//         pem.replace("\r\n", "\n").replace("\\r\\n", "\n").replace("\\n", "\n").replace("\r", "\n");
+//     let secret_key = SecretKey::from_sec1_pem(&pem)
+//         .or_else(|_| SecretKey::from_pkcs8_pem(&pem))
+//         .map_err(|e| anyhow!("invalid EC key: {e}"))?;
+//     let signing_key = SigningKey::from(secret_key);
+
+//     let mut rng = OsRng;
+//     let mut nonce = [0u8; 16];
+    
+//     let _ = rng.try_fill_bytes(&mut nonce);
+
+//     let header = serde_json::json!({
+//         "typ": "JWT",
+//         "alg": "ES256",
+//         "kid": cred_name,
+//         "nonce": hex::encode(nonce),
+//     });
+//     let header_b64 = URL_SAFE_NO_PAD.encode(header.to_string());
+
+//     let now = Utc::now().timestamp();
+//     let claims = serde_json::json!({
+//         "sub": cred_name,
+//         "iss": "cdp",
+//         "nbf": now,
+//         "exp": now + 60,
+//         "uri": format!("{} {}{}", method, host, endpoint),
+//     });
+//     let claims_b64 = URL_SAFE_NO_PAD.encode(claims.to_string());
+
+//     let signing_input = format!("{header_b64}.{claims_b64}");
+//     let sig: p256::ecdsa::Signature = signing_key.sign(signing_input.as_bytes());
+//     let sig_b64 = URL_SAFE_NO_PAD.encode(sig.to_bytes());
+
+//     Ok(format!("{signing_input}.{sig_b64}"))
+// }
+
+fn build_jwt(
+    method: &str,
+    host: &str,
+    endpoint: &str,
+    key_name: &str,
+    pem: &str,
+) -> Result<String> {
+    let pem =
+        pem.replace("\r\n", "\n").replace("\\r\\n", "\n").replace("\\n", "\n").replace("\r", "\n");
+
+    let mut rng = OsRng;
+    let mut nonce = [0u8; 16];
+    
+    let _ = rng.try_fill_bytes(&mut nonce);
+
+    // Try ECDSA (PEM encoded EC key). Fallback to raw Ed25519 base64 key.
+    if let Ok(secret_key) = SecretKey::from_sec1_pem(&pem)
+        .or_else(|_| SecretKey::from_pkcs8_pem(&pem))
+    {
+        let signing_key = SigningKey::from(secret_key);
+        let header = serde_json::json!({
+            "typ": "JWT",
+            "alg": "ES256",
+            "kid": key_name,
+            "nonce": hex::encode(nonce),
+        });
+        let header_b64 = URL_SAFE_NO_PAD.encode(header.to_string());
+
+        let now = Utc::now().timestamp();
+        let claims = serde_json::json!({
+            "sub": key_name,
+            "iss": "cdp",
+            "nbf": now,
+            "exp": now + 120,
+            "uri": format!("{} {}{}", method, host, endpoint),
+        });
+        let claims_b64 = URL_SAFE_NO_PAD.encode(claims.to_string());
+
+        let signing_input = format!("{header_b64}.{claims_b64}");
+        let sig: p256::ecdsa::Signature = signing_key.sign(signing_input.as_bytes());
+        let sig_b64 = URL_SAFE_NO_PAD.encode(sig.to_bytes());
+
+        return Ok(format!("{signing_input}.{sig_b64}"));
+    } else {
+        // Assume base64-encoded Ed25519 keypair
+        let key_bytes = base64::engine::general_purpose::STANDARD
+            .decode(pem.as_bytes())
+            .map_err(|e| anyhow!("invalid base64 key: {e}"))?;
+        let signing_key = match key_bytes.len() {
+            32 => {
+                let arr: [u8; 32] = key_bytes[..32].try_into().unwrap();
+                Ed25519Key::from_bytes(&arr)
+            }
+            64 => {
+                let arr: [u8; 64] = key_bytes[..64].try_into().unwrap();
+                Ed25519Key::from_keypair_bytes(&arr).map_err(|e| anyhow!("invalid Ed25519 key: {e}"))?
+            }
+            _ => return Err(anyhow!("invalid Ed25519 key length")),
+        };
+
+        let header = serde_json::json!({
+            "typ": "JWT",
+            "alg": "EdDSA",
+            "kid": key_name,
+            "nonce": hex::encode(nonce),
+        });
+        let header_b64 = URL_SAFE_NO_PAD.encode(header.to_string());
+
+        let now = Utc::now().timestamp();
+        let claims = serde_json::json!({
+            "sub": key_name,
+            "iss": "cdp",
+            "nbf": now,
+            "exp": now + 120,
+            "uri": format!("{} {}{}", method, host, endpoint),
+        });
+        let claims_b64 = URL_SAFE_NO_PAD.encode(claims.to_string());
+
+        let signing_input = format!("{header_b64}.{claims_b64}");
+        let sig: ed25519_dalek::Signature = signing_key.sign(signing_input.as_bytes());
+        let sig_b64 = URL_SAFE_NO_PAD.encode(sig.to_bytes());
+        return Ok(format!("{signing_input}.{sig_b64}"));
+    }
+}
--- a/src/validation/utils.rs
+++ b/src/validation/utils.rs
@ -2,19 +2,33 @@ use reqwest::Url;
 use tokio::net::lookup_host;

 use crate::validation::SerializableCaptures;
-pub fn process_captures(captures: &SerializableCaptures) -> Vec<(String, String, usize, usize)> {
-    let has_multiple_captures = captures.captures.len() > 1;
+
+/// Return (NAME, value, start, end) for every capture we care about.
+///
+/// * If a capture has a name, use that (upper-cased)  
+/// * If it’s unnamed, fall back to `"TOKEN"`  
+/// * Skip the unnamed “whole-match” capture **only when** there are
+///   additional captures to return.
+pub fn process_captures(
+    captures: &SerializableCaptures,
+) -> Vec<(String, String, usize, usize)> {
+    let multiple = captures.captures.len() > 1;
+
    captures
        .captures
        .iter()
-        .enumerate()
-        .filter(|(idx, _)| !has_multiple_captures || *idx > 0)
-        .map(|(_, capture)| {
-            let name = capture.name.as_ref().map_or("TOKEN".to_string(), |n| n.to_uppercase());
-            (name, capture.value.clone().into_owned(), capture.start, capture.end)
+        .filter(|cap| !multiple || cap.name.is_some())
+        .map(|cap| {
+            let name = cap
+                .name
+                .as_ref()
+                .map(|n| n.to_uppercase())
+                .unwrap_or_else(|| "TOKEN".to_string());
+            (name, cap.value.clone().into_owned(), cap.start, cap.end)
        })
        .collect()
 }
+
 pub fn find_closest_variable(
    captures: &[(String, String, usize, usize)],
    target_value: &String,
@ -47,6 +61,7 @@ pub fn find_closest_variable(
    }
    closest_value
 }
+
 pub async fn check_url_resolvable(url: &Url) -> Result<(), Box<dyn std::error::Error>> {
    let host = url.host_str().ok_or("No host in URL")?;
    let port = url.port().unwrap_or(if url.scheme() == "https" { 443 } else { 80 });