From c6b10f0b47c844ca830f5f6b1900a5bf95f56488 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 16 Nov 2025 23:25:42 -0800 Subject: [PATCH] - Skip reporting MongoDB and Postgres findings when their connection strings cannot be parsed, even when validation is disabled. - Improve MySQL detection by broadening URI coverage and adding live validation that skips clearly invalid connection strings. --- CHANGELOG.md | 1 + Cargo.toml | 1 + data/rules/uri.yml | 4 ++ mysqltest.py | 107 ++++++++++++++++++++++++++++++++++++ src/validation.rs | 31 ++++++++++- tests/live_db_validation.rs | 101 ++++++++++++++++++++++++++++++++++ 6 files changed, 242 insertions(+), 3 deletions(-) create mode 100644 mysqltest.py create mode 100644 tests/live_db_validation.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6286da2..fd69ef5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ All notable changes to this project will be documented in this file. ## [v1.65.0] - Skip reporting MongoDB and Postgres findings when their connection strings cannot be parsed, even when validation is disabled. - Improve MySQL detection by broadening URI coverage and adding live validation that skips clearly invalid connection strings. +- Added a helper to truncate validation response bodies only at UTF-8 character boundaries to prevent panics during validation. ## [v1.64.0] - Fixed a bug when using --redact, that broke validation diff --git a/Cargo.toml b/Cargo.toml index b52f29a..d6b0a0e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -221,6 +221,7 @@ temp-env = "0.3.6" wiremock = "0.6.4" git2 = "0.20.2" rand_chacha = "0.9.0" +testcontainers = "0.15.0" [profile.release] debug = false diff --git a/data/rules/uri.yml b/data/rules/uri.yml index 04c8057..45fde52 100644 --- a/data/rules/uri.yml +++ b/data/rules/uri.yml @@ -16,6 +16,10 @@ rules: (?:\?[A-Za-z0-9\-._~%!$&'()*,;=:@/?%]*)? (?:\#[A-Za-z0-9\-._~%!$&'()*,;=:@/?%]*)? ) + pattern_requirements: + ignore_if_contains: + - "*****" + - "xxxxx" min_entropy: 4.0 confidence: medium examples: diff --git a/mysqltest.py b/mysqltest.py new file mode 100644 index 0000000..bd567a6 --- /dev/null +++ b/mysqltest.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Start a MySQL 8.4 Docker container with a known root password. + +The script ensures the container keeps running and writes the connection +string to /tmp/mysql-creds.log for convenience. +""" +import subprocess +import sys +import time +from pathlib import Path + +IMAGE = "mysql:8.4" +CONTAINER_NAME = "kingfisher-mysql" +ROOT_PASSWORD = "superman123" +HOST = "127.0.0.1" +PORT = 3306 +CREDS_PATH = Path("/tmp/mysql-creds.log") +READY_TIMEOUT = 120 + + +def run(cmd, check=True, capture_output=False, text=True): + return subprocess.run(cmd, check=check, capture_output=capture_output, text=text) + + +def container_exists(name: str) -> bool: + result = run([ + "docker", + "ps", + "-aq", + "--filter", + f"name=^{name}$", + ], capture_output=True) + return bool(result.stdout.strip()) + + +def remove_container(name: str) -> None: + if container_exists(name): + run(["docker", "rm", "-f", name]) + + +def pull_image() -> None: + run(["docker", "pull", IMAGE]) + + +def start_container() -> None: + run( + [ + "docker", + "run", + "-d", + "--name", + CONTAINER_NAME, + "-e", + f"MYSQL_ROOT_PASSWORD={ROOT_PASSWORD}", + "-p", + f"{PORT}:3306", + IMAGE, + ] + ) + + +def wait_for_mysql() -> None: + start = time.time() + while time.time() - start < READY_TIMEOUT: + try: + run( + [ + "docker", + "exec", + CONTAINER_NAME, + "mysqladmin", + "ping", + "-h", + "127.0.0.1", + "-uroot", + f"-p{ROOT_PASSWORD}", + ] + ) + return + except subprocess.CalledProcessError: + time.sleep(2) + raise RuntimeError("MySQL container did not become ready in time") + + +def write_creds_file() -> None: + conn = f"mysql://root:{ROOT_PASSWORD}@{HOST}:{PORT}/" + CREDS_PATH.write_text(conn + "\n", encoding="utf-8") + print(f"Wrote connection string to {CREDS_PATH}: {conn}") + + +def main() -> None: + try: + pull_image() + remove_container(CONTAINER_NAME) + start_container() + wait_for_mysql() + write_creds_file() + print( + "MySQL container is running. Use `docker logs -f %s` to monitor it." % CONTAINER_NAME + ) + except Exception as exc: # noqa: BLE001 + print(f"Failed to start MySQL container: {exc}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/validation.rs b/src/validation.rs index 4270545..8ffcd66 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -34,11 +34,26 @@ mod jwt; mod mongodb; mod mysql; mod postgres; +pub use mysql::validate_mysql; +pub use postgres::validate_postgres; mod utils; const VALIDATION_CACHE_SECONDS: u64 = 1200; // 20 minutes const MAX_VALIDATION_BODY_LEN: usize = 2048; +fn truncate_to_char_boundary(s: &mut String, max_len: usize) { + if s.len() <= max_len { + return; + } + + let mut new_len = max_len; + while new_len > 0 && !s.is_char_boundary(new_len) { + new_len -= 1; + } + + s.truncate(new_len); +} + static USER_AGENT_SUFFIX: OnceCell = OnceCell::new(); const BROWSER_USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \ @@ -550,9 +565,7 @@ async fn timed_validate_single_match<'a>( return; } }; - if body.len() > MAX_VALIDATION_BODY_LEN { - body.truncate(MAX_VALIDATION_BODY_LEN); - } + truncate_to_char_boundary(&mut body, MAX_VALIDATION_BODY_LEN); m.validation_response_status = status; m.validation_response_body = body.clone(); @@ -1139,6 +1152,18 @@ mod tests { assert!(globals.get("TOKEN").is_none()); assert_eq!(globals.get("CHECKSUM"), Some(Value::scalar("123456")).as_ref()); } + + #[test] + fn truncate_to_char_boundary_handles_multibyte_characters() { + let mut body = "a".repeat(MAX_VALIDATION_BODY_LEN); + body.push('é'); + + truncate_to_char_boundary(&mut body, MAX_VALIDATION_BODY_LEN); + + assert_eq!(body.len(), MAX_VALIDATION_BODY_LEN); + assert!(body.is_char_boundary(body.len())); + assert!(body.ends_with('a')); + } } // #[cfg(test)] diff --git a/tests/live_db_validation.rs b/tests/live_db_validation.rs new file mode 100644 index 0000000..faf49ee --- /dev/null +++ b/tests/live_db_validation.rs @@ -0,0 +1,101 @@ +//! Live validation smoke tests that exercise the database validators against +//! real MySQL and Postgres instances provisioned with `testcontainers`. +//! +//! These are ignored by default because they require Docker. Run them with: +//! `cargo test --test live_db_validation -- --ignored`. + +use std::time::{Duration, Instant}; + +use anyhow::{anyhow, Result}; +use kingfisher::validation::{validate_mysql, validate_postgres}; +use testcontainers::{clients::Cli, core::WaitFor, GenericImage}; +use tokio::{net::TcpStream, time::sleep}; + +const HOST_ALIAS: &str = "kingfisherlocal"; +const STARTUP_TIMEOUT: Duration = Duration::from_secs(60); +const STARTUP_POLL_INTERVAL: Duration = Duration::from_millis(250); + +async fn wait_for_port(host: &str, port: u16) -> Result<()> { + let deadline = Instant::now() + STARTUP_TIMEOUT; + let mut last_err = None; + + loop { + match TcpStream::connect((host, port)).await { + Ok(stream) => { + drop(stream); + return Ok(()); + } + Err(err) => { + last_err = Some(err); + if Instant::now() >= deadline { + return Err(anyhow!( + "timed out after {:?} waiting for {host}:{port}: {last_err:?}", + STARTUP_TIMEOUT, + )); + } + sleep(STARTUP_POLL_INTERVAL).await; + } + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore] +async fn validates_mysql_secret_against_testcontainer() -> Result<()> { + let docker = Cli::default(); + let image = GenericImage::new("mysql", "8.4") + .with_env_var("MYSQL_ROOT_PASSWORD", "secret") + .with_env_var("MYSQL_DATABASE", "app") + .with_env_var("MYSQL_ROOT_HOST", "%") + .with_wait_for(WaitFor::message_on_stdout( + "MySQL init process done. Ready for start up.", + )); + + let container = docker.run(image); + let port = container.get_host_port_ipv4(3306); + + wait_for_port(HOST_ALIAS, port).await?; + + let uri = format!("mysql://root:secret@{HOST_ALIAS}:{port}/app"); + let (is_valid, metadata) = validate_mysql(&uri).await?; + + assert!(is_valid, "expected MySQL validation to succeed, got {metadata:?}"); + assert!( + metadata.iter().any(|entry| entry.contains("user=root")), + "expected user metadata in {metadata:?}" + ); + assert!( + metadata.iter().any(|entry| entry.contains("database=app")), + "expected database metadata in {metadata:?}" + ); + + drop(container); + drop(docker); + Ok(()) +} + + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore] +async fn validates_postgres_secret_against_testcontainer() -> Result<()> { + let docker = Cli::default(); + let image = GenericImage::new("postgres", "15") + .with_env_var("POSTGRES_PASSWORD", "secret") + .with_wait_for(WaitFor::message_on_stdout( + "database system is ready to accept connections", + )); + let container = docker.run(image); + let port = container.get_host_port_ipv4(5432); + + wait_for_port(HOST_ALIAS, port).await?; + + let uri = format!("postgres://postgres:secret@{HOST_ALIAS}:{port}/postgres"); + let (is_valid, metadata) = validate_postgres(&uri).await?; + + assert!(is_valid, "expected Postgres validation to succeed"); + assert!(metadata.is_empty(), "expected no metadata but found {metadata:?}"); + + drop(container); + drop(docker); + Ok(()) +} \ No newline at end of file