- Skip reporting MongoDB and Postgres findings when their connection strings cannot be parsed, even when validation is disabled.

- Improve MySQL detection by broadening URI coverage and adding live validation that skips clearly invalid connection strings.

CHANGELOG.md

@@ -5,6 +5,7 @@ All notable changes to this project will be documented in this file.
 ## [v1.65.0]
 - Skip reporting MongoDB and Postgres findings when their connection strings cannot be parsed, even when validation is disabled.
 - Improve MySQL detection by broadening URI coverage and adding live validation that skips clearly invalid connection strings.
+- Added a helper to truncate validation response bodies only at UTF-8 character boundaries to prevent panics during validation.
 
 ## [v1.64.0]
 - Fixed a bug when using --redact, that broke validation

Cargo.toml

@@ -221,6 +221,7 @@ temp-env = "0.3.6"
 wiremock = "0.6.4"
 git2 = "0.20.2"
 rand_chacha = "0.9.0"
+testcontainers = "0.15.0"
 
 [profile.release]
 debug = false

data/rules/uri.yml

@@ -16,6 +16,10 @@ rules:
         (?:\?[A-Za-z0-9\-._~%!$&'()*,;=:@/?%]*)?
         (?:\#[A-Za-z0-9\-._~%!$&'()*,;=:@/?%]*)?
       )
+    pattern_requirements:
+      ignore_if_contains:
+        - "*****"
+        - "xxxxx"
     min_entropy: 4.0
     confidence: medium
     examples:

mysqltest.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""Start a MySQL 8.4 Docker container with a known root password.
+
+The script ensures the container keeps running and writes the connection
+string to /tmp/mysql-creds.log for convenience.
+"""
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+IMAGE = "mysql:8.4"
+CONTAINER_NAME = "kingfisher-mysql"
+ROOT_PASSWORD = "superman123"
+HOST = "127.0.0.1"
+PORT = 3306
+CREDS_PATH = Path("/tmp/mysql-creds.log")
+READY_TIMEOUT = 120
+
+
+def run(cmd, check=True, capture_output=False, text=True):
+    return subprocess.run(cmd, check=check, capture_output=capture_output, text=text)
+
+
+def container_exists(name: str) -> bool:
+    result = run([
+        "docker",
+        "ps",
+        "-aq",
+        "--filter",
+        f"name=^{name}$",
+    ], capture_output=True)
+    return bool(result.stdout.strip())
+
+
+def remove_container(name: str) -> None:
+    if container_exists(name):
+        run(["docker", "rm", "-f", name])
+
+
+def pull_image() -> None:
+    run(["docker", "pull", IMAGE])
+
+
+def start_container() -> None:
+    run(
+        [
+            "docker",
+            "run",
+            "-d",
+            "--name",
+            CONTAINER_NAME,
+            "-e",
+            f"MYSQL_ROOT_PASSWORD={ROOT_PASSWORD}",
+            "-p",
+            f"{PORT}:3306",
+            IMAGE,
+        ]
+    )
+
+
+def wait_for_mysql() -> None:
+    start = time.time()
+    while time.time() - start < READY_TIMEOUT:
+        try:
+            run(
+                [
+                    "docker",
+                    "exec",
+                    CONTAINER_NAME,
+                    "mysqladmin",
+                    "ping",
+                    "-h",
+                    "127.0.0.1",
+                    "-uroot",
+                    f"-p{ROOT_PASSWORD}",
+                ]
+            )
+            return
+        except subprocess.CalledProcessError:
+            time.sleep(2)
+    raise RuntimeError("MySQL container did not become ready in time")
+
+
+def write_creds_file() -> None:
+    conn = f"mysql://root:{ROOT_PASSWORD}@{HOST}:{PORT}/"
+    CREDS_PATH.write_text(conn + "\n", encoding="utf-8")
+    print(f"Wrote connection string to {CREDS_PATH}: {conn}")
+
+
+def main() -> None:
+    try:
+        pull_image()
+        remove_container(CONTAINER_NAME)
+        start_container()
+        wait_for_mysql()
+        write_creds_file()
+        print(
+            "MySQL container is running. Use `docker logs -f %s` to monitor it." % CONTAINER_NAME
+        )
+    except Exception as exc:  # noqa: BLE001
+        print(f"Failed to start MySQL container: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

src/validation.rs

@@ -34,11 +34,26 @@ mod jwt;
 mod mongodb;
 mod mysql;
 mod postgres;
+pub use mysql::validate_mysql;
+pub use postgres::validate_postgres;
 mod utils;
 
 const VALIDATION_CACHE_SECONDS: u64 = 1200; // 20 minutes
 const MAX_VALIDATION_BODY_LEN: usize = 2048;
 
+fn truncate_to_char_boundary(s: &mut String, max_len: usize) {
+    if s.len() <= max_len {
+        return;
+    }
+
+    let mut new_len = max_len;
+    while new_len > 0 && !s.is_char_boundary(new_len) {
+        new_len -= 1;
+    }
+
+    s.truncate(new_len);
+}
+
 static USER_AGENT_SUFFIX: OnceCell<String> = OnceCell::new();
 
 const BROWSER_USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \
@@ -550,9 +565,7 @@ async fn timed_validate_single_match<'a>(
                             return;
                         }
                     };
-                    if body.len() > MAX_VALIDATION_BODY_LEN {
-                        body.truncate(MAX_VALIDATION_BODY_LEN);
-                    }
+                    truncate_to_char_boundary(&mut body, MAX_VALIDATION_BODY_LEN);
 
                     m.validation_response_status = status;
                     m.validation_response_body = body.clone();
@@ -1139,6 +1152,18 @@ mod tests {
         assert!(globals.get("TOKEN").is_none());
         assert_eq!(globals.get("CHECKSUM"), Some(Value::scalar("123456")).as_ref());
     }
+
+    #[test]
+    fn truncate_to_char_boundary_handles_multibyte_characters() {
+        let mut body = "a".repeat(MAX_VALIDATION_BODY_LEN);
+        body.push('é');
+
+        truncate_to_char_boundary(&mut body, MAX_VALIDATION_BODY_LEN);
+
+        assert_eq!(body.len(), MAX_VALIDATION_BODY_LEN);
+        assert!(body.is_char_boundary(body.len()));
+        assert!(body.ends_with('a'));
+    }
 }
 
 // #[cfg(test)]

tests/live_db_validation.rs

@@ -0,0 +1,101 @@
+//! Live validation smoke tests that exercise the database validators against
+//! real MySQL and Postgres instances provisioned with `testcontainers`.
+//!
+//! These are ignored by default because they require Docker. Run them with:
+//! `cargo test --test live_db_validation -- --ignored`.
+
+use std::time::{Duration, Instant};
+
+use anyhow::{anyhow, Result};
+use kingfisher::validation::{validate_mysql, validate_postgres};
+use testcontainers::{clients::Cli, core::WaitFor, GenericImage};
+use tokio::{net::TcpStream, time::sleep};
+
+const HOST_ALIAS: &str = "kingfisherlocal";
+const STARTUP_TIMEOUT: Duration = Duration::from_secs(60);
+const STARTUP_POLL_INTERVAL: Duration = Duration::from_millis(250);
+
+async fn wait_for_port(host: &str, port: u16) -> Result<()> {
+    let deadline = Instant::now() + STARTUP_TIMEOUT;
+    let mut last_err = None;
+
+    loop {
+        match TcpStream::connect((host, port)).await {
+            Ok(stream) => {
+                drop(stream);
+                return Ok(());
+            }
+            Err(err) => {
+                last_err = Some(err);
+                if Instant::now() >= deadline {
+                    return Err(anyhow!(
+                        "timed out after {:?} waiting for {host}:{port}: {last_err:?}",
+                        STARTUP_TIMEOUT,
+                    ));
+                }
+                sleep(STARTUP_POLL_INTERVAL).await;
+            }
+        }
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[ignore]
+async fn validates_mysql_secret_against_testcontainer() -> Result<()> {
+    let docker = Cli::default();
+    let image = GenericImage::new("mysql", "8.4")
+        .with_env_var("MYSQL_ROOT_PASSWORD", "secret")
+        .with_env_var("MYSQL_DATABASE", "app")
+        .with_env_var("MYSQL_ROOT_HOST", "%")
+        .with_wait_for(WaitFor::message_on_stdout(
+            "MySQL init process done. Ready for start up.",
+        ));
+
+    let container = docker.run(image);
+    let port = container.get_host_port_ipv4(3306);
+
+    wait_for_port(HOST_ALIAS, port).await?;
+
+    let uri = format!("mysql://root:secret@{HOST_ALIAS}:{port}/app");
+    let (is_valid, metadata) = validate_mysql(&uri).await?;
+
+    assert!(is_valid, "expected MySQL validation to succeed, got {metadata:?}");
+    assert!(
+        metadata.iter().any(|entry| entry.contains("user=root")),
+        "expected user metadata in {metadata:?}"
+    );
+    assert!(
+        metadata.iter().any(|entry| entry.contains("database=app")),
+        "expected database metadata in {metadata:?}"
+    );
+
+    drop(container);
+    drop(docker);
+    Ok(())
+}
+
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[ignore]
+async fn validates_postgres_secret_against_testcontainer() -> Result<()> {
+    let docker = Cli::default();
+    let image = GenericImage::new("postgres", "15")
+        .with_env_var("POSTGRES_PASSWORD", "secret")
+        .with_wait_for(WaitFor::message_on_stdout(
+            "database system is ready to accept connections",
+        ));
+    let container = docker.run(image);
+    let port = container.get_host_port_ipv4(5432);
+
+    wait_for_port(HOST_ALIAS, port).await?;
+
+    let uri = format!("postgres://postgres:secret@{HOST_ALIAS}:{port}/postgres");
+    let (is_valid, metadata) = validate_postgres(&uri).await?;
+
+    assert!(is_valid, "expected Postgres validation to succeed");
+    assert!(metadata.is_empty(), "expected no metadata but found {metadata:?}");
+
+    drop(container);
+    drop(docker);
+    Ok(())
+}