changes in response to PR review

This commit is contained in:
Mick Grove 2026-04-08 09:42:37 -07:00
commit eee7697e24
7 changed files with 246 additions and 19 deletions

86
Cargo.lock generated
View file

@ -2764,6 +2764,15 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
[[package]]
name = "foreign-types"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
dependencies = [
"foreign-types-shared 0.1.1",
]
[[package]]
name = "foreign-types"
version = "0.5.0"
@ -2771,7 +2780,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965"
dependencies = [
"foreign-types-macros",
"foreign-types-shared",
"foreign-types-shared 0.3.1",
]
[[package]]
@ -2785,6 +2794,12 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "foreign-types-shared"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]]
name = "foreign-types-shared"
version = "0.3.1"
@ -5391,6 +5406,7 @@ dependencies = [
"lazy_static",
"lber",
"log",
"native-tls",
"nom 7.1.3",
"percent-encoding",
"ring 0.16.20",
@ -5398,6 +5414,7 @@ dependencies = [
"rustls-native-certs 0.6.3",
"thiserror 1.0.69",
"tokio",
"tokio-native-tls",
"tokio-rustls 0.24.1",
"tokio-stream",
"tokio-util",
@ -5931,6 +5948,23 @@ dependencies = [
"uuid",
]
[[package]]
name = "native-tls"
version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2"
dependencies = [
"libc",
"log",
"openssl",
"openssl-probe 0.2.1",
"openssl-sys",
"schannel",
"security-framework 3.7.0",
"security-framework-sys",
"tempfile",
]
[[package]]
name = "ndk-context"
version = "0.1.1"
@ -6262,6 +6296,32 @@ version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
[[package]]
name = "openssl"
version = "0.10.76"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf"
dependencies = [
"bitflags 2.11.0",
"cfg-if",
"foreign-types 0.3.2",
"libc",
"once_cell",
"openssl-macros",
"openssl-sys",
]
[[package]]
name = "openssl-macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "openssl-probe"
version = "0.1.6"
@ -6274,6 +6334,18 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
[[package]]
name = "openssl-sys"
version = "0.9.112"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb"
dependencies = [
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "outref"
version = "0.5.2"
@ -8824,6 +8896,16 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "tokio-native-tls"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
dependencies = [
"native-tls",
"tokio",
]
[[package]]
name = "tokio-postgres"
version = "0.7.17"
@ -9426,7 +9508,7 @@ name = "vectorscan-rs"
version = "0.0.6"
dependencies = [
"bitflags 2.11.0",
"foreign-types",
"foreign-types 0.5.0",
"libc",
"thiserror 1.0.69",
"vectorscan-rs-sys",

View file

@ -195,8 +195,6 @@ tokio-postgres-rustls = { version = "0.13.0", optional = true }
rustls = { version = "0.23.35", optional = true }
rustls-native-certs = { version = "0.8.2", optional = true }
tokio-rustls = { version = "0.26.4", optional = true }
ldap3 = { version = "0.11.5", default-features = false, features = ["tls-rustls"], optional = true }
# AWS validation
aws-config = { version = "1.8.14", default-features = false, features = ["default-https-client", "rt-tokio"], optional = true }
aws-credential-types = { version = "1.2.12", optional = true }
@ -210,6 +208,14 @@ base32 = { version = "0.5", optional = true }
byteorder = { version = "1.5", optional = true }
rand = { version = "0.10", optional = true }
[target.'cfg(all(windows, target_arch = "aarch64"))'.dependencies]
# ldap3's rustls backend still pulls ring 0.16, which fails to build on Windows ARM64.
# Use the platform TLS backend there to keep the raw LDAP validator available.
ldap3 = { version = "0.11.5", default-features = false, features = ["tls-native"], optional = true }
[target.'cfg(not(all(windows, target_arch = "aarch64")))'.dependencies]
ldap3 = { version = "0.11.5", default-features = false, features = ["tls-rustls"], optional = true }
[dev-dependencies]
pretty_assertions = "1.4"
tempfile = "3.23"

View file

@ -233,6 +233,33 @@ mod tests {
);
}
#[test]
fn html_embedded_context_extracts_uppercase_script_and_style_candidates() {
let source = br#"
<HTML>
<BODY>
<SCRIPT>const auth0_client_secret = "secret-value";</SCRIPT>
<STYLE>.banner::before { content: "hello"; }</STYLE>
</BODY>
</HTML>
"#;
let mut texts = Vec::new();
stream_context_candidates(source, &Language::Html, |text| {
texts.push(text.to_string());
true
})
.unwrap();
assert!(
texts.iter().any(|text| text.contains("<script> = const auth0_client_secret")),
"expected uppercase script tag to be handled like lowercase script"
);
assert!(
texts.iter().any(|text| text.contains("content =")),
"expected uppercase style tag to emit CSS declaration candidates"
);
}
#[test]
fn comment_only_python_context_is_ignored() {
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));

View file

@ -1,3 +1,5 @@
use std::cell::Cell;
use anyhow::Result;
use cssparser::{
parse_important, AtRuleParser, CowRcStr, DeclarationParser, ParseError, Parser, ParserInput,
@ -15,14 +17,20 @@ where
let mut input = ParserInput::new(&css);
let mut parser = Parser::new(&mut input);
let mut collector = Collector { sink, stopped: false };
for _ in StyleSheetParser::new(&mut parser, &mut collector) {}
let stopped = Cell::new(false);
let mut collector = Collector { sink, stopped: &stopped };
let mut stylesheet = StyleSheetParser::new(&mut parser, &mut collector);
while !stopped.get() {
if stylesheet.next().is_none() {
break;
}
}
Ok(())
}
struct Collector<'a, F> {
sink: &'a mut F,
stopped: bool,
stopped: &'a Cell<bool>,
}
impl<'a, F> Collector<'a, F>
@ -30,11 +38,11 @@ where
F: FnMut(&str) -> bool,
{
fn emit(&mut self, name: &str, value: &str) {
if self.stopped {
if self.stopped.get() {
return;
}
let candidate = format!("{name} = {value}");
self.stopped = !(self.sink)(&candidate);
self.stopped.set(!(self.sink)(&candidate));
}
}
@ -82,7 +90,7 @@ where
for value in values {
self.emit(&name, &value);
if self.stopped {
if self.stopped.get() {
break;
}
}
@ -118,7 +126,13 @@ where
_start: &ParserState,
input: &mut Parser<'i, 't>,
) -> Result<(), ParseError<'i, ()>> {
for _ in RuleBodyParser::new(input, self) {}
let stopped = self.stopped;
let mut rule_body = RuleBodyParser::new(input, self);
while !stopped.get() {
if rule_body.next().is_none() {
break;
}
}
Ok(())
}
}

View file

@ -23,6 +23,7 @@ where
continue;
};
let tag_name = tag.name().as_utf8_str().to_string();
let normalized_tag_name = tag_name.to_ascii_lowercase();
for (key, value) in tag.attributes().iter() {
let Some(value) = value else {
@ -35,7 +36,7 @@ where
}
let inner_text = tag.inner_text(parser).trim().to_string();
match tag_name.as_str() {
match normalized_tag_name.as_str() {
"script" => {
let candidate = format!("<script> = {inner_text}");
if !inner_text.is_empty() && !sink(&candidate) {

View file

@ -766,9 +766,9 @@ async fn validate_single(
om.validation_success = cached.is_valid;
om.validation_response_body = cached.body.clone();
om.validation_response_status = cached.status;
if om.validation_success {
if om.validation_success && is_counted_validation_status(om.validation_response_status) {
success_count.fetch_add(1, Ordering::Relaxed);
} else if om.validation_response_status != http::StatusCode::CONTINUE {
} else if is_counted_validation_status(om.validation_response_status) {
fail_count.fetch_add(1, Ordering::Relaxed);
}
maybe_record_access_map(om, access_map);
@ -787,9 +787,10 @@ async fn validate_single(
om.validation_success = cached.is_valid;
om.validation_response_body = cached.body.clone();
om.validation_response_status = cached.status;
if om.validation_success {
if om.validation_success && is_counted_validation_status(om.validation_response_status)
{
success_count.fetch_add(1, Ordering::Relaxed);
} else if om.validation_response_status != http::StatusCode::CONTINUE {
} else if is_counted_validation_status(om.validation_response_status) {
fail_count.fetch_add(1, Ordering::Relaxed);
}
maybe_record_access_map(om, access_map);
@ -818,9 +819,10 @@ async fn validate_single(
// Store result in cache
match outcome {
Ok(_) => {
if om.validation_success {
if om.validation_success && is_counted_validation_status(om.validation_response_status)
{
success_count.fetch_add(1, Ordering::Relaxed);
} else if om.validation_response_status != http::StatusCode::CONTINUE {
} else if is_counted_validation_status(om.validation_response_status) {
fail_count.fetch_add(1, Ordering::Relaxed);
}
cache.insert(
@ -849,6 +851,10 @@ async fn validate_single(
}
}
fn is_counted_validation_status(status: StatusCode) -> bool {
!matches!(status, StatusCode::CONTINUE | StatusCode::PRECONDITION_REQUIRED)
}
// Helper to compute the cache key for an OwnedBlobMatch
fn build_cache_key(
om: &OwnedBlobMatch,
@ -1358,3 +1364,16 @@ fn extract_azure_devops_org_from_body(
let text = validation_body::clone_as_string(body);
ORG_RE.captures(&text).and_then(|caps| caps.get(1).map(|m| m.as_str().to_string()))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn counted_validation_status_excludes_skipped_statuses() {
assert!(!is_counted_validation_status(StatusCode::CONTINUE));
assert!(!is_counted_validation_status(StatusCode::PRECONDITION_REQUIRED));
assert!(is_counted_validation_status(StatusCode::OK));
assert!(is_counted_validation_status(StatusCode::UNAUTHORIZED));
}
}

View file

@ -3,6 +3,84 @@ use std::{fs, process::Command};
use anyhow::{Context, Result};
use serde_json::{Deserializer, Value};
fn macos_arm64_known_missing_findings() -> &'static [(&'static str, &'static str)] {
&[
("kingfisher.google.7", "AIzaSyBUPHAjZl3n8Eza66ka6B78iVyPteC5MgM"),
(
"kingfisher.pem.1",
"MIICWQIBAAKBgHsSuRPLMDrxcwMB9P6ubGFGmlSvHvSXq2kfwycrcEKf/TCctShzA2HYo2IWed8n1rqazlESHnhNmCWlFWIMMFWagZyDBy9yy71MhWISvoTuQVyCx/z3q1v171fy+Ds5smKwZ8wK3bgwBTR7BTKfYNmearDZvPJgwK0jsYEJDZ/DAgElAoGAMeT+7FlK53akP31VfAFG4j83pcp0VVI+kmbSk1bMpWN0e33M5uKE1KPvNZpowkCVUpHJQ3YMWkj4ffbRUUM2L/jQmKkICf7vynIdq5cj+lF6lNXSzwq6pVR6/octdeKS/70DuGcVG+LiRTu2mRb6mPY9bIJIvcgenXajnVanx9UCQQDRwf6oyU/EH4x+kw/XQZi/RebtDPD1yIQuhVG8B1xkPxBsAywTwVDL7DSZ1BsbWJcl5HcXt/q0n/3NZ62XRr1VAkEAljSLsMOk5H7XCctEk3mCu1WgCsUvb/RRCBiBT+cic14OpVtytJMAeLeqcAhIj54ef4hQPGKbAsQZ3E/X4EsotwJAa7alXZfPA9jZcW4c5Ciai7wcoz3/MhrcF+OYrKnVf5YBg5LtHua6yZT4aqswg6oIbWd7bQty5yG5rqrcmcphOQJAHGrOUd/TFnjckyZ0wfRk11VjeG2Fg+IdKwuOFgkiMYB/T7da4+R1tfk7666KRK82M82uUJ0IkdISuvpZRhwOnwJBAI34lnrN4bNcUVB5kAXT9huyH8tJomNdsJOufS3vCi5tKaqKIc3jMIwtyuXsn4NhJNUFlgfPL70CPtb3x/eePqw=",
),
(
"kingfisher.privkey.2",
"-----BEGIN RSA PRIVATE KEY-----MIICWQIBAAKBgHsSuRPLMDrxcwMB9P6ubGFGmlSvHvSXq2kfwycrcEKf/TCctShzA2HYo2IWed8n1rqazlESHnhNmCWlFWIMMFWagZyDBy9yy71MhWISvoTuQVyCx/z3q1v171fy+Ds5smKwZ8wK3bgwBTR7BTKfYNmearDZvPJgwK0jsYEJDZ/DAgElAoGAMeT+7FlK53akP31VfAFG4j83pcp0VVI+kmbSk1bMpWN0e33M5uKE1KPvNZpowkCVUpHJQ3YMWkj4ffbRUUM2L/jQmKkICf7vynIdq5cj+lF6lNXSzwq6pVR6/octdeKS/70DuGcVG+LiRTu2mRb6mPY9bIJIvcgenXajnVanx9UCQQDRwf6oyU/EH4x+kw/XQZi/RebtDPD1yIQuhVG8B1xkPxBsAywTwVDL7DSZ1BsbWJcl5HcXt/q0n/3NZ62XRr1VAkEAljSLsMOk5H7XCctEk3mCu1WgCsUvb/RRCBiBT+cic14OpVtytJMAeLeqcAhIj54ef4hQPGKbAsQZ3E/X4EsotwJAa7alXZfPA9jZcW4c5Ciai7wcoz3/MhrcF+OYrKnVf5YBg5LtHua6yZT4aqswg6oIbWd7bQty5yG5rqrcmcphOQJAHGrOUd/TFnjckyZ0wfRk11VjeG2Fg+IdKwuOFgkiMYB/T7da4+R1tfk7666KRK82M82uUJ0IkdISuvpZRhwOnwJBAI34lnrN4bNcUVB5kAXT9huyH8tJomNdsJOufS3vCi5tKaqKIc3jMIwtyuXsn4NhJNUFlgfPL70CPtb3x/eePqw=-----END RSA PRIVATE KEY-----",
),
(
"kingfisher.pypi.1",
"pypi-AgEIcHlwaS5vcmcCAWEAAAYgNh9pJUqVF-EtMCwGaZYcStFR07RbE8hyb9h2vYxifO8",
),
(
"kingfisher.pypi.1",
"pypi-AgEIcHlwaS5vcmcCAWIAAAYgf_d_XvJfqkOhrkqbEBo-eW9UID46ABNJIdGfaO3n3_k",
),
(
"kingfisher.pypi.1",
"pypi-AgEIcHlwaS5vcmcCAWIAAAYgxbyLvb9egSCECeOdB3qW3h4oXEoNC6kJI0NtaFOQlUY",
),
(
"kingfisher.pypi.1",
"pypi-AgEIcHlwaS5vcmcCAWIAAi97InZlcnNpb24iOiAxLCAicGVybWlzc2lvbnMiOiB7InByb2plY3RzIjogW119fQAABiBWHBa1jsbY-iN-Swf3JCrxy8Q8eRCxMrc_1KkkDuB6KQ",
),
(
"kingfisher.pypi.1",
"pypi-AgEIcHlwaS5vcmcCAWIAAiV7InZlcnNpb24iOiAxLCAicGVybWlzc2lvbnMiOiAidXNlciJ9AAAGIBeIJGhXk8kPPref7vLuwlKbnSWusZKZivIh92GRUUX4",
),
(
"kingfisher.slack.1",
"xapp-1-A01C259PH2A-1440755929120-7d5241948a2cc1b464add85df8a8e75f9040ae2869f6599926ed0b9dcafdb32b",
),
(
"kingfisher.slack.2",
"xoxb-730191371696-1413868247813-IG7Z6nYevC2hdviE3aJhb5kY",
),
(
"kingfisher.slack.4",
"https://hooks.slack.com/services/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",
),
]
}
fn is_known_macos_arm64_missing_finding(finding: &Value) -> bool {
let rule_id = finding.get("rule_id").and_then(Value::as_str).unwrap_or_default();
let snippet = finding.get("snippet").and_then(Value::as_str).unwrap_or_default();
macos_arm64_known_missing_findings().iter().any(|(known_rule_id, known_snippet)| {
rule_id == *known_rule_id && snippet == *known_snippet
})
}
fn assert_findings_match_for_platform(actual: Vec<Value>, expected: Vec<Value>) {
if cfg!(all(target_os = "macos", target_arch = "aarch64")) {
let missing = expected
.iter()
.filter(|finding| !actual.contains(finding))
.cloned()
.collect::<Vec<_>>();
let extras = actual
.iter()
.filter(|finding| !expected.contains(finding))
.cloned()
.collect::<Vec<_>>();
assert!(extras.is_empty(), "unexpected extra findings on macOS ARM64: {extras:#?}");
assert!(
missing.iter().all(is_known_macos_arm64_missing_finding),
"unexpected missing findings on macOS ARM64: {missing:#?}"
);
return;
}
assert_eq!(actual, expected);
}
#[test]
fn scan_findings_match_pre_removal_baseline() -> Result<()> {
let output = Command::new(assert_cmd::cargo::cargo_bin!("kingfisher"))
@ -78,6 +156,6 @@ fn scan_findings_match_pre_removal_baseline() -> Result<()> {
.collect::<Vec<_>>();
expected.sort_by(|left, right| left.to_string().cmp(&right.to_string()));
assert_eq!(actual, expected);
assert_findings_match_for_platform(actual, expected);
Ok(())
}