From 6b56528a4d524bd4440491c3391b8602682b21fa Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Tue, 24 Jun 2025 17:20:19 -0700 Subject: [PATCH 1/4] preparing for v1.12 --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f17c240..342e0c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,8 @@ All notable changes to this project will be documented in this file. -## [1.12.0] -- Added automatic update checks using GitHub releases +## [1.12.0] +- Added automatic update checks using GitHub releases. - New `--self-update` flag installs updates when available - New `--no-update-check` flag disables update checks - Updated rules From 0354b441e06a7e9290c379e6845eb3b76550829b Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Tue, 24 Jun 2025 19:28:15 -0700 Subject: [PATCH 2/4] preparing for v1.12 --- Cargo.toml | 1 + data/rules/digitalocean.yml | 35 ++++---- data/rules/facebook.yml | 10 +-- src/decompress.rs | 172 ++++++++++++++++++------------------ tests/update.rs | 80 ++++++++--------- 5 files changed, 150 insertions(+), 148 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 880c4af..4d9d103 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -160,6 +160,7 @@ percent-encoding = "2.3.1" trust-dns-resolver = { version = "0.23.2", default-features = false, features = ["tokio-runtime"] } atty = "0.2.14" self_update = { version = "0.42.0", default-features = false, features = ["rustls"] } +semver = "1.0.26" [dependencies.tikv-jemallocator] version = "0.6" diff --git a/data/rules/digitalocean.yml b/data/rules/digitalocean.yml index 9b07cd6..3206ab7 100644 --- a/data/rules/digitalocean.yml +++ b/data/rules/digitalocean.yml @@ -48,20 +48,21 @@ rules: - ' "refresh_token": "dor_v1_d6ce5b93104521c47be0b580e9296454ef4a319b02b5513469f0ec71d99af2e2",' validation: type: Http - content: - request: - method: POST - url: https://cloud.digitalocean.com/v1/oauth/token - headers: - Content-Type: application/json - Accept: application/json - body: | - { - "grant_type": "refresh_token", - "refresh_token": "{{ TOKEN }}" - } - response_matcher: - - report_response: true - - type: StatusMatch - status: [200] - - type: JsonValid + content: + request: + method: POST + url: https://cloud.digitalocean.com/v1/oauth/token + headers: + Content-Type: application/json + Accept: application/json + body: | + { + "grant_type": "refresh_token", + "refresh_token": "{{ TOKEN }}" + } + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 + - type: JsonValid diff --git a/data/rules/facebook.yml b/data/rules/facebook.yml index 413ff8b..82075c4 100644 --- a/data/rules/facebook.yml +++ b/data/rules/facebook.yml @@ -50,11 +50,11 @@ rules: - status: - 200 type: StatusMatch - url: >- - https://graph.facebook.com/v19.0/oauth/access_token - ?client_id={{ APIID }} - &client_secret={{ TOKEN }} - &grant_type=client_credentials + url: >- + https://graph.facebook.com/v19.0/oauth/access_token + ?client_id={{ APIID }} + &client_secret={{ TOKEN }} + &grant_type=client_credentials depends_on_rule: - rule_id: kingfisher.facebook.1 variable: APIID diff --git a/src/decompress.rs b/src/decompress.rs index d695cf3..3795910 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -276,7 +276,7 @@ pub fn decompress_file_to_temp(path: &Path) -> Result<(CompressedContent, TempDi #[cfg(test)] mod tests { - use std::{fs::File, io::Write, path::PathBuf}; + use std::fs::File; use flate2::{write::GzEncoder, Compression}; use tar::Builder; @@ -382,104 +382,104 @@ mod tests { Ok(()) } - // /// 3) Nested archive: - // /// outer.tar.gz ──▶ outer.tar (contains inner.tar.gz) - // /// └──▶ inner.tar.gz ──▶ inner.tar (contains secret.txt) - // #[test] - // fn smoke_decompress_nested_tar_gz_archives() -> anyhow::Result<()> { - // use std::{ - // fs::File, - // io::{Read, Write}, - // path::PathBuf, - // }; + /// 3) Nested archive: + /// outer.tar.gz ──▶ outer.tar (contains inner.tar.gz) + /// └──▶ inner.tar.gz ──▶ inner.tar (contains secret.txt) + #[test] + fn smoke_decompress_nested_tar_gz_archives() -> anyhow::Result<()> { + use std::{ + fs::File, + io::{Read, Write}, + path::PathBuf, + }; - // use flate2::{write::GzEncoder, Compression}; - // use tar::Builder; - // use tempfile::tempdir; + use flate2::{write::GzEncoder, Compression}; + use tar::Builder; + use tempfile::tempdir; - // use super::{decompress_once, CompressedContent}; + use super::{decompress_once, CompressedContent}; - // let tmp = tempdir()?; + let tmp = tempdir()?; - // /* ── build INNER tar.gz ──────────────────────────────────────────────── */ - // let inner_tgz = tmp.path().join("inner.tar.gz"); - // { - // let f = File::create(&inner_tgz)?; - // let gz = GzEncoder::new(f, Compression::default()); - // let mut tar = Builder::new(gz); + /* ── build INNER tar.gz ──────────────────────────────────────────────── */ + let inner_tgz = tmp.path().join("inner.tar.gz"); + { + let f = File::create(&inner_tgz)?; + let gz = GzEncoder::new(f, Compression::default()); + let mut tar = Builder::new(gz); - // let data = b"nested_secret=shh\n"; - // let mut hdr = tar::Header::new_gnu(); - // hdr.set_size(data.len() as u64); - // hdr.set_mode(0o644); - // hdr.set_cksum(); - // tar.append_data(&mut hdr, "secret.txt", &data[..])?; + let data = b"nested_secret=shh\n"; + let mut hdr = tar::Header::new_gnu(); + hdr.set_size(data.len() as u64); + hdr.set_mode(0o644); + hdr.set_cksum(); + tar.append_data(&mut hdr, "secret.txt", &data[..])?; - // tar.into_inner()?.finish()?; - // } + tar.into_inner()?.finish()?; + } - // /* ── read inner archive into memory so we can embed it ──────────────── */ - // let mut inner_bytes = Vec::new(); - // File::open(&inner_tgz)?.read_to_end(&mut inner_bytes)?; + /* ── read inner archive into memory so we can embed it ──────────────── */ + let mut inner_bytes = Vec::new(); + File::open(&inner_tgz)?.read_to_end(&mut inner_bytes)?; - // /* ── build OUTER tar.gz that contains the inner .tar.gz ─────────────── */ - // let outer_tgz = tmp.path().join("outer.tar.gz"); - // { - // let f = File::create(&outer_tgz)?; - // let gz = GzEncoder::new(f, Compression::default()); - // let mut tar = Builder::new(gz); + /* ── build OUTER tar.gz that contains the inner .tar.gz ─────────────── */ + let outer_tgz = tmp.path().join("outer.tar.gz"); + { + let f = File::create(&outer_tgz)?; + let gz = GzEncoder::new(f, Compression::default()); + let mut tar = Builder::new(gz); - // let mut hdr = tar::Header::new_gnu(); - // hdr.set_size(inner_bytes.len() as u64); - // hdr.set_mode(0o644); - // hdr.set_cksum(); - // tar.append_data(&mut hdr, "inner.tar.gz", inner_bytes.as_slice())?; + let mut hdr = tar::Header::new_gnu(); + hdr.set_size(inner_bytes.len() as u64); + hdr.set_mode(0o644); + hdr.set_cksum(); + tar.append_data(&mut hdr, "inner.tar.gz", inner_bytes.as_slice())?; - // tar.into_inner()?.finish()?; - // } + tar.into_inner()?.finish()?; + } - // /* ── Layer 1: gunzip outer.tar.gz ───────────────────────────────────── */ - // let scratch = tempdir()?; // where intermediate layers land - // let tar_path = match decompress_once(&outer_tgz, Some(scratch.path()))? { - // CompressedContent::RawFile(p) => p, - // other => panic!("expected RawFile after gunzip, got {:?}", other), - // }; + /* ── Layer 1: gunzip outer.tar.gz ───────────────────────────────────── */ + let scratch = tempdir()?; // where intermediate layers land + let tar_path = match decompress_once(&outer_tgz, Some(scratch.path()))? { + CompressedContent::RawFile(p) => p, + other => panic!("expected RawFile after gunzip, got {:?}", other), + }; - // /* ── Layer 2: untar outer.tar -> find inner.tar.gz on disk ─────────── */ - // let inner_on_disk: PathBuf = match decompress_once(&tar_path, Some(scratch.path()))? { - // CompressedContent::ArchiveFiles(files) => files - // .into_iter() - // .find(|(logical, _)| logical.ends_with("!inner.tar.gz")) - // .map(|(_, p)| p) - // .expect("inner.tar.gz not found in outer archive"), - // other => panic!("expected ArchiveFiles after untar, got {:?}", other), - // }; + /* ── Layer 2: untar outer.tar -> find inner.tar.gz on disk ─────────── */ + let inner_on_disk: PathBuf = match decompress_once(&tar_path, Some(scratch.path()))? { + CompressedContent::ArchiveFiles(files) => files + .into_iter() + .find(|(logical, _)| logical.ends_with("!inner.tar.gz")) + .map(|(_, p)| p) + .expect("inner.tar.gz not found in outer archive"), + other => panic!("expected ArchiveFiles after untar, got {:?}", other), + }; - // /* ── Layer 3: gunzip inner.tar.gz ───────────────────────────────────── */ - // let inner_tar = match decompress_once(&inner_on_disk, Some(scratch.path()))? { - // CompressedContent::RawFile(p) => p, - // other => panic!("expected RawFile after gunzip inner, got {:?}", other), - // }; + /* ── Layer 3: gunzip inner.tar.gz ───────────────────────────────────── */ + let inner_tar = match decompress_once(&inner_on_disk, Some(scratch.path()))? { + CompressedContent::RawFile(p) => p, + other => panic!("expected RawFile after gunzip inner, got {:?}", other), + }; - // /* ── Layer 4: untar inner.tar -> secret.txt should be present ──────── */ - // match decompress_once(&inner_tar, Some(scratch.path()))? { - // CompressedContent::ArchiveFiles(files) => { - // let mut found = false; - // for (logical, path) in files { - // if logical.ends_with("!secret.txt") { - // let txt = std::fs::read_to_string(&path)?; - // assert!( - // txt.contains("nested_secret=shh"), - // "secret.txt content corrupted" - // ); - // found = true; - // } - // } - // assert!(found, "secret.txt not extracted from nested archive"); - // } - // other => panic!("expected ArchiveFiles after untar inner, got {:?}", other), - // } + /* ── Layer 4: untar inner.tar -> secret.txt should be present ──────── */ + match decompress_once(&inner_tar, Some(scratch.path()))? { + CompressedContent::ArchiveFiles(files) => { + let mut found = false; + for (logical, path) in files { + if logical.ends_with("!secret.txt") { + let txt = std::fs::read_to_string(&path)?; + assert!( + txt.contains("nested_secret=shh"), + "secret.txt content corrupted" + ); + found = true; + } + } + assert!(found, "secret.txt not extracted from nested archive"); + } + other => panic!("expected ArchiveFiles after untar inner, got {:?}", other), + } - // Ok(()) - // } + Ok(()) + } } diff --git a/tests/update.rs b/tests/update.rs index 29ae49c..f785836 100644 --- a/tests/update.rs +++ b/tests/update.rs @@ -1,46 +1,46 @@ -use kingfisher::{cli::global::GlobalArgs, update::check_for_update}; -use tokio; -use wiremock::{ - matchers::{method, path}, - Mock, MockServer, ResponseTemplate, -}; +// use kingfisher::{cli::global::GlobalArgs, update::check_for_update}; +// use tokio; +// use wiremock::{ +// matchers::{method, path}, +// Mock, MockServer, ResponseTemplate, +// }; -#[tokio::test] -async fn no_update_when_flag_set() { - let args = GlobalArgs { no_update_check: true, ..Default::default() }; - assert!(check_for_update(&args, None).is_none()); -} +// #[tokio::test] +// async fn no_update_when_flag_set() { +// let args = GlobalArgs { no_update_check: true, ..Default::default() }; +// assert!(check_for_update(&args, None).is_none()); +// } -#[tokio::test] -async fn detects_new_release() { - let server = MockServer::start().await; +// #[tokio::test] +// async fn detects_new_release() { +// let server = MockServer::start().await; - let body = serde_json::json!({ - "tag_name": "v1.99.0", - "created_at": "2025-01-01T00:00:00Z", - "name": "Kingfisher 1.99.0", - "body": "", - "assets": [{"url": "http://example.com/bin", "name": "bin"}] - }); +// let body = serde_json::json!({ +// "tag_name": "v1.99.0", +// "created_at": "2025-01-01T00:00:00Z", +// "name": "Kingfisher 1.99.0", +// "body": "", +// "assets": [{"url": "http://example.com/bin", "name": "bin"}] +// }); - // Stub HEAD *and* GET - for m in ["HEAD", "GET"] { - Mock::given(method(m)) - .and(path("/repos/mongodb/kingfisher/releases/latest")) - .respond_with(ResponseTemplate::new(200).set_body_json(&body)) - .mount(&server) - .await; - } +// // Stub HEAD *and* GET +// for m in ["HEAD", "GET"] { +// Mock::given(method(m)) +// .and(path("/repos/mongodb/kingfisher/releases/latest")) +// .respond_with(ResponseTemplate::new(200).set_body_json(&body)) +// .mount(&server) +// .await; +// } - // run the update checker on a blocking thread - let msg = tokio::task::spawn_blocking({ - let uri = server.uri(); // move into closure - let args = GlobalArgs::default(); - move || check_for_update(&args, Some(&uri)) - }) - .await - .expect("blocking task panicked") - .expect("update checker returned None"); +// // run the update checker on a blocking thread +// let msg = tokio::task::spawn_blocking({ +// let uri = server.uri(); // move into closure +// let args = GlobalArgs::default(); +// move || check_for_update(&args, Some(&uri)) +// }) +// .await +// .expect("blocking task panicked") +// .expect("update checker returned None"); - assert!(msg.contains("1.99.0")); -} +// assert!(msg.contains("1.99.0")); +// } From 4ea748bac46130989af1e0f5807fe827aa854164 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Tue, 24 Jun 2025 20:17:40 -0700 Subject: [PATCH 3/4] preparing for v1.12 --- src/decompress.rs | 2 +- tests/cli.rs | 4 +-- tests/cli_failure.rs | 4 ++- tests/smoke_archive.rs | 5 +-- tests/smoke_fs.rs | 1 + tests/smoke_git.rs | 2 +- tests/update.rs | 80 +++++++++++++++++++++--------------------- 7 files changed, 51 insertions(+), 47 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 3795910..0ac0e0e 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -389,7 +389,7 @@ mod tests { fn smoke_decompress_nested_tar_gz_archives() -> anyhow::Result<()> { use std::{ fs::File, - io::{Read, Write}, + io::Read, path::PathBuf, }; diff --git a/tests/cli.rs b/tests/cli.rs index a888a6a..f1af007 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -8,7 +8,7 @@ mod test { fn cli_lists_rules_pretty() { Command::cargo_bin("kingfisher") .unwrap() - .args(["rules", "list", "--format", "pretty"]) + .args(["rules", "list", "--format", "pretty", "--no-update-check"]) .assert() .success() .stdout(contains("kingfisher.aws.").and(contains("Pattern"))); @@ -17,7 +17,7 @@ mod test { fn cli_lists_rules_json() { Command::cargo_bin("kingfisher") .unwrap() - .args(["rules", "list", "--format", "json"]) + .args(["rules", "list", "--format", "json", "--no-update-check"]) .assert() .success() .stdout(contains("kingfisher.aws.").and(contains("pattern"))); diff --git a/tests/cli_failure.rs b/tests/cli_failure.rs index 0746f94..ca2e715 100644 --- a/tests/cli_failure.rs +++ b/tests/cli_failure.rs @@ -10,7 +10,7 @@ use tempfile::TempDir; fn scan_fails_for_missing_path() { Command::cargo_bin("kingfisher") .unwrap() - .args(["scan", "no/such/path/here"]) + .args(["scan", "no/such/path/here", "--no-update-check"]) .assert() .failure() // exit-code ≠ 0 .stderr(contains("Invalid input")); // message from run_async_scan @@ -30,6 +30,7 @@ fn scan_fails_for_bad_rule_yaml() { "--rules-path", tmp.path().to_str().unwrap(), // point loader at bad YAML "--no-validate", // keep the test fast + "--no-update-check", // skip update check to avoid network calls ]) .assert() .failure() @@ -71,6 +72,7 @@ rules: tmp.path().to_str().unwrap(), // only the custom rule "--no-dedup", "--load-builtins=false", // skip the builtin rules + "--no-update-check", // skip update check to avoid network calls ]) .assert() .failure() // CLI exits 0 diff --git a/tests/smoke_archive.rs b/tests/smoke_archive.rs index 1ae4c31..addba57 100644 --- a/tests/smoke_archive.rs +++ b/tests/smoke_archive.rs @@ -11,7 +11,7 @@ fn smoke_scan_tar_gz_archive() -> anyhow::Result<()> { // --- build a payload.tar.gz ------------------------------------------------- { - use std::{fs::File, io::Write}; + use std::fs::File; use flate2::{write::GzEncoder, Compression}; use tar::Builder; @@ -30,7 +30,7 @@ fn smoke_scan_tar_gz_archive() -> anyhow::Result<()> { // ── 1) extraction ENABLED -- secret should be found ───────────────────────── Command::cargo_bin("kingfisher")? - .args(["scan", tar_gz.to_str().unwrap(), "--confidence=low", "--format", "json"]) + .args(["scan", tar_gz.to_str().unwrap(), "--confidence=low", "--format", "json", "--no-update-check"]) .assert() .code(findings_code) .stdout(predicates::str::contains(github_pat)); @@ -44,6 +44,7 @@ fn smoke_scan_tar_gz_archive() -> anyhow::Result<()> { "--format", "json", "--no-extract-archives", + "--no-update-check", // skip update check to avoid network calls ]) .assert() .success() // always 0 diff --git a/tests/smoke_fs.rs b/tests/smoke_fs.rs index fce1198..1c1b35b 100644 --- a/tests/smoke_fs.rs +++ b/tests/smoke_fs.rs @@ -26,6 +26,7 @@ fn smoke_scan_filesystem_text_and_binary() -> anyhow::Result<()> { "--confidence=low", "--format", "json", + "--no-update-check", // skip update check to avoid network calls ]) .assert() .code(200) // findings present diff --git a/tests/smoke_git.rs b/tests/smoke_git.rs index 0ce6450..428ce24 100644 --- a/tests/smoke_git.rs +++ b/tests/smoke_git.rs @@ -40,7 +40,7 @@ fn smoke_scan_git_history() -> anyhow::Result<()> { "--confidence=low", // pick up even low-confidence rules "--format", "json", - // add "--no-validate" if the CLI supports it to avoid network I/O + "--no-update-check", // skip update check to avoid network calls ]) .assert() .code(200) // ← kingfisher’s “findings present” status diff --git a/tests/update.rs b/tests/update.rs index f785836..29ae49c 100644 --- a/tests/update.rs +++ b/tests/update.rs @@ -1,46 +1,46 @@ -// use kingfisher::{cli::global::GlobalArgs, update::check_for_update}; -// use tokio; -// use wiremock::{ -// matchers::{method, path}, -// Mock, MockServer, ResponseTemplate, -// }; +use kingfisher::{cli::global::GlobalArgs, update::check_for_update}; +use tokio; +use wiremock::{ + matchers::{method, path}, + Mock, MockServer, ResponseTemplate, +}; -// #[tokio::test] -// async fn no_update_when_flag_set() { -// let args = GlobalArgs { no_update_check: true, ..Default::default() }; -// assert!(check_for_update(&args, None).is_none()); -// } +#[tokio::test] +async fn no_update_when_flag_set() { + let args = GlobalArgs { no_update_check: true, ..Default::default() }; + assert!(check_for_update(&args, None).is_none()); +} -// #[tokio::test] -// async fn detects_new_release() { -// let server = MockServer::start().await; +#[tokio::test] +async fn detects_new_release() { + let server = MockServer::start().await; -// let body = serde_json::json!({ -// "tag_name": "v1.99.0", -// "created_at": "2025-01-01T00:00:00Z", -// "name": "Kingfisher 1.99.0", -// "body": "", -// "assets": [{"url": "http://example.com/bin", "name": "bin"}] -// }); + let body = serde_json::json!({ + "tag_name": "v1.99.0", + "created_at": "2025-01-01T00:00:00Z", + "name": "Kingfisher 1.99.0", + "body": "", + "assets": [{"url": "http://example.com/bin", "name": "bin"}] + }); -// // Stub HEAD *and* GET -// for m in ["HEAD", "GET"] { -// Mock::given(method(m)) -// .and(path("/repos/mongodb/kingfisher/releases/latest")) -// .respond_with(ResponseTemplate::new(200).set_body_json(&body)) -// .mount(&server) -// .await; -// } + // Stub HEAD *and* GET + for m in ["HEAD", "GET"] { + Mock::given(method(m)) + .and(path("/repos/mongodb/kingfisher/releases/latest")) + .respond_with(ResponseTemplate::new(200).set_body_json(&body)) + .mount(&server) + .await; + } -// // run the update checker on a blocking thread -// let msg = tokio::task::spawn_blocking({ -// let uri = server.uri(); // move into closure -// let args = GlobalArgs::default(); -// move || check_for_update(&args, Some(&uri)) -// }) -// .await -// .expect("blocking task panicked") -// .expect("update checker returned None"); + // run the update checker on a blocking thread + let msg = tokio::task::spawn_blocking({ + let uri = server.uri(); // move into closure + let args = GlobalArgs::default(); + move || check_for_update(&args, Some(&uri)) + }) + .await + .expect("blocking task panicked") + .expect("update checker returned None"); -// assert!(msg.contains("1.99.0")); -// } + assert!(msg.contains("1.99.0")); +} From 852cdda83560a7d71ba4b88270481e14b778f241 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Tue, 24 Jun 2025 20:46:20 -0700 Subject: [PATCH 4/4] preparing for v1.12 --- testdata/json_vulnerable.json | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 testdata/json_vulnerable.json diff --git a/testdata/json_vulnerable.json b/testdata/json_vulnerable.json new file mode 100644 index 0000000..8998468 --- /dev/null +++ b/testdata/json_vulnerable.json @@ -0,0 +1,25 @@ +{ + "glossary": { + "title": "example glossary", + "somedata1": ["foo", "bar"], + "GlossDiv": { + "title": "S", + "GlossList": { + "GlossEntry": { + "ID": "SGML", + "SortAs": "SGML", + "password": "blink182", + "Acronym": "qwerty123", + "Abbrev": "ISO 8879:1986", + "aws_key_id": "AKIA6ODU5DHT7VPXGCE4", + "aws_secret": "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", + "GlossDef": { + "para": "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso": ["GML", "XML"] + }, + "GlossSee": "markup" + } + } + } + } +} \ No newline at end of file