eblume/kingfisher
1
0
Fork 0
forked from mirrors/kingfisher
Code Pull requests Activity Actions

Merge pull request #9 from mongodb/development

Browse source 
preparing for v1.12
This commit is contained in:
Mick Grove 2025-06-24 21:20:00 -07:00 • committed by GitHub
commit f51a6c06bf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 162 additions and 131 deletions
Download patch file Download diff file Expand all files Collapse all files

4
CHANGELOG.md
View file

@ -2,8 +2,8 @@
All notable changes to this project will be documented in this file.
## [1.12.0]
- Added automatic update checks using GitHub releases
## [1.12.0]
- Added automatic update checks using GitHub releases.
- New `--self-update` flag installs updates when available
- New `--no-update-check` flag disables update checks
- Updated rules

1
Cargo.toml
View file

@ -160,6 +160,7 @@ percent-encoding = "2.3.1"
trust-dns-resolver = { version = "0.23.2", default-features = false, features = ["tokio-runtime"] }
atty = "0.2.14"
self_update = { version = "0.42.0", default-features = false, features = ["rustls"] }
semver = "1.0.26"
[dependencies.tikv-jemallocator]
version = "0.6"

35
data/rules/digitalocean.yml
View file

@ -48,20 +48,21 @@ rules:
- ' "refresh_token": "dor_v1_d6ce5b93104521c47be0b580e9296454ef4a319b02b5513469f0ec71d99af2e2",'
validation:
type: Http
content:
request:
method: POST
url: https://cloud.digitalocean.com/v1/oauth/token
headers:
Content-Type: application/json
Accept: application/json
body: |
{
"grant_type": "refresh_token",
"refresh_token": "{{ TOKEN }}"
}
response_matcher:
- report_response: true
- type: StatusMatch
status: [200]
- type: JsonValid
content:
request:
method: POST
url: https://cloud.digitalocean.com/v1/oauth/token
headers:
Content-Type: application/json
Accept: application/json
body: |
{
"grant_type": "refresh_token",
"refresh_token": "{{ TOKEN }}"
}
response_matcher:
- report_response: true
- type: StatusMatch
status:
- 200
- type: JsonValid

10
data/rules/facebook.yml
View file

@ -50,11 +50,11 @@ rules:
- status:
- 200
type: StatusMatch
url: >-
https://graph.facebook.com/v19.0/oauth/access_token
?client_id={{ APIID }}
&client_secret={{ TOKEN }}
&grant_type=client_credentials
url: >-
https://graph.facebook.com/v19.0/oauth/access_token
?client_id={{ APIID }}
&client_secret={{ TOKEN }}
&grant_type=client_credentials
depends_on_rule:
- rule_id: kingfisher.facebook.1
variable: APIID

172
src/decompress.rs
View file

@ -276,7 +276,7 @@ pub fn decompress_file_to_temp(path: &Path) -> Result<(CompressedContent, TempDi
#[cfg(test)]
mod tests {
use std::{fs::File, io::Write, path::PathBuf};
use std::fs::File;
use flate2::{write::GzEncoder, Compression};
use tar::Builder;
@ -382,104 +382,104 @@ mod tests {
Ok(())
}
// /// 3) Nested archive:
// /// outer.tar.gz ──▶ outer.tar (contains inner.tar.gz)
// /// └──▶ inner.tar.gz ──▶ inner.tar (contains secret.txt)
// #[test]
// fn smoke_decompress_nested_tar_gz_archives() -> anyhow::Result<()> {
// use std::{
// fs::File,
// io::{Read, Write},
// path::PathBuf,
// };
/// 3) Nested archive:
/// outer.tar.gz ──▶ outer.tar (contains inner.tar.gz)
/// └──▶ inner.tar.gz ──▶ inner.tar (contains secret.txt)
#[test]
fn smoke_decompress_nested_tar_gz_archives() -> anyhow::Result<()> {
use std::{
fs::File,
io::Read,
path::PathBuf,
};
// use flate2::{write::GzEncoder, Compression};
// use tar::Builder;
// use tempfile::tempdir;
use flate2::{write::GzEncoder, Compression};
use tar::Builder;
use tempfile::tempdir;
// use super::{decompress_once, CompressedContent};
use super::{decompress_once, CompressedContent};
// let tmp = tempdir()?;
let tmp = tempdir()?;
// /* ── build INNER tar.gz ──────────────────────────────────────────────── */
// let inner_tgz = tmp.path().join("inner.tar.gz");
// {
// let f = File::create(&inner_tgz)?;
// let gz = GzEncoder::new(f, Compression::default());
// let mut tar = Builder::new(gz);
/* ── build INNER tar.gz ──────────────────────────────────────────────── */
let inner_tgz = tmp.path().join("inner.tar.gz");
{
let f = File::create(&inner_tgz)?;
let gz = GzEncoder::new(f, Compression::default());
let mut tar = Builder::new(gz);
// let data = b"nested_secret=shh\n";
// let mut hdr = tar::Header::new_gnu();
// hdr.set_size(data.len() as u64);
// hdr.set_mode(0o644);
// hdr.set_cksum();
// tar.append_data(&mut hdr, "secret.txt", &data[..])?;
let data = b"nested_secret=shh\n";
let mut hdr = tar::Header::new_gnu();
hdr.set_size(data.len() as u64);
hdr.set_mode(0o644);
hdr.set_cksum();
tar.append_data(&mut hdr, "secret.txt", &data[..])?;
// tar.into_inner()?.finish()?;
// }
tar.into_inner()?.finish()?;
}
// /* ── read inner archive into memory so we can embed it ──────────────── */
// let mut inner_bytes = Vec::new();
// File::open(&inner_tgz)?.read_to_end(&mut inner_bytes)?;
/* ── read inner archive into memory so we can embed it ──────────────── */
let mut inner_bytes = Vec::new();
File::open(&inner_tgz)?.read_to_end(&mut inner_bytes)?;
// /* ── build OUTER tar.gz that contains the inner .tar.gz ─────────────── */
// let outer_tgz = tmp.path().join("outer.tar.gz");
// {
// let f = File::create(&outer_tgz)?;
// let gz = GzEncoder::new(f, Compression::default());
// let mut tar = Builder::new(gz);
/* ── build OUTER tar.gz that contains the inner .tar.gz ─────────────── */
let outer_tgz = tmp.path().join("outer.tar.gz");
{
let f = File::create(&outer_tgz)?;
let gz = GzEncoder::new(f, Compression::default());
let mut tar = Builder::new(gz);
// let mut hdr = tar::Header::new_gnu();
// hdr.set_size(inner_bytes.len() as u64);
// hdr.set_mode(0o644);
// hdr.set_cksum();
// tar.append_data(&mut hdr, "inner.tar.gz", inner_bytes.as_slice())?;
let mut hdr = tar::Header::new_gnu();
hdr.set_size(inner_bytes.len() as u64);
hdr.set_mode(0o644);
hdr.set_cksum();
tar.append_data(&mut hdr, "inner.tar.gz", inner_bytes.as_slice())?;
// tar.into_inner()?.finish()?;
// }
tar.into_inner()?.finish()?;
}
// /* ── Layer 1: gunzip outer.tar.gz ───────────────────────────────────── */
// let scratch = tempdir()?; // where intermediate layers land
// let tar_path = match decompress_once(&outer_tgz, Some(scratch.path()))? {
// CompressedContent::RawFile(p) => p,
// other => panic!("expected RawFile after gunzip, got {:?}", other),
// };
/* ── Layer 1: gunzip outer.tar.gz ───────────────────────────────────── */
let scratch = tempdir()?; // where intermediate layers land
let tar_path = match decompress_once(&outer_tgz, Some(scratch.path()))? {
CompressedContent::RawFile(p) => p,
other => panic!("expected RawFile after gunzip, got {:?}", other),
};
// /* ── Layer 2: untar outer.tar -> find inner.tar.gz on disk ─────────── */
// let inner_on_disk: PathBuf = match decompress_once(&tar_path, Some(scratch.path()))? {
// CompressedContent::ArchiveFiles(files) => files
// .into_iter()
// .find(|(logical, _)| logical.ends_with("!inner.tar.gz"))
// .map(|(_, p)| p)
// .expect("inner.tar.gz not found in outer archive"),
// other => panic!("expected ArchiveFiles after untar, got {:?}", other),
// };
/* ── Layer 2: untar outer.tar -> find inner.tar.gz on disk ─────────── */
let inner_on_disk: PathBuf = match decompress_once(&tar_path, Some(scratch.path()))? {
CompressedContent::ArchiveFiles(files) => files
.into_iter()
.find(|(logical, _)| logical.ends_with("!inner.tar.gz"))
.map(|(_, p)| p)
.expect("inner.tar.gz not found in outer archive"),
other => panic!("expected ArchiveFiles after untar, got {:?}", other),
};
// /* ── Layer 3: gunzip inner.tar.gz ───────────────────────────────────── */
// let inner_tar = match decompress_once(&inner_on_disk, Some(scratch.path()))? {
// CompressedContent::RawFile(p) => p,
// other => panic!("expected RawFile after gunzip inner, got {:?}", other),
// };
/* ── Layer 3: gunzip inner.tar.gz ───────────────────────────────────── */
let inner_tar = match decompress_once(&inner_on_disk, Some(scratch.path()))? {
CompressedContent::RawFile(p) => p,
other => panic!("expected RawFile after gunzip inner, got {:?}", other),
};
// /* ── Layer 4: untar inner.tar -> secret.txt should be present ──────── */
// match decompress_once(&inner_tar, Some(scratch.path()))? {
// CompressedContent::ArchiveFiles(files) => {
// let mut found = false;
// for (logical, path) in files {
// if logical.ends_with("!secret.txt") {
// let txt = std::fs::read_to_string(&path)?;
// assert!(
// txt.contains("nested_secret=shh"),
// "secret.txt content corrupted"
// );
// found = true;
// }
// }
// assert!(found, "secret.txt not extracted from nested archive");
// }
// other => panic!("expected ArchiveFiles after untar inner, got {:?}", other),
// }
/* ── Layer 4: untar inner.tar -> secret.txt should be present ──────── */
match decompress_once(&inner_tar, Some(scratch.path()))? {
CompressedContent::ArchiveFiles(files) => {
let mut found = false;
for (logical, path) in files {
if logical.ends_with("!secret.txt") {
let txt = std::fs::read_to_string(&path)?;
assert!(
txt.contains("nested_secret=shh"),
"secret.txt content corrupted"
);
found = true;
}
}
assert!(found, "secret.txt not extracted from nested archive");
}
other => panic!("expected ArchiveFiles after untar inner, got {:?}", other),
}
// Ok(())
// }
Ok(())
}
}

25
testdata/json_vulnerable.json vendored Normal file
View file

@ -0,0 +1,25 @@
{
"glossary": {
"title": "example glossary",
"somedata1": ["foo", "bar"],
"GlossDiv": {
"title": "S",
"GlossList": {
"GlossEntry": {
"ID": "SGML",
"SortAs": "SGML",
"password": "blink182",
"Acronym": "qwerty123",
"Abbrev": "ISO 8879:1986",
"aws_key_id": "AKIA6ODU5DHT7VPXGCE4",
"aws_secret": "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",
"GlossDef": {
"para": "A meta-markup language, used to create markup languages such as DocBook.",
"GlossSeeAlso": ["GML", "XML"]
},
"GlossSee": "markup"
}
}
}
}
}

4
tests/cli.rs
View file

@ -8,7 +8,7 @@ mod test {
fn cli_lists_rules_pretty() {
Command::cargo_bin("kingfisher")
.unwrap()
.args(["rules", "list", "--format", "pretty"])
.args(["rules", "list", "--format", "pretty", "--no-update-check"])
.assert()
.success()
.stdout(contains("kingfisher.aws.").and(contains("Pattern")));
@ -17,7 +17,7 @@ mod test {
fn cli_lists_rules_json() {
Command::cargo_bin("kingfisher")
.unwrap()
.args(["rules", "list", "--format", "json"])
.args(["rules", "list", "--format", "json", "--no-update-check"])
.assert()
.success()
.stdout(contains("kingfisher.aws.").and(contains("pattern")));

4
tests/cli_failure.rs
View file

@ -10,7 +10,7 @@ use tempfile::TempDir;
fn scan_fails_for_missing_path() {
Command::cargo_bin("kingfisher")
.unwrap()
.args(["scan", "no/such/path/here"])
.args(["scan", "no/such/path/here", "--no-update-check"])
.assert()
.failure() // exit-code ≠ 0
.stderr(contains("Invalid input")); // message from run_async_scan
@ -30,6 +30,7 @@ fn scan_fails_for_bad_rule_yaml() {
"--rules-path",
tmp.path().to_str().unwrap(), // point loader at bad YAML
"--no-validate", // keep the test fast
"--no-update-check", // skip update check to avoid network calls
])
.assert()
.failure()
@ -71,6 +72,7 @@ rules:
tmp.path().to_str().unwrap(), // only the custom rule
"--no-dedup",
"--load-builtins=false", // skip the builtin rules
"--no-update-check", // skip update check to avoid network calls
])
.assert()
.failure() // CLI exits 0

5
tests/smoke_archive.rs
View file

@ -11,7 +11,7 @@ fn smoke_scan_tar_gz_archive() -> anyhow::Result<()> {
// --- build a payload.tar.gz -------------------------------------------------
{
use std::{fs::File, io::Write};
use std::fs::File;
use flate2::{write::GzEncoder, Compression};
use tar::Builder;
@ -30,7 +30,7 @@ fn smoke_scan_tar_gz_archive() -> anyhow::Result<()> {
// ── 1) extraction ENABLED -- secret should be found ─────────────────────────
Command::cargo_bin("kingfisher")?
.args(["scan", tar_gz.to_str().unwrap(), "--confidence=low", "--format", "json"])
.args(["scan", tar_gz.to_str().unwrap(), "--confidence=low", "--format", "json", "--no-update-check"])
.assert()
.code(findings_code)
.stdout(predicates::str::contains(github_pat));
@ -44,6 +44,7 @@ fn smoke_scan_tar_gz_archive() -> anyhow::Result<()> {
"--format",
"json",
"--no-extract-archives",
"--no-update-check", // skip update check to avoid network calls
])
.assert()
.success() // always 0

1
tests/smoke_fs.rs
View file

@ -26,6 +26,7 @@ fn smoke_scan_filesystem_text_and_binary() -> anyhow::Result<()> {
"--confidence=low",
"--format",
"json",
"--no-update-check", // skip update check to avoid network calls
])
.assert()
.code(200) // findings present

2
tests/smoke_git.rs
View file

@ -40,7 +40,7 @@ fn smoke_scan_git_history() -> anyhow::Result<()> {
"--confidence=low", // pick up even low-confidence rules
"--format",
"json",
// add "--no-validate" if the CLI supports it to avoid network I/O
"--no-update-check", // skip update check to avoid network calls
])
.assert()
.code(200) // ← kingfisher’s “findings present” status