eblume/kingfisher
1
0
Fork 0
forked from mirrors/kingfisher
Code Pull requests Activity Actions

fixing rules

Browse source
This commit is contained in:
Mick Grove 2025-11-08 10:48:00 -08:00
commit 8aac161603
22 changed files with 78 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

15
data/rules/aws.yml
View file

@ -22,23 +22,22 @@ rules:
pattern: |
(?xi)
(?:
(?:[^A-Za-z0-9/+=])
\b
(?:AWS|AMAZON|AMZN|A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)
(?:.|[\n\r]){0,64}?
(?:[^A-Za-z0-9/+=])
([A-Za-z0-9+]{40})
(?:[^A-Za-z0-9/+=])
\b
([A-Za-z0-9/+]{40})
\b
|
(?:[^A-Za-z0-9/+=])
(?:AWS|AMAZON|AMZN|A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)
\b(?:AWS|AMAZON|AMZN|A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)
(?:.|[\n\r]){0,96}?
(?:SECRET|PRIVATE|ACCESS)
(?:.|[\n\r]){0,16}?
(?:KEY|TOKEN)
(?:.|[\n\r]){0,64}?
\b
([A-Za-z0-9+]{40})
(?:[^A-Za-z0-9/+=])
([A-Za-z0-9/+]{40})
\b
)
pattern_requirements:
min_digits: 2

4
data/rules/azurestorage.yml
View file

@ -28,7 +28,7 @@ rules:
id: kingfisher.azurestorage.2
pattern: |
(?xi)
(?:[^A-Za-z0-9/+=])
\b
azure
(?:.|[\n\r]){0,128}?
(?:SECRET|PRIVATE|ACCESS|KEY|TOKEN)
@ -51,7 +51,7 @@ rules:
confidence: medium
examples:
- Azure AccountKey=Xy9aB8cD7eF6gH5iJ4kL3mN2oP1qR0sT9uV8wX7yZ6aB5cD4eF3gH2iJ1kL0mN9oP8qR7sT6uV5wX4yZ3aB2cD1q
- Azure AccountKey=Xy9aB8cD7eF6gH5iJ4kL3mN2oP1qR0sT9uV8wX7yZ6aB5cD4eF3gH2iJ1kL0mN9oP8qR7sT6uV5wX4yZ3aB2cD1g==\
- Azure AccountKey=Ky7aC1cD7eF6gH5iJ4kL3mN2oP1qR0sT9uV8wX7yZ6aB5cD4eF3gH2iJ1kL0mN9oP8qR7sT6uV5wX4yZ3aB2cD1g==\
validation:
type: AzureStorage
depends_on_rule:

5
data/rules/confluent.yml
View file

@ -32,12 +32,11 @@ rules:
(
[A-Z0-9\+/]{64}
)
(?:[^A-Za-z0-9/+=])
min_entropy: 3.3
confidence: medium
examples:
- confluent secret=abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890ab
- kafka_token=ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890abcdefghijklmnopqrstuvwxyzAB
- confluent secret=cbadefghijklmnopqrstuvwxyzcbaDEFGHIJKLMNOPQRSTUVWXYZ3214567890ab
- kafka_token=cbaDEFGHIJKLMNOPQRSTUVWXYZ3214567890cbadefghijklmnopqrstuvwxyzAB
references:
- https://docs.confluent.io/cloud/current/api.html#tag/API-Keys-(iamv2)/operation/getIamV2ApiKey
validation:

11
data/rules/github.yml
View file

@ -4,15 +4,16 @@ rules:
pattern: |
(?xi)
(
github_pat
[A-Z0-9]{80,84}
github_pat_
[A-Z0-9_+]{82,84}
)
\b
pattern_requirements:
min_digits: 2
min_lowercase: 2
min_entropy: 3.5
examples:
- "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"
- "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"
references:
- https://docs.github.com/en/rest/users?apiVersion=2022-11-28
validation:
@ -247,11 +248,11 @@ rules:
(?:.|[\n\r]){0,32}?
(?:SECRET|PRIVATE|ACCESS|KEY|TOKEN)
(?:.|[\n\r]){0,32}?
(?:[^A-Za-z0-9/+=])
\b
(
[a-z0-9]{40}
)
(?:[^A-Za-z0-9/+=])
\b
depends_on_rule:
- rule_id: "kingfisher.github.5"
variable: GITHUB_CLIENT_ID

3
data/rules/gitlab.yml
View file

@ -86,11 +86,10 @@ rules:
id: kingfisher.gitlab.3
pattern: |
(?xi)
(?:[^A-Za-z0-9/+=]|\r\n|\\n)
\b
(
glptt-[0-9a-f]{40}
)
\b
pattern_requirements:
min_digits: 2
examples:

2
data/rules/ibm.yml
View file

@ -3,7 +3,7 @@ rules:
id: kingfisher.ibm.1
pattern: |
(?xi)
(?:[^A-Za-z0-9/+=])
\b
(?:ibm(?:cloud)?|bx)
(?:.|[\n\r]){0,32}?
(?:SECRET|PRIVATE|ACCESS|KEY|TOKEN)

12
data/rules/mapbox.yml
View file

@ -1,7 +1,7 @@
rules:
- name: Mapbox Public Access Token
id: kingfisher.mapbox.1
pattern: '(?i)(?s)mapbox.{0,30}(pk\.[a-z0-9\-+/=]{32,128}\.[a-z0-9\-+/=]{20,30})(?:[^a-z0-9\-+/=])'
pattern: '(?i)(?s)mapbox.{0,30}(pk\.[a-z0-9\-+/=]{32,128}\.[a-z0-9\-+/=]{20,30})\b'
pattern_requirements:
min_digits: 2
min_entropy: 3.3
@ -28,7 +28,13 @@ rules:
- name: Mapbox Secret Access Token
id: kingfisher.mapbox.2
pattern: '(?i)(?s)mapbox.{0,30}(sk\.[a-z0-9\-+/=]{32,128}\.[a-z0-9\-+/=]{20,30})(?:[^a-z0-9\-+/=])'
pattern: |
(?xi)(?s)
mapbox.{0,30}
(
sk\.[a-z0-9\-+/=]{32,128}\.[a-z0-9\-+/=]{20,30}
)
\b
pattern_requirements:
min_digits: 2
min_entropy: 3.3
@ -54,7 +60,7 @@ rules:
- name: Mapbox Temporary Access Token
id: kingfisher.mapbox.3
pattern: '(?i)(?s)mapbox.{0,30}(tk\.[a-z0-9\-+/=]{32,128}\.[a-z0-9\-+/=]{20,30})(?:[^a-z0-9\-+/=])'
pattern: '(?i)(?s)mapbox.{0,30}(tk\.[a-z0-9\-+/=]{32,128}\.[a-z0-9\-+/=]{20,30})\b'
pattern_requirements:
min_digits: 2
min_entropy: 3.3

2
data/rules/npm.yml
View file

@ -23,7 +23,7 @@ rules:
min_entropy: 3.3
confidence: medium
examples:
- 'npm_TCllNwh2WLQlMWVhybM1iQrsTj6rMQ0BOh6d'
- "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"
validation:
type: Http
content:

16
data/rules/opsgenie.yml
View file

@ -18,17 +18,23 @@ rules:
min_entropy: 3.5
examples:
- opsgenie_api_key = '12345678-9abc-def0-1234-56789abcdef0'
references:
- https://docs.opsgenie.com/docs/api-overview
- https://support.atlassian.com/security-and-access-policies/docs/send-alerts-to-opsgenie/
- https://support.atlassian.com/opsgenie/docs/european-service-region/
validation:
type: Http
content:
request:
headers:
Authorization: GenieKey {{ TOKEN }}
method: GET
url: https://api.opsgenie.com/v2/alerts
headers:
Authorization: "GenieKey {{ TOKEN }}"
response_matcher:
- report_response: true
- type: WordMatch
words:
- "Could not authenticate"
- type: StatusMatch
status: [401, 403]
negative: true
- type: WordMatch
words: ["Could not authenticate", "is not valid"]
negative: true

3
data/rules/pypi.yml
View file

@ -3,11 +3,10 @@ rules:
id: kingfisher.pypi.1
pattern: |
(?xi)
\b
(
pypi-AgEIcHlwaS5vcmc[A-Z0-9_-]{50,}
)
(?:[^A-Z0-9_-])
\b
pattern_requirements:
min_digits: 2
min_entropy: 3.3

1
data/rules/recaptcha.yml
View file

@ -9,7 +9,6 @@ rules:
(
6l[c-f][a-z0-9_-].{36}
)
(?:[^A-Za-z0-9/])
pattern_requirements:
min_digits: 3
min_entropy: 3

15
data/rules/sentry.yml
View file

@ -12,14 +12,14 @@ rules:
(
[a-f0-9]{64}
)
(?:[^A-Za-z0-9/+=])
\b
pattern_requirements:
min_digits: 2
min_entropy: 3.5
confidence: medium
examples:
- SENTRY_TOKEN=abcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcd
- '"sentry": "1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"'
- SENTRY_TOKEN=cbadefcbadefcbadefcbadefcbadefcbadefcbadefcbadefcbadefcbadefcbad
- '"sentry-key": "3214567890cbadef3214567890cbadef3214567890cbadef3214567890cbadef"'
references:
- https://docs.sentry.io/api/auth/
validation:
@ -41,7 +41,6 @@ rules:
id: kingfisher.sentry.2
pattern: |
(?xi)
\b
(
sntrys_eyJpYXQiO[a-zA-Z0-9+/]{10,200}(?:LCJyZWdpb25fdXJs|InJlZ2lvbl91cmwi|cmVnaW9uX3VybCI6)[a-zA-Z0-9+/]{10,200}={0,2}_[a-zA-Z0-9+/]{43}
)
@ -51,8 +50,8 @@ rules:
min_entropy: 4.2
confidence: medium
examples:
- sntrys_eyJpYXQiOjE2OTA4ODAwMDAsInJlZ2lvbl91cmwiOiJodHRwczovL3NlbnRyeS5pby9vcmdzL215LW9yZy8ifQ==_abcdefghijklmnopqrstuvwx1234567890abcdefabc
- sntrys_eyJpYXQiOiIxNjkwODgwMDAwIiwicmVnaW9uX3VybCI6Imh0dHBzOi8vc2VudHJ5LmlvLyJ9_abcdABCD1234567890abcdABCD1234567890abcdABC
- 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
- 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
references:
- https://docs.sentry.io/api/auth/
validation:
@ -84,8 +83,8 @@ rules:
min_entropy: 3.5
confidence: medium
examples:
- sntryu_abcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcd
- SNTRY_USER="sntryu_1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef"
- 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
- SNTRY_USER="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"
references:
- https://docs.sentry.io/api/auth/
validation:

2
data/rules/telegram.yml
View file

@ -11,7 +11,7 @@ rules:
:
[A-Z0-9_-]{35}
)
(?:[^A-Za-z0-9/+=])
\b
pattern_requirements:
min_digits: 2
confidence: medium

4
src/decompress.rs
View file

@ -340,7 +340,7 @@ mod tests {
fn smoke_decompress_tar_gz_archive() -> anyhow::Result<()> {
let dir = tempdir()?;
let tar_gz = dir.path().join("payload.tar.gz");
let github_pat = "ghp_1wuHFikBKQtCcH3EB2FBUkyn8krXhP2qLqPa"; // this is not a real secret
let github_pat = "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"; // this is not a real secret
// build payload.tar.gz containing secret.txt
{
@ -393,7 +393,7 @@ mod tests {
fn smoke_decompress_without_extract_archives() -> anyhow::Result<()> {
let dir = tempdir()?;
let tar_gz = dir.path().join("payload.tar.gz");
let github_pat = "ghp_1wuHFikBKQtCcH3EB2FBUkyn8krXhP2qLqPa";
let github_pat = "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";
// ── build payload.tar.gz containing secret.txt ──────────────────────────────
{

15
src/validation.rs
View file

@ -996,8 +996,14 @@ mod tests {
let mut globals = Object::new();
populate_globals_from_captures(&mut globals, &captured_values);
assert_eq!(globals.get("TOKEN").map(|v| v.to_string()), Some("longervalue".to_string()));
assert_eq!(globals.get("BODY").map(|v| v.to_string()), Some("body".to_string()));
assert_eq!(
globals.get("TOKEN"),
Some(Value::scalar("longervalue")).as_ref()
);
assert_eq!(
globals.get("BODY"),
Some(Value::scalar("body")).as_ref()
);
}
#[test]
@ -1008,7 +1014,10 @@ mod tests {
populate_globals_from_captures(&mut globals, &captured_values);
assert!(globals.get("TOKEN").is_none());
assert_eq!(globals.get("CHECKSUM").map(|v| v.to_string()), Some("123456".to_string()));
assert_eq!(
globals.get("CHECKSUM"),
Some(Value::scalar("123456")).as_ref()
);
}
}

14
tests/int_base64.rs
View file

@ -8,8 +8,8 @@ use tempfile::tempdir;
fn detects_base64_encoded_secret() -> anyhow::Result<()> {
let dir = tempdir()?;
let file_path = dir.path().join("secret.txt");
// Base64 for ghp_1wuHFikBKQtCcH3EB2FBUkyn8krXhP2qLqPa
let encoded = "Z2hwXzF3dUhGaWtCS1F0Q2NIM0VCMkZCVWt5bjhrclhoUDJxTHFQYQ==";
// Base64 for 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
let encoded = "Z2hwXzF3dUhGaWtCS1F0Q2NIM0VCMkZCVWt5bjhrclhoUDBNV0h4cw==";
fs::write(&file_path, encoded)?;
Command::new(assert_cmd::cargo::cargo_bin!("kingfisher"))
@ -26,7 +26,7 @@ fn detects_base64_encoded_secret() -> anyhow::Result<()> {
.assert()
.code(200)
.stdout(
predicate::str::contains("ghp_1wuHFikBKQtCcH3EB2FBUkyn8krXhP2qLqPa")
predicate::str::contains("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")
.and(predicate::str::contains("\"encoding\": \"base64\"")),
);
@ -39,7 +39,7 @@ fn detects_base64_encoded_secret() -> anyhow::Result<()> {
fn skips_base64_when_disabled() -> anyhow::Result<()> {
let dir = tempdir()?;
let file_path = dir.path().join("secret.txt");
let encoded = "Z2hwXzF3dUhGaWtCS1F0Q2NIM0VCMkZCVWt5bjhrclhoUDJxTHFQYQ==";
let encoded = "Z2hwXzF3dUhGaWtCS1F0Q2NIM0VCMkZCVWt5bjhrclhoUDBNV0h4cw==";
fs::write(&file_path, encoded)?;
Command::new(assert_cmd::cargo::cargo_bin!("kingfisher"))
@ -92,8 +92,8 @@ fn no_base64_skips_empty_files() -> anyhow::Result<()> {
fn detects_base64_in_code_with_tree_sitter() -> anyhow::Result<()> {
let dir = tempdir()?;
let file_path = dir.path().join("secret.py");
// Base64 for ghp_1wuHFikBKQtCcH3EB2FBUkyn8krXhP2qLqPa
let encoded = "Z2hwXzF3dUhGaWtCS1F0Q2NIM0VCMkZCVWt5bjhrclhoUDJxTHFQYQ==";
// Base64 for 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
let encoded = "Z2hwXzF3dUhGaWtCS1F0Q2NIM0VCMkZCVWt5bjhrclhoUDBNV0h4cw==";
fs::write(&file_path, format!("token = \"{}\"\n", encoded))?;
Command::new(assert_cmd::cargo::cargo_bin!("kingfisher"))
@ -110,7 +110,7 @@ fn detects_base64_in_code_with_tree_sitter() -> anyhow::Result<()> {
.assert()
.code(200)
.stdout(
predicate::str::contains("ghp_1wuHFikBKQtCcH3EB2FBUkyn8krXhP2qLqPa")
predicate::str::contains("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")
.and(predicate::str::contains("\"encoding\": \"base64\"")),
);

2
tests/int_slack.rs
View file

@ -159,7 +159,7 @@ async fn test_scan_slack_messages() -> Result<()> {
"messages": {
"matches": [{
"permalink": "https://example.slack.com/archives/C123/p1234",
"text": "This contains a github token ghp_1wuHFikBKQtCcH3EB2FBUkyn8krXhP2qLqPa",
"text": "This contains a github token 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",
"ts": "1234.56",
"channel": {"id": "C123", "name": "general"}
}],

2
tests/smoke_archive.rs
View file

@ -7,7 +7,7 @@ fn smoke_scan_tar_gz_archive() -> anyhow::Result<()> {
let dir = tempfile::tempdir()?;
let tar_gz = dir.path().join("payload.tar.gz");
let github_pat = "ghp_1wuHFikBKQtCcH3EB2FBUkyn8krXhP2qLqPa";
let github_pat = "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";
// --- build a payload.tar.gz -------------------------------------------------
{

2
tests/smoke_baseline.rs
View file

@ -5,7 +5,7 @@ use clap::Parser;
use predicates::prelude::*;
use tempfile::tempdir;
const GH_PAT: &str = "ghp_1wuHFikBKQtCcH3EB2FBUkyn8krXhP2qLqPa";
const GH_PAT: &str = "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";
#[test]
fn manage_baseline_enables_no_dedup() -> anyhow::Result<()> {

2
tests/smoke_exclude.rs
View file

@ -4,7 +4,7 @@ use assert_cmd::Command;
use predicates::prelude::*;
use tempfile::tempdir;
const SECRET: &str = "ghp_1wuHFikBKQtCcH3EB2FBUkyn8krXhP2qLqPa";
const SECRET: &str = "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";
#[test]
fn exclude_pattern_hides_matches() -> anyhow::Result<()> {

2
tests/smoke_fs.rs
View file

@ -5,7 +5,7 @@ use assert_cmd::prelude::*;
use predicates::prelude::*;
use tempfile::tempdir;
const GITHUB_PAT: &str = "ghp_1wuHFikBKQtCcH3EB2FBUkyn8krXhP2qLqPa";
const GITHUB_PAT: &str = "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";
#[test]
fn smoke_scan_filesystem_text_and_binary() -> anyhow::Result<()> {

6
tests/smoke_git.rs
View file

@ -15,7 +15,7 @@ fn smoke_scan_git_history() -> anyhow::Result<()> {
// commit v1
let file_path = repo_dir.join("config.yml");
fs::write(&file_path, b"ghp_1wuHFikBKQtCcH3EB2FBUkyn8krXhP2qLqPa")?;
fs::write(&file_path, b"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")?;
let mut idx = repo.index()?;
idx.add_path(std::path::Path::new("config.yml"))?;
let oid1 = idx.write_tree()?;
@ -23,7 +23,7 @@ fn smoke_scan_git_history() -> anyhow::Result<()> {
repo.commit(Some("HEAD"), &sig, &sig, "init", &tree1, &[])?;
// commit v2 (same leak, will test dedup)
fs::write(&file_path, b"ghp_1wuHFikBKQtCcH3EB2FBUkyn8krXhP2qLqPa # unchanged")?;
fs::write(&file_path, b"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 # unchanged")?;
idx.add_path(std::path::Path::new("config.yml"))?;
let oid2 = idx.write_tree()?;
let tree2 = repo.find_tree(oid2)?;
@ -44,7 +44,7 @@ fn smoke_scan_git_history() -> anyhow::Result<()> {
])
.assert()
.code(200) // ← kingfisher’s “findings present” status
.stdout(predicate::str::contains("ghp_1wuHFikBKQtCcH3EB2FBUkyn8krXhP2qLqPa"));
.stdout(predicate::str::contains("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"));
dir.close()?;
Ok(())