From 9c4e459a149604c062eea222bb8750728be7d71c Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Thu, 6 Nov 2025 16:22:35 -0800 Subject: [PATCH] updated ci --- CHANGELOG.md | 1 + README.md | 4 +- data/rules/github.yml | 118 ++++++++++++++++++++++++++++++---------- data/rules/mongodb.yml | 4 -- data/rules/ngrok.yml | 6 +- data/rules/postgres.yml | 5 -- 6 files changed, 96 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 884beb6..9f94263 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ All notable changes to this project will be documented in this file. ## [v1.62.0] +- This release is focused on further improving detection accuracy, before even attempting to validate findings. - Added `pattern_requirements` checks to rules, providing lightweight post-regex character-class validation without lookarounds. - Added an `ignore_if_contains` option to `PatternRequirements` to drop matches containing case-insensitive placeholder words, with tests covering the new behavior. - Updated rules to adopt the new `pattern_requirements` support. diff --git a/README.md b/README.md index 656e73c..95a4d68 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ See ([docs/COMPARISON.md](docs/COMPARISON.md)) - [ Run Kingfisher in Docker](#-run-kingfisher-in-docker) - [🔐 Detection Rules at a Glance](#-detection-rules-at-a-glance) - [📝 Write Custom Rules!](#-write-custom-rules) + - [Pattern requirements and placeholder filtering](#pattern-requirements-and-placeholder-filtering) - [🎉 Usage](#-usage) - [Basic Examples](#basic-examples) - [Scan with secret validation](#scan-with-secret-validation) @@ -331,8 +332,7 @@ is independent: - `min_digits`, `min_uppercase`, `min_lowercase`, and `min_special_chars` enforce complexity thresholds. - `special_chars` lets you override the set of characters counted as "special" when `min_special_chars` is used. - `ignore_if_contains` lists case-insensitive substrings that should cause a match to be discarded (for example, to drop - `test`, `demo`, or `localhost` values). Kingfisher still accepts the legacy `exclude_words` key as an alias when loading - existing rule files. + `test`, `demo`, or `localhost` values). When a match is skipped because of `ignore_if_contains`, Kingfisher logs the event at the `DEBUG` level alongside the rule that was evaluated. If you need to keep those matches for a particular scan, pass `--no-ignore-if-contains` to `kingfisher scan` to diff --git a/data/rules/github.yml b/data/rules/github.yml index 7941ac5..f4c53d8 100644 --- a/data/rules/github.yml +++ b/data/rules/github.yml @@ -1,34 +1,17 @@ rules: - - name: GitHub Personal Access Token + - name: GitHub Personal Access Token - fine-grained permissions id: kingfisher.github.1 pattern: | - (?xi) - \b + (?xi) ( - (?: # for token prefixes - ghp| # Personal Access Token - gho| # OAuth Token - ghu| # GitHub App User-to-Server Token - ghs| # GitHub App Server-to-Server Token - ghr| # Refresh Token - github_pat # Alternative format for Personal Access Token - )_ - (?: # for token body - [a-z0-9_]{35,235} # 35 to 235 lowercase alphanumeric characters or underscores - ) - ) - \b + github_pat + [A-Z0-9]{80,84} + ) pattern_requirements: min_digits: 2 min_lowercase: 2 min_entropy: 3.5 examples: - - "GITHUB_KEY=ghp_XIxB7KMNdAr3zqWtQqhE94qglHqOzn1D1stg" - - "let g:gh_token='ghp_4U3LSowpDx8XvYE7A8GH56oxU5aWnY2mzIbV'" - - | - ## git developer settings - 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 - - "oauth_token: gho_fq75OMU7UVbS9pTZmoCCzJT6TM5d1w099FgG" - "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" references: - https://docs.github.com/en/rest/users?apiVersion=2022-11-28 @@ -52,8 +35,49 @@ rules: type: WordMatch words: - '"login"' - - name: GitHub OAuth Access Token + - name: GitHub Personal Access Token id: kingfisher.github.2 + pattern: | + (?xi) + \b + ( + ghp_ + [A-Z0-9]{36} + ) + pattern_requirements: + min_digits: 2 + min_lowercase: 2 + min_entropy: 3.5 + examples: + - "GITHUB_KEY=ghp_XIxB7KMNdAr3zqWtQqhE94qglHqOzn1D1stg" + - "let g:gh_token='ghp_4U3LSowpDx8XvYE7A8GH56oxU5aWnY2mzIbV'" + - | + ## git developer settings + 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 + references: + - https://docs.github.com/en/rest/users?apiVersion=2022-11-28 + validation: + type: Http + content: + request: + method: POST + url: https://api.github.com/graphql + headers: + Authorization: token {{ TOKEN }} + Accept: application/vnd.github+json + Content-Type: application/json + body: | + { + "query": "{ viewer { login } }" + } + response_matcher: + - report_response: true + - match_all_words: true + type: WordMatch + words: + - '"login"' + - name: GitHub OAuth Access Token + id: kingfisher.github.3 pattern: | (?xi) \b @@ -90,15 +114,49 @@ rules: type: WordMatch words: - '"login"' - - name: GitHub App Token - id: kingfisher.github.3 + - name: GitHub App User-to-Server Token + id: kingfisher.github.4 pattern: | (?xi) ( - (?:ghu|ghs)_[A-Z0-9]{36} + ghu_[A-Z0-9]{36} ) examples: - ' "token": "ghu_16C7e42F292c69C2E7C10c838347Ae178B4a",' + - | + Example usage: + git clone http://ghu_RguXIkihJjwHAP6eXEYxaPNvywurTr5IOAbg@github.com/username/repo.git + references: + - https://docs.github.com/en/rest/users?apiVersion=2022-11-28 + validation: + type: Http + content: + request: + method: POST + url: https://api.github.com/graphql + headers: + Authorization: token {{ TOKEN }} + Accept: application/vnd.github+json + Content-Type: application/json + body: | + { + "query": "{ viewer { login } }" + } + response_matcher: + - report_response: true + - match_all_words: true + type: WordMatch + words: + - '"login"' + - name: GitHub App Server-to-Server Token + id: kingfisher.github.5 + pattern: | + (?xi) + ( + ghs_[A-Z0-9]{36} + ) + examples: + - ' "token": "ghs_16C7e42F292c69C2E7C10c838347Ae178B4a",' - | Example usage: git clone http://ghs_RguXIkihJjwHAP6eXEYxaPNvywurTr5IOAbg@github.com/username/repo.git @@ -125,7 +183,7 @@ rules: words: - '"login"' - name: GitHub Refresh Token - id: kingfisher.github.4 + id: kingfisher.github.6 pattern: | (?xi) ( @@ -156,7 +214,7 @@ rules: words: - '"login"' - name: GitHub Client ID - id: kingfisher.github.5 + id: kingfisher.github.7 pattern: | (?xi) (?:github) @@ -172,7 +230,7 @@ rules: GITHUB_CLIENT_ID=ac58d6da7d7a84c039b7 GITHUB_SECRET=37d02377a3e9d849e18704c3ec883f9c5787d857 - name: GitHub Secret Key - id: kingfisher.github.6 + id: kingfisher.github.8 pattern: | (?xi) github @@ -209,7 +267,7 @@ rules: GITHUB_CLIENT_ID=ac58d6da7d7a84c039b7 GITHUB_SECRET=37d02377a3e9d849e18704c3ec883f9c5787d857 - name: GitHub Personal Access Token (fine-grained permissions) - id: kingfisher.github.7 + id: kingfisher.github.9 pattern: | (?xi) ( diff --git a/data/rules/mongodb.yml b/data/rules/mongodb.yml index e70adec..28b4031 100644 --- a/data/rules/mongodb.yml +++ b/data/rules/mongodb.yml @@ -81,10 +81,6 @@ rules: mongodb(?:\+srv)?://[\S]{3,50}:(?:[\S]{3,88})@[-.%\w/:]+ ) \b - pattern_requirements: - ignore_if_contains: - - "@localhost" - - "@127.0.0.1" min_entropy: 3 examples: - client = mongoc_client_new ("mongodb+srv://someuser:hunter2@my-atlas-rd941.mongodb.net/test?retryWrites=true&w=majority"); diff --git a/data/rules/ngrok.yml b/data/rules/ngrok.yml index 951cfd5..be45f08 100644 --- a/data/rules/ngrok.yml +++ b/data/rules/ngrok.yml @@ -7,7 +7,11 @@ rules: (?:.|[\\n\r]){0,32}? (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) (?:.|[\n\r]){0,32}? - (?:[a-z0-9]{25,30}_\d[a-z0-9]{20}|(?:cr_|ak_)[a-z0-9]{25,30}) + ( + (?:[a-z0-9]{25,30}_\d[a-z0-9]{20} + | + (?:cr_|ak_)[a-z0-9]{25,30}) + ) \b min_entropy: 4 examples: diff --git a/data/rules/postgres.yml b/data/rules/postgres.yml index bf4d361..5cbb4ba 100644 --- a/data/rules/postgres.yml +++ b/data/rules/postgres.yml @@ -24,11 +24,6 @@ rules: (?: \d+ ) - pattern_requirements: - ignore_if_contains: - - "@localhost" - - "@127.0.0.1" - - "localhost:" min_entropy: 3.3 confidence: medium examples: