updated ci

CHANGELOG.md

@@ -3,6 +3,7 @@
 All notable changes to this project will be documented in this file.
 
 ## [v1.62.0]
+- This release is focused on further improving detection accuracy, before even attempting to validate findings.
 - Added `pattern_requirements` checks to rules, providing lightweight post-regex character-class validation without lookarounds.
 - Added an `ignore_if_contains` option to `PatternRequirements` to drop matches containing case-insensitive placeholder words, with tests covering the new behavior.
 - Updated rules to adopt the new `pattern_requirements` support.

README.md

@@ -67,6 +67,7 @@ See ([docs/COMPARISON.md](docs/COMPARISON.md))
     - [ Run Kingfisher in Docker](#-run-kingfisher-in-docker)
 - [🔐 Detection Rules at a Glance](#-detection-rules-at-a-glance)
   - [📝 Write Custom Rules!](#-write-custom-rules)
+    - [Pattern requirements and placeholder filtering](#pattern-requirements-and-placeholder-filtering)
 - [🎉 Usage](#-usage)
   - [Basic Examples](#basic-examples)
     - [Scan with secret validation](#scan-with-secret-validation)
@@ -331,8 +332,7 @@ is independent:
 - `min_digits`, `min_uppercase`, `min_lowercase`, and `min_special_chars` enforce complexity thresholds.
 - `special_chars` lets you override the set of characters counted as "special" when `min_special_chars` is used.
 - `ignore_if_contains` lists case-insensitive substrings that should cause a match to be discarded (for example, to drop
-  `test`, `demo`, or `localhost` values). Kingfisher still accepts the legacy `exclude_words` key as an alias when loading
-  existing rule files.
+  `test`, `demo`, or `localhost` values).
 
 When a match is skipped because of `ignore_if_contains`, Kingfisher logs the event at the `DEBUG` level alongside the rule that
 was evaluated. If you need to keep those matches for a particular scan, pass `--no-ignore-if-contains` to `kingfisher scan` to

data/rules/github.yml

@@ -1,34 +1,17 @@
 rules:
-  - name: GitHub Personal Access Token
+  - name: GitHub Personal Access Token - fine-grained permissions
     id: kingfisher.github.1
     pattern: |
-      (?xi)                     
-      \b                       
+      (?xi)                 
       (                        
-        (?:                     # for token prefixes
-          ghp|                 # Personal Access Token
-          gho|                 # OAuth Token
-          ghu|                 # GitHub App User-to-Server Token
-          ghs|                 # GitHub App Server-to-Server Token
-          ghr|                 # Refresh Token
-          github_pat           # Alternative format for Personal Access Token
-        )_                     
-        (?:                     # for token body
-          [a-z0-9_]{35,235}    # 35 to 235 lowercase alphanumeric characters or underscores
-        )
-      )                        
-      \b
+        github_pat
+        [A-Z0-9]{80,84}
+      )
     pattern_requirements:
       min_digits: 2
       min_lowercase: 2
     min_entropy: 3.5
     examples:
-      - "GITHUB_KEY=ghp_XIxB7KMNdAr3zqWtQqhE94qglHqOzn1D1stg"
-      - "let g:gh_token='ghp_4U3LSowpDx8XvYE7A8GH56oxU5aWnY2mzIbV'"
-      - |
-        ## git developer settings
-        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
-      - "oauth_token: gho_fq75OMU7UVbS9pTZmoCCzJT6TM5d1w099FgG"
       - "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"
     references:
       - https://docs.github.com/en/rest/users?apiVersion=2022-11-28
@@ -52,8 +35,49 @@ rules:
               type: WordMatch
               words:
                 - '"login"'
-  - name: GitHub OAuth Access Token
+  - name: GitHub Personal Access Token
     id: kingfisher.github.2
+    pattern: |
+      (?xi)                     
+      \b                       
+      (                        
+        ghp_
+        [A-Z0-9]{36}
+      )
+    pattern_requirements:
+      min_digits: 2
+      min_lowercase: 2
+    min_entropy: 3.5
+    examples:
+      - "GITHUB_KEY=ghp_XIxB7KMNdAr3zqWtQqhE94qglHqOzn1D1stg"
+      - "let g:gh_token='ghp_4U3LSowpDx8XvYE7A8GH56oxU5aWnY2mzIbV'"
+      - |
+        ## git developer settings
+        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
+    references:
+      - https://docs.github.com/en/rest/users?apiVersion=2022-11-28
+    validation:
+      type: Http
+      content:
+        request:
+          method: POST
+          url: https://api.github.com/graphql
+          headers:
+            Authorization: token {{ TOKEN }}
+            Accept: application/vnd.github+json
+            Content-Type: application/json
+          body: |
+            {
+              "query": "{ viewer { login } }"
+            }
+          response_matcher:
+            - report_response: true
+            - match_all_words: true
+              type: WordMatch
+              words:
+                - '"login"'
+  - name: GitHub OAuth Access Token
+    id: kingfisher.github.3
     pattern: |
       (?xi)                     
       \b                       
@@ -90,15 +114,49 @@ rules:
               type: WordMatch
               words:
                 - '"login"'
-  - name: GitHub App Token
-    id: kingfisher.github.3
+  - name: GitHub App User-to-Server Token
+    id: kingfisher.github.4
     pattern: |
       (?xi)
       (
-        (?:ghu|ghs)_[A-Z0-9]{36}
+        ghu_[A-Z0-9]{36}
       )
     examples:
       - '            "token": "ghu_16C7e42F292c69C2E7C10c838347Ae178B4a",'
+      - |
+        Example usage:
+        git clone http://ghu_RguXIkihJjwHAP6eXEYxaPNvywurTr5IOAbg@github.com/username/repo.git
+    references:
+      - https://docs.github.com/en/rest/users?apiVersion=2022-11-28
+    validation:
+      type: Http
+      content:
+        request:
+          method: POST
+          url: https://api.github.com/graphql
+          headers:
+            Authorization: token {{ TOKEN }}
+            Accept: application/vnd.github+json
+            Content-Type: application/json
+          body: |
+            {
+              "query": "{ viewer { login } }"
+            }
+          response_matcher:
+            - report_response: true
+            - match_all_words: true
+              type: WordMatch
+              words:
+                - '"login"'
+  - name: GitHub App Server-to-Server Token
+    id: kingfisher.github.5
+    pattern: |
+      (?xi)
+      (
+        ghs_[A-Z0-9]{36}
+      )
+    examples:
+      - '            "token": "ghs_16C7e42F292c69C2E7C10c838347Ae178B4a",'
       - |
         Example usage:
         git clone http://ghs_RguXIkihJjwHAP6eXEYxaPNvywurTr5IOAbg@github.com/username/repo.git
@@ -125,7 +183,7 @@ rules:
               words:
                 - '"login"'
   - name: GitHub Refresh Token
-    id: kingfisher.github.4
+    id: kingfisher.github.6
     pattern: |
       (?xi)
       (
@@ -156,7 +214,7 @@ rules:
               words:
                 - '"login"'
   - name: GitHub Client ID
-    id: kingfisher.github.5
+    id: kingfisher.github.7
     pattern: |
       (?xi)
       (?:github)
@@ -172,7 +230,7 @@ rules:
         GITHUB_CLIENT_ID=ac58d6da7d7a84c039b7
         GITHUB_SECRET=37d02377a3e9d849e18704c3ec883f9c5787d857
   - name: GitHub Secret Key
-    id: kingfisher.github.6
+    id: kingfisher.github.8
     pattern: |
       (?xi)
       github
@@ -209,7 +267,7 @@ rules:
         GITHUB_CLIENT_ID=ac58d6da7d7a84c039b7
         GITHUB_SECRET=37d02377a3e9d849e18704c3ec883f9c5787d857
   - name: GitHub Personal Access Token (fine-grained permissions)
-    id: kingfisher.github.7
+    id: kingfisher.github.9
     pattern: |
       (?xi)
       (

data/rules/mongodb.yml

@@ -81,10 +81,6 @@ rules:
         mongodb(?:\+srv)?://[\S]{3,50}:(?:[\S]{3,88})@[-.%\w/:]+
       )
       \b
-    pattern_requirements:
-      ignore_if_contains:
-        - "@localhost"
-        - "@127.0.0.1"
     min_entropy: 3
     examples:
       - client = mongoc_client_new ("mongodb+srv://someuser:hunter2@my-atlas-rd941.mongodb.net/test?retryWrites=true&w=majority");

data/rules/ngrok.yml

@@ -7,7 +7,11 @@ rules:
       (?:.|[\\n\r]){0,32}?
       (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN)
       (?:.|[\n\r]){0,32}?
-      (?:[a-z0-9]{25,30}_\d[a-z0-9]{20}|(?:cr_|ak_)[a-z0-9]{25,30})
+      (
+        (?:[a-z0-9]{25,30}_\d[a-z0-9]{20}
+        |
+        (?:cr_|ak_)[a-z0-9]{25,30})
+      )
       \b
     min_entropy: 4
     examples:

data/rules/postgres.yml

@@ -24,11 +24,6 @@ rules:
       (?:
         \d+
       )
-    pattern_requirements:
-      ignore_if_contains:
-        - "@localhost"
-        - "@127.0.0.1"
-        - "localhost:"
     min_entropy: 3.3
     confidence: medium
     examples: