From 2265d2b1f06ae4b3ffe95ac3acb9303ed7eb189b Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sat, 28 Mar 2026 22:39:31 -0700 Subject: [PATCH 01/18] added rules --- NOTICE | 6 + .../data/rules/azure-notification-hub.yml | 159 ++++++++++++++++++ .../kingfisher-rules/data/rules/firebase.yml | 58 +++++++ .../kingfisher-rules/data/rules/helpscout.yml | 82 +++++++++ .../data/rules/kubernetes.yml | 118 +++++++++++++ crates/kingfisher-rules/data/rules/zapier.yml | 30 ++++ .../kingfisher-rules/data/rules/zendesk.yml | 102 +++++++++++ 7 files changed, 555 insertions(+) create mode 100644 crates/kingfisher-rules/data/rules/azure-notification-hub.yml create mode 100644 crates/kingfisher-rules/data/rules/firebase.yml create mode 100644 crates/kingfisher-rules/data/rules/helpscout.yml create mode 100644 crates/kingfisher-rules/data/rules/kubernetes.yml create mode 100644 crates/kingfisher-rules/data/rules/zapier.yml create mode 100644 crates/kingfisher-rules/data/rules/zendesk.yml diff --git a/NOTICE b/NOTICE index b95e1b1..6116057 100644 --- a/NOTICE +++ b/NOTICE @@ -47,10 +47,16 @@ Certain detection rules: * crates/kingfisher-rules/data/rules/instagram.yml * crates/kingfisher-rules/data/rules/iterable.yml * crates/kingfisher-rules/data/rules/lokalise.yml + * crates/kingfisher-rules/data/rules/azure-notification-hub.yml + * crates/kingfisher-rules/data/rules/firebase.yml + * crates/kingfisher-rules/data/rules/helpscout.yml + * crates/kingfisher-rules/data/rules/kubernetes.yml * crates/kingfisher-rules/data/rules/pendo.yml * crates/kingfisher-rules/data/rules/razorpay.yml * crates/kingfisher-rules/data/rules/spotify.yml * crates/kingfisher-rules/data/rules/wakatime.yml + * crates/kingfisher-rules/data/rules/zapier.yml + * crates/kingfisher-rules/data/rules/zendesk.yml are derived in part from Titus (https://github.com/praetorian-inc/titus), which is licensed under the Apache License, Version 2.0. diff --git a/crates/kingfisher-rules/data/rules/azure-notification-hub.yml b/crates/kingfisher-rules/data/rules/azure-notification-hub.yml new file mode 100644 index 0000000..5bd443e --- /dev/null +++ b/crates/kingfisher-rules/data/rules/azure-notification-hub.yml @@ -0,0 +1,159 @@ +rules: + - name: Azure Notification Hub Namespace Host + id: kingfisher.azure.notificationhub.1 + pattern: | + (?xi) + \b + (?: + endpoint + \s*=\s* + sb:// + | + notification + (?:.|[\n\r]){0,48}? + https:// + ) + ( + [a-z0-9] + [a-z0-9-]{1,62} + \.servicebus\.windows\.net + ) + (?:/|;|\b) + min_entropy: 2.0 + confidence: medium + visible: false + examples: + - Endpoint=sb://acme-push.servicebus.windows.net/;SharedAccessKeyName=DefaultListenSharedAccessSignature;SharedAccessKey=VGhpcytpcythK3Rlc3Qra2V5K3ZhbHVlLzEyMzQ1Njc4OTA= + - 'notificationHubEndpoint: "https://mobile-prod.servicebus.windows.net"' + references: + - https://learn.microsoft.com/en-us/rest/api/notificationhubs/use-rest-api-backend + + - name: Azure Notification Hub Name + id: kingfisher.azure.notificationhub.2 + pattern: | + (?xi) + \b + (?: + notification + (?:hub)? + (?:name|path) + | + hub + (?:name|path) + ) + \s*[:=]\s* + ["']? + ( + [A-Za-z0-9] + [A-Za-z0-9._-]{1,127} + ) + ["']? + \b + min_entropy: 2.0 + confidence: medium + visible: false + examples: + - NotificationHubPath=my-mobile-hub + - 'notificationHubName: "android-prod"' + references: + - https://learn.microsoft.com/en-us/azure/notification-hubs/create-notification-hub-portal + + - name: Azure Notification Hub SAS Key Name + id: kingfisher.azure.notificationhub.3 + pattern: | + (?xi) + \b + SharedAccessKeyName + \s*[:=]\s* + ["']? + ( + [A-Za-z] + [A-Za-z0-9_-]{2,63} + ) + ["']? + \b + min_entropy: 2.0 + confidence: medium + visible: false + examples: + - SharedAccessKeyName=DefaultListenSharedAccessSignature + - '"SharedAccessKeyName": "DefaultFullSharedAccessSignature"' + references: + - https://learn.microsoft.com/en-us/azure/notification-hubs/notification-hubs-push-notification-security + + - name: Azure Notification Hub Access Key + id: kingfisher.azure.notificationhub.4 + pattern: | + (?xi) + (?: + (?:notification\s*hub|Endpoint\s*=\s*sb://[a-z0-9-]{2,63}\.servicebus\.windows\.net/?) + (?:.|[\n\r]){0,160}? + SharedAccessKey + \s*[:=]\s* + ["']? + ( + [A-Za-z0-9+/]{32,88}={0,2} + ) + | + \b + (?:hubAccessKey|notificationHub(?:Access)?Key) + \b + \s*[:=]\s* + ["']? + ( + [A-Za-z0-9+/]{32,88}={0,2} + ) + ) + ["']? + (?:[^A-Za-z0-9+/=]|$) + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 + ignore_if_contains: + - example + - sample + - document + - placeholder + min_entropy: 3.7 + confidence: medium + examples: + - Endpoint=sb://acme-push.servicebus.windows.net/;SharedAccessKeyName=DefaultListenSharedAccessSignature;SharedAccessKey=Q29udG9zb1Rlc3RLZXkrMTIzNDU2Nzg5MC9BQkNERUZHSEk= + - | + const config = { + notificationHubName: "android-prod", + hubAccessKey: "U2FmZUtleVZhbHVlKzEyMzQ1Njc4OTBBQkNERUYrLz09" + }; + references: + - https://learn.microsoft.com/en-us/azure/notification-hubs/notification-hubs-push-notification-security + - https://learn.microsoft.com/en-us/rest/api/notificationhubs/use-rest-api-backend + depends_on_rule: + - rule_id: kingfisher.azure.notificationhub.1 + variable: NH_HOST + - rule_id: kingfisher.azure.notificationhub.2 + variable: NH_HUB + - rule_id: kingfisher.azure.notificationhub.3 + variable: NH_KEY_NAME + validation: + type: Http + content: + request: + method: GET + url: 'https://{{ NH_HOST }}/{{ NH_HUB }}/registrations/?api-version=2015-01' + headers: + Accept: application/atom+xml + Authorization: | + {%- assign uri = "https://" | append: NH_HOST | append: "/" | append: NH_HUB | append: "/registrations/?api-version=2015-01" -%} + {%- assign se = "" | unix_timestamp | plus: 300 -%} + {%- capture to_sign -%}{{ uri | url_encode }} + {{ se }}{%- endcapture -%} + {%- capture auth -%}SharedAccessSignature sr={{ uri | url_encode }}&sig={{ to_sign | hmac_sha256: TOKEN | url_encode }}&se={{ se }}&skn={{ NH_KEY_NAME | url_encode }}{%- endcapture -%} + {{ auth | strip_newlines }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: XmlValid + - type: WordMatch + words: + - "[a-z0-9]{6}) + (?:.|[\n\r]){0,24}? + \btoken-secret\b + \s*:\s* + (?P[a-z0-9]{16}) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 2.8 + confidence: medium + examples: + - | + token-id: 07402b + token-secret: f395accd245ae53d + references: + - https://kubernetes.io/docs/reference/access-authn-authz/bootstrap-tokens/ + depends_on_rule: + - rule_id: kingfisher.kubernetes.1 + variable: KUBE_API_SERVER + validation: + type: Http + content: + request: + method: GET + url: '{{ KUBE_API_SERVER }}/api/v1/namespaces?limit=1' + headers: + Accept: application/json + Authorization: 'Bearer {{ TOKEN_ID }}.{{ TOKEN_SECRET }}' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200, 403] + - type: JsonValid diff --git a/crates/kingfisher-rules/data/rules/zapier.yml b/crates/kingfisher-rules/data/rules/zapier.yml new file mode 100644 index 0000000..fc84326 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/zapier.yml @@ -0,0 +1,30 @@ +rules: + - name: Zapier Webhook URL + id: kingfisher.zapier.1 + pattern: | + (?x) + \b + ( + https://hooks\.zapier\.com/hooks/catch/ + [0-9]{5,10} + / + [a-z0-9]{5,12} + /? + ) + min_entropy: 3.4 + confidence: medium + examples: + - ZAPIER_WEBHOOK=https://hooks.zapier.com/hooks/catch/11595998/3ouwv7m/ + - webhook_url="https://hooks.zapier.com/hooks/catch/2929690/ztd17n/" + references: + - https://help.zapier.com/hc/en-us/articles/8496288690317-Trigger-Zaps-from-webhooks + validation: + type: Http + content: + request: + method: GET + url: '{{ TOKEN }}' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] diff --git a/crates/kingfisher-rules/data/rules/zendesk.yml b/crates/kingfisher-rules/data/rules/zendesk.yml new file mode 100644 index 0000000..473576d --- /dev/null +++ b/crates/kingfisher-rules/data/rules/zendesk.yml @@ -0,0 +1,102 @@ +rules: + - name: Zendesk Subdomain + id: kingfisher.zendesk.1 + pattern: | + (?xi) + \b + ( + [a-z0-9] + [a-z0-9-]{1,62} + \.zendesk\.com + ) + \b + min_entropy: 2.0 + confidence: medium + visible: false + examples: + - acme-support.zendesk.com + - helpdesk-prod.zendesk.com + references: + - https://developer.zendesk.com/api-reference/introduction/doc-conventions/ + + - name: Zendesk Account Email + id: kingfisher.zendesk.2 + pattern: | + (?xi) + \b + (?:zendesk|zd) + (?:.|[\n\r]){0,32}? + (?:email|user(?:name)?) + (?:.|[\n\r]){0,12}? + ( + [A-Za-z0-9._%+\-]+ + @ + [A-Za-z0-9.\-]+\.[A-Za-z]{2,} + ) + \b + min_entropy: 2.0 + confidence: medium + visible: false + examples: + - ZENDESK_EMAIL=agent@example.com + - 'zendesk_user: "support.bot@example.org"' + references: + - https://developer.zendesk.com/api-reference/introduction/security-and-auth/ + + - name: Zendesk API Token + id: kingfisher.zendesk.3 + pattern: | + (?xi) + \b + (?:zendesk|zd) + (?:.|[\n\r]){0,48}? + (?: + api[\s_.-]*token + | + token + | + api[\s_.-]*key + ) + (?:.|[\n\r]){0,16}? + ( + [A-Za-z0-9]{40} + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 + ignore_if_contains: + - example + - placeholder + - yourtoken + min_entropy: 3.8 + confidence: medium + examples: + - ZENDESK_API_TOKEN=a3B8f29E4d1C6a0578e23D9f41b6C8e2qR7tY4uI + - zendesk_token="E7d2A1f849c3B05d6e81F2a794c3D5b0pQ8wX1zK" + references: + - https://developer.zendesk.com/api-reference/introduction/security-and-auth/ + - https://developer.zendesk.com/api-reference/ticketing/account-configuration/current_user/ + depends_on_rule: + - rule_id: kingfisher.zendesk.1 + variable: ZENDESK_SUBDOMAIN + - rule_id: kingfisher.zendesk.2 + variable: ZENDESK_EMAIL + validation: + type: Http + content: + request: + method: GET + url: 'https://{{ ZENDESK_SUBDOMAIN }}/api/v2/users/me.json' + headers: + Accept: application/json + Authorization: 'Basic {{ ZENDESK_EMAIL | append: "/token:" | append: TOKEN | b64enc }}' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + words: + - '"user"' From a28250be190f18f43dd4417a914dbaff7d0294ef Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 29 Mar 2026 00:03:58 -0700 Subject: [PATCH 02/18] added more rules --- NOTICE | 35 ++++++++ .../data/rules/octopusdeploy.yml | 66 ++++++++++++++ .../kingfisher-rules/data/rules/openshift.yml | 78 +++++++++++++++++ .../data/rules/polymarket.yml | 85 +++++++++++++++++++ .../kingfisher-rules/data/rules/privateai.yml | 58 +++++++++++++ .../kingfisher-rules/data/rules/privkey.yml | 19 +++++ .../data/rules/settlemint.yml | 63 ++++++++++++++ .../kingfisher-rules/data/rules/sidekiq.yml | 58 +++++++++++++ 8 files changed, 462 insertions(+) create mode 100644 crates/kingfisher-rules/data/rules/octopusdeploy.yml create mode 100644 crates/kingfisher-rules/data/rules/openshift.yml create mode 100644 crates/kingfisher-rules/data/rules/polymarket.yml create mode 100644 crates/kingfisher-rules/data/rules/privateai.yml create mode 100644 crates/kingfisher-rules/data/rules/settlemint.yml create mode 100644 crates/kingfisher-rules/data/rules/sidekiq.yml diff --git a/NOTICE b/NOTICE index 6116057..f47f0bb 100644 --- a/NOTICE +++ b/NOTICE @@ -78,3 +78,38 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +-------------------------------------------------------------------- + +Certain detection rules: + * crates/kingfisher-rules/data/rules/octopusdeploy.yml + * crates/kingfisher-rules/data/rules/openshift.yml + * crates/kingfisher-rules/data/rules/polymarket.yml + * crates/kingfisher-rules/data/rules/privateai.yml + * crates/kingfisher-rules/data/rules/settlemint.yml + * crates/kingfisher-rules/data/rules/sidekiq.yml + +are derived in part from Betterleaks +(https://github.com/betterleaks/betterleaks), which is licensed under the MIT +License. + +Betterleaks +Copyright (c) 2026 Zachary Rice + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/kingfisher-rules/data/rules/octopusdeploy.yml b/crates/kingfisher-rules/data/rules/octopusdeploy.yml new file mode 100644 index 0000000..b834bdb --- /dev/null +++ b/crates/kingfisher-rules/data/rules/octopusdeploy.yml @@ -0,0 +1,66 @@ +rules: + - name: Octopus Deploy Server URL + id: kingfisher.octopusdeploy.1 + pattern: | + (?xi) + (?: + \boctopus(?:[_\s.-]?deploy)?(?:[_\s.-]?(?:url|server|host))?\b + (?:.|[\n\r]){0,32}? + [:=] + | + \boctopus(?:url|server|host)\b + \s*[:=] + ) + \s*["']? + ( + https:// + [A-Za-z0-9.-]+ + (?::\d{2,5})? + ) + ["']? + min_entropy: 2.3 + confidence: medium + visible: false + examples: + - OCTOPUS_URL=https://deploy.acme.example + - 'octopus_server: "https://octopus.internal.example:8443"' + references: + - https://octopus.com/docs/octopus-rest-api/getting-started + + - name: Octopus Deploy API Key + id: kingfisher.octopusdeploy.2 + pattern: | + (?x) + \b + ( + API-[A-Z0-9]{26} + ) + \b + pattern_requirements: + min_digits: 4 + min_uppercase: 4 + min_entropy: 3.4 + confidence: medium + examples: + - OCTOPUS_API_KEY=API-ZNRMR7SL6L3ATMOIK7GKJDKLPY + - 'set apikey="API-A1B2C3D4E5F6G7H8J9K0LMNOPQ"' + references: + - https://octopus.com/docs/octopus-rest-api/getting-started + - https://octopus.com/docs/octopus-rest-api/how-to-create-an-api-key + depends_on_rule: + - rule_id: kingfisher.octopusdeploy.1 + variable: OCTOPUS_URL + validation: + type: Http + content: + request: + method: GET + url: '{{ OCTOPUS_URL }}/api' + headers: + X-Octopus-ApiKey: '{{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid diff --git a/crates/kingfisher-rules/data/rules/openshift.yml b/crates/kingfisher-rules/data/rules/openshift.yml new file mode 100644 index 0000000..442c27b --- /dev/null +++ b/crates/kingfisher-rules/data/rules/openshift.yml @@ -0,0 +1,78 @@ +rules: + - name: OpenShift API Server URL + id: kingfisher.openshift.1 + pattern: | + (?xi) + (?: + \boc\s+login\b + (?:.|[\n\r]){0,128}? + --server= + | + \bopenshift\b + (?:.|[\n\r]){0,32}? + \b(?:server|api(?:[_-]?server)?|cluster(?:[_-]?url)?)\b + \s*[:=]\s* + ) + ["']? + ( + https:// + (?: + \[[0-9a-f:.]+\] + | + [a-z0-9] + [a-z0-9.-]{1,253} + ) + (?::\d{2,5})? + ) + ["']? + min_entropy: 2.0 + confidence: medium + visible: false + examples: + - oc login --token=sha256~kV46hPnEYhCWFnB85r5NrprAxggzgb6GOeLbgcKNsH0 --server=https://api.cluster.example.com:6443 + - OPENSHIFT_SERVER=https://api.dev-cluster.example.net:6443 + references: + - https://docs.redhat.com/en/documentation/openshift_container_platform/4.9/html-single/authentication_and_authorization/index + - https://docs.redhat.com/en/documentation/openshift_container_platform/4.17/html/user_and_group_apis/user-user-openshift-io-v1 + + - name: OpenShift OAuth Access Token + id: kingfisher.openshift.2 + pattern: | + (?x) + \b + ( + sha256~[A-Za-z0-9_-]{43} + ) + \b + pattern_requirements: + min_digits: 3 + min_uppercase: 1 + min_lowercase: 3 + ignore_if_contains: + - put_your_token_here + - xxxxxx + min_entropy: 3.8 + confidence: medium + examples: + - 'Authorization: Bearer sha256~kV46hPnEYhCWFnB85r5NrprAxggzgb6GOeLbgcKNsH0' + - oc login --token=sha256~ZBMKw9VAayhdnyANaHvjJeXDiGwA7Fsr5gtLKj3-eh- --server=https://api.cluster.example.com:6443 + references: + - https://docs.redhat.com/en/documentation/openshift_container_platform/4.17/html/oauth_apis/oauthaccesstoken-oauth-openshift-io-v1 + - https://docs.redhat.com/en/documentation/openshift_container_platform/4.9/html-single/authentication_and_authorization/index + depends_on_rule: + - rule_id: kingfisher.openshift.1 + variable: OPENSHIFT_API_SERVER + validation: + type: Http + content: + request: + method: GET + url: '{{ OPENSHIFT_API_SERVER }}/apis/user.openshift.io/v1/users' + headers: + Authorization: 'Bearer {{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200, 403] + - type: JsonValid diff --git a/crates/kingfisher-rules/data/rules/polymarket.yml b/crates/kingfisher-rules/data/rules/polymarket.yml new file mode 100644 index 0000000..fd32d71 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/polymarket.yml @@ -0,0 +1,85 @@ +rules: + - name: Polymarket Builder Secret + id: kingfisher.polymarket.1 + pattern: | + (?xi) + \b + poly(?:market)? + (?:.|[\n\r]){0,32}? + (?:builder|api)? + (?:.|[\n\r]){0,16}? + secret + (?:.|[\n\r]){0,12}? + ( + [A-Za-z0-9+/]{40,88}={0,2} + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 8 + min_entropy: 3.6 + confidence: medium + visible: false + examples: + - POLY_BUILDER_SECRET=QmFzZTY0U2VjcmV0VGVzdEtleTEyMzQ1Njc4OTBBQkNERUY= + - 'polymarket_builder_secret: Q29tcGxleFNlY3JldE1hdGVyaWFsMTIzNDU2Nzg5MDEyMzQ=' + references: + - https://docs.polymarket.com/trading/orders/attribution + + - name: Polymarket Builder Passphrase + id: kingfisher.polymarket.2 + pattern: | + (?xi) + \b + poly(?:market)? + (?:.|[\n\r]){0,32}? + (?:builder|api)? + (?:.|[\n\r]){0,16}? + passphrase + (?:.|[\n\r]){0,12}? + ( + [A-Za-z0-9_]{8,128} + ) + \b + pattern_requirements: + min_digits: 1 + min_uppercase: 1 + min_lowercase: 4 + ignore_if_contains: + - example + - placeholder + min_entropy: 3.0 + confidence: medium + visible: false + examples: + - POLY_BUILDER_PASSPHRASE=BuilderPass_2026 + - 'polymarket_passphrase: AlphaPass_7788' + references: + - https://docs.polymarket.com/trading/orders/attribution + + - name: Polymarket Builder API Key + id: kingfisher.polymarket.3 + pattern: | + (?xi) + \b + poly(?:market)? + (?:.|[\n\r]){0,32}? + (?:builder|api)? + (?:.|[\n\r]){0,16}? + key + (?:.|[\n\r]){0,12}? + ( + [a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12} + ) + \b + pattern_requirements: + min_digits: 6 + min_lowercase: 8 + min_entropy: 3.2 + confidence: medium + examples: + - POLY_BUILDER_API_KEY=12345678-abcd-1234-efab-1234567890ab + - 'polymarket_api_key: a1b2c3d4-e5f6-789a-bcde-f0123456789a' + references: + - https://docs.polymarket.com/trading/orders/attribution diff --git a/crates/kingfisher-rules/data/rules/privateai.yml b/crates/kingfisher-rules/data/rules/privateai.yml new file mode 100644 index 0000000..ce7d5d6 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/privateai.yml @@ -0,0 +1,58 @@ +rules: + - name: Private AI API Key + id: kingfisher.privateai.1 + pattern: | + (?xi) + \b + (?: + private[_-]?ai + | + limina + ) + (?:.|[\n\r]){0,32}? + (?: + api[_-]?key + | + x-api-key + | + token + ) + (?:.|[\n\r]){0,12}? + ( + [a-z0-9]{32} + ) + \b + pattern_requirements: + min_digits: 4 + min_lowercase: 8 + ignore_if_contains: + - example + - placeholder + - insert + - your + min_entropy: 3.5 + confidence: medium + examples: + - PRIVATEAI_API_KEY=4fa2d7c81be9063d4ea8bc1f6d2a7e9c + - 'x-api-key: 2ab4d6e8f0c1a3b5d7e9f1a2b4c6d8e0' + references: + - https://docs.private-ai.com/fundamentals/getting-started + - https://docs.private-ai.com/reference/4.0.0/operation/ner_text_ner_text_post/ + validation: + type: Http + content: + request: + method: POST + url: https://api.private-ai.com/community/v4/process/text + headers: + Content-Type: application/json + x-api-key: '{{ TOKEN }}' + body: '{"text":["Hello Jane Doe"]}' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + words: + - '"processed_text"' diff --git a/crates/kingfisher-rules/data/rules/privkey.yml b/crates/kingfisher-rules/data/rules/privkey.yml index e6fa5e9..8407324 100644 --- a/crates/kingfisher-rules/data/rules/privkey.yml +++ b/crates/kingfisher-rules/data/rules/privkey.yml @@ -108,3 +108,22 @@ rules: -----END ENCRYPTED PRIVATE KEY BLOCK----- references: - https://www.rfc-editor.org/rfc/rfc7468 + + - name: PKCS#12 File Path + id: kingfisher.privkey.3 + pattern: | + (?xi) + (?: + ^|["'\s(=/] + ) + ( + (?:[^"' \t\r\n/]+/)*[^"' \t\r\n/]+\.(?:p12|pfx) + ) + (?: + $|["'\s),] + ) + min_entropy: 2.0 + confidence: medium + examples: + - security/es_certificates/opensearch/es_kibana_client.p12 + - ToDo/ToDo.UWP/ToDo.UWP_TemporaryKey.pfx diff --git a/crates/kingfisher-rules/data/rules/settlemint.yml b/crates/kingfisher-rules/data/rules/settlemint.yml new file mode 100644 index 0000000..2c38e29 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/settlemint.yml @@ -0,0 +1,63 @@ +rules: + - name: SettleMint Personal Access Token + id: kingfisher.settlemint.1 + pattern: | + (?x) + \b + ( + sm_pat_[A-Za-z0-9]{16} + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 4 + min_entropy: 3.2 + confidence: medium + examples: + - SETTLEMINT_ACCESS_TOKEN=sm_pat_A1b2C3d4E5f6G7h8 + - settlemint connect --pat=sm_pat_Z9y8X7w6V5u4T3s2 + references: + - https://console.settlemint.com/documentation/blockchain-platform/platform-components/security-and-authentication/personal-access-tokens + + - name: SettleMint Application Access Token + id: kingfisher.settlemint.2 + pattern: | + (?x) + \b + ( + sm_aat_[A-Za-z0-9]{16} + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 4 + min_entropy: 3.2 + confidence: medium + examples: + - BLOCKSCOUT_SETTLEMINT_APPLICATION_ACCESS_TOKEN=sm_aat_A1b2C3d4E5f6G7h8 + - 'x-auth-token: sm_aat_Z9y8X7w6V5u4T3s2' + references: + - https://console.settlemint.com/documentation/blockchain-platform/platform-components/security-and-authentication/application-access-tokens + + - name: SettleMint Service Access Token + id: kingfisher.settlemint.3 + pattern: | + (?x) + \b + ( + sm_sat_[A-Za-z0-9]{16} + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 4 + min_entropy: 3.2 + confidence: medium + examples: + - SETTLEMINT_SERVICE_TOKEN=sm_sat_A1b2C3d4E5f6G7h8 + - 'Authorization: Bearer sm_sat_Z9y8X7w6V5u4T3s2' + references: + - https://console.settlemint.com/documentation diff --git a/crates/kingfisher-rules/data/rules/sidekiq.yml b/crates/kingfisher-rules/data/rules/sidekiq.yml new file mode 100644 index 0000000..9e0eb2b --- /dev/null +++ b/crates/kingfisher-rules/data/rules/sidekiq.yml @@ -0,0 +1,58 @@ +rules: + - name: Sidekiq Enterprise Credential + id: kingfisher.sidekiq.1 + pattern: | + (?xi) + \b + (?: + BUNDLE_ENTERPRISE__CONTRIBSYS__COM + | + BUNDLE_GEMS__CONTRIBSYS__COM + ) + \s*[:=]\s* + ["']? + ( + [a-f0-9]{8}:[a-f0-9]{8} + ) + ["']? + \b + pattern_requirements: + min_digits: 4 + min_lowercase: 4 + min_entropy: 2.8 + confidence: medium + examples: + - BUNDLE_ENTERPRISE__CONTRIBSYS__COM=cafebabe:deadbeef + - 'export BUNDLE_GEMS__CONTRIBSYS__COM="cafeb4b3:d3adb33f"' + + - name: Sidekiq Sensitive URL + id: kingfisher.sidekiq.2 + pattern: | + (?xi) + ( + https?:// + [a-f0-9]{8}:[a-f0-9]{8} + @ + (?: + gems\.contribsys\.com + | + enterprise\.contribsys\.com + ) + (?: + /[^ \t\r\n"'<>]* + | + \?[^ \t\r\n"'<>]* + | + \#[^ \t\r\n"'<>]* + | + :[0-9]{1,5}(?:/[^ \t\r\n"'<>]*)? + )? + ) + pattern_requirements: + min_digits: 4 + min_lowercase: 4 + min_entropy: 2.8 + confidence: medium + examples: + - https://cafebabe:deadbeef@gems.contribsys.com/ + - http://cafeb4b3:d3adb33f@enterprise.contribsys.com:80/path?param1=true From b9da8e28299095e1613cb75b105e0b44d5878cd5 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 29 Mar 2026 08:19:34 -0700 Subject: [PATCH 03/18] added more rules --- CHANGELOG.md | 4 + Cargo.toml | 2 +- NOTICE | 36 +++++ .../data/rules/azure-notification-hub.yml | 2 + crates/kingfisher-rules/data/rules/etsy.yml | 51 +++++++ .../kingfisher-rules/data/rules/firebase.yml | 2 +- .../data/rules/flutterwave.yml | 43 ++++++ .../kingfisher-rules/data/rules/freemius.yml | 29 ++++ .../kingfisher-rules/data/rules/helpscout.yml | 2 +- crates/kingfisher-rules/data/rules/jfrog.yml | 129 ++++++++++++++++++ crates/kingfisher-rules/data/rules/kraken.yml | 31 +++++ crates/kingfisher-rules/data/rules/kucoin.yml | 62 +++++++++ .../kingfisher-rules/data/rules/openshift.yml | 6 +- .../kingfisher-rules/data/rules/privateai.yml | 2 +- .../kingfisher-rules/data/rules/sidekiq.yml | 8 +- crates/kingfisher-rules/data/rules/trello.yml | 31 +++++ 16 files changed, 429 insertions(+), 11 deletions(-) create mode 100644 crates/kingfisher-rules/data/rules/etsy.yml create mode 100644 crates/kingfisher-rules/data/rules/flutterwave.yml create mode 100644 crates/kingfisher-rules/data/rules/freemius.yml create mode 100644 crates/kingfisher-rules/data/rules/jfrog.yml create mode 100644 crates/kingfisher-rules/data/rules/kraken.yml create mode 100644 crates/kingfisher-rules/data/rules/kucoin.yml create mode 100644 crates/kingfisher-rules/data/rules/trello.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b4462f..6954865 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ All notable changes to this project will be documented in this file. +## [v1.92.0] +- Added new built-in rules for Etsy, Flutterwave, Freemius, JFrog, Kraken, KuCoin, and Trello, plus recent Betterleaks-derived additions including Octopus Deploy, OpenShift, Private AI, SettleMint, Sidekiq, and Polymarket. +- Added live HTTP validation for Etsy, JFrog, Octopus Deploy, OpenShift, and Private AI where provider documentation supported reliable token-only checks. + ## [v1.91.0] - Added SSRF protection for credential validation: outbound HTTP requests now block connections to loopback, private, link-local, and other non-public IP addresses. HTTP redirect targets are DNS-resolved and validated against the same SSRF rules. Use `--allow-internal-ips` to opt out when scanning internal infrastructure. - Consolidated JWT SSRF checks to use the shared `is_ssrf_safe_ip` function, covering additional reserved ranges (CGNAT, documentation, benchmarking, IPv6 unique-local). diff --git a/Cargo.toml b/Cargo.toml index 162bc34..f37bf5a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,7 @@ http = "1.4" [package] name = "kingfisher" -version = "1.91.0" +version = "1.92.0" description = "MongoDB's blazingly fast and accurate secret scanning and validation tool" edition.workspace = true rust-version.workspace = true diff --git a/NOTICE b/NOTICE index f47f0bb..d98a5b3 100644 --- a/NOTICE +++ b/NOTICE @@ -113,3 +113,39 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------- + +Certain detection rules: + * crates/kingfisher-rules/data/rules/adafruit.yml + * crates/kingfisher-rules/data/rules/etsy.yml + * crates/kingfisher-rules/data/rules/flutterwave.yml + * crates/kingfisher-rules/data/rules/freemius.yml + * crates/kingfisher-rules/data/rules/jfrog.yml + * crates/kingfisher-rules/data/rules/kraken.yml + * crates/kingfisher-rules/data/rules/kucoin.yml + * crates/kingfisher-rules/data/rules/trello.yml + +are derived in part from Gitleaks (https://github.com/gitleaks/gitleaks), +which is licensed under the MIT License. + +Gitleaks +Copyright (c) 2019 Zachary Rice + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/kingfisher-rules/data/rules/azure-notification-hub.yml b/crates/kingfisher-rules/data/rules/azure-notification-hub.yml index 5bd443e..c090b03 100644 --- a/crates/kingfisher-rules/data/rules/azure-notification-hub.yml +++ b/crates/kingfisher-rules/data/rules/azure-notification-hub.yml @@ -63,7 +63,9 @@ rules: pattern: | (?xi) \b + ["']? SharedAccessKeyName + ["']? \s*[:=]\s* ["']? ( diff --git a/crates/kingfisher-rules/data/rules/etsy.yml b/crates/kingfisher-rules/data/rules/etsy.yml new file mode 100644 index 0000000..7b89bca --- /dev/null +++ b/crates/kingfisher-rules/data/rules/etsy.yml @@ -0,0 +1,51 @@ +rules: + - name: Etsy Open API Key + id: kingfisher.etsy.1 + pattern: | + (?xi) + \b + (?: + etsy + (?:.|[\n\r]){0,32}? + (?: + api[_-]?key | + keystring | + x-api-key + ) + | + x-api-key + ) + (?:.|[\n\r]){0,12}? + ( + [a-z0-9]{24}:[A-Za-z0-9]{10,64} + ) + \b + pattern_requirements: + min_digits: 4 + min_lowercase: 8 + ignore_if_contains: + - your_api_key + - your_key_here + - placeholder + min_entropy: 3.4 + confidence: medium + examples: + - 'x-api-key: 1aa2bb33c44d55eeeeee6fff:a1b2c3d4e5' + - ETSY_API_KEY=1aa2bb33c44d55eeeeee6fff:a1b2c3d4e5 + references: + - https://developers.etsy.com/documentation/tutorials/quickstart/ + validation: + type: Http + content: + request: + method: GET + url: https://api.etsy.com/v3/application/openapi-ping + headers: + x-api-key: '{{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + diff --git a/crates/kingfisher-rules/data/rules/firebase.yml b/crates/kingfisher-rules/data/rules/firebase.yml index 952b392..1140506 100644 --- a/crates/kingfisher-rules/data/rules/firebase.yml +++ b/crates/kingfisher-rules/data/rules/firebase.yml @@ -52,7 +52,7 @@ rules: confidence: medium examples: - FCM_DEVICE_TOKEN=AbCdEfGhIjKlMnOpQrStUv:APA91bZaYxWvUtSrQpOnMlKjIhGfEdCbA9876543210ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-AaBbCcDdEeFfGgHhIiJj - - 'registrationToken: "ZyXwVuTsRqPoNmLkJiHgFe:APA91bAbCdEfGhIjKlMnOpQrStUvWxYz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-KkLlMmNnOoPpQqRrSs"' + - 'registrationToken: "AbCdEfGhIjKlMnOpQrStUv:APA91bZaYxWvUtSrQpOnMlKjIhGfEdCbA9876543210ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-AaBbCcDdEeFfGgHhIiJj"' references: - https://firebase.google.com/docs/cloud-messaging/manage-tokens # Registration tokens can’t be safely live-validated using only the token value. diff --git a/crates/kingfisher-rules/data/rules/flutterwave.yml b/crates/kingfisher-rules/data/rules/flutterwave.yml new file mode 100644 index 0000000..ae0819d --- /dev/null +++ b/crates/kingfisher-rules/data/rules/flutterwave.yml @@ -0,0 +1,43 @@ +rules: + - name: Flutterwave Public Key + id: kingfisher.flutterwave.1 + pattern: | + (?x) + \b + ( + FLWPUBK(?:_TEST)?-[a-f0-9]{32}-X + ) + \b + pattern_requirements: + min_digits: 4 + min_lowercase: 4 + min_entropy: 3.1 + confidence: medium + examples: + - FLW_PUBLIC_KEY=FLWPUBK_TEST-32193bba8dab84e3d9c4525c85ea7a12-X + - data-PBFPubKey="FLWPUBK_TEST-589490616a6297324231c5e89b58f3f6-X" + references: + - https://developer.flutterwave.com/docs/authentication + - https://developer.flutterwave.com/v2.0/docs/api-keys + + - name: Flutterwave Secret Key + id: kingfisher.flutterwave.2 + pattern: | + (?x) + \b + ( + FLWSECK(?:_TEST)?-[a-f0-9]{32}-X + ) + \b + pattern_requirements: + min_digits: 4 + min_lowercase: 8 + min_entropy: 3.3 + confidence: medium + examples: + - FLW_SECRET_KEY=FLWSECK_TEST-a514d8f1abd080db1502a144f22954dc-X + - 'Authorization: Bearer FLWSECK_TEST-5b1f0a33de9c41748c2a7e9b51d3c6af-X' + - seckey=FLWSECK-e6db11d1f8a6208de8cb2f94e293450e-X + references: + - https://developer.flutterwave.com/docs/authentication + - https://developer.flutterwave.com/v2.0/reference/api-request-and-response-standards diff --git a/crates/kingfisher-rules/data/rules/freemius.yml b/crates/kingfisher-rules/data/rules/freemius.yml new file mode 100644 index 0000000..766d627 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/freemius.yml @@ -0,0 +1,29 @@ +rules: + - name: Freemius Secret Key + id: kingfisher.freemius.1 + pattern: | + (?xi) + ["']secret_key["'] + \s*=>\s* + ["'] + ( + sk_[^"' \t\r\n]{29} + ) + ["'] + pattern_requirements: + min_digits: 2 + min_lowercase: 6 + min_special_chars: 2 + ignore_if_contains: + - xxxxxxxxx + - placeholder + min_entropy: 3.6 + confidence: medium + examples: + - | + $config = array( + "secret_key" => "sk_ubb4yN3mzqGR2x8#P7r5&@*xC$utE", + ); + references: + - https://freemius.com/help/documentation/wordpress-sdk/integrating-freemius-sdk/ + diff --git a/crates/kingfisher-rules/data/rules/helpscout.yml b/crates/kingfisher-rules/data/rules/helpscout.yml index e08a38c..6a63bb7 100644 --- a/crates/kingfisher-rules/data/rules/helpscout.yml +++ b/crates/kingfisher-rules/data/rules/helpscout.yml @@ -56,7 +56,7 @@ rules: confidence: medium examples: - HELPSCOUT_CLIENT_SECRET=a3B8f29E4d1C6a0578e23D9f41b6C8e2 - - 'helpscout_secret: "E7d2A1f849c3B05d6e81F2a794c3D5b0"' + - 'helpscout_client_secret: "E7d2A1f849c3B05d6e81F2a794c3D5b0"' references: - https://developer.helpscout.com/mailbox-api/ depends_on_rule: diff --git a/crates/kingfisher-rules/data/rules/jfrog.yml b/crates/kingfisher-rules/data/rules/jfrog.yml new file mode 100644 index 0000000..4285f49 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/jfrog.yml @@ -0,0 +1,129 @@ +rules: + - name: JFrog Cloud Host + id: kingfisher.jfrog.1 + pattern: | + (?xi) + \b + ( + [a-z0-9] + (?: + [a-z0-9\-]{0,61} + [a-z0-9] + )? + \.jfrog\.io + ) + \b + min_entropy: 2.5 + confidence: medium + visible: false + examples: + - company.jfrog.io + - my-team.jfrog.io + references: + - https://jfrog.com/help/api/khub/documents/xrOb4ANk_fqUw5nctnsIww/content + + - name: JFrog API Key + id: kingfisher.jfrog.2 + pattern: | + (?xi) + \b + (?: + jfrog | + artifactory | + bintray | + xray + ) + (?:.|[\n\r]){0,32}? + (?: + api[_-]?key | + password | + token | + secret + ) + (?:.|[\n\r]){0,12}? + ( + [A-Za-z0-9]{73} + ) + \b + pattern_requirements: + min_digits: 4 + min_uppercase: 2 + min_lowercase: 6 + min_entropy: 3.5 + confidence: medium + examples: + - jfrog_api_key=Ab12Cd34Ef56Gh78Ij90Kl12Mn34Op56Qr78St90Uv12Wx34Yz56Ab78Cd90Ef12Gh34Ij5Kl + - jfrog_password=Ab12Cd34Ef56Gh78Ij90Kl12Mn34Op56Qr78St90Uv12Wx34Yz56Ab78Cd90Ef12Gh34Ij5Kl + references: + - https://jfrog.com/help/api/khub/documents/xrOb4ANk_fqUw5nctnsIww/content + - https://jfrog.com/article/access-service/ + depends_on_rule: + - rule_id: kingfisher.jfrog.1 + variable: JFROG_HOST + validation: + type: Http + content: + request: + method: GET + url: https://{{ JFROG_HOST }}/artifactory/api/repositories + headers: + X-JFrog-Art-Api: '{{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + + - name: JFrog Identity Token + id: kingfisher.jfrog.3 + pattern: | + (?xi) + \b + (?: + jfrog | + artifactory | + bintray | + xray + ) + (?:.|[\n\r]){0,32}? + (?: + identity[_-]?token | + access[_-]?token | + bearer | + token + ) + (?:.|[\n\r]){0,12}? + ( + [A-Za-z0-9]{64} + ) + \b + pattern_requirements: + min_digits: 4 + min_uppercase: 2 + min_lowercase: 6 + min_entropy: 3.4 + confidence: medium + examples: + - jfrog_identity_token=Ab12Cd34Ef56Gh78Ij90Kl12Mn34Op56Qr78St90Uv12Wx34Yz56Ab78Cd90Ef12 + - artifactory_access_token=Zx12Cv34Bn56Mm78Aa90Ss12Dd34Ff56Gg78Hh90Jj12Kk34Ll56Qq78Ww90Ee12 + references: + - https://jfrog.com/help/api/khub/documents/xrOb4ANk_fqUw5nctnsIww/content + - https://jfrog.com/article/access-service/ + depends_on_rule: + - rule_id: kingfisher.jfrog.1 + variable: JFROG_HOST + validation: + type: Http + content: + request: + method: GET + url: https://{{ JFROG_HOST }}/artifactory/api/repositories + headers: + Authorization: 'Bearer {{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid diff --git a/crates/kingfisher-rules/data/rules/kraken.yml b/crates/kingfisher-rules/data/rules/kraken.yml new file mode 100644 index 0000000..bd3fc2e --- /dev/null +++ b/crates/kingfisher-rules/data/rules/kraken.yml @@ -0,0 +1,31 @@ +rules: + - name: Kraken API Secret + id: kingfisher.kraken.1 + pattern: | + (?xi) + \b + kraken + (?:.|[\n\r]){0,32}? + (?: + api[_-]?secret | + secret | + private[_-]?key | + token + ) + (?:.|[\n\r]){0,12}? + ( + [A-Za-z0-9+/=_-]{80,90} + ) + (?:[^A-Za-z0-9+/=_-]|$) + pattern_requirements: + min_digits: 4 + min_uppercase: 2 + min_lowercase: 8 + min_entropy: 4.0 + confidence: medium + examples: + - KRAKEN_API_SECRET=dGhpcy1sb29rcy1saWtlLWEtYmFzZTY0LWtyYWtlbi1zZWNyZXQtdGhhdC1pcy1sb25nLWVub3VnaA== + - kraken_secret="Aq1Bq2Cr3Ds4Et5Fu6Gv7Hw8Ix9Jy0Kz1La2Mb3Nc4Od5Pe6Qf7Rg8Sh9Ti0Uj1Vk2Wm3Xn4Yo5Za6Bc7" + references: + - https://docs.kraken.com/api/docs/guides/spot-rest-auth/ + - https://docs.kraken.com/api/docs/rest-api/get-account-balance/ diff --git a/crates/kingfisher-rules/data/rules/kucoin.yml b/crates/kingfisher-rules/data/rules/kucoin.yml new file mode 100644 index 0000000..c2d92ed --- /dev/null +++ b/crates/kingfisher-rules/data/rules/kucoin.yml @@ -0,0 +1,62 @@ +rules: + - name: KuCoin API Key + id: kingfisher.kucoin.1 + pattern: | + (?xi) + \b + kucoin + (?:.|[\n\r]){0,32}? + (?: + api[_-]?key | + key + ) + (?:.|[\n\r]){0,12}? + ( + [a-f0-9]{24} + ) + \b + pattern_requirements: + min_digits: 4 + min_lowercase: 8 + ignore_if_contains: + - xxxxxx + - your_api_key + min_entropy: 3.0 + confidence: medium + examples: + - KUCOIN_API_KEY=4f4ecb6f11b1a70001c8e2ff + - 'kucoin_api_key: a1b2c3d4e5f60718293a4b5c' + references: + - https://www.kucoin.com/docs-new/authentication + - https://www.kucoin.com/docs-new/api-3470125 + + - name: KuCoin API Secret + id: kingfisher.kucoin.2 + pattern: | + (?xi) + \b + kucoin + (?:.|[\n\r]){0,32}? + (?: + api[_-]?secret | + secret + ) + (?:.|[\n\r]){0,12}? + ( + [a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12} + ) + \b + pattern_requirements: + min_digits: 6 + min_lowercase: 8 + ignore_if_contains: + - 00000000-0000-0000-0000-000000000000 + - xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + min_entropy: 3.3 + confidence: medium + examples: + - KUCOIN_API_SECRET=7d70f6c7-42e9-4261-8a8d-8ca2d5028d4f + - 'kucoin_secret: a1b2c3d4-e5f6-7890-abcd-ef1234567890' + references: + - https://www.kucoin.com/docs-new/authentication + diff --git a/crates/kingfisher-rules/data/rules/openshift.yml b/crates/kingfisher-rules/data/rules/openshift.yml index 442c27b..63aa3c4 100644 --- a/crates/kingfisher-rules/data/rules/openshift.yml +++ b/crates/kingfisher-rules/data/rules/openshift.yml @@ -8,9 +8,9 @@ rules: (?:.|[\n\r]){0,128}? --server= | - \bopenshift\b + \bopenshift(?:[_-]?(?:server|api(?:[_-]?server)?|cluster(?:[_-]?url)?))?\b (?:.|[\n\r]){0,32}? - \b(?:server|api(?:[_-]?server)?|cluster(?:[_-]?url)?)\b + \b(?:server|api(?:[_-]?server)?|cluster(?:[_-]?url)?)?\b \s*[:=]\s* ) ["']? @@ -43,7 +43,7 @@ rules: ( sha256~[A-Za-z0-9_-]{43} ) - \b + (?:[^A-Za-z0-9_-]|$) pattern_requirements: min_digits: 3 min_uppercase: 1 diff --git a/crates/kingfisher-rules/data/rules/privateai.yml b/crates/kingfisher-rules/data/rules/privateai.yml index ce7d5d6..209fb7e 100644 --- a/crates/kingfisher-rules/data/rules/privateai.yml +++ b/crates/kingfisher-rules/data/rules/privateai.yml @@ -34,7 +34,7 @@ rules: confidence: medium examples: - PRIVATEAI_API_KEY=4fa2d7c81be9063d4ea8bc1f6d2a7e9c - - 'x-api-key: 2ab4d6e8f0c1a3b5d7e9f1a2b4c6d8e0' + - 'privateai_x_api_key: 2ab4d6e8f0c1a3b5d7e9f1a2b4c6d8e0' references: - https://docs.private-ai.com/fundamentals/getting-started - https://docs.private-ai.com/reference/4.0.0/operation/ner_text_ner_text_post/ diff --git a/crates/kingfisher-rules/data/rules/sidekiq.yml b/crates/kingfisher-rules/data/rules/sidekiq.yml index 9e0eb2b..0a982dc 100644 --- a/crates/kingfisher-rules/data/rules/sidekiq.yml +++ b/crates/kingfisher-rules/data/rules/sidekiq.yml @@ -22,8 +22,8 @@ rules: min_entropy: 2.8 confidence: medium examples: - - BUNDLE_ENTERPRISE__CONTRIBSYS__COM=cafebabe:deadbeef - - 'export BUNDLE_GEMS__CONTRIBSYS__COM="cafeb4b3:d3adb33f"' + - BUNDLE_ENTERPRISE__CONTRIBSYS__COM=cafe1234:dead5678 + - 'export BUNDLE_GEMS__CONTRIBSYS__COM="ca1eb4b3:d3ad533f"' - name: Sidekiq Sensitive URL id: kingfisher.sidekiq.2 @@ -54,5 +54,5 @@ rules: min_entropy: 2.8 confidence: medium examples: - - https://cafebabe:deadbeef@gems.contribsys.com/ - - http://cafeb4b3:d3adb33f@enterprise.contribsys.com:80/path?param1=true + - https://cafe1234:dead5678@gems.contribsys.com/ + - http://ca1eb4b3:d3ad533f@enterprise.contribsys.com:80/path?param1=true diff --git a/crates/kingfisher-rules/data/rules/trello.yml b/crates/kingfisher-rules/data/rules/trello.yml new file mode 100644 index 0000000..fbe8821 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/trello.yml @@ -0,0 +1,31 @@ +rules: + - name: Trello API Token + id: kingfisher.trello.1 + pattern: | + (?xi) + \b + trello + (?:.|[\n\r]){0,32}? + (?: + token | + api[_-]?token | + access[_-]?token + ) + (?:.|[\n\r]){0,12}? + ( + [A-Za-z0-9]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 6 + ignore_if_contains: + - yourtoken + - placeholder + min_entropy: 3.1 + confidence: medium + examples: + - TRELLO_TOKEN=0a1b2c3d4e5f6g7h8i9j0k1l2m3n4p5q + - trello_access_token="Ab12Cd34Ef56Gh78Ij90Kl12Mn34Op56" + references: + - https://developer.atlassian.com/cloud/trello/guides/rest-api/api-introduction/ From 677c7a5d5fc42b655d38fbf95dc8b814d89ceabb Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Sun, 29 Mar 2026 00:16:28 -0700 Subject: [PATCH 04/18] feat(gitea): add --clone-url-base flag for clone URL rewriting When scanning a self-hosted Gitea/Forgejo instance, the API may be reachable at a different hostname than the git clone endpoint (e.g., internal API vs. public clone URL behind a reverse proxy). The --clone-url-base flag rewrites the scheme, host, and port of clone URLs returned by the API, preserving the path. Example: kingfisher scan gitea \ --api-url https://forge.internal.example.com/api/v1/ \ --clone-url-base https://forge.internal.example.com/ \ --user eblume This avoids routing clone traffic through an external proxy when the API and git endpoints share the same internal host but the instance's ROOT_URL points to the public endpoint. Includes unit tests for the URL rewriting function and an integration test using wiremock to verify the full enumeration path. --- src/cli/commands/inputs.rs | 4 ++ src/cli/commands/scan.rs | 14 +++++- src/direct_validate.rs | 1 + src/gitea.rs | 45 ++++++++++++++++- src/main.rs | 4 +- src/reporter.rs | 1 + src/reporter/json_format.rs | 1 + src/scanner/repos.rs | 2 + tests/int_gitea_clone_url_base.rs | 84 +++++++++++++++++++++++++++++++ 9 files changed, 153 insertions(+), 3 deletions(-) create mode 100644 tests/int_gitea_clone_url_base.rs diff --git a/src/cli/commands/inputs.rs b/src/cli/commands/inputs.rs index 95d3cbd..0fa2d61 100644 --- a/src/cli/commands/inputs.rs +++ b/src/cli/commands/inputs.rs @@ -182,6 +182,10 @@ pub struct InputSpecifierArgs { )] pub gitea_api_url: Url, + /// Override base URL for cloning Gitea repositories + #[arg(long, value_hint = ValueHint::Url, hide = true)] + pub gitea_clone_url_base: Option, + #[arg(long, default_value_t = GiteaRepoType::Source, hide = true)] pub gitea_repo_type: GiteaRepoType, diff --git a/src/cli/commands/scan.rs b/src/cli/commands/scan.rs index c61fc93..ab76967 100644 --- a/src/cli/commands/scan.rs +++ b/src/cli/commands/scan.rs @@ -284,7 +284,7 @@ pub enum ScanOperation { pub enum ListRepositoriesCommand { Github { api_url: Url, specifiers: GitHubRepoSpecifiers }, Gitlab { api_url: Url, specifiers: GitLabRepoSpecifiers }, - Gitea { api_url: Url, specifiers: GiteaRepoSpecifiers }, + Gitea { api_url: Url, clone_url_base: Option, specifiers: GiteaRepoSpecifiers }, Bitbucket { api_url: Url, specifiers: BitbucketRepoSpecifiers }, Azure { base_url: Url, specifiers: AzureRepoSpecifiers }, Huggingface { specifiers: HuggingFaceRepoSpecifiers }, @@ -396,6 +396,7 @@ impl ScanCommandArgs { if args.list_only { Some(ListRepositoriesCommand::Gitea { api_url: args.api_url, + clone_url_base: args.clone_url_base, specifiers: args.specifiers, }) } else { @@ -408,6 +409,8 @@ impl ScanCommandArgs { args.specifiers.all_organizations; scan_args.input_specifier_args.gitea_repo_type = args.specifiers.repo_type; scan_args.input_specifier_args.gitea_api_url = args.api_url; + scan_args.input_specifier_args.gitea_clone_url_base = + args.clone_url_base; None } } @@ -741,6 +744,15 @@ pub struct GiteaScanArgs { value_hint = ValueHint::Url )] pub api_url: Url, + + /// Override the base URL used for cloning repositories. + /// + /// By default, clone URLs returned by the Gitea/Forgejo API are used as-is. + /// When the API is reachable at a different hostname than the git clone + /// endpoint (e.g., internal API vs. public clone URL), use this flag to + /// rewrite the scheme, host, and port of clone URLs. + #[arg(long = "clone-url-base", value_hint = ValueHint::Url)] + pub clone_url_base: Option, } #[derive(Args, Debug, Clone)] diff --git a/src/direct_validate.rs b/src/direct_validate.rs index 5d68d22..e8ca71e 100644 --- a/src/direct_validate.rs +++ b/src/direct_validate.rs @@ -919,6 +919,7 @@ pub(crate) fn create_minimal_scan_args() -> crate::cli::commands::scan::ScanArgs gitea_exclude: Vec::new(), all_gitea_organizations: false, gitea_api_url: Url::parse("https://gitea.com/api/v1/").unwrap(), + gitea_clone_url_base: None, gitea_repo_type: GiteaRepoType::Source, bitbucket_user: Vec::new(), bitbucket_workspace: Vec::new(), diff --git a/src/gitea.rs b/src/gitea.rs index d748685..898b813 100644 --- a/src/gitea.rs +++ b/src/gitea.rs @@ -212,6 +212,7 @@ async fn fetch_authenticated_orgs( pub async fn enumerate_repo_urls( specifiers: &RepoSpecifiers, api_url: Url, + clone_url_base: Option<&Url>, ignore_certs: bool, mut progress: Option<&mut ProgressBar>, ) -> Result> { @@ -291,6 +292,14 @@ pub async fn enumerate_repo_urls( } } + // Rewrite clone URLs if a custom base was provided. + if let Some(base) = clone_url_base { + repos = repos + .into_iter() + .map(|raw| rewrite_clone_url(&raw, base).unwrap_or(raw)) + .collect(); + } + repos.sort(); repos.dedup(); Ok(repos) @@ -298,6 +307,7 @@ pub async fn enumerate_repo_urls( pub async fn list_repositories( api_url: Url, + clone_url_base: Option<&Url>, ignore_certs: bool, progress_enabled: bool, users: &[String], @@ -324,7 +334,7 @@ pub async fn list_repositories( exclude_repos: exclude_repos.to_vec(), }; - let urls = enumerate_repo_urls(&specifiers, api_url, ignore_certs, Some(&mut progress)).await?; + let urls = enumerate_repo_urls(&specifiers, api_url, clone_url_base, ignore_certs, Some(&mut progress)).await?; for url in urls { println!("{}", url); } @@ -332,6 +342,15 @@ pub async fn list_repositories( Ok(()) } +/// Rewrite a clone URL to use a different base (scheme, host, port), preserving the path. +fn rewrite_clone_url(raw: &str, base: &Url) -> Option { + let mut parsed = Url::parse(raw).ok()?; + parsed.set_scheme(base.scheme()).ok()?; + parsed.set_host(base.host_str()).ok()?; + parsed.set_port(base.port()).ok()?; + Some(parsed.to_string()) +} + fn parse_repo(repo_url: &GitUrl) -> Option<(String, String, String)> { let url = Url::parse(repo_url.as_str()).ok()?; let host = url.host_str()?.to_string(); @@ -371,4 +390,28 @@ mod tests { fn normalize_repo_identifier_handles_git_suffix() { assert_eq!(normalize_repo_identifier("owner/repo.git"), Some("owner/repo".into())); } + + #[test] + fn rewrite_clone_url_changes_host() { + let base = Url::parse("https://forge.internal.example.com/").unwrap(); + assert_eq!( + rewrite_clone_url("https://forge.public.example.com/owner/repo.git", &base), + Some("https://forge.internal.example.com/owner/repo.git".to_string()) + ); + } + + #[test] + fn rewrite_clone_url_changes_port() { + let base = Url::parse("https://forge.example.com:3000/").unwrap(); + assert_eq!( + rewrite_clone_url("https://forge.example.com/owner/repo.git", &base), + Some("https://forge.example.com:3000/owner/repo.git".to_string()) + ); + } + + #[test] + fn rewrite_clone_url_returns_none_for_invalid_url() { + let base = Url::parse("https://forge.example.com/").unwrap(); + assert_eq!(rewrite_clone_url("not-a-url", &base), None); + } } diff --git a/src/main.rs b/src/main.rs index 769d8fe..e62010b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -368,9 +368,10 @@ async fn async_main(args: CommandLineArgs) -> Result<()> { ) .await?; } - ListRepositoriesCommand::Gitea { api_url, specifiers } => { + ListRepositoriesCommand::Gitea { api_url, clone_url_base, specifiers } => { gitea::list_repositories( api_url, + clone_url_base.as_ref(), global_args.ignore_certs, global_args.use_progress(), &specifiers.user, @@ -506,6 +507,7 @@ fn create_default_scan_args() -> cli::commands::scan::ScanArgs { gitea_exclude: Vec::new(), all_gitea_organizations: false, gitea_api_url: Url::parse("https://gitea.com/api/v1/").unwrap(), + gitea_clone_url_base: None, gitea_repo_type: GiteaRepoType::Source, bitbucket_user: Vec::new(), diff --git a/src/reporter.rs b/src/reporter.rs index b01c6df..8734450 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -1743,6 +1743,7 @@ mod tests { gitea_exclude: Vec::new(), all_gitea_organizations: false, gitea_api_url: Url::parse("https://gitea.com/api/v1/").unwrap(), + gitea_clone_url_base: None, gitea_repo_type: GiteaRepoType::Source, bitbucket_user: Vec::new(), bitbucket_workspace: Vec::new(), diff --git a/src/reporter/json_format.rs b/src/reporter/json_format.rs index c5b3500..06cf949 100644 --- a/src/reporter/json_format.rs +++ b/src/reporter/json_format.rs @@ -119,6 +119,7 @@ mod tests { gitea_exclude: Vec::new(), all_gitea_organizations: false, gitea_api_url: Url::parse("https://gitea.com/api/v1/").unwrap(), + gitea_clone_url_base: None, gitea_repo_type: GiteaRepoType::Source, // Bitbucket diff --git a/src/scanner/repos.rs b/src/scanner/repos.rs index 3b354b0..8468f43 100644 --- a/src/scanner/repos.rs +++ b/src/scanner/repos.rs @@ -423,9 +423,11 @@ pub async fn enumerate_gitea_repos( let mut num_found: u64 = 0; let api_url = args.input_specifier_args.gitea_api_url.clone(); + let clone_url_base = args.input_specifier_args.gitea_clone_url_base.as_ref(); let repo_strings = gitea::enumerate_repo_urls( &repo_specifiers, api_url, + clone_url_base, global_args.ignore_certs, Some(&mut progress), ) diff --git a/tests/int_gitea_clone_url_base.rs b/tests/int_gitea_clone_url_base.rs new file mode 100644 index 0000000..6ae968e --- /dev/null +++ b/tests/int_gitea_clone_url_base.rs @@ -0,0 +1,84 @@ +// tests/int_gitea_clone_url_base.rs +// +// Integration test: verify that --clone-url-base rewrites clone URLs +// returned by the Gitea API during repository enumeration. +// +// Uses wiremock to mock the Gitea API and assert_cmd to exercise the full +// CLI path: argument parsing → API enumeration → URL rewriting → output. + +use assert_cmd::Command; +use predicates::str::contains; +use wiremock::{ + matchers::{method, path, query_param}, + Mock, MockServer, ResponseTemplate, +}; + +/// Run `kingfisher scan gitea --list-only` against a mock Gitea API with and +/// without --clone-url-base, verifying that clone URLs are rewritten. +#[tokio::test] +async fn clone_url_base_rewrites_listed_urls() { + let mock_server = MockServer::start().await; + + let public_host = "https://forge.public.example.com"; + let repo_json = serde_json::json!([{ + "full_name": "eblume/kingfisher", + "clone_url": format!("{public_host}/eblume/kingfisher.git"), + "fork": false + }]); + + // Page 1: return one repo. + Mock::given(method("GET")) + .and(path("/api/v1/users/eblume/repos")) + .and(query_param("page", "1")) + .respond_with(ResponseTemplate::new(200).set_body_json(&repo_json)) + .mount(&mock_server) + .await; + + // Page 2: return empty array to terminate pagination. + Mock::given(method("GET")) + .and(path("/api/v1/users/eblume/repos")) + .and(query_param("page", "2")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([]))) + .mount(&mock_server) + .await; + + let api_url = format!("{}/api/v1/", mock_server.uri()); + + // WITH --clone-url-base: URLs should be rewritten. + Command::new(assert_cmd::cargo::cargo_bin!("kingfisher")) + .args([ + "scan", + "gitea", + "--api-url", + &api_url, + "--clone-url-base", + "https://forge.internal.example.com/", + "--user", + "eblume", + "--list-only", + "--no-update-check", + "--quiet", + ]) + .assert() + .success() + .stdout(contains("https://forge.internal.example.com/eblume/kingfisher.git")); + + // WITHOUT --clone-url-base: URLs should be unchanged. + Command::new(assert_cmd::cargo::cargo_bin!("kingfisher")) + .args([ + "scan", + "gitea", + "--api-url", + &api_url, + "--user", + "eblume", + "--list-only", + "--no-update-check", + "--quiet", + ]) + .assert() + .success() + .stdout(contains(&format!( + "{public_host}/eblume/kingfisher.git" + ))); +} From 3442e92405515c07789555ade595862a1f72324c Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 29 Mar 2026 08:53:18 -0700 Subject: [PATCH 05/18] added more rules --- .../kingfisher-rules/data/rules/privkey.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/crates/kingfisher-rules/data/rules/privkey.yml b/crates/kingfisher-rules/data/rules/privkey.yml index 8407324..e6fa5e9 100644 --- a/crates/kingfisher-rules/data/rules/privkey.yml +++ b/crates/kingfisher-rules/data/rules/privkey.yml @@ -108,22 +108,3 @@ rules: -----END ENCRYPTED PRIVATE KEY BLOCK----- references: - https://www.rfc-editor.org/rfc/rfc7468 - - - name: PKCS#12 File Path - id: kingfisher.privkey.3 - pattern: | - (?xi) - (?: - ^|["'\s(=/] - ) - ( - (?:[^"' \t\r\n/]+/)*[^"' \t\r\n/]+\.(?:p12|pfx) - ) - (?: - $|["'\s),] - ) - min_entropy: 2.0 - confidence: medium - examples: - - security/es_certificates/opensearch/es_kibana_client.p12 - - ToDo/ToDo.UWP/ToDo.UWP_TemporaryKey.pfx From b1c2953d25b10a55a1d28a1ed5f569d4e3bc8e40 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 29 Mar 2026 08:54:40 -0700 Subject: [PATCH 06/18] added more rules --- crates/kingfisher-rules/data/rules/azure-notification-hub.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/kingfisher-rules/data/rules/azure-notification-hub.yml b/crates/kingfisher-rules/data/rules/azure-notification-hub.yml index c090b03..01321b6 100644 --- a/crates/kingfisher-rules/data/rules/azure-notification-hub.yml +++ b/crates/kingfisher-rules/data/rules/azure-notification-hub.yml @@ -45,7 +45,7 @@ rules: ["']? ( [A-Za-z0-9] - [A-Za-z0-9._-]{1,127} + [A-Za-z0-9._-]{6,64} ) ["']? \b From 3d7629cf8b5f1a51dbd976c1ae4b2df00dba0c42 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 29 Mar 2026 08:55:09 -0700 Subject: [PATCH 07/18] added more rules --- crates/kingfisher-rules/data/rules/azure-notification-hub.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/kingfisher-rules/data/rules/azure-notification-hub.yml b/crates/kingfisher-rules/data/rules/azure-notification-hub.yml index 01321b6..6e3ce2f 100644 --- a/crates/kingfisher-rules/data/rules/azure-notification-hub.yml +++ b/crates/kingfisher-rules/data/rules/azure-notification-hub.yml @@ -45,7 +45,7 @@ rules: ["']? ( [A-Za-z0-9] - [A-Za-z0-9._-]{6,64} + [A-Za-z0-9._-]{6,63} ) ["']? \b @@ -70,7 +70,7 @@ rules: ["']? ( [A-Za-z] - [A-Za-z0-9_-]{2,63} + [A-Za-z0-9_-]{6,63} ) ["']? \b From 482a60bb9d2d74ae13a4da9d92875cfc4eb92258 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 29 Mar 2026 10:41:54 -0700 Subject: [PATCH 08/18] fixed github actions --- .github/workflows/release-provenance.yml | 49 -------- .github/workflows/release.yml | 40 ++++++ CHANGELOG.md | 4 +- README.md | 36 +++++- crates/kingfisher-rules/data/rules/AGENTS.md | 21 +++- .../kingfisher-rules/data/rules/anthropic.yml | 36 +++++- .../data/rules/azurespeech.yml | 84 +++++++++++++ .../data/rules/azuretranslator.yml | 97 +++++++++++++++ .../kingfisher-rules/data/rules/databento.yml | 34 +++++ .../kingfisher-rules/data/rules/datastax.yml | 33 +++++ .../kingfisher-rules/data/rules/devcycle.yml | 117 ++++++++++++++++++ .../kingfisher-rules/data/rules/fullstory.yml | 41 ++++++ .../kingfisher-rules/data/rules/gcnotify.yml | 41 ++++++ crates/kingfisher-rules/data/rules/heroku.yml | 60 +++++++++ crates/kingfisher-rules/data/rules/stytch.yml | 76 ++++++++++++ 15 files changed, 716 insertions(+), 53 deletions(-) delete mode 100644 .github/workflows/release-provenance.yml create mode 100644 crates/kingfisher-rules/data/rules/azurespeech.yml create mode 100644 crates/kingfisher-rules/data/rules/azuretranslator.yml create mode 100644 crates/kingfisher-rules/data/rules/databento.yml create mode 100644 crates/kingfisher-rules/data/rules/datastax.yml create mode 100644 crates/kingfisher-rules/data/rules/devcycle.yml create mode 100644 crates/kingfisher-rules/data/rules/fullstory.yml create mode 100644 crates/kingfisher-rules/data/rules/gcnotify.yml create mode 100644 crates/kingfisher-rules/data/rules/stytch.yml diff --git a/.github/workflows/release-provenance.yml b/.github/workflows/release-provenance.yml deleted file mode 100644 index c3adb37..0000000 --- a/.github/workflows/release-provenance.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: SLSA Provenance - -on: - release: - types: [published] - -permissions: {} - -jobs: - # Compute SHA256 hashes of all release assets - hash: - name: Compute artifact hashes - runs-on: ubuntu-24.04 - permissions: - contents: read - outputs: - hashes: ${{ steps.hash.outputs.hashes }} - steps: - - name: Download release assets - env: - GH_TOKEN: ${{ github.token }} - TAG_NAME: ${{ github.event.release.tag_name }} - run: | - set -euo pipefail - mkdir -p assets - gh release download "${TAG_NAME}" \ - --repo "${{ github.repository }}" \ - --dir assets - - - name: Compute SHA256 hashes - id: hash - run: | - set -euo pipefail - cd assets - # Base64-encode the SHA256 hashes for SLSA provenance - echo "hashes=$(sha256sum -- * | base64 -w0)" >> "$GITHUB_OUTPUT" - - # Generate SLSA provenance for the release artifacts - provenance: - name: Generate SLSA provenance - needs: [hash] - permissions: - actions: read - id-token: write - contents: write - uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@f7dd8c54c2067bafc12ca7a55595d5ee9b75204a # v2.1.0 - with: - base64-subjects: "${{ needs.hash.outputs.hashes }}" - upload-assets: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8ea626f..f58df1d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -408,6 +408,46 @@ jobs: with: subject-path: 'target/release/*' + # ──────────────── SLSA Provenance ──────────────── + hash: + name: Compute artifact hashes + needs: [release] + runs-on: ubuntu-24.04 + permissions: + contents: read + outputs: + hashes: ${{ steps.hash.outputs.hashes }} + steps: + - name: Download release assets + env: + GH_TOKEN: ${{ github.token }} + TAG_NAME: ${{ needs.release.outputs.tag }} + run: | + set -euo pipefail + mkdir -p assets + gh release download "${TAG_NAME}" \ + --repo "${{ github.repository }}" \ + --dir assets + + - name: Compute SHA256 hashes + id: hash + run: | + set -euo pipefail + cd assets + echo "hashes=$(sha256sum -- * | base64 -w0)" >> "$GITHUB_OUTPUT" + + provenance: + name: Generate SLSA provenance + needs: [hash] + permissions: + actions: read + id-token: write + contents: write + uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@f7dd8c54c2067bafc12ca7a55595d5ee9b75204a # v2.1.0 + with: + base64-subjects: "${{ needs.hash.outputs.hashes }}" + upload-assets: true + # ──────────────── Publish Docker image ──────────────── publish-docker: needs: [release] diff --git a/CHANGELOG.md b/CHANGELOG.md index 6954865..97b5ed7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,10 @@ All notable changes to this project will be documented in this file. ## [v1.92.0] -- Added new built-in rules for Etsy, Flutterwave, Freemius, JFrog, Kraken, KuCoin, and Trello, plus recent Betterleaks-derived additions including Octopus Deploy, OpenShift, Private AI, SettleMint, Sidekiq, and Polymarket. +- Added new built-in rules for Etsy, Flutterwave, Freemius, JFrog, Kraken, KuCoin, Trello, Octopus Deploy, OpenShift, Private AI, SettleMint, Sidekiq, and Polymarket. - Added live HTTP validation for Etsy, JFrog, Octopus Deploy, OpenShift, and Private AI where provider documentation supported reliable token-only checks. +- Added detection + validation rules for Anthropic Admin, Azure Speech, Azure Translator, Databento, DataStax Astra, DevCycle, Fullstory, GC Notify, and Stytch; built-in runtime rule count is now 601 with `--confidence=low`. +- Added Heroku token revocation support for both legacy UUID-format tokens and `HRKU-` platform tokens via the OAuth authorizations API. ## [v1.91.0] - Added SSRF protection for credential validation: outbound HTTP requests now block connections to loopback, private, link-local, and other non-public IP addresses. HTTP redirect targets are DNS-resolved and validated against the same SSRF rules. Use `--allow-internal-ips` to opt out when scanning internal infrastructure. diff --git a/README.md b/README.md index d5cb266..e208ae9 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Kingfisher Logo [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -[![Detection Rules](https://img.shields.io/badge/Detection%20Rules-548-2ea043.svg)](https://github.com/mongodb/kingfisher)
+[![Detection Rules](https://img.shields.io/badge/Detection%20Rules-601-2ea043.svg)](https://github.com/mongodb/kingfisher)
[![ghcr downloads](https://ghcr-badge.elias.eu.org/shield/mongodb/kingfisher/kingfisher)](https://github.com/mongodb/kingfisher/pkgs/container/kingfisher)
@@ -302,6 +302,40 @@ Kingfisher supports multiple installation methods: **For complete installation instructions and pre-commit hook setup, see [docs/INSTALLATION.md](docs/INSTALLATION.md).** +## Verifying Releases + +Every Kingfisher release includes [SLSA v3](https://slsa.dev) provenance and GitHub build attestations so you can verify that artifacts were built by our CI pipeline and haven't been tampered with. + +### SLSA provenance + +Each GitHub release includes a `multiple.intoto.jsonl` provenance file. Verify any release artifact with [`slsa-verifier`](https://github.com/slsa-framework/slsa-verifier): + +```bash +# Install slsa-verifier +go install github.com/slsa-framework/slsa-verifier/v2/cli/slsa-verifier@latest + +# Download the artifact and provenance from the release +gh release download v1.91.0 --repo mongodb/kingfisher \ + --pattern 'kingfisher-linux-x64.tgz' \ + --pattern 'multiple.intoto.jsonl' + +# Verify +slsa-verifier verify-artifact kingfisher-linux-x64.tgz \ + --provenance-path multiple.intoto.jsonl \ + --source-uri github.com/mongodb/kingfisher +``` + +### GitHub attestations + +Release artifacts also have GitHub build attestations, verifiable with the GitHub CLI: + +```bash +gh release download v1.91.0 --repo mongodb/kingfisher \ + --pattern 'kingfisher-linux-x64.tgz' + +gh attestation verify kingfisher-linux-x64.tgz --repo mongodb/kingfisher +``` + # Detection Rules Kingfisher ships with [hundreds of rules](crates/kingfisher-rules/data/rules/) that cover everything from classic cloud keys to the latest AI SaaS tokens. Below is an overview: diff --git a/crates/kingfisher-rules/data/rules/AGENTS.md b/crates/kingfisher-rules/data/rules/AGENTS.md index 80a9522..80ecbaf 100644 --- a/crates/kingfisher-rules/data/rules/AGENTS.md +++ b/crates/kingfisher-rules/data/rules/AGENTS.md @@ -30,8 +30,27 @@ Strongly recommended fields: ## Pattern Quality Rules - Prefer specific anchors/prefixes and provider context over broad generic regex. +- When the token format is generic or common-looking (for example bare 32-hex keys), prefer contextual patterns of the form: provider keyword -> short flexible gap -> key/secret label -> short flexible gap -> token. A good default is: + - `\b` + - provider identifier (for example `amplitude`, `azure`, `speech`, `translator`) + - `(?:.|[\n\r]){0,N}?` + - common credential labels such as `(?:SECRET|PRIVATE|ACCESS|KEY|TOKEN|AUTHORIZATION|API)` + - `(?:.|[\n\r]){0,M}?` + - the token capture wrapped in a single unnamed capture group +- Do not add surrounding context when the token is already strongly self-identifying by prefix or structure (for example `sk-ant-api...`, `AstraCS:...`, `dvc_client_...`, `secret-test-...`). In those cases, prefer the tighter self-identifying regex. - Use `pattern_requirements` to enforce quality constraints (`min_digits`, `min_uppercase`, `min_lowercase`, `min_special_chars`, `ignore_if_contains`, `checksum`). -- Use checksum validation in `pattern_requirements.checksum` when token formats support it. +- Use checksum validation in `pattern_requirements.checksum` when token formats support it. This is preferred when the provider token format includes a documented or reverse-engineered check segment, because it can sharply reduce false positives without adding brittle surrounding context. +- For checksum-based rules, prefer named captures for the main token body and checksum suffix/prefix, then compute the expected checksum in Liquid. A typical pattern is: + - `( + prefix_(?P...)(?P...) + )` + - with: + - `actual.template: "{{ checksum }}"` + - `actual.requires_capture: checksum` + - `expected: "{{ body | | }}"` + - `skip_if_missing: true` +- Example: GitHub PATs use a CRC32-derived base62 checksum. The rule in `github.yml` captures `body` and `checksum`, then compares `{{ checksum }}` against `{{ body | crc32 | base62: 6 }}`. +- Prefer checksum validation over extra loose context whenever the token structure itself supports it. If the checksum is only present on some token generations, keep `skip_if_missing: true` so older examples continue to load safely. - Use `visible: false` for helper/non-secret captures used only by dependent rules. - Use `depends_on_rule` for multi-part credential validation (for example ID + secret). diff --git a/crates/kingfisher-rules/data/rules/anthropic.yml b/crates/kingfisher-rules/data/rules/anthropic.yml index 3aed2b1..1a2784a 100644 --- a/crates/kingfisher-rules/data/rules/anthropic.yml +++ b/crates/kingfisher-rules/data/rules/anthropic.yml @@ -47,4 +47,38 @@ rules: words: - '"type":"message"' - 'credit balance is too low' - url: https://api.anthropic.com/v1/messages \ No newline at end of file + url: https://api.anthropic.com/v1/messages + + - name: Anthropic Admin API Key + id: kingfisher.anthropic.2 + pattern: | + (?xi) + ( + sk-ant-admin(?:\d{2,4})?-[A-Za-z0-9_-]{40,} + ) + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 + min_entropy: 3.5 + confidence: medium + examples: + - sk-ant-admin03-4mB9zY2Qx8LmN7pR5sT1uV6wX0aBcDeFgHiJkLmNoPqRsTuVwXyZ1234 + references: + - https://docs.anthropic.com/en/api/administration-api + - https://docs.anthropic.com/en/api/admin-api/organization/get-me + validation: + type: Http + content: + request: + method: GET + url: https://api.anthropic.com/v1/organizations/me + headers: + x-api-key: '{{ TOKEN }}' + anthropic-version: "2023-06-01" + content-type: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid diff --git a/crates/kingfisher-rules/data/rules/azurespeech.yml b/crates/kingfisher-rules/data/rules/azurespeech.yml new file mode 100644 index 0000000..4a661cf --- /dev/null +++ b/crates/kingfisher-rules/data/rules/azurespeech.yml @@ -0,0 +1,84 @@ +rules: + - name: Azure Speech Region + id: kingfisher.azurespeech.1 + visible: false + pattern: | + (?xi) + \b + (?: + SPEECH_REGION + | + AZURE_SPEECH_REGION + | + speech[_-]?region + | + azure[_-]?speech[_-]?region + ) + \b + (?:.|[\n\r]){0,16}? + [=:] + \s*["']? + ( + [a-z0-9-]{4,32} + ) + ["']? + min_entropy: 1.5 + confidence: medium + examples: + - SPEECH_REGION=eastus + - azure_speech_region="westus2" + references: + - https://learn.microsoft.com/en-us/azure/ai-services/speech-service/rest-text-to-speech + + - name: Azure Speech API Key + id: kingfisher.azurespeech.2 + pattern: | + (?xi) + \b + (?: + speech + | + azure[_-]?speech + ) + (?:.|[\n\r]){0,24}? + (?: + key + | + api[_-]?key + | + subscription[_-]?key + | + secret + ) + (?:.|[\n\r]){0,16}? + ( + [a-f0-9]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 2 + min_entropy: 3.5 + confidence: medium + examples: + - AZURE_SPEECH_KEY=1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d + - speech_subscription_key="abcdef0123456789abcdef0123456789" + references: + - https://learn.microsoft.com/en-us/azure/ai-services/speech-service/rest-text-to-speech + depends_on_rule: + - rule_id: kingfisher.azurespeech.1 + variable: AZURE_SPEECH_REGION + validation: + type: Http + content: + request: + method: POST + url: https://{{ AZURE_SPEECH_REGION }}.api.cognitive.microsoft.com/sts/v1.0/issueToken + headers: + Ocp-Apim-Subscription-Key: "{{ TOKEN }}" + Content-Type: application/x-www-form-urlencoded + Content-Length: "0" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] diff --git a/crates/kingfisher-rules/data/rules/azuretranslator.yml b/crates/kingfisher-rules/data/rules/azuretranslator.yml new file mode 100644 index 0000000..7a9ccf3 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/azuretranslator.yml @@ -0,0 +1,97 @@ +rules: + - name: Azure Translator Region + id: kingfisher.azuretranslator.1 + visible: false + pattern: | + (?xi) + \b + (?: + TRANSLATOR_REGION + | + AZURE_TRANSLATOR_REGION + | + translator[_-]?region + | + translation[_-]?region + | + Ocp-Apim-Subscription-Region + ) + \b + (?:.|[\n\r]){0,16}? + [=:] + \s*["']? + ( + [a-z0-9-]{4,32} + ) + ["']? + min_entropy: 1.5 + confidence: medium + examples: + - TRANSLATOR_REGION=eastus + - azure_translator_region="westeurope" + references: + - https://learn.microsoft.com/en-us/azure/ai-services/translator/text-translation/reference/authentication + - https://learn.microsoft.com/en-us/azure/ai-services/translator/reference/v3-0-reference + + - name: Azure Translator API Key + id: kingfisher.azuretranslator.2 + pattern: | + (?xi) + \b + (?: + translator + | + translation + | + azure[_-]?(?:translator|translation) + ) + (?:.|[\n\r]){0,24}? + (?: + key + | + api[_-]?key + | + subscription[_-]?key + | + secret + ) + (?:.|[\n\r]){0,16}? + ( + [a-f0-9]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 2 + min_entropy: 3.5 + confidence: medium + examples: + - AZURE_TRANSLATOR_KEY=1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d + - translator_subscription_key="abcdef0123456789abcdef0123456789" + references: + - https://learn.microsoft.com/en-us/azure/ai-services/translator/text-translation/reference/authentication + - https://learn.microsoft.com/en-us/azure/ai-services/translator/reference/v3-0-reference + depends_on_rule: + - rule_id: kingfisher.azuretranslator.1 + variable: AZURE_TRANSLATOR_REGION + validation: + type: Http + content: + request: + method: POST + url: https://api.cognitive.microsofttranslator.com/translate?api-version=3.0&to=es + headers: + Ocp-Apim-Subscription-Key: "{{ TOKEN }}" + Ocp-Apim-Subscription-Region: "{{ AZURE_TRANSLATOR_REGION }}" + Content-Type: application/json + body: | + [ + { + "Text": "hello" + } + ] + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid diff --git a/crates/kingfisher-rules/data/rules/databento.yml b/crates/kingfisher-rules/data/rules/databento.yml new file mode 100644 index 0000000..2249cb0 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/databento.yml @@ -0,0 +1,34 @@ +rules: + - name: Databento API Key + id: kingfisher.databento.1 + pattern: | + (?x) + \b + ( + db-[A-Za-z0-9]{29} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 2 + min_entropy: 3.3 + confidence: medium + examples: + - DATABENTO_API_KEY=db-abc123def456ghi789jkl012mno34 + references: + - https://databento.com/docs/api-reference-historical + - https://databento.com/docs/portal/api-keys + validation: + type: Http + content: + request: + method: GET + url: https://hist.databento.com/v0/metadata.list_datasets + headers: + Authorization: "Basic {{ TOKEN | append: ':' | b64enc }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid diff --git a/crates/kingfisher-rules/data/rules/datastax.yml b/crates/kingfisher-rules/data/rules/datastax.yml new file mode 100644 index 0000000..62b4537 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/datastax.yml @@ -0,0 +1,33 @@ +rules: + - name: DataStax Astra Application Token + id: kingfisher.datastax.1 + pattern: | + (?x) + \b + ( + AstraCS:[A-Za-z0-9]{20,} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 4.0 + confidence: medium + examples: + - ASTRA_DB_APPLICATION_TOKEN=AstraCS:Q29kZXhWYWxpZGF0aW9uVG9rZW5FeGFtcGxlMTIzNDU2Nzg5 + references: + - https://docs.datastax.com/en/astra-db-serverless/administration/manage-application-tokens.html + - https://docs.datastax.com/en/astra-db-classic/api-reference/devops-api.html + validation: + type: Http + content: + request: + method: GET + url: https://api.astra.datastax.com/v2/databases + headers: + Authorization: Bearer {{ TOKEN }} + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid diff --git a/crates/kingfisher-rules/data/rules/devcycle.yml b/crates/kingfisher-rules/data/rules/devcycle.yml new file mode 100644 index 0000000..0e71e76 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/devcycle.yml @@ -0,0 +1,117 @@ +rules: + - name: DevCycle Client SDK Key + id: kingfisher.devcycle.1 + pattern: | + (?x) + \b + ( + dvc_client_[A-Za-z0-9]{8,32} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 2 + min_entropy: 3.0 + confidence: medium + examples: + - dvc_client_abc12345 + - 'sdkKey: "dvc_client_abcdefg1234"' + references: + - https://docs.devcycle.com/cli-guides/environments/ + - https://docs.devcycle.com/bucketing-api/ + validation: + type: Http + content: + request: + method: POST + url: https://bucketing-api.devcycle.com/v1/variables + headers: + Authorization: Bearer {{ TOKEN }} + Content-Type: application/json + body: | + { + "user_id": "kingfisher-validation-user" + } + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + + - name: DevCycle Mobile SDK Key + id: kingfisher.devcycle.2 + pattern: | + (?x) + \b + ( + dvc_mobile_[A-Za-z0-9]{8,32} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 2 + min_entropy: 3.0 + confidence: medium + examples: + - dvc_mobile_abc12345 + - 'mobileKey: "dvc_mobile_abcdefg1234"' + references: + - https://docs.devcycle.com/cli-guides/environments/ + - https://docs.devcycle.com/bucketing-api/ + validation: + type: Http + content: + request: + method: POST + url: https://bucketing-api.devcycle.com/v1/variables + headers: + Authorization: Bearer {{ TOKEN }} + Content-Type: application/json + body: | + { + "user_id": "kingfisher-validation-user" + } + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + + - name: DevCycle Server SDK Key + id: kingfisher.devcycle.3 + pattern: | + (?x) + \b + ( + dvc_server_[A-Za-z0-9]{8,32} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 2 + min_entropy: 3.0 + confidence: medium + examples: + - dvc_server_abc12345 + - 'serverKey: "dvc_server_abcdefg1234"' + references: + - https://docs.devcycle.com/cli-guides/environments/ + - https://docs.devcycle.com/bucketing-api/ + validation: + type: Http + content: + request: + method: POST + url: https://bucketing-api.devcycle.com/v1/variables + headers: + Authorization: Bearer {{ TOKEN }} + Content-Type: application/json + body: | + { + "user_id": "kingfisher-validation-user" + } + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid diff --git a/crates/kingfisher-rules/data/rules/fullstory.yml b/crates/kingfisher-rules/data/rules/fullstory.yml new file mode 100644 index 0000000..b9c4571 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/fullstory.yml @@ -0,0 +1,41 @@ +rules: + - name: Fullstory API Key + id: kingfisher.fullstory.1 + pattern: | + (?xi) + \b + (?:fullstory|fs_api_key|fullstory_api_key) + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN|AUTHORIZATION|API) + (?:.|[\n\r]){0,16}? + \b + ( + (?:na1|eu1)\.[A-Za-z0-9]{20,} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 2 + min_entropy: 3.3 + confidence: medium + examples: + - FULLSTORY_API_KEY=na1.Abcd1234Efgh5678Ijkl9012Mnop3456 + - 'fs_api_key: "eu1.Abcd1234Efgh5678Ijkl9012Mnop3456"' + references: + - https://developer.fullstory.com/server/v1/getting-started/ + - https://developer.fullstory.com/server/authentication/ + - https://developer.fullstory.com/server/v1/authentication/me/ + validation: + type: Http + content: + request: + method: GET + url: https://api.fullstory.com/me + headers: + Authorization: Basic {{ TOKEN }} + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid diff --git a/crates/kingfisher-rules/data/rules/gcnotify.yml b/crates/kingfisher-rules/data/rules/gcnotify.yml new file mode 100644 index 0000000..1d83850 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/gcnotify.yml @@ -0,0 +1,41 @@ +rules: + - name: GC Notify API Key + id: kingfisher.gcnotify.1 + pattern: | + (?xi) + \b + ( + ApiKey-v1 + \s+ + gcntfy-[a-z0-9_]+ + - + [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} + - + [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} + ) + \b + pattern_requirements: + min_digits: 4 + min_lowercase: 2 + min_entropy: 3.5 + confidence: medium + examples: + - 'Authorization: "ApiKey-v1 gcntfy-my_test_key-26785a09-ab16-4eb0-8407-a37497a57506-3d844edf-8d35-48ac-975b-e847b4f122b0"' + references: + - https://documentation.notification.canada.ca/en/start.html + - https://documentation.notification.canada.ca/en/status.html + - https://documentation.notification.canada.ca/en/keys.html + validation: + type: Http + content: + request: + method: GET + url: https://api.notification.canada.ca/v2/notifications + headers: + Authorization: "{{ TOKEN }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid diff --git a/crates/kingfisher-rules/data/rules/heroku.yml b/crates/kingfisher-rules/data/rules/heroku.yml index 96e94e4..d10ff87 100644 --- a/crates/kingfisher-rules/data/rules/heroku.yml +++ b/crates/kingfisher-rules/data/rules/heroku.yml @@ -20,6 +20,7 @@ rules: - 'HEROKU_API_KEY: c55dbac4-e0e8-4a06-b892-75cac2387ce5' references: - https://devcenter.heroku.com/articles/authentication + - https://devcenter.heroku.com/articles/oauth validation: type: Http content: @@ -33,6 +34,35 @@ rules: - report_response: true - type: StatusMatch status: [200] + revocation: + type: HttpMultiStep + content: + steps: + - name: lookup_authorization_id + request: + method: GET + url: https://api.heroku.com/oauth/authorizations + headers: + Accept: application/vnd.heroku+json; version=3 + Authorization: Bearer {{ TOKEN }} + response_matcher: + - type: StatusMatch + status: [200] + extract: + AUTHORIZATION_ID: + type: Regex + pattern: '"id":"([^"]+)"(?:.|[\n\r]){0,2048}?"token":"{{ TOKEN }}"' + - name: revoke_authorization + request: + method: DELETE + url: https://api.heroku.com/oauth/authorizations/{{ AUTHORIZATION_ID }} + headers: + Accept: application/vnd.heroku+json; version=3 + Authorization: Bearer {{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] - name: Heroku API Key (Platform Key) id: kingfisher.heroku.2 pattern: | @@ -61,8 +91,38 @@ rules: - '"id":' - '"name":' match_all_words: true + revocation: + type: HttpMultiStep + content: + steps: + - name: lookup_authorization_id + request: + method: GET + url: https://api.heroku.com/oauth/authorizations + headers: + Accept: application/vnd.heroku+json; version=3 + Authorization: Bearer {{TOKEN}} + response_matcher: + - type: StatusMatch + status: [200] + extract: + AUTHORIZATION_ID: + type: Regex + pattern: '"id":"([^"]+)"(?:.|[\n\r]){0,2048}?"token":"{{ TOKEN }}"' + - name: revoke_authorization + request: + method: DELETE + url: https://api.heroku.com/oauth/authorizations/{{ AUTHORIZATION_ID }} + headers: + Accept: application/vnd.heroku+json; version=3 + Authorization: "Bearer {{TOKEN}}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] references: - https://devcenter.heroku.com/articles/platform-api-quickstart + - https://devcenter.heroku.com/articles/oauth examples: - "HRKU-AADVTUYvfjT4nhuJ07bEfAUq9GS3PkTdyWuNBiXYmYMg_____wgAf6OTnGyh" - "HRKU-AABW9W1iH9NHEIlAABq9nZUq9GS3PkTdyWuNBiXYmYMg_____wV2XYIXxm5p" diff --git a/crates/kingfisher-rules/data/rules/stytch.yml b/crates/kingfisher-rules/data/rules/stytch.yml new file mode 100644 index 0000000..97c7a57 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/stytch.yml @@ -0,0 +1,76 @@ +rules: + - name: Stytch Project ID + id: kingfisher.stytch.1 + visible: false + pattern: | + (?xi) + \b + ( + project-(?:test|live)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} + ) + \b + min_entropy: 2.0 + confidence: medium + examples: + - project-test-8aed2e54-0266-4793-9b5e-0cc9c56064da + references: + - https://stytch.com/docs/api-reference/consumer/api/overview + - https://stytch.com/docs/api-reference/b2b/api/sessions/authenticate-jwt + + - name: Stytch Project Secret + id: kingfisher.stytch.2 + pattern: | + (?xi) + \b + ( + secret-(?:test|live)-[A-Za-z0-9_-]{35}=? + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 + min_entropy: 3.5 + confidence: medium + examples: + - secret-test-IJ7zLTgXp8xoS7yXO2xavNxZTbYfvm-2nZM= + references: + - https://stytch.com/docs/api-reference/consumer/api/overview + - https://stytch.com/docs/api-reference/b2b/api/m2m/overview + - https://stytch.com/docs/api-reference/b2b/api/sessions/authenticate-jwt + depends_on_rule: + - rule_id: kingfisher.stytch.1 + variable: STYTCH_PROJECT_ID + validation: + type: Http + content: + request: + method: POST + url: > + {%- if TOKEN contains "-live-" -%} + https://api.stytch.com/v1/m2m/clients/search + {%- else -%} + https://test.stytch.com/v1/m2m/clients/search + {%- endif -%} + headers: + Authorization: "Basic {{ STYTCH_PROJECT_ID | append: ':' | append: TOKEN | b64enc }}" + Content-Type: application/json + body: | + { + "query": { + "operator": "AND", + "operands": [ + { + "filter_name": "status", + "filter_value": ["active"] + } + ] + } + } + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"m2m_clients"' From ac2198e3bd3ebdbf0111fd6eadaaf4debdc40a14 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 29 Mar 2026 12:32:14 -0700 Subject: [PATCH 09/18] fixed github actions --- .github/workflows/release.yml | 6 +++++- README.md | 4 ++-- .../kingfisher-rules/data/rules/azure-notification-hub.yml | 3 ++- crates/kingfisher-rules/data/rules/fullstory.yml | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f58df1d..9cf1000 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -427,7 +427,11 @@ jobs: mkdir -p assets gh release download "${TAG_NAME}" \ --repo "${{ github.repository }}" \ - --dir assets + --dir assets \ + --pattern '*.tgz' \ + --pattern '*.deb' \ + --pattern '*.rpm' \ + --pattern '*.zip' - name: Compute SHA256 hashes id: hash diff --git a/README.md b/README.md index e208ae9..c7c1d4f 100644 --- a/README.md +++ b/README.md @@ -315,7 +315,7 @@ Each GitHub release includes a `multiple.intoto.jsonl` provenance file. Verify a go install github.com/slsa-framework/slsa-verifier/v2/cli/slsa-verifier@latest # Download the artifact and provenance from the release -gh release download v1.91.0 --repo mongodb/kingfisher \ +gh release download --repo mongodb/kingfisher \ --pattern 'kingfisher-linux-x64.tgz' \ --pattern 'multiple.intoto.jsonl' @@ -330,7 +330,7 @@ slsa-verifier verify-artifact kingfisher-linux-x64.tgz \ Release artifacts also have GitHub build attestations, verifiable with the GitHub CLI: ```bash -gh release download v1.91.0 --repo mongodb/kingfisher \ +gh release download --repo mongodb/kingfisher \ --pattern 'kingfisher-linux-x64.tgz' gh attestation verify kingfisher-linux-x64.tgz --repo mongodb/kingfisher diff --git a/crates/kingfisher-rules/data/rules/azure-notification-hub.yml b/crates/kingfisher-rules/data/rules/azure-notification-hub.yml index 6e3ce2f..42be867 100644 --- a/crates/kingfisher-rules/data/rules/azure-notification-hub.yml +++ b/crates/kingfisher-rules/data/rules/azure-notification-hub.yml @@ -149,7 +149,8 @@ rules: {%- assign se = "" | unix_timestamp | plus: 300 -%} {%- capture to_sign -%}{{ uri | url_encode }} {{ se }}{%- endcapture -%} - {%- capture auth -%}SharedAccessSignature sr={{ uri | url_encode }}&sig={{ to_sign | hmac_sha256: TOKEN | url_encode }}&se={{ se }}&skn={{ NH_KEY_NAME | url_encode }}{%- endcapture -%} + {%- assign key_bytes = TOKEN | b64dec -%} + {%- capture auth -%}SharedAccessSignature sr={{ uri | url_encode }}&sig={{ to_sign | hmac_sha256: key_bytes | url_encode }}&se={{ se }}&skn={{ NH_KEY_NAME | url_encode }}{%- endcapture -%} {{ auth | strip_newlines }} response_matcher: - report_response: true diff --git a/crates/kingfisher-rules/data/rules/fullstory.yml b/crates/kingfisher-rules/data/rules/fullstory.yml index b9c4571..960c997 100644 --- a/crates/kingfisher-rules/data/rules/fullstory.yml +++ b/crates/kingfisher-rules/data/rules/fullstory.yml @@ -32,7 +32,7 @@ rules: method: GET url: https://api.fullstory.com/me headers: - Authorization: Basic {{ TOKEN }} + Authorization: Basic {{ TOKEN | append: ':' | b64enc }} Accept: application/json response_matcher: - report_response: true From 2b99c6a3544cb3c039be69217f34bc5fa05e3c0d Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 29 Mar 2026 12:48:55 -0700 Subject: [PATCH 10/18] fixed github actions --- crates/kingfisher-rules/data/rules/fullstory.yml | 2 +- crates/kingfisher-rules/data/rules/heroku.yml | 4 ++-- crates/kingfisher-rules/data/rules/zapier.yml | 10 ---------- crates/kingfisher-rules/data/rules/zendesk.yml | 9 ++++++--- 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/crates/kingfisher-rules/data/rules/fullstory.yml b/crates/kingfisher-rules/data/rules/fullstory.yml index 960c997..64481de 100644 --- a/crates/kingfisher-rules/data/rules/fullstory.yml +++ b/crates/kingfisher-rules/data/rules/fullstory.yml @@ -32,7 +32,7 @@ rules: method: GET url: https://api.fullstory.com/me headers: - Authorization: Basic {{ TOKEN | append: ':' | b64enc }} + Authorization: "Basic {{ TOKEN | append: ':' | b64enc }}" Accept: application/json response_matcher: - report_response: true diff --git a/crates/kingfisher-rules/data/rules/heroku.yml b/crates/kingfisher-rules/data/rules/heroku.yml index d10ff87..c929e67 100644 --- a/crates/kingfisher-rules/data/rules/heroku.yml +++ b/crates/kingfisher-rules/data/rules/heroku.yml @@ -51,7 +51,7 @@ rules: extract: AUTHORIZATION_ID: type: Regex - pattern: '"id":"([^"]+)"(?:.|[\n\r]){0,2048}?"token":"{{ TOKEN }}"' + pattern: '"id":"([^"]+)"[^{}]{0,2048}?"token":"{{ TOKEN }}"' - name: revoke_authorization request: method: DELETE @@ -108,7 +108,7 @@ rules: extract: AUTHORIZATION_ID: type: Regex - pattern: '"id":"([^"]+)"(?:.|[\n\r]){0,2048}?"token":"{{ TOKEN }}"' + pattern: '"id":"([^"]+)"[^{}]{0,2048}?"token":"{{ TOKEN }}"' - name: revoke_authorization request: method: DELETE diff --git a/crates/kingfisher-rules/data/rules/zapier.yml b/crates/kingfisher-rules/data/rules/zapier.yml index fc84326..83f37f2 100644 --- a/crates/kingfisher-rules/data/rules/zapier.yml +++ b/crates/kingfisher-rules/data/rules/zapier.yml @@ -18,13 +18,3 @@ rules: - webhook_url="https://hooks.zapier.com/hooks/catch/2929690/ztd17n/" references: - https://help.zapier.com/hc/en-us/articles/8496288690317-Trigger-Zaps-from-webhooks - validation: - type: Http - content: - request: - method: GET - url: '{{ TOKEN }}' - response_matcher: - - report_response: true - - type: StatusMatch - status: [200] diff --git a/crates/kingfisher-rules/data/rules/zendesk.yml b/crates/kingfisher-rules/data/rules/zendesk.yml index 473576d..e4fa89a 100644 --- a/crates/kingfisher-rules/data/rules/zendesk.yml +++ b/crates/kingfisher-rules/data/rules/zendesk.yml @@ -6,7 +6,10 @@ rules: \b ( [a-z0-9] - [a-z0-9-]{1,62} + (?: + [a-z0-9-]{0,61} + [a-z0-9] + )? \.zendesk\.com ) \b @@ -80,7 +83,7 @@ rules: - https://developer.zendesk.com/api-reference/ticketing/account-configuration/current_user/ depends_on_rule: - rule_id: kingfisher.zendesk.1 - variable: ZENDESK_SUBDOMAIN + variable: ZENDESK_HOST - rule_id: kingfisher.zendesk.2 variable: ZENDESK_EMAIL validation: @@ -88,7 +91,7 @@ rules: content: request: method: GET - url: 'https://{{ ZENDESK_SUBDOMAIN }}/api/v2/users/me.json' + url: 'https://{{ ZENDESK_HOST }}/api/v2/users/me.json' headers: Accept: application/json Authorization: 'Basic {{ ZENDESK_EMAIL | append: "/token:" | append: TOKEN | b64enc }}' From 01f3a12106266ada03b4b17b1a33ee49dd40b477 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 29 Mar 2026 15:34:08 -0700 Subject: [PATCH 11/18] fixed github actions --- NOTICE | 1 - crates/kingfisher-rules/data/rules/fullstory.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/NOTICE b/NOTICE index d98a5b3..bd6e957 100644 --- a/NOTICE +++ b/NOTICE @@ -117,7 +117,6 @@ SOFTWARE. -------------------------------------------------------------------- Certain detection rules: - * crates/kingfisher-rules/data/rules/adafruit.yml * crates/kingfisher-rules/data/rules/etsy.yml * crates/kingfisher-rules/data/rules/flutterwave.yml * crates/kingfisher-rules/data/rules/freemius.yml diff --git a/crates/kingfisher-rules/data/rules/fullstory.yml b/crates/kingfisher-rules/data/rules/fullstory.yml index 64481de..e8783b1 100644 --- a/crates/kingfisher-rules/data/rules/fullstory.yml +++ b/crates/kingfisher-rules/data/rules/fullstory.yml @@ -4,7 +4,7 @@ rules: pattern: | (?xi) \b - (?:fullstory|fs_api_key|fullstory_api_key) + (?:fullstory|fs_api|fullstory_api) (?:.|[\n\r]){0,32}? (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN|AUTHORIZATION|API) (?:.|[\n\r]){0,16}? From 92b23b1f82aa7f6fb8c19924727d848587f432f2 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 29 Mar 2026 16:43:45 -0700 Subject: [PATCH 12/18] fixed github actions --- crates/kingfisher-rules/data/rules/kubernetes.yml | 4 +--- crates/kingfisher-rules/data/rules/polymarket.yml | 2 +- crates/kingfisher-rules/data/rules/stytch.yml | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/crates/kingfisher-rules/data/rules/kubernetes.yml b/crates/kingfisher-rules/data/rules/kubernetes.yml index bc8e810..47d05f3 100644 --- a/crates/kingfisher-rules/data/rules/kubernetes.yml +++ b/crates/kingfisher-rules/data/rules/kubernetes.yml @@ -5,8 +5,6 @@ rules: (?xi) \b (?: - server - | kube(?:rnetes)?(?:_api)?_server | api_server @@ -28,7 +26,7 @@ rules: confidence: medium visible: false examples: - - "server: https://10.96.0.1:443" + - "kube_server: https://10.96.0.1:443" - KUBE_API_SERVER=https://api.cluster.example.com:6443 references: - https://kubernetes.io/docs/reference/access-authn-authz/authentication/ diff --git a/crates/kingfisher-rules/data/rules/polymarket.yml b/crates/kingfisher-rules/data/rules/polymarket.yml index fd32d71..fa13685 100644 --- a/crates/kingfisher-rules/data/rules/polymarket.yml +++ b/crates/kingfisher-rules/data/rules/polymarket.yml @@ -13,7 +13,7 @@ rules: ( [A-Za-z0-9+/]{40,88}={0,2} ) - \b + (?:[^A-Za-z0-9+/=]|$) pattern_requirements: min_digits: 2 min_uppercase: 1 diff --git a/crates/kingfisher-rules/data/rules/stytch.yml b/crates/kingfisher-rules/data/rules/stytch.yml index 97c7a57..1fe26cb 100644 --- a/crates/kingfisher-rules/data/rules/stytch.yml +++ b/crates/kingfisher-rules/data/rules/stytch.yml @@ -25,7 +25,7 @@ rules: ( secret-(?:test|live)-[A-Za-z0-9_-]{35}=? ) - \b + (?:[^A-Za-z0-9_=-]|$) pattern_requirements: min_digits: 2 min_uppercase: 1 From fc542afa99013190c704d43491a116ee76a350cb Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 29 Mar 2026 17:08:58 -0700 Subject: [PATCH 13/18] fixed github actions --- CHANGELOG.md | 7 +++ .../data/rules/azure-notification-hub.yml | 18 ++---- crates/kingfisher-rules/src/liquid_filters.rs | 55 +++++++++++++++++++ 3 files changed, 68 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97b5ed7..df09a58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ All notable changes to this project will be documented in this file. - Added live HTTP validation for Etsy, JFrog, Octopus Deploy, OpenShift, and Private AI where provider documentation supported reliable token-only checks. - Added detection + validation rules for Anthropic Admin, Azure Speech, Azure Translator, Databento, DataStax Astra, DevCycle, Fullstory, GC Notify, and Stytch; built-in runtime rule count is now 601 with `--confidence=low`. - Added Heroku token revocation support for both legacy UUID-format tokens and `HRKU-` platform tokens via the OAuth authorizations API. +- Added `hmac_sha256_b64key` Liquid filter for HMAC-SHA256 signing with base64-encoded keys (decodes key to raw bytes before signing), enabling correct Azure Notification Hub SAS validation. +- Integrated SLSA v3 provenance generation into the release workflow; hash computation now scopes to build artifacts only for idempotent re-runs. +- Removed Zapier webhook live validation (GET to a catch hook triggers the Zap). +- Hardened Heroku revocation regex to prevent crossing JSON object boundaries when extracting authorization IDs. +- Fixed Zendesk subdomain regex to reject trailing hyphens; renamed `ZENDESK_SUBDOMAIN` to `ZENDESK_HOST` for clarity. +- Fixed Stytch and Polymarket trailing `\b` boundaries that prevented matching base64-padded secrets ending with `=`. +- Tightened Kubernetes API Server URL pattern to require kube-specific identifiers, preventing bootstrap tokens from binding to unrelated `server:` entries. ## [v1.91.0] - Added SSRF protection for credential validation: outbound HTTP requests now block connections to loopback, private, link-local, and other non-public IP addresses. HTTP redirect targets are DNS-resolved and validated against the same SSRF rules. Use `--allow-internal-ips` to opt out when scanning internal infrastructure. diff --git a/crates/kingfisher-rules/data/rules/azure-notification-hub.yml b/crates/kingfisher-rules/data/rules/azure-notification-hub.yml index 42be867..6b40af1 100644 --- a/crates/kingfisher-rules/data/rules/azure-notification-hub.yml +++ b/crates/kingfisher-rules/data/rules/azure-notification-hub.yml @@ -91,20 +91,15 @@ rules: (?:notification\s*hub|Endpoint\s*=\s*sb://[a-z0-9-]{2,63}\.servicebus\.windows\.net/?) (?:.|[\n\r]){0,160}? SharedAccessKey - \s*[:=]\s* - ["']? - ( - [A-Za-z0-9+/]{32,88}={0,2} - ) | \b (?:hubAccessKey|notificationHub(?:Access)?Key) \b - \s*[:=]\s* - ["']? - ( - [A-Za-z0-9+/]{32,88}={0,2} - ) + ) + \s*[:=]\s* + ["']? + ( + [A-Za-z0-9+/]{32,88}={0,2} ) ["']? (?:[^A-Za-z0-9+/=]|$) @@ -149,8 +144,7 @@ rules: {%- assign se = "" | unix_timestamp | plus: 300 -%} {%- capture to_sign -%}{{ uri | url_encode }} {{ se }}{%- endcapture -%} - {%- assign key_bytes = TOKEN | b64dec -%} - {%- capture auth -%}SharedAccessSignature sr={{ uri | url_encode }}&sig={{ to_sign | hmac_sha256: key_bytes | url_encode }}&se={{ se }}&skn={{ NH_KEY_NAME | url_encode }}{%- endcapture -%} + {%- capture auth -%}SharedAccessSignature sr={{ uri | url_encode }}&sig={{ to_sign | hmac_sha256_b64key: TOKEN | url_encode }}&se={{ se }}&skn={{ NH_KEY_NAME | url_encode }}{%- endcapture -%} {{ auth | strip_newlines }} response_matcher: - report_response: true diff --git a/crates/kingfisher-rules/src/liquid_filters.rs b/crates/kingfisher-rules/src/liquid_filters.rs index 16a8e8b..351678b 100644 --- a/crates/kingfisher-rules/src/liquid_filters.rs +++ b/crates/kingfisher-rules/src/liquid_filters.rs @@ -182,6 +182,45 @@ impl Filter for HmacSha256Filter { } } +// ── HMAC-SHA256 with base64-encoded key ────────────────────────────────── +#[derive(Debug, FilterParameters)] +struct HmacB64KeyArgs { + #[parameter(description = "Base64-encoded HMAC key", arg_type = "str")] + key: Expression, +} + +#[derive(Clone, ParseFilter, FilterReflection, Default)] +#[filter( + name = "hmac_sha256_b64key", + description = "HMAC-SHA256 with a base64-encoded key – decodes the key to raw bytes before signing. Returns Base64.", + parameters(HmacB64KeyArgs), + parsed(HmacSha256B64KeyFilter) +)] +pub struct HmacSha256B64Key; + +#[derive(Debug, FromFilterParameters, Display_filter)] +#[name = "hmac_sha256_b64key"] +struct HmacSha256B64KeyFilter { + #[parameters] + args: HmacB64KeyArgs, +} + +impl Filter for HmacSha256B64KeyFilter { + fn evaluate(&self, input: &dyn ValueView, runtime: &dyn Runtime) -> Result { + let args = self.args.evaluate(runtime)?; + let key_b64 = args.key.to_kstr(); + + let key_bytes = general_purpose::STANDARD.decode(key_b64.as_bytes()).map_err(|e| { + LiquidError::with_msg(format!("hmac_sha256_b64key: invalid base64 key: {e}")) + })?; + + let mut mac = Hmac::::new_from_slice(&key_bytes) + .map_err(|e| LiquidError::with_msg(format!("hmac_sha256_b64key: {e}")))?; + mac.update(input.to_kstr().as_bytes()); + Ok(Value::scalar(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))) + } +} + // ── HMAC-SHA1 ───────────────────────────────────────────── #[derive(Debug, FilterParameters)] struct HmacSha1Args { @@ -923,6 +962,7 @@ pub fn register_all(builder: liquid::ParserBuilder) -> liquid::ParserBuilder { .filter(Base62Filter::default()) .filter(Base36Filter::default()) .filter(HmacSha256::default()) + .filter(HmacSha256B64Key::default()) .filter(HmacSha1::default()) .filter(HmacSha384::default()) } @@ -1073,6 +1113,21 @@ mod tests { assert_eq!(render(r#"{{ "hi!" | hmac_sha256: "secret" }}"#), expect); } + #[test] + fn hmac_sha256_b64key_filter() { + // Key is base64-encoded; the filter must decode it to raw bytes before HMAC. + let raw_key: &[u8] = &[0x00, 0x80, 0xFF, 0x42, 0xDE, 0xAD, 0xBE, 0xEF]; + let b64_key = general_purpose::STANDARD.encode(raw_key); + + let data = b"hello azure"; + let mut mac = Hmac::::new_from_slice(raw_key).unwrap(); + mac.update(data); + let expect = general_purpose::STANDARD.encode(mac.finalize().into_bytes()); + + let template = format!(r#"{{{{ "hello azure" | hmac_sha256_b64key: "{b64_key}" }}}}"#); + assert_eq!(render(&template), expect); + } + #[test] fn hmac_sha384_filter() { let key = b"topsecret"; From 8da61ab553e81d9812a5c173ef070a450d3c423b Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 29 Mar 2026 17:22:23 -0700 Subject: [PATCH 14/18] fixed github actions --- crates/kingfisher-rules/data/rules/azure-notification-hub.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/kingfisher-rules/data/rules/azure-notification-hub.yml b/crates/kingfisher-rules/data/rules/azure-notification-hub.yml index 6b40af1..be7975f 100644 --- a/crates/kingfisher-rules/data/rules/azure-notification-hub.yml +++ b/crates/kingfisher-rules/data/rules/azure-notification-hub.yml @@ -142,8 +142,8 @@ rules: Authorization: | {%- assign uri = "https://" | append: NH_HOST | append: "/" | append: NH_HUB | append: "/registrations/?api-version=2015-01" -%} {%- assign se = "" | unix_timestamp | plus: 300 -%} - {%- capture to_sign -%}{{ uri | url_encode }} - {{ se }}{%- endcapture -%} + {%- assign to_sign = uri | url_encode | append: ' +' | append: se -%} {%- capture auth -%}SharedAccessSignature sr={{ uri | url_encode }}&sig={{ to_sign | hmac_sha256_b64key: TOKEN | url_encode }}&se={{ se }}&skn={{ NH_KEY_NAME | url_encode }}{%- endcapture -%} {{ auth | strip_newlines }} response_matcher: From 49d980acb09407a79fa5f9fe2d7011e92d0d8d95 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 29 Mar 2026 17:29:33 -0700 Subject: [PATCH 15/18] fixed github actions --- .github/workflows/release.yml | 8 +++++++- crates/kingfisher-rules/data/rules/etsy.yml | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9cf1000..af21408 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -438,7 +438,13 @@ jobs: run: | set -euo pipefail cd assets - echo "hashes=$(sha256sum -- * | base64 -w0)" >> "$GITHUB_OUTPUT" + shopt -s nullglob + files=( * ) + if [ ${#files[@]} -eq 0 ]; then + echo "Error: no release assets found to hash in $(pwd)" >&2 + exit 1 + fi + echo "hashes=$(sha256sum -- "${files[@]}" | base64 -w0)" >> "$GITHUB_OUTPUT" provenance: name: Generate SLSA provenance diff --git a/crates/kingfisher-rules/data/rules/etsy.yml b/crates/kingfisher-rules/data/rules/etsy.yml index 7b89bca..b617efd 100644 --- a/crates/kingfisher-rules/data/rules/etsy.yml +++ b/crates/kingfisher-rules/data/rules/etsy.yml @@ -41,7 +41,7 @@ rules: method: GET url: https://api.etsy.com/v3/application/openapi-ping headers: - x-api-key: '{{ TOKEN }}' + x-api-key: '{{ TOKEN | split: ":" | first }}' Accept: application/json response_matcher: - report_response: true From 9c448eec6008fd924029eee4e068438542d4efc1 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 29 Mar 2026 17:36:40 -0700 Subject: [PATCH 16/18] fixed github actions --- .github/workflows/release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index af21408..6a4fb41 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -444,7 +444,8 @@ jobs: echo "Error: no release assets found to hash in $(pwd)" >&2 exit 1 fi - echo "hashes=$(sha256sum -- "${files[@]}" | base64 -w0)" >> "$GITHUB_OUTPUT" + hashes=$(sha256sum -- "${files[@]}" | base64 -w0) + echo "hashes=${hashes}" >> "$GITHUB_OUTPUT" provenance: name: Generate SLSA provenance From ba30b1788fd0c59038369ed41c52db5aa1b5a27e Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 29 Mar 2026 18:24:18 -0700 Subject: [PATCH 17/18] fixed github actions --- .../data/rules/azure-notification-hub.yml | 4 ++-- crates/kingfisher-rules/src/liquid_filters.rs | 9 +++++++++ docs/LIBRARY.md | 6 +++--- docs/RULES.md | 3 +++ 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/crates/kingfisher-rules/data/rules/azure-notification-hub.yml b/crates/kingfisher-rules/data/rules/azure-notification-hub.yml index be7975f..28ca7bc 100644 --- a/crates/kingfisher-rules/data/rules/azure-notification-hub.yml +++ b/crates/kingfisher-rules/data/rules/azure-notification-hub.yml @@ -142,8 +142,8 @@ rules: Authorization: | {%- assign uri = "https://" | append: NH_HOST | append: "/" | append: NH_HUB | append: "/registrations/?api-version=2015-01" -%} {%- assign se = "" | unix_timestamp | plus: 300 -%} - {%- assign to_sign = uri | url_encode | append: ' -' | append: se -%} + {%- assign nl = "" | newline -%} + {%- assign to_sign = uri | url_encode | append: nl | append: se -%} {%- capture auth -%}SharedAccessSignature sr={{ uri | url_encode }}&sig={{ to_sign | hmac_sha256_b64key: TOKEN | url_encode }}&se={{ se }}&skn={{ NH_KEY_NAME | url_encode }}{%- endcapture -%} {{ auth | strip_newlines }} response_matcher: diff --git a/crates/kingfisher-rules/src/liquid_filters.rs b/crates/kingfisher-rules/src/liquid_filters.rs index 351678b..77fba6e 100644 --- a/crates/kingfisher-rules/src/liquid_filters.rs +++ b/crates/kingfisher-rules/src/liquid_filters.rs @@ -478,6 +478,14 @@ impl Filter for B64DecFilter { } } +// {{ "any" | newline }} → "\n" (appends nothing, just returns a newline character) +static_filter!( + /// Returns a single newline character. Useful inside YAML block scalars where + /// a literal newline in the template would break indentation. + NewlineFilter, "newline", + |_input: &dyn ValueView| -> String { "\n".to_string() } +); + // ----------------------------------------------------------------------------- // Authentication & Security // ----------------------------------------------------------------------------- @@ -951,6 +959,7 @@ pub fn register_all(builder: liquid::ParserBuilder) -> liquid::ParserBuilder { .filter(JwtHeaderFilter::default()) .filter(B64EncFilter::default()) .filter(B64DecFilter::default()) + .filter(NewlineFilter::default()) .filter(RandomStringFilter::default()) .filter(SuffixFilter::default()) .filter(PrefixFilter::default()) diff --git a/docs/LIBRARY.md b/docs/LIBRARY.md index 313af28..b4aa0bb 100644 --- a/docs/LIBRARY.md +++ b/docs/LIBRARY.md @@ -315,10 +315,10 @@ let template = parser.parse("{{ secret | sha256 }}")?; Available filters: - **Encoding**: `b64enc`, `b64dec`, `b64url_enc`, `url_encode`, `json_escape` -- **Hashing**: `sha256`, `crc32`, `crc32_dec`, `crc32_hex` -- **HMAC**: `hmac_sha256`, `hmac_sha384`, `hmac_sha1` +- **Hashing**: `sha256`, `crc32`, `crc32_dec`, `crc32_hex`, `crc32_le_b64` +- **HMAC**: `hmac_sha256`, `hmac_sha384`, `hmac_sha1`, `hmac_sha256_b64key` - **Encoding**: `base62`, `base36` -- **Strings**: `prefix`, `suffix`, `replace`, `lstrip_chars`, `random_string` +- **Strings**: `prefix`, `suffix`, `replace`, `lstrip_chars`, `random_string`, `newline` - **Time**: `unix_timestamp`, `iso_timestamp`, `iso_timestamp_no_frac` - **Other**: `uuid`, `jwt_header` diff --git a/docs/RULES.md b/docs/RULES.md index c806aa2..5fe9bad 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -468,6 +468,7 @@ Below is the complete list of Liquid filters available in Kingfisher, along with | `hmac_sha1` | `key` (string) | Computes HMAC-SHA1 over the input, returns Base64-encoded result. | `{{ TOKEN \| hmac_sha1: "secret-key" }}` | | `hmac_sha256` | `key` (string) | Computes HMAC-SHA256 over the input, returns Base64-encoded result. | `{{ TOKEN \| hmac_sha256: "secret-key" }}` | | `hmac_sha384` | `key` (string) | Computes HMAC-SHA384 over the input, returns Base64-encoded result. | `{{ TOKEN \| hmac_sha384: "secret-key" }}` | +| `hmac_sha256_b64key` | `key` (string, base64-encoded) | Decodes the key from Base64 to raw bytes, then computes HMAC-SHA256. Returns Base64. Use for Azure SAS and other protocols where the signing key is base64-encoded. | `{{ to_sign \| hmac_sha256_b64key: TOKEN }}` | | `random_string` | `len` (integer, optional) | Generates a cryptographically-secure random alphanumeric string of the specified length (default: 32). | `{{ "" \| random_string: 16 }}` | | `prefix` | `len` (integer, optional) | Returns the first `len` characters from the string (default: full). | `{{ TOKEN \| prefix: 6 }}` | | `suffix` | `len` (integer, optional) | Returns the last `len` characters from the string (default: full). | `{{ TOKEN \| suffix: 6 }}` | @@ -480,6 +481,8 @@ Below is the complete list of Liquid filters available in Kingfisher, along with | `uuid` | – | Generates a random UUIDv4 string. | `{{ "" \| uuid }}` | | `jwt_header` | – | Builds a minimal JWT header JSON (`{"typ":"JWT","alg":…}`) and Base64URL-encodes it. | `{{ "HS256" \| jwt_header }}` | | `replace` | `from` (string), `to` (string) | Replaces every occurrence of `from` with `to` in the input string. | `{{ "hello world" \| replace: "world", "mars" }}` | +| `newline` | – | Returns a single newline character (`\n`). Useful inside YAML block scalars where a literal newline would break indentation. | `{{ "" \| newline }}` | +| `base36` | `width` (integer, optional) | Encodes the input number as Base36, left-padding with zeros as needed. | `{{ TOKEN \| crc32 \| base36: 6 }}` | **Chaining & Composition:** Filters can be stacked; e.g.: From 4d5ce57a12650ec54c41b909f8623a1d395aa0a9 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Sun, 29 Mar 2026 00:16:28 -0700 Subject: [PATCH 18/18] feat(gitea): add --clone-url-base flag for clone URL rewriting When scanning a self-hosted Gitea/Forgejo instance, the API may be reachable at a different hostname than the git clone endpoint (e.g., internal API vs. public clone URL behind a reverse proxy). The --clone-url-base flag rewrites the scheme, host, and port of clone URLs returned by the API, preserving the path. Example: kingfisher scan gitea \ --api-url https://forge.internal.example.com/api/v1/ \ --clone-url-base https://forge.internal.example.com/ \ --user eblume This avoids routing clone traffic through an external proxy when the API and git endpoints share the same internal host but the instance's ROOT_URL points to the public endpoint. Includes unit tests for the URL rewriting function and an integration test using wiremock to verify the full enumeration path. --- src/cli/commands/inputs.rs | 4 ++ src/cli/commands/scan.rs | 14 +++++- src/direct_validate.rs | 1 + src/gitea.rs | 45 ++++++++++++++++- src/main.rs | 4 +- src/reporter.rs | 1 + src/reporter/json_format.rs | 1 + src/scanner/repos.rs | 2 + tests/int_gitea_clone_url_base.rs | 84 +++++++++++++++++++++++++++++++ 9 files changed, 153 insertions(+), 3 deletions(-) create mode 100644 tests/int_gitea_clone_url_base.rs diff --git a/src/cli/commands/inputs.rs b/src/cli/commands/inputs.rs index 95d3cbd..0fa2d61 100644 --- a/src/cli/commands/inputs.rs +++ b/src/cli/commands/inputs.rs @@ -182,6 +182,10 @@ pub struct InputSpecifierArgs { )] pub gitea_api_url: Url, + /// Override base URL for cloning Gitea repositories + #[arg(long, value_hint = ValueHint::Url, hide = true)] + pub gitea_clone_url_base: Option, + #[arg(long, default_value_t = GiteaRepoType::Source, hide = true)] pub gitea_repo_type: GiteaRepoType, diff --git a/src/cli/commands/scan.rs b/src/cli/commands/scan.rs index c61fc93..ab76967 100644 --- a/src/cli/commands/scan.rs +++ b/src/cli/commands/scan.rs @@ -284,7 +284,7 @@ pub enum ScanOperation { pub enum ListRepositoriesCommand { Github { api_url: Url, specifiers: GitHubRepoSpecifiers }, Gitlab { api_url: Url, specifiers: GitLabRepoSpecifiers }, - Gitea { api_url: Url, specifiers: GiteaRepoSpecifiers }, + Gitea { api_url: Url, clone_url_base: Option, specifiers: GiteaRepoSpecifiers }, Bitbucket { api_url: Url, specifiers: BitbucketRepoSpecifiers }, Azure { base_url: Url, specifiers: AzureRepoSpecifiers }, Huggingface { specifiers: HuggingFaceRepoSpecifiers }, @@ -396,6 +396,7 @@ impl ScanCommandArgs { if args.list_only { Some(ListRepositoriesCommand::Gitea { api_url: args.api_url, + clone_url_base: args.clone_url_base, specifiers: args.specifiers, }) } else { @@ -408,6 +409,8 @@ impl ScanCommandArgs { args.specifiers.all_organizations; scan_args.input_specifier_args.gitea_repo_type = args.specifiers.repo_type; scan_args.input_specifier_args.gitea_api_url = args.api_url; + scan_args.input_specifier_args.gitea_clone_url_base = + args.clone_url_base; None } } @@ -741,6 +744,15 @@ pub struct GiteaScanArgs { value_hint = ValueHint::Url )] pub api_url: Url, + + /// Override the base URL used for cloning repositories. + /// + /// By default, clone URLs returned by the Gitea/Forgejo API are used as-is. + /// When the API is reachable at a different hostname than the git clone + /// endpoint (e.g., internal API vs. public clone URL), use this flag to + /// rewrite the scheme, host, and port of clone URLs. + #[arg(long = "clone-url-base", value_hint = ValueHint::Url)] + pub clone_url_base: Option, } #[derive(Args, Debug, Clone)] diff --git a/src/direct_validate.rs b/src/direct_validate.rs index 5d68d22..e8ca71e 100644 --- a/src/direct_validate.rs +++ b/src/direct_validate.rs @@ -919,6 +919,7 @@ pub(crate) fn create_minimal_scan_args() -> crate::cli::commands::scan::ScanArgs gitea_exclude: Vec::new(), all_gitea_organizations: false, gitea_api_url: Url::parse("https://gitea.com/api/v1/").unwrap(), + gitea_clone_url_base: None, gitea_repo_type: GiteaRepoType::Source, bitbucket_user: Vec::new(), bitbucket_workspace: Vec::new(), diff --git a/src/gitea.rs b/src/gitea.rs index d748685..898b813 100644 --- a/src/gitea.rs +++ b/src/gitea.rs @@ -212,6 +212,7 @@ async fn fetch_authenticated_orgs( pub async fn enumerate_repo_urls( specifiers: &RepoSpecifiers, api_url: Url, + clone_url_base: Option<&Url>, ignore_certs: bool, mut progress: Option<&mut ProgressBar>, ) -> Result> { @@ -291,6 +292,14 @@ pub async fn enumerate_repo_urls( } } + // Rewrite clone URLs if a custom base was provided. + if let Some(base) = clone_url_base { + repos = repos + .into_iter() + .map(|raw| rewrite_clone_url(&raw, base).unwrap_or(raw)) + .collect(); + } + repos.sort(); repos.dedup(); Ok(repos) @@ -298,6 +307,7 @@ pub async fn enumerate_repo_urls( pub async fn list_repositories( api_url: Url, + clone_url_base: Option<&Url>, ignore_certs: bool, progress_enabled: bool, users: &[String], @@ -324,7 +334,7 @@ pub async fn list_repositories( exclude_repos: exclude_repos.to_vec(), }; - let urls = enumerate_repo_urls(&specifiers, api_url, ignore_certs, Some(&mut progress)).await?; + let urls = enumerate_repo_urls(&specifiers, api_url, clone_url_base, ignore_certs, Some(&mut progress)).await?; for url in urls { println!("{}", url); } @@ -332,6 +342,15 @@ pub async fn list_repositories( Ok(()) } +/// Rewrite a clone URL to use a different base (scheme, host, port), preserving the path. +fn rewrite_clone_url(raw: &str, base: &Url) -> Option { + let mut parsed = Url::parse(raw).ok()?; + parsed.set_scheme(base.scheme()).ok()?; + parsed.set_host(base.host_str()).ok()?; + parsed.set_port(base.port()).ok()?; + Some(parsed.to_string()) +} + fn parse_repo(repo_url: &GitUrl) -> Option<(String, String, String)> { let url = Url::parse(repo_url.as_str()).ok()?; let host = url.host_str()?.to_string(); @@ -371,4 +390,28 @@ mod tests { fn normalize_repo_identifier_handles_git_suffix() { assert_eq!(normalize_repo_identifier("owner/repo.git"), Some("owner/repo".into())); } + + #[test] + fn rewrite_clone_url_changes_host() { + let base = Url::parse("https://forge.internal.example.com/").unwrap(); + assert_eq!( + rewrite_clone_url("https://forge.public.example.com/owner/repo.git", &base), + Some("https://forge.internal.example.com/owner/repo.git".to_string()) + ); + } + + #[test] + fn rewrite_clone_url_changes_port() { + let base = Url::parse("https://forge.example.com:3000/").unwrap(); + assert_eq!( + rewrite_clone_url("https://forge.example.com/owner/repo.git", &base), + Some("https://forge.example.com:3000/owner/repo.git".to_string()) + ); + } + + #[test] + fn rewrite_clone_url_returns_none_for_invalid_url() { + let base = Url::parse("https://forge.example.com/").unwrap(); + assert_eq!(rewrite_clone_url("not-a-url", &base), None); + } } diff --git a/src/main.rs b/src/main.rs index 769d8fe..e62010b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -368,9 +368,10 @@ async fn async_main(args: CommandLineArgs) -> Result<()> { ) .await?; } - ListRepositoriesCommand::Gitea { api_url, specifiers } => { + ListRepositoriesCommand::Gitea { api_url, clone_url_base, specifiers } => { gitea::list_repositories( api_url, + clone_url_base.as_ref(), global_args.ignore_certs, global_args.use_progress(), &specifiers.user, @@ -506,6 +507,7 @@ fn create_default_scan_args() -> cli::commands::scan::ScanArgs { gitea_exclude: Vec::new(), all_gitea_organizations: false, gitea_api_url: Url::parse("https://gitea.com/api/v1/").unwrap(), + gitea_clone_url_base: None, gitea_repo_type: GiteaRepoType::Source, bitbucket_user: Vec::new(), diff --git a/src/reporter.rs b/src/reporter.rs index b01c6df..8734450 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -1743,6 +1743,7 @@ mod tests { gitea_exclude: Vec::new(), all_gitea_organizations: false, gitea_api_url: Url::parse("https://gitea.com/api/v1/").unwrap(), + gitea_clone_url_base: None, gitea_repo_type: GiteaRepoType::Source, bitbucket_user: Vec::new(), bitbucket_workspace: Vec::new(), diff --git a/src/reporter/json_format.rs b/src/reporter/json_format.rs index c5b3500..06cf949 100644 --- a/src/reporter/json_format.rs +++ b/src/reporter/json_format.rs @@ -119,6 +119,7 @@ mod tests { gitea_exclude: Vec::new(), all_gitea_organizations: false, gitea_api_url: Url::parse("https://gitea.com/api/v1/").unwrap(), + gitea_clone_url_base: None, gitea_repo_type: GiteaRepoType::Source, // Bitbucket diff --git a/src/scanner/repos.rs b/src/scanner/repos.rs index 3b354b0..8468f43 100644 --- a/src/scanner/repos.rs +++ b/src/scanner/repos.rs @@ -423,9 +423,11 @@ pub async fn enumerate_gitea_repos( let mut num_found: u64 = 0; let api_url = args.input_specifier_args.gitea_api_url.clone(); + let clone_url_base = args.input_specifier_args.gitea_clone_url_base.as_ref(); let repo_strings = gitea::enumerate_repo_urls( &repo_specifiers, api_url, + clone_url_base, global_args.ignore_certs, Some(&mut progress), ) diff --git a/tests/int_gitea_clone_url_base.rs b/tests/int_gitea_clone_url_base.rs new file mode 100644 index 0000000..6ae968e --- /dev/null +++ b/tests/int_gitea_clone_url_base.rs @@ -0,0 +1,84 @@ +// tests/int_gitea_clone_url_base.rs +// +// Integration test: verify that --clone-url-base rewrites clone URLs +// returned by the Gitea API during repository enumeration. +// +// Uses wiremock to mock the Gitea API and assert_cmd to exercise the full +// CLI path: argument parsing → API enumeration → URL rewriting → output. + +use assert_cmd::Command; +use predicates::str::contains; +use wiremock::{ + matchers::{method, path, query_param}, + Mock, MockServer, ResponseTemplate, +}; + +/// Run `kingfisher scan gitea --list-only` against a mock Gitea API with and +/// without --clone-url-base, verifying that clone URLs are rewritten. +#[tokio::test] +async fn clone_url_base_rewrites_listed_urls() { + let mock_server = MockServer::start().await; + + let public_host = "https://forge.public.example.com"; + let repo_json = serde_json::json!([{ + "full_name": "eblume/kingfisher", + "clone_url": format!("{public_host}/eblume/kingfisher.git"), + "fork": false + }]); + + // Page 1: return one repo. + Mock::given(method("GET")) + .and(path("/api/v1/users/eblume/repos")) + .and(query_param("page", "1")) + .respond_with(ResponseTemplate::new(200).set_body_json(&repo_json)) + .mount(&mock_server) + .await; + + // Page 2: return empty array to terminate pagination. + Mock::given(method("GET")) + .and(path("/api/v1/users/eblume/repos")) + .and(query_param("page", "2")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([]))) + .mount(&mock_server) + .await; + + let api_url = format!("{}/api/v1/", mock_server.uri()); + + // WITH --clone-url-base: URLs should be rewritten. + Command::new(assert_cmd::cargo::cargo_bin!("kingfisher")) + .args([ + "scan", + "gitea", + "--api-url", + &api_url, + "--clone-url-base", + "https://forge.internal.example.com/", + "--user", + "eblume", + "--list-only", + "--no-update-check", + "--quiet", + ]) + .assert() + .success() + .stdout(contains("https://forge.internal.example.com/eblume/kingfisher.git")); + + // WITHOUT --clone-url-base: URLs should be unchanged. + Command::new(assert_cmd::cargo::cargo_bin!("kingfisher")) + .args([ + "scan", + "gitea", + "--api-url", + &api_url, + "--user", + "eblume", + "--list-only", + "--no-update-check", + "--quiet", + ]) + .assert() + .success() + .stdout(contains(&format!( + "{public_host}/eblume/kingfisher.git" + ))); +}