From 5f7d82a524217e1766e9dbee277fb1bee3865881 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 5 Apr 2026 16:36:08 -0700 Subject: [PATCH 01/17] fix github action --- .github/workflows/release.yml | 26 +++------------------ .gitignore | 1 + docs-site/docs/assets/stylesheets/extra.css | 3 +++ docs-site/docs/rules/builtin-rules.md | 12 +++++----- docs-site/overrides/home.html | 9 +++++++ 5 files changed, 22 insertions(+), 29 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8fb9341..7d7ee6a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -411,7 +411,7 @@ jobs: provenance: name: Generate SLSA provenance - needs: [hash] + needs: [hash, release] permissions: actions: read id-token: write @@ -419,28 +419,8 @@ jobs: uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@f7dd8c54c2067bafc12ca7a55595d5ee9b75204a # v2.1.0 with: base64-subjects: "${{ needs.hash.outputs.hashes }}" - upload-assets: false - - upload-provenance: - name: Upload provenance to release - needs: [provenance, release] - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - name: Download provenance artifact - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: ${{ needs.provenance.outputs.provenance-name }} - - name: Upload to release - env: - GH_TOKEN: ${{ github.token }} - TAG: ${{ needs.release.outputs.tag }} - PROVENANCE_FILE: ${{ needs.provenance.outputs.provenance-name }} - run: | - gh release upload "${TAG}" "${PROVENANCE_FILE}" \ - --repo "${{ github.repository }}" \ - --clobber + upload-assets: true + upload-tag-name: "${{ needs.release.outputs.tag }}" # ──────────────── Publish Docker image ──────────────── publish-docker: diff --git a/.gitignore b/.gitignore index 1fa6b8b..1490502 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ logs/* *.rej *.html !docs/access-map-viewer/index.html +!docs-site/overrides/*.html *.dot fuzz/* !fuzz/Cargo.toml diff --git a/docs-site/docs/assets/stylesheets/extra.css b/docs-site/docs/assets/stylesheets/extra.css index b3df9bd..c590f93 100644 --- a/docs-site/docs/assets/stylesheets/extra.css +++ b/docs-site/docs/assets/stylesheets/extra.css @@ -99,6 +99,7 @@ max-width: 700px; margin: 0 auto 2rem; color: var(--md-default-fg-color--light); + font-size: 1rem; line-height: 1.6; } @@ -127,11 +128,13 @@ .kf-feature h3 { margin-top: 0; + font-size: 1.3rem; color: var(--md-primary-fg-color); } .kf-feature p { color: var(--md-default-fg-color--light); + font-size: 0.85rem; line-height: 1.6; margin-bottom: 0; } diff --git a/docs-site/docs/rules/builtin-rules.md b/docs-site/docs/rules/builtin-rules.md index f165577..6951c17 100644 --- a/docs-site/docs/rules/builtin-rules.md +++ b/docs-site/docs/rules/builtin-rules.md @@ -2485,7 +2485,7 @@ Of these, **462** include live validation and **43** support direct revocation. Hashicorp -Hashicorp Vault Service Token (< v1.10) +Hashicorp Vault Service Token (< v1.10) kingfisher.hashicorp.1 Medium @@ -2493,7 +2493,7 @@ Of these, **462** include live validation and **43** support direct revocation. Hashicorp -Hashicorp Vault Batch Token (< v1.10) +Hashicorp Vault Batch Token (< v1.10) kingfisher.hashicorp.2 Medium @@ -2501,7 +2501,7 @@ Of these, **462** include live validation and **43** support direct revocation. Hashicorp -Hashicorp Vault Recovery Token (< v1.10) +Hashicorp Vault Recovery Token (< v1.10) kingfisher.hashicorp.3 Medium @@ -2509,7 +2509,7 @@ Of these, **462** include live validation and **43** support direct revocation. Hashicorp -Hashicorp Vault Service Token (>= v1.10) +Hashicorp Vault Service Token (>= v1.10) kingfisher.hashicorp.4 Medium @@ -2517,7 +2517,7 @@ Of these, **462** include live validation and **43** support direct revocation. Hashicorp -Hashicorp Vault Batch Token (>= v1.10) +Hashicorp Vault Batch Token (>= v1.10) kingfisher.hashicorp.5 Medium @@ -2525,7 +2525,7 @@ Of these, **462** include live validation and **43** support direct revocation. Hashicorp -Hashicorp Vault Recovery Token (>= v1.10) +Hashicorp Vault Recovery Token (>= v1.10) kingfisher.hashicorp.6 Medium diff --git a/docs-site/overrides/home.html b/docs-site/overrides/home.html index bcbb144..28bc386 100644 --- a/docs-site/overrides/home.html +++ b/docs-site/overrides/home.html @@ -95,6 +95,15 @@

+
+

Open Source

+

+ Apache 2.0 licensed. Free to use, modify, and distribute. No vendor + lock-in, no usage limits, no telemetry. Fully auditable codebase + backed by MongoDB. +

+
+

Built for Accuracy

From 45a565fa6e24c16b2d62b69b4ef70f64e611f007 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Mon, 6 Apr 2026 22:18:58 -0700 Subject: [PATCH 02/17] added more rules --- AGENTS.md | 15 +- CHANGELOG.md | 5 + Cargo.lock | 359 ++++++++-- Cargo.toml | 2 +- README.md | 12 +- crates/kingfisher-rules/data/rules/AGENTS.md | 7 +- crates/kingfisher-rules/data/rules/agora.yml | 24 +- .../data/rules/amazonoauth.yml | 19 + crates/kingfisher-rules/data/rules/asaas.yml | 18 + crates/kingfisher-rules/data/rules/azure.yml | 22 + .../kingfisher-rules/data/rules/azureapim.yml | 64 ++ .../data/rules/azurebatch.yml | 71 ++ .../data/rules/azurecognitive.yml | 46 ++ .../data/rules/azurecommunication.yml | 20 + .../data/rules/azurecosmosdb.yml | 77 +++ .../data/rules/azureeventgrid.yml | 28 + .../data/rules/azurefunctionkey.yml | 56 ++ .../data/rules/azurelogicapps.yml | 22 + .../kingfisher-rules/data/rules/azuremaps.yml | 21 + .../data/rules/azuremixedreality.yml | 28 + .../data/rules/azuresastoken.yml | 50 ++ .../data/rules/azuresignalr.yml | 20 + .../kingfisher-rules/data/rules/azuresql.yml | 46 ++ .../data/rules/azurewebpubsub.yml | 20 + .../kingfisher-rules/data/rules/bitfinex.yml | 29 +- .../kingfisher-rules/data/rules/bitrise.yml | 35 + .../data/rules/blockprotocol.yml | 19 + crates/kingfisher-rules/data/rules/canva.yml | 20 + crates/kingfisher-rules/data/rules/cfxre.yml | 19 + .../data/rules/cockroachlabs.yml | 28 + crates/kingfisher-rules/data/rules/docker.yml | 40 +- .../kingfisher-rules/data/rules/docusign.yml | 104 ++- .../kingfisher-rules/data/rules/dropbox.yml | 35 + crates/kingfisher-rules/data/rules/dwolla.yml | 47 +- crates/kingfisher-rules/data/rules/ebay.yml | 58 ++ .../kingfisher-rules/data/rules/elastic.yml | 48 ++ .../data/rules/flutterwave.yml | 23 +- crates/kingfisher-rules/data/rules/ftp.yml | 21 +- crates/kingfisher-rules/data/rules/gitlab.yml | 261 ++++++++ .../kingfisher-rules/data/rules/hcaptcha.yml | 24 + .../kingfisher-rules/data/rules/highnote.yml | 20 + crates/kingfisher-rules/data/rules/hop.yml | 38 ++ .../kingfisher-rules/data/rules/iterative.yml | 19 + crates/kingfisher-rules/data/rules/kraken.yml | 39 ++ crates/kingfisher-rules/data/rules/kucoin.yml | 57 ++ crates/kingfisher-rules/data/rules/ldap.yml | 33 +- .../kingfisher-rules/data/rules/lichess.yml | 31 + .../data/rules/localstack.yml | 21 + .../data/rules/mailersend.yml | 32 + crates/kingfisher-rules/data/rules/onfido.yml | 32 + .../kingfisher-rules/data/rules/openvsx.yml | 19 + crates/kingfisher-rules/data/rules/paddle.yml | 34 + crates/kingfisher-rules/data/rules/pangea.yml | 34 + .../kingfisher-rules/data/rules/persona.yml | 34 + .../kingfisher-rules/data/rules/pinterest.yml | 50 ++ crates/kingfisher-rules/data/rules/polar.yml | 20 + crates/kingfisher-rules/data/rules/proof.yml | 22 + .../kingfisher-rules/data/rules/rabbitmq.yml | 23 +- .../data/rules/rainforestpay.yml | 21 + crates/kingfisher-rules/data/rules/redis.yml | 19 +- .../data/rules/ringcentral.yml | 74 ++- crates/kingfisher-rules/data/rules/rootly.yml | 31 + crates/kingfisher-rules/data/rules/runpod.yml | 36 ++ .../kingfisher-rules/data/rules/snowflake.yml | 73 +++ .../kingfisher-rules/data/rules/tableau.yml | 86 ++- crates/kingfisher-rules/data/rules/telnyx.yml | 31 + .../data/rules/thunderstore.yml | 19 + crates/kingfisher-rules/data/rules/trello.yml | 54 ++ .../kingfisher-rules/data/rules/ubidots.yml | 24 +- .../kingfisher-rules/data/rules/valtown.yml | 19 + .../data/rules/volcengine.yml | 19 + crates/kingfisher-rules/data/rules/webex.yml | 53 +- crates/kingfisher-rules/src/liquid_filters.rs | 101 ++- crates/kingfisher-scanner/Cargo.toml | 22 +- .../src/validation/http_validation.rs | 58 ++ .../kingfisher-scanner/src/validation/mod.rs | 10 +- .../kingfisher-scanner/src/validation/raw.rs | 612 ++++++++++++++++++ data/default/rule_cleanup/count_rules.py | 25 +- docs-site/docs/reference/library.md | 2 +- docs-site/docs/usage/advanced.md | 2 +- docs/ADVANCED.md | 2 +- docs/ARCHITECTURE.md | 3 +- docs/LIBRARY.md | 16 +- docs/RULES.md | 34 +- src/direct_validate.rs | 55 +- src/reporter.rs | 10 +- src/validation.rs | 46 +- src/validation_rate_limit.rs | 5 +- 88 files changed, 3824 insertions(+), 159 deletions(-) create mode 100644 crates/kingfisher-rules/data/rules/amazonoauth.yml create mode 100644 crates/kingfisher-rules/data/rules/asaas.yml create mode 100644 crates/kingfisher-rules/data/rules/azureapim.yml create mode 100644 crates/kingfisher-rules/data/rules/azurebatch.yml create mode 100644 crates/kingfisher-rules/data/rules/azurecognitive.yml create mode 100644 crates/kingfisher-rules/data/rules/azurecommunication.yml create mode 100644 crates/kingfisher-rules/data/rules/azurecosmosdb.yml create mode 100644 crates/kingfisher-rules/data/rules/azureeventgrid.yml create mode 100644 crates/kingfisher-rules/data/rules/azurefunctionkey.yml create mode 100644 crates/kingfisher-rules/data/rules/azurelogicapps.yml create mode 100644 crates/kingfisher-rules/data/rules/azuremaps.yml create mode 100644 crates/kingfisher-rules/data/rules/azuremixedreality.yml create mode 100644 crates/kingfisher-rules/data/rules/azuresastoken.yml create mode 100644 crates/kingfisher-rules/data/rules/azuresignalr.yml create mode 100644 crates/kingfisher-rules/data/rules/azuresql.yml create mode 100644 crates/kingfisher-rules/data/rules/azurewebpubsub.yml create mode 100644 crates/kingfisher-rules/data/rules/bitrise.yml create mode 100644 crates/kingfisher-rules/data/rules/blockprotocol.yml create mode 100644 crates/kingfisher-rules/data/rules/canva.yml create mode 100644 crates/kingfisher-rules/data/rules/cfxre.yml create mode 100644 crates/kingfisher-rules/data/rules/cockroachlabs.yml create mode 100644 crates/kingfisher-rules/data/rules/ebay.yml create mode 100644 crates/kingfisher-rules/data/rules/elastic.yml create mode 100644 crates/kingfisher-rules/data/rules/hcaptcha.yml create mode 100644 crates/kingfisher-rules/data/rules/highnote.yml create mode 100644 crates/kingfisher-rules/data/rules/hop.yml create mode 100644 crates/kingfisher-rules/data/rules/iterative.yml create mode 100644 crates/kingfisher-rules/data/rules/lichess.yml create mode 100644 crates/kingfisher-rules/data/rules/localstack.yml create mode 100644 crates/kingfisher-rules/data/rules/mailersend.yml create mode 100644 crates/kingfisher-rules/data/rules/onfido.yml create mode 100644 crates/kingfisher-rules/data/rules/openvsx.yml create mode 100644 crates/kingfisher-rules/data/rules/paddle.yml create mode 100644 crates/kingfisher-rules/data/rules/pangea.yml create mode 100644 crates/kingfisher-rules/data/rules/persona.yml create mode 100644 crates/kingfisher-rules/data/rules/pinterest.yml create mode 100644 crates/kingfisher-rules/data/rules/polar.yml create mode 100644 crates/kingfisher-rules/data/rules/proof.yml create mode 100644 crates/kingfisher-rules/data/rules/rainforestpay.yml create mode 100644 crates/kingfisher-rules/data/rules/rootly.yml create mode 100644 crates/kingfisher-rules/data/rules/runpod.yml create mode 100644 crates/kingfisher-rules/data/rules/telnyx.yml create mode 100644 crates/kingfisher-rules/data/rules/thunderstore.yml create mode 100644 crates/kingfisher-rules/data/rules/valtown.yml create mode 100644 crates/kingfisher-rules/data/rules/volcengine.yml create mode 100644 crates/kingfisher-scanner/src/validation/raw.rs diff --git a/AGENTS.md b/AGENTS.md index 0f6d292..917c3f7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,7 +22,6 @@ Key capabilities: ## Repository Structure - `src/`: main binary source - `src/cli/commands/`: CLI command implementations -- `src/validation/`: provider-specific credential validators - `src/matcher/`: pattern matching engine - `src/scanner/`: core scanning logic - `src/parser/`: language-aware parsing (`tree-sitter`) @@ -32,6 +31,7 @@ Key capabilities: - `crates/kingfisher-rules/`: rule loading and rule data - `crates/kingfisher-rules/data/rules/`: YAML detection rules - `crates/kingfisher-scanner/`: embeddable high-level scanning API +- `crates/kingfisher-scanner/src/validation/`: shared typed and raw credential validators - `tests/`: integration/e2e tests - `testdata/`: test fixtures - `docs/`: user and developer docs @@ -81,18 +81,21 @@ Key capabilities: - `use-mimalloc` (default) - `use-jemalloc` - `system-alloc` -- Validation modules live in `crates/kingfisher-scanner/src/validation/`; optional validation feature sets are defined in `crates/kingfisher-scanner/Cargo.toml` (e.g., `validation-aws`, `validation-gcp`, `validation-database`, `validation-all`). +- Validation modules live in `crates/kingfisher-scanner/src/validation/`; optional validation feature sets are defined in `crates/kingfisher-scanner/Cargo.toml` (e.g., `validation-raw`, `validation-aws`, `validation-gcp`, `validation-database`, `validation-all`). ## Validation and Revocation Policy -- Default rule: define validation logic in rule YAML (`validation:` block), not Rust code. -- Code-based validation in `crates/kingfisher-scanner/src/validation/` is an exception path for cases that cannot be expressed reliably in YAML alone (for example AWS, GCP, Coinbase, MongoDB, and similar complex/provider-specific flows). +- Default rule: define validation logic in rule YAML (`validation:` block), especially `Http` or `Grpc`, not Rust code. +- Typed validators are first-class schema variants (`AWS`, `AzureStorage`, `Coinbase`, `GCP`, `MongoDB`, `MySQL`, `Postgres`, `Jdbc`, `JWT`) for stable, reusable validation families. +- Raw validators use `validation: { type: Raw, content: }` and are the ad-hoc exception path for provider-specific or protocol-specific validation that cannot be expressed reliably in YAML alone. Implement them in `crates/kingfisher-scanner/src/validation/raw.rs`. - Treat Rust validation additions as rare; prefer extending YAML-based validation first. +- If a Rust exception path is required, prefer adding a raw validator before introducing a new typed validator. Add a new typed validator only when it represents a reusable schema-level validation family. +- Do not convert existing typed validators to `Raw` just for consistency. - For rules that include validation, add a `revocation:` section whenever the third-party API safely supports revocation. ## Common Development Tasks - Add a detection rule: follow the workflow below and validate with relevant tests. - Add a CLI command: implement under `src/cli/commands/` and register in the CLI command wiring. -- Add a validator (rare exception path): implement in `crates/kingfisher-scanner/src/validation/` and wire feature flags/dependencies in `crates/kingfisher-scanner/Cargo.toml` only when YAML validation cannot express the required logic. +- Add a validator (rare exception path): implement it in `crates/kingfisher-scanner/src/validation/`, prefer `raw.rs` for one-off provider flows, and wire the narrowest feature/dependencies in `crates/kingfisher-scanner/Cargo.toml` only when YAML validation cannot express the required logic. ## Rule Authoring Workflow Use this when creating or updating rules in `crates/kingfisher-rules/data/rules/`. @@ -105,7 +108,7 @@ Use this when creating or updating rules in `crates/kingfisher-rules/data/rules/ - `pattern_requirements` (e.g., `min_digits`, `min_uppercase`, `min_lowercase`, `min_special_chars`, `ignore_if_contains`) when format constraints are known. - `pattern_requirements.checksum` when provider formats include check digits/signatures. 5. Add `validation` only when a reliable provider/API check exists. -6. Put validation in YAML by default; only use Rust validator logic for rare, justified exceptions. +6. Put validation in YAML by default. If YAML cannot express the check, use an existing typed validator or `type: Raw` exception path; add new Rust validator logic only for rare, justified cases. 7. Add `revocation` when the provider API supports safe revocation and the flow is well understood. 8. If a rule needs context from another match (for example ID + secret pair), use `depends_on_rule` and consider `visible: false` on the helper rule. 9. Verify locally: diff --git a/CHANGELOG.md b/CHANGELOG.md index 6921720..6f507c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. +## [v1.95.0] +- Added 80+ built-in rules, bringing the bundled ruleset to 820 total. New coverage includes Amazon OAuth, Asaas, multiple Azure credential families, Bitrise, Canva, CockroachDB, eBay, Elastic, hCaptcha, Highnote, Lichess, MailerSend, Onfido, Paddle, Pangea, Persona, Pinterest, Proof, Rootly, Runpod, Telnyx, Thunderstore, Valtown, Volcengine, and more. +- Added a `validation: type: Raw` exception path for provider-specific checks, with new raw validators for Azure Batch, FTP, Kraken, LDAP, RabbitMQ, and Redis. Also added stable request-scoped template values plus new Liquid filters for HMAC-SHA384 hex output and timestamp generation. +- Expanded live validation coverage for several built-in rules, including Agora, Bitfinex, DocuSign, Dwolla, GitLab, KuCoin, RingCentral, Snowflake, Tableau, Trello, and Webex, and fixed newly added rule patterns/examples so `kingfisher rules check` passes cleanly. + ## [v1.94.0] - Updated vendored `vectorscan-rs` from v0.0.5 (Vectorscan 5.4.11) to v0.0.6 (Vectorscan 5.4.12). The upstream crate now ships pre-extracted sources instead of a tarball+patch, and fixes the `cpu_native` feature flag. Local Windows and musl build patches have been re-applied. - Added more built-in rules diff --git a/Cargo.lock b/Cargo.lock index 475da1d..54f32a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -191,6 +191,45 @@ dependencies = [ "wax", ] +[[package]] +name = "asn1-rs" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom 7.1.3", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure 0.12.6", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -875,11 +914,11 @@ dependencies = [ "hyper-rustls", "hyper-util", "pin-project-lite", - "rustls", - "rustls-native-certs", + "rustls 0.23.37", + "rustls-native-certs 0.8.3", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tower", "tracing", ] @@ -1215,8 +1254,8 @@ dependencies = [ "num", "pin-project-lite", "rand 0.9.2", - "rustls", - "rustls-native-certs", + "rustls 0.23.37", + "rustls-native-certs 0.8.3", "rustls-pki-types", "serde", "serde_derive", @@ -1768,6 +1807,16 @@ dependencies = [ "url", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -1826,7 +1875,7 @@ dependencies = [ "crc", "digest 0.10.7", "rustversion", - "spin", + "spin 0.10.0", ] [[package]] @@ -2143,6 +2192,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der-parser" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom 7.1.3", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "der_derive" version = "0.7.3" @@ -2863,7 +2926,7 @@ dependencies = [ "regex", "reqwest 0.13.2", "reqwest-middleware 0.5.1", - "ring", + "ring 0.17.14", "serde", "serde_json", "sha2 0.10.9", @@ -4197,7 +4260,7 @@ dependencies = [ "ipnet", "once_cell", "rand 0.9.2", - "ring", + "ring 0.17.14", "thiserror 2.0.18", "tinyvec", "tokio", @@ -4415,11 +4478,11 @@ dependencies = [ "http 1.4.0", "hyper", "hyper-util", - "rustls", - "rustls-native-certs", + "rustls 0.23.37", + "rustls-native-certs 0.8.3", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tower-service", "webpki-roots 1.0.6", ] @@ -4959,7 +5022,7 @@ dependencies = [ "base64 0.22.1", "js-sys", "pem", - "ring", + "ring 0.17.14", "serde", "serde_json", "simple_asn1", @@ -4993,7 +5056,7 @@ dependencies = [ [[package]] name = "kingfisher" -version = "1.94.0" +version = "1.95.0" dependencies = [ "anyhow", "asar", @@ -5089,12 +5152,12 @@ dependencies = [ "reqwest 0.12.28", "reqwest-middleware 0.4.2", "reqwest-middleware 0.5.1", - "ring", + "ring 0.17.14", "roaring", "rusqlite", "rustc-hash", - "rustls", - "rustls-native-certs", + "rustls 0.23.37", + "rustls-native-certs 0.8.3", "schemars 0.8.22", "self_update", "semver", @@ -5122,7 +5185,7 @@ dependencies = [ "tokio", "tokio-postgres", "tokio-postgres-rustls", - "tokio-rustls", + "tokio-rustls 0.26.4", "toon-format", "tracing", "tracing-core", @@ -5241,6 +5304,7 @@ dependencies = [ "jsonwebtoken 10.3.0", "kingfisher-core", "kingfisher-rules", + "ldap3", "liquid", "liquid-core", "mongodb", @@ -5255,10 +5319,10 @@ dependencies = [ "rand 0.10.0", "regex", "reqwest 0.12.28", - "ring", + "ring 0.17.14", "rustc-hash", - "rustls", - "rustls-native-certs", + "rustls 0.23.37", + "rustls-native-certs 0.8.3", "schemars 0.8.22", "serde", "serde_json", @@ -5268,9 +5332,11 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "thread_local", + "time", "tokio", "tokio-postgres", "tokio-postgres-rustls", + "tokio-rustls 0.26.4", "tracing", "url", "vectorscan-rs", @@ -5293,6 +5359,43 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lber" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2df7f9fd9f64cf8f59e1a4a0753fe7d575a5b38d3d7ac5758dcee9357d83ef0a" +dependencies = [ + "bytes", + "nom 7.1.3", +] + +[[package]] +name = "ldap3" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "166199a8207874a275144c8a94ff6eed5fcbf5c52303e4d9b4d53a0c7ac76554" +dependencies = [ + "async-trait", + "bytes", + "futures", + "futures-util", + "lazy_static", + "lber", + "log", + "nom 7.1.3", + "percent-encoding", + "ring 0.16.20", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", + "thiserror 1.0.69", + "tokio", + "tokio-rustls 0.24.1", + "tokio-stream", + "tokio-util", + "url", + "x509-parser", +] + [[package]] name = "leb128fmt" version = "0.1.0" @@ -5710,7 +5813,7 @@ dependencies = [ "percent-encoding", "rand 0.9.2", "rustc_version_runtime", - "rustls", + "rustls 0.23.37", "rustversion", "serde", "serde_bytes", @@ -5723,7 +5826,7 @@ dependencies = [ "take_mut", "thiserror 2.0.18", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tokio-util", "typed-builder", "uuid", @@ -5778,14 +5881,14 @@ dependencies = [ "pem", "percent-encoding", "rand 0.9.2", - "rustls", - "rustls-pemfile", + "rustls 0.23.37", + "rustls-pemfile 2.2.0", "serde", "serde_json", "socket2 0.5.10", "thiserror 2.0.18", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tokio-util", "twox-hash", "url", @@ -6103,7 +6206,7 @@ dependencies = [ "reqwest-middleware 0.4.2", "reqwest-retry", "reqwest-tracing", - "ring", + "ring 0.17.14", "schemars 0.8.22", "serde", "serde_json", @@ -6114,6 +6217,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "oid-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" +dependencies = [ + "asn1-rs", +] + [[package]] name = "olpc-cjson" version = "0.1.4" @@ -6141,6 +6253,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + [[package]] name = "openssl-probe" version = "0.2.1" @@ -6759,7 +6877,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls", + "rustls 0.23.37", "socket2 0.6.3", "thiserror 2.0.18", "tokio", @@ -6778,9 +6896,9 @@ dependencies = [ "getrandom 0.3.4", "lru-slab", "rand 0.9.2", - "ring", + "ring 0.17.14", "rustc-hash", - "rustls", + "rustls 0.23.37", "rustls-pki-types", "slab", "thiserror 2.0.18", @@ -7051,15 +7169,15 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", - "rustls-native-certs", + "rustls 0.23.37", + "rustls-native-certs 0.8.3", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tokio-util", "tower", "tower-http", @@ -7097,7 +7215,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", + "rustls 0.23.37", "rustls-pki-types", "rustls-platform-verifier", "serde", @@ -7105,7 +7223,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tokio-util", "tower", "tower-http", @@ -7222,6 +7340,21 @@ dependencies = [ "subtle", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted 0.7.1", + "web-sys", + "winapi", +] + [[package]] name = "ring" version = "0.17.14" @@ -7302,6 +7435,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom 7.1.3", +] + [[package]] name = "rustix" version = "1.1.4" @@ -7315,6 +7457,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring 0.17.14", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.37" @@ -7324,23 +7478,44 @@ dependencies = [ "aws-lc-rs", "log", "once_cell", - "ring", + "ring 0.17.14", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.10", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe 0.1.6", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework 2.11.1", +] + [[package]] name = "rustls-native-certs" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" dependencies = [ - "openssl-probe", + "openssl-probe 0.2.1", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.7.0", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", ] [[package]] @@ -7368,16 +7543,16 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" dependencies = [ - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "jni 0.21.1", "log", "once_cell", - "rustls", - "rustls-native-certs", + "rustls 0.23.37", + "rustls-native-certs 0.8.3", "rustls-platform-verifier-android", - "rustls-webpki", - "security-framework", + "rustls-webpki 0.103.10", + "security-framework 3.7.0", "security-framework-sys", "webpki-root-certs", "windows-sys 0.61.2", @@ -7389,6 +7564,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring 0.17.14", + "untrusted 0.9.0", +] + [[package]] name = "rustls-webpki" version = "0.103.10" @@ -7396,7 +7581,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ "aws-lc-rs", - "ring", + "ring 0.17.14", "rustls-pki-types", "untrusted 0.9.0", ] @@ -7534,6 +7719,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring 0.17.14", + "untrusted 0.9.0", +] + [[package]] name = "sec1" version = "0.7.3" @@ -7548,6 +7743,19 @@ dependencies = [ "zeroize", ] +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.11.0", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -7555,7 +7763,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags 2.11.0", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -7995,6 +8203,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "spin" version = "0.10.0" @@ -8182,6 +8396,18 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -8622,21 +8848,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27d684bad428a0f2481f42241f821db42c54e2dc81d8c00db8536c506b0a0144" dependencies = [ "const-oid 0.9.6", - "ring", - "rustls", + "ring 0.17.14", + "rustls 0.23.37", "tokio", "tokio-postgres", - "tokio-rustls", + "tokio-rustls 0.26.4", "x509-cert", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls", + "rustls 0.23.37", "tokio", ] @@ -9273,7 +9509,7 @@ dependencies = [ "flate2", "log", "percent-encoding", - "rustls", + "rustls 0.23.37", "rustls-pki-types", "serde", "serde_json", @@ -9637,7 +9873,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe985f41e291eecef5e5c0770a18d28390addb03331c043964d9e916453d6f16" dependencies = [ - "core-foundation", + "core-foundation 0.10.1", "jni 0.22.4", "log", "ndk-context", @@ -10307,6 +10543,23 @@ dependencies = [ "tls_codec", ] +[[package]] +name = "x509-parser" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7069fba5b66b9193bd2c5d3d4ff12b839118f6bcbef5328efafafb5395cf63da" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom 7.1.3", + "oid-registry", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + [[package]] name = "xattr" version = "1.6.1" @@ -10355,7 +10608,7 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.117", - "synstructure", + "synstructure 0.13.2", ] [[package]] @@ -10396,7 +10649,7 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.117", - "synstructure", + "synstructure 0.13.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 37e6d19..e4f8b78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,7 @@ http = "1.4" [package] name = "kingfisher" -version = "1.94.0" +version = "1.95.0" description = "MongoDB's blazingly fast and accurate secret scanning and validation tool" edition.workspace = true rust-version.workspace = true diff --git a/README.md b/README.md index decb35a..5e1aad6 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ License - Detection Rules + Detection Rules
@@ -17,7 +17,7 @@ Kingfisher is an open source secret scanner and **live secret validation** tool built in Rust. -It combines Intel's SIMD-accelerated regex engine (Hyperscan) with language-aware parsing to achieve high accuracy at massive scale, and **ships with 700+ built-in rules** to detect, **validate**, and triage leaked API keys, tokens, and credentials before they ever reach production. +It combines Intel's SIMD-accelerated regex engine (Hyperscan) with language-aware parsing to achieve high accuracy at massive scale, and **ships with 800+ built-in rules** to detect, **validate**, and triage leaked API keys, tokens, and credentials before they ever reach production. Designed for offensive security engineers and blue-team defenders alike, Kingfisher helps you scan repositories, cloud storage, chat, docs, and CI pipelines to find and verify exposed secrets quickly. @@ -49,9 +49,9 @@ Kingfisher is a high-performance, open source secret detection tool for source c

-### Performance, Accuracy, and 700+ Rules +### Performance, Accuracy, and 800+ Rules - **Performance**: multithreaded, Hyperscan‑powered scanning built for huge codebases -- **Extensible rules**: 700+ built-in rules plus YAML-defined custom rules ([docs/RULES.md](/docs/RULES.md)) +- **Extensible rules**: 800+ built-in rules plus YAML-defined custom rules ([docs/RULES.md](/docs/RULES.md)) - **Validate & Revoke**: live validation of discovered secrets, plus direct revocation for supported platforms (GitHub, GitLab, Slack, AWS, GCP, and more) ([docs/USAGE.md](/docs/USAGE.md)) - **Revocation support matrix**: current built-in revocation coverage across providers and rule IDs ([docs/REVOCATION_PROVIDERS.md](/docs/REVOCATION_PROVIDERS.md)) - **Blast Radius Mapping**: instantly map leaked keys to their effective cloud identities and exposed resources with `--access-map`. Supports 39 providers (see table below). @@ -345,7 +345,7 @@ gh attestation verify kingfisher-linux-x64.tgz --repo mongodb/kingfisher # Detection Rules -Kingfisher ships with [700+ built-in rules](crates/kingfisher-rules/data/rules/) covering cloud keys, AI tokens, CI/CD secrets, database credentials, and SaaS API keys. Below is an overview — see the full list in [crates/kingfisher-rules/data/rules/](crates/kingfisher-rules/data/rules/): +Kingfisher ships with [800+ built-in rules](crates/kingfisher-rules/data/rules/) covering cloud keys, AI tokens, CI/CD secrets, database credentials, and SaaS API keys. Below is an overview — see the full list in [crates/kingfisher-rules/data/rules/](crates/kingfisher-rules/data/rules/): | Category | What we catch | |----------|---------------| @@ -362,7 +362,7 @@ Kingfisher ships with [700+ built-in rules](crates/kingfisher-rules/data/rules/) ## Write Custom Rules -Kingfisher ships with 700+ rules with HTTP and service‑specific validation checks (AWS, Azure, GCP, etc.) to confirm if a detected string is a live credential. +Kingfisher ships with 800+ rules with HTTP and service‑specific validation checks (AWS, Azure, GCP, etc.) to confirm if a detected string is a live credential. However, you may want to add your own custom rules, or modify a detection to better suit your needs / environment. diff --git a/crates/kingfisher-rules/data/rules/AGENTS.md b/crates/kingfisher-rules/data/rules/AGENTS.md index cfc2b77..433d951 100644 --- a/crates/kingfisher-rules/data/rules/AGENTS.md +++ b/crates/kingfisher-rules/data/rules/AGENTS.md @@ -57,8 +57,11 @@ Strongly recommended fields: ## Validation Policy (Important) - Default: define validation logic in YAML under `validation:`. - Do not move validation logic into Rust unless YAML cannot reliably express it. -- Code-backed validation types (for example AWS, GCP, Coinbase, MongoDB) are notable exceptions and should remain rare. - For new rules, first attempt `Http`/`Grpc` YAML validation before considering exception paths. +- Typed validation kinds such as `AWS`, `AzureStorage`, `Coinbase`, `GCP`, `MongoDB`, `MySQL`, `Postgres`, `Jdbc`, and `JWT` are schema-level validator families. Use them when an existing typed validator already matches the problem. +- `validation: { type: Raw, content: }` is the ad-hoc exception path for provider-specific or protocol-specific flows that cannot be expressed cleanly in YAML. Raw implementations live in `crates/kingfisher-scanner/src/validation/raw.rs`. +- When Rust validation is unavoidable for a one-off provider, prefer adding a raw validator instead of inventing a new typed validator. +- Do not convert existing typed validators to `Raw` just for consistency. ## Revocation Policy - If a rule has validation and the provider API safely supports revocation, add `revocation:` in the same YAML rule. @@ -70,7 +73,7 @@ Strongly recommended fields: 1. Choose the target provider file (or add a new provider file if no suitable file exists). 2. Copy a structurally similar rule from this directory. 3. Implement/adjust `pattern`, `examples`, and filtering (`pattern_requirements`, `min_entropy`). -4. Add YAML `validation` (default path). +4. Add YAML `validation` (default path). Prefer `Http`/`Grpc`; if that fails, use an existing typed validator or `type: Raw` only when justified. 5. Add YAML `revocation` when supported. 6. Add `references` for token format/API behavior. 7. Verify locally (below). diff --git a/crates/kingfisher-rules/data/rules/agora.yml b/crates/kingfisher-rules/data/rules/agora.yml index 61e1edf..2a328dc 100644 --- a/crates/kingfisher-rules/data/rules/agora.yml +++ b/crates/kingfisher-rules/data/rules/agora.yml @@ -47,6 +47,28 @@ rules: examples: - "agora.app_certificate=397a3af3db1950bdbd84f4e4ec18ebef" - "agora.app_secret = \"127a3af3db1950b8dbd4fe440c28ebef\"" + validation: + type: Http + content: + request: + method: GET + url: https://api.agora.io/dev/v1/projects + headers: + Accept: application/json + Authorization: "Basic {{ AGORA_ID | append: ':' | append: TOKEN | b64enc }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: false + words: + - '"projects"' + - '"vendor_key"' + depends_on_rule: + - rule_id: kingfisher.agora.1 + variable: AGORA_ID references: + - https://docs.agora.io/en/voice-calling/reference/agora-console-rest-api - https://docs.agora.io/en/rtc/restfulapi - - https://docs.agora.io/en/video-calling/reference/authentication-workflow diff --git a/crates/kingfisher-rules/data/rules/amazonoauth.yml b/crates/kingfisher-rules/data/rules/amazonoauth.yml new file mode 100644 index 0000000..b83242b --- /dev/null +++ b/crates/kingfisher-rules/data/rules/amazonoauth.yml @@ -0,0 +1,19 @@ +rules: + - name: Login with Amazon OAuth Client ID + id: kingfisher.amazonoauth.1 + pattern: | + (?x) + \b + ( + amzn1\.application-oa2-client\.[a-f0-9]{20,40} + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.0 + confidence: high + categories: [api, key] + examples: + - 'AMAZON_CLIENT_ID=amzn1.application-oa2-client.1a2b3c4d5e6f7890abcdef1234567890' + references: + - https://developer.amazon.com/docs/login-with-amazon/authorization-code-grant.html diff --git a/crates/kingfisher-rules/data/rules/asaas.yml b/crates/kingfisher-rules/data/rules/asaas.yml new file mode 100644 index 0000000..3b06e7b --- /dev/null +++ b/crates/kingfisher-rules/data/rules/asaas.yml @@ -0,0 +1,18 @@ +rules: + - name: Asaas API Token + id: kingfisher.asaas.1 + pattern: | + (?x) + ( + \$aact_(?:prod|hmlg)_[a-zA-Z0-9_-]{20,100} + ) + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'ASAAS_API_KEY=$aact_prod_abcdefghijklmnop1234567890ABCDEF' + - 'api_token: $aact_hmlg_abcdefghijklmnop1234567890ABCDEF' + references: + - https://docs.asaas.com/docs/authentication-2 diff --git a/crates/kingfisher-rules/data/rules/azure.yml b/crates/kingfisher-rules/data/rules/azure.yml index 45fb88d..f909148 100644 --- a/crates/kingfisher-rules/data/rules/azure.yml +++ b/crates/kingfisher-rules/data/rules/azure.yml @@ -114,3 +114,25 @@ rules: variable: ACR_USERNAME references: - https://learn.microsoft.com/en-us/azure/container-registry/container-registry-authentication + + - name: Azure AD Client Secret (Microsoft Entra ID) + id: kingfisher.azure.6 + pattern: | + (?x) + (?:^|['"\x60\s>=:(,]) + ( + [a-zA-Z0-9_~.]{3} + \d + Q~ + [a-zA-Z0-9_~.\-]{31,34} + ) + (?:$|['"\x60\s<),]) + pattern_requirements: + min_digits: 1 + min_entropy: 3.5 + confidence: medium + examples: + - '"aBc4Q~xY9kLmNpQrStUvWxYz01234567890abcd"' + - 'AZURE_CLIENT_SECRET=xY14Q~abcdefghijklmnopqrstuvwxyz01234' + references: + - https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app diff --git a/crates/kingfisher-rules/data/rules/azureapim.yml b/crates/kingfisher-rules/data/rules/azureapim.yml new file mode 100644 index 0000000..8ebe0da --- /dev/null +++ b/crates/kingfisher-rules/data/rules/azureapim.yml @@ -0,0 +1,64 @@ +rules: + - name: Azure API Management Subscription Key + id: kingfisher.azureapim.1 + pattern: | + (?x) + \b + (?: + (?i:(?:apim|api[_\s-]*management)[_\s-]*(?:subscription[_\s-]*key|key)) + | + (?i:Ocp-Apim-Subscription-Key) + ) + (?:.|[\n\r]){0,16}? + ( + [a-f0-9]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 2 + min_entropy: 3.5 + confidence: medium + categories: [api, key] + examples: + - 'Ocp-Apim-Subscription-Key: 1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d' + - 'APIM_SUBSCRIPTION_KEY=abcdef0123456789abcdef0123456789' + validation: + type: Http + content: + request: + method: GET + url: "{{ APIM_URL }}" + headers: + Ocp-Apim-Subscription-Key: "{{ TOKEN }}" + response_is_html: true + response_matcher: + - report_response: true + - type: StatusMatch + status: [200, 201, 202, 204, 400, 404, 405] + - type: StatusMatch + status: [401, 403] + negative: true + depends_on_rule: + - rule_id: kingfisher.azureapim.2 + variable: APIM_URL + references: + - https://learn.microsoft.com/en-us/azure/api-management/api-management-subscriptions + + - name: Azure API Management Gateway URL + id: kingfisher.azureapim.2 + pattern: | + (?xi) + \b + ( + https://[a-z0-9-]+(?:\.developer)?\.azure-api\.net(?:/[^\s"'<>]{0,200})? + ) + min_entropy: 1.0 + confidence: medium + visible: false + examples: + - https://contoso.azure-api.net/echo + - APIM_URL=https://contoso.developer.azure-api.net/api + references: + - https://learn.microsoft.com/en-us/azure/api-management/api-management-subscriptions + - https://learn.microsoft.com/en-us/troubleshoot/azure/api-mgmt/availability/unauthorized-errors-invoke-apis diff --git a/crates/kingfisher-rules/data/rules/azurebatch.yml b/crates/kingfisher-rules/data/rules/azurebatch.yml new file mode 100644 index 0000000..df34ba2 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/azurebatch.yml @@ -0,0 +1,71 @@ +rules: + - name: Azure Batch Account Key + id: kingfisher.azurebatch.1 + pattern: | + (?x) + \b + (?: + (?i:azure[_\s-]*batch[_\s-]*(?:key|account[_\s-]*key|access[_\s-]*key)) + | + (?i:batch[_\s-]*account[_\s-]*key) + ) + (?:.|[\n\r]){0,16}? + ( + [A-Za-z0-9+/]{86}== + ) + pattern_requirements: + min_digits: 2 + min_uppercase: 2 + min_lowercase: 2 + min_special_chars: 1 + min_entropy: 4.0 + confidence: medium + categories: [api, key] + examples: + - 'AZURE_BATCH_KEY=oqb4TdY9T0hphvktd5fJnMiHuQqzVy1jd5sSuOpAbGkaoqTlrHl0BOJN2okcasinVLOJzfDbZo1L+ASt68RAhA==' + validation: + type: Http + content: + request: + method: GET + url: '{{ BATCH_URL }}/applications?api-version=2020-09-01.12.0' + headers: + Accept: application/json + Content-Type: application/json + Date: '{{ REQUEST_RFC1123_DATE }}' + Authorization: | + {%- assign host = BATCH_URL | split: "://" | last | split: "/" | first -%} + {%- assign account_name = host | split: "." | first -%} + {%- assign resource_path = "/" | append: account_name | append: "/applications" | downcase -%} + {%- assign string_to_sign = "GET\n\n\n\n\napplication/json\n" | append: REQUEST_RFC1123_DATE | append: "\n\n\n\n\n\n" | append: resource_path | append: "\napi-version:2020-09-01.12.0" -%} + {%- assign signature = string_to_sign | hmac_sha256_b64key: TOKEN -%} + SharedKey {{ account_name }}:{{ signature }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + depends_on_rule: + - rule_id: kingfisher.azurebatch.2 + variable: BATCH_URL + references: + - https://learn.microsoft.com/en-us/azure/batch/batch-account-create-portal + - https://learn.microsoft.com/en-us/rest/api/batchservice/authenticate-requests-to-the-azure-batch-service + + - name: Azure Batch Account Endpoint + id: kingfisher.azurebatch.2 + pattern: | + (?xi) + \b + ( + https://[a-z0-9-]+\.[a-z0-9-]+\.batch\.azure\.com + ) + \b + min_entropy: 1.0 + confidence: medium + visible: false + examples: + - BATCH_URL=https://mybatch.westus.batch.azure.com + - batchAccountUrl="https://contoso-prod.eastus.batch.azure.com" + references: + - https://learn.microsoft.com/en-us/rest/api/batchservice/authenticate-requests-to-the-azure-batch-service diff --git a/crates/kingfisher-rules/data/rules/azurecognitive.yml b/crates/kingfisher-rules/data/rules/azurecognitive.yml new file mode 100644 index 0000000..41812a5 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/azurecognitive.yml @@ -0,0 +1,46 @@ +rules: + - name: Azure Cognitive Services / AI Services Key + id: kingfisher.azurecognitive.1 + pattern: | + (?x) + \b + (?: + (?i:azure[_\s-]*(?:cognitive|ai)[_\s-]*(?:service|key)) + | + (?i:cognitive[_\s-]*service[_\s-]*(?:key|secret|subscription)) + | + (?i: + (?:azure[_\s-]*)?(?:anomaly[_\s-]*detector|computer[_\s-]*vision|content[_\s-]*moderator|content[_\s-]*safety + |custom[_\s-]*vision|face[_\s-]*(?:api)?|form[_\s-]*recognizer|document[_\s-]*intelligence + |immersive[_\s-]*reader|language[_\s-]*understanding|luis + |personalizer|qna[_\s-]*maker|text[_\s-]*analytics|video[_\s-]*indexer + |metrics[_\s-]*advisor|health[_\s-]*insights|cognitive[_\s-]*service|ai[_\s-]*service) + [_\s-]*(?:key|api[_\s-]*key|subscription[_\s-]*key|secret) + ) + ) + \b + (?:.|[\n\r]){0,16}? + ( + [a-f0-9]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 2 + min_entropy: 3.5 + confidence: medium + categories: [api, key] + examples: + - AZURE_COGNITIVE_SERVICE_KEY=1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d + - AZURE_COMPUTER_VISION_KEY=abcdef0123456789abcdef0123456789 + - content_moderator_key="1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d" + - AZURE_FACE_KEY=abcdef0123456789abcdef0123456789 + - AZURE_FORM_RECOGNIZER_KEY=1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d + - AZURE_LUIS_KEY=abcdef0123456789abcdef0123456789 + - AZURE_QNA_MAKER_KEY=1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d + - AZURE_TEXT_ANALYTICS_KEY=abcdef0123456789abcdef0123456789 + references: + - https://learn.microsoft.com/en-us/azure/ai-services/ + - https://learn.microsoft.com/en-us/azure/ai-services/computer-vision/ + - https://learn.microsoft.com/en-us/azure/ai-services/content-moderator/ + - https://learn.microsoft.com/en-us/azure/ai-services/luis/ diff --git a/crates/kingfisher-rules/data/rules/azurecommunication.yml b/crates/kingfisher-rules/data/rules/azurecommunication.yml new file mode 100644 index 0000000..ce44bbf --- /dev/null +++ b/crates/kingfisher-rules/data/rules/azurecommunication.yml @@ -0,0 +1,20 @@ +rules: + - name: Azure Communication Services Connection String + id: kingfisher.azurecommunication.1 + pattern: | + (?x) + (?i:endpoint=https://(?:[a-z0-9-]+\.)?(?:communication|comm)\.azure\.com/;accesskey=) + ( + [A-Za-z0-9+/]{40,90}={0,2} + ) + pattern_requirements: + min_digits: 2 + min_uppercase: 2 + min_special_chars: 1 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'endpoint=https://myresource.communication.azure.com/;accesskey=AbCdEfGhIjKlMnOpQrStUvWxYz1234567890ABCDEFGHIJKLMNOPQRS+/==' + references: + - https://learn.microsoft.com/en-us/azure/communication-services/ diff --git a/crates/kingfisher-rules/data/rules/azurecosmosdb.yml b/crates/kingfisher-rules/data/rules/azurecosmosdb.yml new file mode 100644 index 0000000..a531cb1 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/azurecosmosdb.yml @@ -0,0 +1,77 @@ +rules: + - name: Azure CosmosDB Account Key + id: kingfisher.azurecosmosdb.1 + pattern: | + (?x) + \b + (?: + (?i:cosmos(?:db)?[_\s-]*(?:key|account[_\s-]*key|primary[_\s-]*key|secondary[_\s-]*key|master[_\s-]*key)) + | + (?i:azure[_\s-]*cosmos(?:db)?[_\s-]*(?:key|account_key|primary_key|master_key)) + | + (?i:documentdb(?:authkey|key)) + ) + (?:.|[\n\r]){0,16}? + ( + [A-Za-z0-9+/]{86}== + ) + pattern_requirements: + min_digits: 2 + min_uppercase: 2 + min_lowercase: 2 + min_special_chars: 1 + min_entropy: 4.0 + confidence: medium + categories: [api, key] + examples: + - AZURE_COSMOSDB_KEY=oqb4TdY9T0hphvktd5fJnMiHuQqzVy1jd5sSuOpAbGkaoqTlrHl0BOJN2okcasinVLOJzfDbZo1L+ASt68RAhA== + - 'DocumentDbAuthKey=B/1EVX2Ui47X09tqU3GI/j+Nko9r5COPm0Hea9tfzitF9MQX9lZZiNO3tYQckWnt+rtlGIWS+sCx+AStkq8ZLg==' + references: + - https://learn.microsoft.com/en-us/azure/cosmos-db/how-to-obtain-account-keys + + - name: Azure CosmosDB Connection String + id: kingfisher.azurecosmosdb.2 + pattern: | + (?x) + (?i:AccountEndpoint=(?Phttps://[a-z0-9-]+\.documents\.azure\.com(?::\d+)?)/?;) + AccountKey= + (?P + (?P + [A-Za-z0-9+/]{86}== + ) + ) + pattern_requirements: + min_digits: 2 + min_uppercase: 2 + min_special_chars: 1 + min_entropy: 4.0 + confidence: high + categories: [api, key] + examples: + - 'AccountEndpoint=https://myaccount.documents.azure.com:443;AccountKey=oqb4TdY9T0hphvktd5fJnMiHuQqzVy1jd5sSuOpAbGkaoqTlrHl0BOJN2okcasinVLOJzfDbZo1L+ASt68RAhA==;' + validation: + type: Http + content: + request: + method: GET + url: "{{ COSMOS_ENDPOINT }}/dbs" + headers: + Accept: application/json + x-ms-date: '{%- assign x_ms_date = "" | date: "%a, %d %b %Y %H:%M:%S GMT" | downcase -%}{{ x_ms_date }}' + x-ms-version: "2018-12-31" + Authorization: | + {%- assign x_ms_date = "" | date: "%a, %d %b %Y %H:%M:%S GMT" | downcase -%} + {%- assign string_to_sign = "get\ndbs\n\n" | append: x_ms_date | append: "\n\n" -%} + {%- assign signature = string_to_sign | hmac_sha256_b64key: TOKEN | url_encode -%} + type=master&ver=1.0&sig={{ signature }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + words: + - '"Databases"' + references: + - https://learn.microsoft.com/en-us/azure/cosmos-db/how-to-obtain-account-keys + - https://learn.microsoft.com/en-us/rest/api/cosmos-db/access-control-on-cosmosdb-resources diff --git a/crates/kingfisher-rules/data/rules/azureeventgrid.yml b/crates/kingfisher-rules/data/rules/azureeventgrid.yml new file mode 100644 index 0000000..12b1f19 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/azureeventgrid.yml @@ -0,0 +1,28 @@ +rules: + - name: Azure Event Grid Key + id: kingfisher.azureeventgrid.1 + pattern: | + (?x) + \b + (?: + (?i:event[_\s-]*grid[_\s-]*(?:key|access[_\s-]*key|topic[_\s-]*key)) + | + (?i:azure_event_grid[_\s-]*(?:key|access_key|topic_key)) + | + (?i:aeg-sas-key) + ) + (?:.|[\n\r]){0,16}? + ( + [A-Za-z0-9+/]{40,50}={0,2} + ) + pattern_requirements: + min_digits: 2 + min_uppercase: 2 + min_entropy: 3.5 + confidence: medium + categories: [api, key] + examples: + - 'AZURE_EVENT_GRID_KEY=AbCdEfGhIjKlMnOpQrStUvWxYz1234567890ABCDEFG==' + - 'aeg-sas-key: AbCdEfGhIjKlMnOpQrStUvWxYz1234567890ABCDEFG==' + references: + - https://learn.microsoft.com/en-us/azure/event-grid/ diff --git a/crates/kingfisher-rules/data/rules/azurefunctionkey.yml b/crates/kingfisher-rules/data/rules/azurefunctionkey.yml new file mode 100644 index 0000000..f0eddaf --- /dev/null +++ b/crates/kingfisher-rules/data/rules/azurefunctionkey.yml @@ -0,0 +1,56 @@ +rules: + - name: Azure Function Key in URL + id: kingfisher.azurefunctionkey.1 + pattern: | + (?x) + ( + (?i:https://[a-z0-9-]+\.azurewebsites\.net/api/)[a-zA-Z0-9_-]+ + (?i:\?code=)[a-zA-Z0-9_/+=-]{20,100} + ) + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + categories: [api, key] + examples: + - 'https://myfunc.azurewebsites.net/api/HttpTrigger1?code=AbCdEfGhIjKlMnOpQrStUvWxYz1234567890/+==' + validation: + type: Http + content: + request: + method: GET + url: "{{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200, 202, 204, 400, 404, 405] + - type: StatusMatch + status: [401, 403] + negative: true + references: + - https://learn.microsoft.com/en-us/azure/azure-functions/functions-bindings-http-webhook-trigger + + - name: Azure Function Master/Host Key + id: kingfisher.azurefunctionkey.2 + pattern: | + (?x) + \b + (?: + (?i:azure[_\s-]*function[_\s-]*(?:key|master[_\s-]*key|host[_\s-]*key)) + | + (?i:x-functions-key) + ) + (?:.|[\n\r]){0,16}? + ( + [a-zA-Z0-9_/+=-]{40,100} + ) + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + categories: [api, key] + examples: + - 'AZURE_FUNCTION_KEY=AbCdEfGhIjKlMnOpQrStUvWxYz1234567890ABCDEFGH/+==' + - 'x-functions-key: AbCdEfGhIjKlMnOpQrStUvWxYz1234567890ABCDEFGH' + references: + - https://learn.microsoft.com/en-us/azure/azure-functions/functions-bindings-http-webhook-trigger diff --git a/crates/kingfisher-rules/data/rules/azurelogicapps.yml b/crates/kingfisher-rules/data/rules/azurelogicapps.yml new file mode 100644 index 0000000..380fca8 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/azurelogicapps.yml @@ -0,0 +1,22 @@ +rules: + - name: Azure Logic Apps SAS URL + id: kingfisher.azurelogicapps.1 + pattern: | + (?x) + ( + (?i:https://(?:[a-z0-9-]+\.)+logic\.azure\.com) + (?::\d+)?/workflows/[A-Fa-f0-9]+/triggers/[a-zA-Z0-9_-]+/paths/invoke + (?i:\?api-version=)[0-9-]+ + (?i:&sp=)[%a-zA-Z0-9/]+ + (?i:&sv=)[0-9.]+ + (?i:&sig=)[a-zA-Z0-9_/+=-]+ + ) + pattern_requirements: + min_digits: 4 + min_entropy: 3.0 + confidence: high + categories: [api, key] + examples: + - 'https://prod-00.eastus.logic.azure.com/workflows/abcdef1234567890/triggers/manual/paths/invoke?api-version=2016-10-01&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=AbCdEfGhIjKlMnOpQrStUvWxYz123456' + references: + - https://learn.microsoft.com/en-us/azure/logic-apps/logic-apps-securing-a-logic-app diff --git a/crates/kingfisher-rules/data/rules/azuremaps.yml b/crates/kingfisher-rules/data/rules/azuremaps.yml new file mode 100644 index 0000000..177a382 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/azuremaps.yml @@ -0,0 +1,21 @@ +rules: + - name: Azure Maps Subscription Key + id: kingfisher.azuremaps.1 + pattern: | + (?x) + \b + (?i:azure[_\s-]*maps[_\s-]*(?:key|subscription[_\s-]*key|api[_\s-]*key|secret)) + (?:.|[\n\r]){0,16}? + ( + [a-zA-Z0-9_-]{32,44} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + categories: [api, key] + examples: + - AZURE_MAPS_KEY=AbCdEfGhIjKlMnOpQrStUvWxYz123456 + references: + - https://learn.microsoft.com/en-us/azure/azure-maps/how-to-manage-authentication diff --git a/crates/kingfisher-rules/data/rules/azuremixedreality.yml b/crates/kingfisher-rules/data/rules/azuremixedreality.yml new file mode 100644 index 0000000..6694b91 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/azuremixedreality.yml @@ -0,0 +1,28 @@ +rules: + - name: Azure Mixed Reality / Spatial Anchors Key + id: kingfisher.azuremixedreality.1 + pattern: | + (?x) + \b + (?: + (?i: + (?:azure[_\s-]*)?(?:mixed[_\s-]*reality|spatial[_\s-]*anchors?|remote[_\s-]*rendering) + [_\s-]*(?:key|account[_\s-]*key|access[_\s-]*key) + ) + ) + (?:.|[\n\r]){0,16}? + ( + [a-f0-9]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 2 + min_entropy: 3.5 + confidence: medium + categories: [api, key] + examples: + - 'AZURE_SPATIAL_ANCHORS_KEY=1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d' + - 'AZURE_REMOTE_RENDERING_KEY=abcdef0123456789abcdef0123456789' + references: + - https://learn.microsoft.com/en-us/azure/spatial-anchors/ diff --git a/crates/kingfisher-rules/data/rules/azuresastoken.yml b/crates/kingfisher-rules/data/rules/azuresastoken.yml new file mode 100644 index 0000000..448849e --- /dev/null +++ b/crates/kingfisher-rules/data/rules/azuresastoken.yml @@ -0,0 +1,50 @@ +rules: + - name: Azure SAS Token + id: kingfisher.azuresastoken.1 + pattern: | + (?x) + ( + (?i:(?:sv|SharedAccessSignature\s+sr))=[0-9]{4}-[0-9]{2}-[0-9]{2} + (?:&(?i:[a-z]{2,4})=[^&\s"']{1,200}){2,10} + (?i:&sig=)[a-zA-Z0-9%+/=]{20,100} + ) + pattern_requirements: + min_digits: 4 + min_entropy: 3.0 + confidence: medium + categories: [api, key] + examples: + - 'sv=2021-06-08&ss=bfqt&srt=sco&sp=rwdlacupiytfx&se=2024-12-31&st=2024-01-01&spr=https&sig=AbCdEfGhIjKlMnOpQrStUvWxYz1234567890%2BABCDE%3D' + references: + - https://learn.microsoft.com/en-us/azure/storage/common/storage-sas-overview + + - name: Azure SAS Token in URL + id: kingfisher.azuresastoken.2 + pattern: | + (?x) + ( + (?i:https://[a-z0-9-]+\.(?:blob|queue|table|file|dfs)\.core\.windows\.net/)[^\s"']* + \?[^\s"']*(?i:sig=)[a-zA-Z0-9%+/=]{20,100}[^\s"']* + ) + pattern_requirements: + min_digits: 4 + min_entropy: 3.0 + confidence: medium + categories: [api, key] + examples: + - 'https://mystorageaccount.blob.core.windows.net/mycontainer/myblob?sv=2021-06-08&st=2024-01-01&se=2024-12-31&sr=b&sp=r&sig=AbCdEfGhIjKlMnOp%2BQrStUvWxYz%3D' + validation: + type: Http + content: + request: + method: HEAD + url: "{{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200, 206, 404] + - type: StatusMatch + status: [401, 403] + negative: true + references: + - https://learn.microsoft.com/en-us/azure/storage/common/storage-sas-overview diff --git a/crates/kingfisher-rules/data/rules/azuresignalr.yml b/crates/kingfisher-rules/data/rules/azuresignalr.yml new file mode 100644 index 0000000..3b952cc --- /dev/null +++ b/crates/kingfisher-rules/data/rules/azuresignalr.yml @@ -0,0 +1,20 @@ +rules: + - name: Azure SignalR Connection String + id: kingfisher.azuresignalr.1 + pattern: | + (?x) + (?i:Endpoint=https://(?:[a-z0-9-]+\.service\.signalr\.net);AccessKey=) + ( + [A-Za-z0-9+/]{40,90}={0,2} + ) + pattern_requirements: + min_digits: 2 + min_uppercase: 2 + min_special_chars: 1 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'Endpoint=https://myservice.service.signalr.net;AccessKey=AbCdEfGhIjKlMnOpQrStUvWxYz1234567890ABCDEFGHIJKLMNOPQRS+/==;Version=1.0;' + references: + - https://learn.microsoft.com/en-us/azure/azure-signalr/ diff --git a/crates/kingfisher-rules/data/rules/azuresql.yml b/crates/kingfisher-rules/data/rules/azuresql.yml new file mode 100644 index 0000000..e4cb7ba --- /dev/null +++ b/crates/kingfisher-rules/data/rules/azuresql.yml @@ -0,0 +1,46 @@ +rules: + - name: Azure SQL Connection String + id: kingfisher.azuresql.1 + pattern: | + (?x) + (?i:Server=tcp:(?:[a-z0-9-]+\.database\.windows\.net),\d+;) + (?:.|[\n\r]){0,100}? + (?i:Password=)( + [^;'"]{8,128} + ) + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + categories: [api, key] + examples: + - 'Server=tcp:myserver.database.windows.net,1433;Initial Catalog=mydb;Persist Security Info=False;User ID=admin;Password=MyP@ssw0rd!123;' + references: + - https://learn.microsoft.com/en-us/azure/azure-sql/database/connect-query-content-reference-guide + + - name: Azure SQL Password Assignment + id: kingfisher.azuresql.2 + pattern: | + (?x) + \b + (?: + (?i:azure[_\s-]*sql[_\s-]*password) + | + (?i:sql_admin_password) + | + (?i:mssql_sa_password) + ) + \b + (?:.|[\n\r]){0,8}? + [=:] + \s*["']? + ([^\s"']{8,128}) + ["']? + min_entropy: 3.0 + confidence: medium + categories: [password] + examples: + - 'AZURE_SQL_PASSWORD=MyStr0ngP@ssword!' + - 'SQL_ADMIN_PASSWORD="Compl3x!Pass#2024"' + references: + - https://learn.microsoft.com/en-us/azure/azure-sql/database/connect-query-content-reference-guide diff --git a/crates/kingfisher-rules/data/rules/azurewebpubsub.yml b/crates/kingfisher-rules/data/rules/azurewebpubsub.yml new file mode 100644 index 0000000..713c46c --- /dev/null +++ b/crates/kingfisher-rules/data/rules/azurewebpubsub.yml @@ -0,0 +1,20 @@ +rules: + - name: Azure Web PubSub Connection String + id: kingfisher.azurewebpubsub.1 + pattern: | + (?x) + (?i:Endpoint=https://(?:[a-z0-9-]+\.webpubsub\.azure\.com);AccessKey=) + ( + [A-Za-z0-9+/]{40,90}={0,2} + ) + pattern_requirements: + min_digits: 2 + min_uppercase: 2 + min_special_chars: 1 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'Endpoint=https://myservice.webpubsub.azure.com;AccessKey=AbCdEfGhIjKlMnOpQrStUvWxYz1234567890ABCDEFGHIJKLMNOPQRS+/==;Version=1.0;' + references: + - https://learn.microsoft.com/en-us/azure/azure-web-pubsub/ diff --git a/crates/kingfisher-rules/data/rules/bitfinex.yml b/crates/kingfisher-rules/data/rules/bitfinex.yml index d78769f..b79baed 100644 --- a/crates/kingfisher-rules/data/rules/bitfinex.yml +++ b/crates/kingfisher-rules/data/rules/bitfinex.yml @@ -47,9 +47,30 @@ rules: examples: - "bitfinex\nsecret = 8d7c3965318b8d20f7648dbda96fbfa23f4d1c449aa" - "bitfinex\napi-secret = 1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1d2" + validation: + type: Http + content: + request: + method: POST + url: https://api.bitfinex.com/v2/auth/r/wallets + headers: + Accept: application/json + Content-Type: application/json + bfx-apikey: "{{ BITFINEX_KEY }}" + bfx-nonce: "{{ REQUEST_UNIX_MILLIS }}" + bfx-signature: | + {%- assign request_path = "/v2/auth/r/wallets" -%} + {%- assign request_body = "{}" -%} + {%- assign signature_payload = "/api" | append: request_path | append: REQUEST_UNIX_MILLIS | append: request_body -%} + {{ signature_payload | hmac_sha384_hex: TOKEN }} + body: "{}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + depends_on_rule: + - rule_id: kingfisher.bitfinex.1 + variable: BITFINEX_KEY references: - - https://docs.bitfinex.com/docs - https://docs.bitfinex.com/docs/rest-auth - # No simple validation: Bitfinex REST API v2 uses HMAC-SHA384 - # request signing with a nonce and payload. Cannot validate with - # a static Bearer/API-key style header. diff --git a/crates/kingfisher-rules/data/rules/bitrise.yml b/crates/kingfisher-rules/data/rules/bitrise.yml new file mode 100644 index 0000000..b27ce22 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/bitrise.yml @@ -0,0 +1,35 @@ +rules: + - name: Bitrise Personal Access Token + id: kingfisher.bitrise.1 + pattern: | + (?x) + \b + (?i:bitrise) + (?:.|[\n\r]){0,24}? + (?i:token|pat|personal[_\s-]*access) + (?:.|[\n\r]){0,16}? + ( + [a-zA-Z0-9_-]{60,120} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + categories: [api, key] + examples: + - 'BITRISE_TOKEN=AbCdEfGhIjKlMnOpQrStUvWxYz1234567890AbCdEfGhIjKlMnOpQrStUvWxYz12' + validation: + type: Http + content: + request: + method: GET + url: https://api.bitrise.io/v0.1/me + headers: + Authorization: '{{ TOKEN }}' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + references: + - https://devcenter.bitrise.io/en/api/authenticating-with-the-bitrise-api.html diff --git a/crates/kingfisher-rules/data/rules/blockprotocol.yml b/crates/kingfisher-rules/data/rules/blockprotocol.yml new file mode 100644 index 0000000..0797b1e --- /dev/null +++ b/crates/kingfisher-rules/data/rules/blockprotocol.yml @@ -0,0 +1,19 @@ +rules: + - name: Block Protocol API Key + id: kingfisher.blockprotocol.1 + pattern: | + (?x) + \b + ( + b10ck5\.[a-zA-Z0-9]{28,36}\.[a-zA-Z0-9]{32,40} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'BLOCK_PROTOCOL_API_KEY=b10ck5.AbCdEfGhIjKlMnOpQrStUvWxYz1234.AbCdEfGhIjKlMnOpQrStUvWxYz12345678' + references: + - https://blockprotocol.org/docs/hub/api diff --git a/crates/kingfisher-rules/data/rules/canva.yml b/crates/kingfisher-rules/data/rules/canva.yml new file mode 100644 index 0000000..e8d84cb --- /dev/null +++ b/crates/kingfisher-rules/data/rules/canva.yml @@ -0,0 +1,20 @@ +rules: + - name: Canva Connect API Client Secret + id: kingfisher.canva.1 + pattern: | + (?x) + \b + ( + cnvca[a-zA-Z0-9_-]{20,80} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'CANVA_CLIENT_SECRET=cnvcaAbCdEfGhIjKlMnOpQrStUvWxYz123456' + references: + - https://www.canva.dev/docs/connect/authentication/ + - https://www.canva.dev/docs/connect/guidelines/security/ diff --git a/crates/kingfisher-rules/data/rules/cfxre.yml b/crates/kingfisher-rules/data/rules/cfxre.yml new file mode 100644 index 0000000..f0325d2 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/cfxre.yml @@ -0,0 +1,19 @@ +rules: + - name: Cfx.re FiveM Server Key + id: kingfisher.cfxre.1 + pattern: | + (?x) + \b + ( + cfxk_[a-zA-Z0-9_-]{20,100} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'sv_licenseKey "cfxk_AbCdEfGhIjKlMnOpQrStUvWxYz1234567890_abcdef"' + references: + - https://docs.fivem.net/docs/server-manual/setting-up-a-server/ diff --git a/crates/kingfisher-rules/data/rules/cockroachlabs.yml b/crates/kingfisher-rules/data/rules/cockroachlabs.yml new file mode 100644 index 0000000..9fbcc40 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/cockroachlabs.yml @@ -0,0 +1,28 @@ +rules: + - name: CockroachDB Cloud API Key + id: kingfisher.cockroachlabs.1 + pattern: | + (?x) + \b + (?: + (?i:cockroach(?:db)?(?:cloud)?) + (?:.|[\n\r]){0,24}? + (?i:api[_\s-]*key|secret|token) + | + (?i:CC_API_KEY) + ) + (?:.|[\n\r]){0,16}? + ( + [A-Z0-9_]{20,60} + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 4 + min_entropy: 3.0 + confidence: medium + categories: [api, key] + examples: + - 'COCKROACHDB_API_KEY=B81649_8F7D11A_92BCE13_56782D_C53' + references: + - https://www.cockroachlabs.com/docs/cockroachcloud/cloud-api diff --git a/crates/kingfisher-rules/data/rules/docker.yml b/crates/kingfisher-rules/data/rules/docker.yml index 888e717..2ec5055 100644 --- a/crates/kingfisher-rules/data/rules/docker.yml +++ b/crates/kingfisher-rules/data/rules/docker.yml @@ -45,4 +45,42 @@ rules: response_matcher: - report_response: true - type: StatusMatch - status: [200] \ No newline at end of file + status: [200] + + - name: Docker Swarm Join Token + id: kingfisher.docker.2 + pattern: | + (?x) + \b + ( + SWMTKN-1-[a-z0-9]{50,60}-[a-z0-9]{24,30} + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'docker swarm join --token SWMTKN-1-3pu6hszjas19xyp7ghgosyx9k8atbfcr8p2is99znpy26u2lkl-1awxwuwd3z9j1z3puu7rcgdbx 192.168.99.100:2377' + references: + - https://docs.docker.com/engine/swarm/join-nodes/ + + - name: Docker Swarm Unlock Key + id: kingfisher.docker.3 + pattern: | + (?x) + \b + ( + SWMKEY-1-[A-Za-z0-9+/]{40,50} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'docker swarm unlock --key SWMKEY-1-AbCdEfGhIjKlMnOpQrStUvWxYz1234567890ABCDEFG' + references: + - https://docs.docker.com/engine/swarm/swarm_manager_locking/ diff --git a/crates/kingfisher-rules/data/rules/docusign.yml b/crates/kingfisher-rules/data/rules/docusign.yml index 207bdc2..56cf8a1 100644 --- a/crates/kingfisher-rules/data/rules/docusign.yml +++ b/crates/kingfisher-rules/data/rules/docusign.yml @@ -25,7 +25,107 @@ rules: examples: - "docusign.secret_key = 7a39ce6d-94cf-4bf6-9e9e-9213373c15f4" - "docusign\nds_secret = 3d2f18c9-2075-4e78-834b-64f57f8757d0" + validation: + type: Http + content: + request: + method: POST + url: "https://{{ DOCUSIGN_AUTH_HOST }}/oauth/token" + headers: + Accept: application/json + Content-Type: application/x-www-form-urlencoded + body: > + grant_type=authorization_code&code=INVALID_AUTH_CODE&client_id={{ DOCUSIGN_CLIENT_ID | url_encode }}&client_secret={{ TOKEN | url_encode }}&redirect_uri={{ REDIRECT_URI | url_encode }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [400] + - type: WordMatch + match_all_words: false + words: + - invalid_grant + - invalid authorization code + - type: WordMatch + words: + - invalid_client + negative: true + depends_on_rule: + - rule_id: kingfisher.docusign.2 + variable: DOCUSIGN_CLIENT_ID + - rule_id: kingfisher.docusign.3 + variable: DOCUSIGN_AUTH_HOST + - rule_id: kingfisher.docusign.4 + variable: REDIRECT_URI + references: + - https://developers.docusign.com/platform/auth/ + - https://developers.docusign.com/platform/build-integration/ + + - name: DocuSign Integration Key + id: kingfisher.docusign.2 + pattern: | + (?xi) + \b + docusign + (?:.|[\n\r]){0,64}? + (?:integration[_-]?key|client[_-]?id|app[_-]?id)\b + (?:.|[\n\r]){0,16}? + [=:"'\s] + ['"]* + ( + [a-f0-9]{8}- + [a-f0-9]{4}- + [a-f0-9]{4}- + [a-f0-9]{4}- + [a-f0-9]{12} + ) + \b + pattern_requirements: + min_digits: 6 + min_entropy: 3.0 + confidence: medium + visible: false + examples: + - DOCUSIGN_CLIENT_ID=7a39ce6d-94cf-4bf6-9e9e-9213373c15f4 + - 'docusign.integration_key = "3d2f18c9-2075-4e78-834b-64f57f8757d0"' references: - https://developers.docusign.com/platform/build-integration/ - # No public validation endpoint: DocuSign OAuth secret keys cannot be - # validated without a full Authorization Code Grant flow. + + - name: DocuSign Auth Host + id: kingfisher.docusign.3 + pattern: | + (?xi) + \b + ( + account(?:-d)?\.docusign\.com + ) + \b + min_entropy: 1.0 + confidence: medium + visible: false + examples: + - account.docusign.com + - account-d.docusign.com + references: + - https://developers.docusign.com/platform/auth/ + + - name: DocuSign Redirect URI + id: kingfisher.docusign.4 + pattern: | + (?xi) + \b + docusign + (?:.|[\n\r]){0,64}? + (?:redirect[_-]?uri|oauth[_-]?redirect)\b + (?:.|[\n\r]){0,16}? + [=:"'\s] + ( + https?://[^\s"'<>]{6,200} + ) + min_entropy: 1.5 + confidence: medium + visible: false + examples: + - DOCUSIGN_REDIRECT_URI=https://example.com/docusign/callback + - 'docusign.redirect_uri = "https://localhost:3000/oauth/docusign"' + references: + - https://developers.docusign.com/platform/auth/ diff --git a/crates/kingfisher-rules/data/rules/dropbox.yml b/crates/kingfisher-rules/data/rules/dropbox.yml index 9dd5aca..051047a 100644 --- a/crates/kingfisher-rules/data/rules/dropbox.yml +++ b/crates/kingfisher-rules/data/rules/dropbox.yml @@ -33,5 +33,40 @@ rules: - '"account_id":' - '"email":' url: https://api.dropboxapi.com/2/users/get_current_account + references: + - https://www.dropbox.com/developers/documentation/http/documentation#auth + + - name: Dropbox Long-Lived API Token + id: kingfisher.dropbox.2 + pattern: | + (?x) + \b + ( + [a-z0-9]{11} + AAAAAAAAAA + [a-z0-9\-_=]{43} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + examples: + - 'ab3cd5ef7g9AAAAAAAAAAbcdefghij-klmnopqrstuvwxyz01234567890abcdef' + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + method: POST + response_matcher: + - report_response: true + - match_all_words: true + type: WordMatch + words: + - '"account_id":' + - '"email":' + url: https://api.dropboxapi.com/2/users/get_current_account references: - https://www.dropbox.com/developers/documentation/http/documentation#auth \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/dwolla.yml b/crates/kingfisher-rules/data/rules/dwolla.yml index e1509e1..7c66a3b 100644 --- a/crates/kingfisher-rules/data/rules/dwolla.yml +++ b/crates/kingfisher-rules/data/rules/dwolla.yml @@ -47,7 +47,48 @@ rules: examples: - "dwolla secret = 4d1d407752bfd562bZ=9b7c21f8862fdfc57bc1e45\n" - "dwolla\nclient_secret = 7e4e5297691a673cA=0c7d43b997ec1h8dcaf56\n" + validation: + type: Http + content: + request: + method: POST + url: "{{ DWOLLA_API_BASE }}/token" + headers: + Accept: application/json + Content-Type: application/x-www-form-urlencoded + Authorization: "Basic {{ CLIENT_ID | append: ':' | append: TOKEN | b64enc }}" + body: grant_type=client_credentials + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + words: + - '"access_token"' + depends_on_rule: + - rule_id: kingfisher.dwolla.1 + variable: CLIENT_ID + - rule_id: kingfisher.dwolla.3 + variable: DWOLLA_API_BASE references: - - https://developers.dwolla.com/ - # No simple validation: Dwolla OAuth2 requires both client_id and - # client_secret together for the token endpoint. + - https://developers.dwolla.com/docs/api-reference/tokens/create-an-application-access-token + - https://developers.dwolla.com/docs/balance/auth/application-access-tokens + + - name: Dwolla API Base URL + id: kingfisher.dwolla.3 + visible: false + pattern: | + (?xi) + \b + ( + https://api(?:-sandbox)?\.dwolla\.com + ) + \b + min_entropy: 1.0 + confidence: medium + examples: + - DWOLLA_API_BASE=https://api.dwolla.com + - DWOLLA_BASE_URL="https://api-sandbox.dwolla.com" + references: + - https://developers.dwolla.com/docs/api-reference/tokens/create-an-application-access-token diff --git a/crates/kingfisher-rules/data/rules/ebay.yml b/crates/kingfisher-rules/data/rules/ebay.yml new file mode 100644 index 0000000..3e71f77 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/ebay.yml @@ -0,0 +1,58 @@ +rules: + - name: eBay Production Client ID + id: kingfisher.ebay.1 + pattern: | + (?x) + \b + ( + [a-zA-Z0-9_-]+-[a-zA-Z0-9_-]+-PRD-[a-f0-9]{8,12}-[a-f0-9]{8,12} + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.0 + confidence: medium + categories: [api, key] + examples: + - 'EBAY_CLIENT_ID=MyApp-MyApp-PRD-1a2b3c4d-567890ab' + references: + - https://developer.ebay.com/api-docs/static/oauth-credentials.html + + - name: eBay Sandbox Client ID + id: kingfisher.ebay.2 + pattern: | + (?x) + \b + ( + [a-zA-Z0-9_-]+-[a-zA-Z0-9_-]+-SBX-[a-f0-9]{8,12}-[a-f0-9]{8,12} + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.0 + confidence: medium + categories: [api, key] + examples: + - 'EBAY_SANDBOX_CLIENT_ID=MyApp-MyApp-SBX-1a2b3c4d-567890ab' + references: + - https://developer.ebay.com/api-docs/static/oauth-credentials.html + + - name: eBay Client Secret + id: kingfisher.ebay.3 + pattern: | + (?x) + \b + ( + (?:PRD|SBX)-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4,12} + ) + \b + pattern_requirements: + min_digits: 8 + min_entropy: 3.0 + confidence: medium + categories: [api, key] + examples: + - 'EBAY_CLIENT_SECRET=PRD-1a2b3c4d-5e6f-7a8b-9c0d-1e2f3a4b' + - 'EBAY_SANDBOX_SECRET=SBX-1a2b3c4d-5e6f-7a8b-9c0d-1e2f3a4b' + references: + - https://developer.ebay.com/api-docs/static/oauth-credentials.html diff --git a/crates/kingfisher-rules/data/rules/elastic.yml b/crates/kingfisher-rules/data/rules/elastic.yml new file mode 100644 index 0000000..e4040d7 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/elastic.yml @@ -0,0 +1,48 @@ +rules: + - name: Elastic Cloud API Key + id: kingfisher.elastic.1 + pattern: | + (?x) + \b + (?: + (?i:elastic(?:[_\s-]*(?:search|cloud))?) + (?:.|[\n\r]){0,24}? + (?i:api[_\s-]*key|apikey) + | + (?i:EC_API_KEY) + ) + (?:.|[\n\r]){0,16}? + ( + [A-Za-z0-9+/=]{40,120} + ) + pattern_requirements: + min_digits: 2 + min_uppercase: 2 + min_lowercase: 2 + min_entropy: 3.5 + confidence: medium + categories: [api, key] + examples: + - 'ELASTIC_CLOUD_API_KEY=VnVhQ2ZHY0JDZGJrUW0tZTVhT3g6dWkybHAyYXhUTm1zeWFrdzl0dk5udw==' + references: + - https://www.elastic.co/docs/deploy-manage/api-keys/elastic-cloud-api-keys + + - name: Elasticsearch API Key with Prefix + id: kingfisher.elastic.2 + pattern: | + (?x) + \b + (?i:Authorization:\s*ApiKey\s+) + ( + [A-Za-z0-9+/=]{40,120} + ) + pattern_requirements: + min_digits: 2 + min_uppercase: 2 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'Authorization: ApiKey VnVhQ2ZHY0JDZGJrUW0tZTVhT3g6dWkybHAyYXhUTm1zeWFrdzl0dk5udw==' + references: + - https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html diff --git a/crates/kingfisher-rules/data/rules/flutterwave.yml b/crates/kingfisher-rules/data/rules/flutterwave.yml index ae0819d..da74a8c 100644 --- a/crates/kingfisher-rules/data/rules/flutterwave.yml +++ b/crates/kingfisher-rules/data/rules/flutterwave.yml @@ -38,6 +38,27 @@ rules: - FLW_SECRET_KEY=FLWSECK_TEST-a514d8f1abd080db1502a144f22954dc-X - 'Authorization: Bearer FLWSECK_TEST-5b1f0a33de9c41748c2a7e9b51d3c6af-X' - seckey=FLWSECK-e6db11d1f8a6208de8cb2f94e293450e-X + validation: + type: Http + content: + request: + method: POST + url: https://idp.flutterwave.com/realms/flutterwave/protocol/openid-connect/token + headers: + Accept: application/json + Content-Type: application/x-www-form-urlencoded + body: > + client_id={{ CLIENT_ID | url_encode }}&client_secret={{ TOKEN | url_encode }}&grant_type=client_credentials + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + words: + - '"access_token"' + depends_on_rule: + - rule_id: kingfisher.flutterwave.1 + variable: CLIENT_ID references: - https://developer.flutterwave.com/docs/authentication - - https://developer.flutterwave.com/v2.0/reference/api-request-and-response-standards diff --git a/crates/kingfisher-rules/data/rules/ftp.yml b/crates/kingfisher-rules/data/rules/ftp.yml index 7371036..a239e6f 100644 --- a/crates/kingfisher-rules/data/rules/ftp.yml +++ b/crates/kingfisher-rules/data/rules/ftp.yml @@ -4,12 +4,16 @@ rules: pattern: | (?xi) \b - ftps?:// - [^:@\s]{1,64} - : - ([^@\s]{6,128}) - @ - [^\s/"']{4,128} + (?P + (?Pftps?):// + (?P[^:@\s]{1,64}) + : + (?P[^@\s]{6,128}) + @ + (?P[^\s/"':]{4,128}) + (?::(?P\d{2,5}))? + (?:/[^\s"']*)? + ) pattern_requirements: min_digits: 2 min_entropy: 2.5 @@ -17,6 +21,9 @@ rules: examples: - "ftp://johndoe:pg9stqu2018@files.example.edu.cn" - "BACKUP_URL=ftps://backupuser:S5ec4rePassWord@ftp.corp.example.com" + validation: + type: Raw + content: ftp references: - https://datatracker.ietf.org/doc/html/rfc959 - # No public validation endpoint: FTP servers are instance-specific. + - https://datatracker.ietf.org/doc/html/rfc4217 diff --git a/crates/kingfisher-rules/data/rules/gitlab.yml b/crates/kingfisher-rules/data/rules/gitlab.yml index 1f49d88..2fa5320 100644 --- a/crates/kingfisher-rules/data/rules/gitlab.yml +++ b/crates/kingfisher-rules/data/rules/gitlab.yml @@ -191,3 +191,264 @@ rules: - type: StatusMatch status: [204] url: https://gitlab.com/api/v4/personal_access_tokens/self + + - name: GitLab CI/CD Job Token + id: kingfisher.gitlab.5 + pattern: | + (?x) + \b + ( + glcbt- + [0-9a-zA-Z]{1,5} + _ + [0-9a-zA-Z_-]{20} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + examples: + - glcbt-ab12_xY9kLmNpQrStUvWxYz01 + - 'CI_JOB_TOKEN=glcbt-a1b2c_3dEfGhIjKlMnOpQrStUv' + references: + - https://docs.gitlab.com/ci/jobs/ci_job_token/ + + - name: GitLab Deploy Token + id: kingfisher.gitlab.6 + pattern: | + (?x) + \b + ( + gldt- + [0-9a-zA-Z_-]{20} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + examples: + - gldt-xY9kLmNpQrStUvWxYz01 + - 'DEPLOY_TOKEN=gldt-3dEfGhIjK4MnOpQrStUv' + references: + - https://docs.gitlab.com/user/project/deploy_tokens/ + + - name: GitLab Feature Flag Client Token + id: kingfisher.gitlab.7 + pattern: | + (?x) + \b + ( + glffct- + [0-9a-zA-Z_-]{20} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + examples: + - glffct-xY9kLmNpQrStUvWxYz01 + references: + - https://docs.gitlab.com/operations/feature_flags/ + + - name: GitLab Feed Token + id: kingfisher.gitlab.8 + pattern: | + (?x) + \b + ( + glft- + [0-9a-zA-Z_-]{20} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + examples: + - glft-xY9kLmNpQrStUvWxYz01 + references: + - https://docs.gitlab.com/user/profile/contributions_calendar/#feed-token + + - name: GitLab Incoming Mail Token + id: kingfisher.gitlab.9 + pattern: | + (?x) + \b + ( + glimt- + [0-9a-zA-Z_-]{25} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + examples: + - glimt-xY9kLmNpQrStUvWxYz0123456 + references: + - https://docs.gitlab.com/administration/incoming_email/ + + - name: GitLab Kubernetes Agent Token + id: kingfisher.gitlab.10 + pattern: | + (?x) + \b + ( + glagent- + [0-9a-zA-Z_-]{50} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + examples: + - glagent-xY9kLmNpQrStUvWxYz01234567890abcdefghijklmnopqrstu + references: + - https://docs.gitlab.com/user/clusters/agent/ + + - name: GitLab OAuth Application Secret + id: kingfisher.gitlab.11 + pattern: | + (?x) + \b + ( + gloas- + [0-9a-zA-Z_-]{64} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + examples: + - gloas-xY9kLmNpQrStUvWxYz01234567890abcdefghijklmnopqrstuvwxyz012345678 + references: + - https://docs.gitlab.com/integration/oauth_provider/ + + - name: GitLab Runner Authentication Token + id: kingfisher.gitlab.12 + pattern: | + (?x) + \b + ( + glrt- + [0-9a-zA-Z_-]{20} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + examples: + - glrt-xY9kLmNpQrStUvWxYz01 + - | + gitlab-runner register \ + --url https://gitlab.com \ + --token glrt-3dEfGhIjK4MnOpQrStUv + references: + - https://docs.gitlab.com/runner/register/ + validation: + type: Http + content: + request: + method: POST + headers: + Content-Type: application/x-www-form-urlencoded + Accept: application/json + body: token={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: 200 + - type: WordMatch + words: + - '"token is missing"' + - '"403 Forbidden"' + negative: true + url: https://gitlab.com/api/v4/runners/verify + + - name: GitLab Runner Authentication Token - Routable Format + id: kingfisher.gitlab.13 + pattern: | + (?x) + \b + ( + glrt-t + \d + _ + [0-9a-zA-Z_-]{27,300} + \. + [0-9a-z]{2} + [0-9a-z]{7} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 4.0 + confidence: medium + examples: + - glrt-t1_xY9kLmNpQrStUvWxYz01234567890.01abc1234 + references: + - https://docs.gitlab.com/runner/register/ + validation: + type: Http + content: + request: + method: POST + headers: + Content-Type: application/x-www-form-urlencoded + Accept: application/json + body: token={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: 200 + - type: WordMatch + words: + - '"token is missing"' + - '"403 Forbidden"' + negative: true + url: https://gitlab.com/api/v4/runners/verify + + - name: GitLab SCIM Token + id: kingfisher.gitlab.14 + pattern: | + (?x) + \b + ( + glsoat- + [0-9a-zA-Z_-]{20} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + examples: + - glsoat-xY9kLmNpQrStUvWxYz01 + references: + - https://docs.gitlab.com/api/scim/ + + - name: GitLab Session Cookie + id: kingfisher.gitlab.15 + pattern: | + (?x) + (?:^|[;\s]) + _gitlab_session= + ( + [0-9a-f]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + examples: + - '_gitlab_session=a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6' + - 'Cookie: _gitlab_session=0f1e2d3c4b5a69788796a5b4c3d2e1f0' + references: + - https://docs.gitlab.com/ diff --git a/crates/kingfisher-rules/data/rules/hcaptcha.yml b/crates/kingfisher-rules/data/rules/hcaptcha.yml new file mode 100644 index 0000000..cc1a129 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/hcaptcha.yml @@ -0,0 +1,24 @@ +rules: + - name: hCaptcha Site Verify Secret Key + id: kingfisher.hcaptcha.1 + pattern: | + (?x) + \b + ( + (?: + 0x[a-fA-F0-9]{40} + | + ES_[a-fA-F0-9]{32} + ) + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.5 + confidence: medium + categories: [api, key] + examples: + - 'HCAPTCHA_SECRET=0x0000000000000000000000000000000000000000' + - 'hcaptcha_secret: ES_abcdef1234567890abcdef1234567890' + references: + - https://docs.hcaptcha.com/ diff --git a/crates/kingfisher-rules/data/rules/highnote.yml b/crates/kingfisher-rules/data/rules/highnote.yml new file mode 100644 index 0000000..953ff32 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/highnote.yml @@ -0,0 +1,20 @@ +rules: + - name: Highnote API Key + id: kingfisher.highnote.1 + pattern: | + (?x) + \b + ( + (?:sk|rk)_(?:live|test)_[a-zA-Z0-9]{20,60} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + categories: [api, key] + examples: + - 'HIGHNOTE_API_KEY=sk_live_AbCdEfGhIjKlMnOpQrStUvWxYz1234' + - 'highnote_key: rk_test_AbCdEfGhIjKlMnOpQrStUvWxYz1234' + references: + - https://docs.highnote.com/docs/developers/api/using-the-api diff --git a/crates/kingfisher-rules/data/rules/hop.yml b/crates/kingfisher-rules/data/rules/hop.yml new file mode 100644 index 0000000..06fbd0b --- /dev/null +++ b/crates/kingfisher-rules/data/rules/hop.yml @@ -0,0 +1,38 @@ +rules: + - name: HOP Project Token + id: kingfisher.hop.1 + pattern: | + (?x) + \b + ( + ptk_[a-zA-Z0-9_-]{20,80} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'HOP_TOKEN=ptk_AbCdEfGhIjKlMnOpQrStUvWxYz123456' + references: + - https://docs.hop.io/reference/rest-api + + - name: HOP Personal Access Token + id: kingfisher.hop.2 + pattern: | + (?x) + (?:^|['"\x60\s>=:(,]) + ( + hop_pat_[a-zA-Z0-9_-]{20,80} + ) + (?:$|['"\x60\s<),]) + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'HOP_PAT="hop_pat_AbCdEfGhIjKlMnOpQrStUvWxYz123456"' + references: + - https://docs.hop.io/reference/rest-api diff --git a/crates/kingfisher-rules/data/rules/iterative.yml b/crates/kingfisher-rules/data/rules/iterative.yml new file mode 100644 index 0000000..9d132ae --- /dev/null +++ b/crates/kingfisher-rules/data/rules/iterative.yml @@ -0,0 +1,19 @@ +rules: + - name: Iterative DVC Studio Access Token + id: kingfisher.iterative.1 + pattern: | + (?x) + \b + ( + isat_[a-zA-Z0-9_-]{20,80} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + categories: [api, key] + examples: + - 'DVC_STUDIO_TOKEN=isat_AbCdEfGhIjKlMnOpQrStUvWxYz123456' + references: + - https://dvc.org/doc/command-reference/studio/token diff --git a/crates/kingfisher-rules/data/rules/kraken.yml b/crates/kingfisher-rules/data/rules/kraken.yml index bd3fc2e..2f547cd 100644 --- a/crates/kingfisher-rules/data/rules/kraken.yml +++ b/crates/kingfisher-rules/data/rules/kraken.yml @@ -26,6 +26,45 @@ rules: examples: - KRAKEN_API_SECRET=dGhpcy1sb29rcy1saWtlLWEtYmFzZTY0LWtyYWtlbi1zZWNyZXQtdGhhdC1pcy1sb25nLWVub3VnaA== - kraken_secret="Aq1Bq2Cr3Ds4Et5Fu6Gv7Hw8Ix9Jy0Kz1La2Mb3Nc4Od5Pe6Qf7Rg8Sh9Ti0Uj1Vk2Wm3Xn4Yo5Za6Bc7" + validation: + type: Raw + content: kraken + depends_on_rule: + - rule_id: kingfisher.kraken.2 + variable: KRAKEN_API_KEY references: - https://docs.kraken.com/api/docs/guides/spot-rest-auth/ - https://docs.kraken.com/api/docs/rest-api/get-account-balance/ + + - name: Kraken API Key + id: kingfisher.kraken.2 + pattern: | + (?xi) + \b + kraken + (?:.|[\n\r]){0,32}? + (?: + api[_-]?key | + key | + public[_-]?key + ) + (?:.|[\n\r]){0,12}? + ( + [A-Za-z0-9]{16,64} + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 4 + ignore_if_contains: + - your_api_key + - xxxxxx + min_entropy: 3.0 + confidence: medium + visible: false + examples: + - KRAKEN_API_KEY=Ab12Cd34Ef56Gh78Ij90Kl12Mn34Op56 + - 'kraken_key: 5A6b7C8d9E0f1G2h3I4J5K6L7M8N9P0Q' + references: + - https://docs.kraken.com/api/docs/guides/spot-rest-auth/ diff --git a/crates/kingfisher-rules/data/rules/kucoin.yml b/crates/kingfisher-rules/data/rules/kucoin.yml index c2d92ed..0d6fac4 100644 --- a/crates/kingfisher-rules/data/rules/kucoin.yml +++ b/crates/kingfisher-rules/data/rules/kucoin.yml @@ -57,6 +57,63 @@ rules: examples: - KUCOIN_API_SECRET=7d70f6c7-42e9-4261-8a8d-8ca2d5028d4f - 'kucoin_secret: a1b2c3d4-e5f6-7890-abcd-ef1234567890' + validation: + type: Http + content: + request: + method: GET + url: https://api.kucoin.com/api/v1/accounts + headers: + Accept: application/json + Content-Type: application/json + KC-API-KEY: "{{ KUCOIN_KEY }}" + KC-API-TIMESTAMP: '{%- assign ts = "" | unix_timestamp | times: 1000 -%}{{ ts }}' + KC-API-KEY-VERSION: "2" + KC-API-PASSPHRASE: '{%- assign passphrase = KUCOIN_PASSPHRASE | hmac_sha256: TOKEN -%}{{ passphrase }}' + KC-API-SIGN: '{%- assign ts = "" | unix_timestamp | times: 1000 -%}{%- assign prehash = ts | append: "GET" | append: "/api/v1/accounts" -%}{{ prehash | hmac_sha256: TOKEN }}' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: false + words: + - '"data"' + - '"code":"200000"' + depends_on_rule: + - rule_id: kingfisher.kucoin.1 + variable: KUCOIN_KEY + - rule_id: kingfisher.kucoin.3 + variable: KUCOIN_PASSPHRASE references: - https://www.kucoin.com/docs-new/authentication + - name: KuCoin API Passphrase + id: kingfisher.kucoin.3 + pattern: | + (?xi) + \b + kucoin + (?:.|[\n\r]){0,32}? + (?: + api[_-]?passphrase | + passphrase + ) + (?:.|[\n\r]){0,12}? + ( + [A-Za-z0-9!@\#$%^&*()_+=./:-]{6,64} + ) + \b + pattern_requirements: + ignore_if_contains: + - your_passphrase + - xxxxxx + min_entropy: 2.5 + confidence: medium + visible: false + examples: + - KUCOIN_API_PASSPHRASE=my-strong-passphrase + - 'kucoin_passphrase: S3cur3Passphrase123' + references: + - https://www.kucoin.com/docs-new/authentication diff --git a/crates/kingfisher-rules/data/rules/ldap.yml b/crates/kingfisher-rules/data/rules/ldap.yml index 4176173..719a099 100644 --- a/crates/kingfisher-rules/data/rules/ldap.yml +++ b/crates/kingfisher-rules/data/rules/ldap.yml @@ -1,4 +1,33 @@ rules: + - name: LDAP Bind URI Credentials + id: kingfisher.ldap.2 + pattern: | + (?xi) + \b + (?P + (?Pldaps?):// + (?P[^:@\s]{1,128}) + : + (?P[^@\s]{6,128}) + @ + (?P[^\s/"':]{4,128}) + (?::(?P\d{2,5}))? + (?:/[^\s"']*)? + ) + pattern_requirements: + min_digits: 1 + min_entropy: 2.5 + confidence: medium + examples: + - ldap://cn=admin,dc=example,dc=com:Tr0ub4dor%263!@ldap.example.com:389 + - ldaps://uid=svc-reader,ou=people,dc=corp,dc=example,dc=com:S3cur3BindPass@directory.corp.example.com + validation: + type: Raw + content: ldap + references: + - https://datatracker.ietf.org/doc/html/rfc4511 + - https://datatracker.ietf.org/doc/html/rfc4516 + - name: LDAP Credentials id: kingfisher.ldap.1 pattern: | @@ -22,6 +51,4 @@ rules: - "ldap_pwd = 'Tr0ub4dor&3!'" - "LDAP_PASSWORD=s3cur3P@ssw0rd\n" references: - - https://tools.ietf.org/html/rfc2251 - # No public validation endpoint: LDAP servers are self-hosted; - # the host, port, and DN are instance-specific. + - https://datatracker.ietf.org/doc/html/rfc4511 diff --git a/crates/kingfisher-rules/data/rules/lichess.yml b/crates/kingfisher-rules/data/rules/lichess.yml new file mode 100644 index 0000000..efe968c --- /dev/null +++ b/crates/kingfisher-rules/data/rules/lichess.yml @@ -0,0 +1,31 @@ +rules: + - name: Lichess Personal Access Token + id: kingfisher.lichess.1 + pattern: | + (?x) + \b + ( + lip_[a-zA-Z0-9_]{16,60} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + categories: [api, key] + examples: + - 'LICHESS_TOKEN=lip_AbCdEfGhIjKlMnOpQr12' + validation: + type: Http + content: + request: + method: GET + url: https://lichess.org/api/account + headers: + Authorization: Bearer {{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + references: + - https://lichess.org/api diff --git a/crates/kingfisher-rules/data/rules/localstack.yml b/crates/kingfisher-rules/data/rules/localstack.yml new file mode 100644 index 0000000..7f610d4 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/localstack.yml @@ -0,0 +1,21 @@ +rules: + - name: LocalStack Simulated AWS Access Key + id: kingfisher.localstack.1 + pattern: | + (?x) + \b + ( + (?:LSIA|LKIA)[A-Z0-9]{16,} + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 4 + min_entropy: 3.0 + confidence: medium + categories: [api, key] + examples: + - 'AWS_ACCESS_KEY_ID=LSIAQAAAAAAVNCBMPN59' + - 'aws_access_key=LKIAQAAAAAAVNCBMPN59' + references: + - https://docs.localstack.cloud/aws/capabilities/config/credentials/ diff --git a/crates/kingfisher-rules/data/rules/mailersend.yml b/crates/kingfisher-rules/data/rules/mailersend.yml new file mode 100644 index 0000000..7401624 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/mailersend.yml @@ -0,0 +1,32 @@ +rules: + - name: MailerSend API Token + id: kingfisher.mailersend.1 + pattern: | + (?x) + \b + ( + mlsn\.[a-zA-Z0-9]{30,100} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'MAILERSEND_API_TOKEN=mlsn.AbCdEfGhIjKlMnOpQrStUvWxYz1234567890AbCdEfGhIj' + validation: + type: Http + content: + request: + method: GET + url: https://api.mailersend.com/v1/api-quota + headers: + Authorization: Bearer {{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + references: + - https://www.mailersend.com/help/managing-api-tokens + - https://developers.mailersend.com/ diff --git a/crates/kingfisher-rules/data/rules/onfido.yml b/crates/kingfisher-rules/data/rules/onfido.yml new file mode 100644 index 0000000..6f84055 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/onfido.yml @@ -0,0 +1,32 @@ +rules: + - name: Onfido API Token + id: kingfisher.onfido.1 + pattern: | + (?x) + \b + ( + api_(?:live|sandbox)\.[a-zA-Z0-9_-]{20,80} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'ONFIDO_API_TOKEN=api_live.AbCdEfGhIjKlMnOpQrStUvWxYz123456' + - 'onfido_token: api_sandbox.AbCdEfGhIjKlMnOpQrStUvWxYz123456' + validation: + type: Http + content: + request: + method: GET + url: https://api.eu.onfido.com/v3.6/ping + headers: + Authorization: Token token={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + references: + - https://documentation.identity.entrust.com/api/latest/ diff --git a/crates/kingfisher-rules/data/rules/openvsx.yml b/crates/kingfisher-rules/data/rules/openvsx.yml new file mode 100644 index 0000000..981ea9c --- /dev/null +++ b/crates/kingfisher-rules/data/rules/openvsx.yml @@ -0,0 +1,19 @@ +rules: + - name: OpenVSX Access Token + id: kingfisher.openvsx.1 + pattern: | + (?x) + \b + ( + (?:ovsxat|ovsxp)_[a-zA-Z0-9_-]{20,80} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + categories: [api, key] + examples: + - 'OVSX_PAT=ovsxat_AbCdEfGhIjKlMnOpQrStUvWxYz123456' + references: + - https://github.com/eclipse/openvsx/wiki/Publishing-Extensions diff --git a/crates/kingfisher-rules/data/rules/paddle.yml b/crates/kingfisher-rules/data/rules/paddle.yml new file mode 100644 index 0000000..17851c0 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/paddle.yml @@ -0,0 +1,34 @@ +rules: + - name: Paddle API Key + id: kingfisher.paddle.1 + pattern: | + (?x) + \b + ( + pdl_(?:live|sdbx)_apikey_[a-z0-9_]{30,60} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 4 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'PADDLE_API_KEY=pdl_live_apikey_01gtgztp8f4kek3yd4g1wrksa3_q6tgtjyvoiz7ldtxt65bx7_aqo' + - 'paddle_key: pdl_sdbx_apikey_01h9fjk2z4qqw8n3m7xr5bc6y1_p3rstuvwxyz' + validation: + type: Http + content: + request: + method: GET + url: https://api.paddle.com/event-types + headers: + Authorization: Bearer {{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + references: + - https://developer.paddle.com/api-reference/about/api-keys + - https://developer.paddle.com/api-reference/about/authentication diff --git a/crates/kingfisher-rules/data/rules/pangea.yml b/crates/kingfisher-rules/data/rules/pangea.yml new file mode 100644 index 0000000..0300170 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/pangea.yml @@ -0,0 +1,34 @@ +rules: + - name: Pangea Service Token + id: kingfisher.pangea.1 + pattern: | + (?x) + \b + ( + pts_[a-z0-9]{20,60} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'PANGEA_TOKEN=pts_gqmqvvxk4yhirapuhw6bs7nswu' + - 'pangea_token: pts_hpbc3klkkq54tigu4osc5eygthxps6vf' + validation: + type: Http + content: + request: + method: POST + url: https://audit.aws.us.pangea.cloud/v1/log + headers: + Authorization: Bearer {{ TOKEN }} + Content-Type: application/json + body: '{"event":{"message":"test"}}' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200, 401, 403] + references: + - https://pangea.cloud/docs/ diff --git a/crates/kingfisher-rules/data/rules/persona.yml b/crates/kingfisher-rules/data/rules/persona.yml new file mode 100644 index 0000000..db610d9 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/persona.yml @@ -0,0 +1,34 @@ +rules: + - name: Persona API Key + id: kingfisher.persona.1 + pattern: | + (?x) + \b + ( + persona_(?:production|sandbox)_[a-z0-9_-]{20,80} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 4 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'PERSONA_API_KEY=persona_production_abc123def456ghi789jkl012mno345pqr' + - 'api_key: persona_sandbox_abc123def456ghi789jkl012mno345pqr' + validation: + type: Http + content: + request: + method: GET + url: https://withpersona.com/api/v1/accounts + headers: + Authorization: Bearer {{ TOKEN }} + Persona-Version: "2023-01-05" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + references: + - https://docs.withpersona.com/api-keys diff --git a/crates/kingfisher-rules/data/rules/pinterest.yml b/crates/kingfisher-rules/data/rules/pinterest.yml new file mode 100644 index 0000000..04a815f --- /dev/null +++ b/crates/kingfisher-rules/data/rules/pinterest.yml @@ -0,0 +1,50 @@ +rules: + - name: Pinterest Access Token + id: kingfisher.pinterest.1 + pattern: | + (?x) + \b + ( + pina_[a-zA-Z0-9_-]{20,200} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'PINTEREST_ACCESS_TOKEN=pina_AbCdEfGhIjKlMnOpQrStUvWxYz1234567890' + validation: + type: Http + content: + request: + method: GET + url: https://api.pinterest.com/v5/user_account + headers: + Authorization: Bearer {{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + references: + - https://developers.pinterest.com/docs/api/v5/ + + - name: Pinterest Refresh Token + id: kingfisher.pinterest.2 + pattern: | + (?x) + \b + ( + pinr_[a-zA-Z0-9._-]{20,500} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'PINTEREST_REFRESH_TOKEN=pinr_eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.abc123' + references: + - https://developers.pinterest.com/docs/api/v5/oauth-token/ diff --git a/crates/kingfisher-rules/data/rules/polar.yml b/crates/kingfisher-rules/data/rules/polar.yml new file mode 100644 index 0000000..15b3eee --- /dev/null +++ b/crates/kingfisher-rules/data/rules/polar.yml @@ -0,0 +1,20 @@ +rules: + - name: Polar Personal Access Token + id: kingfisher.polar.1 + pattern: | + (?x) + \b + ( + polar_(?:at|oat|rt)_[a-zA-Z0-9_-]{20,100} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'POLAR_TOKEN=polar_at_AbCdEfGhIjKlMnOpQrStUvWxYz1234' + - 'polar_org_token: polar_oat_AbCdEfGhIjKlMnOpQrStUvWx12' + references: + - https://docs.polar.sh/api/authentication diff --git a/crates/kingfisher-rules/data/rules/proof.yml b/crates/kingfisher-rules/data/rules/proof.yml new file mode 100644 index 0000000..965926e --- /dev/null +++ b/crates/kingfisher-rules/data/rules/proof.yml @@ -0,0 +1,22 @@ +rules: + - name: Proof API Key + id: kingfisher.proof.1 + pattern: | + (?x) + \b + ( + prf_(?:(?:cli_)?test_)?[a-zA-Z0-9_-]{20,80} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'PROOF_API_KEY=prf_AbCdEfGhIjKlMnOpQrStUvWxYz123456' + - 'proof_key: prf_test_AbCdEfGhIjKlMnOpQrStUvWxYz123456' + - 'proof_key: prf_cli_AbCdEfGhIjKlMnOpQrStUvWxYz123456' + - 'proof_key: prf_cli_test_AbCdEfGhIjKlMnOpQrStUvWxYz123456' + references: + - https://dev.proof.com/docs/api-keys diff --git a/crates/kingfisher-rules/data/rules/rabbitmq.yml b/crates/kingfisher-rules/data/rules/rabbitmq.yml index 293ec81..9545676 100644 --- a/crates/kingfisher-rules/data/rules/rabbitmq.yml +++ b/crates/kingfisher-rules/data/rules/rabbitmq.yml @@ -3,22 +3,25 @@ rules: id: kingfisher.rabbitmq.1 pattern: | (?xi) - (?: - amqps? + (?P + (?Pamqps?) + :\/\/ + (?P[\S]{3,50}) + : + (?P[\S]{3,50}) + @ + (?P[-.%\w]+) + (?::(?P\d{2,5}))? + (?:\/(?P[-.%\w\/]+))? ) - :\/\/ - [\S]{3,50} - : - ( - [\S]{3,50} - ) - @ - [-.%\w\/:]+ \b min_entropy: 3.5 confidence: medium examples: - amqp://user:password@rabbitmq.example.com/queue - amqps://admin:3eCa3P@192.168.1.10:5671/vhost + validation: + type: Raw + content: rabbitmq references: - https://www.rabbitmq.com/uri-spec.html diff --git a/crates/kingfisher-rules/data/rules/rainforestpay.yml b/crates/kingfisher-rules/data/rules/rainforestpay.yml new file mode 100644 index 0000000..1376c76 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/rainforestpay.yml @@ -0,0 +1,21 @@ +rules: + - name: Rainforest Pay API Key + id: kingfisher.rainforestpay.1 + pattern: | + (?x) + \b + ( + (?:sbx_)?apikey_[a-f0-9]{64} + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'RAINFOREST_API_KEY=apikey_1ad1c535b0c0093e7b9bf093d7e3444cd0e2ddefab36199216f555c3efa65d63' + - 'api_key: sbx_apikey_2bd2c646e7e1194f8c9cf194e8f4555de1f3eefbbc472003276666d4efb76e74' + references: + - https://docs.rainforestpay.com/docs/api-keys + - https://docs.rainforestpay.com/reference/authentication diff --git a/crates/kingfisher-rules/data/rules/redis.yml b/crates/kingfisher-rules/data/rules/redis.yml index 3d6d5b8..3c52e5f 100644 --- a/crates/kingfisher-rules/data/rules/redis.yml +++ b/crates/kingfisher-rules/data/rules/redis.yml @@ -5,13 +5,15 @@ rules: # Host supports hostnames, IPv4, and IPv6 in brackets pattern: | (?xi) - (?: redis | rediss | redis\+sentinel ) :// - (?: (?P[a-zA-Z0-9%;._~!$&'()*+,;=-]*) - : - )? - (?P[a-zA-Z0-9%;._~!$&'()*+,;:=/+-]{8,}) - @ (?P(?:\[[0-9a-fA-F:.]+\]|[a-zA-Z0-9_.-]{1,})) (?: :(?P\d{1,5}))? - (?: / (?P\d{1,2}))? + (?P + (?: redis | rediss | redis\+sentinel ) :// + (?: (?P[a-zA-Z0-9%;._~!$&'()*+,;=-]*) + : + )? + (?P[a-zA-Z0-9%;._~!$&'()*+,;:=/+-]{8,}) + @ (?P(?:\[[0-9a-fA-F:.]+\]|[a-zA-Z0-9_.-]{1,})) (?: :(?P\d{1,5}))? + (?: / (?P\d{1,2}))? + ) \b pattern_requirements: @@ -42,6 +44,9 @@ rules: - https://redis.io/docs/latest/develop/clients/redis-py/connect/ - https://redis.io/docs/latest/commands/auth/ - https://github.com/redis/redis-py/blob/master/redis/client.py + validation: + type: Raw + content: redis - id: kingfisher.redis.2 name: Python Redis Client Debug Output diff --git a/crates/kingfisher-rules/data/rules/ringcentral.yml b/crates/kingfisher-rules/data/rules/ringcentral.yml index 170089f..9244dc2 100644 --- a/crates/kingfisher-rules/data/rules/ringcentral.yml +++ b/crates/kingfisher-rules/data/rules/ringcentral.yml @@ -51,5 +51,77 @@ rules: - 'RINGCENTRAL_CLIENT_SECRET="xY9zW8vU7tS6rQ5pO4nM3l"' negative_examples: - 'RINGCENTRAL_URL="https://platform.ringcentral.com"' + validation: + type: Http + content: + request: + method: POST + url: "{{ RINGCENTRAL_BASE_URL }}/restapi/oauth/token" + headers: + Accept: application/json + Content-Type: application/x-www-form-urlencoded + Authorization: "Basic {{ CLIENT_ID | append: ':' | append: TOKEN | b64enc }}" + body: > + grant_type=authorization_code&code=INVALID_AUTH_CODE&redirect_uri={{ REDIRECT_URI | url_encode }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [400] + - type: WordMatch + match_all_words: false + words: + - invalid_grant + - authentication_error + - type: WordMatch + words: + - invalid_client + negative: true + depends_on_rule: + - rule_id: kingfisher.ringcentral.1 + variable: CLIENT_ID + - rule_id: kingfisher.ringcentral.3 + variable: RINGCENTRAL_BASE_URL + - rule_id: kingfisher.ringcentral.4 + variable: REDIRECT_URI references: - - https://developers.ringcentral.com/api-reference/ + - https://developers.ringcentral.com/guide/authentication/auth-code-flow + + - name: RingCentral OAuth Base URL + id: kingfisher.ringcentral.3 + pattern: | + (?xi) + \b + ( + https://platform(?:\.devtest)?\.ringcentral\.com + ) + \b + min_entropy: 1.0 + confidence: medium + visible: false + examples: + - RINGCENTRAL_BASE_URL=https://platform.ringcentral.com + - RINGCENTRAL_SANDBOX_URL=https://platform.devtest.ringcentral.com + references: + - https://developers.ringcentral.com/guide/authentication/auth-code-flow + + - name: RingCentral Redirect URI + id: kingfisher.ringcentral.4 + pattern: | + (?xi) + \b + ring.?central + (?:.|[\n\r]){0,64}? + (?:redirect[_-]?uri|oauth[_-]?redirect)\b + (?:.|[\n\r]){0,16}? + [=:"'\s] + ( + https?://[^\s"'<>]{6,200} + ) + min_entropy: 1.5 + confidence: medium + visible: false + examples: + - RINGCENTRAL_REDIRECT_URI=https://example.com/ringcentral/callback + - 'ringcentral.redirect_uri = "https://localhost:8080/oauth/ringcentral"' + references: + - https://developers.ringcentral.com/guide/authentication/auth-code-flow diff --git a/crates/kingfisher-rules/data/rules/rootly.yml b/crates/kingfisher-rules/data/rules/rootly.yml new file mode 100644 index 0000000..8fe48ac --- /dev/null +++ b/crates/kingfisher-rules/data/rules/rootly.yml @@ -0,0 +1,31 @@ +rules: + - name: Rootly API Key + id: kingfisher.rootly.1 + pattern: | + (?x) + \b + ( + rootly_[a-f0-9]{64} + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'ROOTLY_API_KEY=rootly_abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890' + validation: + type: Http + content: + request: + method: GET + url: https://api.rootly.com/v1/users/me + headers: + Authorization: Bearer {{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + references: + - https://rootly.com/api diff --git a/crates/kingfisher-rules/data/rules/runpod.yml b/crates/kingfisher-rules/data/rules/runpod.yml new file mode 100644 index 0000000..f8e913a --- /dev/null +++ b/crates/kingfisher-rules/data/rules/runpod.yml @@ -0,0 +1,36 @@ +rules: + - name: RunPod API Key + id: kingfisher.runpod.1 + pattern: | + (?x) + \b + ( + rpa_[a-zA-Z0-9]{20,60} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'RUNPOD_API_KEY=rpa_ABC123DEF456GHI789JKL012MNO345PQR678' + validation: + type: Http + content: + request: + method: POST + url: https://api.runpod.io/graphql + headers: + Authorization: Bearer {{ TOKEN }} + Content-Type: application/json + body: '{"query":"{ myself { id } }"}' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + match_all_words: true + words: ['"myself"'] + references: + - https://docs.runpod.io/get-started/api-keys diff --git a/crates/kingfisher-rules/data/rules/snowflake.yml b/crates/kingfisher-rules/data/rules/snowflake.yml index 5f266af..7585c8d 100644 --- a/crates/kingfisher-rules/data/rules/snowflake.yml +++ b/crates/kingfisher-rules/data/rules/snowflake.yml @@ -24,3 +24,76 @@ rules: - https://docs.snowflake.com/en/ - https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-api # Snowflake credentials are endpoint-specific; no public REST endpoint for standalone validation. + + - name: Snowflake Programmatic Access Token + id: kingfisher.snowflake.2 + pattern: | + (?x) + \b + (?: + (?i:snowflake[_\s-]*(?:programmatic[_\s-]*)?(?:access[_\s-]*)?token) + | + (?i:SF_TOKEN) + ) + \b + (?:.|[\n\r]){0,16}? + [=:] + \s*["']? + ( + [a-zA-Z0-9_-]{100,500} + ) + ["']? + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + categories: [api, key] + examples: + - 'SNOWFLAKE_TOKEN=AbCdEfGhIjKlMnOpQrStUvWxYz1234567890AbCdEfGhIjKlMnOpQrStUvWxYz1234567890AbCdEfGhIjKlMnOpQrStUvWxYz12' + validation: + type: Http + content: + request: + method: POST + url: "https://{{ SNOWFLAKE_HOST }}/api/v2/statements" + headers: + Accept: application/json + Content-Type: application/json + Authorization: "Bearer {{ TOKEN }}" + X-Snowflake-Authorization-Token-Type: PROGRAMMATIC_ACCESS_TOKEN + body: '{"statement":"select 1","timeout":5}' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200, 202] + - type: JsonValid + - type: WordMatch + match_all_words: false + words: + - '"statementHandle"' + - '"resultSetMetaData"' + depends_on_rule: + - rule_id: kingfisher.snowflake.3 + variable: SNOWFLAKE_HOST + references: + - https://docs.snowflake.com/en/user-guide/programmatic-access-tokens + - https://docs.snowflake.com/en/developer-guide/sql-api/submitting-requests + + - name: Snowflake Account Host + id: kingfisher.snowflake.3 + pattern: | + (?xi) + \b + ( + [a-z0-9_-]+(?:\.[a-z0-9_-]+)*\.snowflakecomputing\.com + ) + \b + min_entropy: 1.0 + confidence: medium + visible: false + examples: + - account = "xy12345.us-east-1.snowflakecomputing.com" + - host=acme-prod.eu-west-1.aws.snowflakecomputing.com + references: + - https://docs.snowflake.com/en/user-guide/programmatic-access-tokens + - https://docs.snowflake.com/en/developer-guide/sql-api/submitting-requests diff --git a/crates/kingfisher-rules/data/rules/tableau.yml b/crates/kingfisher-rules/data/rules/tableau.yml index d47df68..803bbcf 100644 --- a/crates/kingfisher-rules/data/rules/tableau.yml +++ b/crates/kingfisher-rules/data/rules/tableau.yml @@ -14,10 +14,11 @@ rules: (?:.|[\n\r]){0,16}? ) ( - [A-Za-z0-9+/]{12,24} + (?P[A-Za-z0-9+/]{12,24} (?:={1,2})? + ) : - [A-Za-z0-9+/=_-]{24,48} + (?P[A-Za-z0-9+/=_-]{24,48}) ) pattern_requirements: min_digits: 2 @@ -28,7 +29,84 @@ rules: examples: - "tableau_auth = TSC.PersonalAccessTokenAuth('prod_svc', 'WLQKWBs1TnuBx4G7gIzz/w==:yDwZ74EWDPIgU6cSlz8RDJHp7CV2rtFP', 'companysite')" - 'curl -H "X-Tableau-Auth:oJzK8bqwPTnmSl1/E2+aXw==:ZvTsRqFmKpWuLdNhYcBjXiGe" https://tableau.example.com/api/3.17/sites' + validation: + type: Http + content: + request: + method: POST + url: "{{ TABLEAU_SERVER }}/api/3.28/auth/signin" + headers: + Accept: application/json + Content-Type: application/json + body: > + {"credentials":{"personalAccessTokenName":"{{ TABLEAU_PAT_NAME }}","personalAccessTokenSecret":"{{ TOKEN }}","site":{"contentUrl":"{{ TABLEAU_SITE | default: "" }}"}}} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: false + words: + - '"token"' + - '"site"' + depends_on_rule: + - rule_id: kingfisher.tableau.2 + variable: TABLEAU_SERVER + - rule_id: kingfisher.tableau.3 + variable: TABLEAU_SITE references: - - https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_ref.htm + - https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_ref_authentication.htm - https://help.tableau.com/current/server/en-us/security_personal_access_tokens.htm - # Tableau PATs are instance-specific; no universal public endpoint for standalone validation. + + - name: Tableau Server URL + id: kingfisher.tableau.2 + pattern: | + (?xi) + \b + ( + https://[a-z0-9.-]{3,200} + ) + (?: + /api/\d+\.\d+ + )? + (?: + /[^\s"'<>]{0,120} + )? + min_entropy: 1.5 + confidence: medium + visible: false + examples: + - https://tableau.example.com + - https://10ax.online.tableau.com + - server="https://analytics.example.com" + references: + - https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_ref_authentication.htm + + - name: Tableau Site Content URL + id: kingfisher.tableau.3 + pattern: | + (?xi) + \b + (?: + tableau + (?:.|[\n\r]){0,48}? + )? + (?: + site | + content[_-]?url + ) + (?:.|[\n\r]){0,12}? + [=:"'\s] + ( + [A-Za-z0-9._-]{1,64} + ) + \b + min_entropy: 1.0 + confidence: medium + visible: false + examples: + - tableau_site=companysite + - contentUrl="default" + references: + - https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_ref_authentication.htm diff --git a/crates/kingfisher-rules/data/rules/telnyx.yml b/crates/kingfisher-rules/data/rules/telnyx.yml new file mode 100644 index 0000000..79f5dc9 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/telnyx.yml @@ -0,0 +1,31 @@ +rules: + - name: Telnyx API V2 Key + id: kingfisher.telnyx.1 + pattern: | + (?x) + \b + ( + KEY[0-9A-Za-z_-]{55} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'TELNYX_API_KEY=KEYabcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRS' + validation: + type: Http + content: + request: + method: GET + url: https://api.telnyx.com/v2/balance + headers: + Authorization: Bearer {{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + references: + - https://developers.telnyx.com/development/api-fundamentals/authentication diff --git a/crates/kingfisher-rules/data/rules/thunderstore.yml b/crates/kingfisher-rules/data/rules/thunderstore.yml new file mode 100644 index 0000000..50a8d81 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/thunderstore.yml @@ -0,0 +1,19 @@ +rules: + - name: Thunderstore API Token + id: kingfisher.thunderstore.1 + pattern: | + (?x) + \b + ( + tss_[a-zA-Z0-9_-]{20,80} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + categories: [api, key] + examples: + - 'THUNDERSTORE_TOKEN=tss_AbCdEfGhIjKlMnOpQrStUvWxYz123456' + references: + - https://thunderstore.io/api/docs/ diff --git a/crates/kingfisher-rules/data/rules/trello.yml b/crates/kingfisher-rules/data/rules/trello.yml index fbe8821..1678f28 100644 --- a/crates/kingfisher-rules/data/rules/trello.yml +++ b/crates/kingfisher-rules/data/rules/trello.yml @@ -27,5 +27,59 @@ rules: examples: - TRELLO_TOKEN=0a1b2c3d4e5f6g7h8i9j0k1l2m3n4p5q - trello_access_token="Ab12Cd34Ef56Gh78Ij90Kl12Mn34Op56" + validation: + type: Http + content: + request: + method: GET + url: "https://api.trello.com/1/members/me?key={{ TRELLO_KEY | url_encode }}&token={{ TOKEN | url_encode }}" + headers: + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: false + words: + - '"id"' + - '"username"' + depends_on_rule: + - rule_id: kingfisher.trello.2 + variable: TRELLO_KEY references: - https://developer.atlassian.com/cloud/trello/guides/rest-api/api-introduction/ + - https://developer.atlassian.com/cloud/trello/guides/rest-api/authorization/ + + - name: Trello API Key + id: kingfisher.trello.2 + visible: false + pattern: | + (?xi) + \b + trello + (?:.|[\n\r]){0,32}? + (?: + api[_-]?key | + app[_-]?key | + key + ) + (?:.|[\n\r]){0,12}? + ( + [A-Za-z0-9]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 6 + ignore_if_contains: + - yourkey + - placeholder + min_entropy: 3.1 + confidence: medium + examples: + - TRELLO_KEY=0a1b2c3d4e5f6g7h8i9j0k1l2m3n4p5q + - trello_api_key="Ab12Cd34Ef56Gh78Ij90Kl12Mn34Op56" + references: + - https://developer.atlassian.com/cloud/trello/guides/rest-api/authorization/ diff --git a/crates/kingfisher-rules/data/rules/ubidots.yml b/crates/kingfisher-rules/data/rules/ubidots.yml index b0b1928..ff7a694 100644 --- a/crates/kingfisher-rules/data/rules/ubidots.yml +++ b/crates/kingfisher-rules/data/rules/ubidots.yml @@ -16,7 +16,25 @@ rules: examples: - "API_KEY = \"BBUS-kDvT2Vrm6JThnHZvgzNyO2K7DAHdWs12abc\"" - "UBIDOTS_TOKEN=BBUS-AbCdEfGhIjKlMnOpQrStUvWxYz0123456" + validation: + type: Http + content: + request: + method: GET + url: https://industrial.api.ubidots.com/api/v2.0/devices + headers: + Accept: application/json + X-Auth-Token: "{{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: false + words: + - '"count"' + - '"results"' references: - - https://docs.ubidots.com/v1.6/reference/authentication - # No API validation available: Ubidots API keys generate tokens but cannot - # themselves be validated against a public endpoint. + - https://docs.ubidots.com/reference/authentication + - https://docs.ubidots.com/reference/get-devices diff --git a/crates/kingfisher-rules/data/rules/valtown.yml b/crates/kingfisher-rules/data/rules/valtown.yml new file mode 100644 index 0000000..fdaa9b5 --- /dev/null +++ b/crates/kingfisher-rules/data/rules/valtown.yml @@ -0,0 +1,19 @@ +rules: + - name: Val Town API Token + id: kingfisher.valtown.1 + pattern: | + (?x) + \b + ( + vtwn_[a-zA-Z0-9_-]{20,80} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: high + categories: [api, key] + examples: + - 'VALTOWN_TOKEN=vtwn_AbCdEfGhIjKlMnOpQrStUvWxYz123456' + references: + - https://docs.val.town/api/authentication/ diff --git a/crates/kingfisher-rules/data/rules/volcengine.yml b/crates/kingfisher-rules/data/rules/volcengine.yml new file mode 100644 index 0000000..68e5dfd --- /dev/null +++ b/crates/kingfisher-rules/data/rules/volcengine.yml @@ -0,0 +1,19 @@ +rules: + - name: VolcEngine Access Key ID + id: kingfisher.volcengine.1 + pattern: | + (?x) + \b + ( + AKLT[a-zA-Z0-9_-]{16,60} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + categories: [api, key] + examples: + - 'VOLCENGINE_ACCESS_KEY=AKLTabcdefghijklmnop1234567890' + references: + - https://www.volcengine.com/docs/6291/65568 diff --git a/crates/kingfisher-rules/data/rules/webex.yml b/crates/kingfisher-rules/data/rules/webex.yml index a66d125..3cf82c4 100644 --- a/crates/kingfisher-rules/data/rules/webex.yml +++ b/crates/kingfisher-rules/data/rules/webex.yml @@ -47,6 +47,57 @@ rules: examples: - "webex.secret = 8ab9b3c77035e1121e2d7d64529749682b3ce5b93dc1f1e6677f0800dcf00d1e" - "webex\nclient_secret=1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b" + validation: + type: Http + content: + request: + method: POST + url: https://webexapis.com/v1/access_token + headers: + Accept: application/json + Content-Type: application/x-www-form-urlencoded + body: > + grant_type=authorization_code&client_id={{ CLIENT_ID | url_encode }}&client_secret={{ TOKEN | url_encode }}&code=INVALID_AUTH_CODE&redirect_uri={{ REDIRECT_URI | url_encode }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [400] + - type: WordMatch + match_all_words: false + words: + - invalid_grant + - Invalid authorization code + - type: WordMatch + words: + - invalid_client + negative: true + depends_on_rule: + - rule_id: kingfisher.webex.1 + variable: CLIENT_ID + - rule_id: kingfisher.webex.3 + variable: REDIRECT_URI references: - - https://developer.webex.com/docs/platform-introduction + - https://developer.webex.com/create/docs/authentication - https://developer.webex.com/docs/integrations + + - name: Webex Redirect URI + id: kingfisher.webex.3 + pattern: | + (?xi) + \b + webex + (?:.|[\n\r]){0,64}? + (?:redirect[_-]?uri|oauth[_-]?redirect)\b + (?:.|[\n\r]){0,16}? + [=:"'\s] + ( + https?://[^\s"'<>]{6,200} + ) + min_entropy: 1.5 + confidence: medium + visible: false + examples: + - WEBEX_REDIRECT_URI=https://example.com/webex/callback + - 'webex.redirect_uri = "https://localhost:3000/oauth/webex"' + references: + - https://developer.webex.com/create/docs/authentication diff --git a/crates/kingfisher-rules/src/liquid_filters.rs b/crates/kingfisher-rules/src/liquid_filters.rs index 77fba6e..cf6b73e 100644 --- a/crates/kingfisher-rules/src/liquid_filters.rs +++ b/crates/kingfisher-rules/src/liquid_filters.rs @@ -12,7 +12,10 @@ use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC}; use rand::{distr::Alphanumeric, RngExt}; use sha1::Sha1; use sha2::{Digest, Sha256, Sha384}; -use time::{format_description::well_known::Iso8601, OffsetDateTime}; +use time::{ + format_description::well_known::{Iso8601, Rfc2822}, + OffsetDateTime, +}; use uuid::Uuid; // ----------------------------------------------------------------------------- @@ -297,6 +300,42 @@ impl Filter for HmacSha384Filter { } } +#[derive(Clone, ParseFilter, FilterReflection, Default)] +#[filter( + name = "hmac_sha384_hex", + description = "HMAC-SHA384 - returns lowercase hex.", + parameters(Hmac384Args), + parsed(HmacSha384HexFilter) +)] +pub struct HmacSha384Hex; + +#[derive(Debug, FromFilterParameters, Display_filter)] +#[name = "hmac_sha384_hex"] +struct HmacSha384HexFilter { + #[parameters] + args: Hmac384Args, +} + +impl Filter for HmacSha384HexFilter { + fn evaluate(&self, input: &dyn ValueView, runtime: &dyn Runtime) -> Result { + use std::fmt::Write as _; + + let args = self.args.evaluate(runtime)?; + let key = args.key.to_kstr(); + + let mut mac = Hmac::::new_from_slice(key.as_bytes()).unwrap(); + mac.update(input.to_kstr().as_bytes()); + + let bytes = mac.finalize().into_bytes(); + let mut hex = String::with_capacity(bytes.len() * 2); + for byte in bytes { + let _ = write!(&mut hex, "{byte:02x}"); + } + + Ok(Value::scalar(hex)) + } +} + // ── random_string ──────────────────────────────── #[derive(Debug, FilterParameters)] struct RandomStringArgs { @@ -903,6 +942,15 @@ static_filter!( } ); +// {{ "" | unix_timestamp_ms }} +static_filter!( + /// Current Unix epoch milliseconds. + UnixTimestampMsFilter, "unix_timestamp_ms", + |_input: &dyn ValueView| -> i64 { + (OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000) as i64 + } +); + // {{ "" | iso_timestamp_no_frac }} static_filter!( /// Current ISO-8601 timestamp (UTC) with no fractional seconds. @@ -933,6 +981,21 @@ static_filter!( } ); +// {{ "" | rfc1123_date }} +static_filter!( + /// Current RFC-1123 timestamp in GMT. + Rfc1123DateFilter, "rfc1123_date", + |_input: &dyn ValueView| -> String { + let rendered = OffsetDateTime::now_utc() + .format(&Rfc2822) + .unwrap_or_else(|_| "Thu, 01 Jan 1970 00:00:00 +0000".into()); + rendered + .strip_suffix(" +0000") + .map(|prefix| format!("{prefix} GMT")) + .unwrap_or(rendered) + } +); + // ----------------------------------------------------------------------------- // Request Uniqueness // ----------------------------------------------------------------------------- @@ -953,8 +1016,10 @@ pub fn register_all(builder: liquid::ParserBuilder) -> liquid::ParserBuilder { .filter(UrlEncodeFilter::default()) .filter(JsonEscapeFilter::default()) .filter(UnixTimestampFilter::default()) + .filter(UnixTimestampMsFilter::default()) .filter(IsoTimestampFilter::default()) .filter(IsoTimestampNoFracFilter::default()) + .filter(Rfc1123DateFilter::default()) .filter(UuidFilter::default()) .filter(JwtHeaderFilter::default()) .filter(B64EncFilter::default()) @@ -974,6 +1039,7 @@ pub fn register_all(builder: liquid::ParserBuilder) -> liquid::ParserBuilder { .filter(HmacSha256B64Key::default()) .filter(HmacSha1::default()) .filter(HmacSha384::default()) + .filter(HmacSha384Hex::default()) } #[cfg(test)] @@ -1148,6 +1214,24 @@ mod tests { assert_eq!(render(r#"{{ "payload" | hmac_sha384: "topsecret" }}"#), expect); } + #[test] + fn hmac_sha384_hex_filter() { + use std::fmt::Write as _; + + let key = b"topsecret"; + let data = b"payload"; + let mut mac = Hmac::::new_from_slice(key).unwrap(); + mac.update(data); + + let bytes = mac.finalize().into_bytes(); + let mut expect = String::with_capacity(bytes.len() * 2); + for byte in bytes { + let _ = write!(&mut expect, "{byte:02x}"); + } + + assert_eq!(render(r#"{{ "payload" | hmac_sha384_hex: "topsecret" }}"#), expect); + } + // ------------------------------------------------------------------------- // Random string // ------------------------------------------------------------------------- @@ -1174,6 +1258,13 @@ mod tests { assert!((now - tmpl_val).abs() < 5, "timestamp differs by >5 s"); } + #[test] + fn unix_timestamp_ms_filter_is_nowish() { + let tmpl_val: i64 = render(r#"{{ "" | unix_timestamp_ms }}"#).parse().unwrap(); + let now = (OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000) as i64; + assert!((now - tmpl_val).abs() < 5_000, "timestamp differs by >5 s"); + } + #[test] fn iso_timestamp_filter_parses() { let out = render(r#"{{ "" | iso_timestamp }}"#); @@ -1192,6 +1283,14 @@ mod tests { let v = render(r#"{{ "" | uuid }}"#); assert!(uuid_re.is_match(&v)); } + + #[test] + fn rfc1123_date_filter_format() { + let out = render(r#"{{ "" | rfc1123_date }}"#); + assert!(out.ends_with(" GMT"), "unexpected RFC-1123 date: {out}"); + let normalized = out.replace(" GMT", " +0000"); + assert!(OffsetDateTime::parse(&normalized, &Rfc2822).is_ok()); + } // ------------------------------------------------------------------------- // Replace filter // ------------------------------------------------------------------------- diff --git a/crates/kingfisher-scanner/Cargo.toml b/crates/kingfisher-scanner/Cargo.toml index 09521ca..f5b5dd2 100644 --- a/crates/kingfisher-scanner/Cargo.toml +++ b/crates/kingfisher-scanner/Cargo.toml @@ -24,6 +24,22 @@ validation-http = [ "dep:liquid-core", "dep:quick-xml", "dep:sha1", + "dep:time", +] + +# Provider/protocol-specific validation flows that need custom network logic. +validation-raw = [ + "validation-http", + "dep:chrono", + "dep:hmac", + "dep:sha2", + "dep:hex", + "dep:url", + "dep:percent-encoding", + "dep:rustls", + "dep:rustls-native-certs", + "dep:tokio-rustls", + "dep:ldap3", ] # AWS credential validation @@ -94,6 +110,7 @@ validation-database = [ # All validation features validation-all = [ "validation", + "validation-raw", "validation-aws", "validation-azure", "validation-coinbase", @@ -153,11 +170,12 @@ tracing.workspace = true reqwest = { version = "0.12", default-features = false, features = [ "json", "gzip", "brotli", "deflate", "stream", "rustls-tls", "rustls-tls-native-roots", "multipart" ], optional = true } -tokio = { version = "1.48", features = ["net", "time", "sync"], optional = true } +tokio = { version = "1.48", features = ["net", "time", "sync", "io-util"], optional = true } liquid = { version = "0.26", optional = true } liquid-core = { version = "0.26", optional = true } quick-xml = { version = "0.39", features = ["serde", "serialize"], optional = true } sha1 = { workspace = true, optional = true } +time = { workspace = true, optional = true } chrono = { version = "0.4.42", optional = true } hmac = { workspace = true, optional = true } sha2 = { workspace = true, optional = true } @@ -176,6 +194,8 @@ tokio-postgres = { version = "0.7", default-features = false, features = ["runti tokio-postgres-rustls = { version = "0.13.0", optional = true } rustls = { version = "0.23.35", optional = true } rustls-native-certs = { version = "0.8.2", optional = true } +tokio-rustls = { version = "0.26.4", optional = true } +ldap3 = { version = "0.11.5", default-features = false, features = ["tls-rustls"], optional = true } # AWS validation aws-config = { version = "1.8.14", default-features = false, features = ["default-https-client", "rt-tokio"], optional = true } diff --git a/crates/kingfisher-scanner/src/validation/http_validation.rs b/crates/kingfisher-scanner/src/validation/http_validation.rs index 2dc70de..2fb1f0e 100644 --- a/crates/kingfisher-scanner/src/validation/http_validation.rs +++ b/crates/kingfisher-scanner/src/validation/http_validation.rs @@ -3,6 +3,7 @@ use std::{collections::BTreeMap, future::Future, net::IpAddr, str::FromStr, time use anyhow::{anyhow, Error, Result}; use http::StatusCode; use liquid::Object; +use liquid_core::Value; use quick_xml::de::from_str as xml_from_str; use reqwest::{ header, @@ -11,6 +12,7 @@ use reqwest::{ }; use serde::de::IgnoredAny; use sha1::{Digest, Sha1}; +use time::{format_description::well_known::Rfc2822, OffsetDateTime}; use tokio::{net::lookup_host, time::sleep}; use tracing::debug; @@ -68,6 +70,33 @@ pub fn parse_http_method(method_str: &str) -> Result { Method::from_str(method_str).map_err(|_| format!("Invalid HTTP method: {}", method_str)) } +fn format_rfc1123(now: OffsetDateTime) -> String { + let rendered = + now.format(&Rfc2822).unwrap_or_else(|_| "Thu, 01 Jan 1970 00:00:00 +0000".to_string()); + rendered.strip_suffix(" +0000").map(|prefix| format!("{prefix} GMT")).unwrap_or(rendered) +} + +/// Clone `globals` and add stable request-scoped values for templated request rendering. +/// +/// These values are computed once so the same generated timestamp can be reused across the URL, +/// headers, body, and multipart parts of a single request. +pub fn with_request_template_globals(globals: &Object) -> Object { + let mut out = globals.clone(); + let now = OffsetDateTime::now_utc(); + + if !out.contains_key("REQUEST_RFC1123_DATE") { + out.insert("REQUEST_RFC1123_DATE".into(), Value::scalar(format_rfc1123(now))); + } + if !out.contains_key("REQUEST_UNIX_MILLIS") { + out.insert( + "REQUEST_UNIX_MILLIS".into(), + Value::scalar((now.unix_timestamp_nanos() / 1_000_000).to_string()), + ); + } + + out +} + /// Build a reqwest RequestBuilder using the provided parameters. pub fn build_request_builder( client: &Client, @@ -566,7 +595,36 @@ pub async fn check_url_resolvable_safe(url: &Url) -> Result<(), Box 0); + } + + #[test] + fn request_template_globals_preserve_explicit_overrides() { + let mut globals = Object::new(); + globals.insert("REQUEST_RFC1123_DATE".into(), Value::scalar("custom-date")); + globals.insert("REQUEST_UNIX_MILLIS".into(), Value::scalar("123")); + + let rendered = with_request_template_globals(&globals); + + assert_eq!(rendered.get("REQUEST_RFC1123_DATE").unwrap().to_kstr(), "custom-date"); + assert_eq!(rendered.get("REQUEST_UNIX_MILLIS").unwrap().to_kstr(), "123"); + } #[test] fn rejects_ipv4_loopback() { diff --git a/crates/kingfisher-scanner/src/validation/mod.rs b/crates/kingfisher-scanner/src/validation/mod.rs index 443f821..b242e90 100644 --- a/crates/kingfisher-scanner/src/validation/mod.rs +++ b/crates/kingfisher-scanner/src/validation/mod.rs @@ -20,6 +20,8 @@ //! - **Azure**: Azure Storage credential validation (requires `validation-azure` feature) //! - **Databases**: MongoDB, MySQL, Postgres, JDBC (requires `validation-database` feature) //! - **JWT**: JWT token validation (requires `validation-jwt` feature) +//! - **Raw**: provider/protocol-specific validators that need custom logic +//! (requires `validation-raw` feature) mod utils; mod validation_body; @@ -54,6 +56,9 @@ pub mod mysql; #[cfg(feature = "validation-database")] pub mod postgres; +#[cfg(feature = "validation-raw")] +pub mod raw; + // Re-exports pub use utils::{find_closest_variable, process_captures}; pub use validation_body::{as_str, clone_as_string, from_string, ValidationResponseBody}; @@ -62,9 +67,12 @@ pub use validation_body::{as_str, clone_as_string, from_string, ValidationRespon pub use http_validation::{ build_request_builder, check_url_resolvable, generate_http_cache_key_parts, is_ssrf_safe_ip, parse_http_method, process_headers, retry_multipart_request, retry_request, validate_response, - SsrfBlockedError, + with_request_template_globals, SsrfBlockedError, }; +#[cfg(feature = "validation-raw")] +pub use raw::{required_vars as raw_required_vars, validate_raw, RawValidationOutcome}; + #[cfg(feature = "validation-http")] #[allow(deprecated)] pub use http_validation::check_url_resolvable_safe; diff --git a/crates/kingfisher-scanner/src/validation/raw.rs b/crates/kingfisher-scanner/src/validation/raw.rs new file mode 100644 index 0000000..dc006e6 --- /dev/null +++ b/crates/kingfisher-scanner/src/validation/raw.rs @@ -0,0 +1,612 @@ +//! Provider-specific raw validators for secret formats that need custom protocol logic. + +use std::{ + collections::BTreeSet, + sync::{Arc, OnceLock}, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use anyhow::{anyhow, Context, Result}; +use base64::{engine::general_purpose::STANDARD as B64, Engine}; +use hmac::{digest::KeyInit, Hmac, Mac}; +use http::StatusCode; +use ldap3::LdapConnSettings; +use liquid::Object; +use liquid_core::ValueView; +use once_cell::sync::OnceCell; +use percent_encoding::percent_decode_str; +use reqwest::Client; +use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; +use rustls::crypto::{ring, verify_tls12_signature, verify_tls13_signature, CryptoProvider}; +use rustls::pki_types::{CertificateDer, ServerName, UnixTime}; +use rustls::{ClientConfig, DigitallySignedStruct, RootCertStore, SignatureScheme}; +use sha2::{Digest, Sha256, Sha512}; +use tokio::{ + io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufStream}, + net::TcpStream, + time::timeout, +}; +use tokio_rustls::TlsConnector; +use url::Url; + +pub struct RawValidationOutcome { + pub valid: bool, + pub status: StatusCode, + pub body: String, +} + +static INIT_PROVIDER: OnceCell<()> = OnceCell::new(); +static LAX_PROVIDER: OnceLock> = OnceLock::new(); + +fn ensure_crypto_provider() { + INIT_PROVIDER.get_or_init(|| { + let _ = CryptoProvider::install_default(ring::default_provider()); + }); +} + +#[derive(Debug)] +struct LaxCertVerifier(Arc); + +impl ServerCertVerifier for LaxCertVerifier { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> std::result::Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> std::result::Result { + verify_tls12_signature(message, cert, dss, &self.0.signature_verification_algorithms) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> std::result::Result { + verify_tls13_signature(message, cert, dss, &self.0.signature_verification_algorithms) + } + + fn supported_verify_schemes(&self) -> Vec { + self.0.signature_verification_algorithms.supported_schemes() + } +} + +pub fn required_vars(kind: &str) -> BTreeSet { + let mut vars = BTreeSet::new(); + vars.insert("TOKEN".to_string()); + + match kind { + "azurebatch" => { + vars.insert("BATCH_URL".to_string()); + } + "kraken" => { + vars.insert("KRAKEN_API_KEY".to_string()); + } + _ => {} + } + + vars +} + +pub async fn validate_raw( + kind: &str, + globals: &Object, + client: &Client, + use_lax_tls: bool, +) -> Result { + match kind { + "azurebatch" => validate_azure_batch(globals, client).await, + "ftp" => validate_ftp(globals, use_lax_tls).await, + "kraken" => validate_kraken(globals, client).await, + "ldap" => validate_ldap(globals, use_lax_tls).await, + "rabbitmq" => validate_rabbitmq(globals, use_lax_tls).await, + "redis" => validate_redis(globals, use_lax_tls).await, + other => Ok(RawValidationOutcome { + valid: false, + status: StatusCode::NOT_IMPLEMENTED, + body: format!("Raw validator `{other}` is not implemented."), + }), + } +} + +fn string_var(globals: &Object, name: &str) -> Option { + globals.get(name).map(|v| v.to_kstr().to_string()).filter(|s| !s.is_empty()) +} + +fn decode_userinfo(input: &str) -> String { + percent_decode_str(input).decode_utf8_lossy().to_string() +} + +fn current_unix_millis() -> String { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_else(|_| Duration::from_millis(0)) + .as_millis() + .to_string() +} + +fn rfc1123_now() -> String { + chrono::Utc::now().format("%a, %d %b %Y %H:%M:%S GMT").to_string() +} + +fn build_root_store() -> Result { + let mut roots = RootCertStore::empty(); + let native = rustls_native_certs::load_native_certs(); + for cert in native.certs { + roots.add(cert).map_err(|e| anyhow!("failed to add native root cert: {e:?}"))?; + } + Ok(roots) +} + +fn lax_provider() -> Arc { + LAX_PROVIDER.get_or_init(|| Arc::new(ring::default_provider())).clone() +} + +fn tls_connector(use_lax_tls: bool) -> Result { + let cfg = if use_lax_tls { + ensure_crypto_provider(); + ClientConfig::builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(LaxCertVerifier(lax_provider()))) + .with_no_client_auth() + } else { + ClientConfig::builder().with_root_certificates(build_root_store()?).with_no_client_auth() + }; + Ok(TlsConnector::from(Arc::new(cfg))) +} + +trait AsyncStream: AsyncRead + AsyncWrite + Unpin + Send {} +impl AsyncStream for T where T: AsyncRead + AsyncWrite + Unpin + Send {} +type DynStream = Box; + +async fn connect_plain(host: &str, port: u16) -> Result { + let stream = timeout(Duration::from_secs(10), TcpStream::connect((host, port))) + .await + .context("connection timed out")??; + Ok(Box::new(stream)) +} + +async fn connect_tls(host: &str, port: u16, use_lax_tls: bool) -> Result { + let stream = timeout(Duration::from_secs(10), TcpStream::connect((host, port))) + .await + .context("connection timed out")??; + let server_name = + ServerName::try_from(host.to_string()).map_err(|_| anyhow!("invalid TLS host: {host}"))?; + let tls = + timeout(Duration::from_secs(10), tls_connector(use_lax_tls)?.connect(server_name, stream)) + .await + .context("TLS handshake timed out")??; + Ok(Box::new(tls)) +} + +async fn connect_from_url( + url: &Url, + tls_default_port: u16, + plain_default_port: u16, + use_lax_tls: bool, +) -> Result { + let host = url.host_str().ok_or_else(|| anyhow!("URL is missing host"))?; + let tls = matches!(url.scheme(), "ftps" | "amqps" | "rediss" | "ldaps"); + let port = url.port().unwrap_or(if tls { tls_default_port } else { plain_default_port }); + if tls { + connect_tls(host, port, use_lax_tls).await + } else { + connect_plain(host, port).await + } +} + +async fn validate_azure_batch(globals: &Object, client: &Client) -> Result { + let endpoint = string_var(globals, "BATCH_URL").ok_or_else(|| anyhow!("missing BATCH_URL"))?; + let account_key = string_var(globals, "TOKEN").ok_or_else(|| anyhow!("missing TOKEN"))?; + let parsed = Url::parse(&endpoint).context("invalid BATCH_URL")?; + let host = parsed.host_str().ok_or_else(|| anyhow!("BATCH_URL is missing host"))?; + let account_name = host + .split('.') + .next() + .filter(|s| !s.is_empty()) + .ok_or_else(|| anyhow!("failed to derive Batch account name from host"))?; + + let api_version = "2020-09-01.12.0"; + let url = format!("{endpoint}/applications?api-version={api_version}"); + let date = rfc1123_now(); + let string_to_sign = format!( + "GET\n\n\n\n\napplication/json\n{}\n\n\n\n\n\n{}\napi-version:{}", + date, + format!("/{account_name}/applications").to_lowercase(), + api_version + ); + + let key = B64.decode(account_key.as_bytes()).context("Azure Batch key is not valid base64")?; + let mut mac = as KeyInit>::new_from_slice(&key) + .map_err(|e| anyhow!("invalid HMAC key: {e}"))?; + mac.update(string_to_sign.as_bytes()); + let signature = B64.encode(mac.finalize().into_bytes()); + + let resp = client + .get(&url) + .header("Content-Type", "application/json") + .header("Date", &date) + .header("Authorization", format!("SharedKey {account_name}:{signature}")) + .send() + .await + .context("Azure Batch validation request failed")?; + + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + let valid = status == StatusCode::OK; + + Ok(RawValidationOutcome { valid, status, body }) +} + +async fn validate_ftp(globals: &Object, use_lax_tls: bool) -> Result { + let token = string_var(globals, "TOKEN").ok_or_else(|| anyhow!("missing TOKEN"))?; + let url = Url::parse(&token).context("invalid FTP URI")?; + let host = url.host_str().ok_or_else(|| anyhow!("FTP URI is missing host"))?; + let username = decode_userinfo(url.username()); + let password = + decode_userinfo(url.password().ok_or_else(|| anyhow!("FTP URI is missing password"))?); + let scheme = url.scheme().to_ascii_lowercase(); + + let mut stream = if scheme == "ftp" { + BufStream::new(connect_plain(host, url.port().unwrap_or(21)).await?) + } else { + let port = url.port().unwrap_or(990); + if url.port().unwrap_or(990) == 990 { + BufStream::new(connect_tls(host, port, use_lax_tls).await?) + } else { + let tcp = timeout(Duration::from_secs(10), TcpStream::connect((host, port))) + .await + .context("connection timed out")??; + let mut plain = BufStream::new(tcp); + let _ = read_ftp_reply(&mut plain).await?; + plain.write_all(b"AUTH TLS\r\n").await?; + plain.flush().await?; + let (code, auth_body) = read_ftp_reply(&mut plain).await?; + if code != 234 { + return Ok(RawValidationOutcome { + valid: false, + status: StatusCode::UNAUTHORIZED, + body: auth_body, + }); + } + let tcp = plain.into_inner(); + let server_name = ServerName::try_from(host.to_string()) + .map_err(|_| anyhow!("invalid TLS host: {host}"))?; + let tls = timeout( + Duration::from_secs(10), + tls_connector(use_lax_tls)?.connect(server_name, tcp), + ) + .await + .context("TLS handshake timed out")??; + BufStream::new(Box::new(tls) as DynStream) + } + }; + + let _ = read_ftp_reply(&mut stream).await?; + stream.write_all(format!("USER {username}\r\n").as_bytes()).await?; + stream.flush().await?; + let (user_code, user_body) = read_ftp_reply(&mut stream).await?; + if user_code == 230 { + return Ok(RawValidationOutcome { valid: true, status: StatusCode::OK, body: user_body }); + } + if user_code != 331 { + return Ok(RawValidationOutcome { + valid: false, + status: StatusCode::UNAUTHORIZED, + body: user_body, + }); + } + + stream.write_all(format!("PASS {password}\r\n").as_bytes()).await?; + stream.flush().await?; + let (pass_code, pass_body) = read_ftp_reply(&mut stream).await?; + let _ = stream.write_all(b"QUIT\r\n").await; + let _ = stream.flush().await; + + Ok(RawValidationOutcome { + valid: pass_code == 230, + status: if pass_code == 230 { StatusCode::OK } else { StatusCode::UNAUTHORIZED }, + body: pass_body, + }) +} + +async fn read_ftp_reply(stream: &mut BufStream) -> Result<(u16, String)> +where + S: AsyncRead + AsyncWrite + Unpin, +{ + let mut body = String::new(); + let mut code_prefix: Option = None; + + loop { + let mut line = String::new(); + let read = timeout(Duration::from_secs(10), stream.read_line(&mut line)) + .await + .context("FTP server did not reply in time")??; + if read == 0 { + return Err(anyhow!("FTP server closed the connection")); + } + + body.push_str(&line); + let trimmed = line.trim_end_matches(['\r', '\n']); + if trimmed.len() < 4 { + continue; + } + + let code = &trimmed[0..3]; + if !code.chars().all(|c| c.is_ascii_digit()) { + continue; + } + + match trimmed.as_bytes()[3] { + b' ' => return Ok((code.parse().unwrap_or(0), body)), + b'-' => { + code_prefix = Some(code.to_string()); + } + _ => {} + } + + if let Some(prefix) = &code_prefix { + if trimmed.starts_with(prefix) && trimmed.as_bytes()[3] == b' ' { + return Ok((code.parse().unwrap_or(0), body)); + } + } + } +} + +async fn validate_kraken(globals: &Object, client: &Client) -> Result { + let api_key = + string_var(globals, "KRAKEN_API_KEY").ok_or_else(|| anyhow!("missing KRAKEN_API_KEY"))?; + let api_secret = string_var(globals, "TOKEN").ok_or_else(|| anyhow!("missing TOKEN"))?; + let secret = B64.decode(api_secret.as_bytes()).context("Kraken secret is not valid base64")?; + + let nonce = current_unix_millis(); + let body = format!("nonce={nonce}"); + let mut sha = Sha256::new(); + sha.update(format!("{nonce}{body}").as_bytes()); + let shasum = sha.finalize(); + + let path = "/0/private/Balance"; + let mut mac = as KeyInit>::new_from_slice(&secret) + .map_err(|e| anyhow!("invalid HMAC key: {e}"))?; + let mut payload = Vec::with_capacity(path.len() + shasum.len()); + payload.extend_from_slice(path.as_bytes()); + payload.extend_from_slice(&shasum); + mac.update(&payload); + let signature = B64.encode(mac.finalize().into_bytes()); + + let resp = client + .post(format!("https://api.kraken.com{path}")) + .header("Content-Type", "application/x-www-form-urlencoded") + .header("API-Key", api_key) + .header("API-Sign", signature) + .body(body) + .send() + .await + .context("Kraken validation request failed")?; + + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + let valid = status == StatusCode::OK && body.contains(r#""error":[]"#); + + Ok(RawValidationOutcome { valid, status, body }) +} + +async fn validate_ldap(globals: &Object, use_lax_tls: bool) -> Result { + let token = string_var(globals, "TOKEN").ok_or_else(|| anyhow!("missing TOKEN"))?; + let url = Url::parse(&token).context("invalid LDAP URI")?; + let scheme = url.scheme().to_ascii_lowercase(); + let host = url.host_str().ok_or_else(|| anyhow!("LDAP URI is missing host"))?; + let port = url.port().unwrap_or(if scheme == "ldaps" { 636 } else { 389 }); + let bind_dn = if let Some(bind_dn) = string_var(globals, "LDAP_BIND_DN") { + bind_dn + } else { + decode_userinfo(url.username()) + }; + let password = if let Some(password) = string_var(globals, "LDAP_PASSWORD") { + password + } else { + decode_userinfo(url.password().ok_or_else(|| anyhow!("LDAP URI is missing password"))?) + }; + + let ldap_url = format!("{scheme}://{host}:{port}"); + let settings = LdapConnSettings::new().set_no_tls_verify(use_lax_tls); + let (conn, mut ldap) = ldap3::LdapConnAsync::with_settings(settings, &ldap_url) + .await + .with_context(|| format!("failed to connect to LDAP server {ldap_url}"))?; + ldap3::drive!(conn); + let bind_result = ldap.simple_bind(&bind_dn, &password).await; + let _ = ldap.unbind().await; + + match bind_result { + Ok(res) => match res.success() { + Ok(_) => Ok(RawValidationOutcome { + valid: true, + status: StatusCode::OK, + body: "LDAP bind succeeded.".to_string(), + }), + Err(err) => Ok(RawValidationOutcome { + valid: false, + status: StatusCode::UNAUTHORIZED, + body: err.to_string(), + }), + }, + Err(err) => Ok(RawValidationOutcome { + valid: false, + status: StatusCode::BAD_GATEWAY, + body: err.to_string(), + }), + } +} + +async fn validate_rabbitmq(globals: &Object, use_lax_tls: bool) -> Result { + let token = string_var(globals, "TOKEN").ok_or_else(|| anyhow!("missing TOKEN"))?; + let url = Url::parse(&token).context("invalid AMQP URI")?; + let _host = url.host_str().ok_or_else(|| anyhow!("AMQP URI is missing host"))?; + let username = decode_userinfo(url.username()); + let password = + decode_userinfo(url.password().ok_or_else(|| anyhow!("AMQP URI is missing password"))?); + + let mut stream = connect_from_url(&url, 5671, 5672, use_lax_tls).await?; + timeout(Duration::from_secs(10), stream.write_all(b"AMQP\x00\x00\x09\x01")) + .await + .context("failed to write AMQP protocol header")??; + timeout(Duration::from_secs(10), stream.flush()).await.context("flush timed out")??; + + let (_, _, start_payload) = read_amqp_frame(&mut stream).await?; + let (class_id, method_id) = amqp_method_ids(&start_payload)?; + if class_id != 10 || method_id != 10 { + return Ok(RawValidationOutcome { + valid: false, + status: StatusCode::BAD_GATEWAY, + body: format!("unexpected AMQP frame {class_id}.{method_id}"), + }); + } + + let start_ok = build_amqp_start_ok_frame(&username, &password); + timeout(Duration::from_secs(10), stream.write_all(&start_ok)) + .await + .context("failed to write AMQP start-ok frame")??; + timeout(Duration::from_secs(10), stream.flush()).await.context("flush timed out")??; + + let (_, _, next_payload) = read_amqp_frame(&mut stream).await?; + let (class_id, method_id) = amqp_method_ids(&next_payload)?; + let valid = class_id == 10 && method_id == 30; + Ok(RawValidationOutcome { + valid, + status: if valid { StatusCode::OK } else { StatusCode::UNAUTHORIZED }, + body: format!("received AMQP method frame {class_id}.{method_id}"), + }) +} + +fn build_amqp_start_ok_frame(username: &str, password: &str) -> Vec { + let mut payload = Vec::new(); + payload.extend_from_slice(&10u16.to_be_bytes()); + payload.extend_from_slice(&11u16.to_be_bytes()); + payload.extend_from_slice(&0u32.to_be_bytes()); // empty client properties table + + payload.extend_from_slice(&(5u32).to_be_bytes()); + payload.extend_from_slice(b"PLAIN"); + + let mut response = Vec::with_capacity(username.len() + password.len() + 2); + response.push(0); + response.extend_from_slice(username.as_bytes()); + response.push(0); + response.extend_from_slice(password.as_bytes()); + payload.extend_from_slice(&(response.len() as u32).to_be_bytes()); + payload.extend_from_slice(&response); + + payload.extend_from_slice(&(5u32).to_be_bytes()); + payload.extend_from_slice(b"en_US"); + + let mut frame = Vec::with_capacity(payload.len() + 8); + frame.push(1); // method frame + frame.extend_from_slice(&0u16.to_be_bytes()); + frame.extend_from_slice(&(payload.len() as u32).to_be_bytes()); + frame.extend_from_slice(&payload); + frame.push(0xCE); + frame +} + +async fn read_amqp_frame(stream: &mut DynStream) -> Result<(u8, u16, Vec)> { + let mut header = [0u8; 7]; + timeout(Duration::from_secs(10), stream.read_exact(&mut header)) + .await + .context("timed out while reading AMQP frame header")??; + let frame_type = header[0]; + let channel = u16::from_be_bytes([header[1], header[2]]); + let size = u32::from_be_bytes([header[3], header[4], header[5], header[6]]) as usize; + let mut payload = vec![0u8; size]; + timeout(Duration::from_secs(10), stream.read_exact(&mut payload)) + .await + .context("timed out while reading AMQP frame payload")??; + let mut end = [0u8; 1]; + timeout(Duration::from_secs(10), stream.read_exact(&mut end)) + .await + .context("timed out while reading AMQP frame terminator")??; + if end[0] != 0xCE { + return Err(anyhow!("invalid AMQP frame terminator")); + } + Ok((frame_type, channel, payload)) +} + +fn amqp_method_ids(payload: &[u8]) -> Result<(u16, u16)> { + if payload.len() < 4 { + return Err(anyhow!("AMQP payload too short")); + } + Ok((u16::from_be_bytes([payload[0], payload[1]]), u16::from_be_bytes([payload[2], payload[3]]))) +} + +async fn validate_redis(globals: &Object, use_lax_tls: bool) -> Result { + let token = string_var(globals, "TOKEN").ok_or_else(|| anyhow!("missing TOKEN"))?; + let url = Url::parse(&token).context("invalid Redis URI")?; + let username = if let Some(username) = string_var(globals, "USERNAME") { + username + } else if !url.username().is_empty() { + decode_userinfo(url.username()) + } else { + String::new() + }; + let password = if let Some(password) = string_var(globals, "PASSWORD") { + password + } else { + decode_userinfo(url.password().ok_or_else(|| anyhow!("Redis URI is missing password"))?) + }; + + let mut stream = BufStream::new(connect_from_url(&url, 6380, 6379, use_lax_tls).await?); + let auth_cmd = if username.is_empty() { + format!("*2\r\n$4\r\nAUTH\r\n${}\r\n{}\r\n", password.len(), password) + } else { + format!( + "*3\r\n$4\r\nAUTH\r\n${}\r\n{}\r\n${}\r\n{}\r\n", + username.len(), + username, + password.len(), + password + ) + }; + stream.write_all(auth_cmd.as_bytes()).await?; + stream.flush().await?; + let auth_reply = read_resp_line(&mut stream).await?; + if !auth_reply.starts_with("+OK") { + return Ok(RawValidationOutcome { + valid: false, + status: StatusCode::UNAUTHORIZED, + body: auth_reply, + }); + } + + stream.write_all(b"*1\r\n$4\r\nPING\r\n").await?; + stream.flush().await?; + let ping_reply = read_resp_line(&mut stream).await?; + Ok(RawValidationOutcome { + valid: ping_reply.starts_with("+PONG"), + status: if ping_reply.starts_with("+PONG") { + StatusCode::OK + } else { + StatusCode::UNAUTHORIZED + }, + body: ping_reply, + }) +} + +async fn read_resp_line(stream: &mut BufStream) -> Result +where + S: AsyncRead + AsyncWrite + Unpin, +{ + let mut line = String::new(); + timeout(Duration::from_secs(10), stream.read_line(&mut line)) + .await + .context("Redis server did not reply in time")??; + Ok(line) +} diff --git a/data/default/rule_cleanup/count_rules.py b/data/default/rule_cleanup/count_rules.py index a390464..7a54ffe 100644 --- a/data/default/rule_cleanup/count_rules.py +++ b/data/default/rule_cleanup/count_rules.py @@ -38,6 +38,11 @@ def parse_args() -> argparse.Namespace: default=DEFAULT_RULES_DIR, help="Directory containing rule YAML files (default: %(default)s)", ) + parser.add_argument( + "--list-validators", + action="store_true", + help="Print the names of detectors with and without a validator", + ) return parser.parse_args() @@ -74,6 +79,8 @@ def main() -> int: total_rules = 0 dependent_rules = 0 + with_validator: list[str] = [] + without_validator: list[str] = [] for path in rule_files: try: @@ -86,14 +93,28 @@ def main() -> int: dependent_rules += sum( 1 for rule in rules if rule.get("depends_on_rule") ) + if any(rule.get("validation") for rule in rules): + with_validator.append(path.stem) + else: + without_validator.append(path.stem) detector_rules = total_rules - dependent_rules print(f"Rules directory: {rules_dir}") - print(f"Rule files: {len(rule_files)}") + print(f"Detectors: {len(rule_files)}") + print(f"Detectors with validator: {len(with_validator)}") + print(f"Detectors without validator: {len(without_validator)}") print(f"Total rules: {total_rules}") print(f"Dependent rules: {dependent_rules}") - print(f"Detectors: {detector_rules}") + print(f"Non-dependent rules: {detector_rules}") + + if args.list_validators: + print(f"\nWith validator ({len(with_validator)}):") + for name in with_validator: + print(f" {name}") + print(f"\nWithout validator ({len(without_validator)}):") + for name in without_validator: + print(f" {name}") return 0 diff --git a/docs-site/docs/reference/library.md b/docs-site/docs/reference/library.md index 8b4d871..15b5f15 100644 --- a/docs-site/docs/reference/library.md +++ b/docs-site/docs/reference/library.md @@ -262,7 +262,7 @@ flowchart TD ### Loading Builtin Rules -Kingfisher comes with 700+ builtin rules for common secret types: +Kingfisher comes with 800+ builtin rules for common secret types: ```rust use kingfisher_rules::{get_builtin_rules, Confidence}; diff --git a/docs-site/docs/usage/advanced.md b/docs-site/docs/usage/advanced.md index 6d06d28..5e95c8c 100644 --- a/docs-site/docs/usage/advanced.md +++ b/docs-site/docs/usage/advanced.md @@ -300,7 +300,7 @@ kingfisher scan ./my-project \ ## Custom Rules -Kingfisher ships with 700+ rules, but you may want to add your own custom rules or modify existing detection to better suit your needs. +Kingfisher ships with 800+ rules, but you may want to add your own custom rules or modify existing detection to better suit your needs. First, review [RULES.md](../rules/overview.md) to learn how to create custom Kingfisher rules. diff --git a/docs/ADVANCED.md b/docs/ADVANCED.md index bf9bcf0..cb35da1 100644 --- a/docs/ADVANCED.md +++ b/docs/ADVANCED.md @@ -297,7 +297,7 @@ kingfisher scan ./my-project \ ## Custom Rules -Kingfisher ships with 700+ rules, but you may want to add your own custom rules or modify existing detection to better suit your needs. +Kingfisher ships with 800+ rules, but you may want to add your own custom rules or modify existing detection to better suit your needs. First, review [RULES.md](RULES.md) to learn how to create custom Kingfisher rules. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 448e20a..3a2366f 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -110,7 +110,7 @@ flowchart LR - `src/parser.rs`: tree-sitter integration for language-aware parsing, supporting 17+ languages (Bash, C, C#, C++, CSS, Go, HTML, Java, JavaScript, PHP, Python, Ruby, Rust, TOML, TypeScript, YAML, and regex). - `src/scanner_pool.rs`: thread-local vectorscan `BlockScanner` pool, providing safe reuse of compiled pattern databases across scan threads. - `src/reporter.rs` and `src/reporter/*`: report rendering for pretty, JSON, BSON, TOON, SARIF, and HTML outputs, plus the data model used by the viewer. -- `src/direct_validate.rs`: direct validation of a known secret without going through pattern matching. Supports HTTP, AWS, Azure, GCP, JDBC, MongoDB, MySQL, PostgreSQL, JWT, and Coinbase validators, with Liquid template integration for custom validation logic. +- `src/direct_validate.rs`: direct validation of a known secret without going through pattern matching. Supports HTTP, gRPC, plus schema-level typed validators such as AWS, AzureStorage, GCP, JDBC, MongoDB, MySQL, PostgreSQL, JWT, and Coinbase, and delegates ad-hoc `Raw` validators to `crates/kingfisher-scanner/src/validation/raw.rs`. - `src/direct_revoke.rs`: direct revocation of a known secret without going through the scan pipeline. Uses Liquid templates for revocation configurations and supports multi-step HTTP revocation flows. - `src/access_map.rs` and `src/access_map/*`: standalone blast-radius mapping with 24 provider implementations including AWS, Azure, GCP, GitHub, GitLab, Slack, Bitbucket, Gitea, Hugging Face, Buildkite, Anthropic, OpenAI, and more. @@ -118,6 +118,7 @@ flowchart LR - The main CLI scan path is implemented primarily in the application modules under `src/`, not in `kingfisher-scanner`. - `kingfisher-scanner` is still important: it provides the embeddable scanner API plus shared validation and primitive functionality reused by the application. +- The shared validation layer in `crates/kingfisher-scanner/src/validation/` contains both reusable typed validator families and the `Raw` exception-path validators used by rule YAML. - Direct `validate`, `revoke`, and standalone `access-map` are sibling command paths. They are not downstream stages of `FindingsStore`. - Reporting is downstream from the datastore, which lets Kingfisher emit multiple output formats and drive the local viewer from the same finding set. - The matching layer is intentionally hybrid: vectorscan provides high-throughput SIMD-accelerated pattern detection, while regex helpers, Base64 support, and tree-sitter verification improve accuracy and reduce false positives. diff --git a/docs/LIBRARY.md b/docs/LIBRARY.md index 1b51fff..b2174fb 100644 --- a/docs/LIBRARY.md +++ b/docs/LIBRARY.md @@ -36,7 +36,13 @@ The `kingfisher-scanner` crate supports optional validation features: | ------- | ----------- | | `validation` | Core validation support (includes HTTP validation) | | `validation-http` | HTTP-based validation for API tokens | +| `validation-raw` | Provider/protocol-specific raw validation flows for `validation: type: Raw` rules | | `validation-aws` | AWS credential validation via STS GetCallerIdentity | +| `validation-azure` | Azure storage credential validation | +| `validation-coinbase` | Coinbase credential validation | +| `validation-gcp` | GCP credential validation | +| `validation-jwt` | JWT validation | +| `validation-database` | MongoDB, MySQL, PostgreSQL, and JDBC validation | | `validation-all` | Enable all validation features | ## Quick Start @@ -259,7 +265,7 @@ flowchart TD ### Loading Builtin Rules -Kingfisher comes with 700+ builtin rules for common secret types: +Kingfisher comes with 800+ builtin rules for common secret types: ```rust use kingfisher_rules::{get_builtin_rules, Confidence}; @@ -724,9 +730,17 @@ kingfisher-scanner = { git = "https://github.com/mongodb/kingfisher", features = | ------- | ----------- | | `validation` | Core validation support with HTTP validation | | `validation-http` | HTTP-based validation for API tokens | +| `validation-raw` | Provider/protocol-specific raw validation flows for `validation: type: Raw` rules | | `validation-aws` | AWS credential validation via STS | +| `validation-azure` | Azure storage credential validation | +| `validation-coinbase` | Coinbase credential validation | +| `validation-gcp` | GCP credential validation | +| `validation-jwt` | JWT validation | +| `validation-database` | MongoDB, MySQL, PostgreSQL, and JDBC validation | | `validation-all` | Enable all validation features | +`validation: type: Raw` is the ad-hoc validator path for provider-specific or protocol-specific checks that are not generic enough to become schema-level validator families. Typed validators such as `AWS`, `GCP`, `MongoDB`, and `JWT` remain separate validator kinds in the rule schema. + ### HTTP Validation Example ```rust diff --git a/docs/RULES.md b/docs/RULES.md index 5fe9bad..6386da9 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -168,9 +168,29 @@ revocation: | visible | false to hide non‑secret captures (e.g. IDs) | | depends_on_rule | Chain rules: use captures from one rule in another's validation | | pattern_requirements | Require character types and/or exclude placeholder words from matches | -| validation | Configure HTTP, AWS, GCP, etc. checks to verify live validity | +| validation | Configure `Http`, `Grpc`, typed validators (`AWS`, `GCP`, etc.), or `Raw` exception-path checks to verify live validity | | revocation | Configure HTTP, AWS, or multi-step revocation for a detected secret | +## Validation Types + +Kingfisher supports three validation buckets: + +1. `Http` and `Grpc`: YAML-native validation flows. Prefer these first. +2. Typed validators: schema-level validation families already modeled in the rule schema, such as `AWS`, `AzureStorage`, `Coinbase`, `GCP`, `MongoDB`, `MySQL`, `Postgres`, `Jdbc`, and `JWT`. +3. Raw validators: provider-specific or protocol-specific exception paths dispatched through `validation: type: Raw`. + +Raw validation looks like this: + +```yaml +validation: + type: Raw + content: kraken +``` + +Use `Raw` only when the provider check cannot be expressed reliably with `Http` or `Grpc` and does not justify a new reusable validator family. Raw validator implementations live in `crates/kingfisher-scanner/src/validation/raw.rs`. + +Typed validators are safer and more reusable because the validator kind is part of the schema. `Raw` validators are string-dispatched and fail at runtime if the `content` name is unknown. If you need a Rust-backed exception path for one provider, prefer `Raw`; reserve new typed validators for stable validation families that can be reused across rules. + ## gRPC Validation (Grpc) Some services (notably CLI/SDK control planes) are **gRPC-only**. For these, `validation: type: Http` @@ -468,6 +488,7 @@ Below is the complete list of Liquid filters available in Kingfisher, along with | `hmac_sha1` | `key` (string) | Computes HMAC-SHA1 over the input, returns Base64-encoded result. | `{{ TOKEN \| hmac_sha1: "secret-key" }}` | | `hmac_sha256` | `key` (string) | Computes HMAC-SHA256 over the input, returns Base64-encoded result. | `{{ TOKEN \| hmac_sha256: "secret-key" }}` | | `hmac_sha384` | `key` (string) | Computes HMAC-SHA384 over the input, returns Base64-encoded result. | `{{ TOKEN \| hmac_sha384: "secret-key" }}` | +| `hmac_sha384_hex` | `key` (string) | Computes HMAC-SHA384 over the input, returns lowercase hexadecimal output. | `{{ TOKEN \| hmac_sha384_hex: "secret-key" }}` | | `hmac_sha256_b64key` | `key` (string, base64-encoded) | Decodes the key from Base64 to raw bytes, then computes HMAC-SHA256. Returns Base64. Use for Azure SAS and other protocols where the signing key is base64-encoded. | `{{ to_sign \| hmac_sha256_b64key: TOKEN }}` | | `random_string` | `len` (integer, optional) | Generates a cryptographically-secure random alphanumeric string of the specified length (default: 32). | `{{ "" \| random_string: 16 }}` | | `prefix` | `len` (integer, optional) | Returns the first `len` characters from the string (default: full). | `{{ TOKEN \| prefix: 6 }}` | @@ -476,8 +497,10 @@ Below is the complete list of Liquid filters available in Kingfisher, along with | `url_encode` | – | Percent-encodes the input according to RFC 3986. | `{{ TOKEN \| url_encode }}` | | `json_escape` | – | Escapes special characters so a string can be safely injected into JSON contexts. | `{{ TOKEN \| json_escape }}` | | `unix_timestamp` | – | Returns the current Unix epoch time in seconds (UTC). | `{{ "" \| unix_timestamp }}` | +| `unix_timestamp_ms` | – | Returns the current Unix epoch time in milliseconds (UTC). | `{{ "" \| unix_timestamp_ms }}` | | `iso_timestamp` | – | Returns the current UTC timestamp in full ISO-8601 format (may include fractional seconds). | `{{ "" \| iso_timestamp }}` | | `iso_timestamp_no_frac` | – | Current ISO-8601 timestamp (UTC) **without** fractional seconds. | `{{ "" \| iso_timestamp_no_frac }}` | +| `rfc1123_date` | – | Returns the current RFC-1123 timestamp in GMT. | `{{ "" \| rfc1123_date }}` | | `uuid` | – | Generates a random UUIDv4 string. | `{{ "" \| uuid }}` | | `jwt_header` | – | Builds a minimal JWT header JSON (`{"typ":"JWT","alg":…}`) and Base64URL-encodes it. | `{{ "HS256" \| jwt_header }}` | | `replace` | `from` (string), `to` (string) | Replaces every occurrence of `from` with `to` in the input string. | `{{ "hello world" \| replace: "world", "mars" }}` | @@ -492,6 +515,11 @@ Authorization: Basic {{ "api:" | append: TOKEN | b64enc }} ``` **Runtime Values:** Filters like unix_timestamp and uuid are evaluated at runtime, enabling nonces, timestamps, and unique IDs in your requests. + +**Stable Request Values:** HTTP and gRPC validation requests also expose stable per-request template variables. Use these when the same generated value must appear in multiple places within one request. Currently: +- `REQUEST_RFC1123_DATE` +- `REQUEST_UNIX_MILLIS` + ### How depends_on_rule Works - **Dependency Declaration:** @@ -738,7 +766,7 @@ When writing custom rules, consider the following best practices: 1. **Multi-line Regex:** Write your regex patterns over multiple lines for clarity. Use the `(?x)` flag to enable free-spacing mode. 2. **Optimize for Performance:** Structure your regex to minimize backtracking. Use non-capturing groups where possible and keep the pattern as concise as possible. -3. **Validation Integration:** Define a `validation` section if you want to verify the detected secret. You can use Liquid templating to insert dynamic values—use the unnamed capture as `TOKEN` and any named captures in uppercase. +3. **Validation Integration:** Define a `validation` section if you want to verify the detected secret. Prefer `Http` or `Grpc`; use an existing typed validator when the rule matches a supported validator family; use `Raw` only for rare provider-specific exception paths. You can use Liquid templating to insert dynamic values where supported. Use the unnamed capture as `TOKEN` and any named captures in uppercase. 4. **Revocation Integration:** Define a `revocation` section if you want to revoke a detected secret. It uses the same HTTP request format and template variables as `validation`. 5. **Test with Examples:** Always include examples that should match and, optionally, negative examples to ensure your rule behaves as expected. @@ -915,4 +943,4 @@ rules: words: ['"Arn"'] depends_on_rule: - rule_id: kingfisher.alibabacloud.1 - variable: AKID``` \ No newline at end of file + variable: AKID``` diff --git a/src/direct_validate.rs b/src/direct_validate.rs index 5d68d22..1cea920 100644 --- a/src/direct_validate.rs +++ b/src/direct_validate.rs @@ -133,6 +133,10 @@ fn extract_template_vars(text: &str) -> BTreeSet { re.captures_iter(text).filter_map(|cap| cap.get(1).map(|m| m.as_str().to_uppercase())).collect() } +fn is_auto_provided_request_var(var: &str) -> bool { + matches!(var, "REQUEST_RFC1123_DATE" | "REQUEST_UNIX_MILLIS") +} + /// Extract all template variables used in a validation configuration. fn extract_validation_vars(validation: &Validation) -> BTreeSet { let mut vars = BTreeSet::new(); @@ -201,11 +205,13 @@ fn extract_validation_vars(validation: &Validation) -> BTreeSet { vars.insert("TOKEN".to_string()); vars.insert("CRED_NAME".to_string()); } - Validation::Raw(_) => { - vars.insert("TOKEN".to_string()); + Validation::Raw(raw) => { + vars.extend(kingfisher_scanner::validation::raw::required_vars(raw)); } } + vars.retain(|var| !is_auto_provided_request_var(var)); + vars } @@ -298,8 +304,10 @@ async fn execute_http_validation( retries: u32, allow_internal_ips: bool, ) -> Result { + let request_globals = kingfisher_scanner::validation::with_request_template_globals(globals); + // Render the URL - let url = render_and_parse_url(parser, globals, &http_validation.request.url).await?; + let url = render_and_parse_url(parser, &request_globals, &http_validation.request.url).await?; // SSRF check: verify the resolved IP is public before making the request crate::validation::utils::check_url_resolvable(&url, allow_internal_ips) @@ -317,7 +325,7 @@ async fn execute_http_validation( &http_validation.request.body, timeout, parser, - globals, + &request_globals, ) .map_err(|e| anyhow!("Failed to build request: {}", e))?; @@ -359,8 +367,11 @@ async fn execute_grpc_validation( timeout: Duration, allow_internal_ips: bool, ) -> Result { + let request_globals = kingfisher_scanner::validation::with_request_template_globals(globals); + // Render the URL - let url = render_and_parse_url(parser, globals, &grpc_validation_cfg.request.url).await?; + let url = + render_and_parse_url(parser, &request_globals, &grpc_validation_cfg.request.url).await?; // SSRF check: verify the resolved IP is public before making the request crate::validation::utils::check_url_resolvable(&url, allow_internal_ips) @@ -374,7 +385,7 @@ async fn execute_grpc_validation( &grpc_validation_cfg.request.headers, &grpc_validation_cfg.request.body, parser, - globals, + &request_globals, timeout, ) .await @@ -840,13 +851,31 @@ pub async fn run_direct_validation( } } - Validation::Raw(_) => DirectValidationResult { - rule_id: String::new(), - rule_name: String::new(), - is_valid: false, - status_code: None, - message: "Raw validation type is not supported via direct validation.".to_string(), - }, + Validation::Raw(raw) => { + match kingfisher_scanner::validation::raw::validate_raw( + raw, + &globals, + &client, + use_lax_tls, + ) + .await + { + Ok(result) => DirectValidationResult { + rule_id: String::new(), + rule_name: String::new(), + is_valid: result.valid, + status_code: Some(result.status.as_u16()), + message: result.body, + }, + Err(e) => DirectValidationResult { + rule_id: String::new(), + rule_name: String::new(), + is_valid: false, + status_code: None, + message: format!("Raw validation error: {}", e), + }, + } + } }; result.rule_id = rule_id; diff --git a/src/reporter.rs b/src/reporter.rs index b01c6df..d1a5a24 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -86,6 +86,10 @@ fn extract_template_vars(text: &str) -> BTreeSet { vars } +fn is_auto_provided_request_var(var: &str) -> bool { + matches!(var, "REQUEST_RFC1123_DATE" | "REQUEST_UNIX_MILLIS") +} + fn required_vars_for_validation(validation: &crate::rules::Validation) -> BTreeSet { use crate::rules::Validation; let mut vars = BTreeSet::new(); @@ -133,11 +137,13 @@ fn required_vars_for_validation(validation: &crate::rules::Validation) -> BTreeS vars.insert("TOKEN".to_string()); vars.insert("CRED_NAME".to_string()); } - Validation::Raw(_) => { - vars.insert("TOKEN".to_string()); + Validation::Raw(raw) => { + vars.extend(kingfisher_scanner::validation::raw::required_vars(raw)); } } + vars.retain(|var| !is_auto_provided_request_var(var)); + vars } diff --git a/src/validation.rs b/src/validation.rs index fd33662..3e90b07 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -614,10 +614,11 @@ async fn timed_validate_single_match<'a>( let request_timeout = validation_timeout; let multipart_timeout = validation_timeout; let max_retries: u32 = validation_retries; + let request_globals = httpvalidation::with_request_template_globals(&globals); // render URL let url = match render_and_parse_url( parser, - &globals, + &request_globals, &rule_syntax.name, &http_validation.request.url, clients.allow_internal_ips, @@ -643,7 +644,7 @@ async fn timed_validate_single_match<'a>( &http_validation.request.body, request_timeout, parser, - &globals, + &request_globals, ) { Ok(rb) => rb, Err(e) => { @@ -663,7 +664,7 @@ async fn timed_validate_single_match<'a>( let rendered_headers = httpvalidation::process_headers( &http_validation.request.headers, parser, - &globals, + &request_globals, &url, ) .unwrap_or_default(); @@ -681,7 +682,7 @@ async fn timed_validate_single_match<'a>( parser .parse(body_template) .ok() - .and_then(|template| template.render(&globals).ok()) + .and_then(|template| template.render(&request_globals).ok()) }); cache_key = httpvalidation::generate_http_cache_key_parts( @@ -726,7 +727,7 @@ async fn timed_validate_single_match<'a>( if let Ok(mut headers) = httpvalidation::process_headers( &http_validation.request.headers, parser, - &globals, + &request_globals, &url, ) { // add realistic UA & accept headers @@ -752,7 +753,7 @@ async fn timed_validate_single_match<'a>( "file" => { let path = render_template( parser, - &globals, + &request_globals, &rule_syntax.name, &part.content, ) @@ -771,7 +772,7 @@ async fn timed_validate_single_match<'a>( "text" => { let txt = render_template( parser, - &globals, + &request_globals, &rule_syntax.name, &part.content, ) @@ -872,11 +873,12 @@ async fn timed_validate_single_match<'a>( // ---------------------------------------------------- gRPC validator Some(Validation::Grpc(grpc_validation_cfg)) => { let request_timeout = validation_timeout; + let request_globals = httpvalidation::with_request_template_globals(&globals); // Render URL let url = match render_and_parse_url( parser, - &globals, + &request_globals, &rule_syntax.name, &grpc_validation_cfg.request.url, clients.allow_internal_ips, @@ -899,7 +901,7 @@ async fn timed_validate_single_match<'a>( &grpc_validation_cfg.request.headers, &grpc_validation_cfg.request.body, parser, - &globals, + &request_globals, request_timeout, ) .await @@ -1481,11 +1483,27 @@ async fn timed_validate_single_match<'a>( } // --------------------------------------------------------- Raw / none Some(Validation::Raw(raw)) => { - debug!("Raw validation not implemented: {}", raw); - m.validation_success = false; - m.validation_response_body = - validation_body::from_string("Validator not implemented".to_string()); - m.validation_response_status = StatusCode::NOT_IMPLEMENTED; + match kingfisher_scanner::validation::raw::validate_raw( + raw, + &globals, + client, + clients.should_use_lax(rule_syntax.tls_mode), + ) + .await + { + Ok(result) => { + m.validation_success = result.valid; + m.validation_response_body = validation_body::from_string(result.body); + m.validation_response_status = result.status; + } + Err(e) => { + debug!("Raw validation error for {}: {}", raw, e); + m.validation_success = false; + m.validation_response_body = + validation_body::from_string(format!("Raw validation error: {}", e)); + m.validation_response_status = StatusCode::BAD_GATEWAY; + } + } } None => { /* no validation specified */ } } diff --git a/src/validation_rate_limit.rs b/src/validation_rate_limit.rs index 8b1ac1c..9eb9eb8 100644 --- a/src/validation_rate_limit.rs +++ b/src/validation_rate_limit.rs @@ -118,7 +118,10 @@ fn selector_matches(rule_id: &str, selector: &str) -> bool { } pub fn should_rate_limit_validation(validation: &Validation) -> bool { - !matches!(validation, Validation::Raw(_)) + match validation { + Validation::Raw(raw) => raw != "custom", + _ => true, + } } #[cfg(test)] From 4cc1a3413065798aef2667115b489a014671295d Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Mon, 6 Apr 2026 22:25:54 -0700 Subject: [PATCH 03/17] added more rules --- crates/kingfisher-rules/data/rules/highnote.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/kingfisher-rules/data/rules/highnote.yml b/crates/kingfisher-rules/data/rules/highnote.yml index 953ff32..cfdffcb 100644 --- a/crates/kingfisher-rules/data/rules/highnote.yml +++ b/crates/kingfisher-rules/data/rules/highnote.yml @@ -4,6 +4,9 @@ rules: pattern: | (?x) \b + (?i:highnote) + (?:.|[\n\r]){0,24}? + \b ( (?:sk|rk)_(?:live|test)_[a-zA-Z0-9]{20,60} ) From f72d0c06225dd5e5c0f81fc8d1935d543072f681 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Mon, 6 Apr 2026 22:36:09 -0700 Subject: [PATCH 04/17] Bump jsonwebtoken to 10.3.0 --- Cargo.toml | 2 +- crates/kingfisher-scanner/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e4f8b78..b8607b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -226,7 +226,7 @@ percent-encoding = "2.3.2" self_update = { version = "0.43.1", default-features = false, features = ["reqwest", "rustls", "archive-tar", "archive-zip", "compression-flate2"] } semver = "1.0.27" globset = "0.4.18" -jsonwebtoken = { version = "10.2.0", features = ["aws-lc-rs"] } +jsonwebtoken = { version = "10.3.0", features = ["aws-lc-rs"] } ipnet = "2.11.0" gouqi = { version = "0.20.0", features = ["async"] } oci-client = { version = "0.16", default-features = false, features = ["rustls-tls"] } diff --git a/crates/kingfisher-scanner/Cargo.toml b/crates/kingfisher-scanner/Cargo.toml index f5b5dd2..013d0a7 100644 --- a/crates/kingfisher-scanner/Cargo.toml +++ b/crates/kingfisher-scanner/Cargo.toml @@ -183,7 +183,7 @@ pem = { version = "3.0.6", optional = true } percent-encoding = { workspace = true, optional = true } ring = { version = "0.17", optional = true } -jsonwebtoken = { version = "10.2.0", features = ["aws-lc-rs"], optional = true } +jsonwebtoken = { version = "10.3.0", features = ["aws-lc-rs"], optional = true } p256 = { version = "0.13.2", optional = true } ed25519-dalek = { version = "2.2", features = ["pkcs8"], optional = true } hex = { workspace = true, optional = true } From 413798e27d7f9de8a16481712b87e6799dea8b46 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Mon, 6 Apr 2026 23:58:55 -0700 Subject: [PATCH 05/17] Apply open Dependabot updates --- .github/workflows/docs.yml | 6 +++--- Cargo.lock | 24 ++++++++++++------------ Cargo.toml | 2 +- crates/kingfisher-scanner/Cargo.toml | 4 ++-- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f6cf749..4111f6b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -27,7 +27,7 @@ jobs: - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0 - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.12' @@ -46,7 +46,7 @@ jobs: CI: true - name: Upload artifact - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 + uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4.0.0 with: path: docs-site/site @@ -59,4 +59,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 diff --git a/Cargo.lock b/Cargo.lock index 54f32a1..36fd53a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1611,9 +1611,9 @@ checksum = "de0758edba32d61d1fd9f4d69491b47604b91ee2f7e6b33de7e54ca4ebe55dc3" [[package]] name = "color-backtrace" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308329d5d62e877ba02943db3a8e8c052de9fde7ab48283395ba0e6494efbabd" +checksum = "83c39683d44e712e45134c852c21c2f60139c3846047c9dde39cddf7066c78c6" dependencies = [ "backtrace", "termcolor", @@ -7156,7 +7156,6 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", "http 1.4.0", "http-body 1.0.1", "http-body-util", @@ -7202,6 +7201,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", + "h2", "http 1.4.0", "http-body 1.0.1", "http-body-util", @@ -7792,9 +7792,9 @@ dependencies = [ [[package]] name = "self_update" -version = "0.43.1" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6644febaa58f323b28f7321d04e24d0020d117c27619ab869d6abdf76be9aac6" +checksum = "2e79722b5a505d4ddc77527455a97244e9e8c4c07533ff44cf4421cce7bb6d17" dependencies = [ "either", "flate2", @@ -7803,7 +7803,7 @@ dependencies = [ "log", "quick-xml 0.38.4", "regex", - "reqwest 0.12.28", + "reqwest 0.13.2", "self-replace", "semver", "serde", @@ -7818,9 +7818,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" dependencies = [ "serde", "serde_core", @@ -8789,9 +8789,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.50.0" +version = "1.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "2bd1c4c0fc4a7ab90fc15ef6daaa3ec3b893f004f915f2392557ed23237820cd" dependencies = [ "bytes", "libc", @@ -8806,9 +8806,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index b8607b8..3bc7011 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -223,7 +223,7 @@ bloomfilter = "3.0.1" uuid = "1.19.0" rand = "0.10.0" percent-encoding = "2.3.2" -self_update = { version = "0.43.1", default-features = false, features = ["reqwest", "rustls", "archive-tar", "archive-zip", "compression-flate2"] } +self_update = { version = "0.44.0", default-features = false, features = ["reqwest", "rustls", "archive-tar", "archive-zip", "compression-flate2"] } semver = "1.0.27" globset = "0.4.18" jsonwebtoken = { version = "10.3.0", features = ["aws-lc-rs"] } diff --git a/crates/kingfisher-scanner/Cargo.toml b/crates/kingfisher-scanner/Cargo.toml index 013d0a7..69d0d8c 100644 --- a/crates/kingfisher-scanner/Cargo.toml +++ b/crates/kingfisher-scanner/Cargo.toml @@ -170,7 +170,7 @@ tracing.workspace = true reqwest = { version = "0.12", default-features = false, features = [ "json", "gzip", "brotli", "deflate", "stream", "rustls-tls", "rustls-tls-native-roots", "multipart" ], optional = true } -tokio = { version = "1.48", features = ["net", "time", "sync", "io-util"], optional = true } +tokio = { version = "1.51", features = ["net", "time", "sync", "io-util"], optional = true } liquid = { version = "0.26", optional = true } liquid-core = { version = "0.26", optional = true } quick-xml = { version = "0.39", features = ["serde", "serialize"], optional = true } @@ -213,4 +213,4 @@ rand = { version = "0.10", optional = true } [dev-dependencies] pretty_assertions = "1.4" tempfile = "3.23" -tokio = { version = "1.48", features = ["macros", "rt"] } +tokio = { version = "1.51", features = ["macros", "rt"] } From afee0b7181228fd5de87f3796d9a12db5a8b626f Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Tue, 7 Apr 2026 10:42:44 -0700 Subject: [PATCH 06/17] updated rules --- CHANGELOG.md | 2 +- crates/kingfisher-rules/data/rules/AGENTS.md | 4 + crates/kingfisher-rules/data/rules/adobe.yml | 51 ++++++++++- crates/kingfisher-rules/data/rules/asaas.yml | 29 ++++++ crates/kingfisher-rules/data/rules/asana.yml | 26 ++++++ crates/kingfisher-rules/data/rules/azure.yml | 24 +++++ .../kingfisher-rules/data/rules/azuremaps.yml | 21 +++++ .../kingfisher-rules/data/rules/branchio.yml | 14 +++ .../data/rules/cockroachlabs.yml | 23 +++++ .../data/rules/databricks.yml | 24 ++++- crates/kingfisher-rules/data/rules/gitlab.yml | 21 +++++ crates/kingfisher-rules/data/rules/google.yml | 90 ++++++++++++++++++- .../data/rules/googleoauth2.yml | 17 +++- .../kingfisher-rules/data/rules/highnote.yml | 30 +++++++ .../kingfisher-rules/data/rules/langfuse.yml | 11 +-- .../kingfisher-rules/data/rules/posthog.yml | 16 ++++ crates/kingfisher-rules/data/rules/proof.yml | 27 ++++++ .../kingfisher-rules/data/rules/tableau.yml | 17 ++-- .../kingfisher-scanner/src/validation/raw.rs | 27 ++++++ data/default/rule_cleanup/count_rules.py | 63 +++++++++---- docs-site/docs/changelog.md | 5 ++ src/direct_validate.rs | 1 + src/reporter.rs | 8 +- src/validation.rs | 3 +- 24 files changed, 513 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f507c0..5ee1dde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. ## [v1.95.0] - Added 80+ built-in rules, bringing the bundled ruleset to 820 total. New coverage includes Amazon OAuth, Asaas, multiple Azure credential families, Bitrise, Canva, CockroachDB, eBay, Elastic, hCaptcha, Highnote, Lichess, MailerSend, Onfido, Paddle, Pangea, Persona, Pinterest, Proof, Rootly, Runpod, Telnyx, Thunderstore, Valtown, Volcengine, and more. - Added a `validation: type: Raw` exception path for provider-specific checks, with new raw validators for Azure Batch, FTP, Kraken, LDAP, RabbitMQ, and Redis. Also added stable request-scoped template values plus new Liquid filters for HMAC-SHA384 hex output and timestamp generation. -- Expanded live validation coverage for several built-in rules, including Agora, Bitfinex, DocuSign, Dwolla, GitLab, KuCoin, RingCentral, Snowflake, Tableau, Trello, and Webex, and fixed newly added rule patterns/examples so `kingfisher rules check` passes cleanly. +- Expanded live validation coverage for several built-in rules, including Agora, Bitfinex, DocuSign, Dwolla, GitLab, KuCoin, RingCentral, Snowflake, Tableau, Trello, and Webex. Also tightened newly added helper regex to avoid high-match scan regressions, and made preflight-blocked raw validations report as skipped/not attempted instead of failed. ## [v1.94.0] - Updated vendored `vectorscan-rs` from v0.0.5 (Vectorscan 5.4.11) to v0.0.6 (Vectorscan 5.4.12). The upstream crate now ships pre-extracted sources instead of a tarball+patch, and fixes the `cpu_native` feature flag. Local Windows and musl build patches have been re-applied. diff --git a/crates/kingfisher-rules/data/rules/AGENTS.md b/crates/kingfisher-rules/data/rules/AGENTS.md index 433d951..0287d66 100644 --- a/crates/kingfisher-rules/data/rules/AGENTS.md +++ b/crates/kingfisher-rules/data/rules/AGENTS.md @@ -30,6 +30,7 @@ Strongly recommended fields: ## Pattern Quality Rules - Prefer specific anchors/prefixes and provider context over broad generic regex. +- Keep helper/context regex narrow. Avoid patterns that match generic URLs, hostnames, query params, or assignments without strong provider-specific constraints; broad helpers can create huge match counts and cause major memory/time regressions on large repos and git history. - When the token format is generic or common-looking (for example bare 32-hex keys), prefer contextual patterns of the form: provider keyword -> short flexible gap -> key/secret label -> short flexible gap -> token. A good default is: - `\b` - provider identifier (for example `amplitude`, `azure`, `speech`, `translator`) @@ -83,6 +84,9 @@ Strongly recommended fields: - `cargo test -p kingfisher-rules` - Broader regression check: - `cargo test --workspace --all-targets` +- Match-volume check on a realistic large target: + - `kingfisher scan --rule-stats` + - Review unexpected high-match helper/generic rules before submitting. - **Warning-free build**: `cargo check` (or `make darwin` / `make linux`) must produce zero warnings. Address all `dead_code`, `unused_*`, and other warnings before submitting. Use `#[allow(dead_code)]` on individual struct fields kept for deserialization completeness, and remove truly unused code. - Behavioral check against sample content: - `kingfisher scan ./testdata --rule --rule-stats` diff --git a/crates/kingfisher-rules/data/rules/adobe.yml b/crates/kingfisher-rules/data/rules/adobe.yml index ec72a85..63a2062 100644 --- a/crates/kingfisher-rules/data/rules/adobe.yml +++ b/crates/kingfisher-rules/data/rules/adobe.yml @@ -73,4 +73,53 @@ rules: "client_credentials": { "client_id": "a65b0146769d433a835f36660881db50", "client_secret": "p8e-ibndcvsmAp9ZgPBZ606FSlYIZVlsZ-g5" - }, \ No newline at end of file + }, + depends_on_rule: + - rule_id: "kingfisher.adobe.4" + variable: ADOBE_CLIENT_ID + validation: + type: Http + content: + request: + method: POST + url: https://ims-na1.adobelogin.com/ims/token/v3 + headers: + Authorization: 'Basic {{ ADOBE_CLIENT_ID | append: ":" | append: TOKEN | b64enc }}' + Content-Type: application/x-www-form-urlencoded + Accept: application/json + body: 'code=invalid_code&grant_type=authorization_code' + response_matcher: + - report_response: true + - type: StatusMatch + status: [400] + - type: WordMatch + words: + - invalid_client + negative: true + # Revocation not added: Adobe documents revocation for access and refresh + # tokens, not for the OAuth client secret itself. + references: + - https://developer.adobe.com/developer-console/docs/guides/authentication/UserAuthentication/ims + + - name: Adobe OAuth Client ID + id: kingfisher.adobe.4 + pattern: | + (?xi) + \b + adobe + (?:.|[\n\r]){0,64}? + client_id + (?:.|[\n\r]){0,16}? + ( + [a-f0-9]{32} + ) + \b + min_entropy: 3.0 + visible: false + examples: + - | + { + "client_credentials": { + "client_id": "a65b0146769d433a835f36660881db50", + "client_secret": "p8e-ibndcvsmAp9ZgPBZ606FSlYIZVlsZ-g5" + }, diff --git a/crates/kingfisher-rules/data/rules/asaas.yml b/crates/kingfisher-rules/data/rules/asaas.yml index 3b06e7b..1746083 100644 --- a/crates/kingfisher-rules/data/rules/asaas.yml +++ b/crates/kingfisher-rules/data/rules/asaas.yml @@ -14,5 +14,34 @@ rules: examples: - 'ASAAS_API_KEY=$aact_prod_abcdefghijklmnop1234567890ABCDEF' - 'api_token: $aact_hmlg_abcdefghijklmnop1234567890ABCDEF' + validation: + type: Http + content: + request: + method: GET + url: > + {%- if TOKEN contains "$aact_hmlg_" -%} + https://api-sandbox.asaas.com/v3/myAccount/commercialInfo/ + {%- else -%} + https://api.asaas.com/v3/myAccount/commercialInfo/ + {%- endif -%} + headers: + Accept: application/json + User-Agent: kingfisher + access_token: "{{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: + - '"object"' + - '"commercialInfo"' + # Revocation not added: Asaas documents key deletion in the dashboard and + # parent-driven sub-account key management, but not a self-revoke endpoint + # for the current access_token alone. references: - https://docs.asaas.com/docs/authentication-2 + - https://docs.asaas.com/docs/change-the-name-of-a-business-subaccount-via-api diff --git a/crates/kingfisher-rules/data/rules/asana.yml b/crates/kingfisher-rules/data/rules/asana.yml index 0def56b..0c767d9 100644 --- a/crates/kingfisher-rules/data/rules/asana.yml +++ b/crates/kingfisher-rules/data/rules/asana.yml @@ -41,6 +41,32 @@ rules: examples: - "asana :'20c2F0d03201af478ca1aBE9515A1A4FEfb'" - ASANA_PAT = 1234567890abcdef1234567890abcdef12 + depends_on_rule: + - rule_id: kingfisher.asana.1 + variable: ASANA_CLIENT_ID + validation: + type: Http + content: + request: + method: POST + url: https://app.asana.com/-/oauth_token + headers: + Content-Type: application/x-www-form-urlencoded + Accept: application/json + body: > + grant_type=authorization_code&client_id={{ ASANA_CLIENT_ID | url_encode }}&client_secret={{ TOKEN | url_encode }}&redirect_uri={{ "https://example.com/oauth/callback" | url_encode }}&code=invalid_code + response_matcher: + - report_response: true + - type: StatusMatch + status: [400] + - type: WordMatch + words: + - invalid_client + negative: true + # Revocation not added: Asana's revoke endpoint deauthorizes refresh tokens, + # not OAuth client secrets. + references: + - https://developers.asana.com/docs/oauth - name: Asana OAuth / Personal Access Token (Legacy) id: kingfisher.asana.3 diff --git a/crates/kingfisher-rules/data/rules/azure.yml b/crates/kingfisher-rules/data/rules/azure.yml index f909148..dd782e1 100644 --- a/crates/kingfisher-rules/data/rules/azure.yml +++ b/crates/kingfisher-rules/data/rules/azure.yml @@ -70,6 +70,30 @@ rules: - | if __name__ == "__main__": ado_pat = "iyfmob6xjrfmit67anxbot64umfx2clwx7dz5ynxi4q2z3uqegvq" + validation: + type: Http + content: + request: + method: GET + url: https://app.vssps.visualstudio.com/_apis/profile/profiles/me?api-version=7.1 + headers: + Authorization: 'Basic {{ ":" | append: TOKEN | b64enc }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: + - '"id"' + - '"displayName"' + # Revocation not added: Azure DevOps PAT lifecycle management is documented + # separately and is not a self-revoke flow driven solely by the PAT itself. + references: + - https://learn.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops + - https://learn.microsoft.com/en-us/rest/api/azure/devops/profile/profiles/get?view=azure-devops-rest-7.1 - name: Azure Container Registry URL id: kingfisher.azure.4 pattern: | diff --git a/crates/kingfisher-rules/data/rules/azuremaps.yml b/crates/kingfisher-rules/data/rules/azuremaps.yml index 177a382..762e41e 100644 --- a/crates/kingfisher-rules/data/rules/azuremaps.yml +++ b/crates/kingfisher-rules/data/rules/azuremaps.yml @@ -17,5 +17,26 @@ rules: categories: [api, key] examples: - AZURE_MAPS_KEY=AbCdEfGhIjKlMnOpQrStUvWxYz123456 + validation: + type: Http + content: + request: + method: GET + url: https://atlas.microsoft.com/geocode?api-version=2025-01-01&addressLine=15127%20NE%2024th%20Street%20Redmond%20WA&countryRegion=US&subscription-key={{ TOKEN }} + headers: + Accept: application/geo+json, application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: + - '"FeatureCollection"' + - '"features"' + # Revocation not added: Azure Maps shared-key docs cover rotation and + # authentication, but I did not find a token self-revoke API. references: - https://learn.microsoft.com/en-us/azure/azure-maps/how-to-manage-authentication + - https://learn.microsoft.com/en-us/rest/api/maps/search/get-geocoding diff --git a/crates/kingfisher-rules/data/rules/branchio.yml b/crates/kingfisher-rules/data/rules/branchio.yml index 04cf378..7a0a655 100644 --- a/crates/kingfisher-rules/data/rules/branchio.yml +++ b/crates/kingfisher-rules/data/rules/branchio.yml @@ -45,6 +45,20 @@ rules: - 'branch.init("key_test_plqYW3Aq9Xija1cobGMieipndBzO5y7J");' references: - https://help.branch.io/developers-hub/docs/deep-linking-api + - https://help.branch.io/apidocs/app-api + depends_on_rule: + - rule_id: kingfisher.branchio.3 + variable: BRANCH_SECRET + validation: + type: Http + content: + request: + method: GET + url: "https://api2.branch.io/v1/app/{{ TOKEN }}?branch_secret={{ BRANCH_SECRET }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] - name: Branch.io Secret id: kingfisher.branchio.3 diff --git a/crates/kingfisher-rules/data/rules/cockroachlabs.yml b/crates/kingfisher-rules/data/rules/cockroachlabs.yml index 9fbcc40..a4b8d7e 100644 --- a/crates/kingfisher-rules/data/rules/cockroachlabs.yml +++ b/crates/kingfisher-rules/data/rules/cockroachlabs.yml @@ -24,5 +24,28 @@ rules: categories: [api, key] examples: - 'COCKROACHDB_API_KEY=B81649_8F7D11A_92BCE13_56782D_C53' + validation: + type: Http + content: + request: + method: GET + url: https://cockroachlabs.cloud/api/v1/clusters?show_inactive=true + headers: + Authorization: Bearer {{ TOKEN }} + Accept: application/json + Cc-Version: "2024-09-16" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: + - '"clusters"' + - '"pagination"' + # Revocation not added: the public Cloud API docs describe bearer-token + # authentication for service-account secret keys, but not a documented + # self-revocation endpoint for the current secret key value. references: - https://www.cockroachlabs.com/docs/cockroachcloud/cloud-api diff --git a/crates/kingfisher-rules/data/rules/databricks.yml b/crates/kingfisher-rules/data/rules/databricks.yml index 2405d5c..4190d22 100644 --- a/crates/kingfisher-rules/data/rules/databricks.yml +++ b/crates/kingfisher-rules/data/rules/databricks.yml @@ -22,6 +22,26 @@ rules: - secret references: - https://docs.databricks.com/dev-tools/api/latest/authentication.html + - https://docs.databricks.com/en/dev-tools/auth/pat.html + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://{{ DOMAIN }}/api/2.0/clusters/list + depends_on_rule: + - rule_id: "kingfisher.databricks.3" + variable: DOMAIN + # Revocation not added: Databricks PAT docs describe token creation and + # use, but I did not find a PAT-only self-revoke endpoint suitable for YAML + # revocation here. - name: Databricks API Token id: kingfisher.databricks.2 @@ -51,7 +71,7 @@ rules: type: StatusMatch url: https://{{ DOMAIN }}/api/2.0/clusters/list depends_on_rule: - - rule_id: "kingfisher.databricks.2" + - rule_id: "kingfisher.databricks.3" variable: DOMAIN - name: Databricks Domain @@ -83,4 +103,4 @@ rules: references: - https://docs.databricks.com/workspace/workspace-details.html - https://docs.gcp.databricks.com/workspace/workspace-details.html - - https://docs.microsoft.com/en-us/azure/databricks/scenarios/what-is-azure-databricks \ No newline at end of file + - https://docs.microsoft.com/en-us/azure/databricks/scenarios/what-is-azure-databricks diff --git a/crates/kingfisher-rules/data/rules/gitlab.yml b/crates/kingfisher-rules/data/rules/gitlab.yml index 2fa5320..2ac05bb 100644 --- a/crates/kingfisher-rules/data/rules/gitlab.yml +++ b/crates/kingfisher-rules/data/rules/gitlab.yml @@ -213,6 +213,27 @@ rules: - 'CI_JOB_TOKEN=glcbt-a1b2c_3dEfGhIjKlMnOpQrStUv' references: - https://docs.gitlab.com/ci/jobs/ci_job_token/ + - https://docs.gitlab.com/api/jobs/ + validation: + type: Http + content: + request: + method: GET + url: https://gitlab.com/api/v4/job + headers: + JOB-TOKEN: '{{ TOKEN }}' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: + - '"id"' + - '"status"' + # Revocation not added: CI/CD job tokens are short-lived and automatically + # invalidated when the job finishes. - name: GitLab Deploy Token id: kingfisher.gitlab.6 diff --git a/crates/kingfisher-rules/data/rules/google.yml b/crates/kingfisher-rules/data/rules/google.yml index c6c9e66..4cfbde6 100644 --- a/crates/kingfisher-rules/data/rules/google.yml +++ b/crates/kingfisher-rules/data/rules/google.yml @@ -23,6 +23,32 @@ rules: confidence: medium examples: - 'const CLIENTSECRET = "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"' + depends_on_rule: + - rule_id: "kingfisher.google.1" + variable: GOOGLE_CLIENT_ID + validation: + type: Http + content: + request: + method: POST + url: https://oauth2.googleapis.com/token + headers: + Content-Type: application/x-www-form-urlencoded + Accept: application/json + body: > + code=invalid_code&client_id={{ GOOGLE_CLIENT_ID | url_encode }}&client_secret={{ TOKEN | url_encode }}&redirect_uri={{ "https://example.com/oauth/callback" | url_encode }}&grant_type=authorization_code + response_matcher: + - report_response: true + - type: StatusMatch + status: [400] + - type: WordMatch + words: + - invalid_client + negative: true + # Revocation not added: Google's OAuth revocation endpoint revokes tokens, + # not client secrets. + references: + - https://developers.google.com/identity/protocols/oauth2/web-server - name: Google OAuth Client Secret id: kingfisher.google.3 @@ -36,6 +62,32 @@ rules: examples: - " //$google_client_secret = 'fnhqAakzWrX-mtFQ4PRdMoy0';" - " 'clientSecret' : 'Ufvuj-d6alhwGKvvLh_8Nq0K'" + depends_on_rule: + - rule_id: "kingfisher.google.1" + variable: GOOGLE_CLIENT_ID + validation: + type: Http + content: + request: + method: POST + url: https://oauth2.googleapis.com/token + headers: + Content-Type: application/x-www-form-urlencoded + Accept: application/json + body: > + code=invalid_code&client_id={{ GOOGLE_CLIENT_ID | url_encode }}&client_secret={{ TOKEN | url_encode }}&redirect_uri={{ "https://example.com/oauth/callback" | url_encode }}&grant_type=authorization_code + response_matcher: + - report_response: true + - type: StatusMatch + status: [400] + - type: WordMatch + words: + - invalid_client + negative: true + # Revocation not added: Google's OAuth revocation endpoint revokes tokens, + # not client secrets. + references: + - https://developers.google.com/identity/protocols/oauth2/web-server - name: Google OAuth Access Token id: kingfisher.google.4 @@ -61,6 +113,42 @@ rules: - | -- Clear login if it's a new connection. --propertyTable.access_token = 'ya29.Ci_UA7aEsvT6-oVI8f96kvB6i8oO13WgdZUviLaCVtpEPYZqhQcQycR-u2X9xtmYGA' + validation: + type: Http + content: + request: + method: GET + url: https://www.googleapis.com/oauth2/v3/tokeninfo?access_token={{ TOKEN | url_encode }} + headers: + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: + - '"aud"' + - '"expires_in"' + revocation: + type: Http + content: + request: + method: POST + url: https://oauth2.googleapis.com/revoke + headers: + Content-Type: application/x-www-form-urlencoded + Accept: application/json + body: token={{ TOKEN | url_encode }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + references: + - https://developers.google.com/identity/openid-connect/openid-connect + - https://developers.google.com/data-portability/user-guide/quickstart + - https://developers.google.com/identity/protocols/oauth2/web-server - name: Google OAuth Credentials id: kingfisher.google.6 @@ -118,4 +206,4 @@ rules: match_all_words: true words: - '"models"' - - '"name"' \ No newline at end of file + - '"name"' diff --git a/crates/kingfisher-rules/data/rules/googleoauth2.yml b/crates/kingfisher-rules/data/rules/googleoauth2.yml index ddbd412..be4664b 100644 --- a/crates/kingfisher-rules/data/rules/googleoauth2.yml +++ b/crates/kingfisher-rules/data/rules/googleoauth2.yml @@ -30,5 +30,20 @@ rules: - type: WordMatch words: - '"email":' + revocation: + type: Http + content: + request: + method: POST + url: https://oauth2.googleapis.com/revoke + headers: + Content-Type: application/x-www-form-urlencoded + Accept: application/json + body: token={{ TOKEN | url_encode }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] references: - - https://developers.google.com/identity/protocols/oauth2 \ No newline at end of file + - https://developers.google.com/identity/protocols/oauth2 + - https://developers.google.com/identity/protocols/oauth2/web-server diff --git a/crates/kingfisher-rules/data/rules/highnote.yml b/crates/kingfisher-rules/data/rules/highnote.yml index cfdffcb..c8ddf35 100644 --- a/crates/kingfisher-rules/data/rules/highnote.yml +++ b/crates/kingfisher-rules/data/rules/highnote.yml @@ -19,5 +19,35 @@ rules: examples: - 'HIGHNOTE_API_KEY=sk_live_AbCdEfGhIjKlMnOpQrStUvWxYz1234' - 'highnote_key: rk_test_AbCdEfGhIjKlMnOpQrStUvWxYz1234' + validation: + type: Http + content: + request: + method: POST + url: > + {%- if TOKEN contains "_test_" -%} + https://api.us.test.highnote.com/graphql + {%- else -%} + https://api.us.highnote.com/graphql + {%- endif -%} + headers: + Authorization: "Basic {{ TOKEN | b64enc }}" + Content-Type: application/json + Accept: application/json + body: '{"query":"query { ping }"}' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: + - '"data"' + - '"ping"' + - '"pong"' + # Revocation not added: the public Highnote docs I found describe API key + # usage and rotation guidance, but not an API endpoint to revoke the + # current key directly. references: - https://docs.highnote.com/docs/developers/api/using-the-api diff --git a/crates/kingfisher-rules/data/rules/langfuse.yml b/crates/kingfisher-rules/data/rules/langfuse.yml index 3a010d8..845e6cb 100644 --- a/crates/kingfisher-rules/data/rules/langfuse.yml +++ b/crates/kingfisher-rules/data/rules/langfuse.yml @@ -2,10 +2,10 @@ rules: - name: Langfuse Secret Key id: kingfisher.langfuse.1 pattern: | - (?xi) + (?x) \b ( - sk-lf-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} + sk-lf-[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12} ) \b pattern_requirements: @@ -42,10 +42,10 @@ rules: - name: Langfuse Public Key id: kingfisher.langfuse.2 pattern: | - (?xi) + (?x) \b ( - pk-lf-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} + pk-lf-[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12} ) \b pattern_requirements: @@ -57,9 +57,6 @@ rules: examples: - pk-lf-a1b2c3d4-e5f6-7890-abcd-ef1234567890 - 'LANGFUSE_PUBLIC_KEY="pk-lf-9f8e7d6c-5b4a-3210-fedc-ba0987654321"' - negative_examples: - - pk-lf-test - - pk-lf- references: - https://langfuse.com/docs/sdk/typescript - https://langfuse.com/docs/get-started diff --git a/crates/kingfisher-rules/data/rules/posthog.yml b/crates/kingfisher-rules/data/rules/posthog.yml index 904c202..6f65b0e 100644 --- a/crates/kingfisher-rules/data/rules/posthog.yml +++ b/crates/kingfisher-rules/data/rules/posthog.yml @@ -57,6 +57,22 @@ rules: examples: - "pha_XgrXUnvwyoPLmjwHES5lc8scZUtheBpa1QV1qmssutB" - "pha_35kHVLA1E068nvrwUTgabkh8xvGGTpSpsVjGcpVNfis" + validation: + type: Http + content: + request: + method: GET + url: https://app.posthog.com/api/users/@me/ + headers: + Authorization: "Bearer {{ TOKEN }}" + Content-Type: "application/json" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + # Revocation not added: I did not find a documented token self-revoke + # endpoint for OAuth access tokens in the public PostHog API docs. references: - https://posthog.com/docs/api - https://github.com/PostHog/posthog/blob/e408aac5debe02b39a6a67cfd028f16a2ca7bc90/posthog/models/utils.py#L260-L290 diff --git a/crates/kingfisher-rules/data/rules/proof.yml b/crates/kingfisher-rules/data/rules/proof.yml index 965926e..6fddb22 100644 --- a/crates/kingfisher-rules/data/rules/proof.yml +++ b/crates/kingfisher-rules/data/rules/proof.yml @@ -18,5 +18,32 @@ rules: - 'proof_key: prf_test_AbCdEfGhIjKlMnOpQrStUvWxYz123456' - 'proof_key: prf_cli_AbCdEfGhIjKlMnOpQrStUvWxYz123456' - 'proof_key: prf_cli_test_AbCdEfGhIjKlMnOpQrStUvWxYz123456' + validation: + type: Http + content: + request: + method: POST + url: > + {%- if TOKEN contains "_test_" -%} + https://api.fairfax.proof.com/v1/transactions + {%- else -%} + https://api.proof.com/v1/transactions + {%- endif -%} + headers: + ApiKey: "{{ TOKEN }}" + Content-Type: application/json + Accept: application/json + body: '{}' + response_matcher: + - report_response: true + - type: StatusMatch + status: [422] + - type: WordMatch + words: + - signer + # Revocation not added: the public Proof docs describe dashboard key + # management and secret-scanning guidance, but not a self-revoke API. references: - https://dev.proof.com/docs/api-keys + - https://dev.proof.com/docs/environments + - https://dev.proof.com/reference/createtransaction diff --git a/crates/kingfisher-rules/data/rules/tableau.yml b/crates/kingfisher-rules/data/rules/tableau.yml index 803bbcf..14b0510 100644 --- a/crates/kingfisher-rules/data/rules/tableau.yml +++ b/crates/kingfisher-rules/data/rules/tableau.yml @@ -65,7 +65,11 @@ rules: (?xi) \b ( - https://[a-z0-9.-]{3,200} + https://(?: + (?:[a-z0-9-]+\.)?online\.tableau\.com + | + (?:[a-z0-9-]+\.)*tableau(?:\.[a-z0-9-]+)+ + ) ) (?: /api/\d+\.\d+ @@ -79,7 +83,7 @@ rules: examples: - https://tableau.example.com - https://10ax.online.tableau.com - - server="https://analytics.example.com" + - server="https://analytics.tableau.example.com" references: - https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_ref_authentication.htm @@ -89,12 +93,11 @@ rules: (?xi) \b (?: + tableau[_-]?(?:site|content[_-]?url) + | tableau (?:.|[\n\r]){0,48}? - )? - (?: - site | - content[_-]?url + (?:site|content[_-]?url) ) (?:.|[\n\r]){0,12}? [=:"'\s] @@ -107,6 +110,6 @@ rules: visible: false examples: - tableau_site=companysite - - contentUrl="default" + - tableau_content_url="default" references: - https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_ref_authentication.htm diff --git a/crates/kingfisher-scanner/src/validation/raw.rs b/crates/kingfisher-scanner/src/validation/raw.rs index dc006e6..3436272 100644 --- a/crates/kingfisher-scanner/src/validation/raw.rs +++ b/crates/kingfisher-scanner/src/validation/raw.rs @@ -29,6 +29,8 @@ use tokio::{ use tokio_rustls::TlsConnector; use url::Url; +use crate::validation::http_validation::check_url_resolvable; + pub struct RawValidationOutcome { pub valid: bool, pub status: StatusCode, @@ -104,7 +106,20 @@ pub async fn validate_raw( globals: &Object, client: &Client, use_lax_tls: bool, + allow_internal_ips: bool, ) -> Result { + if let Some(url) = raw_validation_target_url(kind, globals)? { + if let Err(e) = check_url_resolvable(&url, allow_internal_ips).await { + return Ok(RawValidationOutcome { + valid: false, + status: StatusCode::PRECONDITION_REQUIRED, + body: format!( + "Validation skipped - raw validation target blocked or not resolvable: {e}" + ), + }); + } + } + match kind { "azurebatch" => validate_azure_batch(globals, client).await, "ftp" => validate_ftp(globals, use_lax_tls).await, @@ -120,6 +135,18 @@ pub async fn validate_raw( } } +fn raw_validation_target_url(kind: &str, globals: &Object) -> Result> { + match kind { + "azurebatch" => string_var(globals, "BATCH_URL") + .map(|s| Url::parse(&s).context("invalid BATCH_URL")) + .transpose(), + "ftp" | "ldap" | "rabbitmq" | "redis" => string_var(globals, "TOKEN") + .map(|s| Url::parse(&s).context("invalid raw validation URI")) + .transpose(), + _ => Ok(None), + } +} + fn string_var(globals: &Object, name: &str) -> Option { globals.get(name).map(|v| v.to_kstr().to_string()).filter(|s| !s.is_empty()) } diff --git a/data/default/rule_cleanup/count_rules.py b/data/default/rule_cleanup/count_rules.py index 7a54ffe..f9363ac 100644 --- a/data/default/rule_cleanup/count_rules.py +++ b/data/default/rule_cleanup/count_rules.py @@ -27,8 +27,8 @@ DEFAULT_RULES_DIR = ( def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description=( - "Count total rules and detector rules. " - "Detector rules are rules that do not " + "Count total rules and standalone detector rules. " + "Standalone detector rules are rules that do not " "declare depends_on_rule." ) ) @@ -41,7 +41,10 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--list-validators", action="store_true", - help="Print the names of detectors with and without a validator", + help=( + "Print the IDs of standalone detectors with and " + "without a validator" + ), ) return parser.parse_args() @@ -64,6 +67,14 @@ def iter_rule_entries(path: Path) -> list[dict]: return entries +def rule_identifier(rule: dict, path: Path, index: int) -> str: + if isinstance(rule.get("id"), str) and rule["id"].strip(): + return rule["id"] + if isinstance(rule.get("name"), str) and rule["name"].strip(): + return rule["name"] + return f"{path.stem}#{index}" + + def main() -> int: args = parse_args() rules_dir = args.rules_dir.resolve() @@ -79,8 +90,8 @@ def main() -> int: total_rules = 0 dependent_rules = 0 - with_validator: list[str] = [] - without_validator: list[str] = [] + standalone_with_validator: list[str] = [] + standalone_without_validator: list[str] = [] for path in rule_files: try: @@ -93,27 +104,43 @@ def main() -> int: dependent_rules += sum( 1 for rule in rules if rule.get("depends_on_rule") ) - if any(rule.get("validation") for rule in rules): - with_validator.append(path.stem) - else: - without_validator.append(path.stem) + for index, rule in enumerate(rules, start=1): + if rule.get("depends_on_rule"): + continue - detector_rules = total_rules - dependent_rules + identifier = rule_identifier(rule, path, index) + if rule.get("validation"): + standalone_with_validator.append(identifier) + else: + standalone_without_validator.append(identifier) + + standalone_detector_rules = total_rules - dependent_rules print(f"Rules directory: {rules_dir}") - print(f"Detectors: {len(rule_files)}") - print(f"Detectors with validator: {len(with_validator)}") - print(f"Detectors without validator: {len(without_validator)}") print(f"Total rules: {total_rules}") print(f"Dependent rules: {dependent_rules}") - print(f"Non-dependent rules: {detector_rules}") + print(f"Standalone detectors: {standalone_detector_rules}") + print( + "Standalone detectors with validator: " + f"{len(standalone_with_validator)}" + ) + print( + "Standalone detectors without validator: " + f"{len(standalone_without_validator)}" + ) if args.list_validators: - print(f"\nWith validator ({len(with_validator)}):") - for name in with_validator: + print( + "\nStandalone detectors with validator " + f"({len(standalone_with_validator)}):" + ) + for name in standalone_with_validator: print(f" {name}") - print(f"\nWithout validator ({len(without_validator)}):") - for name in without_validator: + print( + "\nStandalone detectors without validator " + f"({len(standalone_without_validator)}):" + ) + for name in standalone_without_validator: print(f" {name}") return 0 diff --git a/docs-site/docs/changelog.md b/docs-site/docs/changelog.md index 6439801..323e91b 100644 --- a/docs-site/docs/changelog.md +++ b/docs-site/docs/changelog.md @@ -7,6 +7,11 @@ description: "Kingfisher release history: new features, rules, bug fixes, and im All notable changes to this project will be documented in this file. +## [v1.95.0] +- Added 80+ built-in rules, bringing the bundled ruleset to 820 total. New coverage includes Amazon OAuth, Asaas, multiple Azure credential families, Bitrise, Canva, CockroachDB, eBay, Elastic, hCaptcha, Highnote, Lichess, MailerSend, Onfido, Paddle, Pangea, Persona, Pinterest, Proof, Rootly, Runpod, Telnyx, Thunderstore, Valtown, Volcengine, and more. +- Added a `validation: type: Raw` exception path for provider-specific checks, with new raw validators for Azure Batch, FTP, Kraken, LDAP, RabbitMQ, and Redis. Also added stable request-scoped template values plus new Liquid filters for HMAC-SHA384 hex output and timestamp generation. +- Expanded live validation coverage for several built-in rules, including Agora, Bitfinex, DocuSign, Dwolla, GitLab, KuCoin, RingCentral, Snowflake, Tableau, Trello, and Webex. Also tightened newly added helper regex to avoid high-match scan regressions, and made preflight-blocked raw validations report as skipped/not attempted instead of failed. + ## [v1.94.0] - Updated vendored `vectorscan-rs` from v0.0.5 (Vectorscan 5.4.11) to v0.0.6 (Vectorscan 5.4.12). The upstream crate now ships pre-extracted sources instead of a tarball+patch, and fixes the `cpu_native` feature flag. Local Windows and musl build patches have been re-applied. - Added more built-in rules diff --git a/src/direct_validate.rs b/src/direct_validate.rs index 1cea920..9a03bbe 100644 --- a/src/direct_validate.rs +++ b/src/direct_validate.rs @@ -857,6 +857,7 @@ pub async fn run_direct_validation( &globals, &client, use_lax_tls, + global_args.allow_internal_ips, ) .await { diff --git a/src/reporter.rs b/src/reporter.rs index d1a5a24..9233cc7 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -942,7 +942,11 @@ impl DetailsReporter { let validation_status = if rm.validation_success { "Active Credential".to_string() - } else if rm.validation_response_status == StatusCode::CONTINUE.as_u16() { + } else if matches!( + rm.validation_response_status, + status if status == StatusCode::CONTINUE.as_u16() + || status == StatusCode::PRECONDITION_REQUIRED.as_u16() + ) { "Not Attempted".to_string() } else { "Inactive Credential".to_string() @@ -1975,7 +1979,7 @@ mod tests { let (report_match, _) = sample_report_match( "(skip list entry) AWS validation not attempted for account 111122223333.", - StatusCode::CONTINUE.as_u16(), + StatusCode::PRECONDITION_REQUIRED.as_u16(), false, ); let scan_args = sample_scan_args(); diff --git a/src/validation.rs b/src/validation.rs index 3e90b07..5cdf823 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -1311,7 +1311,7 @@ async fn timed_validate_single_match<'a>( "(skip list entry) AWS validation not attempted for account {}.", account_id )); - m.validation_response_status = StatusCode::CONTINUE; + m.validation_response_status = StatusCode::PRECONDITION_REQUIRED; cache.insert( cache_key, CachedResponse { @@ -1488,6 +1488,7 @@ async fn timed_validate_single_match<'a>( &globals, client, clients.should_use_lax(rule_syntax.tls_mode), + clients.allow_internal_ips, ) .await { From 0cb854872bb66bdec03c37a75829e0c61605768b Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Tue, 7 Apr 2026 23:20:17 -0700 Subject: [PATCH 07/17] Replaced tree-sitter with a lighter parser-based context verifier built from handwritten lexers plus tl/cssparser, preserving context-dependent matching while cutting about 19 MB from the release binary. --- AGENTS.md | 2 +- CHANGELOG.md | 1 + Cargo.lock | 249 +--- Cargo.toml | 21 +- README.md | 8 +- crates/kingfisher-rules/data/rules/adobe.yml | 4 +- crates/kingfisher-scanner/src/scanner.rs | 2 +- .../assets/images/binary-size-comparison.png | Bin 0 -> 35660 bytes docs-site/docs/changelog.md | 1 + docs-site/docs/features/parsing.md | 51 +- docs-site/docs/reference/architecture.md | 8 +- docs-site/docs/reference/comparison.md | 90 +- docs/ARCHITECTURE.md | 8 +- docs/COMPARISON.md | 89 +- docs/CONTEXT_VERIFICATION.md | 49 + docs/PARSING.md | 50 +- docs/TREE_SITTER.md | 100 -- docs/binary-size-comparison.png | Bin 0 -> 35660 bytes src/cli/commands/scan.rs | 2 +- src/matcher/mod.rs | 131 +- src/parser.rs | 494 ++----- src/parser/css.rs | 173 +++ src/parser/html.rs | 67 + src/parser/lexer.rs | 1276 +++++++++++++++++ src/parser/queries.rs | 1105 -------------- src/scanner/processing.rs | 4 +- testdata/css_vulnerable.css | 8 + testdata/parsers/comment_only_context.py | 2 + tests/int_base64.rs | 6 +- tests/int_context_verification.rs | 83 ++ 30 files changed, 2056 insertions(+), 2028 deletions(-) create mode 100644 docs-site/docs/assets/images/binary-size-comparison.png create mode 100644 docs/CONTEXT_VERIFICATION.md delete mode 100644 docs/TREE_SITTER.md create mode 100644 docs/binary-size-comparison.png create mode 100644 src/parser/css.rs create mode 100644 src/parser/html.rs create mode 100644 src/parser/lexer.rs delete mode 100644 src/parser/queries.rs create mode 100644 testdata/css_vulnerable.css create mode 100644 testdata/parsers/comment_only_context.py create mode 100644 tests/int_context_verification.rs diff --git a/AGENTS.md b/AGENTS.md index 917c3f7..4d39dbc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,7 +24,7 @@ Key capabilities: - `src/cli/commands/`: CLI command implementations - `src/matcher/`: pattern matching engine - `src/scanner/`: core scanning logic -- `src/parser/`: language-aware parsing (`tree-sitter`) +- `src/parser/`: language-aware context verification (lightweight lexers, `tl` for HTML, `cssparser` for CSS) - `src/reporter/`: TOON/JSON/SARIF/HTML report generation - `src/access_map/`: access mapping analysis - `crates/kingfisher-core/`: shared types and core logic diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ee1dde..94ec62d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file. ## [v1.95.0] - Added 80+ built-in rules, bringing the bundled ruleset to 820 total. New coverage includes Amazon OAuth, Asaas, multiple Azure credential families, Bitrise, Canva, CockroachDB, eBay, Elastic, hCaptcha, Highnote, Lichess, MailerSend, Onfido, Paddle, Pangea, Persona, Pinterest, Proof, Rootly, Runpod, Telnyx, Thunderstore, Valtown, Volcengine, and more. +- Replaced tree-sitter with a lighter parser-based context verifier built from handwritten lexers plus `tl`/`cssparser`, preserving context-dependent matching while cutting about 19 MB from the release binary. - Added a `validation: type: Raw` exception path for provider-specific checks, with new raw validators for Azure Batch, FTP, Kraken, LDAP, RabbitMQ, and Redis. Also added stable request-scoped template values plus new Liquid filters for HMAC-SHA384 hex output and timestamp generation. - Expanded live validation coverage for several built-in rules, including Agora, Bitfinex, DocuSign, Dwolla, GitLab, KuCoin, RingCentral, Snowflake, Tableau, Trello, and Webex. Also tightened newly added helper regex to avoid high-match scan regressions, and made preflight-blocked raw validations report as skipped/not attempted instead of failed. diff --git a/Cargo.lock b/Cargo.lock index 36fd53a..a8b5e17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1994,6 +1994,17 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "cssparser" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9cdaae01d5ed7882b04d795e7f752f46ff52d2fa3b50a20d28c464510bba98" +dependencies = [ + "dtoa-short", + "itoa", + "smallvec", +] + [[package]] name = "ctutils" version = "0.4.0" @@ -2418,6 +2429,21 @@ dependencies = [ "litrs", ] +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + [[package]] name = "dunce" version = "1.0.5" @@ -5097,6 +5123,7 @@ dependencies = [ "crc32fast", "crossbeam-channel", "crossbeam-skiplist", + "cssparser", "dashmap", "ed25519-dalek", "fixedbitset", @@ -5168,7 +5195,6 @@ dependencies = [ "sha1 0.11.0", "sha2 0.11.0", "smallvec", - "streaming-iterator", "strum 0.28.0", "strum_macros 0.28.0", "sysinfo", @@ -5181,6 +5207,7 @@ dependencies = [ "thread_local", "tikv-jemallocator", "time", + "tl", "tokei", "tokio", "tokio-postgres", @@ -5190,24 +5217,6 @@ dependencies = [ "tracing", "tracing-core", "tracing-subscriber", - "tree-sitter", - "tree-sitter-bash", - "tree-sitter-c", - "tree-sitter-c-sharp", - "tree-sitter-cpp", - "tree-sitter-css", - "tree-sitter-go", - "tree-sitter-html", - "tree-sitter-java", - "tree-sitter-javascript", - "tree-sitter-php", - "tree-sitter-python", - "tree-sitter-regex", - "tree-sitter-ruby", - "tree-sitter-rust", - "tree-sitter-toml-ng", - "tree-sitter-typescript", - "tree-sitter-yaml", "tree_magic_mini", "url", "uuid", @@ -8249,12 +8258,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" - [[package]] name = "stringprep" version = "0.1.5" @@ -8724,6 +8727,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tl" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b130bd8a58c163224b44e217b4239ca7b927d82bf6cc2fea1fc561d15056e3f7" + [[package]] name = "tls_codec" version = "0.4.2" @@ -9150,196 +9159,6 @@ dependencies = [ "tracing-log", ] -[[package]] -name = "tree-sitter" -version = "0.26.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "887bd495d0582c5e3e0d8ece2233666169fa56a9644d172fc22ad179ab2d0538" -dependencies = [ - "cc", - "regex", - "regex-syntax", - "serde_json", - "streaming-iterator", - "tree-sitter-language", -] - -[[package]] -name = "tree-sitter-bash" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5ec769279cc91b561d3df0d8a5deb26b0ad40d183127f409494d6d8fc53062" -dependencies = [ - "cc", - "tree-sitter-language", -] - -[[package]] -name = "tree-sitter-c" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3aad8f0129083a59fe8596157552d2bb7148c492d44c21558d68ca1c722707" -dependencies = [ - "cc", - "tree-sitter-language", -] - -[[package]] -name = "tree-sitter-c-sharp" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67f06accca7b45351758663b8215089e643d53bd9a660ce0349314263737fcb0" -dependencies = [ - "cc", - "tree-sitter-language", -] - -[[package]] -name = "tree-sitter-cpp" -version = "0.23.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2196ea9d47b4ab4a31b9297eaa5a5d19a0b121dceb9f118f6790ad0ab94743" -dependencies = [ - "cc", - "tree-sitter-language", -] - -[[package]] -name = "tree-sitter-css" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5cbc5e18f29a2c6d6435891f42569525cf95435a3e01c2f1947abcde178686f" -dependencies = [ - "cc", - "tree-sitter-language", -] - -[[package]] -name = "tree-sitter-go" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8560a4d2f835cc0d4d2c2e03cbd0dde2f6114b43bc491164238d333e28b16ea" -dependencies = [ - "cc", - "tree-sitter-language", -] - -[[package]] -name = "tree-sitter-html" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "261b708e5d92061ede329babaaa427b819329a9d427a1d710abb0f67bbef63ee" -dependencies = [ - "cc", - "tree-sitter-language", -] - -[[package]] -name = "tree-sitter-java" -version = "0.23.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa6cbcdc8c679b214e616fd3300da67da0e492e066df01bcf5a5921a71e90d6" -dependencies = [ - "cc", - "tree-sitter-language", -] - -[[package]] -name = "tree-sitter-javascript" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68204f2abc0627a90bdf06e605f5c470aa26fdcb2081ea553a04bdad756693f5" -dependencies = [ - "cc", - "tree-sitter-language", -] - -[[package]] -name = "tree-sitter-language" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782" - -[[package]] -name = "tree-sitter-php" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d8c17c3ab69052c5eeaa7ff5cd972dd1bc25d1b97ee779fec391ad3b5df5592" -dependencies = [ - "cc", - "tree-sitter-language", -] - -[[package]] -name = "tree-sitter-python" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bf85fd39652e740bf60f46f4cda9492c3a9ad75880575bf14960f775cb74a1c" -dependencies = [ - "cc", - "tree-sitter-language", -] - -[[package]] -name = "tree-sitter-regex" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd8a59be9f0ac131fd8f062eaaba14882b2fa5a6a7882a20134cb1d60df2e625" -dependencies = [ - "cc", - "tree-sitter-language", -] - -[[package]] -name = "tree-sitter-ruby" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be0484ea4ef6bb9c575b4fdabde7e31340a8d2dbc7d52b321ac83da703249f95" -dependencies = [ - "cc", - "tree-sitter-language", -] - -[[package]] -name = "tree-sitter-rust" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439e577dbe07423ec2582ac62c7531120dbfccfa6e5f92406f93dd271a120e45" -dependencies = [ - "cc", - "tree-sitter-language", -] - -[[package]] -name = "tree-sitter-toml-ng" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9adc2c898ae49730e857d75be403da3f92bb81d8e37a2f918a08dd10de5ebb1" -dependencies = [ - "cc", - "tree-sitter-language", -] - -[[package]] -name = "tree-sitter-typescript" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff" -dependencies = [ - "cc", - "tree-sitter-language", -] - -[[package]] -name = "tree-sitter-yaml" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c223db85f05e34794f065454843b0668ebc15d240ada63e2b5939f43ce7c97" -dependencies = [ - "cc", - "tree-sitter-language", -] - [[package]] name = "tree_magic_mini" version = "3.2.2" diff --git a/Cargo.toml b/Cargo.toml index 3bc7011..a9cbb83 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -168,28 +168,11 @@ reqwest-middleware = "0.5.1" reqwest-middleware-octorust = { package = "reqwest-middleware", version = "0.4.2" } tracing-subscriber = {version = "0.3.22", features = ["env-filter"] } tracing-core = "0.1.35" -tree-sitter = "0.26.5" aws-smithy-http-client = "1.1.10" aws-smithy-runtime-api = "1.11.4" aws-smithy-types = "1.4.4" -tree-sitter-bash = "0.25.1" -tree-sitter-c = "0.24.1" -tree-sitter-c-sharp = "0.23.1" -tree-sitter-cpp = "0.23.4" -tree-sitter-css = "0.25.0" -tree-sitter-go = "0.25.0" -tree-sitter-html = "0.23.2" -tree-sitter-java = "0.23.5" -tree-sitter-javascript = "0.25.0" -tree-sitter-php = "0.24.2" -tree-sitter-python = "0.25.0" -tree-sitter-ruby = "0.23.1" -tree-sitter-rust = "0.24.0" -tree-sitter-toml-ng = "0.7.0" -tree-sitter-typescript = "0.23.2" -tree-sitter-yaml = "0.7.2" -streaming-iterator = "0.1.9" -tree-sitter-regex = "0.25.0" +cssparser = { version = "0.37.0", default-features = false } +tl = "0.7.8" tree_magic_mini = "3.2" content_inspector = "0.2.4" rustc-hash = "2.1.1" diff --git a/README.md b/README.md index 5e1aad6..c89ea8f 100644 --- a/README.md +++ b/README.md @@ -401,7 +401,7 @@ kingfisher scan /path/to/code kingfisher scan ~/src/myrepo --no-validate # Turbo mode: run as fast as possible by disabling Git commit metadata, Base64 decoding, -# MIME sniffing, language detection, and tree-sitter parsing +# MIME sniffing, language detection, and parser-based context verification # (findings omit commit context, Base64-only matches, MIME type, and language metadata) kingfisher scan ~/src/myrepo --turbo @@ -510,7 +510,7 @@ cat /path/to/file.py | kingfisher scan - kingfisher scan /some/file --max-file-size 500 # Turbo mode: equivalent to --commit-metadata=false --no-base64 and disables MIME sniffing, -# language detection/tree-sitter parsing for maximum speed +# language detection/parser-based context verification for maximum speed # No Git commit metadata (author, date, hash), Base64 decoding, MIME, or language metadata in findings kingfisher scan /path/to/repo --turbo @@ -725,7 +725,7 @@ kingfisher scan /tmp/repo --branch feature-1 \ | [FINGERPRINT.md](docs/FINGERPRINT.md) | Understanding finding fingerprints and deduplication | | [COMPARISON.md](docs/COMPARISON.md) | Benchmark results and performance comparisons | | [PARSING.md](docs/PARSING.md) | Language-aware parsing details | -| [TREE_SITTER.md](docs/TREE_SITTER.md) | Tree-sitter scanning flow, verification gates, and fallback behavior | +| [CONTEXT_VERIFICATION.md](docs/CONTEXT_VERIFICATION.md) | Context-verification flow, gates, and parser backends | # Library Usage @@ -751,7 +751,7 @@ Since then it has evolved far beyond that starting point, introducing live valid - **Live validation** of detected secrets directly within rules - **Hundreds of new built-in rules** and an expanded YAML rule schema - **Baseline management** to suppress known findings over time -- **Tree-sitter parsing** layered on Hyperscan for language-aware detection +- **Parser-based context verification** layered on Hyperscan for language-aware detection - **More scan targets** (GitLab, Bitbucket, Gitea, Jira, Confluence, Slack, Microsoft Teams, S3, GCS, Docker, Hugging Face, etc.) - **Compressed Files**, **SQLite database**, and **Python bytecode (.pyc)** scanning support - **New storage model** (in-memory + Bloom filter, replacing SQLite) diff --git a/crates/kingfisher-rules/data/rules/adobe.yml b/crates/kingfisher-rules/data/rules/adobe.yml index 63a2062..5983e12 100644 --- a/crates/kingfisher-rules/data/rules/adobe.yml +++ b/crates/kingfisher-rules/data/rules/adobe.yml @@ -70,7 +70,7 @@ rules: examples: - | { - "client_credentials": { + "adobe_client_credentials": { "client_id": "a65b0146769d433a835f36660881db50", "client_secret": "p8e-ibndcvsmAp9ZgPBZ606FSlYIZVlsZ-g5" }, @@ -119,7 +119,7 @@ rules: examples: - | { - "client_credentials": { + "adobe_client_credentials": { "client_id": "a65b0146769d433a835f36660881db50", "client_secret": "p8e-ibndcvsmAp9ZgPBZ606FSlYIZVlsZ-g5" }, diff --git a/crates/kingfisher-scanner/src/scanner.rs b/crates/kingfisher-scanner/src/scanner.rs index dc8ef6c..55dff65 100644 --- a/crates/kingfisher-scanner/src/scanner.rs +++ b/crates/kingfisher-scanner/src/scanner.rs @@ -26,7 +26,7 @@ pub struct ScannerConfig { /// Override the minimum entropy threshold for all rules. pub min_entropy_override: Option, - /// Language hint for tree-sitter parsing (e.g., "python", "javascript"). + /// Language hint for parser-based context verification (e.g., "python", "javascript"). pub language_hint: Option, /// Whether to redact secrets in findings. diff --git a/docs-site/docs/assets/images/binary-size-comparison.png b/docs-site/docs/assets/images/binary-size-comparison.png new file mode 100644 index 0000000000000000000000000000000000000000..1353d6d42e90249c6c7479f853d1f0efbc2c8fb0 GIT binary patch literal 35660 zcmdSBbyU>t`!0$iph$>d(5Q4O-6192Ag$6ZEg(aTA}FPF_s}4n17gq&DKMmTGxPvM zoM-s{&R*w_z4lsrt$ohkXV!XOS;Xh_^nKmeb=^;xrn(|AAr&DG4i2%>3ps5ZoEua) zIM-j^#0TH$P5pQs{3Gfi|I$Op#oELBwVM@=+G~%ujxHXKc5fKGtlZq~T%7s1pFQOk z=47z-@ObMk%EROI-+qJJ#m$DN&DawGo^tE$3qyAtoCkNY|6Q}m7y|G5T(2Z2qx&v( zYx>rd>LKQGH;SnemDhRqE0aq0M`g@o2V0#-SrIP^@5Cv`D^EPpp0JI%bBpPb3cCv9 zBc-DkM<-G<8K}!%RBuE76uNh6WyZX}!(wXN5RockgP7WHKzN#Fm}F_PT7vh3&-G}r z(2xK5@{a9){tbGZbM5M@8zIuTS6|#D3c~gs2j|Yi>%>=IFg(SM3HV6Ay2*d_Mc|MB z_V)_s7fwl58^kN2?u?L%N09e<*m&rWSh=`eV6_M@LGH$2>Mh z`&5(pyJ$q*7?-W+gx~%MrR`6~R*U%M*?u1`srW0& z%QJWPgVn*r1QE}5-53soGR6d$S_;hGbfq_Gp~+R%g>);$T45`ebjZ zi@QoYPnoP|0CjUF!5m^I>b>n~KUU_X9LxGTQfyloqtoFH#V79d95b~^_g!b>bDH3< zav0;DwYqT*=?*_a&ALcos!I{V3qoofyAn z(sFr$@$l(G_6=mo%F2JH2(zPhnXKi4SzYY+OJ(V(i~RAXd|>wObWs zmKtgXzD4OFl_KcGn_%jdgsiY2ND(2n>xJ>b7TP0jzejdI9)R?!mV1IpfqASA9fgYw zMOD5!5UF_8c3seqx7KCWd7{?IVHuG|J?3Rp=S(|!59j)Gu$Fmc>|MUx=lXVoGM3J_ zPhf0ZI!GIJ@8vu6w09q9$m4BvFJFHmxw>%M!PcZuS0w=nO(J7wrmlnkEgpliRJFR% zguM*NX;*KGP!!L^Eh<6Bm){sDffNw?A&h>4nUK4h;G_U&8C9DtE|5}4pYP~!E$BSMer=MKNKhn~Z zo!nujxxwYBLf&0=`eHM9BUi>T2G!aJmQG)965rNtSZ|?qU$W9n@u?U-Lrl+VEFn(k z8r{2pL5y};c!U%;)iNX{y8PqHAd^mYObr3sQw_2l`X!pR05QYWzF%j1QpLSM-5e5;;#vPp9*(G($>5K}jsJpg5Bd9FV`zFjfWx3O5ZHzJGeG&<6E!nU0@+*X zr!UgXdcJxz=|Wm+T*v)J`n;Xm7Ccn{X+2Zac#dlL=r zHX};~_+w2MR1ZYFyI~ko zrX@ZFdjcjO(4l88!K_9!p6Qnw$xj(<lo~t$2BHE~}Qze=F_X zeArPUr%Cm6 z5eV}dBtHYNuup*~?osIb<6UH1i18qWUt z>S^{;DydH8=hNVH9VpT(&gLfVECNxvvpNXh|MdXu())n#&|5 ztBx?Vgqh z*g#Y262j+~JKchU*A8Aa_YzG)- z$KuU`@ZtCUU!$!D&%l8egh%DkjIna6uX5$bc~NX3Vw9Ps7)94I`65Gt>PLOK3wZtr z-|`G|IoOY+e|j)8V+LCj7DbQe&;cb7PeF{B)x{ORjJ&Z`LM!7Q?K6 z$_d$NBiO1{w>v+)NRtX^LAjmH?1Pi|bED}2WY9}TM}>GhMa*}s0mNc|* zcyb1241?csHj2*rL|1w3qiv7QKeYg8;N}Nu+(1=o+Z{2!rIuBqk3wFhi?F;26#D4`Vr zHru%tvyR-^Bt+dTM68Q9mWiy~7Aip;e$bG`hoCCS)s0_x5hV>p9CYlz!@~$Vj^9!K zz-!dpGrXdeeQ?n_7eYO66hMnM4>B?ni=kkr?U3g3Hdm?vok~pcbv(jEa7Z)S5uHrBv?kr= zX&$(c^4^}RU0a}bY5uynYp-1JB}XpoZtbq+SedEQ{?8E*hk_eGu<8XN!@jOGK45j= z(d2tI%!~bOu@T|7s{>ifHDhKft_}9%urbeR&#|O0+>Mytik9Q{HMo;nyy?*1m9+i0FNmpoaFDcHz{ zrPwQY12{1`wyD@haoBPfKKjg}ef&^~hg#6_d&G;FoCSrAdmiMidrLi&7AL>SHTYdx zPOQHf;+la8-3J|qbyaM3*xh+q=WOaetR{rC4r{%<@T+$%9@RsU=g(zF9)g20`QjX- z$hiU~l?**Ljd=NYQNYZ1HUPi_rLs!^C!KO$m@a~yQfR zUm?V5*#`~(oSqjr0c@y<5k0&Z@~`Ha$`G2+ICUIb@o@8)74=*n@1K667?}u;!AZ(c zn$xCybWCkbx=sKo=mdt5_)X31q2!2bbSj7llgp*x|3ZfwV|B!4e@FhFry7w?mnmdB z8-23>TZHr+SZRu^nE&%-azPo@+AoReTDnw+h{|VmFyM@XX83&HMbCvyf4~~F~xf7 zN^XY@kt*B`jPoKr2|O8#l7k2s`?N4sYt;}0Mb_$+&;KqiML2eC4&D!90!;-?sfzgI zsJ|96JfWLoNxi={6r4H#3n284*FaJ|fPE>w3d4Eqm?eOhi9H&+ zNd^pGe2SEB-K0umBR+i@sbDkf+0NS#9_Y8-z%uV`CBDiDUjTc%#-?oA!%K9~!i&B+5EB|{q5^+bG~?s^P7PZ7naR(8eu=k_bYS!yCY!|VBeypd>6?U%5s&kZ?> zm;?uwOI3iN^VGwy0u0@qn$`-+n2oQBbQP1ce$5VquY|139YLmyJRDwW@D?+Fzx#&` z(sIp^GLIaGm&>KQSX~lCCy9TP>ENTn$L^%^{4#e{VK<4<6>w_ z5yNR#(V}>!UE?B%Y0Ab$4IdQXwLrd_w*j(xuZUwq)PpEI)ejx9PXC~tDK)L8BD+0X_xzwJM2 z4{cw6PAi(QvxwnYa{rKgkJguxE;(^ukMV??W#T*N1E%7+d{l00`-L-|Hf-m~Ge&|ziTGQyz1*4(fnY#DF5*;@Sw&XaDBh^|IAbyRnV(sRhLiWKP)Emo z9mHW6cFUNiKqcX|V&uf2OOnHRRJz2G^R$V=QJ$-gH3_^o=d;pr%+^<;@A^~jZlM0v ziGQ*$^%PXbHT!#?$EdoKI4$u4-V!xX9C0If93uM+G&D2o7-UQ9&%Y{JK z4}{}&_94T5a{dxi6tRPp0kk%RH|l+=9JI;r4DG9@izD%A1vIq-pG;twzorv76V{u0 z{?#Lco64OlP`upV!@LjgINF+AfP|alC}y~ay+z|HukBCsJiOPlp;6rGIw*%}-t#jB zY?j9HC7~wA4vj{%%o}*XBplf{jmN5Eq*ooiTHQ>`bKxpdjQ@mL_*L+1%+dF&7i+{5dS!|A~q z8%Z!o;o=unQd=Gte%0r!2D@K{ZCttZNJT3OT_cREfqKA{7ta183ucX|)ywIWV;rJ* zHi~}n7{!!^+!On?hMw6G^IKA1^GtXxO#E-^L4IM9_s;5+B*-md;t%8{L#q(Go0G9a zGf>g^1w$4o_K$B9kTjC-#|5i&f5@oGO|PxkBNFfPcaCo8BijKqdDywTIpCn#4fh@> z%Y0J}gF!l^#jA-ImMp*4v*j=vh3UA(hAl2}Jy7CycpN{Lp&zi29aeh*8EdMlnW9wR z?w7QxT=n0ku>mB;3tM=wy;rfDO57G(W~MeL-uuIbH7-8g0acV~eqr(K#i(Dq9X|E% z{~A2jG@R&#`c@NR59Z%~`fdPnc5y43zl!~~#+~u9(W6l9!I79B8JCqVl=LrGQ~#nJ zjL2VWFQ=`r<)2TD?ch&*>SiQajs0I`FItcLYUTqgeJKxI{l#fpbDWI*8?+NVT*5-v z8>(>ABy=V!%yLp@aAzvXU8ovHQ^jx{JBgA@k@JD~v7MY5YjD7HH zLiPr!n0v3ZGKa-vL+IoDgl&GGcefv$lx@usbNfaGD5?3V8tocuxc1+lD=@0DA6ZbX zg0ZeHp8MsvrH8}N4c+cjOrpM0tow>6(iL}ADXFs^S!1J>P1MQ$iXx{<`R+FrZl$|{ z1Di6=qIA=fw|`tlgU#EqE2guoRyTsfp2q_tyz{R011q@=1 zBFmHMwR9VyyGD;zfh`=<(5&e3;ph8xKd-01i@&I5+tPJCwa)j|b(YgM&=5 z%lcQ*M4;xYv!LZ-e(RI^O+bY6c6ropw|^l-_!Lr>3%P5y$+;Lt`Fj607QD51!xB&i zu4hAHeuwK;8%uwmDyz}`4dsd7?k_Udql7k@NKZE&(yV)wS23PuXY=$c=Zqrn_#1la zK)Le7n);~rP~rb9q9m+!4B^#gguJWSnIWQ18!BJzRpYuc z+wF>44@XUlul((z26k?Z}&}|E*TXjosA!R3r^umi0#W4mjw>f2P7K z3QBMJ;79%GXm#+(d{5(0OiYT8tc~^IL)6izpC%y3X*)LtMeCu2gUK?Vt?4_tJKXH; z7|JdFN-TW9_?Ho}wuU5x5#85&ob@B*6p{)!Gq1o>oG;q{^n90y?Ug{#n}=`{Bj88| zWil6#fPMQ1l|{*}zzcqa;#X`36BP@-iANEkZkqGe5@T#}noDWl=k7`C99 z7ud}@I4izoLwniS=XYJ2QiuevAcUCu{$R08o}p2^b(NM~tn%Ljy1l&=&l^83FK%9! z4jDe%nmNgZpDj0xrB0b8n@{>l^*L;%qniPn$%c?J2P6{3%66n+xmUHypU?Ml5F4f- z*G^1WXaVQVlXWjH-2%<$Jv2hjCah!td8)Z0efF1${}yOH={lTf-Q84_l7TDIOqf?} zwt^pgB)rf0aQ{dGvsopJL8GD^CQsplv_gesPA%w{+#hd0P_OKNGi^P=!nSpJAQ^Ar zk7BghTkO&rn;TT5C(hHavhKS(VUQ;3xo2vojACi+Uy9|P*%I~I9RDziM4jI$2ADD= zncwa%+8y8lp*JEOk+ipKTxKQ9tboA7!*7E~{V=;fZpmx5n53>Pg+`CVeLjBjgnm!IBB7! zrGGOuJc~{%b7Rf@pN(cRq_6#pH7c|_0y@Jhf>h-^J=*-CCB2YL3bnjf_6&C;T3&KQ z=)vz(NDn-)5^x_G@l^h0z#z*+YUC~8G7|XLMU-Etk2OpCHkS(9Pj&|&0YTF~t5^}A z{>^jymoZu~>F)TRf*W(fr)eAQ1!OVz*rcdPhb{y~47W}JelK#xeg4Aw3C%$n34TI{(N~bxuQ2qk~hLpV`fDm%B=Uxyxa`V z*fV$@t6e_gi>GvGHe^#IDSJ=&?@H6C0aTZ`9hxljs8k$m7D0xRRsmnqHBZ*3IjwAj zcQFDLBN(G=p&aGSL-CK-P@jD-R#C~ytOt;>k#30E=ix8GcA_(Sf;mI?wJxn!WJ6w@8kg zV*mtg1~E;B{*<+>s(C_%>ud7ocvHl*@-ju$>G%(DXkyiYo_pi>B!xyFIl0~Mn>fyjP!5bR5PN2PgXGl}>zXXyp4K)lwl$tYNtT2glOj zKTlFsyU`Iv?~;Od9VdGDN;SS2Xe~N$5a0cFK=omA7A{U;YN#R0m540;s`LMrrT@pW za&E(YE~D|=*gs;x^rQ@O1a7)aH+oXSu{u6ZpvQBC|DS5v45lB(v1S435aIWWH;w7m z*m)+Y*B&K2gL$`h^Z|JY`49V3(XJSlD4*r9YmP3t9M_3o2odrEA~2kb1pE*O`4D87 z*w6pzx;?gy|2-!`%oAMV{gr+p0~6PeY+BhLy3)kzfT)}Rs7j4Dji#4lc-X(h$#u_c zqm1qFI{~7>uw$BjxtSAp!{S{&i*_QN@2_Osw;NWHK{A>GRK!0XegpW#RF>xMKws8r zF0b+j>RSx+2G-qyBW=GVVD|#oqZ~>=nvC7kml>X;hRgvMXG+`dKyA)Pibl{$Owa(+ z0M=gc9!N!k7LU>=uj~%e_;^&yeJKfxqE;M0K}TK$%>NF9fmbq1l$taQUFh0G#-u>m=rBk+TlsgxN6 zm7$L$R>vzWep>*dk^;g#&6782aR`X8$;vO4Rz1n@Pxg=*fWH?2Ivl^96A)ryK>Z-= zpN6SX1G#CI5mOJ?)E<8OkxmMB-A?BX>=9#~BaC?huflG292`Rf7$JW>F|Ftb$v3@j z%^G_>FzqeZtl+8ro06{Ev#dL&I^0k8nK!4;4xJ?1T z!`Ar|T@4uW@xWWmR$U69+xe!VwWRb&qA9n=D}GtOvc-?kkUAf+z9Y#@}j1izUn@WeSM7*237-k6zR>8+2?t#|&lkwp!Qei>Cm zK18_NiaAIC{{7*P(W_BeWV93Akh3qad9MEqxJ#-6E+Ot%#sF}^$20jrI$cKEWJF9D zJZGY2i?e+X8mLr;0s6%+iH86Wn9X*u9cS(lPit%#g6&b|GTXAV=iIaJV&ZtYBdrUHz8y=lXm;pMw zD(3b(_&!2xIfU9dG&ygk+`N@>Y@?=$h2lMg>Iy{jn9c5Kd)a zGa(JNmhNQtLd`e@toN0xxTUV~K`XscW|*qCb3X-j+M=9?k93cq?g6wW*&Auvx61+T zdZV!n+hJa*L$`fu)(%+~*?UBqvl4NxO>2wA+A-eHw8#>l^OBvf%X>n(u=A=aYqrF@W_fv+W6{6^zQl^_a*r;0T>FaFz>BJUh z0Z`V4TX_5&MKKoRZxRyS5C{SOi}b$Col{^T=mj*+T_aefxVreRoZoJGi2=?6kla2e z>t)1OT4uH(1Cwdn`hjs~$kRXd+sBN63uLa4nqna=s!O8P-5-d$Te?}<8QqqeBM!i^ z(h{YUdk5Pxgf@W3?pV2t?1{p7+b(cf-5Fb$_1)mw-)=&M1Yo^3X;~U=*Gx(1#BDp= z)}MGItrFzL_3O}U!z}ZnX>peTuJi&2fFrq|cgQcG+moX=ILw;sx%4y*L*2FYdkmPN z)`T8li<@Th5wsl7e^Dyak!IFfN%KxmbJBbOk0+HpX2@PNoAd>Q6*JtbOBxD`rpTLk zS4i3h_HlaIM$U@~a7?(QaIwoXNw8JspHhbj=KSI77qnwio;iS)%)ml|O7j=dvdz-U^hoO7hKWnBMYA1ICF3gscy>yZ#c=aYy5HNBo(*Y8tbn{? zD@9*G-C5mIyG9Hv3m*GgiP*wEgBk8HDJdlBUY*bimBR*lz%i5?wBINfm<7CZ1#C zhs2TgEIztcxw2^&4KisXc8Aio5h0j#%^@GI*Jec>N5p0k1S7IN@ zh9m;1m37{g@N>bf&y~^?uQ`F6=h&4{a`oAKto79fa$D}t0MW=gr5=PoC!Ie)9-P#^ zk+ur1ttw{NC+en zE5!+|W@R@hX%<{zCvA#JsjR5Q&_V+(n z9#qHL9kzmUJ?I2ZH4oI+cz`{kZa9hcoRy2USO!UpNC5+5<&}1TnQCy)8P#LMeeU9n zC+D?2Wj9)8>Tp?pkj7C~|8u4zZ63gV9GpSNsrqNjhUi$Ct+bD?fjxk%6{E(p21j5L zU#`iO`pU1)r)mBJJHNKd#dbNI!DD@d&E$C&emvoif`pGc3R!ECdX;y1A`bbthvh8> z=p=kUxD0o_v+|QmNqcIn=2FOq8Eo`o_D(SXZnmwnEQot3`|%wo5%=X@C9ejy>GYw~ zLi6BZzDM_y3G@#qtY)ep4!kXq$z0-Z=G8k?WA{+CQ~gqi04+A-eEEO@_@%#FtK zKlBg2t~iN30IO0!JWV`^x8kFUK1#KYuE19ZNNCA#keUX(P5LOG{HGa^bxUX;P;lt? zBJR2Z>z2aQcKW?GzoY5$C@8v7PKBD{_du7)VXe-s=3qMZmA6#Vk<>zn`1Ru7k8gT6 zcC+v=gz$-F!nSnTlU zKgB|TPEp|cT1y#>ow=JP6Xxnv_)?j5`p(lBX$=PEob zRs+0L8M$Bcs+}eU=a{3n*SKm-brsys562Xz6#49IlY0*2u&)ft74lmf%5!?s>3g`2 zgsQ@+DQIX9@)dgvR*seu%oWbcU=cInjwp+}Yft7hlRkK# z2lwLJ0@Chxm&T8&q!f#pNv%h<{lHnaa!_cM5e%l4~q~Z_;94 zP{~c44IgF$Fl5Yewq=?mkVQS!*19*5?CbOZ&_s7o${24U^O0U(tl7-Jgi3yWv(<0l z{h<7VB(MPF#WSc>MERbr&O05Khi^|hH}xG^zU>#Ru9xzXyg%s$juJ-{ zfhkywHo>_=w@|8-fLc;VbMAX&{0Ch{`}Ac<%>wH&^E&cvT7ieqTL7GL8@3xaezua* zOHSdFrxsER;N@Ju!>+P}V9#2P-j?}pZSzpjF>wtrV8@MswE86|APzt!yk3PH>-s}) z49BL}zc2rCoy=alaN~^k0lPt&Bi|3G2QaHns%T=0ur|fZ2|kT{LY39d-7)+}?USlyrgpxh!1y(=%(Q8d-0s))G`2(-l=1Xd_@D7mTIel;jVb&UZhALB(MrkLMH7rwMsC2~i!iBZk zRZ{A1D>7pDXs$#Aj{*thZkJA1mR+8u|H!r)niBr23{iQuZZhrYUZ~k zVC+qy@#{ATjDHli1f3=<2@Zf8uvQBWc+-i%HsENtmIQfC;$wB5=AGx?oQrowS5n=7 zx{uno7?(!b`;J&00_BrGM=`3FO=HH#`6^8a4KZZN))dka;2KmV>J}f+jlz0(aNiPl z-t)jlC$$oOwQk^ZaDWm4qhtk}Xm|X}2Ddz7TWkzfP2&9{el%{u#tksJ_Yi}Vp zWf=+q{lVt%MQKKrmSJiB&x7!}jLWsw1h-E)uv-h~n{4JI!oNhOZ`<$$_kV1K^C6yE zw1-z*8brSW9v0^NfJYVpC&_oHpBn9&2fk}yOtkdn?d?Ni zm3FZ(806~X&c%`7ARhll2=~G>n4-11>e~yFppYB34BIJioeGajuN+N4=gC%!9W zd7kzB!W0>ZwsG`pZ4?h8--s@qHcM9_B6{^mb_|cC?0r#unevzQ>hu;S2YUj z6FWC7GifM5yE+e>6bw=+70KjuGtmW{xyFp9+SvnoJ8`-HigDI zf^9|MCNw;RLL4|qYXgFOAF(b-4{H`3yq>7Gt9@-*0vy7&0Y$snr*!WMY32c2P;)uf zOKL{=wxCWH7}6b~?bRN~JFndVKApHUcl~qEi_Y+>Z*OEWAK?*l^E*$CUnw5RDtNS5 z9)S;Xc1)2X*jNMDi+FJw@pl6z0q2KaN4Gv%SFZvC`bZ1uscBvDCn&5&)d4gr%COq@ zYv0dzAfp&QJGpu^k{~FAqS|2$ab)@O`>UW6tXV5sM&=X9V8={62Iajbx?kGXh5?}b z0~-~&i-CQq7MTqZ1f&B#Ip~-LYJ=;`>TGLk+8Z4mbF8(2nQKji!ernSwD40c8{^e$ zDjpBq$p6$Egm54H69t}j}SkaigoBKEi z3)|Jihw|b>F3-J<6|(SSEIJ}zpfA)JlH2zd@RHy?0?P6YS(%&X+m`ab zRhTIKPY){KtTPad({QuT{)hN0!m$&X==oSiF_I?iILTQu`1C2OZB`*8 zut3~tF|)b{YI)ZGtYu}EfOeD^l*Ga_6=HlhvE>5J@WWAQGg9nB7;GL7{IdW&5SC7; zZEkY_k=?7iL&EQ9TpM5Di^uRtLL;^q#+$ImrS)8JUbN4|6$M96xDocTlO1r9Ca3Fc z4CgAvFxL5Fxvpr}&GBb^Zi_F5ffOU1L5+Py^uDI(oCSyp*1#ltl-)u=$XehXbovOy znLjg%V`|_Oq2M+Vp0B_jJyPf0G$78tCOE*7Nzed8xNi-X6JgUUilsevVE)X&7Cl!I zGe#^l^$H^eJeC89(5x>R*hnsZw(~2f?7>y9{6Mc3u%5Pl@%?ZvSxsoWfWv4ah)9Wd znUu%bxc;(MfT|NESeB|1sIce-v9b@4A1;vdLxV~3EFdt$!0WN;mf(1FI>o;`YNcy^ z)o+Wz4@ArEUA=*$=bpB!WON7!_f`RB6U}K{!&jY^bX!3CCGm z4#!dwh_l$we~%mepR2Ok!>?my@bTGOd`*sh_F5?FU+2w+=U^4drc;V2lp589&scI8 zlyt7@VSnzH`U{vK7xvsDDc3#?-OBx9;%EcD%^-WnQrh>)|NcTR(3bxNgYrL9!^ZGm zo#v(I{Xe<{?*IFd`CnEh_&+_wYmY6TJKE(Y$tsdoc!FgcsQTDNp&Z98#DqCP1hZwm z+*$F263*Ufy|;{TxbJl=o@0RBvOg{rGq0^F$b>w}%5ikPvGI_a0(`eq6?ks0K0*Iu z$?58M{?99kjIO#)47hXN2Dc?%z^^E}BR^%<=wYV=_w~eFiB*BA%sMxyK(1kprM9poFrYTeMe_I` z?r^sJmz^zO6W~qy13-E&5W)Ixv7~}hHwJ({>fl$6i6*ytmpKP4?#saYDu8OFY&axfMwg%*0i%f`ZAA^|J z=ELoi@4#a)T<2o$zFwF=Q4$JO;RyTV?=vjcv_wU1FI0o2eP?vv9cCdRsG3w%9c}C zTwL>BH%Be$hf!>~4Ga+Z*1(t1p`r?(Vpu{k2&5xC-O#HTLtk*{hy?&Gic>&sv+-Nji!mqbsCEY>4%*4)qq~JA_ z7?d&z@W2ngod8m-HyH2s<}0My`Mt9GdBS~=luvAa0E0b|DVHJXziH7t^5nT1{hohHj83_kE&=*j6cE^C4i>wV7SP7TVeY?#@)m z>4E8p&{f|8Rg(rY+x)>gaEz9LlkmS%=kH|TX*tej(=CWLtaDByzFuNf)6HpI+lx)J zj#g4!S}0vF&oSr}Y&~SzC&b7lAa}fely&r{iVXLHIuNYnG5@R^KM}ClPWo#X(+!M5 z8y|QXvAO%@<7LG@W5BlWyTR9b@?7M z4oVl9eR>z7q@1Bd`DtFs{;BCM0CrgenQR%57`14v0ofo*sq&W7qQ1ssuuXb_Ok{~g z=hUlLE0{qCpDNvNTVx;9*&(F^@qdaoWl#a?KoCqui^Kv7NN;?-KdfD%uO0Ct@=H8_KuAu$AZ6_8cq+F_Xm9x^>T_|M(o z56@VI=T!ICU_^v?m#;RL|D>w3y1?9bFp;iwgLCFbK)%4N#fjNH4S zX&zi$*4X_Xl%y0Ir7Pb!V?Cd}W&19_AQmt+r;_Sz*jkIEOjc+Xz;m^QIl+%@(tTik zwa4Jd(IKjShvBI&-29C>1=Yfe7@dg6Dg1WxeuIsjK+&3OnzFq{2s{uWI$#$&K2`5i z{~J1LhbY2HuZ+~4SX4}s#`=c2G#Y_$=oF|!Y;<@GXoN|k?AO0SkeP4n*)}%fjh&wX z>Y$NLJ2$kCe@Np{Rcm1s_=#DnP4ZutqUXm_+OyU^ozsz*%9PK_XB4xG4BHG~nIU~F z%CF2PTHDFz=|K>Y6Y>_^nlCY|`nKgBO><}ys;~Ws>R*!xu6wDX+s4COZ22VTwZjI9!N5C?i98$FAi+KcZB#M!cq1N}2_lj)Z-3%?4q`zFZW4E^zFQ>%9E zN`WADl+Vkat+ZV$w;G;K^|a_(Zn8*G{s+X}b;p8>^5sOCgO6{`CeyrU>~Qa}nFgk+ z8eU)bnZL2sd@T%^(MA6_GFi#JD{R{YvT_V#3j6(a>IGun)%{=7D(w;}eONLi_^Io> zTorJ%Gmb@J2~`WG=S&veoWp;NBVp3{yg-%|2Y?{j)5g_$?e({130hV`+c|lY>;MX z!1h7XkAGb6M}JD!qEM)T=5w8WcFm$|_1%p8Pr|)@jqoT0-bE#h_NnFJ8yVnP3OP^N z=GS`;+t}XWpWa>_%z09#^He*DBuqOX30wYS#^2MwGxzI>v%xD_B=%~BPBUKq0fs!~ zI9_b&V4+@dd|KKZFtt8wJ=;@Bmdrn86t2=>E%=wm|GIT!IM}{9?c1$1JTG8kHS2WBqWy5Acx$|;2R*vG9cll6zD>(z+I53F9o z$%C^wBY@w2TF}@%fn)XdIj}MGft=HtK-~Url9963RYB&yt#o!(&dl8~Vrs#0zAq4g zf70r?y&+giu}hV6#zLFeGUw2A0LLifwX-_I6S|ot4vwEmh6)*m*!qupIB+2{3zW5- zuJ8Oa@1va_zE+$cz5vR&`3v*?|8{8{|z)E*@o?l*v*y|4TVIE@!V3Uq+!%wwWPey{^?kfnvI|Ho& zmjzWezpsitY@7hJa2am&bVw1%UJ(PX;9ez`G|T=L`_bEtEl*g6pJiKUc6{ni&?TAM z?#s4Nu)1F%>U&VY<@8nz9O?6BQEAjY$M1Dk^ zarkv6i6}sA@HhdeaJZW#c_-OS2MY1N^II)Q8IKiIJkQm~OMqGdYz<>bfPo;?I)0#6 zYCU1WlBeKPgidAFR&HjV5Nl>;A#W}XD66e*`}mTzNjYd5;0uwmL!gOBFDDf8Qt{sI z+bUy#4yE7X#X$~(HSffCgCB1R0Ht26WKH*iBS)H25Ab)N?L@X2px7HjZV^y|(RaIP zCCb(b?&iR)@>tV0c6;hFBd!C^gtyYwpaxwF$`ocPQ18blEQ25C#CU2nHEir#^$WO$ z-#}^D%>gu$KKf>H<(ERI(L;c<76z1~pb^HE-@SOPUxvO1^(spcVk^%3%ZkZL{2Dv3 zNCh*Fy)0$Qt%qxl0a{6ft>K)!e53{B=<+n@VF~gNzLW*|;mR#jO9a`yUl1MKF|$#eFboG zpTtr20wk%<;RgyUKdIn>+C=(Ehu2sO4;qcAe*5poHDi*@yBwcjsi~6wjUu3i)xwBC zKtuOdkh!dUpRg84fUvPFRQvPpLdRq{$FzYz<&>^I8O5Q6X8C7umjvQso15uYur--B zb7)Fs_dzl?xcM)RgBtSG$qe}igN7bzuM5MOrNC1nj!pTL>Y+s0YtQ0tEh$#%{~q4d z>|>47`^U|b_{<_C4WZpfaqrG9VPir#?1-ov-1XrCeX1Va?jptk4c~l*oy3pe!89C3 zE;x7$vF8H-mV|Dc!;+_(yvyOoT6Z3s*-M`Qs*&-Q<~t}6RVo@aD>X;?Xb2j7yHO5n zL}KoG3>qjs3i52}$gFR-;#h16l@2M80L?pH6qVrTp^DFBpyr3RTK8oXu;)?Q8-pYq z_LTwM6ksH&#r5io&eA6Clg#S@{!zOczy)y;t$N-m|*ccm|f;v+%t~7|juZA#kmSh0; zyYU&a3xPe3O}``&UvU_pyWP#7Yg2#vlPXYUMoX~qg`1I@B zWujZa=ReT{;GOn#n5n=lA$tM-SyjIzANGEZF^A-)crN2Bx8nA0ayxf2!1;Ni#Xr@8 zZs1X3Q$nnNq{yJ$x!l0u94MYC0Ig1p1`C0+<@?H&yl?3et1f5vP{@5*bswb36V_JA z#*zX+cxBoD^Wd`h5j~%^s&FTR{pQhfipxa!4FQNTply{%u}(N_xzD?_R0e1@s5ue` zweUxm;C>?I1nhYY;t}Z$s-d7t{9NE@z}6tExHX+VVI7w77ZE_c_tgYD{rT1HG%Qd9 zu)jC^vDOekHstk)Uj2Ec=ev~8fZs*mp54OiR zSaHlF8%Yb6mNh6qcJiYyV1ahi@%t}>u~Y+wp{;XJf=WH+mS)I_rD{N(dR-#DLAG32 z=v_i7gcHL5Y6f520*~?dd5a|xYuO^gkK+bP%`dFtFjAz~Q}VI`m`vCzLgShau#C+! zsWL#pAFTi&Np&cRsE+@CJVo^%b|V5}=I9a#zTq1c+YnL&MN)+@P(e5Q0q#4QDQk~Q zcKzvZe$@f(=b4YlHRFE)qU7ot1)J#54p3C=6abXl;aCm>?&S`)OxG#b4&)}`q4$n2 zP`I`<`W&+$;8Hg5`5i1R8wiXgZ@|?JmLM5#V60gggGXCOo5}sV^+`t*>g}suiqsOUk`hL2Z z4!`FzxJYa3|EliGqp9xyeMN>uAq^-r5E_&*WQtVgDN`9Lwt0w@c}z4IGNf%DH(N4o zp@ft&&*L^0l9`H4w7Kuketv7+b=N)ju5;Eo>z;M}^E}VuVSk6u=ktEQruQ~Vi~8C} z`Ky!4o}yl~BAFVA5=7BIqj+BNWJ`b!KK0wHURx`T=4^2W9BF>E3Q{+Emy6+OyzkWn zkx@gVjGV*v(rTtTf~SSwO=p~l>=ps$$<~WBZ%ac5SuMpop1@u~Okn~Pp{(`^@^Uxc zc{6H~YD4B?S2qG87ePM`sSM&-!+>*DNPdD&;@|^8B=Z0c>3GLiC1d(gCTi=V#cDIf zAR}?d!5$F#Gy-fg2HK#`vvaX}n&%Bo-Q$=Q^z)56*Y2Vi2J%~%ZlGV>RfOU(|E?b=sdh)0qQ~A|EFj=DvB8aF5vgk|K?=7NoFS<1xDwx zVb5@>F9w%5-xs1AYsu-ObmbibHp+P5n_HPgS6zB$sCxCqKLhXJn+p62GdsI3pN)BX zCR|J@3azGwi~kY6;=c@)|1bMR|6zC^-6X1=|No{DqL2TdXYT$Z5B>kqEmnBPP@Mc} z50BJG%Dxrl_`P*{zd~D7^P%p_v56_bJNQpF|I3%cQAHsaqA(0%ydwe)F2Ep{T|jrn z@(#ro^#A%6{g=JY@QWU8I|Dt>Z&+{>2cZaeo$ElNF(OxnJRh019&tN@)NG*)sC0!u>aww*CjR%ordsXK zR(z%Q242H?Tqu?DlT=b!S3Uu8aOcrp9;a=2m#+$AyutlYvpJuwTIsS(RLm0&E^y1; zsZ=9#P?rCI(8No~C!<7c!Xc;uk~(!z$2-=9*%-5f<3keY3{g;X6yuGGGa8|XKun71 z5;~2ozNoclcd@+oj^vhA7hq=`Bf;^O00}g8u|n6INTnTKc{!J%fa zJtM-2YGv>j+z~w>c5crTy6@p!k`kaWc}*sR+PH+qFO*XqX98e)zQ(?!+`^~@8fH!pHB^8ZZ>QZ` z944A2<#4ZSH#C!xD*}tN&1ThxMe3p$2oYRl;OA1VHLmHyw){SQ}lU6!lZmh^HY<(-DwF zt6&zVCYK99${h=0(tT|rUa795qO8h(ky6iFC~ZHW=TOG(x68uwh6iyi>axm~t4byS zC$lU;x%$b|V7Dovm@eRLt}dU;uc|B$a?=r1HqpKIaD&%I;u%WeT$2MiLy0_jLl$bK zq4yiJugcTly3RR8kmf~(&F%Nlwop)v zw&`&``j+<-xVIei>hXFxDvnY3t-XvY&;iNdfPI|;Rl7K#CeKIA5|WRbEYrm!NS@f{ejbD?{VFpd={^iEI0o7@GTOF=?7*`V@`+2Y2a@#RjJF2Dhc;eakG_Ew+#q#3X1XQqfMztN94q(Y#ix5o9C95eK0|%c@|~ zxP}+)XB0ur3@+v5K?j%I^eOP#IUu;!WsHGV4!9cjetQKMcFT7l?QH?j$N|#^NMO`* ziN4*}*qMqPJ7!%VS6g5s`R=X5p%VqNU44nzT!_@=dtZkPUi?D^FJYzAlblQLuFTws(PYj7qWHUe zGq=Im%if=+-rV>56yYP6!%K1d7tCE3!cUE?&Ir|9@K{bel8>SY3i=~jF1H?E9o0!w zH5QqzPmjiyLbxN(OQc`t&`1h+l%iFSVG#>{I@+GmKe1CQ$?v$P!g6W1=59+2$Abxt zHT5%a1ym)TX^O3^;%SS6Z(begU^;Z}L!WUfL_$;WOPUDe4@t@HC$IP?AC35o1Zw`H zck7Q#A%QFOYj5I>-uHYN z;tF~SBJpz9lN4(>2+M|5wox2tD^SQzk;xv-VHutFE|L$fyMP)K5@H2?s=Rcj$%z7I z^lIfEybZ-v=mhJd3nlvW?F#Ms+nbpVdX1@Y$-I9{39+v_!O|MvbH!}3Gv?Ur z!{6DBi+2a>mh3<0LZ{Z2&(^5pTMlerhhdb0)sb0?!Y>hd^!;9?5EOY6q^GQr-hf*y zsIzb?<8`tU)&;x2fkmY}Fv!FDs7nwttgs@Qle9--Thn@$0iq&qzrs$zyX9~oIEWKG zY}q3(5(f7g_oD>lsG~|BttaB}(X998r+o|NIcSNxN=tW+$V->?QWv7=tQP z;|opv8*m1Xn-s_iXV9LEJ(@mI4B!af!14zG{~RwuNozXrsD@ zG@b($nhwvLOAuFamzy0wgXv#oZ@HYnJEaz%An-WmrauzP?=In?ZHU$Qnzen32e z4Rc1yJ|P)h7OTc+KGt$R>5v?12Gq%K8_q>_XH*s`Z0kE=W@k~JF^Evf5bH~_z(0WXlOU}2DN^v%1fgfq(57hVe-9R4`SxU4Ss z<`MzniB!=Kr{Rj?zm_+4P-ACG+L@$lKTxph)~f3v6s`0bs>&|9#e-5>%Wzp`|ao$bP5A) z#K@VIJ{el6au}dvcD>HTdG$K5y+c;k`;Yc%G|u+ZD7NoAVRde2+~>3zk!0ej_pQ7~ zP!J?Vpt=P;8#W1ZeE_(?o}s;9mKEO98WEKuw*OV4*P-cg!%M#*@`P4y!7?f~{P}QM z0aUP_@EE$>kpe9sA6ni3eGafAqvE#}DBKn(R2NPk2R61u?{XfKlJ&N@OdZ;TKpK^r zT~^j-Sg5EN4c3H2VBZo!!h>0{L+={aq##OC_z=U(Zl+&dQw!9|cX>~Bef6GoIyYX$L6snRT9ay*N>0^@WOz#A~o+j z1(S8<7U9f5$md(I^R@!QIBz@+M*|krU4x}A?iq$_WIzjFf>!PG`{Moj0|e_x$5`mG zq&7F<3K!fiB+py7CRovLm?6_KG97S>9V?+`Rexb~A0#qU(4nS(LiZLqWveoc5 z3|7?8%H@4zmYJ}*Ur@Ks2Z$ORY(xCw4L0lY|EO#m78`8BGJrFaph~OVMxq@0F=&Vg zG~OB$IZtBfxjjD6ow%JHu?5s!NwAl!rrr(|LMb6Lj{1Q98PBSfn@~m)6dxg5J>AI{ zr;Ts>?}2kfV)y$QT;j(aPmRb>UzFW9o6CGg_qrKrpj=VqO(XP9wkmNq<>r(CiYq7f z0hr)2@^;a!PwD`TeJaOWD)wK0ZPKghzXl1Jb?=MeWijh6x~1f8F!UPKc*yPz-T<)> zMyd*5pmxhHW*8OUo&teIv3#=AcBCxnB?^>vD6#aK++{XbgLQ@^Lq>s=OZ#6^LP=@c-e|IBcCo# z(!+gWuy^b4*WkV5UYgc#XSve-Se=CGwGEVxHB;8x)6#yZ05);u6B8-1?w|vw7y1pmcU1DrM+<72lmu`3;55&<`DhTEG<# z-sM>}Ux(cy6Jr>ba7Bnl5hba!Cwj;}KuB{fIrR)(=gy)@6Od5jt+Dg$XD`?l$fx&R3`3z4JyZ@BI ziPBE-kSvj-;Lbh`+6E5q5lDBnM<#6ZQywUnT(X;$jda zBM^N!F6aW^zRnkw&~Ur#x?H*d`yXtkTtK41niem1U6a0gruEXdtr-_iWj{ESsw3SB z`eT#YN|OW|Aw3^thKlOVjsr#M5UHZoQuxp7<_4eTnntvG6PgpSScEvzCXkcnVTsYvNIF5tYuE$4G#T*B(=vm-JR>HpgP zV1?^69n-wLO-J*tpSod?jz25Dfb(lYg4pAc7jGwCvy*rRiJK>k%L=+Tkj3Yk zQ};>|NQ2&F?*mL}ueJ4bj-TO79dwQkP-;j2t)yT%0o%gDH;&;U9Spl`AOFCPYRn_F z+!u7Tv#zqr2#nSE`+H!Xz>ckQv9B>;0H}hxRm9g4j8^7}-K0AmROY_C$Uq|#6Vd}b z3y9NBew&KiLoQGVNl?*DlxPL!r7a0=T$yAr=C$^|YZ~GBw1D^*uibTvYM0!FcwIm5 zH+%_+uAh9ss(pX971?_*nqVLqK;4~07zM9VI1L;1IJm>!5VVF-q;>2iEJMPSEHs+9T4+oc69oa8^VGh<7)&(2V z#3R;=nd^J4aZK!T4sPK+nvC#_(`yfz;w&1S+54|Q>>6^<*puW z%;MC)ErXW2;(gsZ`Y}tbSWRBj$=foatd{7%qmo%|&_AiD{+mVfRR0=P6!iy^A1Pn| za|{u`Az*;C!@ zVU7`=Z%04sTO`~~FH3>I|D8jV_zlB6>Hm4%6#ktdSd{Diw;c(*^PpT7jlB3P3~%r{ zK?-oHRZ4)31vGFo6;6HSBGcR?5GejIp z-%uh5$s&e>p^}z_X)TY@Xe4!D^A;qK@*01mynq5_LJ{x7BZ&q&IL_YW_F1m2=*xtL z>sx*G=3=GJQaLE>!+!kybt4|3-_`W6Wtt#-IVO3gDs|WoE9dLgQ;$sYl$at|`LU;s zyKX`j^DakWlGV8kUnfstY0Z%f5~6K@u(d$5s19f)f`&8wG-=)c`a^3sXRMq*@^bNH4xvLe#&qpFAZL^zPcYeLM>mMD z^kCDAk@xol3D$nVKpzl*@O>EZ+pSINWWg%NoQ@!`w`tr|jD1V8I_<9p_}zF|y^90p zEagp-Lh5_&e24}-2LmFaZHXn>%!l9<=tH(zGk<`~zf|2Mv-VS!LX%NlUjK`B3PtHd zCv3I8hQ#Jb@H`b~Xm%c{zR91oGgX*lVi1Bj1$uz;?S6ab&WsQ!KmF$q4ug>Fn$BQO>PxG-wDs46Rpqd@aFKe5djm z3?k^Xv}Vk+`TFfT)Y)U3W1uU?Og22n0iZtxuyJ|)PiTx=AqA-g+<@v}s$(}m` z2ye0)69p|sCWoAl$ECts5S2x7j(}+On$sqfZLM&I>fZda-|*(Gu^LU6+YoL+S_(Uh zv@-r~grVxcsKfAMsk-mGw`&(0Z9UAWxxQI`AFK{Ih~%q|G*tMwBw?B=K(d;RY*m(1 zlade}tA0i*@A-3csM^=NrzZPwB379JTq^+Q?Vee10bA)K(&7l$C_idw`^|x^KJT1^ z!aM504Sf`%LGaq}A$*t0eg3$5*C1KaniQrz<;d#WHxY=2s6y&fDJLFybR;wKgTAU8hDMSw=u~9l8*hlrj+@OsF z9^|Tf$z!XKnZ>*cVFjbOwJrcQFA<-QPI!e&-@h#m9QyKoaLsG1Z$M=o3Xk=mklA|= zl!I*dR#LsLt_lB8N3F4C$bIc?d52!-FMOYCY}v9&Uf|x-jfN zW6~qH2qYAf`8I$CA;>vY*gsZfG=j9J?dD_@Oy;}P57K+>l#8D8W4CL5xXAhcP1yuJ z((dm~am}OEXim)ip?K)c?56<@+zs{3sqvQ=f$hl!OYOHMk4lU2K3z>reOrj3eTSduM>!
EM z3M}Ai15EpblSlLps2(Bg54*Me$KGwSFkp4!IjJ=lC zLy_Bc4EuzwqGt%Y*sEqHdy}8nOh&8S_H%UVzv-s2{>;~4{RY=kb&s(YIb4@Nc=M(H zpE{U8WC@BYD<61}8+Sxm4<{Pin+wfdF*X#^arNKA@kO-OvDgHhkwq;sIrv76 zANjIu^^AFYNuXdaK4y2(6~NpBz2so2%8ikgEBD;|dUY5;GxWtY?IL_J)eL zI`ZEXCJWd6?RsHRr?dmZ9D98#x8}POkI!--`9{>$G|8Op*VnH-^D<}zJ2v)gdt-IW zA-II6zfDDDx8b4l1I*#@B@lt6)i*x9HCxU6DGk(+I=n`X$rIZmliOD~*r21ol4rMg zP}t1Df2wqgskkRenuE`7>iZS*=I>rIwg5oBlBa+Rlz;}srw5Y*>iVZgE+Xre#C67BFRjdR%vOg4 z2huE@;H}LZjl3iJTWw`sZPEEPbsmaiP<#Y3vAQZv#>v^@JkbYwFO;BZ)1cIi>^Gr6 z<#C-HGf8G4LZT_zzf`mcIjd0A{tRHsp74D~QW$qU=X|q8-EVYqxKDdO=d75ON#7Vz zhJ90t#YVy7+{9W)qi@x3dP#2SnAa9TziU~Pi4PNMrcZ}=6|rFjk5=lriRjR1WczP; z&b=<5C+qv2xA;JXoODEjLnr1{SO`IQWTw(?JeNa+jo5B-P@LC(th$P9N_d85AzE5* zh^)YtKJ+UdK!(8&PdEiWd$-upu{oX+575(V^(Yn=U1Vaa$4tdH{LG2ob#jRJ%vN9C z^ZcNPi}@lymI_qG|0cBXU$J@sr$7nesDMX46}D2)O*1WDEsiXhmKDSAr>~o?xdAQZ zJ5NLlV^l#OInR4i2VM6Dy6!jAGC0kfW)Mh^gVtYXss5j9Usp1;md)_BjaH^RJ)YGf zL!VLkEPp;BbQZS-Je<|gNPfYO z4c#o^6~>lnae08+%Opc^N$x>cin$4RStnwAE$op`lZ_Qd<}+JcKEhKw;rjP2y`=5W za)Z?PJm0=CHEUJdAGh>>W{W+66eV4&us{6ZGC?>t0S4 zOw=#WlmAfc_vfdkGhXETzkt(C2-pA9%p zZUlLL;jn(GCHJS!@9vWadt7uxMVK>sPH1#TqTB`2(|Ef&NXTPd(^pZ|=)+R7Mufcv z>z`OyU-9yFV*DV;Y-nvE_C0Be*3hn?%?L+NvRbxQ5r2~{SD%q>;pTAc%?h_}S!xi? z#!zl;+{P)J4zAs6D=$>H4b7>^vK>UPGgrce-KgU(;Mui_F$nk2WOK9t~?*=%Jp}^o$y~I8vrafuO9};%$ss(3Lo6nau$T0c#pi+{bbTP1GJHN zczuN8=VU4BjR)%O?Q)Hz+TQ{OytxUsu7xFtMPOQTTN87FTRd}`0HZ)jvIO|6+L~x} z@nO9dK|#o@Wq&Xn*Wc}nu)minp}yGrNi*~(Vd7%Pgp@?Lp=x&lpX*;Vd*-iH&i1Bl zZ(dxLX?M+MQND5Hg7ohnQ$()|VIAI%U1Qs53Xiu}GgN#s7im{PG;qmSCV9Yhem6O$mwn(2#9&1B!1XNFESWYWWwm^-3m9qB-q7&&q4 zm5H0?VAgwB-UqOObNAV}&T~-DyFnOUh>Y>c*|H%gg0)Ya)8aj*w#O*+tKDd3-XBmJ zhbD_-NrFMKQ^^9CqvL9SLEr`!>Ctlub_RM%&GR5?bX3J{(MpAn`CXfN*~=lNVV|SU zmi>)C$d`l3$_#{=Eqx7h^EStzIFDD3kdNDtBK*ke9W#jo4Q4p!kl=AuUD2V3x%!=r z6}rKL^XkTjoASCJu&)|xldO&zTPhwdoNq4uR_=U3f?xEg% zT=Zo9+iGwW;8PB$m@vNZU%Dy|<8~GnRyWx&ww6}X}PQjr)6RAL>4pN{6>;Wsf3*AO<;h{EeJu}$5|CD>7 zL8y^?pXN%7St5A?|Fx>+P?z2Z=mMqSvy}n85Eth5@My_YK0=p;x$V2K~q~ z%x<+`(*c3QB`BH0z2qZW^9^N^8h(~T<*5y(rkIriotg~L45%!`u-KdWtFkS0eqA{< z10AA(%%WOOYSh-&+chiK6c`iG0fMUI1yB_mO+y=2OnwL^+x;dGrq(joSj~j-=cgQH zZ@tBA{P#nbIx3%SW(u#)Zh|K7;nr<9VHe&L@^0yxM0fu*7f(P|lSgg1GzW$UZv$pi zuAk+FUD+}s(S00hT!t%Cak4Iu;i=cj*Ht z=S7{zObbxGvI{-&c_$6V_Bo0shwIIwh}>N{TXaWHE;id(Tf}l?U@vBT8)z|_{W+1c z$7sp$?t^Tea~2B9HCSlkP!w@Z_G~6JDmNfAKrhiVt)^;rrUQy{Z&MY-RdGlOD+IWSMBmR47Z=uovn5d1&X**5BeL824erg& z8ow&pzTWl>(8us+TEe#4u%*6&j>6vyYLcz&8B1G(IYOptd+IU#F`du%SaQ!6T!gwO z@o|)}xg6JOFUK>Zc7vSjD#2?R1X{?cbj<};J{@cqLb3bscS^>}ewvjEAAYjD27Q#1 zb@Hw2Vs9;KgHRYu2o!{JGH#dO2g&VTteym9y^P81GuA~DFP>*5H?pgJL8<~!Bs(kT;TtsEJMu#2T6fpuB1VfyF&d2KbcwzID zd0#ltUqqzG%Bi`hrbVEE3B=8rEw3cZi)$xtoSG(l9evuyE0n0{@0`&Z`@DC}s&t7% zZ~;Y@%H2alUBSz7SO{L1r}5m!gS;!x_>yjJwhlgXc-z z?JbbC=R1`ZqbY(6(^6o1Xv{}h3FXM3kL`R#O?8zXjTdjoN|120qqiPyMoFs3!XkZd z+M*iTkY{GC>L}eE1x-*WWpEm!bZ#7?D?t`TDeY0n6B51|cY;&Q3IJbWQRqEkPfE#M zN(CZFlK36)1$i7O$i8iE(i;68efrKO(QXKu|DnHdrdK!D05H|^_+$ztJOeJKM!(vQ zN&y3ACeWKNBB?v*0{57+TMj^Bh-RpQ_1&!n_K=v1Qxy{tqKJtEC&_s!BE0GIyzB5w zC0`j9N}#dg0GAPhXHk9?U4m8CP~ zc1|L&517+~)i&DYp1(2Ou!P+}hU;zM2i_XiNI}TWJz@lbyv>vPT#`;==y;KG`5N#R z65?7xWYB_w8PG6j==E+Rb)+0j-*JP$R_yTG<27uyiFTVaFpvcX9S_b2fGCyU*k3a4 zdIRWCGr`?q>#~@b1Waql*i)3`4#i61AGjeJ;hf?qidP*D0biaR-{+FfX1`J38@#@C z%ZxjyjKTc`qMS5%iLgjYxdYdx><`c$N0lnJ4fYdz`f-3I-vm3*nh(eJc0j_pC?sLK zp+TJa$~5i^{&8oK0;6$s4Y*E8KnF#^ZCnP)+F;vBQ-EEDne!zqe!fyUMiHbm+cFP4 z9CzD8@b@%-}x@tBQoetYi`X?8c!tEM?a$gYJlrDU|B z^2uKyr|p;k#?}G%Fg}49P2hqda~V5+0cq>l zZ{82_aU&!R?Z_QgN0l>Uy+_d55ox<=Lv;qcP#Rq|yw5RFZ4w$Y8B8?7RbN0KIN&6? zDyAK!#CLeMn8m&^Xb28lNBVg$$NfV3SI-N$qfG2)dlq&J)81ka(Eg%mS21Sa>1BrU zL$-Y2l+j(i31(VKCZlmIT^Ag2VN%~m)MOd&9oPh2V_7R}Z;irt>b)1210b~gO7!L{ z;fHij-~@Yw417p#E45JCJi(rw~HdXktx!DpnIBqEB%RFV9JS zbAe#-vZ32vzKoV(UUVKb=O1fpe;$FEiaJ!a?Lg5Zg4QwS&34| zxE-!YfwSPzF1uFu=By^Z1(FN*casxgrajjUyP<8^alXyf{F&k!D!()&^!!Ua;~rZc zyBb@rFwBcE6^?r@+*_Pjc* z2Q4> z9Ckfw+6PwpS0^H`2PlbU=QN$ZeHS87L+F=gZak+=8NH)Fq*pBS84PN&**ovDI&HH(KUCh1q=Db& zl9@t1n)pjilh-m-bxZenndj1fg6ijMBJ)HC*W430i7xq7?H04b#q9d}1GWYNk%hBD5#m(A`c%nWklN8O)l zH3%w5!+bYR=3U?3m z4QoF+vm_k`I_Lz)1y5=}dMNPexWVYJT~-zX9qCp#-CD)Z3)jn5WuFxa)Oiuq06m~LZ{XS@^5wy6bqkHeL1kKCG|RB+{WZuNX6 zYA|or6xnI@lxMa!Ovxtz>_$<)RHkAyA*#yv zbX=yuvshZiTI$=Iq(eOX6KN#R+~~x5(wg=J*J|)FYAQ7W1=CK-Se}T@t&8bQ`}(w7 z@|b}TLAx*vGE0Vj=%$8(*gFBG8)075se-}#!z6C43319?_wx>0LKxAv7FuCizTv_h zR8eF^wJ99Pa;3?xd?lv9w+o;y?cwH*uhN4hpNCE<;(zjTqHb)ogZgyS(hXyEY_!7WCq^mL3z?q4OVV|EOA za_nvT@tt1Ju9iM}4qG`iVz8CJ)a_4prA@x2rIwJa(6g=;yajaN!*8rML z6yWxB8^f^q2mf;?a0cuetk{N?hd%|tCPK!~Fja}^Jyd1YRin{fwPT617U+nwAu6A4 zrAxLw21wC!hi!&X+<-nNgs#WVWJav*84i5(q#wYB+zGk|iiE+`PR#n%2uw)FX`Xii zpN{M zspT9R<(mjz2f@$o9rK5U++K=UJy{6(%G`BW`d$MBJsm$B6&AUJ^K+oF#3pU*x6z*FKJn_= zEoi1l4>ZoqB>&_+gEDgO%Zhypq00s0oUnubbu))`(yi|0UY2uj#J*?Ib#VsBC{odK zk7w802x*$G_@6N}-fi*&YRE^{vK?t_P~~df^-SQ`lRaK~n%1y%glzL@H|;THMLI`e zS^@TD?e43sq;w|61$m95c@|3VlfGb~nk^f;){Zsn3FtBXk|DCo@u0Q0(47px{;zT8 zupQFS%L~x7~U*mK+SlUF%V}{)W_aqJub5+J<52?RnLsAS&hqY8VWB z_%iug6#a%$uaH)Jz|EF7Z%3(_VJTsv*iuX&EDeA9`t1zGDr%;LCLMx?&ATgciZ)N0 zJZwJ9$1<$rp`42LW1Q_wW1>tx$%kaLwt;llgCqCh5!P4jfo$~$TsGRv$7Dg<Fo z@QZezJ_hM3bbA!(s2a8?iGSE9UK}>}>66nTw$4+fP!orfV-;!~fH+|2QEcADcAU5k z2tVfQhrksKRlwn(E-)-aeM60fciD|_PH66&3cUe}r%v``h0y1{NB{z}_ny(B0w(;d zc;aO^tRM>n#{0lXS;`l+lIT^+zx>;zJ>^^fH|HQ_dKUZ?b}t89EPVYp4S0H)_E-La Xg` / ` + + + + + diff --git a/testdata/html_vulnerable.html b/testdata/html_vulnerable.html new file mode 100644 index 0000000..0b1cbef --- /dev/null +++ b/testdata/html_vulnerable.html @@ -0,0 +1,7 @@ + + + +
hunter2
+ + + From 17c57e96e333c47dcfb7e39c5b54ca6b92e8cbd2 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Wed, 8 Apr 2026 08:29:50 -0700 Subject: [PATCH 09/17] changes in response to PR review --- crates/kingfisher-rules/data/rules/adobe.yml | 14 ++++++++------ crates/kingfisher-rules/data/rules/tableau.yml | 2 +- .../src/validation/http_validation.rs | 4 ++++ docs/RULES.md | 3 ++- src/direct_validate.rs | 5 +---- src/parser/html.rs | 7 ++----- src/reporter.rs | 6 ++---- src/validation_rate_limit.rs | 11 ++++------- 8 files changed, 24 insertions(+), 28 deletions(-) diff --git a/crates/kingfisher-rules/data/rules/adobe.yml b/crates/kingfisher-rules/data/rules/adobe.yml index 5983e12..02ea4cb 100644 --- a/crates/kingfisher-rules/data/rules/adobe.yml +++ b/crates/kingfisher-rules/data/rules/adobe.yml @@ -71,9 +71,10 @@ rules: - | { "adobe_client_credentials": { - "client_id": "a65b0146769d433a835f36660881db50", - "client_secret": "p8e-ibndcvsmAp9ZgPBZ606FSlYIZVlsZ-g5" - }, + "client_id": "a65b0146769d433a835f36660881db50", + "client_secret": "p8e-ibndcvsmAp9ZgPBZ606FSlYIZVlsZ-g5" + } + } depends_on_rule: - rule_id: "kingfisher.adobe.4" variable: ADOBE_CLIENT_ID @@ -120,6 +121,7 @@ rules: - | { "adobe_client_credentials": { - "client_id": "a65b0146769d433a835f36660881db50", - "client_secret": "p8e-ibndcvsmAp9ZgPBZ606FSlYIZVlsZ-g5" - }, + "client_id": "a65b0146769d433a835f36660881db50", + "client_secret": "p8e-ibndcvsmAp9ZgPBZ606FSlYIZVlsZ-g5" + } + } diff --git a/crates/kingfisher-rules/data/rules/tableau.yml b/crates/kingfisher-rules/data/rules/tableau.yml index 14b0510..57127c5 100644 --- a/crates/kingfisher-rules/data/rules/tableau.yml +++ b/crates/kingfisher-rules/data/rules/tableau.yml @@ -13,7 +13,7 @@ rules: X-Tableau-Auth (?:.|[\n\r]){0,16}? ) - ( + (?: (?P[A-Za-z0-9+/]{12,24} (?:={1,2})? ) diff --git a/crates/kingfisher-scanner/src/validation/http_validation.rs b/crates/kingfisher-scanner/src/validation/http_validation.rs index 2fb1f0e..0862323 100644 --- a/crates/kingfisher-scanner/src/validation/http_validation.rs +++ b/crates/kingfisher-scanner/src/validation/http_validation.rs @@ -76,6 +76,10 @@ fn format_rfc1123(now: OffsetDateTime) -> String { rendered.strip_suffix(" +0000").map(|prefix| format!("{prefix} GMT")).unwrap_or(rendered) } +pub fn is_auto_provided_request_var(var: &str) -> bool { + matches!(var, "REQUEST_RFC1123_DATE" | "REQUEST_UNIX_MILLIS") +} + /// Clone `globals` and add stable request-scoped values for templated request rendering. /// /// These values are computed once so the same generated timestamp can be reused across the URL, diff --git a/docs/RULES.md b/docs/RULES.md index 6386da9..9fca4a8 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -943,4 +943,5 @@ rules: words: ['"Arn"'] depends_on_rule: - rule_id: kingfisher.alibabacloud.1 - variable: AKID``` + variable: AKID +``` diff --git a/src/direct_validate.rs b/src/direct_validate.rs index 9a03bbe..77365df 100644 --- a/src/direct_validate.rs +++ b/src/direct_validate.rs @@ -29,6 +29,7 @@ use crate::{ azure::validate_azure_storage_credentials, coinbase::validate_cdp_api_key, gcp::GcpValidator, + httpvalidation::is_auto_provided_request_var, httpvalidation::validate_response, httpvalidation::{build_request_builder, retry_request}, jdbc::validate_jdbc, @@ -133,10 +134,6 @@ fn extract_template_vars(text: &str) -> BTreeSet { re.captures_iter(text).filter_map(|cap| cap.get(1).map(|m| m.as_str().to_uppercase())).collect() } -fn is_auto_provided_request_var(var: &str) -> bool { - matches!(var, "REQUEST_RFC1123_DATE" | "REQUEST_UNIX_MILLIS") -} - /// Extract all template variables used in a validation configuration. fn extract_validation_vars(validation: &Validation) -> BTreeSet { let mut vars = BTreeSet::new(); diff --git a/src/parser/html.rs b/src/parser/html.rs index 0a2aeac..02a3482 100644 --- a/src/parser/html.rs +++ b/src/parser/html.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use tl::{Node, ParserOptions}; +use tl::ParserOptions; use super::{css, lexer, Language}; @@ -53,10 +53,7 @@ where } } _ => { - if !inner_text.is_empty() - && !matches!(node, Node::Comment(_)) - && !sink(&format!("{tag_name} = {inner_text}")) - { + if !inner_text.is_empty() && !sink(&format!("{tag_name} = {inner_text}")) { return Ok(()); } } diff --git a/src/reporter.rs b/src/reporter.rs index 9233cc7..7999651 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -12,6 +12,8 @@ use schemars::JsonSchema; use serde::Serialize; use url::Url; +use kingfisher_scanner::validation::http_validation::is_auto_provided_request_var; + use crate::{ access_map::{AccessSummary, AccessTokenDetails, ProviderMetadata, ResourceExposure}, blob::BlobMetadata, @@ -86,10 +88,6 @@ fn extract_template_vars(text: &str) -> BTreeSet { vars } -fn is_auto_provided_request_var(var: &str) -> bool { - matches!(var, "REQUEST_RFC1123_DATE" | "REQUEST_UNIX_MILLIS") -} - fn required_vars_for_validation(validation: &crate::rules::Validation) -> BTreeSet { use crate::rules::Validation; let mut vars = BTreeSet::new(); diff --git a/src/validation_rate_limit.rs b/src/validation_rate_limit.rs index 9eb9eb8..e7413cb 100644 --- a/src/validation_rate_limit.rs +++ b/src/validation_rate_limit.rs @@ -117,11 +117,8 @@ fn selector_matches(rule_id: &str, selector: &str) -> bool { || rule_id.strip_prefix(selector).is_some_and(|suffix| suffix.starts_with('.')) } -pub fn should_rate_limit_validation(validation: &Validation) -> bool { - match validation { - Validation::Raw(raw) => raw != "custom", - _ => true, - } +pub fn should_rate_limit_validation(_validation: &Validation) -> bool { + true } #[cfg(test)] @@ -178,7 +175,7 @@ mod tests { } #[test] - fn should_skip_rate_limit_for_raw_validation() { - assert!(!should_rate_limit_validation(&Validation::Raw("custom".to_string()))); + fn should_rate_limit_raw_validation() { + assert!(should_rate_limit_validation(&Validation::Raw("azurebatch".to_string()))); } } From 51b3b65706d0cea4cb79702d9ad3c10bfbff0e77 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Wed, 8 Apr 2026 08:57:12 -0700 Subject: [PATCH 10/17] changes in response to PR review --- .gitignore | 2 + .../data/rules/azurecosmosdb.yml | 4 +- crates/kingfisher-rules/data/rules/kucoin.yml | 4 +- testdata/parsers/context_verifier_golden.json | 459 ++++++++++++++++++ testdata/parsers/scan_findings_baseline.json | 150 ++++++ 5 files changed, 615 insertions(+), 4 deletions(-) create mode 100644 testdata/parsers/context_verifier_golden.json create mode 100644 testdata/parsers/scan_findings_baseline.json diff --git a/.gitignore b/.gitignore index d6bc06a..1f27a8f 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ *.json !webserver/static/sample-report.json !docs/access-map-viewer/sample-report.json +!testdata/parsers/context_verifier_golden.json +!testdata/parsers/scan_findings_baseline.json !testdata/parsers/tree_sitter_capture_baseline.json *.jsonl *.bson diff --git a/crates/kingfisher-rules/data/rules/azurecosmosdb.yml b/crates/kingfisher-rules/data/rules/azurecosmosdb.yml index a531cb1..c1cf6e5 100644 --- a/crates/kingfisher-rules/data/rules/azurecosmosdb.yml +++ b/crates/kingfisher-rules/data/rules/azurecosmosdb.yml @@ -57,10 +57,10 @@ rules: url: "{{ COSMOS_ENDPOINT }}/dbs" headers: Accept: application/json - x-ms-date: '{%- assign x_ms_date = "" | date: "%a, %d %b %Y %H:%M:%S GMT" | downcase -%}{{ x_ms_date }}' + x-ms-date: '{{ REQUEST_RFC1123_DATE | downcase }}' x-ms-version: "2018-12-31" Authorization: | - {%- assign x_ms_date = "" | date: "%a, %d %b %Y %H:%M:%S GMT" | downcase -%} + {%- assign x_ms_date = REQUEST_RFC1123_DATE | downcase -%} {%- assign string_to_sign = "get\ndbs\n\n" | append: x_ms_date | append: "\n\n" -%} {%- assign signature = string_to_sign | hmac_sha256_b64key: TOKEN | url_encode -%} type=master&ver=1.0&sig={{ signature }} diff --git a/crates/kingfisher-rules/data/rules/kucoin.yml b/crates/kingfisher-rules/data/rules/kucoin.yml index 0d6fac4..dbf142d 100644 --- a/crates/kingfisher-rules/data/rules/kucoin.yml +++ b/crates/kingfisher-rules/data/rules/kucoin.yml @@ -67,10 +67,10 @@ rules: Accept: application/json Content-Type: application/json KC-API-KEY: "{{ KUCOIN_KEY }}" - KC-API-TIMESTAMP: '{%- assign ts = "" | unix_timestamp | times: 1000 -%}{{ ts }}' + KC-API-TIMESTAMP: "{{ REQUEST_UNIX_MILLIS }}" KC-API-KEY-VERSION: "2" KC-API-PASSPHRASE: '{%- assign passphrase = KUCOIN_PASSPHRASE | hmac_sha256: TOKEN -%}{{ passphrase }}' - KC-API-SIGN: '{%- assign ts = "" | unix_timestamp | times: 1000 -%}{%- assign prehash = ts | append: "GET" | append: "/api/v1/accounts" -%}{{ prehash | hmac_sha256: TOKEN }}' + KC-API-SIGN: '{%- assign prehash = REQUEST_UNIX_MILLIS | append: "GET" | append: "/api/v1/accounts" -%}{{ prehash | hmac_sha256: TOKEN }}' response_matcher: - report_response: true - type: StatusMatch diff --git a/testdata/parsers/context_verifier_golden.json b/testdata/parsers/context_verifier_golden.json new file mode 100644 index 0000000..d3393f2 --- /dev/null +++ b/testdata/parsers/context_verifier_golden.json @@ -0,0 +1,459 @@ +{ + "bash:testdata/shell_vulnerable.sh": [ + "IPADDRESS = 8.8.8.8", + "PASSWORD = s3cr3tp@ssw0rd", + "PWD = a9lah209la81la3", + "PASSPHRASE = all along the watchtower", + "KEY = qpsbnoewdmdsoeg", + "SECRET_KEY = 402750613792034973", + "PRIVATE_KEY = ja4wALsaho20af21dS", + "another_password = blink182", + "backup_password = letmein123", + "API_KEY = 932" + ], + "c:testdata/c_vulnerable.c": [ + "id = 0", + "secret_key = my voice is my passport", + "employee_default = 0", + "employee_default = 8934#@hafRhzj13!d<2$F5q", + "age = 30", + "secret_key = John", + "strdup = John", + "password = Doe", + "strdup = Doe", + "msg = sunshine19", + "s1 = blink182", + "printf = values: %s; Age: %u\\n", + "age = 25", + "secret_key = 449a@QL#cha0213aKL:HF#@9;+_345Awd", + "strdup = 449a@QL#cha0213aKL:HF#@9;+_345Awd", + "printf = values: %s; Age: %u\\n", + "firstName = Marty", + "password = McFly", + "key_id = AKIA6ODU5DHT7VPXGCE4", + "aws_secret = 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", + "printf = values: %s; Age: %u\\n" + ], + "c_sharp:testdata/csharp_vulnerable.cs": [ + "user = John", + "user = Doe", + "user = john@email.com", + "User = John", + "User = Doe", + "User = john@email.com", + "John = Doe", + "FirsName = Bob", + "ipAddress = 8.8.8.8", + "String = 8.8.8.8", + "password = s3cr3tp@ssw0rd", + "String = s3cr3tp@ssw0rd", + "passwd = 9043hfdlasf023", + "String = 9043hfdlasf023", + "pwd = a9lah209la81la3", + "String = a9lah209la81la3", + "password = all along the watchtower", + "String = all along the watchtower", + "key = qpsbnoewdmdsoeg", + "String = qpsbnoewdmdsoeg", + "secretKey = 402750613792034973", + "String = 402750613792034973", + "privateKey = ja4wALsaho20af21dS", + "String = ja4wALsaho20af21dS", + "ip = 8.8.8.8", + "pass = s3cr3tp@ssw0rd 2", + "password = 9043hfdlasf023", + "secret = a9lah209la81la3", + "phrase = all along the watchtower", + "myKey = qpsbnoewdmdsoeg", + "secretKey = 402750613792034973", + "privateKey = ja4wALsaho20af21dS", + "key_id = AKIA6ODU5DHT7VPXGCE4", + "aws_secret = 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", + "hidden_passphrase = blink182", + "escaped = Hello \\\"World\\\"", + "name = John", + "firstName = John ", + "lastName = Doe", + "score = The score is {0}", + "score = 42", + "Format = The score is {0}", + "Format = 42" + ], + "cpp:testdata/cpp_vulnerable.cpp": [ + "my_api_key = foo", + "setMyNum = 15", + "setMyString = p@ssw0rd123", + "setSecretKey = 23847601237597123230895", + "secret_pass = my voice is my passport", + "temp_password = short line for testing", + "s5 = 6", + "s5 = 4", + "6 = 4", + "szHackerProof = 15", + "szHackerProof = *", + "15 = *", + "strForFunc = Passing a string" + ], + "css:testdata/css_vulnerable.css": [ + "password = blink182", + "background-image = url(", + "background-image = all-along-the-watchtower", + "content = abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234" + ], + "go:testdata/go_vulnerable.go": [ + "Println = hello world", + "ipAddress = 8.8.8.8", + "password = s3cr3tp@ssw0rd", + "passwd = 9043hfdlasf023", + "pwd = a9lah209la81la3", + "passphrase = all along the watchtower", + "key = qpsbnoewdmdsoeg", + "secret_key = 402750613792034973", + "private_key = ja4wALsaho20af21dS", + "ipAddress = 8.8.8.8", + "password = s3cr3tp@ssw0rd 2", + "passwd = 9043hfdlasf023", + "pwd = a9lah209la81la3", + "passphrase = all along the watchtower", + "key = qpsbnoewdmdsoeg", + "secret_key = 402750613792034973", + "private_key = ja4wALsaho20af21dS", + "ipAddress = 1a2w3eqwerty", + "password = space2001", + "passwd = space1958", + "pwd = qwertyuiop123", + "passphrase = trustno1", + "key_id = AKIA6ODU5DHT7VPXGCE4", + "aws_secret = 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", + "hidden_passphrase = blink182", + "badPassword = sunshine123", + "goodPassword = kingpin987", + "bestPassword = kingpin987", + "Printf = %s %s %s %s %s %s %s %s", + "AccessKey = 924JSR1PGW2D4MNRZX45", + "SecretKey = wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "Println = >>done<<" + ], + "html:testdata/html_embedded_vulnerable.html": [ + "html = .auth0_client_secret {\n content: \"abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234\";\n }\n \n \n \n \n const auth0_client_secret = \"abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234\";\n const password = \"superSecret123\";", + "head = .auth0_client_secret {\n content: \"abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234\";\n }", + "content = abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234", + "body = const auth0_client_secret = \"abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234\";\n const password = \"superSecret123\";", + " + + + + "#; + let mut texts = Vec::new(); + stream_context_candidates(source, &Language::Html, |text| { + texts.push(text.to_string()); + true + }) + .unwrap(); + + assert!( + texts.iter().any(|text| text.contains(" +
visible text
+ + + "#; + let mut texts = Vec::new(); + stream_context_candidates(source, &Language::Html, |text| { + texts.push(text.to_string()); + true + }) + .unwrap(); + + assert!( + !texts.iter().any(|text| text.contains("AIzaSyBUPHAjZl3n8Eza66ka6B78iVyPteC5MgM")), + "expected commented-out script secrets to stay ignored" + ); + assert!( + texts.iter().any(|text| text.contains("div = visible text")), + "expected visible non-script HTML text to remain available for verification" + ); + } + #[test] fn comment_only_python_context_is_ignored() { let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); diff --git a/src/parser/html.rs b/src/parser/html.rs index f10840b..f0ad094 100644 --- a/src/parser/html.rs +++ b/src/parser/html.rs @@ -1,5 +1,5 @@ use anyhow::Result; -use tl::ParserOptions; +use tl::{HTMLTag, Node, Parser, ParserOptions}; use super::{css, lexer, Language}; @@ -35,25 +35,27 @@ where } } - let inner_text = tag.inner_text(parser).trim().to_string(); match normalized_tag_name.as_str() { "script" => { - let candidate = format!("