diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c46ebf6..5fcf36c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,42 +1,148 @@ name: CI Pull Request on: - pull_request: - branches: - - main + pull_request: + branches: + - main + +env: + VCPKG_ROOT: C:\vcpkg + VCPKG_DOWNLOADS: C:\vcpkg\downloads + VCPKG_FEATURE_FLAGS: binarycaching + VCPKG_BINARY_SOURCES: clear;x-gha,readwrite + RUST_TOOLCHAIN: "1.90" -# This workflow runs on pull requests to the main branch -# It builds the project for 2 platforms, Linux arm64 and macOS arm64, -# and runs tests for each platform. All platforms tested on merge to main jobs: - linux-arm64: - name: Linux arm64 - runs-on: ubuntu-24.04-arm - steps: - - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 - with: - toolchain: 1.85.0 - profile: minimal - override: true - - uses: swatinem/rust-cache@v2 - - name: Build (Makefile linux-arm64) - run: make ubuntu-arm64 - - name: Run tests - run: make tests + linux-arm64: + name: Linux arm64 + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@v4 + - uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + profile: minimal + override: true + - name: Build (Makefile linux-arm64) + run: make ubuntu-arm64 + - name: Run tests + run: make tests + env: + CARGO_BUILD_JOBS: 1 - macos-arm64: - name: macOS arm64 - runs-on: macos-14 - steps: - - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 - with: - toolchain: 1.85.0 - profile: minimal - override: true - - uses: swatinem/rust-cache@v2 - - name: Build (Makefile darwin-arm64) - run: make darwin-arm64 - - name: Run tests - run: make tests + macos-arm64: + name: macOS arm64 + runs-on: macos-14 + steps: + - uses: actions/checkout@v4 + - uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + profile: minimal + override: true + - uses: swatinem/rust-cache@v2 + - name: Build (Makefile darwin-arm64) + run: make darwin-arm64 + - name: Run tests + run: make tests + + windows: + name: Windows x64 + runs-on: windows-latest + + # Windows-only env to keep vcpkg consistent and enable caching + env: + VCPKG_ROOT: C:\vcpkg + VCPKG_DOWNLOADS: C:\vcpkg\downloads + VCPKG_FEATURE_FLAGS: binarycaching + VCPKG_BINARY_SOURCES: clear;x-gha,readwrite + + steps: + - uses: actions/checkout@v4 + - uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ env.RUST_TOOLCHAIN }} + profile: minimal + override: true + + # Cache vcpkg artifacts & downloads (so we only fetch PCRE once) + - name: Cache vcpkg artifacts + uses: actions/cache@v4 + with: + path: | + C:\vcpkg\buildtrees + C:\vcpkg\packages + C:\vcpkg\installed + C:\vcpkg\downloads + C:\vcpkg\archives + C:\Users\runneradmin\AppData\Local\vcpkg\archives + key: vcpkg-${{ runner.os }}-hs-542 + restore-keys: | + vcpkg-${{ runner.os }}- + vcpkg- + + # Ensure downloads dir exists and seed PCRE 8.45 zip from a working mirror + - name: Pre-seed PCRE 8.45 for vcpkg + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path "$env:VCPKG_DOWNLOADS" | Out-Null + $dst = Join-Path $env:VCPKG_DOWNLOADS "pcre-8.45.zip" + + if (-not (Test-Path $dst)) { + $sf = "https://sourceforge.net/projects/pcre/files/pcre/8.45/pcre-8.45.zip/download" + + # Resolve to the final mirror URL (follow redirects without downloading the whole file) + $handler = New-Object System.Net.Http.HttpClientHandler + $handler.AllowAutoRedirect = $true + $client = New-Object System.Net.Http.HttpClient($handler) + + try { + $req = New-Object System.Net.Http.HttpRequestMessage([System.Net.Http.HttpMethod]::Head, $sf) + $resp = $client.SendAsync($req).GetAwaiter().GetResult() + + # Some mirrors don’t like HEAD; fall back to GET headers only. + if (-not $resp.IsSuccessStatusCode) { + $req.Dispose() + $req = New-Object System.Net.Http.HttpRequestMessage([System.Net.Http.HttpMethod]::Get, $sf) + $resp = $client.SendAsync($req, [System.Net.Http.HttpCompletionOption]::ResponseHeadersRead).GetAwaiter().GetResult() + } + + $finalUrl = $resp.RequestMessage.RequestUri.AbsoluteUri + Write-Host "Resolved SourceForge URL to: $finalUrl" + + # Download the actual file + Invoke-WebRequest -Uri $finalUrl -OutFile $dst + } + finally { + $client.Dispose() + $handler.Dispose() + } + } + + Get-ChildItem $env:VCPKG_DOWNLOADS + + - uses: swatinem/rust-cache@v2 + + - name: Build + run: .\buildwin.bat + shell: cmd + + - name: Run tests + shell: pwsh + run: | + if (-not (Get-Command cargo-nextest -ErrorAction SilentlyContinue)) { + cargo install --locked cargo-nextest + } + Write-Host "▶ cargo nextest run --release --workspace --all-targets" + cargo nextest run --release --workspace --all-targets + + - name: Move artifact to dist + shell: bash + run: | + mkdir -p dist + cp target/release/kingfisher-windows-x64.zip dist/ + + - uses: actions/upload-artifact@v4 + with: + name: kingfisher-windows-x64 + path: dist/kingfisher-*windows-x64*.* diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml new file mode 100644 index 0000000..6a35aa7 --- /dev/null +++ b/.github/workflows/release-docker.yml @@ -0,0 +1,101 @@ +# .github/workflows/release-docker.yml +name: Publish Docker image + +############################################################################### +# Triggers +############################################################################### +on: + # 1️⃣ Traditional: run automatically when a GitHub Release is published + release: + types: [published] + + # 2️⃣ Option 2: run every time the build-and-release workflow + # completes successfully on the main branch + workflow_run: + workflows: ["build-and-release"] + types: [completed] + branches: [main] + + # 3️⃣ Manual: “Run workflow” button or `gh workflow run` + workflow_dispatch: + inputs: + tag: + description: "Tag to push (leave blank → latest release)" + required: false + type: string + +############################################################################### +permissions: + contents: read # needed for checkout + GH API + packages: write # push to ghcr.io + +############################################################################### +jobs: + build-and-push: + # Run if: + # - event is NOT workflow_run (release, workflow_dispatch) + # - OR workflow_run completed successfully + # - OR this is a re-run (run_attempt > 1) so we force it to run + if: > + github.event_name != 'workflow_run' || + github.event.workflow_run.conclusion == 'success' || + github.run_attempt > 1 + runs-on: ubuntu-latest + + steps: + # ----------------------------------------------------------------------- + # Check out the exact commit that produced the artifacts (workflow_run), + # otherwise just use the SHA tied to the release / manual dispatch. + # ----------------------------------------------------------------------- + - uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }} + + # ----------------------------------------------------------------------- + # Decide which tag we’re going to publish + # ----------------------------------------------------------------------- + - name: Determine tag + id: tag + shell: bash + env: + # populated only for workflow_dispatch + MANUAL_TAG: ${{ github.event.inputs.tag }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + if [[ "${GITHUB_EVENT_NAME}" == "release" ]]; then + RAW_TAG="${{ github.event.release.tag_name }}" + elif [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" && -n "${MANUAL_TAG}" ]]; then + RAW_TAG="${MANUAL_TAG}" + else + # workflow_run (or manual w/o tag) → ask GitHub API for latest release tag + RAW_TAG=$(curl -sSL -H "Authorization: Bearer ${GH_TOKEN}" \ + "https://api.github.com/repos/${{ github.repository }}/releases/latest" \ + | jq -r .tag_name) + fi + + # Strip a leading "v" so v1.2.3 → 1.2.3 + TAG=${RAW_TAG#v} + echo "Selected tag: ${TAG}" + echo "tag=${TAG}" >> "${GITHUB_OUTPUT}" + + # ----------------------------------------------------------------------- + # Build & push + # ----------------------------------------------------------------------- + - uses: docker/setup-buildx-action@v3 + + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ghcr.io/mongodb/kingfisher:latest + ghcr.io/mongodb/kingfisher:${{ steps.tag.outputs.tag }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cc46915..e918d13 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,20 @@ on: push: branches: - main + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: "Tag to publish (leave blank to use Cargo.toml version)" + required: false + type: string +env: + VCPKG_ROOT: C:\vcpkg + VCPKG_DOWNLOADS: C:\vcpkg\downloads + VCPKG_FEATURE_FLAGS: binarycaching + VCPKG_BINARY_SOURCES: clear;x-gha,readwrite + RUST_TOOLCHAIN: "1.90" jobs: # ──────────────── Linux (via Makefile) ──────────────── @@ -15,25 +29,51 @@ jobs: - uses: actions-rs/toolchain@v1 with: - toolchain: 1.85.0 + toolchain: ${{ env.RUST_TOOLCHAIN }} profile: minimal override: true - uses: swatinem/rust-cache@v2 + - name: Install packaging tools + run: cargo install cargo-deb cargo-generate-rpm + - name: Build (Makefile linux-x64) run: make linux-x64 + - name: Fix permissions + run: sudo chown -R $(id -u):$(id -g) target + + - name: Build Debian package + run: | + cargo deb --no-build --target x86_64-unknown-linux-musl \ + --output target/release/kingfisher-linux-x64.deb + + - name: Build RPM package + run: | + cargo generate-rpm --target x86_64-unknown-linux-musl \ + --output target/release/kingfisher-linux-x64.rpm + - name: Move artifact to dist shell: bash run: | mkdir -p dist cp target/release/kingfisher-linux-x64.tgz dist/ + cp target/release/kingfisher-linux-x64.deb dist/ + cp target/release/kingfisher-linux-x64.rpm dist/ - uses: actions/upload-artifact@v4 with: - name: kingfisher-linux-x64 - path: dist/kingfisher-*linux-x64*.* + name: kingfisher-linux-x64.tgz + path: dist/kingfisher-linux-x64.tgz + - uses: actions/upload-artifact@v4 + with: + name: kingfisher-linux-x64.deb + path: dist/kingfisher-linux-x64.deb + - uses: actions/upload-artifact@v4 + with: + name: kingfisher-linux-x64.rpm + path: dist/kingfisher-linux-x64.rpm linux-arm64: name: Linux arm64 @@ -43,36 +83,59 @@ jobs: - uses: actions-rs/toolchain@v1 with: - toolchain: 1.85.0 + toolchain: ${{ env.RUST_TOOLCHAIN }} profile: minimal override: true - - uses: swatinem/rust-cache@v2 + - name: Install packaging tools + run: cargo install cargo-deb cargo-generate-rpm - name: Build (Makefile linux-arm64) run: make linux-arm64 + - name: Fix permissions + run: sudo chown -R $(id -u):$(id -g) target + + - name: Build Debian package + run: | + cargo deb --no-build --target aarch64-unknown-linux-musl \ + --output target/release/kingfisher-linux-arm64.deb + + - name: Build RPM package + run: | + cargo generate-rpm --target aarch64-unknown-linux-musl \ + --output target/release/kingfisher-linux-arm64.rpm + - name: Move artifact to dist shell: bash run: | mkdir -p dist cp target/release/kingfisher-linux-arm64.tgz dist/ + cp target/release/kingfisher-linux-arm64.deb dist/ + cp target/release/kingfisher-linux-arm64.rpm dist/ - uses: actions/upload-artifact@v4 with: - name: kingfisher-linux-arm64 - path: dist/kingfisher-*linux-arm64*.* - + name: kingfisher-linux-arm64.tgz + path: dist/kingfisher-linux-arm64.tgz + - uses: actions/upload-artifact@v4 + with: + name: kingfisher-linux-arm64.deb + path: dist/kingfisher-linux-arm64.deb + - uses: actions/upload-artifact@v4 + with: + name: kingfisher-linux-arm64.rpm + path: dist/kingfisher-linux-arm64.rpm macos-x64: name: macOS x64 - runs-on: macos-13 + runs-on: macos-15-intel steps: - uses: actions/checkout@v4 - uses: actions-rs/toolchain@v1 with: - toolchain: 1.85.0 + toolchain: ${{ env.RUST_TOOLCHAIN }} profile: minimal override: true @@ -80,8 +143,6 @@ jobs: - name: Build Darwin x64 run: make darwin-x64 - - name: Run tests - run: make tests - name: Move artifacts to dist shell: bash @@ -94,7 +155,6 @@ jobs: name: kingfisher-darwin-x64.tgz path: dist/kingfisher-darwin-x64.tgz - macos-arm64: name: macOS arm64 runs-on: macos-14 @@ -103,7 +163,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: - toolchain: 1.85.0 + toolchain: ${{ env.RUST_TOOLCHAIN }} profile: minimal override: true @@ -125,34 +185,87 @@ jobs: name: kingfisher-darwin-arm64.tgz path: dist/kingfisher-darwin-arm64.tgz - # ──────────────── Windows ──────────────── windows: name: Windows x64 runs-on: windows-latest + + # Windows-only env to keep vcpkg consistent and enable caching + env: + VCPKG_ROOT: C:\vcpkg + VCPKG_DOWNLOADS: C:\vcpkg\downloads + VCPKG_FEATURE_FLAGS: binarycaching + VCPKG_BINARY_SOURCES: clear;x-gha,readwrite + steps: - uses: actions/checkout@v4 - uses: actions-rs/toolchain@v1 with: - toolchain: 1.85.0 + toolchain: ${{ env.RUST_TOOLCHAIN }} profile: minimal override: true + # Cache vcpkg artifacts & downloads (so we only fetch PCRE once) - name: Cache vcpkg artifacts - uses: actions/cache@v3 + uses: actions/cache@v4 with: - # Adjust these paths if your vcpkg root is somewhere else path: | C:\vcpkg\buildtrees C:\vcpkg\packages C:\vcpkg\installed - key: ${{ runner.os }}-vcpkg-${{ hashFiles('vcpkg.json', 'vcpkg-configuration.cmake') }} + C:\vcpkg\downloads + C:\vcpkg\archives + C:\Users\runneradmin\AppData\Local\vcpkg\archives + key: vcpkg-${{ runner.os }}-hs-542 restore-keys: | - ${{ runner.os }}-vcpkg- + vcpkg-${{ runner.os }}- + vcpkg- + + # Ensure downloads dir exists and seed PCRE 8.45 zip from a working mirror + - name: Pre-seed PCRE 8.45 for vcpkg + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path "$env:VCPKG_DOWNLOADS" | Out-Null + $dst = Join-Path $env:VCPKG_DOWNLOADS "pcre-8.45.zip" + + if (-not (Test-Path $dst)) { + $sf = "https://sourceforge.net/projects/pcre/files/pcre/8.45/pcre-8.45.zip/download" + + # Resolve to the final mirror URL (follow redirects without downloading the whole file) + $handler = New-Object System.Net.Http.HttpClientHandler + $handler.AllowAutoRedirect = $true + $client = New-Object System.Net.Http.HttpClient($handler) + + try { + $req = New-Object System.Net.Http.HttpRequestMessage([System.Net.Http.HttpMethod]::Head, $sf) + $resp = $client.SendAsync($req).GetAwaiter().GetResult() + + # Some mirrors don’t like HEAD; fall back to GET headers only. + if (-not $resp.IsSuccessStatusCode) { + $req.Dispose() + $req = New-Object System.Net.Http.HttpRequestMessage([System.Net.Http.HttpMethod]::Get, $sf) + $resp = $client.SendAsync($req, [System.Net.Http.HttpCompletionOption]::ResponseHeadersRead).GetAwaiter().GetResult() + } + + $finalUrl = $resp.RequestMessage.RequestUri.AbsoluteUri + Write-Host "Resolved SourceForge URL to: $finalUrl" + + # Download the actual file + Invoke-WebRequest -Uri $finalUrl -OutFile $dst + } + finally { + $client.Dispose() + $handler.Dispose() + } + } + + Get-ChildItem $env:VCPKG_DOWNLOADS + + + - uses: swatinem/rust-cache@v2 - - uses: Swatinem/rust-cache@v2 - name: Build - run: .\buildwin.bat -force + run: .\buildwin.bat shell: cmd - name: Run tests @@ -183,11 +296,20 @@ jobs: contents: write steps: - uses: actions/checkout@v4 - - name: Read version from Cargo.toml + - name: Determine tag id: version + shell: bash run: | - VERSION=$(grep -m1 '^version\s*=' Cargo.toml | cut -d '"' -f2) - echo "version=$VERSION" >> "$GITHUB_OUTPUT" + set -euo pipefail + if [[ "${GITHUB_EVENT_NAME}" == "release" ]]; then + TAG="${{ github.event.release.tag_name }}" + elif [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" && -n "${{ github.event.inputs.tag }}" ]]; then + TAG="${{ github.event.inputs.tag }}" + else + VERSION=$(grep -m1 '^version\s*=' Cargo.toml | cut -d '"' -f2) + TAG="v${VERSION}" + fi + echo "tag=${TAG}" >> "$GITHUB_OUTPUT" - uses: actions/download-artifact@v4 with: path: target/release/kingfisher-* @@ -207,8 +329,9 @@ jobs: - name: Create release & upload assets uses: ncipollo/release-action@v1 with: - tag: v${{ steps.version.outputs.version }} - name: "Kingfisher v${{ steps.version.outputs.version }}" - bodyFile: .latest_changelog.md # ← only the most-recent entry + tag: ${{ steps.version.outputs.tag }} + name: "Kingfisher ${{ steps.version.outputs.tag }}" + bodyFile: .latest_changelog.md # ← only the most-recent entry + allowUpdates: true generateReleaseNotes: false - artifacts: target/release/** \ No newline at end of file + artifacts: target/release/** diff --git a/.gitignore b/.gitignore index 5e9dde0..ec3fdbb 100644 --- a/.gitignore +++ b/.gitignore @@ -5,11 +5,19 @@ *.sarif *.profile.json *.json +!webserver/static/sample-report.json +!docs/access-map-viewer/sample-report.json *.jsonl *.bson .prettierrc custom.py logs/* +*.patch +*.orig +*.rej +*.html +!docs/access-map-viewer/index.html +*.dot ### macOS ### # General @@ -20,6 +28,7 @@ logs/* # Icon must end with two \r Icon +node_modules/ # Thumbnails ._* @@ -68,6 +77,7 @@ Cargo.lock !.vscode/launch.json !.vscode/extensions.json !.vscode/*.code-snippets +.vscode/launch.json # Local History for Visual Studio Code .history/ diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml new file mode 100644 index 0000000..91f652c --- /dev/null +++ b/.pre-commit-hooks.yaml @@ -0,0 +1,18 @@ +- id: kingfisher-docker + name: kingfisher (docker) + description: Run Kingfisher in Docker against staged changes at the repository root. No local install required. + entry: ghcr.io/mongodb/kingfisher:latest + language: docker + args: ["scan", ".", "--staged", "--quiet", "--no-update-check"] + pass_filenames: false + stages: [commit] + +- id: kingfisher + name: kingfisher + description: Scan staged changes with the locally installed Kingfisher binary. + entry: kingfisher + language: system + args: ["scan", ".", "--staged", "--quiet", "--no-update-check"] + pass_filenames: false + types: [file] + stages: [commit] \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b14fae..0804eb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,309 @@ All notable changes to this project will be documented in this file. +## [v1.76.0] +- Fixed validation deduplication for rules with nested unnamed captures (e.g. `(?...(ABC|DEF)...)`) to use the primary capture for grouping, ensuring each unique match triggers a separate validation request. +- Added trace-level (`-vv`) logging for internal validation dedup keys and grouping to aid debugging. +- Switched compression dependencies to pure-Rust bzip2/lzma implementations and pared zip features to avoid C-based codecs for bz2/xz handling. + +## [v1.75.0] +- Enhanced Access Map View: added fingerprint display, enabled searching by fingerprint, and implemented bidirectional navigation between Findings and Access Map nodes. +- Added Slack Access Map support with granular permissions in the tree view. +- Improved HTML report +- Improved several rules +- Added new rules for Apollo, Clay, CodeRabbit, Customer.io, Instantly, Vast.ai +- Skipped per-repository report writes when an output file is specified and emit a single aggregated report after multi-repository scans to preserve full output content in files. + +## [v1.74.0] +- Added new rules: cursor, definednetworking, filezilla, harness, intra42, klingai, lark, mergify, naver, plaid, resend, retellai + +## [v1.73.0] +- Will now prefer git history findings when identical secrets appear in both current files and git history (dedup only). +- Fixed report viewer to add support for opening JSONL. +- Add opt-in contributor repository enumeration for GitHub/GitLab `--git-url` scans with `--include-contributors`, plus `--repo-clone-limit` to cap repo cloning. +- Add `--git-clone-dir` to set the parent clone directory and `--keep-clones` to preserve cloned repos after scans. +- Added several new rules. +- Added configurable validation timeout and retry settings for `kingfisher scan`. + +## [v1.72.0] +- Fixed deduplication for dependency-provider rules so dependent validations run per blob +- Updated Artifactory rule entropy and added new artifactory rule +- Aliased "kingfisher self-update" as "kingfisher update" +- Map SARIF result levels from rule confidence +- Added tag selection support to the bash and PowerShell install scripts. + +## [v1.71.0] +- Improved Report Viewer layout +- Improved Salesforce rule + +## [v1.70.0] +- Added `--staged` argument to support new `pre-commit` behavior and added integration coverage to ensure validated secrets block commits when used as pre-commit hook +- Added new rules for AWS Bedrock, Voyage.ai, Posthog, Atlassian +- Added an embedded web-based report and access-map viewer via `kingfisher view` subcommand that can load JSON or JSONL reports passed on the CLI (or upload them in the browser) +- Updated Jira create to gouqi, which supports Jira api v2 and v3 + +## [v1.69.0] +- Reduced per-match memory usage by compacting stored source locations and interning repeated capture names. +- Stored optional validation response bodies as boxed strings to avoid allocating empty payloads and to streamline validator caches. +- Parallelized git cloning based on the configured job count and begin scanning repositories as soon as each clone finishes to reduce end-to-end scan times. +- Combined per-repository results into a single aggregate summary after scans complete. +- Added initial access-map support and report viewer html file. Currently beta features. + +## [v1.68.0] +- Fixed Bitbucket authenticated cloning bug + +## [v1.67.0] +- Added checksum to GitLab rule +- Fixed deduplication to consider rule identifiers so overlapping patterns are not merged before validation +- After scan summaries, emit the styled outdated-version notice to stderr when a newer release is available +- Reduced false positives across a number of rules +- Updated Summary to include scan date, kingfisher version ran, and latest kingfisher version available + +## [v1.66.0] +- Updating to support Bitbucket App Passwords +- Improved boundaries for several rules +- Added more rules + +## [v1.65.0] +- Skip reporting MongoDB and Postgres findings when their connection strings cannot be parsed, even when validation is disabled. +- Improve MySQL detection by broadening URI coverage and adding live validation that skips clearly invalid connection strings. +- Added a helper to truncate validation response bodies only at UTF-8 character boundaries to prevent panics during validation. + +## [v1.64.0] +- Fixed a bug when using --redact, that broke validation +- Added JDBC rule with validator +- Filter out empty 'KF_BITBUCKET_*' environment values when constructing the Bitbucket authentication configuration so blank variables no longer override valid credentials + +## [v1.63.1] +- Updated allocator + +## [v1.63.0] +- Fixed bug when retrieving some finding values and injecting them as TOKENS in the rule templates +- Improved Datadog rule +- Improved AWS rule + +## [v1.62.0] +- Added `pattern_requirements` checks to rules, providing lightweight post-regex character-class validation without lookarounds. See docs/RULES.md for detail +- Added an `ignore_if_contains` option to `pattern_requirements` to drop matches containing case-insensitive placeholder words, with tests covering the new behavior. +- Updated rules to adopt the new `pattern_requirements` support. +- Added checksum comparisons to `pattern_requirements`, new `suffix`, `crc32`, and `base62` Liquid filters, and verbose logging so mismatched checksums are skipped with context rather than reported as findings. +- Split GitHub token detections into fine-grained/fixed-format variants and enforce checksum validation for modern GitHub token families (PAT, OAuth, App, refresh) while preserving legacy coverage. +- Added a rule for Zuplo tokens. +- Added checksum calculation for Confluent, GitHub, and Zuplo tokens, which can drastically reduce false positive reports. +- Improved OpsGenie validation. +- Automatically enable `--no-dedup` when `--manage-baseline` is supplied so baseline management keeps every finding. +- This release is focused on further improving detection accuracy, before even attempting to validate findings. +- Updated GitHub Actions CI for Windows and buildwin.bat script + +## [v1.61.0] +- Fixed local filesystem scans to keep `open_path_as_is` enabled when opening Git repositories and only disable it for diff-based scans. +- Created Linux and Windows specific installer script +- Updated diff-focused scanning so `--branch-root-commit` can be provided alongside `--branch`, letting you diff from a chosen commit while targeting a specific branch tip (still defaulting back to the `--branch` ref when the commit is omitted). +- Updated rules + +## [v1.60.0] +- Removed the `--bitbucket-username`, `--bitbucket-token`, and `--bitbucket-oauth-token` flags in favour of `KF_BITBUCKET_*` environment variables when authenticating to Bitbucket. +- Added provider-specific `kingfisher scan` subcommands (for example `kingfisher scan github …`) that translate into the legacy flags under the hood. The new layout keeps backwards compatibility while removing the wall of provider options from `kingfisher scan --help`. +- Updated the README so every provider example (GitHub, GitLab, Bitbucket, Azure Repos, Gitea, Hugging Face, Slack, Jira, Confluence, S3, GCS, Docker) uses the new subcommand style. +- Legacy provider flags (for example `--github-user`, `--gitlab-group`, `--bitbucket-workspace`, `--s3-bucket`) still work but now emit a deprecation warning to encourage migration to the new `kingfisher scan ` flow. +- Kept the direct `kingfisher scan /path/to/dir` flow for local filesystem / local git repo scans while adding a `--list-only` switch to each provider subcommand so repository enumeration no longer requires the standalone `github repos`, `gitlab repos`, etc. commands. +- Removed the legacy top-level provider commands (`kingfisher github`, `kingfisher gitlab`, `kingfisher gitea`, `kingfisher bitbucket`, `kingfisher azure`, `kingfisher huggingface`) now that enumeration lives under `kingfisher scan --list-only`. + +## [v1.59.0] +- Fixed `kingfisher scan github …` (and other provider-specific subcommands) so they no longer demand placeholder path arguments before the CLI accepts the request. +- Fixed `kingfisher scan` so that providing `--branch` without `--since-commit` now diffs the branch against the empty tree and scans every commit reachable from that branch. +- Added rules for meraki, duffel, finnhub, frameio, freshbooks, gitter, infracost, launchdarkly, lob, maxmind, messagebird, nytimes, prefect, scalingo, sendinblue, sentry, shippo, twitch, typeform + +- ## [v1.58.0] +- Added first-class Hugging Face scanning support, including CLI enumeration, token authentication, and integration with remote scans. +- Condensed GitError formatting to report the exit status and the first informative lines from stdout/stderr, producing concise git clone failure logs. +- Added support for scanning Google Cloud Storage buckets via `--gcs-bucket`, including optional prefixes and service-account authentication. +- Added `--skip-aws-account` (now accepting comma-separated values) and `--skip-aws-account-file` to bypass live AWS validation for known canary/honey-token account IDs without triggering alerts. Kingfisher now ships with several canary AWS account IDs pre-seeded in the skip list and now reports matching findings as "Not Attempted" with the "Response" containing "(skip list entry)" so it's clear that validation was intentionally skipped and why. + +## [v1.57.0] +- Added inline ignore directive detection to treat suppression tokens anywhere on surrounding lines, including multi-line handling +- Added a `--no-ignore` CLI flag to disable inline directives when you need every potential secret reported +- Added: repeatable `--ignore-comment ` flag to reuse inline directives from other scanners (for example `NOSONAR`, `kics-scan ignore`, `gitleaks:allow`, etc) +- Respect user color settings in update messages by using the same color helper as the main reporter, ensuring consistent output and no ANSI codes on update check, when color is disabled + +## [v1.56.0] +- Fixed tree-sitter scanning bug where passing --no-base64 caused errors to be printed when the file type couldn’t be determined + +## [v1.55.0] +- Added first-class Azure Repos support, including CLI commands, enumeration, and documentation updates +- Improved performance of tree-sitter parsing +- Updated Windows build script to ensure static binary is produced + +## [v1.54.0] +- Added first-class Gitea support, including CLI commands, environment-based authentication, documentation, and integration with scans and repository enumeration. +- Populate the finding path from git blob metadata so history-derived secrets display their file location instead of an empty path +- Replaced Match::finding_id’s SHA1-based hashing with a fast xxh3_64 digest that keeps IDs deterministic while eliminating a hot-path SHA1 dependency + +## [v1.53.0] +- Added first-class Bitbucket support, including CLI commands, authentication helpers, documentation, and integration testing. + +## [v1.52.0] +- Enabled ANSI formatting in the tracing formatter whenever stderr is attached to a terminal so colorized updater messages render correctly instead of showing escape sequences. +- Added a new CLI flag, `--user-agent-suffix` to allow developers to append additional information to the user-agent +- Removed the unused --rlimit-nofile flag + +## [1.51.0] +- Added diff-only Git scanning via `--since-commit` and `--branch`, including remote-aware ref resolution so CI jobs can pair `--git-url` clones with pull request branches + +## [1.50.0] +- Added `--github-exclude` and `--gitlab-exclude` options to skip specific repositories when scanning or listing GitHub and GitLab sources, including support for gitignore-style glob patterns + +## [1.49.0] +- Enabled MongoDB URI validation +- AWS + GCP validators now respect HTTPS_PROXY and share a consistent user agent across AWS, GCP, and HTTP validation +- Increase max-file-size default to 256 mb (up from 64 mb) +- Improved AWS rule + +## [1.48.0] +- Improved error message when self-update cannot find the current binary +- Optimized memory usage via string interning and extensive data sharing +- Replaced quadratic match filtering with a per-rule span map, fixing missed secrets in extremely large files and improving scan performance +- Support scanning extremely large files by chunking input into 1 GiB segments with small overlaps, avoiding vectorscan buffer limits while preserving match offsets +- Always use chunked vectorscan, eliminating the slow regex fallback for blobs over 4 GiB +- Skip Base64 scanning for blobs over 64 MB to avoid a second pass over massive files +- Increased max-file-size default to 64 MB (up from 25 MB) + +## [1.47.0] +- MongoDB validator now validates `mongodb+srv://` URIs with a fast timeout instead of skipping them +- Improved rules: github oauth2, diffbot, mailchimp, aws +- Added validation to SauceLabs rule +- Added rules: shodan, bitly, flickr +- Decode Base64 blobs and scan their contents for secrets while skipping short strings for performance. This has a small performance impact and can be disabled with `--no-base64` + +## [1.46.0] +- Improved rules: AWS, pem +- Added rule for Ollama, Weights and Biases, Cerebras, Friendli, Fireworks.ai, NVIDIA NIM, together.ai, zhipu +- Added `self-update` command to update the binary independently. Now supports updating over homebrew managed binary +- MongoDB validator now checks `mongodb+srv://` URIs with fast-fail timeouts + +## [1.45.0] +- Added `--repo-artifacts` flag to scan repository issues, gists/snippets, and wikis when cloning via `--git-url` +- Added rules for sendbird, mattermost, langchain, notion +- JWT validation hardened to reject alg:none by default (only allowed if explicitly configured), require iss for OIDC/JWKS verification, ensuring "Active Credential" means cryptographically verified and time-valid, not just unexpired +- Updated the Git cloning logic to include all refs and minimize clone output, allowing Kingfisher to analyze pull request and deleted branch history + +## [1.44.0] +- Fixed issue with self-update on Linux +- Reverted the change to json and jsonl outputs by rule +- Added `--skip-regex` and `--skip-word` flags to ignore secrets matching custom patterns or skipwords + +## [1.43.0] +- Added rules for clearbit, kickbox, azure container registry, improved Azure Storage key +- Grouped JSON and JSONL outputs by rule, restoring `matches` arrays in reports + +## [1.42.0] +- Fixed pagination issue when calling gitlab api +- Expanded directory exclusion handling to interpret plain patterns as prefixes, ensuring options like --exclude .git also skip all nested paths +- Updated baseline management to track encountered findings and remove entries that are no longer present, saving the baseline file whenever entries are pruned or new matches are added +- Added rules for authress, clickhouse, codecov, contentful, curl, dropbox, fly.io, hubspot, firecrawl +- Internal refactoring of rule loader, git enumerator, and filetype guesser +- Improved language detection + +## [1.41.0] +- Added support for scanning gitlab subgroups, with `kingfisher scan --gitlab-group my-group --gitlab-include-subgroups` +- Added rule for Vercel + +## [1.40.0] +- Dropped the “prevalidated” flag from rule definitions and validation logic so every finding now flows through the standard active/inactive/unknown pipeline, simplifying rule configuration and preventing special‑case bypasses +- Improved Tailscale api key detectors + +## [1.39.0] +- Added support for scanning Confluence pages via `--confluence-url` and `--cql` + +## [1.38.0] +- `--quiet` now suppresses scan summaries and rule statistics unless `--rule-stats` is explicitly provided +- Added X Consumer key detection and validation + +## [1.37.0] +- GitLab: Matched GitLab group repository listings to glab by only enumerating projects that belong directly to each group, without automatically traversing subgroups + +## [1.36.0] +- Fixed GitHub organization and GitLab group scans when using `--git-history=none` +- JWT tokens without both `iss` and `aud` are no longer reported as active credentials + +## [1.35.0] +- Remote scans with `--git-history=none` now clone repositories with a working tree and scan the current files instead of erroring with "No inputs to scan". +- Fixed issue where `--redact` did not function properly +- Fixed validation logic for clarifai rule + +## [1.34.0] +- Use system TLS root certificates to support self-hosted GitLab instances with internal CAs +- Added new rule: Coze personal access token +- Updated Supabase rule to detect project url's and validate their corresponding tokens + +## [1.33.0] +- Fixed header precedence so custom HTTP validation headers like `Accept` are preserved +- Added new Heroku rule + +## [1.32.0] +- Added support for scanning AWS S3 buckets via `--s3-bucket` and optional `--s3-prefix` +- Added `--role-arn` and `--aws-local-profile` flags for S3 authentication alongside `KF_AWS_KEY`/`KF_AWS_SECRET` +- Added progress bar for scanning s3 buckets +- Refactored output reporting and formatting logic + +## [1.31.0] +- New rules: Telegram bot token, OpenWeatherMap, Apify, Groq +- New OpenAI detectors added (@joshlarsen) +- Fixed bug that broke validation when using unnamed group captures + +## [1.30.0] +- Fixed validation caching for HTTP validators to include rendered headers so inactive secrets no longer appear active. +- Removed pre-commit installation hook, due to bugs + +## [1.29.0] +- Fixed issue when more than 1 named capture group is used in a rule variable +- Added a new liquid template filters: `b64dec` +- Added custom validator for Coinbase, and a Coinbase rule that uses it + +## [1.28.0] +- Added support for scanning Slack + +## [1.27.0] +- Added Buildkite rule +- Added support for scanning Docker images via `--docker-image` + +## [1.26.0] +- Added rule for ElevenLabs +- Added support for scanning Jira issues via a given JQL (Jira Query Language) + +## [1.25.0] +- Fixed GitLab authentication bug +- Added pre-commit and pre-receive installation hooks +- MongoDB validator now skips `mongodb+srv://` URIs and returns a message that validation was skipped +- Fixed noisy Baseten rule + +## [1.24.0] +- Now generating DEB and RPM packages +- Now releasing Docker images, and updated README +- Added rule for Scale, Deepgram, AssemblyAI + + +## [1.23.0] +- Updating GitHub Action to generate Docker image +- Added rules for Diffbot, ai21, baseten +- Fixed supabase rule +- Added 'alg' to JWT validation output + +## [1.22.0] +- Added rules for Google Gemini AI, Cohere, Stability.ai, Replicate, Runway, Clarifai +- Upgraded dependencies + +## [1.21.0] +- Improved Azure Storage rule +- Added rule to detect TravisCI encrypted values +- Added baseline feature with `--baseline-file` and `--manage-baseline` flags +- Introduced `--exclude` option for skipping paths +- Added tests covering baseline and exclude workflow +- Added validation for JWT tokens that checks `exp` and `nbf` claims +- JWT validation performs OpenID Connect discovery using the `iss` claim and verifies signatures via JWKS +- Removed `--ignore-tests` argument, because the `--exclude` flag provides more granular functionality +- DigitalOcean rule update +- Adafruit rule update ## [1.20.0] - Removed confirmation prompt when user provides --self-update flag diff --git a/Cargo.toml b/Cargo.toml index 19c6f95..896bfe5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace.package] edition = "2021" -rust-version = "1.83" +rust-version = "1.90" license = "Apache-2.0" authors = ["Mick Grove "] homepage = "https://github.com/mongodb/kingfisher" @@ -10,7 +10,8 @@ publish = false [package] name = "kingfisher" -version = "1.20.0" +version = "1.76.0" +description = "MongoDB's blazingly fast and accurate secret scanning and validation tool" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -19,18 +20,38 @@ homepage.workspace = true repository.workspace = true publish.workspace = true +[package.metadata.deb] +name = "kingfisher" +maintainer = "Mick Grove " +depends = "$auto" +section = "utils" +priority = "optional" +assets = [ + ["target/release/kingfisher", "/usr/bin/kingfisher", "755"] +] + +[package.metadata.generate-rpm] +package = "kingfisher" +summary = "MongoDB's blazingly fast and accurate secret scanning and validation tool" +license = "Apache-2.0" +url = "https://github.com/mongodb/kingfisher" +assets = [ + { source = "target/release/kingfisher", dest = "/usr/bin/kingfisher", mode = "755" } +] + [dependencies] -clap = { version = "4.3", features = [ +clap = { version = "4.5", features = [ "cargo", "derive", "env", "unicode", "wrap_help", ] } + anyhow = "1.0" -bstr = { version = "1.0", features = ["serde"] } +bstr = { version = "1.12", features = ["serde"] } fixedbitset = "0.5" -gix = { version = "0.72", features = ["max-performance", "serde", "blocking-network-client"] } +gix = { version = "0.73", features = ["max-performance", "serde", "blocking-network-client"] } ignore = "0.4" petgraph = "0.6" roaring = "0.10" @@ -41,132 +62,162 @@ smallvec = { version = "1", features = [ "const_new", "union", ] } -tracing = "0.1.41" +tracing = "0.1.43" indicatif = { version = "0.17", features = ["improved_unicode"] } -rayon = "1.10" -sha1 = "0.10.6" +rayon = "1.11" hex = "0.4.3" vectorscan-rs = "0.0.5" -regex = "1.10.6" -serde_json = "1.0.128" +regex = "1.12.2" +serde_json = "1.0.145" lazy_static = "1.5.0" -url = "2.5.2" +url = "2.5.7" include_dir = { version = "0.7", features = ["glob"] } strum = { version = "0.26", features = ["derive"] } -sysinfo = "0.31.2" +sysinfo = "0.31.4" +webbrowser = "1.0.5" reqwest = { version = "0.12", default-features = false, features = [ - "json", - "gzip", - "brotli", + "json", + "gzip", + "brotli", "deflate", "stream", "rustls-tls", + "rustls-tls-native-roots", "blocking", "multipart", - "rustls-tls", ] } +axum = { version = "0.7", default-features = false, features = ["tokio", "http1"] } -chrono = "0.4.38" -thiserror = "1.0.63" -tokio = { version = "1.39.2", features = ["full"] } +chrono = "0.4.42" +thiserror = "1.0.69" +tokio = { version = "1.48.0", features = ["full"] } base64 = "0.22.1" -crossbeam-channel = "0.5.13" -indenter = "0.3.3" +crossbeam-channel = "0.5.15" +indenter = "0.3.4" serde-sarif = "0.4" -console = "0.15.8" -time = "0.3.36" -tempfile = "3.12.0" -num_cpus = "1.16.0" -once_cell = "1.19.0" -http = "1.1.0" -liquid = "0.26.4" -liquid-core = "0.26.4" -flate2 = "1.0.33" -brotli = "6.0.0" +console = "0.15.11" +time = "0.3.44" +tempfile = "3.23.0" +num_cpus = "1.17.0" +once_cell = "1.21.3" +http = "1.4.0" +liquid = "0.26.11" +liquid-core = "0.26.11" +flate2 = "1.1" thousands = "0.2.0" base32 = "0.5.1" crossbeam-skiplist = "0.1.3" tokio-postgres = { version = "0.7", default-features = false, features = ["runtime"] } -mongodb = { version = "3.2", default-features = false, features = ["rustls-tls", "aws-auth", "compat-3-0-0", "dns-resolver"] } -bson = "2.13.0" -ring = "0.17.8" -pem = "3.0.4" -aws-config = "1.5.10" -aws-credential-types = "1.2.1" -aws-sdk-sts = "1.21.0" -aws-types = "1.3.3" +mongodb = { version = "3.4", default-features = false, features = ["rustls-tls", "aws-auth", "compat-3-0-0", "dns-resolver"] } +mysql_async = { version = "0.34.2", default-features = false, features = ["default-rustls"] } +bson = "2.15.0" +ring = "0.17.14" +pem = "3.0.6" +aws-config = "1.8.12" +aws-credential-types = "1.2.11" +aws-sdk-sts = "1.95.0" +aws-types = "1.3.11" byteorder = "1.5.0" -parking_lot = "0.12.3" +parking_lot = "0.12.5" octorust = "0.9.0" -reqwest-middleware = "0.4.1" -tracing-subscriber = {version = "0.3.19", features = ["env-filter"] } -tracing-core = "0.1.33" -tree-sitter = "0.24.4" -tree-sitter-bash = "0.23.3" -tree-sitter-c = "0.23.2" +reqwest-middleware = "0.4.2" +tracing-subscriber = {version = "0.3.22", features = ["env-filter"] } +tracing-core = "0.1.35" +tree-sitter = "0.25.10" +aws-smithy-http-client = "1.1.5" +aws-smithy-runtime-api = "1.9.3" +aws-smithy-types = "1.3.5" +tree-sitter-bash = "0.25.1" +tree-sitter-c = "0.24.1" tree-sitter-c-sharp = "0.23.1" tree-sitter-cpp = "0.23.4" -tree-sitter-css = "0.23.1" +tree-sitter-css = "0.23.2" tree-sitter-go = "0.23.4" tree-sitter-html = "0.23.2" -tree-sitter-java = "0.23.4" +tree-sitter-java = "0.23.5" tree-sitter-javascript = "0.23.1" tree-sitter-php = "0.23.11" -tree-sitter-python = "0.23.4" +tree-sitter-python = "0.23.6" tree-sitter-ruby = "0.23.1" -tree-sitter-rust = "0.23.2" +tree-sitter-rust = "0.24.0" tree-sitter-toml-ng = "0.7.0" tree-sitter-typescript = "0.23.2" -tree-sitter-yaml = "0.6.1" +tree-sitter-yaml = "0.7.2" streaming-iterator = "0.1.9" tree-sitter-regex = "0.24.3" +tree_magic_mini = "3.2" content_inspector = "0.2.4" -rustc-hash = "2.1.0" +rustc-hash = "2.1.1" term_size = "0.3.2" -bzip2 = "0.5.0" -zip = "2.2.2" -tar = "0.4.43" -xz2 = "0.1.7" +bzip2-rs = "0.1.2" +zip = { version = "2.4.2", default-features = false, features = ["deflate", "deflate64", "time"] } +tar = "0.4.44" +lzma-rs = "0.3.0" asar = "0.3.0" -blake3 = "1.5.5" -memmap2 = "0.9.5" +blake3 = "1.8.2" +memchr = "2.7" +memmap2 = "0.9.9" futures = "0.3.31" dashmap = "6.1.0" xxhash-rust = { version = "0.8.15", features = ["xxh3", "const_xxh3"] } serde_yaml = "0.9.34" hmac = "0.12.1" -sha2 = "0.10.8" -strum_macros = "0.27.1" -humantime = "2.2.0" +sha2 = "0.10.9" +strum_macros = "0.27.2" +humantime = "2.3.0" path-dedot = "3.1.1" -quick-xml = {version = "0.37.5", features = ["serde","serialize"] } -rustls = "0.23.26" +quick-xml = {version = "0.38.4", features = ["serde","serialize"] } +rustls = "0.23.35" tokio-postgres-rustls = "0.13.0" -rustls-native-certs = "0.8.1" +rustls-native-certs = "0.8.2" predicates = "3.1.3" -assert_cmd = "2.0.17" -proptest = "1.6.0" -color-backtrace = "0.7.0" -gitlab = "0.1711.0" -mimalloc = {version = "0.1.46", features = ["override"]} -thread_local = "1.1.8" -crc32fast = "1.4.2" +assert_cmd = "2.1.1" +proptest = "1.9.0" +color-backtrace = "0.7.2" +gitlab = "0.1801.0" +mimalloc = {version = "0.1.48", features = ["override"]} +thread_local = "1.1.9" bloomfilter = "3.0.1" -uuid = "1.17.0" -urlencoding = "2.1.3" -rand = "0.9.1" -percent-encoding = "2.3.1" -trust-dns-resolver = { version = "0.23.2", default-features = false, features = ["tokio-runtime"] } +uuid = "1.19.0" +rand = "0.9.2" +percent-encoding = "2.3.2" atty = "0.2.14" self_update = { version = "0.42.0", default-features = false, features = ["rustls", "archive-tar", "archive-zip", "compression-flate2"] } -semver = "1.0.26" +semver = "1.0.27" +globset = "0.4.18" +jsonwebtoken = { version = "10.2.0", features = ["aws-lc-rs"] } +ipnet = "2.11.0" +gouqi = { version = "0.20.0", features = ["async"] } +oci-client = { version = "0.15", default-features = false, features = ["rustls-tls"] } +walkdir = "2.5.0" +p256 = "0.13.2" +ed25519-dalek = { version = "2.2", features = ["pkcs8"] } +aws-sdk-s3 = "1.117.0" +aws-sdk-iam = "1.101.0" +aws-sdk-ec2 = "1.196.0" +aws-sdk-dynamodb = "1.101.0" +aws-sdk-lambda = "1.111.0" +aws-sdk-kms = "1.97.0" +aws-sdk-secretsmanager = "1.96.0" +gcloud-storage = { version = "1.1.1", default-features = false, features = [ + "rustls-tls", + "auth", + "jwt-aws-lc-rs", +] } +tokei = "12.1.2" +crc32fast = "1.5.0" + +[target.'cfg(not(windows))'.dependencies] +sha1 = { version = "0.10.6", features = ["asm"] } + +[target.'cfg(windows)'.dependencies] +sha1 = "0.10.6" [dependencies.tikv-jemallocator] version = "0.6" optional = true - [features] default = ["use-mimalloc"] use-mimalloc = ["mimalloc/override"] @@ -174,15 +225,16 @@ use-jemalloc = ["tikv-jemallocator"] system-alloc = [] # forces System allocator [dev-dependencies] -pretty_assertions = "1.3" +pretty_assertions = "1.4" temp-env = "0.3.6" -wiremock = "0.6.2" -git2 = "0.20.2" +wiremock = "0.6.5" +git2 = "0.20.3" rand_chacha = "0.9.0" +testcontainers = "0.15.0" [profile.release] debug = false -strip = "debuginfo" +strip = true #"debuginfo" opt-level = 3 # Maximum optimization for performance lto = true # Enable Link Time Optimization codegen-units = 1 # Optimize for size but slower compilation diff --git a/Makefile b/Makefile index da375d4..b41b470 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,7 @@ SHELL := /usr/bin/env bash .SHELLFLAGS := -eu -o pipefail -c -# Detect project name from Cargo.toml -PROJECT_NAME := $(shell grep '^name' Cargo.toml | cut -d '"' -f 2) +PROJECT_NAME := kingfisher # Determine OS and whether to use gtar on darwin OS := $(shell uname) @@ -25,6 +24,7 @@ endif ifeq ($(OS),darwin) export HOMEBREW_NO_INSTALL_CLEANUP=1 export HOMEBREW_NO_ENV_HINTS=1 + export HOMEBREW_NO_AUTO_UPDATE=1 endif # detect host architecture and map to our target suffixes @@ -110,11 +110,11 @@ setup-zig: ubuntu-x64: setup-zig # ensures Zig & cargo-zigbuild exist @echo "Checking Rust toolchain…" @$(MAKE) check-rust || { \ - echo "🦀 Installing Rust 1.85.0 …"; \ + echo "🦀 Installing Rust 1.90.0 …"; \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y; \ . $$HOME/.cargo/env; \ - rustup toolchain install 1.85.0; \ - rustup default 1.85.0; \ + rustup toolchain install 1.90.0; \ + rustup default 1.90.0; \ } @echo "📦 Installing build dependencies (musl, cmake, etc.)…" @@ -150,11 +150,11 @@ ubuntu-x64: setup-zig # ensures Zig & cargo-zigbuild exist ubuntu-arm64: setup-zig # ensures Zig & cargo-zigbuild exist @echo "Checking Rust toolchain…" @$(MAKE) check-rust || { \ - echo "🦀 Installing Rust 1.85.0 …"; \ + echo "🦀 Installing Rust 1.90.0 …"; \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y; \ . $$HOME/.cargo/env; \ - rustup toolchain install 1.85.0; \ - rustup default 1.85.0; \ + rustup toolchain install 1.90.0; \ + rustup default 1.90.0; \ } @echo "📦 Installing build dependencies (musl, cmake, etc.)…" @@ -183,14 +183,15 @@ ubuntu-arm64: setup-zig # ensures Zig & cargo-zigbuild exist $(MAKE) list-archives - darwin-arm64: @echo "Checking Rust for darwin-arm64..." @$(MAKE) check-rust || ( \ echo "Rust not found or out-of-date. Installing via Homebrew..." && \ brew install rust \ ) - @brew install boost cmake gcc libpcap pkg-config ragel sqlite coreutils gnu-tar || true + @brew list cmake >/dev/null 2>&1 || brew install cmake + @brew list boost >/dev/null 2>&1 || brew install boost + @brew install gcc libpcap pkg-config ragel sqlite coreutils gnu-tar @rustup target add aarch64-apple-darwin cargo build --release --target aarch64-apple-darwin --features system-alloc @cd target/aarch64-apple-darwin/release && \ @@ -206,13 +207,18 @@ darwin-arm64: fi $(MAKE) list-archives +darwin-dev: + cargo build --profile=dev --target aarch64-apple-darwin --features system-alloc + darwin-x64: @echo "Checking Rust for darwin-x64..." @$(MAKE) check-rust || ( \ echo "Rust not found or out-of-date. Installing via Homebrew..." && \ brew install rust \ ) - @brew install boost cmake gcc libpcap pkg-config ragel sqlite coreutils gnu-tar || true + @brew list cmake >/dev/null 2>&1 || brew install cmake + @brew list boost >/dev/null 2>&1 || brew install boost + @brew install gcc libpcap pkg-config ragel sqlite coreutils gnu-tar @rustup target add x86_64-apple-darwin source $$HOME/.cargo/env && cargo build --release --target x86_64-apple-darwin --features system-alloc @cd target/x86_64-apple-darwin/release && \ @@ -242,7 +248,7 @@ endif linux-x64: check-docker create-dockerignore @mkdir -p target/release docker run --platform linux/amd64 --rm \ - -v "$$(pwd):/src" -w /src rust:1.85-alpine sh -eu -c '\ + -v "$$(pwd):/src" -w /src rust:1.90-alpine sh -eu -c '\ apk add --no-cache \ musl-dev \ gcc g++ make cmake pkgconfig \ @@ -253,7 +259,7 @@ linux-x64: check-docker create-dockerignore patch perl ragel && \ git openssl-dev curl && \ \ - cargo test --workspace --all-targets --release ; \ + cargo test --workspace --all-targets ; \ \ rustup target add x86_64-unknown-linux-musl && \ \ @@ -262,24 +268,16 @@ linux-x64: check-docker create-dockerignore \ cargo build --release --target x86_64-unknown-linux-musl && \ cd target/x86_64-unknown-linux-musl/release && \ - find "./$(PROJECT_NAME)" -type f -executable \ - -not -name "*.d" -not -name "*.rlib" \ - -exec sha256sum {} \; > CHECKSUM.txt \ + sha256sum kingfisher > CHECKSUM.txt && \ + tar -czf /src/target/release/kingfisher-linux-x64.tgz \ + kingfisher CHECKSUM.txt \ ' - @cd target/release && \ - rm -rf $(PROJECT_NAME)-linux-x64.tgz && \ - cp ../x86_64-unknown-linux-musl/release/$(PROJECT_NAME) . && \ - cp ../x86_64-unknown-linux-musl/release/CHECKSUM.txt CHECKSUM-linux-x64.txt && \ - tar --no-xattrs -czf $(PROJECT_NAME)-linux-x64.tgz \ - $(PROJECT_NAME) CHECKSUM-linux-x64.txt && \ - rm $(PROJECT_NAME) && \ - sha256sum $(PROJECT_NAME)-linux-x64.tgz >> CHECKSUM-linux-x64.txt $(MAKE) list-archives linux-arm64: check-docker create-dockerignore @mkdir -p target/release docker run --platform linux/arm64 --rm \ - -v "$$(pwd):/src" -w /src rust:1.85-alpine sh -eu -c '\ + -v "$$(pwd):/src" -w /src rust:1.90-alpine sh -eu -c '\ apk add --no-cache \ musl-dev \ gcc g++ make cmake pkgconfig \ @@ -292,7 +290,7 @@ linux-arm64: check-docker create-dockerignore \ rustup target add aarch64-unknown-linux-musl && \ \ - cargo test --workspace --all-targets --release ; \ + cargo test --workspace --all-targets ; \ \ export PKG_CONFIG_ALLOW_CROSS=1 ; \ export RUSTFLAGS="-C target-feature=+crt-static" ; \ @@ -300,18 +298,10 @@ linux-arm64: check-docker create-dockerignore cargo build --release --target aarch64-unknown-linux-musl && \ \ cd target/aarch64-unknown-linux-musl/release && \ - find "./$(PROJECT_NAME)" -type f -executable \ - -not -name "*.d" -not -name "*.rlib" \ - -exec sha256sum {} \; > CHECKSUM.txt \ + sha256sum kingfisher > CHECKSUM.txt && \ + tar -czf /src/target/release/kingfisher-linux-arm64.tgz \ + kingfisher CHECKSUM.txt \ ' - @cd target/release && \ - rm -rf $(PROJECT_NAME)-linux-arm64.tgz && \ - cp ../aarch64-unknown-linux-musl/release/$(PROJECT_NAME) . && \ - cp ../aarch64-unknown-linux-musl/release/CHECKSUM.txt CHECKSUM-linux-arm64.txt && \ - tar --no-xattrs -czf $(PROJECT_NAME)-linux-arm64.tgz \ - $(PROJECT_NAME) CHECKSUM-linux-arm64.txt && \ - rm $(PROJECT_NAME) && \ - sha256sum $(PROJECT_NAME)-linux-arm64.tgz >> CHECKSUM-linux-arm64.txt $(MAKE) list-archives @@ -366,6 +356,13 @@ all: linux darwin @echo -e "\nCombined Checksums:" @cat target/release/CHECKSUMS.txt +dockerfile: +# Build for the host architecture (default) + docker build -f docker/Dockerfile -t kingfisher:latest . + +# Cross‑build for arm64 from an x64 machine + docker buildx build -f docker/Dockerfile --platform linux/arm64 -t kingfisher:arm64 . + list-archives: @echo -e "\n=== Built archives ===" @found=0; \ @@ -391,7 +388,7 @@ check-rust: echo "Rust not found."; \ exit 1; \ fi; \ - required=1.85.0; \ + required=1.90.0; \ if [ $$(printf '%s\n' "$$required" "$$version" | sort -V | head -n1) != "$$required" ]; then \ echo "Rust version $$version is older than required $$required."; \ exit 1; \ @@ -424,6 +421,3 @@ notices: @echo "Generating third-party notices..." @cargo install cargo-bundle-licenses @cargo bundle-licenses --format yaml --output THIRD_PARTY_NOTICES - -evergreen-patch: - @evergreen patch --project kingfisher --variants all --tasks build \ No newline at end of file diff --git a/NOTICE b/NOTICE index 62c3bf2..6dd6a63 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,14 @@ NOTICE file corresponding to Section 4 (d) of the Apache License, Version 2.0 +-------------------------------------------------------------------- +Notices for Kingfisher +-------------------------------------------------------------------- +Copyright 2025 MongoDB, Inc. +https://www.mongodb.com + +Source repository: https://github.com/mongodb/kingfisher + + -------------------------------------------------------------------- Upstream notices -------------------------------------------------------------------- @@ -21,11 +30,3 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - --------------------------------------------------------------------- -Additional notices for Kingfisher --------------------------------------------------------------------- -Copyright 2025 MongoDB, Inc. -https://www.mongodb.com - -Source repository: https://github.com/mongodb/kingfisher diff --git a/README.md b/README.md index 40bbce6..f92e38a 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,410 @@ -# Kingfisher + # Kingfisher

Kingfisher Logo -[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+[![ghcr downloads](https://ghcr-badge.elias.eu.org/shield/mongodb/kingfisher/kingfisher)](https://github.com/mongodb/kingfisher/pkgs/container/kingfisher)
-Kingfisher is a blazingly fast secret‑scanning and validation tool built in Rust. It combines Intel’s hardware‑accelerated Hyperscan regex engine with language‑aware parsing via Tree‑Sitter, and **ships with hundreds of built‑in rules** to detect, validate, and triage secrets before they ever reach production + +Kingfisher is a blazingly fast secret-scanning and **live validation** tool built in Rust. + +It combines Intel’s SIMD-accelerated regex engine (Hyperscan) with language-aware parsing to achieve high accuracy at massive scale, and **ships with hundreds of built-in rules** to detect, **validate**, and triage secrets before they ever reach production. + +Designed for offensive security engineers and blue-teamers alike, Kingfisher helps you pivot across repo ecosystems, validate exposure paths, and hunt for developer-owned leaks that spill beyond the primary codebase.

-Kingfisher originated as a fork of [Nosey Parker](https://github.com/praetorian-inc/noseyparker) by Praetorian Security, Inc, and is built atop their incredible work and the work contributed by the Nosey Parker community. - -Kingfisher extends Nosey Parker with live secret validation via cloud-provider APIs, augments regex detection with tree-sitter for code parsing, adds GitLab support, and builds a Windows x64 binary. - -**MongoDB Blog**: [Introducing Kingfisher: Real-Time Secret Detection and Validation](https://www.mongodb.com/blog/post/product-release-announcements/introducing-kingfisher-real-time-secret-detection-validation) - ## Key Features -- **Performance**: Multi‑threaded, Hyperscan‑powered scanning for massive codebases -- **Language‑Aware Accuracy**: AST parsing in 20+ languages via Tree‑Sitter reduces contextless regex matches. see [docs/PARSING.md](/docs/PARSING.md) -- **Built-In Validation**: Hundreds of built-in detection rules, many with live-credential validators that call the relevant service APIs (AWS, Azure, GCP, Stripe, etc.) to confirm a secret is active. You can extend or override the library by adding YAML-defined rules on the command line—see [docs/RULES.md](/docs/RULES.md) for details -- **Git History Scanning**: Scan local repos, remote GitHub/GitLab orgs/users, or arbitrary GitHub/GitLab repos +### Multiple Scan Targets +
-## Getting Started +| Files / Dirs | Local Git | GitHub | GitLab | Azure Repos | Bitbucket | Gitea | Hugging Face | +|:-------------:|:----------:|:------:|:------:|:-------------:|:----------:|:------:|:-------------:| +| Files / Dirs
Files / Dirs | Local Git
Local Git | GitHub
GitHub | GitLab
GitLab | Azure Repos
Azure Repos | Bitbucket
Bitbucket | Gitea
Gitea |Hugging Face
Hugging Face | -### Installation +| Docker | Jira | Confluence | Slack | AWS S3 | Google Cloud | +|:------:|:----:|:-----------:|:-----:|:------:|:---:| +| Docker
Docker | Jira
Jira | Confluence
Confluence | Slack
Slack | AWS S3
AWS S3 | Google Cloud Storage
Cloud Storage | -On macOS, you can simply +
+ +### Performance, Accuracy, and Hundreds of Rules +- **Performance**: multithreaded, Hyperscan‑powered scanning built for huge codebases +- **Extensible rules**: hundreds of built-in detectors plus YAML-defined custom rules ([docs/RULES.md](/docs/RULES.md)) +- **Blast Radius Mapping**: instantly map leaked keys to their effective cloud identities and exposed resources with `--access-map`. Supports AWS, GCP, Azure, GitHub, Gitlab, and more token support coming. +- **Broad AI SaaS coverage**: finds and validates tokens for OpenAI, Anthropic, Google Gemini, Cohere, AWS Bedrock, Voyage AI, Mistral, Stability AI, Replicate, xAI (Grok), Ollama, Langchain, Perplexity, Weights & Biases, Cerebras, Friendli, Fireworks.ai, NVIDIA NIM, Together.ai, Zhipu, and many more +- **Compressed Files**: Supports extracting and scanning compressed files for secrets +- **Baseline management**: generate and track baselines to suppress known secrets ([docs/BASELINE.md](/docs/BASELINE.md)) +- **Checksum-aware detection**: verifies tokens with built-in checksums (e.g., GitHub, Confluent, Zuplo) — no API calls required +- **Built-in Report Viewer**: Visualize and triage findings locally with `kingisher view ./report-file.json` + +**Learn more:** [Introducing Kingfisher: Real‑Time Secret Detection and Validation](https://www.mongodb.com/blog/post/product-release-announcements/introducing-kingfisher-real-time-secret-detection-validation) + +# Benchmark Results + +See ([docs/COMPARISON.md](docs/COMPARISON.md)) + +

+ Kingfisher Runtime Comparison +

+ +## Basic Usage Demo +```bash +kingfisher scan /path/to/scan --view-report +``` +NOTE: Replay has been slowed down for demo +![alt text](docs/kingfisher-usage-01.gif) + +## Report Viewer Demo +Explore Kingfisher’s built-in report viewer and its `--access-map`, which can show what the token (AWS, GCP, Azure, GitHub, and GitLab...more coming) can actually access : [Access map outputs and viewer](#access-map-outputs-and-viewer) + +Note: when you pass `--view-report`, Kingfisher starts a **localhost-only** web server on port `7890` and opens it in your default browser. You’ll see this near the end of the scan output, and **Kingfisher will keep running** until you stop it. + +```bash +INFO kingfisher::cli::commands::view: Starting access-map viewer address=127.0.0.1:7890 +Serving access-map viewer at http://127.0.0.1:7890 (Ctrl+C to stop) +``` + +**Usage:** +```bash +kingfisher scan /path/to/scan --access-map --view-report +``` + +![alt text](docs/kingfisher-usage-access-map-01.gif) + +**Click to view video** +[![Demo](docs/demos/findings-thumbnail.png)](https://github.com/user-attachments/assets/d33ee7a6-c60a-4e42-88e0-ac03cb429a46) + + +# Table of Contents + +
+ +- [Kingfisher](#kingfisher) + - [Key Features](#key-features) + - [Multiple Scan Targets](#multiple-scan-targets) + - [Performance, Accuracy, and Hundreds of Rules](#performance-accuracy-and-hundreds-of-rules) +- [Benchmark Results](#benchmark-results) + - [Basic Usage Demo](#basic-usage-demo) + - [Report Viewer Demo](#report-viewer-demo) +- [Table of Contents](#table-of-contents) +- [Getting Started](#getting-started) + - [Installation](#installation) + - [Pre-built Releases](#pre-built-releases) + - [Homebrew](#homebrew) + - [Linux and macOS](#linux-and-macos) + - [Windows](#windows) + - [Pre-commit hooks](#pre-commit-hooks) + - [macOS and Linux](#macos-and-linux) + - [Windows PowerShell](#windows-powershell) + - [Using the `pre-commit` framework](#using-the-pre-commit-framework) + - [Compile](#compile) + - [ Run Kingfisher in Docker](#-run-kingfisher-in-docker) +- [🔐 Detection Rules at a Glance](#-detection-rules-at-a-glance) + - [📝 Write Custom Rules!](#-write-custom-rules) + - [Pattern requirements and placeholder filtering](#pattern-requirements-and-placeholder-filtering) + - [🔍 Checksum Intelligence](#-checksum-intelligence) +- [🎉 Usage](#-usage) + - [Basic Examples](#basic-examples) + - [Scan with secret validation](#scan-with-secret-validation) + - [Scan a directory containing multiple Git repositories](#scan-a-directory-containing-multiple-git-repositories) + - [Scan a Git repository without validation](#scan-a-git-repository-without-validation) + - [Display only secrets confirmed active by third‑party APIs](#display-only-secrets-confirmed-active-by-thirdparty-apis) + - [Output JSON and capture to a file](#output-json-and-capture-to-a-file) + - [Output SARIF directly to disk](#output-sarif-directly-to-disk) + - [Access map outputs and viewer](#access-map-outputs-and-viewer) + - [View access-map reports locally](#view-access-map-reports-locally) + - [Pipe any text directly into Kingfisher by passing `-`](#pipe-any-text-directly-into-kingfisher-by-passing--) + - [Limit maximum file size scanned (`--max-file-size`)](#limit-maximum-file-size-scanned---max-file-size) + - [Scan using a rule _family_ with one flag](#scan-using-a-rule-family-with-one-flag) + - [Display rule performance statistics](#display-rule-performance-statistics) + - [Scan while ignoring likely test files](#scan-while-ignoring-likely-test-files) + - [Exclude specific paths](#exclude-specific-paths) + - [Scan changes in CI pipelines](#scan-changes-in-ci-pipelines) + - [ Scanning an AWS S3 Bucket](#-scanning-an-aws-s3-bucket) + - [ Scanning a Google Cloud Storage Bucket](#-scanning-a-google-cloud-storage-bucket) + - [ Scanning Docker Images](#-scanning-docker-images) + - [ Scanning GitHub](#-scanning-github) + - [Scan GitHub organization (requires `KF_GITHUB_TOKEN`)](#scan-github-organization-requires-kf_github_token) + - [Skip specific GitHub repositories during enumeration](#skip-specific-github-repositories-during-enumeration) + - [Scan remote GitHub repository](#scan-remote-github-repository) + - [ Scanning GitLab](#-scanning-gitlab) + - [Scan GitLab group (requires `KF_GITLAB_TOKEN`)](#scan-gitlab-group-requires-kf_gitlab_token) + - [Scan GitLab user](#scan-gitlab-user) + - [Skip specific GitLab projects during enumeration](#skip-specific-gitlab-projects-during-enumeration) + - [Scan remote GitLab repository by URL](#scan-remote-gitlab-repository-by-url) + - [List GitLab repositories](#list-gitlab-repositories) + - [ Scanning Azure Repos](#-scanning-azure-repos) + - [Scan Azure Repos organization or collection (requires `KF_AZURE_TOKEN` or `KF_AZURE_PAT`)](#scan-azure-repos-organization-or-collection-requires-kf_azure_token-or-kf_azure_pat) + - [Scan specific Azure Repos projects](#scan-specific-azure-repos-projects) + - [Skip specific Azure repositories during enumeration](#skip-specific-azure-repositories-during-enumeration) + - [List Azure repositories](#list-azure-repositories) + - [ Scanning Gitea](#-scanning-gitea) + - [Scan Gitea organization (requires `KF_GITEA_TOKEN`)](#scan-gitea-organization-requires-kf_gitea_token) + - [Scan Gitea user](#scan-gitea-user) + - [Skip specific Gitea repositories during enumeration](#skip-specific-gitea-repositories-during-enumeration) + - [Scan remote Gitea repository by URL](#scan-remote-gitea-repository-by-url) + - [List Gitea repositories](#list-gitea-repositories) + - [ Scanning Bitbucket](#-scanning-bitbucket) + - [Scan Bitbucket workspace](#scan-bitbucket-workspace) + - [Scan Bitbucket user](#scan-bitbucket-user) + - [Skip specific Bitbucket repositories during enumeration](#skip-specific-bitbucket-repositories-during-enumeration) + - [Scan remote Bitbucket repository by URL](#scan-remote-bitbucket-repository-by-url) + - [List Bitbucket repositories](#list-bitbucket-repositories) + - [Authenticate to Bitbucket](#authenticate-to-bitbucket) + - [Self-hosted Bitbucket Server](#self-hosted-bitbucket-server) + - [ Scanning Hugging Face](#-scanning-hugging-face) + - [Scan Hugging Face user](#scan-hugging-face-user) + - [Scan Hugging Face organization](#scan-hugging-face-organization) + - [Scan specific Hugging Face resources](#scan-specific-hugging-face-resources) + - [List Hugging Face repositories](#list-hugging-face-repositories) + - [Authenticate to Hugging Face](#authenticate-to-hugging-face) + - [ Scanning Jira](#-scanning-jira) + - [Scan Jira issues matching a JQL query](#scan-jira-issues-matching-a-jql-query) + - [Scan the last 1,000 Jira issues:](#scan-the-last-1000-jira-issues) + - [ Scanning Confluence](#-scanning-confluence) + - [Scan Confluence pages matching a CQL query](#scan-confluence-pages-matching-a-cql-query) + - [ Scanning Slack](#-scanning-slack) + - [Scan Slack messages matching a search query](#scan-slack-messages-matching-a-search-query) + - [Environment Variables for Tokens](#environment-variables-for-tokens) + - [Exit Codes](#exit-codes) + - [Update Checks](#update-checks) +- [🤓 Advanced Options](#-advanced-options) + - [Build a Baseline / Detect New Secrets](#build-a-baseline--detect-new-secrets) + - [List Builtin Rules](#list-builtin-rules) + - [To scan using **only** your own `my_rules.yaml` you could run:](#to-scan-using-only-your-own-my_rulesyaml-you-could-run) + - [To add your rules alongside the built‑ins:](#to-add-your-rules-alongside-the-builtins) + - [Other Examples](#other-examples) + - [Customize the HTTP User-Agent](#customize-the-http-user-agent) + - [Validation tuning flags](#validation-tuning-flags) + - [Notable Scan Options](#notable-scan-options) + - [Understanding `--confidence`](#understanding---confidence) + - [Ignore known false positives](#ignore-known-false-positives) + - [Skip Canary Tokens (AWS)](#skip-canary-tokens-aws) + - [Common CLI flows](#common-cli-flows) + - [Inline ignore directives](#inline-ignore-directives) + - [Finding Fingerprint](#finding-fingerprint) + - [Rule Performance Profiling](#rule-performance-profiling) + - [CLI Options](#cli-options) + - [Lineage and Evolution](#lineage-and-evolution) +- [Roadmap](#roadmap) +- [License](#license) + +
+ + +# Getting Started +## Installation +### Pre-built Releases +Pre-built binaries are available from the [Releases](https://github.com/mongodb/kingfisher/releases) section. + +### Homebrew +![Homebrew Formula Version](https://img.shields.io/homebrew/v/kingfisher) ```bash brew install kingfisher ``` -Pre-built binaries are also available on the [Releases](https://github.com/mongodb/kingfisher/releases) section of this page. +### Linux and macOS -Or you may compile for your platform via `make`: +
+ +Use the bundled installer script to fetch the latest release and place it in +`~/.local/bin` (or a directory of your choice): + +```bash +# Linux, macOS +curl --silent --location \ + https://raw.githubusercontent.com/mongodb/kingfisher/main/scripts/install-kingfisher.sh | \ + bash +``` + +To install into a custom location, pass the desired directory as an argument: + +```bash +curl --silent --location \ + https://raw.githubusercontent.com/mongodb/kingfisher/main/scripts/install-kingfisher.sh | \ + bash -s -- /opt/kingfisher +``` + +To install a specific tag: + +```bash +curl --silent --location \ + https://raw.githubusercontent.com/mongodb/kingfisher/main/scripts/install-kingfisher.sh | \ + bash -s -- --tag v1.71.0 +``` + +
+ +### Windows + +
+ +Download and run the PowerShell installer to place the binary in +`$env:USERPROFILE\bin` (or another directory you specify): + +```powershell +# Windows +Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force +Invoke-WebRequest -Uri 'https://raw.githubusercontent.com/mongodb/kingfisher/main/scripts/install-kingfisher.ps1' -OutFile install-kingfisher.ps1 +./install-kingfisher.ps1 +``` + +You can provide a custom destination using the `-InstallDir` parameter: + +```powershell +./install-kingfisher.ps1 -InstallDir 'C:\Tools\Kingfisher' +``` + +To install a specific tag: + +```powershell +./install-kingfisher.ps1 -Tag v1.71.0 +``` +
+ + +### Pre-commit hooks + +Install a Git pre-commit hook to block commits that introduce new secrets. + +The installer: + +- Preserves any existing `pre-commit` hook by chaining it **before** Kingfisher. +- Supports custom hook directories via `--hooks-path` (or Git’s `core.hooksPath`). +- Can be installed either **per-repository** or as a **global** hook. + +#### macOS and Linux + +
+ +Install a **per-repository** hook from the root of the repo you want to protect: + +```bash +curl --silent --location \ + https://raw.githubusercontent.com/mongodb/kingfisher/main/scripts/install-kingfisher-pre-commit.sh | \ + bash +``` + +Uninstall from that repository: + +```bash +curl --silent --location \ + https://raw.githubusercontent.com/mongodb/kingfisher/main/scripts/install-kingfisher-pre-commit.sh | \ + bash -s -- --uninstall +``` + +Install as a **global** pre-commit hook (using core.hooksPath): + +```bash +curl --silent --location \ + https://raw.githubusercontent.com/mongodb/kingfisher/main/scripts/install-kingfisher-pre-commit.sh | \ + bash -s -- --global +``` + +Uninstall the **global** hook: + +```bash +curl --silent --location \ + https://raw.githubusercontent.com/mongodb/kingfisher/main/scripts/install-kingfisher-pre-commit.sh | \ + bash -s -- --global --uninstall +``` + +
+ +#### Windows PowerShell + +
+ +Install a **per-repository** hook from the root of the target repo: + +```powershell +Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force +Invoke-WebRequest -Uri 'https://raw.githubusercontent.com/mongodb/kingfisher/main/scripts/install-kingfisher-pre-commit.ps1' -OutFile install-kingfisher-pre-commit.ps1 +./install-kingfisher-pre-commit.ps1 +``` + +Uninstall from that repository: + +```powershell +./install-kingfisher-pre-commit.ps1 -Uninstall +``` + +Install as a **global** hook (using core.hooksPath): + +```powershell +./install-kingfisher-pre-commit.ps1 -Global +``` + +Uninstall the **global** hook: +```powershell +./install-kingfisher-pre-commit.ps1 -Global -Uninstall +``` + +> The installer automatically runs any existing `pre-commit` hook first, then +> executes `kingfisher scan . --staged --quiet --no-update-check` +> against the staged diff (anchored to `HEAD` when no commits exist yet). + +
+ +#### Using the `pre-commit` framework + +Add Kingfisher as a hook in your `.pre-commit-config.yaml`: + +
+ +```yaml +repos: + - repo: https://github.com/mongodb/kingfisher + rev: + hooks: + # No local install required; runs Kingfisher from Docker at the repo root + - id: kingfisher-docker + + # Fastest when you already have Kingfisher installed locally + - id: kingfisher +``` + +Then install the hook via `pre-commit install`. Every hook now drives Kingfisher +directly with the built-in `--staged` flag: + +```bash +kingfisher scan . --staged --quiet --no-update-check +``` + +When `--staged` is set, Kingfisher snapshots the staged index into a temporary +commit, diffs it against `HEAD` (or an empty tree if no commits exist yet), and +scans only those staged changes. + +> Exit codes: Kingfisher exits `0` when no findings are present and returns +> `205` when validated credentials are discovered (other findings use codes in +> the `200` range). The hook surfaces those exit codes directly to `pre-commit`, +> so no extra handling is required—the commit will fail automatically on +> non-zero exits. + +To trigger a hook in CI without installing to `.git/hooks`, run (for example): + +```bash +pre-commit run kingfisher-pre-commit --all-files +``` + +
+ +### Compile +You may compile for your platform via `make` + +
```bash # NOTE: Requires Docker make linux -``` -```bash -# macOS +# macOS --- must build from a macOS host make darwin -``` -```bash # Windows x64 --- requires building from a Windows host with Visual Studio installed ./buildwin.bat -force ``` @@ -58,7 +416,88 @@ make darwin-all # builds both x64 and arm64 make all # builds for every OS and architecture supported ``` -# Write Custom Rules! +
+ +### Docker Run Kingfisher in Docker + +Run the dockerized Kingfisher container + +
+ +```bash +# GitHub Container Registry +docker run --rm ghcr.io/mongodb/kingfisher:latest --version + +# Scan the current working directory +# (mounts your code at /src and scans it) +docker run --rm \ + -v "$PWD":/src \ + ghcr.io/mongodb/kingfisher:latest scan /src + + +# Scan while providing a GitHub token +# Mounts your working dir at /proj and passes in the token: +docker run --rm \ + -e KF_GITHUB_TOKEN=ghp_… \ + -v "$PWD":/proj \ + ghcr.io/mongodb/kingfisher:latest \ + scan --git-url https://github.com/org/private_repo.git + +# Scan an S3 bucket +# Credentials can come from KF_AWS_KEY/KF_AWS_SECRET, --role-arn, or --profile +docker run --rm \ + -e KF_AWS_KEY=AKIA... \ + -e KF_AWS_SECRET=g5nYW... \ + ghcr.io/mongodb/kingfisher:latest \ + scan s3 bucket-name + + +# Scan and write a JSON report locally +# Here we: +# 1. Mount $PWD → /proj +# 2. Tell Kingfisher to write findings.json inside /proj/reports +# 3. Ensure ./reports exists on your host so Docker can mount it +mkdir -p reports + +# run and output into host’s ./reports directory +docker run --rm \ + -v "$PWD":/proj \ + ghcr.io/mongodb/kingfisher:latest \ + scan /proj \ + --format json \ + --output /proj/reports/findings.json + + +# Tip: you can combine multiple mounts if you prefer separating source vs. output: +# Here /src is read‑only, and /out holds your generated reports +docker run --rm \ + -v "$PWD":/src:ro \ + -v "$PWD/reports":/out \ + ghcr.io/mongodb/kingfisher:latest \ + scan /src \ + --format json \ + --output /out/findings.json + +``` + +
+ +# 🔐 Detection Rules at a Glance + +Kingfisher ships with [hundreds of rules](/data/rules/) that cover everything from classic cloud keys to the latest AI SaaS tokens. Below is an overview: + +| Category | What we catch | +|----------|---------------| +| **AI SaaS APIs** | OpenAI, Anthropic, Google Gemini, Cohere, Mistral, Stability AI, Replicate, xAI (Grok), Ollama, Langchain, Perplexity, Weights & Biases, Cerebras, Friendli, Fireworks.ai, NVIDIA NIM, together.ai, Zhipu, and more | +| **Cloud Providers** | AWS, Azure, GCP, Alibaba Cloud, DigitalOcean, IBM Cloud, Cloudflare, and more | +| **Dev & CI/CD** | GitHub/GitLab tokens, CircleCI, TravisCI, TeamCity, Docker Hub, npm, PyPI, and more | +| **Messaging & Comms** | Slack, Discord, Microsoft Teams, Twilio, Mailgun, SendGrid, Mailchimp, and more | +| **Databases & Data Ops** | MongoDB Atlas, PlanetScale, Postgres DSNs, Grafana Cloud, Datadog, Dynatrace, and more | +| **Payments & Billing** | Stripe, PayPal, Square, GoCardless, and more | +| **Security & DevSecOps** | Snyk, Dependency-Track, CodeClimate, Codacy, OpsGenie, PagerDuty, and more | +| **Misc. SaaS & Tools** | 1Password, Adobe, Atlassian/Jira, Asana, Netlify, Baremetrics, and more | + +## 📝 Write Custom Rules! Kingfisher ships with hundreds of rules with HTTP and service‑specific validation checks (AWS, Azure, GCP, etc.) to confirm if a detected string is a live credential. @@ -66,13 +505,44 @@ However, you may want to add your own custom rules, or modify a detection to bet First, review [docs/RULES.md](/docs/RULES.md) to learn how to create custom Kingfisher rules. +### Pattern requirements and placeholder filtering + +Every rule can declare optional `pattern_requirements` to enforce additional character checks after a regex matches. Each field +is independent: + +- `min_digits`, `min_uppercase`, `min_lowercase`, and `min_special_chars` enforce complexity thresholds. +- `special_chars` lets you override the set of characters counted as "special" when `min_special_chars` is used. +- `ignore_if_contains` lists case-insensitive substrings that should cause a match to be discarded (for example, to drop + `test`, `demo`, or `localhost` values). +- `checksum` lets you compare an extracted portion of the match against a Liquid-rendered expectation. Provide `actual.template` + and `expected` Liquid snippets (with access to `{{ MATCH }}`, `{{ FULL_MATCH }}`, and any named capture as both its original + case and uppercase alias) and Kingfisher will skip the finding when the rendered values differ. Optional keys such as + `requires_capture` and `skip_if_missing` help you guard against legacy formats while onboarding the checksum-aware variant. + +When a match is skipped because of `ignore_if_contains` or a checksum mismatch, Kingfisher logs the event at the `DEBUG` level alongside the rule that was evaluated. If you need to keep those matches for a particular scan, pass `--no-ignore-if-contains` to `kingfisher scan` to disable the substring filter without editing any rule files. Verbose mode (`-v`) will also show you the +checksum mismatch lengths so you can confirm why a finding was suppressed. + Once you've done that, you can provide your custom rules (defined in a YAML file) and provide it to Kingfisher at runtime --- no recompiling required! -# Usage +### 🔍 Checksum Intelligence + +Modern API tokens increasingly include **built-in checksums**, short internal digests that make each credential self-verifiable. (For background, see [GitHub’s write-up on their newer token formats](https://github.blog/engineering/platform-security/behind-githubs-new-authentication-token-formats/) and why checksums slash false positives.) + +Kingfisher supports **checksum-aware matching** in rules, enabling **offline structural verification** of credentials *without* calling third-party APIs. + +By validating each token’s internal checksum (for tokens that support checksums), Kingfisher eliminates nearly all false positives—automatically skipping structurally invalid or fake tokens before validation ever runs. + +**Why this matters** +- **Offline verification** — no API call required +- **Industry-aligned** — compatible with prefix + checksum token designs (e.g., modern PATs) +- **Lower false positives** — invalid tokens are filtered out by structure alone + +**Learn more**: implementation details and templating are documented in **[docs/RULES.md](docs/RULES.md)** +# 🎉 Usage ## Basic Examples -> **Note**  `kingfisher scan` detects whether the input is a Git repository or a plain directory—no extra flags required. +> **Note**  `kingfisher scan` detects whether the input is a Git repository or a plain directory, no extra flags required. ### Scan with secret validation @@ -117,15 +587,55 @@ kingfisher scan . --format json | tee kingfisher.json kingfisher scan /path/to/repo --format sarif --output findings.sarif ``` +### Access map outputs and viewer + +**Stop Guessing, Start Mapping: Understand Your True Blast Radius** + +Finding a leaked credential is only the first step. The critical question isn’t just “Is this a secret?”—it’s “What can an attacker do with it?” + +Kingfisher's `--access-map` feature transforms secret detection from a simple alert into a comprehensive threat assessment. Instead of leaving you with a cryptic API key, Kingfisher actively authenticates against your cloud provider (AWS, GCP, Azure Storage, Azure DevOps, GitHub, or GitLab) to map the full extent of the credential's power. + +* Instant Identity Resolution: Immediately identify who the key belongs to—whether it's a specific IAM user, an assumed role, or a service account. +* Visualize the Blast Radius: See exactly which resources (S3 buckets, EC2 instances, projects, storage containers) are exposed and at risk. + + +Add `--access-map` to enrich JSON, JSONL, BSON, pretty, and SARIF reports with an `access_map` containing the resources and the permissions that the key can access - for each resource (grouped when identical). +- If you validated cloud credentials without `--access-map`, Kingfisher will remind you on stderr to rerun with the flag so the access map appears in the output. +- Run `kingfisher view ./kingfisher.json` to explore a report locally in a local web UI (opens your browser automatically when a report is provided). +- Or use `kingfisher scan --view-report ...` to generate a JSON report, start the viewer at `http://127.0.0.1:7890`, and open it in your browser. + +> **Use the access map functionality only when you are authorized to inspect the target account, as Kingfisher will issue additional network requests to determine what access the secret grants** + +![alt text](docs/kingfisher-usage-access-map.gif) + +### View access-map reports locally + +```bash +kingfisher view kingfisher.json +``` + +The `view` subcommand starts a local-only server (default port `7890`) that bundles the HTML, CSS, and JavaScript for the access-map viewer directly into the Kingfisher binary. Provide a JSON or JSONL report to load it automatically and Kingfisher will open your browser, or open the page and upload a report in the browser. If port 7890 is already in use, Kingfisher will exit and tell you to re-run with `--port `. + + ### Pipe any text directly into Kingfisher by passing `-` ```bash cat /path/to/file.py | kingfisher scan - + +``` + +### Limit maximum file size scanned (`--max-file-size`) + +By default, Kingfisher skips files larger than **256 MB**. You can raise or lower this cap per run with `--max-file-size`, which takes a value in **megabytes**. + +```bash +# Scan files up to 500 mb in size +kingfisher scan /some/file --max-file-size 500 ``` ### Scan using a rule _family_ with one flag -_(prefix matching: `--rule kingfisher.aws` loads `kingfisher.aws._`)\* +_(prefix matching: `--rule kingfisher.aws` loads `kingfisher.aws.*`)_ ```bash # Only apply AWS-related rules (kingfisher.aws.1 + kingfisher.aws.2) @@ -140,60 +650,651 @@ kingfisher scan /path/to/repo --rule-stats ### Scan while ignoring likely test files +`--exclude` skips any file or directory whose path matches this glob pattern (repeatable, uses gitignore-style syntax, case sensitive) + ```bash # Scan source but skip likely unit / integration tests -kingfisher scan ./my-project --ignore-tests +kingfisher scan ./my-project \ + --exclude='[Tt]est' \ + --exclude='spec' \ + --exclude='[Ff]ixture' \ + --exclude='example' \ + --exclude='sample' ``` ---- +### Exclude specific paths +```bash +# Skip all Python files and any directory named tests +kingfisher scan ./my-project \ + --exclude '*.py' \ + --exclude '[Tt]ests' +``` -## Scanning GitHub +### Scan changes in CI pipelines -### Scan GitHub organisation (requires `KF_GITHUB_TOKEN`) +Limit scanning to the delta between your default branch and a pull request branch by combining `--since-commit` with `--branch` (defaults to `HEAD`). This only scans files that differ between the two references, which keeps CI runs fast while still blocking new secrets. + +Use `--branch-root-commit` alongside `--branch` when you need to include a specific commit (and everything after it) in a diff-focused scan without re-examining earlier history. Provide the branch tip (or other comparison ref) via `--branch`, and pass the commit or merge-base you want to include with `--branch-root-commit`. If you omit `--branch-root-commit`, you can still enable `--branch-root` to fall back to treating the `--branch` ref itself as the inclusive root for backwards compatibility. This is especially useful in long-lived branches where you want to resume scanning from a previous review point or from the commit where a hotfix forked. + +> **How is this different from `--since-commit`?** +> `--since-commit` computes a diff between the branch tip and another ref, so it only inspects files that changed between those two points in history. `--branch-root-commit` rewinds to the parent of the commit you provide and then scans everything introduced from that commit forward, even if the files are unchanged relative to another baseline. Reach for `--since-commit` to keep CI scans fast by checking only the latest delta, and use `--branch-root-commit` when you want to re-audit the full contents of a branch starting at a specific commit. ```bash -kingfisher scan --github-organization my-org +kingfisher scan . \ + --since-commit origin/main \ + --branch "$CI_BRANCH" +``` + +Another example: +```bash +cd /tmp +git clone https://github.com/micksmix/SecretsTest.git + +cd /tmp/SecretsTest +git checkout feature-1 +# +# scan diff between main and feature-1 branch +kingfisher scan /tmp/SecretsTest --branch feature-1 \ + --since-commit=$(git -C /tmp/SecretsTest merge-base main feature-1) +# +# scan only a specific commit +kingfisher scan /tmp/SecretsTest \ + --branch baba6ccb453963d3f6136d1ace843e48d7007c3f +# +# scan feature-1 starting at a specific commit (inclusive) +kingfisher scan /tmp/SecretsTest --branch feature-1 \ + --branch-root-commit baba6ccb453963d3f6136d1ace843e48d7007c3f +# +# scan feature-1 starting from the commit where the branch diverged from main +kingfisher scan /tmp/SecretsTest --branch feature-1 \ + --branch-root-commit $(git -C /tmp/SecretsTest merge-base main feature-1) +# +# scan from a hotfix commit that should be re-checked before merging +HOTFIX_COMMIT=$(git -C /tmp/SecretsTest rev-parse hotfix~1) +kingfisher scan /tmp/SecretsTest --branch hotfix \ + --branch-root-commit "$HOTFIX_COMMIT" +``` + +When the branch under test is already checked out, `--branch HEAD` or omitting `--branch` entirely is sufficient. Kingfisher exits with `200` when any findings are discovered and `205` when validated secrets are present, allowing CI jobs to fail automatically if new credentials slip in. + +> **Tip:** You can point Kingfisher at a local working tree and scan another branch or commit without changing checkouts. The CLI now resolves repositories from their worktree roots, so commands like the following work without needing to pass the `.git` directory explicitly: + +```bash +kingfisher scan /path/to/local/repo --branch +kingfisher scan C:\\src\\repo --branch +``` + +The same diff-focused workflow works when cloning repositories on the fly with `--git-url`. Kingfisher automatically tries remote-tracking names like `origin/main` and `origin/feature-1`, so you can target the branches involved in a pull request without performing a local checkout first. + +```bash +kingfisher scan \ + --git-url https://github.com/org/repo.git \ + --since-commit main \ + --branch development +``` + +When `--since-commit` is omitted, specifying `--branch` scans the requested ref directly. This makes it easy to analyze a feature branch without checking it out locally. + +```bash +# Scan a branch from an existing checkout +kingfisher scan ~/tmp/repo --branch feature-123 + +# Or scan a branch when cloning on the fly +kingfisher scan \ + --git-url https://github.com/org/repo.git \ + --branch origin/feature-123 +``` + +In CI systems that expose the base and head commits explicitly, you can pass those SHAs directly while still using `--git-url`: + +```bash +kingfisher scan \ + --git-url git@github.com:org/repo.git \ + --since-commit "$BASE_COMMIT" \ + --branch "$PR_HEAD_COMMIT" +``` + +If you want to know which files are being skipped, enable verbose debugging (-v) when scanning, which will report any files being skipped by the baseline file (or via --exclude): + +```bash +# Skip all Python files and any directory named tests, and report to stderr any skipped files +kingfisher scan ./my-project \ + --exclude '*.py' \ + --exclude tests \ + -v +``` + +## GitHub Scanning an AWS S3 Bucket +You can scan S3 objects directly: + +```bash +kingfisher scan s3 bucket-name [--prefix path/] +``` + +Credential resolution happens in this order: + +1. `KF_AWS_KEY` and `KF_AWS_SECRET` environment variables +2. `--profile` pointing to a profile in `~/.aws/config` (works with AWS SSO) +3. anonymous access for public buckets + +If `--role-arn` is supplied, the credentials from steps 1–2 are used to assume that role. + +Examples + +```bash +# using explicit keys +export KF_AWS_KEY=AKIA... +export KF_AWS_SECRET=g5nYW... +kingfisher scan s3 some-example-bucket + +# Above can also be run as: +KF_AWS_KEY=AKIA... KF_AWS_SECRET=g5nYW... kingfisher scan s3 some-example-bucket + +# using a local profile (e.g., SSO) that exists in your AWS profile (~/.aws/config) +kingfisher scan s3 some-example-bucket --profile default + +# anonymous scan of a bucket, while providing an object prefix to only scan subset of the s3 bucket +kingfisher scan s3 awsglue-datasets \ + --prefix examples/us-legislators/all + +# assuming a role when scanning +kingfisher scan s3 some-example-bucket \ + --role-arn arn:aws:iam::123456789012:role/MyRole + +# anonymous scan of a public bucket +kingfisher scan s3 some-example-bucket +``` + +Docker example: + +```bash +docker run --rm \ + -e KF_AWS_KEY=AKIA... \ + -e KF_AWS_SECRET=g5nYW... \ + ghcr.io/mongodb/kingfisher:latest \ + scan s3 bucket-name +``` + +## Google Cloud Storage Scanning a Google Cloud Storage Bucket + +Use the `gcs` scan subcommand to stream objects directly from Google Cloud Storage. Authentication +uses Application Default Credentials, so you can provide a service-account JSON file via the +`GOOGLE_APPLICATION_CREDENTIALS` environment variable or by passing `--service-account`. Public +buckets work without credentials. + +```bash +kingfisher scan gcs bucket-name + +# scan a sub-tree inside the bucket +kingfisher scan gcs bucket-name --prefix path/to/data/ + +# supply a service-account key explicitly +kingfisher scan gcs bucket-name --service-account /path/to/key.json +``` + +Functional example: +```bash +kingfisher scan gcs cloud-samples-data --prefix "storage/" +``` + + +## Docker Scanning Docker Images + +Kingfisher will first try to use any locally available image, then fall back to pulling via OCI. + +Authentication happens *in this order*: + +1. **`KF_DOCKER_TOKEN`** env var + - If it contains `user:pass`, it’s used as Basic auth + - Otherwise it’s sent as a Bearer token +2. **Docker CLI credentials** + - Checks `credHelpers` (per-registry) and `credsStore` in `~/.docker/config.json`. + - Falls back to the legacy `auths` → `auth` (base64) entries. +3. **Anonymous** (no credentials) + + +```bash +# 1) Scan public or already-pulled image +kingfisher scan docker ghcr.io/owasp/wrongsecrets/wrongsecrets-master:latest-master + +# 2) For private registries, explicitly set KF_DOCKER_TOKEN: +# - Basic auth: "user:pass" +# - Bearer only: "TOKEN" +export KF_DOCKER_TOKEN="AWS:$(aws ecr get-login-password --region us-east-1)" +kingfisher scan docker some-private-registry.dkr.ecr.us-east-1.amazonaws.com/base/amazonlinux2023:latest + +# 3) Or rely on your Docker CLI login/keychain: +# (e.g. aws ecr get-login-password … | docker login …) +kingfisher scan docker private.registry.example.com/my-image:tag +``` + +> **Deprecated** +> Legacy scan flags such as `--github-user`, `--gitlab-group`, +> `--bitbucket-workspace`, `--azure-organization`, `--huggingface-user`, +> `--slack-query`, `--jira-url`, `--confluence-url`, `--s3-bucket`, +> `--gcs-bucket`, and `--docker-image` still work for now, but they trigger a +> warning and will be removed in a future release. Migrate to the +> `kingfisher scan ` subcommands below to future-proof your automations. + +## GitHub Scanning GitHub + +### Scan GitHub organization (requires `KF_GITHUB_TOKEN`) + +```bash +kingfisher scan github --organization my-org +kingfisher scan github --organization my-org --repo-clone-limit 500 +``` + +### Skip specific GitHub repositories during enumeration + +Repeat `--github-exclude` for every repository you want to ignore when scanning +users or organizations. You can provide exact repositories like +`OWNER/REPO` or gitignore-style glob patterns such as `owner/*-archive` +(matching is case-insensitive). + +```bash +kingfisher scan github --organization my-org \ + --github-exclude my-org/huge-repo \ + --github-exclude my-org/*-archive ``` ### Scan remote GitHub repository +`--git-url` clones the repository and scans its files and history. When the URL +targets GitHub and you pass `--include-contributors`, Kingfisher enumerates +repository contributors and attempts to clone **all public repos owned by those +contributors**—a common offensive and blue-team pivot when developers leak +secrets in personal or side projects. Use `--repo-clone-limit` to cap how many +repositories are cloned during this enumeration. + +**NOTE**: This may cause you to be temporarily rate-limited by GitHub. +Providing a token (`KF_GITHUB_TOKEN`) will provide a higher rate limit. + +To inspect related server-side data, supply `--repo-artifacts`. This flag pulls +down the repository's issues (including pull requests), wiki, and any public +gists owned by the repository owner and scans them for secrets. Fetching these +extras counts against API rate limits and private artifacts require a +`KF_GITHUB_TOKEN`. + +Use `--git-clone-dir` to choose where cloned repositories land and +`--keep-clones` to preserve them for follow-on analysis. + +> **Why does `--git-url` sometimes report fewer findings than scanning a local checkout?**. +> +> Remote clones created via `--git-url` default to `--mirror`/bare mode so Kingfisher only +> reads the Git history. When you point Kingfisher at an existing working tree (for example +> `kingfisher scan ./repo`), it enumerates both the filesystem contents *and* the Git +> history. Any secrets that are present in the checked-out files therefore appear twice: +> once from the working tree path and once from the commit where the secret entered the +> history. To replicate the remote behavior locally, either scan a bare clone or disable +> history scanning with `--git-history none` when targeting a working tree. + + ```bash +# Scan the repository only kingfisher scan --git-url https://github.com/org/repo.git -# Optionally provide a GitHub Token -KF_GITHUB_TOKEN="ghp_…" kingfisher scan --git-url https://github.com/org/private_repo.git +# Scan the repository plus contributor repos, but cap the crawl +kingfisher scan --git-url https://github.com/org/repo.git \ + --include-contributors \ + --repo-clone-limit 250 +# Keep clones for later manual inspection +kingfisher scan --git-url https://github.com/org/repo.git \ + --git-clone-dir ./kingfisher-clones \ + --keep-clones + +# Include issues, wiki, and owner gists +kingfisher scan --git-url https://github.com/org/repo.git --repo-artifacts + +# Private repositories or artifacts +KF_GITHUB_TOKEN="ghp_…" kingfisher scan --git-url https://github.com/org/private_repo.git --repo-artifacts ``` --- -## Scanning GitLab +## GitLab Scanning GitLab ### Scan GitLab group (requires `KF_GITLAB_TOKEN`) ```bash -kingfisher scan --gitlab-group my-group +kingfisher scan gitlab --group my-group +# include repositories from all nested subgroups +kingfisher scan gitlab --group my-group --include-subgroups +kingfisher scan gitlab --group my-group --repo-clone-limit 500 ``` ### Scan GitLab user ```bash -kingfisher scan --gitlab-user johndoe +kingfisher scan gitlab --user johndoe +``` + +### Skip specific GitLab projects during enumeration + +Repeat `--gitlab-exclude` for every project path you want to ignore when scanning +users or groups. Specify project paths as `group/project` (case-insensitive) or +use gitignore-style glob patterns like `group/**/archive-*` to drop families of +projects across nested subgroups. + +```bash +kingfisher scan gitlab --group my-group \ + --gitlab-exclude my-group/huge-project \ + --gitlab-exclude my-group/**/archive-* ``` ### Scan remote GitLab repository by URL +`--git-url` by itself clones the project repository. When the URL targets +GitLab and you pass `--include-contributors`, Kingfisher enumerates contributors +and tries to clone **their other public projects** to catch secrets that escape +the main repo. Apply `--repo-clone-limit` to cap the total repos cloned during +this pivot. + +**NOTE**: This may cause you to be temporarily rate-limited by GitLab. +Providing a token (`KF_GITLAB_TOKEN`) will provide a higher rate limit. + +To include server-side artifacts owned by the project, add `--repo-artifacts`. +Kingfisher will retrieve the project's issues, wiki, and snippets and scan them +for secrets. These extra requests may take longer and require a +`KF_GITLAB_TOKEN` for private projects. + +Use `--git-clone-dir` to choose where cloned projects land and `--keep-clones` +to preserve them for later review. + ```bash +# Scan the repository only kingfisher scan --git-url https://gitlab.com/group/project.git + +# Scan the repository plus contributor projects, but cap the crawl +kingfisher scan --git-url https://gitlab.com/group/project.git \ + --include-contributors \ + --repo-clone-limit 250 + +# Keep clones for later manual inspection +kingfisher scan --git-url https://gitlab.com/group/project.git \ + --git-clone-dir ./kingfisher-clones \ + --keep-clones + +# Include issues, wiki, and snippets +kingfisher scan --git-url https://gitlab.com/group/project.git --repo-artifacts + +# Private projects or artifacts +KF_GITLAB_TOKEN="glpat-…" kingfisher scan --git-url https://gitlab.com/group/private_project.git --repo-artifacts ``` ### List GitLab repositories ```bash -kingfisher gitlab repos list --group my-group +kingfisher scan gitlab --group my-group --list-only +# include repositories from all nested subgroups +kingfisher scan gitlab --group my-group --include-subgroups --list-only +# skip specific projects when listing or scanning (supports glob patterns) +kingfisher scan gitlab --group my-group --gitlab-exclude my-group/**/legacy-* --list-only +``` +## Azure Repos Scanning Azure Repos + +### Scan Azure Repos organization or collection (requires `KF_AZURE_TOKEN` or `KF_AZURE_PAT`) + +```bash +kingfisher scan azure --organization my-org + +# Azure Repos Server example +KF_AZURE_PAT="pat" kingfisher scan azure --organization DefaultCollection --azure-base-url https://ado.internal.example/tfs/ ``` ---- +### Scan specific Azure Repos projects + +Projects are specified as `ORGANIZATION/PROJECT`. Repeat the flag for multiple projects. + +```bash +kingfisher scan azure --project my-org/payments \ + --project my-org/core-platform +``` + +### Skip specific Azure repositories during enumeration + +Repeat `--azure-exclude` to ignore repositories when scanning organizations or projects. +Use identifiers like `ORGANIZATION/PROJECT/REPOSITORY`. Repositories that share the same +name as their project can be excluded with `ORGANIZATION/PROJECT`, and gitignore-style +patterns such as `my-org/*/archive-*` are also supported. + +```bash +kingfisher scan azure --organization my-org \ + --azure-exclude my-org/payments/legacy-service \ + --azure-exclude my-org/**/archive-* +``` + +### List Azure repositories + +```bash +kingfisher scan azure --organization my-org --list-only +# list repositories for specific projects +kingfisher scan azure --project my-org/app --project my-org/api --list-only +# skip specific repositories while listing (supports glob patterns) +kingfisher scan azure --organization my-org --azure-exclude my-org/**/experimental-* --list-only +``` +## Gitea Scanning Gitea + +### Scan Gitea organization (requires `KF_GITEA_TOKEN`) + +```bash +kingfisher scan gitea --organization my-org +# self-hosted example +KF_GITEA_TOKEN="gtoken" kingfisher scan gitea --organization platform --gitea-api-url https://gitea.internal.example/api/v1/ +``` + +### Scan Gitea user + +```bash +kingfisher scan gitea --user johndoe +``` + +### Skip specific Gitea repositories during enumeration + +Repeat `--gitea-exclude` for each repository you want to ignore when scanning users +or organizations. Accepts `owner/repo` identifiers or gitignore-style glob patterns +like `team/**/archive-*`. + +```bash +kingfisher scan gitea --organization my-org \ + --gitea-exclude my-org/legacy-repo \ + --gitea-exclude my-org/**/archive-* +``` + +### Scan remote Gitea repository by URL + +`--git-url` clones the repository and scans its history. Adding `--repo-artifacts` +also clones the repository wiki if one exists. Private repositories and wikis +require `KF_GITEA_TOKEN` (and `KF_GITEA_USERNAME` when cloning via HTTPS). + +```bash +# Scan the repository only +kingfisher scan --git-url https://gitea.com/org/repo.git + +# Include the repository wiki (if present) +KF_GITEA_TOKEN="gtoken" KF_GITEA_USERNAME="org" \ + kingfisher scan --git-url https://gitea.com/org/repo.git --repo-artifacts +``` + +### List Gitea repositories + +```bash +kingfisher scan gitea --organization my-org --list-only +# enumerate every organization visible to the authenticated user +KF_GITEA_TOKEN="gtoken" kingfisher scan gitea --all-gitea-organizations --list-only +# self-hosted example +KF_GITEA_TOKEN="gtoken" kingfisher scan gitea --user johndoe --gitea-api-url https://gitea.internal.example/api/v1/ --list-only +``` +## Bitbucket Scanning Bitbucket +### Scan Bitbucket workspace + +```bash +kingfisher scan bitbucket --workspace my-team +# include Bitbucket Cloud repositories from every accessible workspace +KF_BITBUCKET_TOKEN="$BITBUCKET_TOKEN" \ + kingfisher scan bitbucket --all-workspaces +``` + +### Scan Bitbucket user + +```bash +kingfisher scan bitbucket --user johndoe +``` + +### Skip specific Bitbucket repositories during enumeration + +Use `--bitbucket-exclude` to ignore repositories while scanning users, workspaces, +or projects. Patterns accept either `owner/repo` (case-insensitive) or +gitignore-style globs such as `workspace/**/archive-*`. + +```bash +kingfisher scan bitbucket --workspace my-team \ + --bitbucket-exclude my-team/legacy-repo \ + --bitbucket-exclude my-team/**/archive-* +``` + +### Scan remote Bitbucket repository by URL + +`--git-url` clones the repository and scans its files and history. To inspect +Bitbucket artifacts such as issues, add `--repo-artifacts`. Private artifacts +require credentials (see [Authenticate to Bitbucket](#authenticate-to-bitbucket)). + +```bash +# Scan the repository only +kingfisher scan --git-url https://bitbucket.org/hashashash/secretstest.git + +# Include repository issues +KF_BITBUCKET_TOKEN="$BITBUCKET_TOKEN" \ + kingfisher scan --git-url https://bitbucket.org/workspace/project.git --repo-artifacts +``` + +### List Bitbucket repositories + +```bash +kingfisher scan bitbucket --workspace my-team --list-only +# enumerate all accessible workspaces or projects +KF_BITBUCKET_TOKEN="$BITBUCKET_TOKEN" \ + kingfisher scan bitbucket --all-workspaces --list-only +# filter out repositories using glob patterns +kingfisher scan bitbucket --workspace my-team --bitbucket-exclude my-team/**/experimental-* --list-only +``` + +### Authenticate to Bitbucket + +Kingfisher supports Bitbucket Cloud and Bitbucket Server credentials: + +- **Workspace API token (Cloud)** – set `KF_BITBUCKET_TOKEN`. Kingfisher automatically uses the token for Bitbucket REST APIs and authenticates git operations as `x-token-auth`. +- **Bitbucket Server token** – set `KF_BITBUCKET_USERNAME` and either + `KF_BITBUCKET_TOKEN` or `KF_BITBUCKET_PASSWORD`. +- **Legacy app password (Cloud)** – set `KF_BITBUCKET_USERNAME` and + `KF_BITBUCKET_APP_PASSWORD`. +- **OAuth/PAT token** – set `KF_BITBUCKET_OAUTH_TOKEN`. + +These credentials match the options described in the [ghorg setup +guide](https://github.com/gabrie30/ghorg/blob/master/README.md#bitbucket-setup). + +Bitbucket no longer supports App Tokens as of September 9, 2025: +https://support.atlassian.com/bitbucket-cloud/docs/api-tokens/ + +> As of September 9, 2025, app passwords can no longer be created. Use API tokens with scopes instead. All existing app passwords will be disabled on June 9, 2026. Migrate any integrations before then to avoid disruptions. + +### Self-hosted Bitbucket Server + +Use `--bitbucket-api-url` to point Kingfisher at your server's REST endpoint, for example +`https://bitbucket.example.com/rest/api/1.0/`. Provide credentials with +`KF_BITBUCKET_USERNAME` plus either `KF_BITBUCKET_TOKEN` or `KF_BITBUCKET_PASSWORD`, +and pass `--ignore-certs` when connecting to HTTP or otherwise insecure instances. +## Hugging Face Scanning Hugging Face + +Hugging Face hosts git repositories for models, datasets, and Spaces. Kingfisher can enumerate and scan all three resource types. + +### Scan Hugging Face user + +```bash +kingfisher scan huggingface --user +``` + +### Scan Hugging Face organization + +```bash +kingfisher scan huggingface --organization +``` + +### Scan specific Hugging Face resources + +Scan individual repositories by ID (owner/name) or by passing the full HTTPS URL: + +```bash +kingfisher scan huggingface --model +kingfisher scan huggingface --dataset https://huggingface.co/datasets// +kingfisher scan huggingface --space +``` + +Use `--huggingface-exclude` to omit results returned by user or organization enumeration. Prefix values with `model:`, `dataset:`, or `space:` when you only want to skip a specific resource type. + +### List Hugging Face repositories + +```bash +kingfisher scan huggingface --user --list-only +``` + +### Authenticate to Hugging Face + +Private repositories require an access token provided through the `KF_HUGGINGFACE_TOKEN` environment variable. For git authentication the helper also honours `KF_HUGGINGFACE_USERNAME` (default `hf_user`). + +## Jira Scanning Jira + +### Scan Jira issues matching a JQL query + +```bash +KF_JIRA_TOKEN="token" kingfisher scan jira --url https://jira.company.com \ + --jql "project = TEST AND status = Open" \ + --max-results 500 +``` + +### Scan the last 1,000 Jira issues: +```bash +KF_JIRA_TOKEN="token" kingfisher scan jira --url https://jira.mongodb.org \ + --jql 'ORDER BY created DESC' \ + --max-results 1000 +``` + +## Confluence Scanning Confluence +### Scan Confluence pages matching a CQL query + +```bash +# Bearer token +KF_CONFLUENCE_TOKEN="token" kingfisher scan confluence --url https://confluence.company.com \ + --cql "label = secret" \ + --max-results 500 + +# Basic auth with username and token +KF_CONFLUENCE_USER="user@example.com" KF_CONFLUENCE_TOKEN="token" \ + kingfisher scan confluence --url https://confluence.company.com \ + --cql "text ~ 'password'" \ + --max-results 500 +``` + +Use the base URL of your Confluence site for `--confluence-url`. Kingfisher +automatically adds `/rest/api` to the end, so `https://example.com/wiki` and +`https://example.com` both work depending on your server configuration. + +Generate a personal access token and set it in the `KF_CONFLUENCE_TOKEN` environment variable. By default, Kingfisher sends the token as a bearer token in the `Authorization` header. + +To use basic authentication instead, also set `KF_CONFLUENCE_USER` to your Confluence email address; Kingfisher will then send the username and `KF_CONFLUENCE_TOKEN` as a Basic auth header. If the server responds with a redirect to a login page, the credentials are invalid or lack the required permissions. + +## Slack Scanning Slack +### Scan Slack messages matching a search query + +```bash +KF_SLACK_TOKEN="xoxp-1234..." kingfisher scan slack "from:username has:link" \ + --max-results 1000 + +KF_SLACK_TOKEN="xoxp-1234..." kingfisher scan slack "akia" \ + --max-results 1000 +``` +*The Slack token must be a user token with the `search:read` scope. Bot tokens (those beginning with `xoxb-`) cannot call the Slack search API.* ## Environment Variables for Tokens @@ -201,11 +1302,26 @@ kingfisher gitlab repos list --group my-group | ----------------- | ---------------------------- | | `KF_GITHUB_TOKEN` | GitHub Personal Access Token | | `KF_GITLAB_TOKEN` | GitLab Personal Access Token | +| `KF_GITEA_TOKEN` | Gitea Personal Access Token | +| `KF_GITEA_USERNAME` | Username for private Gitea clones (used with `KF_GITEA_TOKEN`) | +| `KF_AZURE_TOKEN` / `KF_AZURE_PAT` | Azure Repos Personal Access Token | +| `KF_AZURE_USERNAME` | Username to use with Azure Repos PATs (defaults to `pat` when unset) | +| `KF_BITBUCKET_TOKEN` | Bitbucket Cloud workspace API token or Bitbucket Server PAT | +| `KF_BITBUCKET_USERNAME` | Optional Bitbucket username for legacy app passwords or server tokens | +| `KF_BITBUCKET_APP_PASSWORD` | Legacy Bitbucket app password (deprecated September 9, 2025; disabled June 9, 2026) | +| `KF_BITBUCKET_OAUTH_TOKEN` | Bitbucket OAuth or PAT token | +| `KF_HUGGINGFACE_TOKEN` | Hugging Face access token for API enumeration and git cloning | +| `KF_HUGGINGFACE_USERNAME` | Optional username for Hugging Face git operations (defaults to `hf_user`) | +| `KF_JIRA_TOKEN` | Jira API token | +| `KF_CONFLUENCE_TOKEN` | Confluence API token | +| `KF_SLACK_TOKEN` | Slack API token | +| `KF_DOCKER_TOKEN` | Docker registry token (`user:pass` or bearer token). If unset, credentials from the Docker keychain are used | +| `KF_AWS_KEY` and `KF_AWS_SECRET` | AWS Credentials to use with S3 bucket scanning | Set them temporarily per command: ```bash -KF_GITLAB_TOKEN="glpat-…" kingfisher scan --gitlab-group my-group +KF_GITLAB_TOKEN="glpat-…" kingfisher scan gitlab --group my-group ``` Or export for the session: @@ -214,6 +1330,16 @@ Or export for the session: export KF_GITLAB_TOKEN="glpat-…" ``` +To authenticate Jira requests: +```bash +export KF_JIRA_TOKEN="token" +``` + +To authenticate Confluence requests: +```bash +export KF_CONFLUENCE_TOKEN="token" +``` + _If no token is provided Kingfisher still works for public repositories._ --- @@ -226,28 +1352,47 @@ _If no token is provided Kingfisher still works for public repositories._ | 200 | Findings discovered | | 205 | Validated findings discovered | ---- - -### Update Checks +## Update Checks Kingfisher automatically queries GitHub for a newer release when it starts and tells you whether an update is available. -- **Hands-free updates** – Add `--self-update` to any Kingfisher command - - * If a newer version exists, Kingfisher will download it, replace the running binary, and re-launch itself with the **exact same arguments**. - * If the update fails or no newer release is found, the current run proceeds as normal +- **Manual update** – Run `kingfisher update` to update the binary without scanning - **Disable version checks** – Pass `--no-update-check` to skip both the startup and shutdown checks entirely ---- +# 🤓 Advanced Options -### List Builtin Rules +## Build a Baseline / Detect New Secrets + +There are situations where a repository already contains checked‑in secrets, but you want to ensure no **new** secrets are introduced. A baseline file lets you document the known findings so future scans only report anything that is not already in that list. + +The easiest way to create a baseline is to run a normal scan with the `--manage-baseline` flag (typically at a low confidence level to capture all potential matches): + +```bash +kingfisher scan /path/to/code \ + --confidence low \ + --manage-baseline \ + --baseline-file ./baseline-file.yml +``` + +`--manage-baseline` automatically enables `--no-dedup` so the baseline captures every individual occurrence. + +Use the same YAML file with the `--baseline-file` option on future scans to hide all recorded findings: + +```bash +kingfisher scan /path/to/code \ + --baseline-file /path/to/baseline-file.yaml +``` + +Running the scan again with `--manage-baseline` refreshes the baseline by adding new findings and pruning entries for secrets that no longer appear. See [docs/BASELINE.md](docs/BASELINE.md) for full detail. + +## List Builtin Rules ```bash kingfisher rules list ``` -### To scan using **only** your own `my_rules.yaml` you could run: +## To scan using **only** your own `my_rules.yaml` you could run: ```bash kingfisher scan \ @@ -256,7 +1401,7 @@ kingfisher scan \ ./src/ ``` -### To add your rules alongside the built‑ins: +## To add your rules alongside the built‑ins: ```bash kingfisher scan \ @@ -272,21 +1417,168 @@ kingfisher scan \ kingfisher rules check --rules-path ./my_rules.yml # List GitHub repos -kingfisher github repos list --user my-user -kingfisher github repos list --organization my-org +kingfisher scan github --user my-user --list-only +kingfisher scan github --organization my-org --list-only +# Skip specific repositories when listing or scanning (supports glob patterns) +kingfisher scan github --organization my-org --github-exclude my-org/*-archive --list-only ``` +## Customize the HTTP User-Agent + +Kingfisher identifies its HTTP requests with a user-agent that includes the binary name and version followed by a browser-style +string. Some environments require extra context, such as a contact address, a change-ticket number, or a temporary test label. +Use the global `--user-agent-suffix` flag to append this information between the Kingfisher identifier and the browser portion: + +```bash +# Attach a contact email to all outbound validation requests +kingfisher --user-agent-suffix "contact=security@example.com" scan path/ + +# Label a one-off experiment +kingfisher --user-agent-suffix "Sept 2025 testing" scan github --user my-user --list-only +``` + +When omitted, Kingfisher defaults to `kingfisher/ Mozilla/5.0 ...`. The suffix is trimmed; passing an empty string + +## Validation tuning flags + +Use these options with `kingfisher scan` to customize live validation behavior: + +- `--validation-timeout SECONDS`: per-request and per-match timeout for validation (default: 10, range: 1-60). +- `--validation-retries N`: number of retry attempts for validation requests (default: 1, range: 0-5). ## Notable Scan Options - `--no-dedup`: Report every occurrence of a finding (disable the default de-duplicate behavior) +- `--no-base64`: By default, Kingfisher finds and decodes base64 blobs and scans them for secrets. This adds a slight performance overhead; use this flag to disable - `--confidence `: (low|medium|high) - `--min-entropy `: Override default threshold +- `--include-contributors`: When using `--git-url` for GitHub or GitLab, include contributor-owned repos in the scan +- `--git-clone-dir `: Choose the parent directory for cloned repos and scan artifacts (use with `--git-url`) +- `--keep-clones`: Preserve cloned repositories on disk after a scan completes +- `--repo-clone-limit `: Cap the number of GitHub/GitLab repositories cloned when enumerating orgs/groups or contributor repos - `--no-binary`: Skip binary files - `--no-extract-archives`: Do not scan inside archives - `--extraction-depth `: Specifies how deep nested archives should be extracted and scanned (default: 2) - `--redact`: Replaces discovered secrets with a one-way hash for secure output -- `--ignore-tests`: Skip files or directories whose path component contains _test_, _spec_, _fixture_, _example_, or _sample_ (case-insensitive) +- `--exclude `: Skip any file or directory whose path matches this glob pattern (repeatable, uses gitignore-style syntax, case sensitive) +- `--baseline-file `: Ignore matches listed in a baseline YAML file +- `--manage-baseline`: Create or update the baseline file with current findings (automatically enables `--no-dedup`) +- `--skip-regex `: Ignore findings whose text matches this regex (repeatable) +- `--skip-word `: Ignore findings containing this case-insensitive word (repeatable) +- `--skip-aws-account `: Skip live AWS validation for findings tied to the specified AWS account number (repeatable, accepts comma-separated lists) +- `--skip-aws-account-file `: Load AWS account numbers to skip from a file (one account per line; `#` comments allowed) +- `--ignore-comment `: Honor additional inline directives from other scanners (repeatable; e.g. `--ignore-comment "gitleaks:allow"`) +- `--no-ignore`: Disable inline directives entirely so every match is reported +- `--no-ignore-if-contains`: Ignore the `ignore_if_contains` filter in rules so placeholder words still produce findings +- `--validation-timeout SECONDS`: per-request and per-match timeout for validation (default: 10, range: 1-60). +- `--validation-retries N`: number of retry attempts for validation requests (default: 1, range: 0-5). + +## Understanding `--confidence` + +The `--confidence` flag sets a minimum confidence threshold, not an exact match. + +- If you pass `--confidence medium`, findings with **medium and higher** confidence (medium + high) will be included. +- If you pass `--confidence low`, you’ll see **all levels** (low, medium, high). + + +### Ignore known false positives + +Use `--skip-regex` and `--skip-word` to suppress findings you know are benign. Both flags may be provided multiple times and are tested against the secret value **and** the full match context. + +With `--skip-regex`, these should be Rust compatible regular expressions, which you can test out at [regex101](https://regex101.com) + +```bash +# Skip any finding where the finding mentions TEST_KEY +kingfisher scan --skip-regex '(?i)TEST_KEY' path/ + +# Skip findings that contain the word "dummy" anywhere in the match +kingfisher scan --skip-word dummy path/ + +# Combine multiple patterns +kingfisher scan \ + --skip-regex 'AKIA[0-9A-Z]{16}' \ + --skip-word placeholder \ + --skip-word dummy \ + path/ +``` + +If a `--skip-regex` regular expression fails to compile, the scan aborts with an error so that typos are caught early. + +### Skip Canary Tokens (AWS) + +Canary/honey tokens are intentionally leaked credentials used to catch misuse. Kingfisher can **recognize and skip** known AWS canary accounts so hygiene scans don’t set off alerts. + +**How to skip** +Pass the 12-digit AWS account IDs for your canaries via `--skip-aws-account` (comma-separated) or `--skip-aws-account-file` (one ID per line; blank lines and `#` comments allowed). Kingfisher also ships with a **pre-seeded (but not exhaustive)** list of Thinkst Canary account IDs used by canarytokens.org, so many are skipped automatically. + +```bash +kingfisher scan /path/to/code \ + --skip-aws-account "171436882533,534261010715" + +# or combine preloaded canary IDs with a just-created decoy account +printf '999900001111 \n534261010715' > /tmp/canary_accounts.txt + +kingfisher scan /path/to/repo \ + --skip-aws-account-file /tmp/canary_accounts.txt + +``` + +**What you’ll see** +Findings tied to a skip-listed account report `Validation: Not Attempted` and note in the `Response:` that the entry came from the skip list: + +```bash +AWS SECRET ACCESS KEY => [KINGFISHER.AWS.2] + |Finding.......: + |Fingerprint...: 2141074333616819500 + |Confidence....: medium + |Entropy.......: 5.00 + |Validation....: Not Attempted + |__Response....: (skip list entry) AWS validation not attempted for account 171436882533. + |Language......: Unknown + |Line Num......: 21 + |Path..........: /tmp/test_canary_accounts.log +``` + +**Why this matters** +Skipping prevents noisy tripwires in prod telemetry while keeping the status explicit—“Not Attempted” isn’t a pass. If needed, verify these credentials out-of-band or with a safe, non-triggering method. + + +#### Common CLI flows + +```bash +# Skip a few in-house canaries during a filesystem scan +kingfisher scan repo/ \ + --skip-aws-account "111122223333,444455556666" + +# Read a longer list from disk +kingfisher scan repo/ \ + --skip-aws-account-file /tmp/scripts/canary_accounts.txt + +# Combine preloaded canary IDs with a just-created decoy account +printf '999900001111\n534261010715\n' > /tmp/new_canary.txt + +kingfisher scan /path/to/repo \ + --skip-aws-account-file /tmp/new_canary.txt + +``` + +Tip: if you manage multiple canary fleets (Thinkst, self-hosted alternatives, or bespoke decoys), checkpoint the account IDs alongside your infrastructure-as-code so security teams can rotate or expand the skip list without editing pipelines. + +### Inline ignore directives + +Add `kingfisher:ignore` anywhere on the same line as a finding to silence it. Multi-line strings and PEM-style blocks may also be ignored by placing the directive on the closing delimiter line (for example, `""" # kingfisher:ignore`), on the next logical line after the string, **or** on a comment immediately before the value: + +```python +# kingfisher:ignore +API_KEY = """ +line 1 +line 2 +""" +# kingfisher:ignore +``` + +Kingfisher searches the surrounding lines for these tokens without requiring language-specific comment markers. To reuse existing inline directives from other scanners, add them with repeatable `--ignore-comment` flags (for example `--ignore-comment "gitleaks:allow" --ignore-comment "NOSONAR"`). Use `--no-ignore` when you want to disable inline suppressions entirely. + ## Finding Fingerprint @@ -303,59 +1595,29 @@ Use `--rule-stats` to collect timing information for every rule. After scanning, kingfisher scan --help ``` -## Business Value -By integrating Kingfisher into your development lifecycle, you can: +## Lineage and Evolution -- **Prevent Costly Breaches** - Early detection of embedded credentials avoids expensive incident response, legal fees, and reputation damage -- **Automate Compliance** - Enforce secret‑scanning policies across GitOps, CI/CD, and pull requests to help satisfy SOC 2, PCI‑DSS, GDPR, and other standards -- **Reduce Noise, Focus on Real Threats** - Validation logic filters out false positives and highlights only active, valid secrets (`--only-valid`) -- **Accelerate Dev Workflows** - Run in parallel across dozens of languages, integrate with GitHub Actions or any pipeline, and shift security left to minimize delays +Kingfisher began as an internal fork of Nosey Parker, used as a high-performance foundation for secret detection. -## The Risk of Leaked Secrets +Since then it has evolved far beyond that starting point, introducing live validation, hundreds of new rules, additional scan targets, and major architectural changes across nearly every subsystem. -Embedding credentials in code repositories is a pervasive, ever‑present risk that leads directly to data breaches: - -1. **Uber (2016)** - - - _Incident_: Attackers stole GitHub credentials, retrieved an AWS key from a developer’s private repo, and accessed data on 57 million riders and 600 000 drivers. - - _Sources_: [BBC News](https://www.bbc.com/news/technology-42075306), [Ars Technica](https://arstechnica.com/tech-policy/2017/11/report-uber-paid-hackers-100000-to-keep-2016-data-breach-quiet/) - -2. **AWS** - - - _Incident_: An AWS engineer accidentally published log files and CloudFormation templates containing AWS key pairs (including “rootkey.csv”) to a public GitHub repo. - - _Sources_: [The Register](https://www.theregister.com/2020/01/23/aws_engineer_credentials_github/), [UpGuard](https://www.upguard.com/breaches/identity-and-access-misstep-how-an-amazon-engineer-exposed-credentials-and-more) - -3. **Infosys** - - - _Incident_: Infosys published an internal PyPI package embedding a FullAdminAccess AWS key for a Johns Hopkins data bucket; the key remained active for over a year. - - _Sources_: [The Stack](https://www.thestack.technology/infosys-leak-aws-key-exposed-on-pypi/), [Tom Forbes Blog](https://tomforb.es/blog/infosys-leak/) - -4. **Microsoft** - - - _Incident_: Microsoft’s AI research GitHub repo included an overly permissive Azure SAS token, exposing 38 TB of private data (workstation backups, 30,000+ Teams messages). - - _Sources_: [Wiz Blog](https://www.wiz.io/blog/38-terabytes-of-private-data-accidentally-exposed-by-microsoft-ai-researchers), [TechCrunch](https://techcrunch.com/2023/09/18/microsoft-ai-researchers-accidentally-exposed-terabytes-of-internal-sensitive-data/) - -5. **GitHub** - - _Incident_: GitHub discovered its RSA SSH host private key was briefly exposed in a public repository and rotated it out of caution. - - _Sources_: [GitHub Blog](https://github.blog/news-insights/company-news/we-updated-our-rsa-ssh-host-key/) - -Left unchecked, leaked secrets can lead to unauthorized access, pivoting within your environment, regulatory fines, and brand‑damaging incident response costs. - -# Benchmark Results - -See ([docs/COMPARISON.md](docs/COMPARISON.md)) +**Key areas of evolution** +- **Live validation** of detected secrets directly within rules +- **Hundreds of new built-in rules** and an expanded YAML rule schema +- **Baseline management** to suppress known findings over time +- **Tree-sitter parsing** layered on Hyperscan for language-aware detection +- **More scan targets** (GitLab, Bitbucket, Gitea, Jira, Confluence, Slack, S3, GCS, Docker, Hugging Face, etc.) +- **Compressed Files** scanning support added +- **New storage model** (in-memory + Bloom filter, replacing SQLite) +- **Unified workflow** with JSON/BSON/SARIF outputs +- **Cross-platform builds** for Linux, macOS, and Windows # Roadmap - More rules -- Auto-updater -- Packages for Linux (deb, rpm) -- Please file a [feature request](https://github.com/mongodb/kingfisher/issues) if you have specific features you'd like added +- More targets +- Please file a [feature request](https://github.com/mongodb/kingfisher/issues), or open a PR, if you have features you'd like added # License diff --git a/buildwin.bat b/buildwin.bat index 1a1f77e..7311058 100644 --- a/buildwin.bat +++ b/buildwin.bat @@ -1,120 +1,133 @@ @echo off -REM This script builds a Windows x64 release binary and creates a tarball with checksum. -REM It requires vcpkg to be installed at root of C: drive (https://github.com/microsoft/vcpkg). -REM This script will install Rust (using chocolatey) if it is not already installed. -REM -REM Call with -force to clone and bootstrap vcpkg if it is not found -REM +REM ============================================================================ +REM buildwin.bat — Windows x64 release build + archive for Kingfisher +REM - Forces a single vcpkg root (C:\vcpkg) and avoids VS-integrated vcpkg +REM - Installs Hyperscan for x64-windows-static into that root +REM - Verifies hs.lib is present before building, then builds & packages +REM ============================================================================ setlocal - -REM Set your Cargo project name manually here if desired: set "PROJECT_NAME=kingfisher" +set "FORCE_VCPKG=0" +if /I "%~1"=="-force" set "FORCE_VCPKG=1" +if "%VCPKG_TRIPLET%"=="" set "VCPKG_TRIPLET=x64-windows-static" -REM Optional check for OS: +REM --- Ensure Windows --- if NOT "%OS%"=="Windows_NT" ( - echo This script must be run on Windows. - exit /b 1 + echo This script must be run on Windows. + exit /b 1 ) + +REM --- Find MSVC / init toolchain --- if "%VCINSTALLDIR%"=="" ( - echo VCINSTALLDIR not set - attempting auto-detection… - for %%P in ( - "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC" - "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC" - "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC" - "C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\VC" - ) do ( - if exist "%%~P\Auxiliary\Build\vcvars64.bat" ( - set "VCINSTALLDIR=%%~P" - echo Found Visual C++ Build Tools at: %%~P - goto :vc_found - ) + echo VCINSTALLDIR not set - attempting auto-detection… + for %%P in ( + "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC" + "C:\Program Files\Microsoft Visual Studio\2022\Professional\VC" + "C:\Program Files\Microsoft Visual Studio\2022\Community\VC" + "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC" + "C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\VC" + ) do ( + if exist "%%~P\Auxiliary\Build\vcvars64.bat" ( + set "VCINSTALLDIR=%%~P" + echo Found Visual C++ Build Tools at: %%~P + goto :vc_found ) - echo ERROR: Could not find a suitable Visual Studio installation. - echo Install “Desktop development with C++” or set VCINSTALLDIR. - exit /b 1 + ) + echo ERROR: Could not find a suitable Visual Studio installation. + echo Install "Desktop development with C++" or set VCINSTALLDIR. + exit /b 1 ) :vc_found - -REM Strip trailing backslash if present if "%VCINSTALLDIR:~-1%"=="\" set "VCINSTALLDIR=%VCINSTALLDIR:~0,-1%" - echo Initialising MSVC environment… call "%VCINSTALLDIR%\Auxiliary\Build\vcvars64.bat" || ( - echo ERROR: Failed to initialise MSVC toolchain. - exit /b 1 + echo ERROR: Failed to initialise MSVC toolchain. + exit /b 1 ) -REM Locate vcpkg.exe -where vcpkg.exe >nul 2>nul -if %ERRORLEVEL% NEQ 0 ( - if exist "%HOMEDRIVE%\vcpkg\vcpkg.exe" ( - set "VCPKG_EXE=%HOMEDRIVE%\vcpkg\vcpkg.exe" - echo Found vcpkg at: %VCPKG_EXE% - ) else ( - if "%~1"=="-force" ( - echo Cloning and bootstrapping vcpkg... - if exist "%HOMEDRIVE%\vcpkg" ( - rmdir /s /q "%HOMEDRIVE%\vcpkg" - ) - git clone https://github.com/microsoft/vcpkg.git "%HOMEDRIVE%\vcpkg" - pushd "%HOMEDRIVE%\vcpkg" - dir - call .\bootstrap-vcpkg.bat - set "VCPKG_EXE=%CD%\vcpkg.exe" - popd - echo Installed vcpkg at: %VCPKG_EXE% - ) else ( - echo ERROR: vcpkg not found. Please install it or re-run script with -force. - exit /b 1 - ) - ) -) else ( - for /f "tokens=*" %%i in ('where vcpkg.exe') do ( - set "VCPKG_EXE=%%i" - goto :found_vcpkg - ) - :found_vcpkg - echo Found vcpkg at: %VCPKG_EXE% -) +REM --- CRITICAL: Force our own vcpkg root (ignore VS-integrated vcpkg) ---- +set "VCPKG_ROOT=C:\vcpkg" +set "VCPKG_DISABLE_METRICS=1" -REM Check if LOCALAPPDATA starts with a drive letter, if not set it to APPDATA +REM --- Ensure LOCALAPPDATA sane for tools that use it --- if /I not "%LOCALAPPDATA:~1,1%"==":" ( - echo LOCALAPPDATA does not start with a drive letter. Setting it to APPDATA. - set "LOCALAPPDATA=%APPDATA%" + echo LOCALAPPDATA not drive-qualified; using APPDATA instead. + set "LOCALAPPDATA=%APPDATA%" ) -echo Installing hyperscan via vcpkg... -set -"%HOMEDRIVE%\vcpkg\vcpkg.exe" install hyperscan:x64-windows -set "LIBHS_NO_PKG_CONFIG=1" +REM --- Find/Install vcpkg into C:\vcpkg --- +call :ensure_vcpkg || exit /b 1 +echo Using vcpkg root: "%VCPKG_ROOT%" +echo vcpkg executable: "%VCPKG_EXE%" -REM Point vectorscan-rs-sys at the Hyperscan install from vcpkg -set "HYPERSCAN_ROOT=%HOMEDRIVE%\vcpkg\installed\x64-windows" -set "LIB=%HYPERSCAN_ROOT%\lib;%LIB%" +REM --- Install Hyperscan into THIS root --- +echo Installing Hyperscan (%VCPKG_TRIPLET%) via vcpkg... +pushd "%VCPKG_ROOT%" || ( + echo ERROR: Cannot cd into "%VCPKG_ROOT%". + exit /b 1 +) +"%VCPKG_EXE%" --vcpkg-root "%VCPKG_ROOT%" install hyperscan:%VCPKG_TRIPLET% || ( + echo ERROR: vcpkg install failed. + popd + exit /b 1 +) +popd + +REM --- Point build to installed Hyperscan --- +set "LIBHS_NO_PKG_CONFIG=1" +set "HYPERSCAN_ROOT=%VCPKG_ROOT%\installed\%VCPKG_TRIPLET%" +set "HS_LIB_DIR=%HYPERSCAN_ROOT%\lib" +set "LIB=%HS_LIB_DIR%;%LIB%" set "INCLUDE=%HYPERSCAN_ROOT%\include;%INCLUDE%" -REM Check for Rust, install if missing +REM Verify hs.lib (or libhs.lib) +set "HS_LIB_FILE=%HS_LIB_DIR%\hs.lib" +if not exist "%HS_LIB_FILE%" if exist "%HS_LIB_DIR%\libhs.lib" set "HS_LIB_FILE=%HS_LIB_DIR%\libhs.lib" + +echo. +echo [DIAG] HYPERSCAN_ROOT = %HYPERSCAN_ROOT% +echo [DIAG] HS_LIB_DIR = %HS_LIB_DIR% +dir /b "%HS_LIB_DIR%\hs.lib" 2>nul +dir /b "%HS_LIB_DIR%\libhs.lib" 2>nul + +if not exist "%HS_LIB_FILE%" ( + echo ERROR: Hyperscan library not found under "%HS_LIB_DIR%". + echo Check that hyperscan:%VCPKG_TRIPLET% installed correctly. + exit /b 1 +) + +REM --- Ensure Rust/CMake present (fallback for local runs) --- where rustc.exe >nul 2>nul if %ERRORLEVEL% NEQ 0 ( - echo Installing Rust... - choco install rust-ms -y - choco install cmake -y --installargs "ADD_CMAKE_TO_PATH=System" - call refreshenv - + echo Installing Rust via Chocolatey... + choco install rust-ms -y + choco install cmake -y --installargs "ADD_CMAKE_TO_PATH=System" + call refreshenv ) else ( - echo Rust is already installed. + echo Rust is already installed. ) -echo Building for Windows x64... +REM --- Build (static CRT) --- +if "%RUSTFLAGS%"=="" ( + set "RUSTFLAGS=-C target-feature=+crt-static" +) else ( + echo Using existing RUSTFLAGS: %RUSTFLAGS% +) + +echo. +echo Building static Windows x64 binary... cargo build --release --target x86_64-pc-windows-msvc || ( - echo Cargo build failed. - exit /b 1 + echo Cargo build failed. + exit /b 1 ) +REM --- Package & checksums --- echo Generating CHECKSUM.txt... powershell -Command ^ - "Get-FileHash .\target\x86_64-pc-windows-msvc\release\%PROJECT_NAME%.exe -Algorithm SHA256 | Out-File .\target\x86_64-pc-windows-msvc\release\CHECKSUM.txt" + "$h=Get-FileHash '.\target\x86_64-pc-windows-msvc\release\%PROJECT_NAME%.exe' -Algorithm SHA256;" ^ + "$l='{0} {1}' -f $h.Hash,(Split-Path -Leaf $h.Path);" ^ + "Set-Content '.\target\x86_64-pc-windows-msvc\release\CHECKSUM.txt' $l" if not exist "target\release" mkdir "target\release" copy /Y "target\x86_64-pc-windows-msvc\release\%PROJECT_NAME%.exe" "target\release\" >nul @@ -126,15 +139,60 @@ if exist "%PROJECT_NAME%-windows-x64.zip" del /f /q "%PROJECT_NAME%-windows-x64. powershell -Command "Compress-Archive -Path '%PROJECT_NAME%.exe','CHECKSUM-windows-x64.txt' -DestinationPath '%PROJECT_NAME%-windows-x64.zip' -Force" if exist "%PROJECT_NAME%-windows-x64.zip" ( - REM -- append the ZIP’s SHA-256 to the existing checksum file ---- - certutil -hashfile "%PROJECT_NAME%-windows-x64.zip" SHA256 >> "CHECKSUM-windows-x64.txt" - echo Created: %PROJECT_NAME%-windows-x64.zip + powershell -Command ^ + "$h=Get-FileHash '.\%PROJECT_NAME%-windows-x64.zip' -Algorithm SHA256;" ^ + "$l='{0} {1}' -f $h.Hash,(Split-Path -Leaf $h.Path);" ^ + "Add-Content '.\CHECKSUM-windows-x64.txt' $l" + echo Created: %PROJECT_NAME%-windows-x64.zip ) else ( - echo ERROR: Archive not created. + echo ERROR: Archive not created. + exit /b 1 ) echo Archives in target\release: dir /b *.zip 2>nul || echo None found. -endlocal +goto :eof + +REM ====================== helpers ============================================ +:ensure_vcpkg +REM If vcpkg.exe already exists under our chosen root, use it +if exist "%VCPKG_ROOT%\vcpkg.exe" ( + set "VCPKG_EXE=%VCPKG_ROOT%\vcpkg.exe" + echo Found vcpkg at: %VCPKG_EXE% + exit /b 0 +) + +REM If on PATH (and not the VS one), still force install into C:\vcpkg +where vcpkg.exe >nul 2>nul +if %ERRORLEVEL%==0 ( + for /f "tokens=*" %%i in ('where vcpkg.exe') do ( + set "VCPKG_EXE=%%i" + ) + echo Found vcpkg on PATH: %VCPKG_EXE% +) + +REM Clone/bootstrap into our root if missing +if not exist "%VCPKG_ROOT%" mkdir "%VCPKG_ROOT%" +if "%FORCE_VCPKG%"=="1" if exist "%VCPKG_ROOT%" rmdir /s /q "%VCPKG_ROOT%" & mkdir "%VCPKG_ROOT%" + +if not exist "%VCPKG_ROOT%\vcpkg.exe" ( + echo Cloning and bootstrapping vcpkg into "%VCPKG_ROOT%"... + git clone https://github.com/microsoft/vcpkg.git "%VCPKG_ROOT%" || ( + echo ERROR: Failed to clone vcpkg repository. + exit /b 1 + ) + pushd "%VCPKG_ROOT%" + call .\bootstrap-vcpkg.bat || ( + popd + echo ERROR: Failed to bootstrap vcpkg. + exit /b 1 + ) + set "VCPKG_EXE=%VCPKG_ROOT%\vcpkg.exe" + popd + echo Installed vcpkg at: %VCPKG_EXE% +) else ( + set "VCPKG_EXE=%VCPKG_ROOT%\vcpkg.exe" +) + exit /b 0 diff --git a/data/rules/adafruitio.yml b/data/rules/adafruitio.yml index 4982d86..a3e4408 100644 --- a/data/rules/adafruitio.yml +++ b/data/rules/adafruitio.yml @@ -2,11 +2,13 @@ rules: - name: Adafruit IO Key id: kingfisher.adafruitio.1 pattern: | - (?xi) + (?x) \b ( aio_ - [A-Z0-9]{28} + [a-zA-Z]{4} + [0-9]{2} + [a-zA-Z0-9]{22} ) \b min_entropy: 3.5 @@ -28,4 +30,4 @@ rules: type: StatusMatch - type: WordMatch words: - - '"username"' \ No newline at end of file + - '"username"' diff --git a/data/rules/adobe.yml b/data/rules/adobe.yml index 03b54d0..ec5584c 100644 --- a/data/rules/adobe.yml +++ b/data/rules/adobe.yml @@ -6,11 +6,13 @@ rules: \b adobe (?:.|[\n\r]){0,32}? - \b + \b ( [A-F0-9]{32} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 examples: - adobeKey = 1a2b3c4d5e6f7890abcdef1234567890 @@ -44,6 +46,7 @@ rules: \b adobe (?:.|[\n\r]){0,64}? + \b ( [a-z0-9]{12} ) @@ -60,7 +63,7 @@ rules: ( p8e-[A-Z0-9-]{32} ) - (?:[^A-Z0-9-]|$) + (?:[^A-Z0-9-]) min_entropy: 3.5 examples: - | diff --git a/data/rules/age.yml b/data/rules/age.yml index dfd5a44..daf1430 100644 --- a/data/rules/age.yml +++ b/data/rules/age.yml @@ -2,12 +2,14 @@ rules: - name: Age Recipient (X25519 public key) id: kingfisher.age.1 pattern: | - (?xi) - \b + (?x) ( - age1[0-9a-z]{58} + age1[qpzry9x8gf2tvdw0s3jn54khce6mua7l]{58} ) \b + pattern_requirements: + min_digits: 2 + min_lowercase: 1 min_entropy: 3.3 confidence: medium examples: @@ -20,12 +22,10 @@ rules: - name: Age Identity (X22519 secret key) id: kingfisher.age.2 pattern: | - (?xi) - \b + (?x) ( - AGE-SECRET-KEY-1[0-9A-Z]{58} + AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58} ) - \b min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/ai21.yml b/data/rules/ai21.yml new file mode 100644 index 0000000..899e91c --- /dev/null +++ b/data/rules/ai21.yml @@ -0,0 +1,48 @@ +rules: + - name: AI21 Studio API Key + id: kingfisher.ai21studio.1 + pattern: | + (?xi) + \b + ai21 + (?:.|[\n\r]){0,32}? + \b + ( + [0-9a-f]{8} + - + [0-9a-f]{4} + - + [0-9a-f]{4} + - + [0-9a-f]{4} + - + [0-9a-f]{12} + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 + min_entropy: 3.2 + confidence: medium + examples: + - ai21 = 90cd6930-a9ae-4f15-8da0-dc1bbcd814b9 + - 'ai21_key: befa7ec1-1129-4713-8e92-bb53d1a4f632' + - ai21_token = ec2e14e9-0309-459b-ba76-1e59e1f42b87 + references: + - https://docs.ai21.com/reference/authentication + - https://docs.ai21.com/reference/manage-library-ref/list-library-files + + validation: + type: Http + content: + request: + method: GET + url: https://api.ai21.com/studio/v1/library/files + headers: + Authorization: Bearer {{ TOKEN }} + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] \ No newline at end of file diff --git a/data/rules/airbrake.yml b/data/rules/airbrake.yml index 9d55e24..f261ffb 100644 --- a/data/rules/airbrake.yml +++ b/data/rules/airbrake.yml @@ -5,14 +5,16 @@ rules: (?xi) \b airbrake - (?:.|[\n\r]){0,16}? - (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) - (?:.|[\n\r]){0,16}? + (?:.|[\n\r]){0,32}? ( [A-Z0-9-]{40} ) - \b - min_entropy: 4.5 + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 + min_entropy: 4.0 confidence: medium examples: - airbrake secretKey= a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8s9T0 diff --git a/data/rules/airtable.yml b/data/rules/airtable.yml index 7b8b037..3f77c16 100644 --- a/data/rules/airtable.yml +++ b/data/rules/airtable.yml @@ -2,15 +2,19 @@ rules: - name: Airtable Personal Access Token id: kingfisher.airtable.1 pattern: | - (?xi) + (?x) \b ( pat - [a-z0-9]{14} + [A-Za-z0-9]{14} \. - [a-z0-9]{62,66} + [a-f0-9]{64} ) - \b + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.3 confidence: medium examples: @@ -37,7 +41,6 @@ rules: id: kingfisher.airtable.2 pattern: | (?xi) - \b ( [A-Z0-9]+\.v1\.[A-Z0-9_-]+\.[a-f0-9]+ ) diff --git a/data/rules/aiven.yml b/data/rules/aiven.yml index 0f4c727..0b98079 100644 --- a/data/rules/aiven.yml +++ b/data/rules/aiven.yml @@ -2,15 +2,18 @@ rules: - name: Aiven API Key id: kingfisher.aiven.1 pattern: | - (?xi) + (?xi) aiven (?:.|[\n\r]){0,32}? - (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) - (?:.|[\n\r]){0,32}? \b ( [a-z0-9/+=]{372} ) + (?:[^A-Za-z0-9/+=]) + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/alchemy.yml b/data/rules/alchemy.yml new file mode 100644 index 0000000..75d72e9 --- /dev/null +++ b/data/rules/alchemy.yml @@ -0,0 +1,46 @@ +rules: + - name: Alchemy API Key + id: kingfisher.alchemy.1 + pattern: | + (?xi) + \balchemy + (?:.|[\n\r]){0,96}? + (?: + /v2/ + | + api[_-]?key|key|token|secret|url|endpoint|rpc + ) + (?:.|[\n\r]){0,96}? + \b + ( + [A-Za-z0-9_-]{24,64} + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.5 + confidence: medium + examples: + - alchemy_key="PajdHzB75s1V_7aldcQ6XbodqDCWMC7m" + - https://eth-mainnet.alchemyapi.io/v2/PajdHzB75s1V_7aldcQ6XbodqDCWMC7m + - https://eth-goerli.alchemyapi.io/v2/AGtF3w2AsccY_bfsdDleaVRehW2xGS7W + references: + - https://www.alchemy.com/rpc/ethereum + - https://www.alchemy.com/docs/reference/nft-api-endpoints/nft-api-endpoints/nft-ownership-endpoints/get-nf-ts-for-owner-v-3 + validation: + type: Http + content: + request: + method: GET + url: "https://eth-mainnet.g.alchemy.com/nft/v3/{{ TOKEN }}/getNFTsForOwner?owner=0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045" + headers: + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"ownedNfts"'] + - type: WordMatch + negative: true + words: ['"error"'] diff --git a/data/rules/algolia.yml b/data/rules/algolia.yml index dd65cf7..150cda9 100644 --- a/data/rules/algolia.yml +++ b/data/rules/algolia.yml @@ -5,11 +5,13 @@ rules: (?xi) algolia (?:.|[\n\r]){0,32}? - \b ( [a-z0-9]{32} ) \b + pattern_requirements: + min_digits: 2 + min_lowercase: 1 min_entropy: 3.5 confidence: medium examples: @@ -38,11 +40,12 @@ rules: (?xi) algolia (?:.|[\n\r]){0,16}? - \b ( [A-Z0-9]{10} ) - \b + \b + pattern_requirements: + min_digits: 2 min_entropy: 2.0 visible: false confidence: medium diff --git a/data/rules/alibaba.yml b/data/rules/alibaba.yml index b4807ec..990ab91 100644 --- a/data/rules/alibaba.yml +++ b/data/rules/alibaba.yml @@ -3,11 +3,14 @@ rules: id: kingfisher.alibabacloud.1 pattern: | (?xi) - \b ( LTAI[a-z0-9]{17,21} ) \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 4.0 confidence: medium visible: false @@ -21,11 +24,9 @@ rules: \b alibaba (?:.|[\n\r]){0,32}? - \b ( [a-z0-9]{30} ) - \b min_entropy: 4.2 confidence: medium examples: diff --git a/data/rules/anthropic.yml b/data/rules/anthropic.yml index 0bf11fa..860de92 100644 --- a/data/rules/anthropic.yml +++ b/data/rules/anthropic.yml @@ -2,23 +2,22 @@ rules: - name: Anthropic API Key id: kingfisher.anthropic.1 pattern: | - (?xi) - \b - ( + (?xi) + ( sk-ant-api \d{2,4} - [\w\-]{93} AA - ) - \b + ) + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.3 confidence: medium examples: - sk-ant-api668-Clm512odot9WDD7itfUU9R880nefA1EtYZDbpE-C9b0XQEWpqFKf9DQUo03vOfXl16oSmyar1CLF1SzV3YzpZJ6bahcpLAA - categories: - - api - - secret references: - https://docs.anthropic.com/claude/reference/authentication validation: @@ -46,5 +45,5 @@ rules: - report_response: true - type: WordMatch words: - - '"type":"invalid_request_error"' + - '"type":"message"' url: https://api.anthropic.com/v1/messages \ No newline at end of file diff --git a/data/rules/anypoint.yml b/data/rules/anypoint.yml index 4b3b8ab..d7d420b 100644 --- a/data/rules/anypoint.yml +++ b/data/rules/anypoint.yml @@ -18,8 +18,12 @@ rules: [0-9a-z]{4} - [0-9a-z]{12} - ) - \b + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/apify.yml b/data/rules/apify.yml new file mode 100644 index 0000000..1cf959c --- /dev/null +++ b/data/rules/apify.yml @@ -0,0 +1,38 @@ +rules: + - name: Apify API Token + id: kingfisher.apify.1 + pattern: | + (?xi) + ( + apify_api_[A-Z0-9]{34,38} + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 + confidence: medium + min_entropy: 3.5 + validation: + type: Http + content: + request: + method: GET + url: "https://api.apify.com/v2/users/me" + headers: + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"data"' + - '"username"' + match_all_words: true + references: + - https://docs.gitguardian.com/secrets-detection/secrets-detection-engine/detectors/specifics/apify_token + - https://docs.apify.com/api/v2#/reference/users/user-object/get-user-public-profile-or-me + - https://docs.apify.com/api/v2/users-me-get + examples: + - "apify_api_NcjXcxEz2XL1irjppyWSHvjghalQOd1LXOHv" + - "apify_api_9uyewBxQUF1EXWdKVc4lNaTSM461Ls4oQouz" \ No newline at end of file diff --git a/data/rules/apollo.yml b/data/rules/apollo.yml new file mode 100644 index 0000000..14f10d1 --- /dev/null +++ b/data/rules/apollo.yml @@ -0,0 +1,52 @@ +rules: + - name: Apollo API Key + id: kingfisher.apollo.1 + pattern: | + (?xi) + \b + apollo + (?:.|[\n\r]){0,16}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9_-]{22} + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 + min_entropy: 3.0 + confidence: medium + examples: + - 'APOLLO_API_KEY="ZNh-14foqIiscbz24oKwww"' + - apollo_key=8ku3EoDJxz8fOSCdxYozdA + - apollo.io api_key oD8GCL8MNZIyg0tzeSDuhw + references: + - https://docs.apollo.io/reference/people-api-search + validation: + type: Http + content: + request: + method: POST + url: "https://api.apollo.io/api/v1/mixed_people/api_search" + headers: + accept: "application/json" + content-type: "application/json" + x-api-key: "{{ TOKEN }}" + body: | + {"page":1,"per_page":1} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200, 403] + - type: WordMatch + words: + - '"total_entries"' + - '"API_INACCESSIBLE"' + match_all_words: false + - type: WordMatch + negative: true + words: + - '"Invalid access credentials"' diff --git a/data/rules/artifactory.yml b/data/rules/artifactory.yml index f53174e..0c63435 100644 --- a/data/rules/artifactory.yml +++ b/data/rules/artifactory.yml @@ -8,6 +8,10 @@ rules: AKC[A-Z0-9]{64,74} ) \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.5 confidence: medium examples: @@ -36,21 +40,55 @@ rules: - name: Artifactory JFrog URL id: kingfisher.artifactory.2 pattern: | - (?xi) - \b - ( + (?xi) + \b + ( [a-z0-9] (?: [a-z0-9\-]{0,61} [a-z0-9] )? \.jfrog\.io - ) - \b - min_entropy: 3.5 + ) + \b + min_entropy: 2.5 visible: false confidence: medium examples: - mycompany.jfrog.io - my-company-name.jfrog.io - - a.jfrog.io \ No newline at end of file + - a.jfrog.io + + - name: Artifactory Identity Reference Token + id: kingfisher.artifactory.3 + pattern: | + (?xi) + \b + ( + cmVmd[A-Z0-9]{59} + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 + min_entropy: 3.5 + confidence: medium + examples: + - export HOMEBREW_ARTIFACTORY_API_REFERENCE_TOKEN=cmVmdawqkxT1EE4bMepW0zmTVmBYYdv264WVufgipR9CQAW2xwSnY4CTKap8H5u0 + validation: + type: Http + content: + request: + headers: + Authorization: 'Bearer {{ TOKEN }}' + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://{{ JFROGURL }}/artifactory/api/repositories + depends_on_rule: + - rule_id: "kingfisher.artifactory.2" + variable: JFROGURL diff --git a/data/rules/asana.yml b/data/rules/asana.yml index 0824711..64a7fd3 100644 --- a/data/rules/asana.yml +++ b/data/rules/asana.yml @@ -6,13 +6,13 @@ rules: \b asana (?:.|[\n\r]){0,32}? - (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) - (?:.|[\n\r]){0,32}? \b ( [0-9]{16} ) - \b + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: @@ -32,35 +32,35 @@ rules: ( [a-z0-9]{30,40} ) - \b + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.5 confidence: medium examples: - "asana :'20c2F0d03201af478ca1aBE9515A1A4FEfb'" - ASANA_PAT = 1234567890abcdef1234567890abcdef12 - - name: Asana OAuth / Personal Access Token + - name: Asana OAuth / Personal Access Token (Legacy) id: kingfisher.asana.3 pattern: | - (?xi) - \b + (?xi) + \b asana (?:.|[\n\r]){0,64}? - \b - ( - [01]{1,} - \/ - [0-9a-f]{16,32} - (?: - : - [a-z0-9]{32,64} - )? - ) - \b + \b + ( + 0/ + [a-f0-9]{32} + ) + \b + pattern_requirements: + min_digits: 4 min_entropy: 3.5 confidence: medium examples: - - asana_pat = 1/1248440223456784:d3d7e52e5c4a5d4c9bc424d2d882324d - asana token = 0/d6f1e29e5b4b4d8c9bb419b2d882154d categories: - api @@ -83,4 +83,94 @@ rules: - 'data:' - email - name - url: https://app.asana.com/api/1.0/users/me \ No newline at end of file + url: https://app.asana.com/api/1.0/users/me + + - name: Asana OAuth / Personal Access Token (V1) + id: kingfisher.asana.4 + pattern: | + (?xi) + \b + asana + (?:.|[\n\r]){0,64}? + \b + ( + 1/ + [0-9]{14,16} + : + [a-f0-9]{32} + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.5 + confidence: medium + examples: + - asana_pat = 1/1248440223456784:d3d7e52e5c4a5d4c9bc424d2d882324d + categories: + - api + - key + - asana + references: + - https://developers.asana.com/docs/personal-access-token#example + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + method: GET + response_matcher: + - report_response: true + - match_all_words: true + type: WordMatch + words: + - 'data:' + - email + - name + url: https://app.asana.com/api/1.0/users/me + + - name: Asana OAuth / Personal Access Token (V2) + id: kingfisher.asana.5 + pattern: | + (?xi) + \b + asana + (?:.|[\n\r]){0,64}? + \b + ( + 2/ + [0-9]{16} + / + [0-9]{16} + : + [a-f0-9]{32} + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.5 + confidence: medium + examples: + - ASANA_TOKEN = "2/1208779539612523/1208824174176866:99d6decca6ce6ef503bf0c5bca554e1a" + categories: + - api + - key + - asana + references: + - https://developers.asana.com/docs/personal-access-token#example + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + method: GET + response_matcher: + - report_response: true + - match_all_words: true + type: WordMatch + words: + - 'data:' + - email + - name + url: https://app.asana.com/api/1.0/users/me diff --git a/data/rules/assemblyai.yml b/data/rules/assemblyai.yml new file mode 100644 index 0000000..38136d0 --- /dev/null +++ b/data/rules/assemblyai.yml @@ -0,0 +1,41 @@ +rules: + - name: AssemblyAI API Key + id: kingfisher.assemblyai.1 + pattern: | + (?xi) + \b + assemblyai + (?:.|[\n\r]){0,32}? + \b + ( + [0-9a-z]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 1 + min_entropy: 3.0 + confidence: medium + examples: + - assemblyai = fa0ed91518b345468f9df7570f31f18a + - assemblyai_token = a741b921ae1f4446826a784726b6a71a + references: + - https://www.assemblyai.com/docs/api-reference/overview + - https://www.assemblyai.com/docs/api-reference/transcripts/list + + validation: + type: Http + content: + request: + method: GET + url: https://api.assemblyai.com/v2/transcript?limit=1 + headers: + Authorization: '{{ TOKEN }}' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: ['"page_details"', '"transcripts"'] diff --git a/data/rules/atlassian.yml b/data/rules/atlassian.yml index 6d4ac9c..a92b279 100644 --- a/data/rules/atlassian.yml +++ b/data/rules/atlassian.yml @@ -6,13 +6,14 @@ rules: \b atlassian (?:.|[\n\r]){0,32}? - (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) - (?:.|[\n\r]){0,32}? \b ( [a-z0-9]{24} ) - \b + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 1 min_entropy: 3.5 confidence: medium examples: @@ -37,4 +38,65 @@ rules: - type: WordMatch words: - "Unauthorized" - negative: true \ No newline at end of file + negative: true + # - name: Atlassian Organization ID + # id: kingfisher.atlassian.2 + # pattern: | + # (?xi) + # admin\.atlassian\.com + # /o/ + # ( + # [0-9a-f]{8} + # - + # [0-9a-f]{4} + # - + # [0-9a-f]{4} + # - + # [0-9a-f]{4} + # - + # [0-9a-f]{12} + # ) + # min_entropy: 2.0 + # confidence: medium + # visible: false + # examples: + # - https://admin.atlassian.com/o/12345678-9abc-def0-1234-56789abcdef0/api-keys + - name: Atlassian Admin API Key + id: kingfisher.atlassian.3 + pattern: | + (?x) + (?:atlassian|api\.atlassian\.com) + (?:.|[\n\r]){0,128}? + \b + ( + AT + [A-Za-z0-9_\-=]{60,260} + ) + \b + min_entropy: 3.8 + confidence: medium + examples: + - | + # Example usage calling the Atlassian admin APIs + curl --request GET \ + 'https://api.atlassian.com/admin/v1/orgs' \ + --header 'Authorization: Bearer ATEXAMPLE1234567890abcdefghijklmnopqrstuvwxyz_-ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789foobarAA11BB22' + references: + - https://developer.atlassian.com/cloud/admin/organization/rest/ + - https://developer.atlassian.com/cloud/admin/api-access/rest/ + - https://support.atlassian.com/organization-administration/docs/manage-an-organization-with-the-admin-apis/ + - https://community.atlassian.com/learning/lesson/what-are-admin-apis + validation: + type: Http + content: + request: + method: GET + url: https://api.atlassian.com/admin/v1/orgs + headers: + Authorization: "Bearer {{ TOKEN }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid \ No newline at end of file diff --git a/data/rules/auth0.yml b/data/rules/auth0.yml index 39a0b43..f77497f 100644 --- a/data/rules/auth0.yml +++ b/data/rules/auth0.yml @@ -10,7 +10,10 @@ rules: ( [a-z0-9_-]{32,60} ) - \b + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 1 min_entropy: 3.5 confidence: medium visible: false diff --git a/data/rules/authress.yml b/data/rules/authress.yml new file mode 100644 index 0000000..1df4922 --- /dev/null +++ b/data/rules/authress.yml @@ -0,0 +1,34 @@ +rules: + - name: Authress Service Client Access Key + id: kingfisher.authress.1 + pattern: | + (?xi) + ( + (?:sc|ext|scauth|authress)_[a-z0-9]{5,30}\.[a-z0-9]{4,6}\.acc[_-][a-z0-9-]{10,32}\.[a-z0-9+/_=-]{30,120} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 1 + confidence: medium + min_entropy: 4.0 + validation: + type: Http + content: + request: + method: GET + url: "https://api.authress.io/v1/users/me" + headers: + Authorization: "Bearer {{TOKEN}}" + response_matcher: + - report_response: true + - type: JsonValid + - type: WordMatch + words: + - '"Unauthorized"' + negative: true + references: + - https://authress.io/knowledge-base/docs/authorization/service-clients/access-keys/ + - https://authress.io/knowledge-base/docs/usage-guides/api-keys-as-a-service-setup/ + examples: + - "sc_a6DTktFwMEvh87xstYV1BXl.ihwj.acc-0xd1a47h1rr0f.MC4CAQAwBQYDKAVwBCIEIB1wYB62EK24FKxEPHbW0ishcstwp2qs30uLXdWgu4V0" diff --git a/data/rules/aws.yml b/data/rules/aws.yml index b324dcc..6015285 100644 --- a/data/rules/aws.yml +++ b/data/rules/aws.yml @@ -3,12 +3,17 @@ rules: id: kingfisher.aws.1 pattern: | (?xi) - \b + \b ( - (?:AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA) - [0-9A-Z]{16} + (?:A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA) + [2-7A-Z]{16} ) - \b + \b + pattern_requirements: + min_digits: 2 + ignore_if_contains: + - "EXAMPLE" + - "TEST" min_entropy: 3.2 visible: false confidence: medium @@ -21,27 +26,28 @@ rules: (?xi) (?: \b - (?:AWS|AMAZON|AMZN|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA) - (?:.|[\n\r]){0,32}? - \b - ( - [A-Z0-9/+=]{40} - ) - \b + (?:AWS|AMAZON|AMZN|A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA) + (?:.|[\n\r]){0,64}? + [^A-Za-z0-9_+!@\#$%^&*()\]./] + ([A-Za-z0-9/+]{40}) + [^A-Za-z0-9_+!@\#$%^&*()\]./] | - \b(?:AWS|AMAZON|AMZN|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA) + \b(?:AWS|AMAZON|AMZN|A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA) (?:.|[\n\r]){0,96}? (?:SECRET|PRIVATE|ACCESS) (?:.|[\n\r]){0,16}? (?:KEY|TOKEN) - (?:.|[\n\r]){0,32}? + (?:.|[\n\r]){0,64}? \b - ( - [A-Z0-9/+=]{40} - ) + ([A-Za-z0-9/+]{40}) \b ) - min_entropy: 4.5 + pattern_requirements: + min_digits: 3 + ignore_if_contains: + - "EXAMPLE" + - "TEST" + min_entropy: 4.0 confidence: medium examples: - foo.backup.archive.aws.secretkey=sBmHlDFrNcsz35N+LRjwlUxF8/wypT4tiJCQ0wP4 @@ -64,10 +70,62 @@ rules: - name: AWS Session Token id: kingfisher.aws.4 pattern: '(?i)(?:aws.?session|aws.?session.?token|aws.?token)["''`]?\s{0,30}(?::|=>|=)\s{0,30}["''`]?([a-z0-9/+=]{16,200})[^a-z0-9/+=]' + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: - | export AWS_ACCESS_KEY_ID="I08BCX2ACV45ED1DOC9J" export AWS_SECRET_ACCESS_KEY="0qk+o7XctJMmG6ydO8537c9+TofLJU1K0PiVBXSg" - export AWS_SESSION_TOKEN="eyJhbGciOiJIUzUxMi53InR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJJMDhCQ1gySkpWNDVFRDFET0M5SiIsImFjciI6Ij53LCJhdWQiOiJhY2NvdW50IiwiYXV0aF90aW1lIjowLCJhenAiOiJtaW5pbyIsImVtYWlsIjoiYWlkYW4uY29wZUBnbWFpbC5jb20iLCJlbWFpbF92ZXJpZmllZCI6ZmFsc2UsImV4cCI6MTU4MDUwMDIzOCwiZmFtaWx5X25hbWUiOiJDb3BlIiwiZ2l2ZW5fbmFtZSI6IkFpZGFuIENvcGUiLCJpYXQiOjE1ODA0OTk5MzgsImlzcyI6Imh0dHBzOi8vYXV0aHN0Zy5wb3BkYXRhLmJjLmNhL2F1dGgvcmVhbG1zL3NhbXBsZSIsImp0aSI6IjU5ZTM5ODAxLWQxMmUtNDVhYS04NmQzLWVhMmNmZDU0NmE2MiIsIm1pbmlvX3BvbGljeSI6ImRhdGFzZXRfMV9ybyIsIm5hbWUiOiJBaWRhbiBDb3BlIENvcGUiLCJuYmYiOjAsInByZWZlcnJlZF91c2VybmFtZSI6ImFjb3BlLTk5LXQwNSIsInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJvZmZsaW5lX2FjY2VzcyIsInVtYV9hdXRob3JpemF0aW9uIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiYWNjb3VudCI6eyJyb2xlcyI6WyJtYW5hZ2UtYWNjb3VudCIsIm1hbmFnZS1hY2NvdW50LWxpbmtzIiwidmlldy1wcm9maWxlIl19fSwic2NvcGUiOiJwcm9maWxlIGVtYWlsIiwic2Vzc2lvbl9zdGF0ZSI6IjcxYjczZWJjLThlMzMtNGMyMi04NmE2LWI0MzhhNDM4ZmI2MiIsInN1YiI6IjVkOTBlOTgzLTA1NDItNDYyYS1hZWIwLWYxZWVmNjcwYzdlNSIsInR5cCI6IkJlYXJlciJ9.J-a9PORJToz7MUrnPQlOywcqtVMNkXy53Gedp_V4PW-Gbf1_BAMjwuw_X7fKRd6hkNfEn43CKKju7muzi_d1Ig" \ No newline at end of file + export AWS_SESSION_TOKEN="eyJhbGciOiJIUzUxMi53InR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJJMDhCQ1gySkpWNDVFRDFET0M5SiIsImFjciI6Ij53LCJhdWQiOiJhY2NvdW50IiwiYXV0aF90aW1lIjowLCJhenAiOiJtaW5pbyIsImVtYWlsIjoiYWlkYW4uY29wZUBnbWFpbC5jb20iLCJlbWFpbF92ZXJpZmllZCI6ZmFsc2UsImV4cCI6MTU4MDUwMDIzOCwiZmFtaWx5X25hbWUiOiJDb3BlIiwiZ2l2ZW5fbmFtZSI6IkFpZGFuIENvcGUiLCJpYXQiOjE1ODA0OTk5MzgsImlzcyI6Imh0dHBzOi8vYXV0aHN0Zy5wb3BkYXRhLmJjLmNhL2F1dGgvcmVhbG1zL3NhbXBsZSIsImp0aSI6IjU5ZTM5ODAxLWQxMmUtNDVhYS04NmQzLWVhMmNmZDU0NmE2MiIsIm1pbmlvX3BvbGljeSI6ImRhdGFzZXRfMV9ybyIsIm5hbWUiOiJBaWRhbiBDb3BlIENvcGUiLCJuYmYiOjAsInByZWZlcnJlZF91c2VybmFtZSI6ImFjb3BlLTk5LXQwNSIsInJlYWxtX2FjY2VzcyI6eyJyb2xlcyI6WyJvZmZsaW5lX2FjY2VzcyIsInVtYV9hdXRob3JpemF0aW9uIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiYWNjb3VudCI6eyJyb2xlcyI6WyJtYW5hZ2UtYWNjb3VudCIsIm1hbmFnZS1hY2NvdW50LWxpbmtzIiwidmlldy1wcm9maWxlIl19fSwic2NvcGUiOiJwcm9maWxlIGVtYWlsIiwic2Vzc2lvbl9zdGF0ZSI6IjcxYjczZWJjLThlMzMtNGMyMi04NmE2LWI0MzhhNDM4ZmI2MiIsInN1YiI6IjVkOTBlOTgzLTA1NDItNDYyYS1hZWIwLWYxZWVmNjcwYzdlNSIsInR5cCI6IkJlYXJlciJ9.J-a9PORJToz7MUrnPQlOywcqtVMNkXy53Gedp_V4PW-Gbf1_BAMjwuw_X7fKRd6hkNfEn43CKKju7muzi_d1Ig" + - name: AWS Bedrock API Key (Long-lived) + id: kingfisher.aws.bedrock.long_lived + pattern: | + (?x) + ( + ABSKQmVkcm9ja0FQSUtleS[A-Za-z0-9+/=]{110} + ) + min_entropy: 3.0 + confidence: medium + examples: + - "ABSKQmVkcm9ja0FQSUtleS1GU9MjAyNTEyMDVUMjE1MTUxWiZYLUFtei1FeHBpcmVzPTQzMjAwJlgtQW16LVNlY3VyaXR5LVRva2VuPUlRb0piM0pwWjJsdVgyVmpFSjclMk" + references: + - https://aws.amazon.com/blogs/security/securing-amazon-bedrock-api-keys-best-practices-for-implementation-and-management/ + - https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys-how.html + validation: + type: Http + content: + request: + method: GET + url: https://bedrock.us-east-1.amazonaws.com/foundation-models + headers: + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - type: StatusMatch + status: [200] + + - name: AWS Bedrock API Key (Short-lived) + id: kingfisher.aws.6 + pattern: | + (?x) + ( + bedrock-api-key-YmVkcm9jay5hbWF6b25hd3MuY29t[A-Za-z0-9+/]+={0,2} + ) + min_entropy: 3.0 + confidence: medium + examples: + - "AWS_BEARER_TOKEN_BEDROCK=bedrock-api-key-YmVkcm9jay5hbWF6b25hd3MuY29tLz9BY3Rpb249Q2FsbFdpdGhCZWFyZXJUb2tlbiZYLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFTSUFWRzRBNFpCSk5YUzRJSEZTJTJGMjAyNTEyMDUlMkZ1cy1lYXN0LTElMkZiZWRyb2NrJTJGYXdzNF9yZXF1ZXN0JlgtQW16LURhdGU9MjAyNTEyMDVUMjE1MTUxWiZYLUFtei1FeHBpcmVzPTQzMjAwJlgtQW16LVNlY3VyaXR5LVRva2VuPUlRb0piM0pwWjJsdVgyVmpFSjclMkYlMkYlMkYlMkYlMkYlMkYlMkYlMkYlMkYlMkZ3RWFDWFZ6TFdWaGMzUXRNU0pITUVVQ0lRQ1Z1dkhPTmZLMHVxY3lPeUhibkJ5SUlRQnl3Sm9RV2VTRko4RlclMkIyNkxGQUlnUXJXSU9SbWd1bUozd3ZxTElMZVJ4SXczYzglMkIxaUJ2U1E0R1d6T21rY2VNcTN3UUlaeEFBR2d3ek5UZ3pOak15TWpBd05UQWlEQ05ZYkhtZVVTOE01Rjl6MGlxOEJOOWg4TEZLd0tqTHJDdzZ1eUVDYTd3YjFDbnpGQndyVlFkSXFPU0ZrNWJGbHg5Sjc0cnJ2TjNCYUZQOHR2S3lQcnJCeUJ3bGU3dTIwRjBXVDJoWHlQVTM4cUtpRDclMkZzaTNydnFkT2ptR3pNdERuazRHbEpEN3ZnM01SMWd3cFZJM2Z5ZjU0WU5aZ0lWcm9RZ1g4UVZ4aGNZeHNuSEx6Y3llelh1aGZWbElRMk1LVXUyOTh0c2NqcnF3aEs3WmMlMkZ0Mjc5TWRvengzVkFveUgzdFpocE9oVHhud1VkMHRtQ3REOU5QNHdIN1pOQjRIR2xaZWtidjBoUGIxV012azlhVGF2QkRUZlFCcERueEFHVG5KbXpicm8lMkJod2M0SDB5Skwwb1lVbGplalB3JTJCRVY4ZlJzU3hrVUliOHVRTWRBNDdhUmFzNGpPWkwzZVRlNTdvUXI1Rlo1ekJLJTJGdzBmc1p3RlY5JTJGMTE5Mzc3S2huSnFPRTMxdjBRJTJGYWV1YVk5YThIZnFVNlZ4MD14cVIyM1VxUExxaUVhUnJiTXlQSjVHRUdNSzk0RG5zMDF5cmFjNzU5UGF2Zko2QnpjaEFPSklJeFdXeXBiYmY4dUJKYTdyTldOQUF6S1R4NHFSVm9VdHljS2txciUyQlFyajZ2b1NNOHBoJTJCRnpZOXFEJTJCaCUyQkNEbkk4M0xMRDRkVnJVN0Jla05QbjNXSFpEN0twRVdVZWJ1UlpoZGVNSVU4R0hVVlpGa3FCV3Q0djk5QVdNdlFydEFJVzlHUWN3UkhZM3FaMFo2ZHI3cHpIOGNoZWRyMWdyJTJGUnBkT3lBdFIlMkZ3OE9HeU1LeklaSzRBdTZVeEhRaGdOVjJKdDh0ZnFVSlNCS281UVhiV1RmakFSNFlQSFcwbEREaEtRTTZYWWJsJTJGY0hSM3pIMG1WMGUyc92OJTJGVTJTc1Q3MVhCb1Z1Y2d3WU56RXFkM2M0ZUZzdjFaelBTQ2lMVWUyaDhPZTI5Q0F2VHF5eEZBTUFaMVpKNyUyRk5MSzVRSldNT09uemNrR09zTUNqQXhOVFdXUXdMUjd5NmR2TlMzQmh6UVlMJTJGeXpJWEdaVnhZYm9mY3IlMkJLbCUyRnVveSUyQkFlWCUyRkxLaXFwWDk5RWc2cSUyQm1tazNIZ1Q0WWNueVU4VW5Ya2FxMUNxcXVFVVBuRllyMklpbE1UYjlIOUVzanJMRDU4TnBhSTB2OENxNUVRQkIlMkZLMUtkMDdzRks5V1B6cTZaeCUyQmZEVjdYZ0NobG41UDZxQjBFJTJGem5QenRTRWNHMlViS0pHaE4yWjZ2TGtQOVU0STJQODk5WFF4enhVSUIxOTAzUWhjcGp3cGRDN2ZZWEZZVkxqS253bTFiRGlMdFIxMTVnbUpoSUVUM3NheE5zUnpSQkIlMkZjWlMwY1FiTm1wUSUyQldrbXo4ekdXUkc1ZTc1cGclMkY1dUVRMW5aN1ZGTk95UTg1M2Jrb0ZLM0lnNzR3MUpPQllPemlYTVI3ZDF6MSUyRkFNa3hQYWFrWE5YWEd2Z3BsaldBYlR1Wm5Jb1N6UFdEcWIlMkZRaFowUWNxM1JaSm1JdUhTd05oaWs2SFJiZ0NvQUlHZ2sxR21iZUZXZDRoZlhVZWNDOUxvcExzRzEzbUklM0QmWC1BbXotU2lnbmF0dXJlPTU4NTk1MjRjN2RlNGZjMWQ1ODlmZmViOTVlYWI5N2NhYjRmNTQyYWY2MmVkOGExMGYyYzlhZDYyZDQ5ZWY3ODkmWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0JlZlcnNpb249MQ==" + references: + - https://docs.aws.amazon.com/bedrock/latest/userguide/api-keys-how.html + validation: + type: Http + content: + request: + method: GET + url: https://bedrock.us-east-1.amazonaws.com/foundation-models + headers: + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - type: StatusMatch + status: [200] \ No newline at end of file diff --git a/data/rules/azure.yml b/data/rules/azure.yml index 277bea7..6348610 100644 --- a/data/rules/azure.yml +++ b/data/rules/azure.yml @@ -8,6 +8,8 @@ rules: (?: AccountKey | SharedAccessKey | SharedSecretValue) \s*=\s* ([^;]{1,100}) (?: ;|$ ) min_entropy: 3.3 + pattern_requirements: + min_digits: 2 confidence: medium examples: - | @@ -27,17 +29,10 @@ rules: "AZURE_STORAGE_CONNECTION_STRING": { "value": "DefaultEndpointsProtocol=https;AccountName=d1biblobstor521;AccountKey=NjEwGHd9+piK+iCi2C2XURWPmeDDjif9UKN1HAszYptL4iQ+yD7/dgjLMZc3VOpURsa53aJ4HZfbVWzL429C5g==;EndpointSuffix=core.windows.net" } - negative_examples: - - 'InstrumentationKey=00000000-0000-0000-0000-000000000000;EndpointSuffix=ai.contoso.com;' - - 'InstrumentationKey=00000000-0000-0000-0000-000000000000;IngestionEndpoint=https://custom.com:111/;LiveEndpoint=https://custom.com:222/;ProfilerEndpoint=https://custom.com:333/;SnapshotEndpoint=https://custom.com:444/;' references: - https://azure.microsoft.com/en-us/blog/windows-azure-web-sites-how-application-strings-and-connection-strings-work/ - https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string - https://learn.microsoft.com/en-us/azure/service-bus-messaging/service-bus-sas#best-practices-when-using-sas - categories: - - api - - fuzzy - - secret - name: Azure App Configuration Connection String id: kingfisher.azure.2 @@ -53,18 +48,10 @@ rules: - 'https://foo-nonprod-appconfig.azconfig.io;Id=ABCD-E6-s0:tl6ABcdefGHi7kLMno/p;Secret=abCD1EF+GHIJxLMnOA53ST8uVWX05zaBCdE/fg9hi4k=' - 'Endpoint=https://appconfig-test01.azconfig.io;Id=09pv-l0-s0:opFCQMC6+9485xJgN5Ws;Secret=GcoEA53t7GLRNJ910M46IrbHO/Vg0tt4HujRdsaCoTY=' - ' private static string appConfigurationConnectionString = "Endpoint=https://appcs-fg-pwc.azconfig.io;Id=pi5x-l9-s0:SZLlhHA53Nz2MpAl04cU;Secret=CQ+mlfQqkzfZv4XA53gigJ/seeXMKwNsqW/rM3wmtuE=";' - negative_examples: - - | - text: - az appconfig feature delete --connection-string Endpoint=https://contoso.azconfig.io;Id=xxx;Secret=xxx --feature color --label MyLabel references: - https://docs.microsoft.com/en-us/azure/azure-app-configuration/ - https://docs.microsoft.com/en-us/azure/azure-app-configuration/howto-best-practices - https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/appconfiguration/azure-appconfiguration/azure/appconfiguration/_utils.py - categories: - - api - - fuzzy - - secret - name: Azure Personal Access Token id: kingfisher.azure.3 @@ -84,4 +71,48 @@ rules: $token = "58oo4mvqr2tpw7b4w3loeckwfu5o6nw3sihfckvlwoxgqimlddza" - | if __name__ == "__main__": - ado_pat = "iyfmob6xjrfmit67anxbot64umfx2clwx7dz5ynxi4q2z3uqegvq" \ No newline at end of file + ado_pat = "iyfmob6xjrfmit67anxbot64umfx2clwx7dz5ynxi4q2z3uqegvq" + - name: Azure Container Registry URL + id: kingfisher.azure.4 + pattern: | + (?xi) + ( + [a-z0-9][a-z0-9-]{1,100}[a-z0-9] + )\.azurecr\.io + confidence: medium + visible: false + min_entropy: 2.0 + examples: + - "myregistry.azurecr.io" + - name: Azure Container Registry Password + id: kingfisher.azure.5 + pattern: | + (?xi) + \b + ( + [A-Z0-9+/]{42}\+ACR[A-Z0-9]{6} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 4.0 + validation: + type: Http + content: + request: + method: GET + url: "https://{{ACR_USERNAME}}.azurecr.io/v2/_catalog" + headers: + Authorization: "Basic {{ ACR_USERNAME | append: ':' | append: TOKEN | b64enc }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + examples: + - "Abcdefghijklmnopqrstuvwxyz123456789012ABCD+ACRefg123" + depends_on_rule: + - rule_id: "kingfisher.azure.4" + variable: ACR_USERNAME + references: + - https://learn.microsoft.com/en-us/azure/container-registry/container-registry-authentication diff --git a/data/rules/azuredevops.yml b/data/rules/azuredevops.yml index 4188999..90fa4e8 100644 --- a/data/rules/azuredevops.yml +++ b/data/rules/azuredevops.yml @@ -1,15 +1,31 @@ rules: - - name: Azure DevOps Personal Access Token + - name: Azure DevOps Organization id: kingfisher.azure.devops.1 pattern: | (?xi) \b - azure - (?:.|[\n\r]){0,32}? + dev\.azure\.com/ ( - [a-z0-9]{75}AZDO[a-z0-9]{5} + [a-z0-9][a-z0-9-]{0,61}[a-z0-9] + ) + confidence: medium + min_entropy: 2.5 + visible: false + examples: + - https://dev.azure.com/contoso + - dev.azure.com/somebody123 + + - name: Azure DevOps Personal Access Token + id: kingfisher.azure.devops.2 + pattern: | + (?xi) + \b + ( + [a-z0-9]{76}AZDO[a-z0-9]{4,5} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3 confidence: medium examples: @@ -17,16 +33,20 @@ rules: references: - https://learn.microsoft.com/en-us/rest/api/azure/devops/profile/profiles/get?view=azure-devops-rest-7.1&tabs=HTTP - https://learn.microsoft.com/en-us/azure/devops/release-notes/2024/general/sprint-241-update + depends_on_rule: + - rule_id: kingfisher.azure.devops.1 + variable: AZURE_DEVOPS_ORG validation: type: Http content: request: headers: Authorization: 'Basic {{ ":" | append: TOKEN | b64enc }}' + Accept: application/json method: GET - url: https://app.vssps.visualstudio.com/_apis/profile/profiles/me?api-version=7.1-preview.1 + url: "https://dev.azure.com/{{ AZURE_DEVOPS_ORG | split: '/' | last }}/_apis/projects?api-version=7.1-preview.1" response_matcher: - report_response: true - type: StatusMatch status: - - 200 + - 200 \ No newline at end of file diff --git a/data/rules/azureopenai.yml b/data/rules/azureopenai.yml index 87e8127..050e7a2 100644 --- a/data/rules/azureopenai.yml +++ b/data/rules/azureopenai.yml @@ -14,6 +14,9 @@ rules: [a-f0-9]{32} ) \b + pattern_requirements: + min_digits: 2 + min_lowercase: 2 min_entropy: 3.5 confidence: medium examples: @@ -46,7 +49,7 @@ rules: (?xi) \b ( - [a-z0-9-]+ + [a-z0-9-]{3,32} \.openai\.azure\.com ) \b diff --git a/data/rules/azuresearchquery.yml b/data/rules/azuresearchquery.yml index 45b84b3..6e37d82 100644 --- a/data/rules/azuresearchquery.yml +++ b/data/rules/azuresearchquery.yml @@ -12,6 +12,10 @@ rules: [0-9A-Z]{52} ) \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 2 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/azurestorage.yml b/data/rules/azurestorage.yml index 8f2921f..57704e3 100644 --- a/data/rules/azurestorage.yml +++ b/data/rules/azurestorage.yml @@ -1,42 +1,59 @@ rules: - name: Azure Storage Account Name - id: kingfisher.azurestorage.name.1 + id: kingfisher.azurestorage.1 pattern: | (?xi) (?: - (?i: - (?:Account|Storage) - (?:[._-]Account)? - [._-]?Name - ) - (?:.|[\n\r]){0,20}? - ([a-z0-9]{3,24}) + # A) Connection string: AccountName= + (?i:AccountName)\s*=\s*([a-z0-9]{3,24})(?:\b|[^a-z0-9]) | - ([a-z0-9]{3,24}) - (?i:\.blob\.core\.windows\.net) - )\b - min_entropy: 2.5 + # B) Blob endpoint URL: .blob.core.windows.net + ([a-z0-9]{3,24})\.blob\.core\.windows\.net\b + | + # C) Explicit KV labels near 'azure storage/account name' with tight separators + \bazure(?:[_\s-]*)(?:storage|account)(?:[_\s-]*)(?:name)\b + [\s:=\"']{0,6} + ([a-z0-9]{3,24})(?:\b|[^a-z0-9]) + | + # D) Explicit KV labels near 'azure storage/account name' with tight separators + (?i:Account[_.-]?Name|Storage[_.-]?(?:Name))(?:.|\s){0,32}?\b([A-Z0-9]{3,32})\b|([A-Z0-9]{3,32})(?i:\.blob\.core\.windows\.net) + ) + min_entropy: 2.0 visible: false confidence: medium examples: - - storage_name=mystorageaccount123 + - AccountName=mystorageaccount - mystorageaccount.blob.core.windows.net - + - azure_storage_name="prodblob2024" - name: Azure Storage Account Key - id: kingfisher.azurestorage.key.1 + id: kingfisher.azurestorage.2 pattern: | (?xi) - (?i:(?:Access|Account|Storage)[_.-]?Key) - (?:.|[\n\r]){0,25}? + azure + (?:.|[\n\r]){0,128}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,128}? + ["':\s=}\]\)] ( - [A-Z0-9+\\/-]{86,88}={0,2} + (?: + [A-Z0-9+\\/-]{86,88}={1,2} + ) + | + (?: + [A-Z0-9+\\/-]{86,88}\b + ) ) + pattern_requirements: + min_digits: 2 + min_uppercase: 2 + min_lowercase: 2 min_entropy: 4.0 confidence: medium examples: - - AccountKey=Xy9aB8cD7eF6gH5iJ4kL3mN2oP1qR0sT9uV8wX7yZ6aB5cD4eF3gH2iJ1kL0mN9oP8qR7sT6uV5wX4yZ3aB2cD1eF0gH9iJ8kL7mN6oP5q==\ + - Azure AccountKey=Xy9aB8cD7eF6gH5iJ4kL3mN2oP1qR0sT9uV8wX7yZ6aB5cD4eF3gH2iJ1kL0mN9oP8qR7sT6uV5wX4yZ3aB2cD1q + - Azure AccountKey=Ky7aC1cD7eF6gH5iJ4kL3mN2oP1qR0sT9uV8wX7yZ6aB5cD4eF3gH2iJ1kL0mN9oP8qR7sT6uV5wX4yZ3aB2cD1g==\ validation: type: AzureStorage depends_on_rule: - - rule_id: kingfisher.azurestorage.name.1 - variable: AZURENAME + - rule_id: kingfisher.azurestorage.1 + variable: AZURENAME \ No newline at end of file diff --git a/data/rules/baremetrics.yml b/data/rules/baremetrics.yml index 415731e..2844cd3 100644 --- a/data/rules/baremetrics.yml +++ b/data/rules/baremetrics.yml @@ -2,17 +2,17 @@ rules: - name: Baremetrics API Key id: kingfisher.baremetrics.1 pattern: | - (?xi) - \b + (?xi) + \b baremetrics (?:.|[\n\r]){0,32}? - (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) - (?:.|[\n\r]){0,32}? \b ( [a-z0-9_-]{25} ) - \b + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium references: diff --git a/data/rules/baseten.yml b/data/rules/baseten.yml new file mode 100644 index 0000000..b3ba12c --- /dev/null +++ b/data/rules/baseten.yml @@ -0,0 +1,49 @@ +rules: + - name: Baseten API Key + id: kingfisher.baseten.1 + pattern: | + (?x) + \b + baseten + (?:.|[\n\r]){0,32}? + \b + ( + [A-Za-z0-9]{8} + \. + [A-Za-z0-9]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 + min_entropy: 3.4 + confidence: medium + examples: + - baseten_key = WSsDXzCD.uOcxAp7k82IvCKyY36TnpVbP4ZszP1qw + - baseten_key = crXCQC3W.CgCGGY1b9IfJan5TppW0Z07C9oMN2DmR + - baseten_key = h2wFkhFC.3WFVwVcxGFr4Qup0gyhvIuONwQxEpL0A + - baseten_key = XqbIpj04.x73j1zLUOEgGIKROqVbxsmggPdL8JvAY + references: + - https://docs.baseten.co/examples/vllm + - https://docs.baseten.co/reference/management-api/api-keys/lists-the-users-api-keys + - https://docs.baseten.co/reference/training-api/overview#authentication + - https://docs.baseten.co/reference/management-api/api-keys/creates-an-api-key + validation: + type: Http + content: + request: + method: GET + url: https://api.baseten.co/v1/api_keys + headers: + Authorization: Api-Key {{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: + - '"name"' + - '"type"' diff --git a/data/rules/beamer.yml b/data/rules/beamer.yml index f051ff2..e3904d3 100644 --- a/data/rules/beamer.yml +++ b/data/rules/beamer.yml @@ -9,7 +9,11 @@ rules: \b ( b_[A-Z0-9=_\\/\\\-+]{44} - ) + ) + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.0 confidence: medium examples: diff --git a/data/rules/bitbucket.yml b/data/rules/bitbucket.yml index ad7e74a..37912de 100644 --- a/data/rules/bitbucket.yml +++ b/data/rules/bitbucket.yml @@ -8,9 +8,10 @@ rules: (?:.|[\n\r]){0,16}? (?:client|id) (?:.|[\n\r]){0,16}? - \b ([a-z0-9]{30,40}) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: @@ -43,11 +44,14 @@ rules: ( [a-z0-9+_\-+]{44} ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: - bitbucket_key=HedmnK9h6KD_eh9KK8FlI9ahUc8WfaNZ4gulbrtN2ouV - - bitbucket_secret=kd8j2h4jf9s8mf6l4k9j2h4jf9s8mf6l4k9j2h4jf9s8mf6l + - bitbucket_secret=kd8j2h4jf9s8mf6l4k9j2h4jf9s8mf6l4k9j2h4jf9s8 validation: type: Http content: diff --git a/data/rules/bitly.yml b/data/rules/bitly.yml new file mode 100644 index 0000000..df79eb3 --- /dev/null +++ b/data/rules/bitly.yml @@ -0,0 +1,38 @@ +rules: + - name: Bitly Access Token + id: kingfisher.bitly.1 + pattern: | + (?xi) + \b + bitly + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + ( + [a-f0-9]{40} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 2 + confidence: medium + min_entropy: 3.0 + validation: + type: Http + content: + request: + method: GET + url: "https://api-ssl.bitly.com/v4/user" + headers: + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"login":' + references: + - https://dev.bitly.com/api-reference#Authentication + examples: + - "bitly_token = 20e9817b9c5ddde1b0cec7622bfc557dbc823791" diff --git a/data/rules/blynk.yml b/data/rules/blynk.yml index 4af12c6..1f58e85 100644 --- a/data/rules/blynk.yml +++ b/data/rules/blynk.yml @@ -6,6 +6,10 @@ rules: https://(?:fra1\.|lon1\.|ny3\.|sgp1\.|blr1\.)*blynk\.cloud/external/api/[A-Z0-9/]*\?token= ([A-Z0-9_\-]{32}) & + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.3 confidence: medium examples: @@ -21,6 +25,10 @@ rules: -H\s*"Authorization:\s*Bearer\s* ([A-Z0-9_\-]{40}) " + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.3 confidence: medium examples: @@ -35,6 +43,10 @@ rules: -H\s*"Authorization:\s*Bearer\s* ([A-Z0-9_\-]{40}) "[\s\\]*https://(?:fra1\.|lon1\.|ny3\.|sgp1\.|blr1\.)*blynk\.cloud/api + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.3 confidence: medium examples: @@ -50,6 +62,10 @@ rules: (oa2-client-id_[A-Z0-9_\-]{32}) (?: : | &client_secret= ) ([A-Z0-9_\-]{40}) + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.3 confidence: medium examples: @@ -66,6 +82,10 @@ rules: (oa2-client-id_[A-Z0-9_\-]{32}) :([A-Z0-9_\-]{40}) [\s\\]*https://(fra1\.|lon1\.|ny3\.|sgp1\.|blr1\.)*blynk\.cloud/oauth2 + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/buildkite.yml b/data/rules/buildkite.yml new file mode 100644 index 0000000..045fa94 --- /dev/null +++ b/data/rules/buildkite.yml @@ -0,0 +1,33 @@ +rules: + - name: Buildkite API Key + id: kingfisher.buildkite.1 + pattern: | + (?xi) + ( + bkua_[a-z0-9]{40} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 1 + min_entropy: 3.5 + confidence: medium + examples: + - bkua_3c7019c2e4b6e76fe2e8bdde7c154e3c1a211743 + references: + - https://buildkite.com/docs/apis/rest-api/access-token + validation: + type: Http + content: + request: + method: GET + url: https://api.buildkite.com/v2/access-token + headers: + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"uuid"', '"user"'] + \ No newline at end of file diff --git a/data/rules/cerebras.yml b/data/rules/cerebras.yml new file mode 100644 index 0000000..044506e --- /dev/null +++ b/data/rules/cerebras.yml @@ -0,0 +1,38 @@ +rules: + - name: Cerebras AI API Key + id: kingfisher.cerebras.1 + pattern: | + (?xi) + ( + csk-[a-z0-9]{48} + ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 2 + confidence: medium + min_entropy: 3.0 + validation: + type: Http + content: + request: + method: GET + url: "https://api.cerebras.ai/v1/models" + headers: + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"object"' + - '"data"' + match_all_words: true + references: + - https://docs.cerebras.net/ + examples: + - "csk-6nptf4w5cx36fw58t3hkx48jvm52wm693pex5tjm29kn55yt" + - "csk-e2knhj8h3h4erp6crfx6rh52tvecj4xnwmtjf3mtrvtt54et" + - "csk-rhw8npjrp6kpv9phm55n5nv5rkkm4492jepx3yh65dc9cwe9" + - "csk-w6p3nxk3dc5249mrpmv642fffert28rwdkepffrpn8rtfr9h" diff --git a/data/rules/circleci.yml b/data/rules/circleci.yml index f3f2d2a..78ef95e 100644 --- a/data/rules/circleci.yml +++ b/data/rules/circleci.yml @@ -2,31 +2,22 @@ rules: - name: CircleCI API Personal Access Token id: kingfisher.circleci.1 pattern: | - (?xi) + (?x) \b ( CCIPAT_ - [a-z0-9]{4} - [a-z]{5} - [a-z0-9]{3} - [0-9]{3} - [a-z]{2} - [A-Z]{2} - [0-9]{1} - [a-z]{1} - [a-z0-9]{1} - [0-9]{1} - [a-z]{1} + [a-zA-Z0-9]{22} _ [a-z0-9]{40} ) - \b + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: - - CircleCI_PAT = "CCIPAT_lZyPAuThWn2G908ssDT0g33e_t7qh0r5hrvsqzmuraqzduq6qco5onxgrtcn7y2z4" - | - export CIRCLECI_TOKEN=CCIPAT_lZyPAuThWn2G908ssDT0g33e_t7qh0r5hrvsqzmuraqzduq6qco5onxgrtcn7y2z4 + export CIRCLECI_TOKEN=CCIPAT_FERZRjTN451xnDCy1y9gWn_79fb6ca4d0e5f833612eee17de397a9dca0a9e9f references: - https://circleci.com/docs/api-developers-guide/#using-the-api-securely-wtih-curl validation: @@ -56,7 +47,10 @@ rules: ( [a-f0-9]{40} ) - \b + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 2 min_entropy: 3.3 confidence: medium examples: @@ -80,4 +74,4 @@ rules: - type: WordMatch words: - '"vcs_url"' - url: https://circleci.com/api/v1.1/projects \ No newline at end of file + url: https://circleci.com/api/v1.1/projects diff --git a/data/rules/ciscomeraki.yml b/data/rules/ciscomeraki.yml new file mode 100644 index 0000000..03f9d3a --- /dev/null +++ b/data/rules/ciscomeraki.yml @@ -0,0 +1,37 @@ +rules: + - name: Cisco Meraki API Key + id: kingfisher.ciscomeraki.1 + pattern: | + (?xi) + meraki + (?:.|[\n\r]){0,32}? + ( + [0-9a-f]{40} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.3 + confidence: medium + examples: + - MERAKI_API_KEY=1234567890abcdef1234567890abcdef12345678 + - |- + // Meraki configuration + const MERAKI_KEY = "abcdefabcdefabcdefabcdefabcdefabcdefabcd"; + references: + - https://developer.cisco.com/meraki/api-v1/overview/ + validation: + type: Http + content: + request: + method: GET + url: https://api.meraki.com/api/v1/organizations + headers: + X-Cisco-Meraki-API-Key: '{{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 + - type: JsonValid diff --git a/data/rules/clarifai.yml b/data/rules/clarifai.yml new file mode 100644 index 0000000..0942f19 --- /dev/null +++ b/data/rules/clarifai.yml @@ -0,0 +1,40 @@ +rules: + - name: Clarifai API Key + id: kingfisher.clarifai.1 + pattern: | + (?xi) + \b + clarifai + (?:.|[\n\r]){0,32}? + \b + ( + [0-9a-f]{32,36} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + examples: + - clarifai_key = 29ee853d47364107b9edf5e5ad4374c2 + - "clarifai_token: eb3cf5440b7b45f8954bb4a1fcea0ea5" + - clarifai-secret = 8e43e018f61b493c8104024ee124a57f + - clarifai_api = cf3cacafabe747988298298bffcbb459 + references: + - https://docs.gitguardian.com/secrets-detection/secrets-detection-engine/detectors/specifics/clarifai_key + - https://docs.clarifai.com/control/authentication/key/ + validation: + type: Http + content: + request: + method: GET + url: https://api.clarifai.com/v2/models + headers: + Authorization: Key {{ TOKEN }} + response_matcher: + - report_response: true + - type: WordMatch + match_all_words: true + words: + - '"code":10000' + - '"description":"Ok"' diff --git a/data/rules/clay.yml b/data/rules/clay.yml new file mode 100644 index 0000000..ff86401 --- /dev/null +++ b/data/rules/clay.yml @@ -0,0 +1,25 @@ +rules: + - name: Clay API Key + id: kingfisher.clay.1 + pattern: | + (?xi) + \b + clay + (?:.|[\n\r]){0,64}? + \b + ( + [a-f0-9]{20} + ) + \b + pattern_requirements: + min_digits: 6 + min_entropy: 3.0 + confidence: medium + examples: + - clay_api_key=ce1abceaffe7d7958a41 + - "CLAY_KEY: bdc55270455ca0a892e4" + - export CLAY_TOKEN=e9b711a5acbb99b8f099 + - 'clay key: f6fd04ab6b4f7992adc2' + - CLAY_API_KEY=d8dfd14ec83e4e17a7d2 + references: + - https://university.clay.com/docs/http-api-integration-overview diff --git a/data/rules/clearbit.yml b/data/rules/clearbit.yml new file mode 100644 index 0000000..02e4469 --- /dev/null +++ b/data/rules/clearbit.yml @@ -0,0 +1,35 @@ +rules: + - name: Clearbit API Key + id: kingfisher.clearbit.1 + pattern: | + (?xi) + \b + clearbit + (?:.|[\n\r]){0,16}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [0-9a-z_]{35} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - clearbit_token = tq50141fm92fl4nid9c1c7liouhbertbvg1 + validation: + type: Http + content: + request: + method: GET + url: https://discovery.clearbit.com/v1/companies/entities?name=kingfisher + headers: + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - report_response: true + - type: WordMatch + words: + - '"Invalid API key provided"' + negative: true \ No newline at end of file diff --git a/data/rules/clickhouse.yml b/data/rules/clickhouse.yml new file mode 100644 index 0000000..9f22697 --- /dev/null +++ b/data/rules/clickhouse.yml @@ -0,0 +1,57 @@ +rules: + - name: ClickHouse Cloud Secret Key + id: kingfisher.clickhouse.1 + pattern: | + (?xi) + \b + ( + 4b1d[a-z0-9]{38} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.5 + examples: + - "4b1dwEZ8aNo1U9ODBqffSci1INBrltLHM2d1bHF4dq" + validation: + type: Http + content: + request: + method: GET + url: "https://api.clickhouse.cloud/v1/organizations" + headers: + Authorization: "Basic {{ CLICKHOUSE_ID | append: ':' | append: TOKEN | b64enc }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"id":' + - '"name":' + match_all_words: true + depends_on_rule: + - rule_id: "kingfisher.clickhouse.2" + variable: CLICKHOUSE_ID + references: + - https://clickhouse.com/docs/en/cloud/security/service-accounts + - name: ClickHouse Cloud Key ID + id: kingfisher.clickhouse.2 + pattern: | + (?xi) + \b + clickhouse + (?:.|[\n\r]){0,16}? + (?:ID|USER) + (?:.|[\n\r]){0,16}? + ( + [a-z0-9]{20} + ) + pattern_requirements: + min_digits: 2 + confidence: medium + visible: false + min_entropy: 3.0 + examples: + - "clickhouse_id = 4ywspD2Tb0gJh4QbLnDI" diff --git a/data/rules/clojars.yml b/data/rules/clojars.yml new file mode 100644 index 0000000..5dfaf53 --- /dev/null +++ b/data/rules/clojars.yml @@ -0,0 +1,41 @@ +rules: + - name: Clojars Username + id: kingfisher.clojars.1 + pattern: | + (?xi) + \b + clojars + (?:.|[\n\r]){0,32}? + (?:ID|USER) + (?:.|[\n\r]){0,16}? + \b + ( + [a-z0-9_-]{3,} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 1.5 + visible: false + examples: + - "clojars_user = my-username" + + - name: Clojars API Token + id: kingfisher.clojars.2 + pattern: | + (?xi) + \b + ( + CLOJARS_[a-z0-9]{60} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 4.0 + examples: + - "CLOJARS_7a0b5c647fdca75616e99a5629ba28955e53faf6391b1b4461ccd972b0e2" + references: + - https://github.com/clojars/clojars-web/wiki/API + - https://github.com/clojars/clojars-web/wiki/Tutorial-for-library-authors \ No newline at end of file diff --git a/data/rules/cloudflare.yml b/data/rules/cloudflare.yml index 776c1bd..442231c 100644 --- a/data/rules/cloudflare.yml +++ b/data/rules/cloudflare.yml @@ -12,7 +12,11 @@ rules: ( [a-z0-9_-]{38,42} ) - \b + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.5 confidence: medium examples: @@ -47,9 +51,10 @@ rules: ( v1\.0-[a-z0-9._-]{160,} ) - ["'`]? - \b - min_entropy: 4.5 + \b + pattern_requirements: + min_digits: 2 + min_entropy: 4.0 confidence: medium examples: - 'X-Auth-User-Service-Key = v1.0-e26de050e02ddeaeef6de8d5ee267df5e78f68666ddd0ee76f22d26a0d20756f-eda77de60e8e76077e162727656787de2005d25e2f6e502e2d067657ed65722eade065275001a0f6f6e521e5e1fd76a6e8d7e2d6da8a2ee01e66e061e22570e2-07f2ede0aed78e82e8d2e620aaef8656d81e762266d7d226a205de7e18e2256a' @@ -58,9 +63,6 @@ rules: references: - https://developers.cloudflare.com/api/keys/ - https://developers.cloudflare.com/fundamentals/api/get-started/keys/ - categories: - - api - - secret validation: type: Http content: diff --git a/data/rules/cloudsight.yml b/data/rules/cloudsight.yml index b06e922..f3bd7f6 100644 --- a/data/rules/cloudsight.yml +++ b/data/rules/cloudsight.yml @@ -11,8 +11,10 @@ rules: \b ( [a-z0-9]{20,24} - ) - \b + ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/codacy.yml b/data/rules/codacy.yml index 121dbf4..e0a68d5 100644 --- a/data/rules/codacy.yml +++ b/data/rules/codacy.yml @@ -12,6 +12,9 @@ rules: ( [0-9A-Z]{20,24} ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/codeclimate.yml b/data/rules/codeclimate.yml index 677cbc5..aeaa805 100644 --- a/data/rules/codeclimate.yml +++ b/data/rules/codeclimate.yml @@ -5,10 +5,13 @@ rules: (?xi) (?: CODECLIMATE| CC_TEST_REPORTER_ID) (?:.|[\n\r]){0,64}? + \b ( [a-f0-9]{64} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/codecov.yml b/data/rules/codecov.yml new file mode 100644 index 0000000..a2ab1e7 --- /dev/null +++ b/data/rules/codecov.yml @@ -0,0 +1,39 @@ +rules: + - name: Codecov Access Token + id: kingfisher.codecov.1 + pattern: | + (?xi) + \b + codecov + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9-]{36} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.5 + validation: + type: Http + content: + request: + method: GET + url: "https://api.codecov.io/api/v2/github/" + headers: + Authorization: "Bearer {{TOKEN}}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"count":' + references: + - https://docs.codecov.com/reference/api-overview + examples: + - "codecov_token = 52acf265-3fc6-4ecd-304a-15940bd04653" diff --git a/data/rules/coderabbit.yml b/data/rules/coderabbit.yml new file mode 100644 index 0000000..f404722 --- /dev/null +++ b/data/rules/coderabbit.yml @@ -0,0 +1,39 @@ +rules: + - name: CodeRabbit API Key + id: kingfisher.coderabbit.1 + pattern: | + (?xi) + \b + ( + cr-[a-f0-9]{58} + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.5 + confidence: medium + examples: + - "cr-33420bb12fddf6cde6fba5414df88b07f75b2258e30c956b95f2ddbb2d" + references: + - https://coderabbit.ai/ + - https://api.coderabbit.ai/docs + validation: + type: Http + content: + request: + method: GET + url: "https://api.coderabbit.ai/v1/seats/" + headers: + accept: "application/json" + x-coderabbitai-api-key: "{{TOKEN}}" + response_matcher: + - report_response: true + - type: WordMatch + words: + - '"success"' + - '"errors"' + match_all_words: false + - type: WordMatch + negative: true + words: + - '"Invalid or inactive API key"' diff --git a/data/rules/cohere.yml b/data/rules/cohere.yml new file mode 100644 index 0000000..f8136f4 --- /dev/null +++ b/data/rules/cohere.yml @@ -0,0 +1,41 @@ +rules: + - name: Cohere API Key + id: kingfisher.cohere.1 + pattern: | + (?xi) + \b + cohere + (?:.|[\n\r]){0,16}? + \b + ( + [A-Z0-9]{40} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - 'cohere_key = 5fNtU1ytdUcOX3jfvgjlr61EPxBqxOojOklDD6BG' + - "cohere secret key = QfsfCM0HdHH9x5ZlhsGzeignSk4pCeBwBrzYqgGV" + - 'cohere_token: x7PX0fac8a2GW2fgnNqdtqIwMQvFbrL6E7lKrKOv' + references: + - https://docs.cohere.com/reference/list-connectors + validation: + type: Http + content: + request: + method: GET + url: https://api.cohere.com/v1/connectors + headers: + Authorization: Bearer {{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: + - '"connectors"' + - '"total_count"' \ No newline at end of file diff --git a/data/rules/coinbase.yml b/data/rules/coinbase.yml new file mode 100644 index 0000000..159dc90 --- /dev/null +++ b/data/rules/coinbase.yml @@ -0,0 +1,82 @@ +rules: + - name: Coinbase Access Token + id: kingfisher.coinbase.1 + pattern: | + (?xi) + \b + coinbase + (?:.|[\n\r]){0,16}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,16}? + \b + ( + [a-z-0-9]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + examples: + - coinbase_token = 32iAkQCcHHYxXGx20VogBZoj27PC1ouI + references: + - https://docs.cloud.coinbase.com/wallet-sdk/docs/api-keys + validation: + type: Http + content: + request: + method: GET + url: https://api.coinbase.com/v2/user + headers: + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - name: Coinbase CDP API Key (ECDSA) + id: kingfisher.coinbase.2 + pattern: | + (?xims) + "name"\s*:\s*" + (?Porganizations/[0-9a-f-]{36}/apiKeys/[0-9a-f-]{36})" + .*"privateKey"\s*:\s*" + (?P + -----BEGIN\sEC\s{0,1} + PRIVATE\sKEY + (\sBLOCK)? + ----- + [a-z0-9 /+=\r\n\\n]{32,}? + -----END\s + (?: + RSA | + PGP | + DSA | + OPENSSH | + ENCRYPTED | + EC + )? + \s{0,1} + PRIVATE\sKEY + (\sBLOCK)? + ----- + ) + validation: + type: Coinbase + examples: + - | + { + "name": "organizations/243873d8-c14e-436d-9cea-10d530cbe201/apiKeys/d29bb143-ad4c-234f-9bd7-c705c16b6d19", + "privateKey": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIDs+vr9F40Mer+qYksK3QhkSMfUXOZsbRVSrelWGnMh3oAoGCCqGSM49\nAwEHoUQDQgAEOXj2qKzLYx21D3plbOa81ilURS/4K1jzLXBvgwfUe4hWDgBdKQvq\nIiet5qqZEwVlR/LqKQEUlP8YLrjLFU8Unw==\n-----END EC PRIVATE KEY-----\n" + } + - name: Coinbase CDP API Key (Ed25519) + id: kingfisher.coinbase.3 + pattern: | + (?xis) + "id"\s*:\s*"(?P[0-9a-f-]{36})"[^{]*?"privateKey"\s*:\s*"(?P[A-Za-z0-9+/=]{88})" + validation: + type: Coinbase + examples: + - | + { + "id": "413b23bf-4582-4e57-b33a-85d9527d9972", + "privateKey": "ygWq07YCO8UkmC9BE0PDBJNGhiu80yslsMUF9WnjPaIF5DBxb/wljjRuHhfuR/AMPC+kdgtL+mWKq/HOnq/YcQ==" + } \ No newline at end of file diff --git a/data/rules/confluent.yml b/data/rules/confluent.yml index 7deda55..49e9b69 100644 --- a/data/rules/confluent.yml +++ b/data/rules/confluent.yml @@ -10,6 +10,8 @@ rules: [A-Z0-9]{16} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3 confidence: medium visible: false @@ -33,8 +35,45 @@ rules: min_entropy: 3.3 confidence: medium examples: - - confluent secret=abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890ab - - kafka_token=ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890abcdefghijklmnopqrstuvwxyzABCD + - confluent secret=cbadefghijklmnopqrstuvwxyzcbaDEFGHIJKLMNOPQRSTUVWXYZ3214567890ab + - kafka_token=cbaDEFGHIJKLMNOPQRSTUVWXYZ3214567890cbadefghijklmnopqrstuvwxyzAB + references: + - https://docs.confluent.io/cloud/current/api.html#tag/API-Keys-(iamv2)/operation/getIamV2ApiKey + validation: + type: Http + content: + request: + headers: + Authorization: 'Basic {{ CLIENTID | append: ":" | append: TOKEN | b64enc }}' + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://api.confluent.cloud/iam/v2/api-keys/{{ CLIENTID }} + depends_on_rule: + - rule_id: "kingfisher.confluent.1" + variable: CLIENTID + - name: Confluent API Secret - Updated Format + id: kingfisher.confluent.3 + pattern: | + (?xi) + \b + ( + cflt(?P[A-Za-z0-9\+/]{54})(?P[A-Za-z0-9\+/]{6}) + ) + pattern_requirements: + checksum: + actual: + template: "{{ MATCH | suffix: 6 }}" + requires_capture: checksum + expected: "{{ BODY | crc32_le_b64: 6 }}" + skip_if_missing: true + min_entropy: 3.3 + confidence: medium + examples: + - confluent secret=cfltqPLd2lLPAtWtHGNhN32WlZxoEj30pcg8mzaPlPJ937JlMa7n9YCRLooqgifw references: - https://docs.confluent.io/cloud/current/api.html#tag/API-Keys-(iamv2)/operation/getIamV2ApiKey validation: diff --git a/data/rules/contentful.yml b/data/rules/contentful.yml new file mode 100644 index 0000000..3c91b77 --- /dev/null +++ b/data/rules/contentful.yml @@ -0,0 +1,77 @@ +rules: + - name: Contentful Delivery API Token + id: kingfisher.contentful.1 + pattern: | + (?xi) + \b + contentful + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9_-]{43,45} + ) + \b + pattern_requirements: + min_digits: 2 + min_special_chars: 1 + confidence: medium + min_entropy: 4.0 + validation: + type: Http + content: + request: + method: GET + url: "https://cdn.contentful.com/spaces" + headers: + Authorization: "Bearer {{TOKEN}}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"sys":' + - '"type":"Array"' + - '"items":' + match_all_words: true + references: + - https://www.contentful.com/developers/docs/references/content-delivery-api/ + examples: + - "contentful_delivery_token = wJz-g_tqZ-8n_abcdefghijklmnopqrstuvwxyz12345" + + - name: Contentful Personal Access Token + id: kingfisher.contentful.2 + pattern: | + (?xi) + ( + CFPAT-[A-Z0-9_-]{43} + ) + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.5 + validation: + type: Http + content: + request: + method: GET + url: "https://api.contentful.com/users/me" + headers: + Authorization: "Bearer {{TOKEN}}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"firstName":' + - '"lastName":' + match_all_words: true + references: + - https://www.contentful.com/developers/docs/references/content-management-api/#/reference/users/user + examples: + - "CFPAT-Cq3AarsJCDvdG9PYAJ3Y00crCG5nEPAAfVZ2LAldCsQ" + - "CFPAT-WFWEK_dWYIW0-uamOjhUBAJJ9NqQisr1x_ylb4z1vcQ" + - "CFPAT-lkITY2pqeiE0_p9TxUZrGAhfHJIiwYtbtr769y4_AYY" diff --git a/data/rules/coveralls.yml b/data/rules/coveralls.yml new file mode 100644 index 0000000..77e8f74 --- /dev/null +++ b/data/rules/coveralls.yml @@ -0,0 +1,81 @@ +rules: + - name: Coveralls Repo Identifier + id: kingfisher.coveralls.1 + visible: false + confidence: medium + min_entropy: 2.0 + pattern: | + (?xi) + (?: + coveralls\.io/ + (?: + (?: + github|bitbucket|gitlab + ) + / + ( + [A-Z0-9_.-]+ + ) + / + ( + [A-Z0-9_.-]+ + ) + ) + | + api/v1/repos/ + ( + github|bitbucket|gitlab + ) + / + ( + [A-Z0-9_.-]+ + ) + ) + examples: + - https://coveralls.io/github/lemurheavy/coveralls-public + - https://coveralls.io/gitlab/group/project + - https://coveralls.io/api/v1/repos/github/octocat/hello-world + + - name: Coveralls Personal API Token + id: kingfisher.coveralls.2 + pattern: | + (?xi) + \b + coveralls + (?:.|[\n\r]){0,1}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9-]{37} + ) + \b + pattern_requirements: + min_digits: 3 + min_entropy: 3.3 + confidence: medium + examples: + - coveralls_SECRETTOKEN abcdefghijklmnopqrstuvwxyzab12345cdef + - coveralls-SECRET-KEY mnopqrstuvwxyzabcdefghi12345678901234 + - coveralls_PRIVATEKEY-1234567890abcdefghijklmnopqrstuvwxyza + references: + - https://docs.coveralls.io/api-repos-endpoint + - https://docs.coveralls.io/api-introduction + depends_on_rule: + - rule_id: kingfisher.coveralls.1 + variable: COVERALLS_REPO_ID + validation: + type: Http + content: + request: + method: GET + url: "https://coveralls.io/api/v1/repos/{{ COVERALLS_REPO_ID }}" + headers: + Authorization: "token {{ TOKEN }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"service"', '"name"', '"id"'] diff --git a/data/rules/coze.yml b/data/rules/coze.yml new file mode 100644 index 0000000..c9783f3 --- /dev/null +++ b/data/rules/coze.yml @@ -0,0 +1,41 @@ +rules: + - name: Coze Personal Access Token + id: kingfisher.coze.1 + pattern: | + (?xi) + coze + (?:.|[\n\r]){0,32}? + \b + ( + pat_[A-Z0-9]{64} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 5.0 + validation: + type: Http + content: + request: + method: GET + url: "https://api.coze.com/v1/workspaces?" + headers: + Authorization: "Bearer {{TOKEN}}" + Content-Type: application/json + response_matcher: + - type: StatusMatch + status: [200, 403] # API returns 403 for a valid token without permission to route + - type: JsonValid + - type: WordMatch + words: + - '"access token invalid"' + - '"does not have permission"' + negative: true + references: + - https://www.coze.com/docs/developer_guides/coze_api_overview + - https://www.coze.com/docs/developer_guides/retrieve_files + examples: + - "key_coze = pat_DlOG7fNcVfmw8cYhPWNcdfwrjjzwDr9EkV8EBjzHdgRWU2DzqHC1pPe0x590NN5f" + - "coze_token = pat_93QiTdIvZGuRCFcfGTQJJ1VIYZ9dNHanX88wKoMojwMk3tX5tKqfFtxUp0ux8CjI" + - "coze-key: pat_WvUTLYq5yZyaqegkyLSxXJMjXAJotjYEuC1sqT8daFlfwM3BiaRVJIZsER42DnhV" diff --git a/data/rules/crates.io.yml b/data/rules/crates.io.yml index 7f6b8f2..4aa7ef6 100644 --- a/data/rules/crates.io.yml +++ b/data/rules/crates.io.yml @@ -8,6 +8,8 @@ rules: cio[A-Z0-9]{32} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/credentials.yml b/data/rules/credentials.yml index 38c1057..cb7e866 100644 --- a/data/rules/credentials.yml +++ b/data/rules/credentials.yml @@ -20,6 +20,6 @@ rules: [a-z0-9\/._~-]* )? min_entropy: 3.0 - confidence: low + confidence: medium examples: - https://eaRIWNkE:qyOIhJiM@j2LYY414Q5cCYD \ No newline at end of file diff --git a/data/rules/curl.yml b/data/rules/curl.yml new file mode 100644 index 0000000..4f28cfe --- /dev/null +++ b/data/rules/curl.yml @@ -0,0 +1,24 @@ +rules: + - name: Curl Basic Authentication Credentials + id: kingfisher.curl.1 + pattern: '(?i)\bcurl\s.*(?:-u|--user)\s+[''"]?(?P[^:''"\s]+:[^''"\s]+)[''"]?' + confidence: low + min_entropy: 3.0 + references: + - https://curl.se/docs/manpage.html#-u + examples: + - 'curl --cacert ca.crt -u elastic:P@ssw0rd$1 https://localhost:9200' + - 'curl -u developer:yqDVtkqPECriaLRi' + - 'curl --user roger23@gmail.com:pQ9wTxu4Fg https://www.dropbox.com' + + - name: Curl Header Authentication + id: kingfisher.curl.2 + pattern: '(?i)\bcurl\s.*(?:-H|--header)\s+[''"]Authorization:\s*(?:Bearer|Basic|Token)\s+(?P[a-zA-Z0-9+/=_-]{20,})[''"]' + confidence: low + min_entropy: 3.5 + references: + - https://curl.se/docs/manpage.html#-H + examples: + - 'curl -H ''Authorization: Basic YnJvd3Nlcjo=''' + - 'curl -H "Authorization: Bearer cfcabd11c7ed9a41b1a3e063c32d5114"' + - 'curl -H "Authorization: Token 22cb987851bc5659229114c62e60c79abd0d2c08"' diff --git a/data/rules/cursor.yml b/data/rules/cursor.yml new file mode 100644 index 0000000..2dacb35 --- /dev/null +++ b/data/rules/cursor.yml @@ -0,0 +1,32 @@ +rules: + - name: Cursor Integrations (User) API Key + id: kingfisher.cursor.1 + pattern: | + (?xi) + \b + ( + key_ + [0-9a-f]{64} + ) + \b + min_entropy: 3.8 + confidence: medium + examples: + - key_8c5a7657fc397e114def1b51dd520410ad50ece61e30b64261ff369ab275ef29 + - key_86aed0092d14dd6aae4e8ad107d52f760c5efa4bc3753730ca6983babc2b1072 + - key_d9ba91e29d81f1bf2085b55b06f5ac43885749005cff565bee7f406ca2e2f3f9 + references: + - https://cursor.com/docs/cloud-agent/api/endpoints + validation: + type: Http + content: + request: + method: GET + url: https://api.cursor.com/v0/me + headers: + Accept: application/json + Authorization: 'Basic {{ TOKEN | b64enc }}' + response_matcher: + - report_response: true + - type: WordMatch + words: ['"userEmail"'] diff --git a/data/rules/customerio.yml b/data/rules/customerio.yml new file mode 100644 index 0000000..5d8181d --- /dev/null +++ b/data/rules/customerio.yml @@ -0,0 +1,68 @@ +rules: + - name: Customer.io Tracking API Key + id: kingfisher.customerio.1 + pattern: | + (?xi) + \b + (?:customer(?:\.?io)?|customerio|cio|tracking|track) + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN|API[_-]?KEY) + (?:.|[\n\r]){0,16}? + \b + ( + [0-9a-f]{20} + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.0 + confidence: medium + examples: + - "tracking api key: f3b0c2b92eca01472efe" + - "customerio_key = a98eab982f4692ceb78f" + - "customer.io tracking_api_key d24d3915959b4d793a67" + references: + - https://docs.customer.io/integrations/api/#track-api + + - name: Customer.io App API Key + id: kingfisher.customerio.2 + pattern: | + (?xi) + \b + (?:customer(?:\.?io)?|customerio|cio) + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN|API) + (?:.|[\n\r]){0,16}? + \b + ( + [0-9a-f]{32} + ) + \b + pattern_requirements: + min_digits: 6 + min_entropy: 3.0 + confidence: medium + examples: + - "customerio_app_key=6e86f5734527548b7477a8b627bf4855" + - "customer.io api key 8363e3ca7e897cae7d76b8f46632e155" + - "cio_app_key: 801b93d4c8627282bbd3524362f1ea9d" + references: + - https://docs.customer.io/integrations/api/#app-api + - https://api.customer.io/v1/workspaces + validation: + type: Http + content: + request: + method: GET + url: https://api.customer.io/v1/workspaces + headers: + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: + - '"workspaces"' diff --git a/data/rules/databricks.yml b/data/rules/databricks.yml index 9cec5c3..294cb20 100644 --- a/data/rules/databricks.yml +++ b/data/rules/databricks.yml @@ -2,12 +2,17 @@ rules: - name: Databricks API token id: kingfisher.databricks.1 pattern: | - (?xi) + (?xi) + \b ( dapi [a-f0-9]{32} ) - \b + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/datadog.yml b/data/rules/datadog.yml index 1fc5420..eb83d04 100644 --- a/data/rules/datadog.yml +++ b/data/rules/datadog.yml @@ -1,64 +1,117 @@ rules: - - name: Datadog API Key + # Helper: extract the Datadog site domain from common config/env/URLs. + # We capture the "site parameter" (domain), then validation uses https://api.. + - name: Datadog Site Domain id: kingfisher.datadog.1 + visible: false + confidence: medium + min_entropy: 2.0 pattern: | - (?xi) - \b - (?:datadog|dd-|dd_) - (?:.|[\n\r]){0,32}? - (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) - (?:.|[\n\r]){0,32}? - \b + (?xi) + (?: + # env/config patterns + \b(?:DD_SITE|DATADOG_SITE|DATADOG_HOST)\b\s*[:=]\s*["']? + (?:https?://)? + (?:api\.|app\.)? + | + # raw URLs in code/docs + \bhttps?://(?:api\.|app\.)? + )? ( - [a-z0-9]{32} + datadoghq\.com + | us3\.datadoghq\.com + | us5\.datadoghq\.com + | datadoghq\.eu + | ap1\.datadoghq\.com + | ap2\.datadoghq\.com + | ddog-gov\.com ) \b + examples: + - DD_SITE=datadoghq.eu + - DATADOG_HOST=https://api.us3.datadoghq.com + - https://app.datadoghq.com + - https://api.ddog-gov.com + + - name: Datadog API Key + id: kingfisher.datadog.2 + pattern: | + (?xi) + \b(?:datadog|dd) + (?:.|[\n\r]){0,64}? + (?:api[_-]?key|dd[_-]?api[_-]?key|secret|private|access|token) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9]{32} + ) + \b + pattern_requirements: + min_digits: 3 min_entropy: 3.3 confidence: medium examples: - - dd-apikey-dd52c29224affe29d163c6bf99e5c34f - - datadog-secrettoken-0024a29224affe29d173c0bf99e5a89d + - DD_API_KEY=0024a29224affe29d173c0bf99e5a89d references: - https://docs.datadoghq.com/account_management/api-app-keys/ validation: type: Http content: request: + method: GET + url: https://api.datadoghq.com/api/v1/validate headers: Accept: application/json - DD-API-KEY: '{{ TOKEN }}' - DD-APPLICATION-KEY: '{{ APPKEY }}' - method: GET + DD-API-KEY: "{{ TOKEN }}" response_matcher: - report_response: true - status: - 200 type: StatusMatch - url: https://api.datadoghq.com/api/v2/current_user - depends_on_rule: - - rule_id: kingfisher.datadog.2 - variable: APPKEY + - type: WordMatch + words: + - '"Forbidden"' + negative: true - - name: Datadog Application Secret - id: kingfisher.datadog.2 + - name: Datadog Application Key + id: kingfisher.datadog.3 pattern: | (?xi) - \b - (?: - dd[_-]?\w{0,8}[_-]?(?:key|secret) | - datadog | - dog - ) + \b(?:datadog|dd) (?:.|[\n\r]){0,64}? + (?:app(?:lication)?[_-]?key|dd[_-]?application[_-]?key|secret|private|access|token) + (?:.|[\n\r]){0,32}? \b ( - [a-z0-9]{40} + [A-Za-z0-9-]{40} ) - \b - min_entropy: 3.3 + \b + pattern_requirements: + min_digits: 3 + min_entropy: 3.5 confidence: medium examples: - - dd_secret_key-3c0c3965368a6b10f7640dbda46abfdca981c2d3 - - datadog_token = BzHpkcs7LujMb3Q1vLRRjbpBNxxYV0ousumYoKJS + - DD_APPLICATION_KEY=abcDEF0123456789abcDEF0123456789abcDEF01 references: - - https://docs.datadoghq.com/account_management/api-app-keys/ \ No newline at end of file + - https://docs.datadoghq.com/account_management/api-app-keys/ + - https://docs.datadoghq.com/getting_started/site/ + depends_on_rule: + - rule_id: kingfisher.datadog.2 + variable: DD_API_KEY + - rule_id: kingfisher.datadog.1 + variable: DD_SITE_DOMAIN + validation: + type: Http + content: + request: + method: GET + # Datadog recommends /api/v2/validate_keys to verify app keys with the key pair + url: "https://api.{{ DD_SITE_DOMAIN }}/api/v2/validate_keys" + headers: + Accept: application/json + DD-API-KEY: "{{ DD_API_KEY }}" + DD-APPLICATION-KEY: "{{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] diff --git a/data/rules/datagov.yml b/data/rules/datagov.yml new file mode 100644 index 0000000..8e10481 --- /dev/null +++ b/data/rules/datagov.yml @@ -0,0 +1,40 @@ +rules: + - name: Data.gov API Key + id: kingfisher.datagov.1 + pattern: | + (?xi) + \b + data\.gov + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [a-zA-Z0-9]{40} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - data.gov_api_key=pBZm2kXbuPdRfzYyarRT0bvcWAisnJg98YJzBJyJ + - data.gov_token=plZJDnKs4OrPeV8wgBr2fYO6VnXb1YPEcVaZbnYI + references: + - https://api.data.gov/docs/developer-manual/ + - https://developer.nrel.gov/docs/api-key/ + - https://developer.nrel.gov/docs/errors/ + validation: + type: Http + content: + request: + method: GET + # NREL (developer.nrel.gov) uses api.data.gov-managed keys and accepts api_key as a query param. + url: "https://developer.nrel.gov/api/alt-fuel-stations/v1.json?limit=1&api_key={{ TOKEN }}" + headers: + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + diff --git a/data/rules/deepgram.yml b/data/rules/deepgram.yml new file mode 100644 index 0000000..5a474f6 --- /dev/null +++ b/data/rules/deepgram.yml @@ -0,0 +1,43 @@ +rules: + - name: Deepgram API Key + id: kingfisher.deepgram.1 + pattern: | + (?xi) + \b + deepgram + (?:.|[\n\r]){0,32}? + \b + ( + [0-9a-f]{40} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - deepgram_key = 948c19ecde2818a1a357fffb14d2fc2a03d3c56e + - 'deepgram-api: 6c8ba06cb14a32d508948606d8b5d9c8f70e493b' + - deepgram = 12e217f37eb173f0c8f1b7309f4207c7dca20186 + - deepgram token 1 == 1f8946087e64b14dffd069b78554e217b3ed34d4 + references: + - https://developers.deepgram.com/docs/authenticating + - https://developers.deepgram.com/reference/management-api/models/list + - https://developers.deepgram.com/reference/list-keys + + validation: + type: Http + content: + request: + method: GET + url: https://api.deepgram.com/v1/projects + headers: + Authorization: Token {{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: ['"name"'] diff --git a/data/rules/deepseek.yml b/data/rules/deepseek.yml index f303c48..fac865e 100644 --- a/data/rules/deepseek.yml +++ b/data/rules/deepseek.yml @@ -2,12 +2,14 @@ rules: - name: DeepSeek API Key id: kingfisher.deepseek.1 pattern: | - (?xi) + (?x) \b ( sk-[a-f0-9]{32} - ) + ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.7 confidence: medium examples: diff --git a/data/rules/definednetworking.yml b/data/rules/definednetworking.yml new file mode 100644 index 0000000..03d65df --- /dev/null +++ b/data/rules/definednetworking.yml @@ -0,0 +1,41 @@ +rules: + - name: Defined Networking API Token + id: kingfisher.definednetworking.1 + pattern: | + (?x) + \b + ( + dnkey- + [A-Z0-9]{26} + - + [A-Z0-9]{52} + ) + \b + min_entropy: 3.6 + confidence: medium + examples: + - 'defined_api_token="dnkey-AHBDSNIG5ATR4LPUX4XTEVXEP4-ACW2JQ45HAWA2XA6FIJNSNBRY2Q4WMSCNNIFSL6VRZQYFZKI2VHA"' + references: + - https://docs.defined.net/api/defined-networking-api/ + - https://docs.defined.net/api/networks-list/ + - https://docs.defined.net/guides/rotating-api-keys/ + validation: + type: Http + content: + request: + method: GET + url: https://api.defined.net/v1/networks + headers: + Accept: application/json + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - report_response: true + # 200 = valid + authorized + # 403 = valid token but missing required scope (still proves the token is real) + - type: StatusMatch + status: [200, 403] + - type: WordMatch + words: + - '"cidr"' + match_all_words: true + diff --git a/data/rules/dependency_track.yml b/data/rules/dependency_track.yml index e41aa32..10f397c 100644 --- a/data/rules/dependency_track.yml +++ b/data/rules/dependency_track.yml @@ -8,6 +8,8 @@ rules: odt_[A-Z0-9]{32,255} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/diffbot.yml b/data/rules/diffbot.yml new file mode 100644 index 0000000..01c4114 --- /dev/null +++ b/data/rules/diffbot.yml @@ -0,0 +1,36 @@ +rules: + - name: Diffbot API Key + id: kingfisher.diffbot.1 + pattern: | + (?xi) + \b + diffbot + (?:.|[\n\r]){0,32}? + \b + ( + [0-9a-z]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + examples: + - diffbot_key = a7424adbafc4624e61482d0f60e43016 + references: + - https://docs.diffbot.com/reference/account + validation: + type: Http + content: + request: + method: GET + url: >- + https://api.diffbot.com/v4/account?token={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"name"' + - '"email"' + - '"planCredits"' \ No newline at end of file diff --git a/data/rules/digitalocean.yml b/data/rules/digitalocean.yml index 1ae7471..b6ca932 100644 --- a/data/rules/digitalocean.yml +++ b/data/rules/digitalocean.yml @@ -2,13 +2,15 @@ rules: - name: DigitalOcean API Key id: kingfisher.digitalocean.1 pattern: | - (?xi) + (?x) \b ( (?:dop|doo)_v1_ [a-f0-9]{64} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: @@ -32,13 +34,11 @@ rules: - name: DigitalOcean Refresh Token id: kingfisher.digitalocean.2 pattern: | - (?xi) - \b + (?x) ( dor_v1_ [a-f0-9]{64} ) - \b min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/discord.yml b/data/rules/discord.yml index 6b8809b..59df775 100644 --- a/data/rules/discord.yml +++ b/data/rules/discord.yml @@ -4,12 +4,15 @@ rules: pattern: | (?xi) ( - https://discord\.com/api/webhooks/ - \d{18} + https://discord(app)?\.com/api/webhooks/ + [0-9]{17,20} )/ ( - [0-9a-z_\-]{68} + [0-9a-z_\-]{60,68} ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: @@ -32,7 +35,6 @@ rules: id: kingfisher.discord.2 pattern: | (?xi) - \b ( [MNO][A-Z0-9_-]{23}\.[A-Z0-9_-]{6}\.[A-Z0-9_-]{27} ) @@ -61,17 +63,14 @@ rules: id: kingfisher.discord.3 pattern: | (?xi) - \b (?:discord|botid|bot_id) (?:.|[\n\r]){0,64}? - \b ( \d{17,19} ) - \b min_entropy: 3.5 visible: false confidence: medium examples: - discord = 12345678901234567 - - 'bot_id: "123456789012345678"' \ No newline at end of file + - 'bot_id: "123456789012345678"' diff --git a/data/rules/disqus.yml b/data/rules/disqus.yml new file mode 100644 index 0000000..8ca8fa8 --- /dev/null +++ b/data/rules/disqus.yml @@ -0,0 +1,39 @@ +rules: + - name: Disqus API Key + id: kingfisher.disqus.1 + pattern: | + (?xi) + \b + disqus + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [a-zA-Z0-9]{64} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.3 + confidence: medium + examples: + - disqus_secret_key = jK5HbxY2QrPn7vMNL8tADcF3mWg4kXqR9sBdZyE1hVuT6fGwJpC0nI9vUxY2aM3K + - DISQUS_PRIVATE_TOKEN = Nh7vRf3mKp9wXc5tJq2YbL8sAg4dB6TzWeUx1nGQjCkPyDHVME0aI1FSx2Z5vY3n + references: + - https://disqus.com/api/docs/requests/ + - https://disqus.com/api/docs/threads/list/ + validation: + type: Http + content: + request: + method: GET + url: "https://disqus.com/api/3.0/threads/list.json?limit=1&api_secret={{ TOKEN }}" + headers: + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"code":0', '"response"'] diff --git a/data/rules/django.yml b/data/rules/django.yml index c335542..c9efa82 100644 --- a/data/rules/django.yml +++ b/data/rules/django.yml @@ -5,7 +5,6 @@ rules: (?x) [DJANGO]\w{0,8}SECRET_KEY .{1,16}? - \b ( [A-Za-z0-9*!$@\#&_%^-]{45,55} ) diff --git a/data/rules/docker.yml b/data/rules/docker.yml new file mode 100644 index 0000000..1503cd4 --- /dev/null +++ b/data/rules/docker.yml @@ -0,0 +1,50 @@ +rules: + - name: Docker Registry Credentials (auths JSON) + id: kingfisher.docker.1 + pattern: | + (?xis) + "auths"\s*:\s*\{ + [^}]*? + " (?P (?:https?:\/\/)? [a-z0-9.\-:+/]+ ) "\s*:\s*\{ + [^}]*? + "auth"\s*:\s*"(?P [A-Za-z0-9+/=]{16,} )" + [^}]*? + \} + [^}]*? + \} + pattern_requirements: + min_digits: 2 + min_entropy: 2.0 + confidence: medium + examples: + - | + { + "auths": { + "quay.io": { + "auth": "cmhkaCtyaHRhcDowM1BERl1RQTJQTDlaQUE5T1gzSU9IQjFYTUlXOVNGNU1XRzNSRVRHNThKVXpKMzEwV0ZZRVMOQTdGMExMNOYx" + } + } + } + - | + {"auths":{"index.docker.io/v1/":{"auth":"dXNlcjp0b2tlbg=="}}} + references: + - https://distribution.github.io/distribution/spec/api/ + validation: + type: Http + content: + request: + method: GET + url: > + {%- assign r = REG -%} + {%- if r contains "://" -%} + {{ r | replace: "/$", "" }}/v2/auth + {%- else -%} + https://{{ r }}/v2/auth + {%- endif -%} + headers: + Authorization: "Basic {{ B64 }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] \ No newline at end of file diff --git a/data/rules/dockerhub.yml b/data/rules/dockerhub.yml index e26d108..5e0be16 100644 --- a/data/rules/dockerhub.yml +++ b/data/rules/dockerhub.yml @@ -2,12 +2,14 @@ rules: - name: Docker Hub Personal Access Token id: kingfisher.dockerhub.1 pattern: | - (?xi) + (?x) \b ( - dckr_pat_[A-Z0-9_-]{27} + dckr_pat_[A-Za-z0-9_-]{27} ) - (?: $ | [^A-Z0-9_-] ) + (?: $ | [^A-Za-z0-9_-] ) + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: @@ -29,4 +31,35 @@ rules: - status: - 200 type: StatusMatch - url: https://hub.docker.com/v2/access-tokens?page_size=1 \ No newline at end of file + url: https://hub.docker.com/v2/access-tokens?page_size=1 + - name: Docker Hub Organization Access Token + id: kingfisher.dockerhub.2 + pattern: | + (?x) + \b + ( + dckr_oat_[A-Za-z0-9_-]{32} + ) + (?: $ | [^A-Za-z0-9_-] ) + pattern_requirements: + min_digits: 2 + min_entropy: 3.3 + confidence: medium + examples: + - docker login -u docker-test -p dckr_oat_7bA9zRt5-JqX3vP0l_MnY8sK2wE-dF6h + references: + - https://docs.docker.com/enterprise/security/access-tokens/ + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + Accept: application/json + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://hub.docker.com/v2/access-tokens?page_size=1 diff --git a/data/rules/doppler.yml b/data/rules/doppler.yml index c8c302a..fde9282 100644 --- a/data/rules/doppler.yml +++ b/data/rules/doppler.yml @@ -4,8 +4,12 @@ rules: pattern: | (?xi) \b - (dp\.ct\.[A-Z0-9]{40,44}) + ( + dp\.ct\.[A-Z0-9]{40,44} + ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/droneci.yml b/data/rules/droneci.yml index 8e26b66..e6a4bd1 100644 --- a/data/rules/droneci.yml +++ b/data/rules/droneci.yml @@ -15,6 +15,8 @@ rules: [a-f0-9]{32,64} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/dropbox.yml b/data/rules/dropbox.yml new file mode 100644 index 0000000..d21eb80 --- /dev/null +++ b/data/rules/dropbox.yml @@ -0,0 +1,35 @@ +rules: + - name: Dropbox API secret/key + id: kingfisher.dropbox.1 + pattern: | + (?xi) + \b + ( + sl\.[A-Z0-9\-\_]{130,152} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.3 + confidence: medium + examples: + - 'curl -X POST https://api.dropboxapi.com/2/users/get_current_account --header "Authorization: Bearer sl.hAi61Jx1hs3XlhrnsCxnctrEmxK2Q-UK29hbdxxHyAykldSeHmipBAauxTzuBEIqt2jdyyUZw8kgY3t_ars-PNIPS27ySa1ab22132U3sUuqYTXHzf2XlvMxSesUhkzx2G11_9W1f-eo"' + - ' "access_token": "sl.AbX9y6Fe3AuH5o66-gmJpR032jwAwQPIVVzWXZNkdzcYT02akC2de219dZi6gxYPVnYPrpvISRSf9lxKWJzYLjtMPH-d9fo_0gXex7X37VIvpty4-G8f4-WX45AcEPfRnJJDwzv-",' + - 'curl -X POST https://api.dropboxapi.com/2/users/get_current_account --header "Authorization: Bearer sl.hAi61Jx1hs3XlhrnsCxnctrEmxK2Q-UK29hbdxxHyAykldSeHmipBAauxTzuBEIqt2jdyyUZw8kgY3t_ars-PNIPS27ySa1ab22132U3sUuqYTXHzf2XlvMxSesUhkzx2G11_9W1f-eo"' + - ' "access_token": "sl.AbX9y6Fe3AuH5o66-gmJpR032jwAwQPIVVzWXZNkdzcYT02akC2de219dZi6gxYPVnYPrpvISRSf9lxKWJzYLjtMPH-d9fo_0gXex7X37VIvpty4-G8f4-WX45AcEPfRnJJDwzv-",' + - sl.lMcWXvCOCR9yRAOXwDg1V_VTsbEXdu3Xpsgg35GQTrDahoIuUnij4H5b6bMnwobZx4XnniaQappZU9j-CCWd-LYW81juiU04-yvLhOPV47Nj7Fs8XSE-CweZp6j9nw4E2oWWJ5 + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + method: POST + response_matcher: + - report_response: true + - match_all_words: true + type: WordMatch + words: + - '"account_id":' + - '"email":' + url: https://api.dropboxapi.com/2/users/get_current_account \ No newline at end of file diff --git a/data/rules/duffel.yml b/data/rules/duffel.yml new file mode 100644 index 0000000..da0a928 --- /dev/null +++ b/data/rules/duffel.yml @@ -0,0 +1,34 @@ +rules: + - name: Duffel API Token + id: kingfisher.duffel.1 + pattern: | + (?xi) + \b + ( + duffel_(?:test|live)_[a-z0-9_\-=]{43} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.2 + confidence: medium + examples: + - DUFFEL_TOKEN=duffel_test_qwertyuiopasdfghjklzxcvbnm123456789abcdefgh + - 'Authorization: "Bearer duffel_live_abcd1234efgh5678ijkl9012mnop3456qrstuvwxyza"' + references: + - https://duffel.com/docs/api + validation: + type: Http + content: + request: + method: GET + url: https://api.duffel.com/airlines + headers: + Authorization: 'Bearer {{ TOKEN }}' + Accept: application/json + Duffel-Version: v1 + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 diff --git a/data/rules/dynatrace.yml b/data/rules/dynatrace.yml index af8a415..29226f4 100644 --- a/data/rules/dynatrace.yml +++ b/data/rules/dynatrace.yml @@ -12,6 +12,8 @@ rules: [A-Z0-9]{64} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/easypost.yml b/data/rules/easypost.yml index 757a70a..0c85fb6 100644 --- a/data/rules/easypost.yml +++ b/data/rules/easypost.yml @@ -9,6 +9,8 @@ rules: [A-Za-z0-9]{54} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/elevenlabs.yml b/data/rules/elevenlabs.yml new file mode 100644 index 0000000..9378a34 --- /dev/null +++ b/data/rules/elevenlabs.yml @@ -0,0 +1,38 @@ +rules: + - name: ElevenLabs API Key + id: kingfisher.elevenlabs.1 + pattern: | + (?xi) + \b + ( + sk_ + [0-9a-f]{48} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - sk_2a30e5a0d39d5f2c5f6a9d2f95cd016049a6323985479bfd + - sk_da9c0613fdeecfab10b302d6f39a3e371f774feb9eafed56 + - sk_82a331629e2128ef70396600809b6a2ff4e433154fa27e1b + references: + - https://elevenlabs.io/docs/api-reference/authentication + - https://elevenlabs.io/docs/api-reference/user/subscription/get + + validation: + type: Http + content: + request: + method: GET + url: https://api.elevenlabs.io/v1/user/subscription + headers: + xi-api-key: '{{ TOKEN }}' + response_matcher: + - report_response: true + - type: WordMatch + match_all_words: false + words: + - '"tier"' + - '"missing_permissions"' diff --git a/data/rules/endorlabs.yml b/data/rules/endorlabs.yml new file mode 100644 index 0000000..3be6b1c --- /dev/null +++ b/data/rules/endorlabs.yml @@ -0,0 +1,61 @@ +rules: + - name: Endor Labs API Key + id: kingfisher.endorlabs.1 + visible: false + confidence: medium + min_entropy: 3.0 + pattern: | + (?xi) + \b + ENDOR_API_CREDENTIALS_KEY + (?:.|[\n\r]){0,32}? + ( + endr\+[A-Za-z0-9-]{16} + ) + \b + examples: + - ENDOR_API_CREDENTIALS_KEY=endr+foo1234567890abc + pattern_requirements: + min_digits: 2 + + - name: Endor Labs API Secret + id: kingfisher.endorlabs.2 + pattern: | + (?xi) + \b + ENDOR_API_CREDENTIALS_SECRET + (?:.|[\n\r]){0,32}? + ( + endr\+[A-Za-z0-9-]{16} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - ENDOR_API_CREDENTIALS_SECRET=endr+bar1234567890abc + references: + - https://docs.endorlabs.com/rest-api/authentication/ + depends_on_rule: + - rule_id: kingfisher.endorlabs.1 + variable: ENDOR_API_KEY + validation: + type: Http + content: + request: + method: POST + # Endor Labs exchanges key+secret for an ENDOR_TOKEN via this endpoint + url: https://api.endorlabs.com/v1/auth/api-key + headers: + Content-Type: application/json + Accept: application/json + body: | + {"key":"{{ ENDOR_API_KEY }}","secret":"{{ TOKEN }}"} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + words: ['"token"'] diff --git a/data/rules/eraserio.yml b/data/rules/eraserio.yml new file mode 100644 index 0000000..f56612f --- /dev/null +++ b/data/rules/eraserio.yml @@ -0,0 +1,34 @@ +rules: + - name: Eraser API Key + id: kingfisher.eraser.1 + pattern: | + (?xi) + \b + eraser + (?:[^A-Za-z0-9]{0,16})? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:[^A-Za-z0-9]{0,16})? + \b + ( + [A-Za-z0-9]{20} + ) + \b + min_entropy: 3.5 + confidence: medium + examples: + - eraser_token = Q7MD4J9L2X0B6R3T8W1P + references: + - https://eraser.io/docs/api/authentication + validation: + type: Http + content: + request: + method: GET + url: https://app.eraser.io/api/reports/usage?rangeDays=1 + headers: + Authorization: "Bearer {{ TOKEN }}" + accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200, 403] \ No newline at end of file diff --git a/data/rules/eventbrite.yml b/data/rules/eventbrite.yml new file mode 100644 index 0000000..c0856df --- /dev/null +++ b/data/rules/eventbrite.yml @@ -0,0 +1,42 @@ +rules: + - name: Eventbrite API Key + id: kingfisher.eventbrite.1 + pattern: | + (?xi) + \b + eventbrite + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [0-9A-Z]{20} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - eventbrite secretkey X7W8HTTHLVYXPPVRZJZS + - eventbrite privatekey YTR4GR5T89WQP8HJKLDF + - '"eventbrite private access key ZXC2JK3HV4TY5UIO6PLK"' + - eventbrite token ABCDEF1234567890QRST + references: + - https://www.eventbrite.com/platform/docs/authentication + - https://www.eventbrite.com/platform/docs/organizations + validation: + type: Http + content: + request: + method: GET + url: https://www.eventbriteapi.com/v3/users/me/organizations/ + headers: + Authorization: "Bearer {{ TOKEN }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"organizations"'] diff --git a/data/rules/exaai.yml b/data/rules/exaai.yml new file mode 100644 index 0000000..1110512 --- /dev/null +++ b/data/rules/exaai.yml @@ -0,0 +1,46 @@ +rules: + - name: Exa AI API Key + id: kingfisher.exa.1 + pattern: | + (?xi) + (?: + \b(?:exa|exa[_-]?api|exa[_-]?key|exa[_-]?api[_-]?key)\b + (?:.|[\n\r]){0,96}? + | + \bx-api-key\b + (?:\s*[:=]\s*|(?:.|[\n\r]){0,16}?) + ) + \b + ( + [a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12} + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.0 + confidence: medium + examples: + - EXA_API_KEY=3f5a9c1e-2b4d-4a6f-8c10-1d2e3f4a5b6c + - 'exa_api_key: "3f5a9c1e-2b4d-4a6f-8c10-1d2e3f4a5b6c"' + - 'x-api-key: 3f5a9c1e-2b4d-4a6f-8c10-1d2e3f4a5b6c' + references: + - https://docs.exa.ai/reference/answer + - https://docs.exa.ai/reference/getting-started + validation: + type: Http + content: + request: + method: POST + url: https://api.exa.ai/answer + headers: + x-api-key: "{{ TOKEN }}" + Content-Type: application/json + Accept: application/json + body: | + {"query":"ping","text":false} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"answer"'] diff --git a/data/rules/facebook.yml b/data/rules/facebook.yml index d3ef582..7fca27a 100644 --- a/data/rules/facebook.yml +++ b/data/rules/facebook.yml @@ -2,17 +2,19 @@ rules: - name: Facebook App ID id: kingfisher.facebook.1 pattern: | - (?xi) - \b + (?xi) + \b (?:facebook|fb) - (?:.|[\n\r]){0,8}? - (?:APP|APPLICATION) - (?:.|[\n\r]){0,16}? - \b + (?:.|[\n\r]){0,32}? + (?:APP|APPLICATION|CLIENT) + (?:.|[\n\r]){0,32}? + \b ( \d{15} - ) - \b + ) + \b + pattern_requirements: + min_digits: 15 min_entropy: 2.0 visible: false confidence: medium @@ -25,18 +27,24 @@ rules: id: kingfisher.facebook.2 pattern: | (?xi) - \b (?: facebook | fb ) - .? - (?: api | app | application | client | consumer | customer | secret | key ) - .? - (?: key | oauth | sec | secret )? + \b + (?: facebook | fb ) + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN|AUTH) + (?:.|[\n\r]){0,48}? .{0,2} \s{0,20} .{0,2} \s{0,20} .{0,2} - \b ([a-z0-9]{32}) \b + \b + ( + [a-z0-9]{32} + ) + \b examples: - ' # config.facebook.key = "34cebc81c056a21bc66e212f947d73ec"' - " var fbApiKey = '0278fc1adf6dc1d82a156f306ce2c5cc';" - ' fbApiKey: "171e84fd57f430fc59afa8fad3dbda2a",' - '"facebook appSecret = "ce3f9f0362bbe5ab01dfc8ee565e4372"' + pattern_requirements: + min_digits: 2 validation: type: Http content: @@ -69,9 +77,12 @@ rules: (?:access_token|access[\s-]token) (?:.|[\n\r]){0,32}? )? + ( + EAACEdEose0cBA[A-Z0-9]{20,} + ) \b - (EAACEdEose0cBA[A-Z0-9]{20,}) - \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/fastly.yml b/data/rules/fastly.yml index c5d4fcb..07cadce 100644 --- a/data/rules/fastly.yml +++ b/data/rules/fastly.yml @@ -9,8 +9,12 @@ rules: (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) (?:.|[\n\r]){0,32}? \b - ([a-z0-9_-]{32}) + ( + [a-z0-9_-]{32} + ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/figma.yml b/data/rules/figma.yml index 7cc6f00..b5faf90 100644 --- a/data/rules/figma.yml +++ b/data/rules/figma.yml @@ -8,6 +8,8 @@ rules: figd_[A-Z0-9_-]{38,42} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: @@ -36,14 +38,14 @@ rules: (?xi) figma (?:.|[\n\r]){0,32}? - \b ( [0-9A-F]{4} -[0-9A-F]{8} (?:-[0-9A-F]{4}){3} -[0-9A-F]{12} ) - \b + pattern_requirements: + min_digits: 2 examples: - "--header='X-Figma-Token: 1394-0ca7a5be-8e22-40ee-8c40-778d41ab2313'" references: diff --git a/data/rules/fileio.yml b/data/rules/fileio.yml index 7bf255e..804fcd4 100644 --- a/data/rules/fileio.yml +++ b/data/rules/fileio.yml @@ -14,6 +14,8 @@ rules: \.[A-Z0-9]{20} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/filezilla.yml b/data/rules/filezilla.yml new file mode 100644 index 0000000..1fb3308 --- /dev/null +++ b/data/rules/filezilla.yml @@ -0,0 +1,38 @@ +rules: + - name: FileZilla base64 encoded password + id: kingfisher.filezilla.1 + pattern: ]*\bencoding\s*=\s*"(?:base64|radix64)"[^>]*>\s*([A-Za-z0-9+/]{8,}={0,2})\s* + min_entropy: 3.2 + confidence: low + pattern_requirements: + ignore_if_contains: + - "ZXhhbXBsZQ==" # "example" (base64) + - "cGFzc3dvcmQ=" # "password" (base64) + - "Y2hhbmdlbWU=" # "changeme" (base64) + examples: + - 'VGhpc0lzQVRlc3RQYXNzd29yZA==' + - 'NjllNWU5ZWMwZDU0MmU5Y2QwOTY4MWM5YzZhMDdkYWVmNjg3OWE3MDMzM2Q4MWJmCg==' + references: + - https://forum.filezilla-project.org/viewtopic.php?style=246&t=38820 + - https://forum.filezilla-project.org/viewtopic.php?p=133138 + - https://forum.filezilla-project.org/viewtopic.php?t=24758 + + - name: FileZilla stored password (Pass plaintext) + id: kingfisher.filezilla.2 + pattern: \s*([^<\r\n]{4,128})\s* + min_entropy: 2.8 + confidence: low + pattern_requirements: + ignore_if_contains: + - example + - Example + - password + - Password + - changeme + - ChangeMe + examples: + - "ExamplePas123" + - "superS3cret!" + references: + - https://stackoverflow.com/questions/29790136/filezilla-plain-text-password + - https://forum.filezilla-project.org/viewtopic.php?t=24758 diff --git a/data/rules/finicity.yml b/data/rules/finicity.yml index 5273fbf..4a29310 100644 --- a/data/rules/finicity.yml +++ b/data/rules/finicity.yml @@ -13,6 +13,8 @@ rules: [a-f0-9]{32} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/finnhub.yml b/data/rules/finnhub.yml new file mode 100644 index 0000000..3d71fb6 --- /dev/null +++ b/data/rules/finnhub.yml @@ -0,0 +1,35 @@ +rules: + - name: Finnhub API Token + id: kingfisher.finnhub.1 + pattern: | + (?xi) + \b + finnhub + (?:.|[\n\r]){0,24}? + \b + ( + [a-z0-9]{20} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + examples: + - FINNHUB_API_KEY=cd3f1a2b3c4d5e6f7a8b + - '"finnhubToken": "9b8a7c6d5e4f3a2b1c0d"' + references: + - https://finnhub.io/docs/api + validation: + type: Http + content: + request: + method: GET + url: https://finnhub.io/api/v1/stock/profile2?symbol=MDB&token={{ TOKEN }} + headers: + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 diff --git a/data/rules/firecrawl.yml b/data/rules/firecrawl.yml new file mode 100644 index 0000000..80665b3 --- /dev/null +++ b/data/rules/firecrawl.yml @@ -0,0 +1,34 @@ +rules: + - name: Firecrawl API Key + id: kingfisher.firecrawl.1 + pattern: | + (?xi) + \b + ( + fc-[a-f0-9]{32} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.0 + validation: + type: Http + content: + request: + method: GET + url: "https://api.firecrawl.dev/v1/crawl/active" + headers: + Authorization: "Bearer {{TOKEN}}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"success"' + references: + - https://www.firecrawl.dev + - https://docs.firecrawl.dev/api-reference/introduction#authentication + examples: + - 'app = FirecrawlApp(api_key="fc-7da8b1ca1d2150c496e91440d777fea8")' diff --git a/data/rules/fireworksai.yml b/data/rules/fireworksai.yml new file mode 100644 index 0000000..f96d251 --- /dev/null +++ b/data/rules/fireworksai.yml @@ -0,0 +1,37 @@ +rules: + - name: Fireworks.ai API Key + id: kingfisher.fireworks.1 + pattern: | + (?xi) + \b + ( + fw_[A-Z0-9]{24} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.5 + validation: + type: Http + content: + request: + method: GET + url: "https://api.fireworks.ai/inference/v1/models" + headers: + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"owned_by"' + - '"data"' + match_all_words: true + references: + - https://readme.fireworks.ai/reference/getting-started-with-the-api + examples: + - "fw_3ZL5ji26Tp7baYrW5S2pA5xi" + - "fw_3ZaW5fSpx5GTnHpRGb8CPu2V" + - "fw_3ZSU8ymvmZ38YPv8uwbZHAyW" diff --git a/data/rules/fleetbase.yml b/data/rules/fleetbase.yml new file mode 100644 index 0000000..e4d05c3 --- /dev/null +++ b/data/rules/fleetbase.yml @@ -0,0 +1,36 @@ +rules: + - name: Fleetbase API Key + id: kingfisher.fleetbase.1 + pattern: | + (?xi) + \b + ( + flb_(?:live|test)_[0-9a-zA-Z]{20,64} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - flb_live_1234567890abcdefGHIJ + - flb_test_1234567890abcdefGHIJ + - 'Authorization: Bearer flb_live_1234567890abcdefGHIJ' + categories: + - api + - secret + references: + - https://docs.fleetbase.io/developers/api/ + validation: + type: Http + content: + request: + method: GET + url: "https://api.fleetbase.io/v1" + headers: + Authorization: "Bearer {{ TOKEN }}" + Accept: "application/json" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] diff --git a/data/rules/flickr.yml b/data/rules/flickr.yml new file mode 100644 index 0000000..faac4c7 --- /dev/null +++ b/data/rules/flickr.yml @@ -0,0 +1,74 @@ +rules: + - name: Flickr API Key + id: kingfisher.flickr.1 + pattern: | + (?xi) + \b + flickr + (?:.|[\n\r]){0,16}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [a-f0-9]{32} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.0 + validation: + type: Http + content: + request: + method: GET + url: "https://www.flickr.com/services/rest/?method=flickr.test.login&api_key={{TOKEN}}&format=json&nojsoncallback=1" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"Invalid API Key' + negative: true + references: + - https://www.flickr.com/services/api/ + - https://www.flickr.com/services/api/flickr.test.login.html + examples: + - "flickr_api_key: d1953fb62a9798593bfdb4287ed2423e" + - name: Flickr OAuth Token + id: kingfisher.flickr.2 + pattern: | + (?xi) + \b + flickr + (?:.|[\n\r]){0,32}? + (?:OAUTH|ACCESS|TOKEN)? + (?:.|[\n\r]){0,32}? + ( + [a-f0-9]{32} + ) + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.0 + validation: + type: Http + content: + request: + method: GET + url: "https://www.flickr.com/services/rest/?method=flickr.auth.oauth.checkToken&api_key={{TOKEN}}&oauth_token={{TOKEN}}&format=json&nojsoncallback=1" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"stat":"ok"' + - '"oauth":' + match_all_words: true + references: + - https://www.flickr.com/services/api/ + - https://www.flickr.com/services/api/flickr.auth.oauth.checkToken.html + examples: + - "flickr_oauth_token: a8c1b9f1d9d34aa5a1edbd43234bcdef" \ No newline at end of file diff --git a/data/rules/flyio.yml b/data/rules/flyio.yml new file mode 100644 index 0000000..4764279 --- /dev/null +++ b/data/rules/flyio.yml @@ -0,0 +1,38 @@ +rules: + - name: Fly.io API Token + id: kingfisher.flyio.1 + pattern: | + (?x) + \b + ( + FlyV1\s[A-Za-z0-9=_\-,/+]{100,} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 4.0 + validation: + type: Http + content: + request: + method: POST + url: "https://api.fly.io/graphql" + headers: + Authorization: "Bearer {{TOKEN}}" + Content-Type: "application/json" + body: '{"query":"query { viewer { id email name } }"}' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"data":' + - '"viewer":' + - '"email":' + match_all_words: true + references: + - https://fly.io/docs/reference/graphql/ + examples: + - "FlyV1 fm2_lJPECAAAAAAACcIGxBBr3eFBrCTMuIb8FOeUWTf0wrVodHRwczovL2FwaS5mbHkuaW8vdjGUAJLOABLVch8Lk7lodHRwczovL2FwaS5mbHkuaW8vYWFhL3YxxDymEU+hbM4EZ6KeG6k1EWHesm6buoSgwBS8yBSLRiHumjZXcxZdCJ2gJ3PN//X8DKdsEFfZyS03lbJtPpwETgpeWpRbsMxfQb2ZkR4zYNi/IOOKYxW2h6DYBHahN3DBS7wY54AIgap8IMAtBJ3imo77+vAeAeZ/0aPq3XJoGdg4+WHDWkxectcZoZROe8Qgnq4tV2yiT2Mx5wmoK+Kw1u33egtwLCEBDC5ZakEM7pI=,fm2_lJPETgpeWpRbsMxfQb2ZkR4zYNi/IOOKYxW2h6DYBHahN3DBS7wY54AIgap8IMAtBJ3imo77+vAeAeZ/0aPq3XJoGdg4+WHDWkxectcZoZROe8QQCY7oFTr+3MOM0p5/Cww1AsO5aHR0cHM6Ly9hcGkuZmx5LmlvL2FhYS92MZgEks5ooPjrzwAAAAEkmRcJF84AEhmjCpHOABIZowzEEMLj9PMThJElQN/ARptX7D3EILDtbLx/4cBgt6fX+zb6/FxsLxV2A/y4e4BeU1SunG+O" diff --git a/data/rules/foursquare.yml b/data/rules/foursquare.yml new file mode 100644 index 0000000..799a540 --- /dev/null +++ b/data/rules/foursquare.yml @@ -0,0 +1,62 @@ +rules: + - name: Foursquare Client ID + id: kingfisher.foursquare.1 + visible: false + confidence: low + min_entropy: 0.0 + pattern: | + (?xi) + (?: + \bclient_id\b\s*[:=]\s*["']? + | + \bclient_id= + ) + ( + [0-9A-Z]{48} + ) + \b + examples: + - client_id=0F12A345BB67C8D901EFG23H45IJKL67MNO89PQ12RST34UV + - 'client_id: "0F12A345BB67C8D901EFG23H45IJKL67MNO89PQ12RST34UV"' + + - name: Foursquare Client Secret + id: kingfisher.foursquare.2 + pattern: | + (?xi) + \b + foursquare + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,16}? + \b + ( + [0-9A-Z]{48} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - 'foursquare_secret=0F12A345BB67C8D901EFG23H45IJKL67MNO89PQ12RST34UV' + - 'foursquare client_secret: "0F12A345BB67C8D901EFG23H45IJKL67MNO89PQ12RST34UV"' + references: + - https://docs.foursquare.com/developer/reference/v2-authentication + - https://docs.foursquare.com/developer/reference/upcoming-changes + depends_on_rule: + - rule_id: kingfisher.foursquare.1 + variable: FOURSQUARE_CLIENT_ID + validation: + type: Http + content: + request: + method: GET + url: "https://api.foursquare.com/v2/venues/search?ll=34.0522,-118.2437&query=coffee&client_id={{ FOURSQUARE_CLIENT_ID }}&client_secret={{ TOKEN }}&v=20211019&limit=1" + headers: + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"response"', '"venues"'] diff --git a/data/rules/frame.io.yml b/data/rules/frame.io.yml index a5ef94b..035aefc 100644 --- a/data/rules/frame.io.yml +++ b/data/rules/frame.io.yml @@ -7,7 +7,11 @@ rules: ( fio-u-(?:[A-Z0-9_-]{16}){4} ) - \b + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/frameio.yml b/data/rules/frameio.yml new file mode 100644 index 0000000..cfbfe10 --- /dev/null +++ b/data/rules/frameio.yml @@ -0,0 +1,33 @@ +rules: + - name: Frame.io API Token + id: kingfisher.frameio.1 + pattern: | + (?xi) + \b + ( + fio-u-[a-z0-9\-_=]{64} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.3 + confidence: medium + examples: + - FRAMEIO_TOKEN=fio-u-a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6q7r8s9t0u1v2w3x4y5z6a7b8c9d0e1f2 + - '"Authorization": "Bearer fio-u-b1c2d3e4f5g6h7i8j9k0l1m2n3o4p5q6r7s8t9u0v1w2x3y4z5a6b7c8d9e0f123"' + references: + - https://developer.frame.io/docs/api/authentication + validation: + type: Http + content: + request: + method: GET + url: https://api.frame.io/v2/me + headers: + Authorization: 'Bearer {{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 diff --git a/data/rules/freshbooks.yml b/data/rules/freshbooks.yml new file mode 100644 index 0000000..d63c14a --- /dev/null +++ b/data/rules/freshbooks.yml @@ -0,0 +1,36 @@ +rules: + - name: FreshBooks Access Token + id: kingfisher.freshbooks.1 + pattern: | + (?xi) + \b + freshbooks + (?:.|[\n\r]){0,32}? + \b + ( + [a-z0-9]{64} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - FRESHBOOKS_TOKEN=0f1e2d3c4b5a69788776655443322110ffeeddccbbaa00998877665544332211 + - '"freshbooksAccess": "abcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcd"' + references: + - https://www.freshbooks.com/api/authentication + validation: + type: Http + content: + request: + method: GET + url: https://api.freshbooks.com/auth/api/v1/users/me + headers: + Authorization: 'Bearer {{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 diff --git a/data/rules/freshdesk.yml b/data/rules/freshdesk.yml new file mode 100644 index 0000000..8cd3f50 --- /dev/null +++ b/data/rules/freshdesk.yml @@ -0,0 +1,62 @@ +rules: + - name: Freshdesk Domain + id: kingfisher.freshdesk.1 + visible: false + confidence: low + min_entropy: 0.0 + pattern: | + (?xi) + \b + ( + [0-9a-z-]{1,63}\.freshdesk\.com + ) + \b + examples: + - acme-support.freshdesk.com + - mycompany-helpdesk.freshdesk.com + + - name: Freshdesk API Key + id: kingfisher.freshdesk.2 + pattern: | + (?xi) + \b + freshdesk + (?:.|[\n\r]){0,64}? + (?:api[_-]?key|secret|private|access|key|token) + (?:.|[\n\r]){0,32}? + \b + ( + [0-9A-Z]{20} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.3 + confidence: medium + examples: + - 'FRESHDESK_API_KEY=abcdefghij1234567890' + - 'freshdesk token: ABCDEFGHIJ1234567890' + references: + - https://developers.freshdesk.com/api/#authentication + - https://developers.freshworks.com/docs/app-sdk/v3.0/support_agent/rest-apis/ + depends_on_rule: + - rule_id: kingfisher.freshdesk.1 + variable: FRESHDESK_DOMAIN + validation: + type: Http + content: + request: + method: GET + url: "https://{{ FRESHDESK_DOMAIN }}/api/v2/agents/me" + headers: + Accept: application/json + # Freshdesk API key auth is HTTP Basic where username=apikey and password can be any dummy value (commonly "X"). + # Docs note you can use a dummy password and (when using Authorization header) base64("apikey:X") + Authorization: "Basic {{ TOKEN | append: ':X' | b64enc }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + words: ['"id"'] diff --git a/data/rules/friendli.yml b/data/rules/friendli.yml new file mode 100644 index 0000000..6881ffc --- /dev/null +++ b/data/rules/friendli.yml @@ -0,0 +1,37 @@ +rules: + - name: Friendli.ai API Key + id: kingfisher.friendli.1 + pattern: | + (?xi) + \b + ( + flp_[A-Z0-9]{46} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.0 + validation: + type: Http + content: + request: + method: GET + url: "https://api.friendli.ai/dedicated/beta/endpoint" + headers: + Authorization: "Bearer {{ TOKEN }}" + Content-Type: "application/json" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"data"' + - '"status"' + references: + - https://docs.friendli.ai/reference/authentication + examples: + - "flp_eb8CAc1OHdVISFraFZXFYQeH1CYtqM2VdYFvV1duniWw32" + - "flp_fYvncz2Ahh4YEfSKbNoT09DWlwPq5I7svZG2l1bdbpOg1c" + - "flp_kGcjWhZQ4zYQnY7b3O6nukAhflKZJeS7pNDhs79IRrfodc" diff --git a/data/rules/gcp.yml b/data/rules/gcp.yml index e23acb4..30d4d1d 100644 --- a/data/rules/gcp.yml +++ b/data/rules/gcp.yml @@ -15,6 +15,8 @@ rules: "auth_provider_x509_cert_url":\s*".+?" (?:[^{}]|\{(?:[^{}]|\{[^{}]*\})*\})* \} + pattern_requirements: + min_digits: 2 min_entropy: 4.5 confidence: high examples: @@ -48,9 +50,13 @@ rules: [=:] \s{0,8} ["']? - ([0-9a-z]{35,40}) + ( + [0-9a-z]{35,40} + ) ["']? - \b + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/generic.yml b/data/rules/generic.yml index 9338797..75d8fc5 100644 --- a/data/rules/generic.yml +++ b/data/rules/generic.yml @@ -5,9 +5,12 @@ rules: (?xi) secret .{0,20} + ( + [0-9a-z]{32,64} + ) \b - ([0-9a-z]{32,64}) - \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: low examples: @@ -18,9 +21,12 @@ rules: (?xi) (?: api_key | apikey | access_key | accesskey ) (?:.|[\n\r]){0,8}? + ( + [0-9a-z][0-9a-z\-._/+]{30,62}[0-9a-z] + ) \b - ([0-9a-z][0-9a-z\-._/+]{30,62}[0-9a-z]) - \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: low examples: @@ -31,7 +37,6 @@ rules: pattern: | (?xi) (?: username | user) - \b (?:.|[\n\r]){0,16}? (?: password | pass ) (?:.|[\n\r]){0,16}? @@ -66,7 +71,6 @@ rules: pattern: | (?xi) (?: username | user) - \b (?:.|[\n\r]){0,16}? (?: password | pass ) (?:.|[\n\r]){0,16}? @@ -89,14 +93,12 @@ rules: pattern: | (?xi) password - \b (?:.|[\n\r]){0,16}? ["'] ([^$<%@.,\s'"(){}&/\#\-][^\s'"(){}/]{4,}) (?# password ) ["'] min_entropy: 3.3 confidence: low - categories: [fuzzy, generic, secret] examples: - | password = "super$ecret" @@ -128,10 +130,9 @@ rules: 'policy_path': os.path.join(TEST_DIR, 'policy.json') }) - name: Weak Password Pattern - id: kingfisher.weak_password.1 + id: kingfisher.generic.6 pattern: | (?xi) - \b ( blink\d{3,6} |correcthorsebatterystaple\d{0,6} @@ -145,7 +146,6 @@ rules: |qwerty\d{3,6} |trustno\d{1,6} ) - \b min_entropy: 1.0 confidence: low examples: @@ -159,7 +159,6 @@ rules: pattern: | (?xi) (?: db_user | db_USERNAME | db_name) - \b (?:.|[\n\r]){0,8}? ["'] ([^"']{5,40}) ["'] (?:.|[\n\r]){0,32}? @@ -193,4 +192,21 @@ rules: password = 'abuser123456' # some other comment - | user = 'Aladdin' - password = 'open sesame' \ No newline at end of file + password = 'open sesame' + - name: Docker Robot Credentials (plaintext pair) + id: kingfisher.generic.9 + pattern: | + (?xi) + ( + (?P [a-z0-9._-]+ \+ [a-z0-9._-]+ ) + : + (?P [A-Z0-9]{32,80} ) + ) + \b + min_entropy: 2.0 + confidence: low + examples: + - some+thing:02PDFMQN2PL2ZAB9OX3IOHC1XMIW1SE5NWG3RETG58JUZJ310WFYESRA7F0LM461 + - org+builder:1C2F9D0BB1E67E9F6B3B5B9A2A3D4E5F6A7B8C9D0E1F2A3B4C5D6E7F8A9B0C1 + references: + - https://docs.quay.io/use_quay.html#robot-accounts \ No newline at end of file diff --git a/data/rules/gitalk.yml b/data/rules/gitalk.yml new file mode 100644 index 0000000..d052855 --- /dev/null +++ b/data/rules/gitalk.yml @@ -0,0 +1,21 @@ +rules: + - name: Gitalk OAuth Credentials + id: kingfisher.gitalk.1 + pattern: | + (?x) + \b + new \s+ Gitalk \s* \( \s* \{ \s* + clientID: \s* '([a-f0-9]{20})', \s* + clientSecret: \s* '([a-f0-9]{40})', + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.0 + references: + - https://gitalk.github.io + - https://github.com/gitalk/gitalk + examples: + - | + new Gitalk({ + clientID: 'd17d49be2e680b78a83d', + clientSecret:'9363cb456dda6402cb71d65092490e75c9f11873', diff --git a/data/rules/github.yml b/data/rules/github.yml index 90c9c3b..fb6ce23 100644 --- a/data/rules/github.yml +++ b/data/rules/github.yml @@ -1,23 +1,53 @@ rules: - - name: GitHub Personal Access Token + - name: GitHub Personal Access Token - fine-grained permissions id: kingfisher.github.1 pattern: | - (?xi) - \b - ( - (?: # for token prefixes - ghp| # Personal Access Token - gho| # OAuth Token - ghu| # GitHub App User-to-Server Token - ghs| # GitHub App Server-to-Server Token - ghr| # Refresh Token - github_pat # Alternative format for Personal Access Token - )_ - (?: # for token body - [a-z0-9_]{35,235} # 35 to 235 lowercase alphanumeric characters or underscores - ) - ) + (?xi) + ( + github_pat_ + [A-Z0-9_+]{82,84} + ) \b + pattern_requirements: + min_digits: 2 + min_lowercase: 2 + min_entropy: 3.5 + examples: + - "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" + references: + - https://docs.github.com/en/rest/users?apiVersion=2022-11-28 + validation: + type: Http + content: + request: + method: GET + url: https://api.github.com/user + headers: + Authorization: token {{ TOKEN }} + Accept: application/vnd.github+json + response_matcher: + - report_response: true + - match_all_words: true + type: WordMatch + words: + - '"login"' + - '"id"' + - name: GitHub Personal Access Token + id: kingfisher.github.2 + pattern: | + (?xi) + ( + ghp_(?P[A-Z0-9]{30})(?P[A-Z0-9]{6}) + ) + pattern_requirements: + min_digits: 2 + min_lowercase: 2 + checksum: + actual: + template: "{{ MATCH | suffix: 6 }}" + requires_capture: checksum + expected: "{{ BODY | crc32 | base62: 6 }}" + skip_if_missing: true min_entropy: 3.5 examples: - "GITHUB_KEY=ghp_XIxB7KMNdAr3zqWtQqhE94qglHqOzn1D1stg" @@ -25,40 +55,39 @@ rules: - | ## git developer settings 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 - - "oauth_token: gho_fq75OMU7UVbS9pTZmoCCzJT6TM5d1w099FgG" - - "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" references: - https://docs.github.com/en/rest/users?apiVersion=2022-11-28 validation: type: Http content: request: - method: POST - url: https://api.github.com/graphql + method: GET + url: https://api.github.com/user headers: Authorization: token {{ TOKEN }} Accept: application/vnd.github+json - Content-Type: application/json - body: | - { - "query": "{ viewer { login } }" - } response_matcher: - report_response: true - match_all_words: true type: WordMatch words: - '"login"' + - '"id"' - name: GitHub OAuth Access Token - id: kingfisher.github.2 + id: kingfisher.github.3 pattern: | - (?xi) - \b - ( - gho_ - [A-Z0-9]{36} - ) - \b + (?xi) + ( + gho_(?P[A-Z0-9]{30})(?P[A-Z0-9]{6}) + ) + pattern_requirements: + min_digits: 2 + checksum: + actual: + template: "{{ MATCH | suffix: 6 }}" + requires_capture: checksum + expected: "{{ BODY | crc32 | base62: 6 }}" + skip_if_missing: true min_entropy: 3.5 confidence: medium examples: @@ -70,33 +99,57 @@ rules: type: Http content: request: - method: POST - url: https://api.github.com/graphql + method: GET + url: https://api.github.com/user headers: Authorization: token {{ TOKEN }} Accept: application/vnd.github+json - Content-Type: application/json - body: | - { - "query": "{ viewer { login } }" - } response_matcher: - report_response: true - match_all_words: true type: WordMatch words: - '"login"' - - name: GitHub App Token - id: kingfisher.github.3 + - '"id"' + - name: GitHub App User-to-Server Token + id: kingfisher.github.4 pattern: | (?xi) - \b ( - (?:ghu|ghs)_[A-Z0-9]{36} + ghu_(?P[A-Z0-9]{30})(?P[A-Z0-9]{6}) ) - \b examples: - ' "token": "ghu_16C7e42F292c69C2E7C10c838347Ae178B4a",' + - | + Example usage: + git clone http://ghu_RguXIkihJjwHAP6eXEYxaPNvywurTr5IOAbg@github.com/username/repo.git + references: + - https://docs.github.com/en/rest/users?apiVersion=2022-11-28 + validation: + type: Http + content: + request: + method: GET + url: https://api.github.com/user + headers: + Authorization: token {{ TOKEN }} + Accept: application/vnd.github+json + response_matcher: + - report_response: true + - match_all_words: true + type: WordMatch + words: + - '"login"' + - '"id"' + - name: GitHub App Server-to-Server Token + id: kingfisher.github.5 + pattern: | + (?xi) + ( + ghs_(?P[A-Z0-9]{30})(?P[A-Z0-9]{6}) + ) + examples: + - ' "token": "ghs_16C7e42F292c69C2E7C10c838347Ae178B4a",' - | Example usage: git clone http://ghs_RguXIkihJjwHAP6eXEYxaPNvywurTr5IOAbg@github.com/username/repo.git @@ -106,31 +159,25 @@ rules: type: Http content: request: - method: POST - url: https://api.github.com/graphql + method: GET + url: https://api.github.com/user headers: Authorization: token {{ TOKEN }} Accept: application/vnd.github+json - Content-Type: application/json - body: | - { - "query": "{ viewer { login } }" - } response_matcher: - report_response: true - match_all_words: true type: WordMatch words: - '"login"' + - '"id"' - name: GitHub Refresh Token - id: kingfisher.github.4 + id: kingfisher.github.6 pattern: | (?xi) - \b ( - ghr_[A-Z0-9]{76} + ghr_(?P[A-Z0-9]{30})(?P[A-Z0-9]{6}) ) - \b examples: - ' "refresh_token": "ghr_1B4a2e77838347a7E420ce178F2E7c6912E169246c3CE1ccbF66C46812d16D5B1A9Dc86A1498",' references: @@ -139,24 +186,21 @@ rules: type: Http content: request: - method: POST - url: https://api.github.com/graphql + method: GET + url: https://api.github.com/user headers: Authorization: token {{ TOKEN }} Accept: application/vnd.github+json - Content-Type: application/json - body: | - { - "query": "{ viewer { login } }" - } response_matcher: - report_response: true - match_all_words: true type: WordMatch words: - '"login"' + - '"id"' + - name: GitHub Client ID - id: kingfisher.github.5 + id: kingfisher.github.7 pattern: | (?xi) (?:github) @@ -166,53 +210,46 @@ rules: (?: id | identifier | key ) .{0,2} \s{0,20} .{0,2} \s{0,20} .{0,2} \b ([a-z0-9]{20}) \b + visible: false examples: - | GITHUB_CLIENT_ID=ac58d6da7d7a84c039b7 GITHUB_SECRET=37d02377a3e9d849e18704c3ec883f9c5787d857 - - name: GitHub Secret Key - id: kingfisher.github.6 - pattern: | - (?xi) - github - .? - (?: api | app | application | client | consumer | customer | secret | key ) - .? - (?: key | oauth | sec | secret )? - .{0,2} \s{0,20} .{0,2} \s{0,20} .{0,2} - \b ([a-z0-9]{40}) \b - examples: - - | - GITHUB_CLIENT_ID=ac58d6da7d7a84c039b7 - GITHUB_SECRET=37d02377a3e9d849e18704c3ec883f9c5787d857 - - name: GitHub Personal Access Token (fine-grained permissions) - id: kingfisher.github.7 + - name: GitHub Legacy Secret Key + id: kingfisher.github.8 pattern: | (?xi) \b + (?:github|gh) + (?:.|[\n\r]){0,4}? + (?:oauth|pat|token|key|secret|api[_-]?key|access[_-]?token)\b + (?:.|[\n\r]){0,32}? + \b ( - github_pat_[0-9A-Z_]{82} + [a-z0-9]{40} ) \b - examples: - - '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' + depends_on_rule: + - rule_id: "kingfisher.github.5" + variable: GITHUB_CLIENT_ID validation: type: Http content: request: method: POST - url: https://api.github.com/graphql + url: "https://github.com/login/oauth/access_token" headers: - Authorization: token {{ TOKEN }} - Accept: application/vnd.github+json - Content-Type: application/json - body: | - { - "query": "{ viewer { login } }" - } + Accept: "application/json" + Content-Type: "application/json" + body: '{"client_id":"{{GITHUB_CLIENT_ID}}","client_secret":"{{TOKEN}}","code":"invalid_code"}' response_matcher: - report_response: true - - match_all_words: true - type: WordMatch + - type: StatusMatch + status: [200] + - type: WordMatch words: - - '"login"' \ No newline at end of file + - '"error":"bad_verification_code"' + examples: + - | + GITHUB_CLIENT_ID=ac58d6da7d7a84c039b7 + GITHUB_SECRET=37d02377a3e9d849e18704c3ec883f9c5787d857 diff --git a/data/rules/gitlab.yml b/data/rules/gitlab.yml index c7475d6..ad1a1e7 100644 --- a/data/rules/gitlab.yml +++ b/data/rules/gitlab.yml @@ -2,13 +2,15 @@ rules: - name: GitLab Private Token id: kingfisher.gitlab.1 pattern: | - (?xi) - \b - ( + (?xi) + \b + ( glpat- [0-9A-Z_-]{20} - ) - (?:\b|$) + ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: @@ -43,6 +45,8 @@ rules: GR1348941[0-9A-Z_-]{20} ) \b + pattern_requirements: + min_digits: 2 examples: - | sudo gitlab-runner register \ @@ -86,7 +90,8 @@ rules: ( glptt-[0-9a-f]{40} ) - \b + pattern_requirements: + min_digits: 2 examples: - | curl \ @@ -114,4 +119,51 @@ rules: - '"token is missing"' - '"403 Forbidden"' negative: true - url: https://gitlab.com/api/v4/ci/pipeline_triggers/{{ TOKEN }} \ No newline at end of file + url: https://gitlab.com/api/v4/ci/pipeline_triggers/{{ TOKEN }} + - name: GitLab Private Token - Routable Format + id: kingfisher.gitlab.4 + pattern: | + (?xi) + \b + ( + glpat- + (?[0-9A-Za-z_-]{27,300}) + \. + (?01) + \. + (?[0-9a-z]{2}) + (?[0-9a-z]{7}) + ) + \b + pattern_requirements: + min_digits: 2 + # GitLab's RoutableTokenGenerator renders the CRC32 digest as lowercase + # base36 with a fixed width of 7 characters. The regex and checksum + # expectation mirror that encoding so we only report matches that carry a + # valid GitLab-style checksum. + checksum: + actual: + template: "{{ MATCH | suffix: 7 }}" + requires_capture: crc32 + expected: "{{ \"glpat-\" | append: BASE64_PAYLOAD | append: \".01.\" | append: BASE36_PAYLOAD_LENGTH | crc32 | base36: 7 }}" + skip_if_missing: true + min_entropy: 3.5 + confidence: medium + examples: + - glpat-ymiBP0-I-J6ghspoBPoZxtSC3g7MyHYG0X0r.01.101erjmwl + references: + - https://github.com/diffblue/gitlab/blob/39c63ee83369bf5353256a6b95f3116728edd102/doc/api/personal_access_tokens.md + - https://docs.gitlab.com/api/personal_access_tokens/ + validation: + type: Http + content: + request: + headers: + PRIVATE-TOKEN: '{{ TOKEN }}' + method: GET + response_matcher: + - report_response: true + - type: WordMatch + words: + - '"id"' + url: https://gitlab.com/api/v4/personal_access_tokens/self diff --git a/data/rules/gitter.yml b/data/rules/gitter.yml new file mode 100644 index 0000000..3da37a3 --- /dev/null +++ b/data/rules/gitter.yml @@ -0,0 +1,36 @@ +rules: + - name: Gitter Access Token + id: kingfisher.gitter.1 + pattern: | + (?xi) + \b + gitter + (?:.|[\n\r]){0,24}? + \b + ( + [a-z0-9_-]{40} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.2 + confidence: medium + examples: + - GITTER_TOKEN=abcd1234efgh5678ijkl9012mnop3456qrst7890 + - '"gitterToken": "a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6q7r8s9t0"' + references: + - https://developer.gitter.im/docs/authentication + validation: + type: Http + content: + request: + method: GET + url: https://api.gitter.im/v1/user + headers: + Authorization: 'Bearer {{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 diff --git a/data/rules/gocardless.yml b/data/rules/gocardless.yml index e261372..525b20b 100644 --- a/data/rules/gocardless.yml +++ b/data/rules/gocardless.yml @@ -13,7 +13,11 @@ rules: (?:[A-Z0-9=_-]{8}){3} [A-Z0-9=_-]{0,2} ) - \b + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/google.yml b/data/rules/google.yml index 812a903..51d5103 100644 --- a/data/rules/google.yml +++ b/data/rules/google.yml @@ -15,9 +15,12 @@ rules: id: kingfisher.google.2 pattern: | (?xi) - \b (GOCSPX-[A-Z0-9_-]{28}) (?:[^A-Z0-9_-] | $) + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.3 confidence: medium examples: @@ -28,9 +31,14 @@ rules: pattern: | (?xi) client.?secret .{0,10} - \b - ([a-z0-9_-]{24}) + ( + [a-z0-9_-]{24} + ) (?: [^a-z0-9_-] |$) + pattern_requirements: + min_digits: 4 + min_uppercase: 3 + min_lowercase: 3 min_entropy: 3.3 confidence: medium examples: @@ -42,9 +50,12 @@ rules: id: kingfisher.google.4 pattern: | (?xi) - \b (ya29\.[0-9A-Z_-]{20,1024}) - (?: [^0-9A-Z_-]|$) + (?: [^0-9A-Z_-]) + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.3 confidence: medium examples: @@ -66,12 +77,10 @@ rules: id: kingfisher.google.6 pattern: | (?xi) - \b ([0-9]+-[a-z0-9_]{32}\.apps\.googleusercontent\.com) (?: (?s).{0,40} ) - \b (?: (GOCSPX-[A-Z0-9_-]{28}) | @@ -80,6 +89,10 @@ rules: ) ) (?:[^A-Z0-9_-] | $) + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.3 confidence: medium examples: @@ -93,4 +106,43 @@ rules: client_id: '132261435625-69ubohrvppjr9hcc5t9uighsb7j2cqhv.apps.googleusercontent.com', client_secret: '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', redirect_uri: `http://localhost:${Config.OAUTH_HTTP_PORT}/oauth2callback` - }; \ No newline at end of file + }; + - name: Google Gemini API Key + id: kingfisher.google.7 + pattern: | + (?xi) + \b + ( + AIza + [A-Za-z0-9_-]{35} + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 + min_entropy: 3.5 + confidence: medium + examples: + - AIzaSyByz6BGQf8QtcQLml8spbyy8x5_327PTow + - AIzaSyDhISgbccTi6mfp2GOSmTtqdU__IdevJes + - AIzaSyA_uW1h2CF4ak3vHr7si_RFD_yWxM4tMAM + references: + - https://ai.google.dev/docs/gemini_api_overview + validation: + type: Http + content: + request: + method: GET + url: https://generativelanguage.googleapis.com/v1/models + headers: + X-goog-api-key: '{{ TOKEN }}' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + match_all_words: true + words: + - '"models"' + - '"name"' \ No newline at end of file diff --git a/data/rules/grafana.yml b/data/rules/grafana.yml index 4cb5461..ecbfe45 100644 --- a/data/rules/grafana.yml +++ b/data/rules/grafana.yml @@ -2,50 +2,79 @@ rules: - name: Grafana API Token id: kingfisher.grafana.1 pattern: | - (?xi) + (?x) \b ( - eyJrIjoi[a-z0-9]{60,100} + eyJrIjoi + [A-Za-z0-9+/]{40,380} + ={0,2} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: - - 'Authorization: Bearer eyJrIjoiWHZiSWd5NzdCYUZnNUtibE8obUpESmE2bzJYNDRIc1UiLCJuIjoibXlrZXkiLCJpZCI7MX1' + - 'Authorization: Bearer eyJrIjoiWHZiSWd5NzdCYUZnNUtibE8obUpESmE2bzJYNDRIc1UiLCJuIjoibXlrZXkiLCJpZCI6MX0=' - 'admin_client = GrafanaClient("eyJrIjoiY21sM1JRYjB6RnVYSTNLenRWQkFEaWN2bXI2V202U2IiLCJuIjoiYWRtaW5rZXkiLCJpZCI6MX0=", host=grafana_host, port=3000, protocol="http")' references: - - https://grafana.com/docs/grafana/latest/developers/http_api/auth/ - + - https://grafana.com/docs/grafana/latest/developer-resources/api-reference/http-api/authentication/ + - https://grafana.com/docs/grafana/latest/developer-resources/api-reference/http-api/org/ + depends_on_rule: + - rule_id: kingfisher.grafana.4 + variable: GRAFANADOMAIN + validation: + type: Http + content: + request: + method: GET + url: "https://{{ GRAFANADOMAIN }}/api/org" + headers: + Authorization: "Bearer {{ TOKEN }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + words: ['"id"', '"name"'] + - name: Grafana Cloud API Token id: kingfisher.grafana.2 pattern: | - (?xi) - \b + (?xi) + \b ( - glc_ + glc_ [a-z0-9+/]{40,150} ={0,2} ) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 2 min_entropy: 3.3 confidence: medium examples: - ' "token": "glc_eyJrIjoiZjI0YzZkNGEwZDBmZmZjMmUzNTU3ODcxMmY0ZWZlNTQ1NTljMDFjOCIsIm6iOiJteXRva3VuIiwiaWQiOjF8"' - 'grafana = glc_etLvNLoNMLt7MTczNNwNbN6Nm1ldGEtbW9paxRvcmlpZt14ZXN4NNwNatN6NLCxdKeH7KTUvWpNqCrHlMKE9EhLcZH7to' references: - - https://grafana.com/docs/grafana-cloud/developer-resources/api-reference/cloud-api/#regions + - https://grafana.com/docs/grafana/latest/developer-resources/api-reference/cloud-api/ validation: type: Http content: request: - headers: - Authorization: Bearer {{ TOKEN }} method: GET + url: https://grafana.com/api/stack-regions + headers: + Authorization: "Bearer {{ TOKEN }}" + Accept: application/json response_matcher: - report_response: true - - status: - - 200 - type: StatusMatch - url: https://grafana.com/api/stack-regions + - type: StatusMatch + status: [200] + - type: JsonValid - name: Grafana Service Account Token id: kingfisher.grafana.3 @@ -56,54 +85,56 @@ rules: glsa_[A-Z0-9]{32}_[A-F0-9]{8} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: - | - curl -H "Authorization: Bearer glsa_HOruNAb7SOiCdshU7algkrq7FDsNSLAa_55e2f8be" -X GET '/api/access-control/user/permissions' | jq + curl -H "Authorization: Bearer glsa_HOruNAb7SOiCdshU7algkrq7FDsNSLAa_55e2f8be" -X GET '/api/org' | jq - | - // getData() - // { - // let url="http://localhost:4200/api/search" - // const headers = new HttpHeaders({ - // 'Content-Type': 'application/json', - // 'Authorization': `Bearer glsa_Sof0HKi3agxrQP9qm5r2G98VacBNwV5P_9b638c45` - // }) - // return this.http.get(url, {headers: headers}); - // } + // headers: { Authorization: `Bearer glsa_Sof0HKi3agxrQP9qm5r2G98VacBNwV5P_9b638c45` } references: + - https://grafana.com/blog/new-in-grafana-9-1-service-accounts-are-now-ga/ - https://grafana.com/docs/grafana/latest/administration/service-accounts/ + - https://grafana.com/docs/grafana/latest/developer-resources/api-reference/http-api/org/ + depends_on_rule: + - rule_id: kingfisher.grafana.4 + variable: GRAFANADOMAIN validation: type: Http content: request: method: GET + url: "https://{{ GRAFANADOMAIN }}/api/org" headers: - Authorization: Bearer {{ TOKEN }} + Authorization: "Bearer {{ TOKEN }}" + Accept: application/json response_matcher: - - report_response: true - - status: - - 200 - type: StatusMatch - url: "{{ GRAFANADOMAIN }}/api/access-control/me" - depends_on_rule: - - rule_id: kingfisher.grafana.4 - variable: GRAFANADOMAIN + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + words: ['"id"', '"name"'] - name: Grafana Domain id: kingfisher.grafana.4 pattern: | - (?xi) - (?:https?://)? - (?:[A-Z0-9-]+\.)* - grafana\.[A-Z0-9.-]+ - (?::\d{2,5})? - (?:[/?\#]\S*)? - min_entropy: 3.0 + (?xi) + (?:https?://)? + \b + ( + (?:[a-z0-9-]+\.){0,16} + grafana\.[a-z0-9.-]{2,64} + (?::\d{2,5})? + ) + \b + min_entropy: 3.0 visible: false confidence: medium examples: - - https://grafana.example.com - - http://grafana.prod.eu-west.mycorp.internal:3000/login - - https://api.team1.grafana.services.cluster.local/health + - grafana.example.com + - grafana.prod.eu-west.mycorp.internal:3000 + - api.team1.grafana.services.cluster.local - grafana.dev.foo-bar.co.uk diff --git a/data/rules/groq.yml b/data/rules/groq.yml new file mode 100644 index 0000000..8160b2b --- /dev/null +++ b/data/rules/groq.yml @@ -0,0 +1,36 @@ +rules: + - name: Groq API Key + id: kingfisher.groq.1 + pattern: | + (?xi) + \b + ( + gsk_[A-Z0-9]{52} + ) + \b + pattern_requirements: + min_digits: 4 + confidence: medium + min_entropy: 3.5 + validation: + type: Http + content: + request: + method: GET + url: "https://api.groq.com/openai/v1/models" + headers: + Authorization: "Bearer {{TOKEN}}" + response_matcher: + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"id"' + - '"data"' + match_all_words: true + references: + - https://console.groq.com/docs/api-keys + - https://console.groq.com/docs/api-reference#models + examples: + - "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" + diff --git a/data/rules/guardian.yml b/data/rules/guardian.yml new file mode 100644 index 0000000..5344a09 --- /dev/null +++ b/data/rules/guardian.yml @@ -0,0 +1,39 @@ +rules: + - name: Guardian API Key + id: kingfisher.guardian.1 + pattern: | + (?xi) + \b + guardian + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN|API) + (?:.|[\n\r]){0,32}? + \b + ( + [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - guardian SECRET=abcdef12-1234-abcd-5678-abcdef123456 + - guardianPRIVATEKEY=abcdef12-1234-abcd-5678-abcdef123456 + references: + - https://open-platform.theguardian.com/documentation/ + - https://open-platform.theguardian.com/documentation/section + validation: + type: Http + content: + request: + method: GET + url: "https://content.guardianapis.com/sections?api-key={{ TOKEN }}" + headers: + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"status":"ok"'] diff --git a/data/rules/gumroad.yml b/data/rules/gumroad.yml new file mode 100644 index 0000000..ee5ab92 --- /dev/null +++ b/data/rules/gumroad.yml @@ -0,0 +1,42 @@ +rules: + - name: Gumroad Access Token + id: kingfisher.gumroad.1 + pattern: | + (?xi) + \b + gumroad + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN|ACCESS_TOKEN|OAUTH) + (?:.|[\n\r]){0,48}? + \b + ( + [a-f0-9]{64} + | + [A-Z0-9-]{43} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - gumroad_access_token=abf11e4ab2850ffd50ef690257f7a1c998a443059513d1a4826f2b3159620505 + - gumroadSECRET = abf11e4ab2850ffd50ef690257f7a1c998a443059513d1a4826f2b3159620505 + - gumroadPRIVATE-abf11e4ab2850ffd50ef690257f7a1c998a443059513d1a4826f2b3159620505 + references: + - https://gumroad.com/api + - https://gumroad.com/help/article/280-create-application-api + validation: + type: Http + content: + request: + method: GET + url: "https://api.gumroad.com/v2/user?access_token={{ TOKEN }}" + headers: + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"success":true', '"user"'] diff --git a/data/rules/harness.yml b/data/rules/harness.yml new file mode 100644 index 0000000..aeb81d0 --- /dev/null +++ b/data/rules/harness.yml @@ -0,0 +1,37 @@ +rules: + - name: Harness Personal Access Token (PAT) + id: kingfisher.harness.pat.1 + pattern: | + (?xi) + \b + ( + pat\. + [A-Z0-9]{22} + \. + [0-9a-f]{24} + \. + [A-Z0-9]{20} + ) + \b + min_entropy: 3.4 + confidence: medium + examples: + - 'HARNESS_TOKEN="pat.AbCdEfGhIjKlMnOpQrStUv.0123abcd4567ef890123abcd.ZyXwVuTsRqPoNmLkJiHg"' + references: + - https://developer.harness.io/docs/platform/automation/api/api-quickstart/ + - https://apidocs.harness.io/ + validation: + type: Http + content: + request: + method: GET + url: https://app.harness.io/ng/api/apikey/aggregate + headers: + Accept: application/json + x-api-key: "{{ TOKEN }}" + response_matcher: + # Valid token + authorized OR valid token but missing params/perms + - type: StatusMatch + status: [200, 400, 403] + negative: true + - type: JsonValid diff --git a/data/rules/hashes.yml b/data/rules/hashes.yml index 60b0f4a..bf6f5af 100644 --- a/data/rules/hashes.yml +++ b/data/rules/hashes.yml @@ -7,6 +7,8 @@ rules: - https://unix.stackexchange.com/a/511017 - https://hashcat.net/wiki/doku.php?id=example_hashes - https://passwordvillage.org/salted.html#md5crypt + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: # generated with `openssl passwd -1 -salt 'OKgLCmVl' 'a'` @@ -38,6 +40,8 @@ rules: \$ [./A-Za-z0-9]{8,16} \$ [./A-Za-z0-9]{43} ) + pattern_requirements: + min_digits: 2 references: - https://en.wikipedia.org/wiki/Crypt_(C)#Key_derivation_functions_supported_by_crypt - https://hashcat.net/wiki/doku.php?id=example_hashes @@ -58,6 +62,8 @@ rules: \$ [./A-Za-z0-9]{8,16} \$ [./A-Za-z0-9]{86} ) + pattern_requirements: + min_digits: 2 references: - https://en.wikipedia.org/wiki/Crypt_(C)#Key_derivation_functions_supported_by_crypt - https://hashcat.net/wiki/doku.php?id=example_hashes @@ -76,6 +82,8 @@ rules: \$ [./A-Za-z0-9]{8,16} \$ [./A-Za-z0-9]{43} ) + pattern_requirements: + min_digits: 2 references: - https://en.wikipedia.org/wiki/Crypt_(C)#Key_derivation_functions_supported_by_crypt - https://hashcat.net/wiki/doku.php?id=example_hashes @@ -96,7 +104,8 @@ rules: [0-9a-f]{32} \$ [0-9a-f]{64,} ) - \b + pattern_requirements: + min_digits: 2 references: - https://hashcat.net/wiki/doku.php?id=example_hashes min_entropy: 3.3 diff --git a/data/rules/hashicorp.yml b/data/rules/hashicorp.yml new file mode 100644 index 0000000..b6a0de1 --- /dev/null +++ b/data/rules/hashicorp.yml @@ -0,0 +1,143 @@ +rules: + - name: Hashicorp Vault Service Token (< v1.10) + id: kingfisher.hashicorp.1 + + pattern: | + (?x) + (?i: hashicorp | vault | token | key | secret ) + (?:.|[\n\r]){0,32}? + \b + ( + s\.[A-Za-z0-9_-]{24,128} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.0 + + examples: + - 'VAULT_CLIENT_TOKEN="s.Z4bTMtngfLeQ18AqVoBBkUAOD1"' + - 'vaultToken="s.CAESIP2jTxc9S3K7Z6CtcFWQv7-044m_oS.0H3nF89l3GiYKHGh3cy5sQmlIZVNyTWJNcDRsYWJpQjlhYjVlb2cQh6PL8wEYAg"`' + + references: + - https://developer.hashicorp.com/vault/docs/concepts/tokens + + - name: Hashicorp Vault Batch Token (< v1.10) + id: kingfisher.hashicorp.2 + + pattern: | + (?x) + (?i: hashicorp | vault | token | key | secret ) + ["':=\ ]{0,5} + (b\.[A-Za-z0-9_-]{24,500}) + (?: [^A-Za-z0-9_-] | $ ) + pattern_requirements: + min_digits: 2 + + examples: + - 'VAULT_CLIENT_TOKEN="b.Z4bTMtngfLeQ18AqVoBBkUAOD1"' + confidence: medium + min_entropy: 3.0 + references: + - https://developer.hashicorp.com/vault/docs/concepts/tokens + + - name: Hashicorp Vault Recovery Token (< v1.10) + id: kingfisher.hashicorp.3 + + pattern: | + (?x) + (?i: hashicorp | vault | token | key | secret ) + ["':=\ ]{0,5} + (r\.[A-Za-z0-9_-]{24,500}) + (?: [^A-Za-z0-9_-] | $ ) + pattern_requirements: + min_digits: 2 + + examples: + - 'VAULT_CLIENT_TOKEN="r.Z4bTMtngfLeQ18AqVoBBkUAOD1"' + + confidence: medium + min_entropy: 3.0 + references: + - https://developer.hashicorp.com/vault/docs/concepts/tokens + - https://developer.hashicorp.com/vault/docs/concepts/recovery-mode + + - name: Hashicorp Vault Service Token (>= v1.10) + id: kingfisher.hashicorp.4 + + pattern: | + (?x) + (hvs\.[A-Za-z0-9]{24,130}) + (?: [^A-Za-z0-9_-] | $ ) + pattern_requirements: + min_digits: 2 + + examples: + - "apikey: hvs.JGbZZaCkOSgsZ56uhGlTK2zyC1j2mwhy0VLp4" + + confidence: medium + min_entropy: 3.0 + references: + - https://developer.hashicorp.com/vault/docs/concepts/tokens + + - name: Hashicorp Vault Batch Token (>= v1.10) + id: kingfisher.hashicorp.5 + + pattern: | + (?x) + (hvb\.[A-Za-z0-9_-]{24,500}) + (?: [^A-Za-z0-9_-] | $ ) + pattern_requirements: + min_digits: 2 + + examples: + - "apikey: hvb.JGbZZaCkOSgsZ56uhGlTK2zyC1j2mwhy0VLp4" + - "hvb.AAAAAQJgxDgqsGNorpoOR8hPZ5SU-ynBvCl764jyRP_fnX8WvkdkDzGjbLNGdPdtlY32Als2P36yDZueqzfdGw9RsaTeaYXSH5E4RYSWuRoQ9YRKIw9o7mDDY2ZcT3KOB7RwtW2w1FN2eDqcy_sbCjXPaM1iBVH-mqMSYRmRd2nb5D1SJPeBzIYRqSglLc32wUGN7xEzyrKUczqOKsIcybQA" + + confidence: medium + min_entropy: 3.0 + + references: + - https://developer.hashicorp.com/vault/docs/concepts/tokens + + - name: Hashicorp Vault Recovery Token (>= v1.10) + id: kingfisher.hashicorp.6 + + pattern: | + (?x) + (hvr\.[A-Za-z0-9]{24,130}) + (?: [^A-Za-z0-9_-] | $ ) + pattern_requirements: + min_digits: 2 + + examples: + - "apikey: hvr.JGbZZaCkOSgsZ56uhGlTK2zyC1j2mwhy0VLp4" + + confidence: medium + min_entropy: 3.0 + references: + - https://developer.hashicorp.com/vault/docs/concepts/tokens + - https://developer.hashicorp.com/vault/docs/concepts/recovery-mode + + - name: Hashicorp Vault Unseal Key + id: kingfisher.hashicorp.7 + + pattern: | + (?x) + (?i: unseal ) + \b + .{1,10} + ([a-zA-Z0-9+/]{44}) + (?: [^a-zA-Z0-9+/] | $ ) + pattern_requirements: + min_digits: 2 + + examples: + - "Unseal Key 2: 0tZn+7QQCxphpHwTm7/dC3LpP5JGIbYl3PK8Sy81R+P2" + - "oc -n vault exec -ti vault-0 -- vault operator unseal 98m+o2ylRhVbOi+3o5ub6PbP343ocFUVORgSYeypMDjh" + + confidence: medium + min_entropy: 3.0 + references: + - https://developer.hashicorp.com/vault/docs/concepts/seal diff --git a/data/rules/hereapi.yml b/data/rules/hereapi.yml new file mode 100644 index 0000000..c6974ba --- /dev/null +++ b/data/rules/hereapi.yml @@ -0,0 +1,40 @@ +rules: + - name: HERE API Key + id: kingfisher.hereapi.1 + pattern: | + (?xi) + \b + hereapi + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN|APIKEY) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9_-]{43} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - "hereapi_key=XxK6G3m_pQ8nR2vT4wY9jL5bN7cA1dF3hJ0iM4eP9su" + - "HEREAPI_SECRET=ZzY8xW6vU4tS2qP0nM5kJ9hF7dC1bA3gL8iK4eR9wQm" + references: + - https://stackoverflow.com/questions/65610274/here-geocoding-api-not-working-inside-my-react-app + - https://github.com/spara/geocoding_tutorial + validation: + type: Http + content: + request: + method: GET + url: "https://geocode.search.hereapi.com/v1/geocode?q=Berlin&apiKey={{ TOKEN }}" + headers: + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + # Successful geocode responses include an "items" array. + - type: WordMatch + words: ['"items"'] diff --git a/data/rules/heroku.yml b/data/rules/heroku.yml index 817a2a4..96e94e4 100644 --- a/data/rules/heroku.yml +++ b/data/rules/heroku.yml @@ -2,17 +2,18 @@ rules: - name: Heroku API Key id: kingfisher.heroku.1 pattern: | - (?xi) + (?xi) + \b heroku - (?:.|[\n\r]){0,32}? - (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) - (?:.|[\n\r]){0,32}? + (?:.|[\n\r]){0,32}? \b ( [0-9a-f]{8}-[0-9a-f]{4}- [0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.0 confidence: medium examples: @@ -32,3 +33,37 @@ rules: - report_response: true - type: StatusMatch status: [200] + - name: Heroku API Key (Platform Key) + id: kingfisher.heroku.2 + pattern: | + (?xi) + ( + HRKU-[A-Z0-9_]{60} + ) + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 4.0 + validation: + type: Http + content: + request: + method: GET + url: "https://api.heroku.com/apps" + headers: + Authorization: "Bearer {{TOKEN}}" + Accept: "application/vnd.heroku+json;version=3" + response_matcher: + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"id":' + - '"name":' + match_all_words: true + references: + - https://devcenter.heroku.com/articles/platform-api-quickstart + examples: + - "HRKU-AADVTUYvfjT4nhuJ07bEfAUq9GS3PkTdyWuNBiXYmYMg_____wgAf6OTnGyh" + - "HRKU-AABW9W1iH9NHEIlAABq9nZUq9GS3PkTdyWuNBiXYmYMg_____wV2XYIXxm5p" + - "HRKU-AAWpqREEr2V1gqh6urSXWYUq9GS3PkTdyWuNBiXYmYMg_____wNI1VGijd8y" diff --git a/data/rules/honeycomb.yml b/data/rules/honeycomb.yml new file mode 100644 index 0000000..278d92b --- /dev/null +++ b/data/rules/honeycomb.yml @@ -0,0 +1,41 @@ +rules: + - name: Honeycomb API Key + id: kingfisher.honeycomb.1 + pattern: | + (?xi) + \b + honeycomb + (?:.|[\n\r]){0,16}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [0-9a-f]{32}| + [0-9a-zA-Z]{22} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - honeycomb_secret_key=8f14e45fceea167a5a36dedd4bea2543 + - honeycomb_token=z0d1f2bcaloumn3456789P + references: + - https://api-docs.honeycomb.io/api/auth + - https://docs.honeycomb.io/api/ + validation: + type: Http + content: + request: + method: GET + url: https://api.honeycomb.io/1/auth + headers: + X-Honeycomb-Team: "{{ TOKEN }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"id"', '"type"', '"team"', '"environment"'] diff --git a/data/rules/http.yml b/data/rules/http.yml new file mode 100644 index 0000000..a3b523b --- /dev/null +++ b/data/rules/http.yml @@ -0,0 +1,36 @@ +rules: + - name: HTTP Basic Authentication + id: kingfisher.http.1 + + pattern: | + (?x)(?i) + Authorization (?: :\s+ | \s*.{1, 5}\s*) Basic \s+ + ( [A-Za-z0-9+/]{6,} ={0,2} ) + (?: [^A-Za-z0-9+/=] | $ ) + confidence: low + min_entropy: 3.0 + examples: + - "Authorization: Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==" + + references: + - https://datatracker.ietf.org/doc/html/rfc7617 + - https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication + + - name: HTTP Bearer Token + id: kingfisher.http.2 + pattern: | + (?x)(?i) + Authorization (?: :\s+ | \s*.{1, 5}\s*) Bearer \s+ + ([a-zA-z0-9._~+/-]{6,} =*) + (?: [^a-zA-z0-9._~+/=-] | $ ) + confidence: low + min_entropy: 3.0 + examples: + - | + GET /resource HTTP/1.1 + Host: server.example.com + Authorization: Bearer mF_9.B5f-4.1JqM + + references: + - https://datatracker.ietf.org/doc/html/rfc6750 + - https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication diff --git a/data/rules/hubspot.yml b/data/rules/hubspot.yml new file mode 100644 index 0000000..5cb5372 --- /dev/null +++ b/data/rules/hubspot.yml @@ -0,0 +1,33 @@ +rules: + - name: HubSpot Private App Token + id: kingfisher.hubspot.1 + pattern: | + (?xi) + \b + ( + pat-[a-z0-9]{2,3}-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.0 + validation: + type: Http + content: + request: + method: GET + url: "https://api.hubapi.com/crm/v3/owners/" + headers: + Authorization: "Bearer {{TOKEN}}" + response_matcher: + - report_response: true + - type: JsonValid + - type: WordMatch + words: + - '"INVALID_AUTHENTICATION"' + negative: true + references: + - https://developers.hubspot.com/docs/api/private-apps + examples: + - "pat-na2-3b124f92-f4cb-4d1d-8d1c-7fc3f3512dba" diff --git a/data/rules/huggingface.yml b/data/rules/huggingface.yml index d103438..c70b993 100644 --- a/data/rules/huggingface.yml +++ b/data/rules/huggingface.yml @@ -3,7 +3,6 @@ rules: id: kingfisher.huggingface.1 pattern: | (?xi) - \b (?: ( (?:api_org|hf)_ @@ -11,6 +10,8 @@ rules: ) ) \b + pattern_requirements: + min_digits: 2 references: - https://huggingface.co/docs/hub/security-tokens min_entropy: 3.3 diff --git a/data/rules/ibm.yml b/data/rules/ibm.yml index 55a33be..390d927 100644 --- a/data/rules/ibm.yml +++ b/data/rules/ibm.yml @@ -8,17 +8,17 @@ rules: (?:.|[\n\r]){0,32}? (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) (?:.|[\n\r]){0,32}? - \b ( [0-9A-Z_-]{42,44} ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium - examples: - ibmcloud_apikey = abcdef0123_56789abcdef0123456789abcdef01234 - ibm_platform_key="f-_RrJDVnuVh07HNTcmnQx_b6CbcQsxmEarVm9P_RWtF" - references: - https://cloud.ibm.com/docs/account?topic=account-userapikey - https://cloud.ibm.com/apidocs/iam-identity-token-api diff --git a/data/rules/imagekit.yml b/data/rules/imagekit.yml new file mode 100644 index 0000000..050168b --- /dev/null +++ b/data/rules/imagekit.yml @@ -0,0 +1,40 @@ +rules: + - name: ImageKit Private API Key + id: kingfisher.imagekit.1 + pattern: | + (?xi) + \b + imagekit + (?:.|[\n\r]){0,64}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN|PRIVATE_KEY) + (?:.|[\n\r]){0,64}? + \b + ( + private_[A-Z0-9_-]{8,128} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.2 + confidence: medium + examples: + - IMAGEKIT_PRIVATE_KEY=private_rGAPQJbhBx + - imagekit token private_AbCdEf0123456789GhIjKlMn + references: + - https://imagekit.io/docs/api-keys + - https://imagekit.io/docs/api-reference/account-management-api/url-endpoints/list-url-endpoints + validation: + type: Http + content: + request: + method: GET + url: "https://api.imagekit.io/v1/accounts/url-endpoints" + headers: + Authorization: "Basic {{ TOKEN | append: ':' | b64enc }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"urlEndpoint"'] diff --git a/data/rules/infracost.yml b/data/rules/infracost.yml new file mode 100644 index 0000000..42dc8d6 --- /dev/null +++ b/data/rules/infracost.yml @@ -0,0 +1,35 @@ +rules: + - name: Infracost API Token + id: kingfisher.infracost.1 + pattern: | + (?xi) + \b + ( + ico-[a-z0-9]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.3 + confidence: medium + examples: + - export INFRACOST_API_KEY=ico-abcdefabcdefabcdefabcdefabcdefab + - '"infracost": "ico-1234567890abcdef1234567890abcdef"' + references: + - https://www.infracost.io/docs/api_reference/ + validation: + type: Http + content: + request: + method: POST + url: https://pricing.api.infracost.io/graphql + headers: + X-Api-Key: '{{ TOKEN }}' + Content-Type: application/json + Accept: application/json + body: '{"query":"{ ping }"}' + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 diff --git a/data/rules/infura.yml b/data/rules/infura.yml new file mode 100644 index 0000000..0f999de --- /dev/null +++ b/data/rules/infura.yml @@ -0,0 +1,43 @@ +rules: + - name: Infura API Key + id: kingfisher.infura.1 + pattern: | + (?xi) + \b + infura + (?:.|[\n\r]){0,32}? + \b + ( + [0-9a-z]{32} + ) + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - https://mainnet.infura.io/v3/7238211010344719ad14a89db874158c + - infuraKEYwithspecial-abcdef1234567890abcdef1234567890 + references: + - https://www.infura.io/docs + - https://docs.metamask.io/services/reference/ethereum/json-rpc-methods/ + validation: + type: Http + content: + request: + method: POST + url: "https://mainnet.infura.io/v3/{{ TOKEN }}" + headers: + Content-Type: application/json + Accept: application/json + body: | + {"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + words: ['"result"'] + - type: WordMatch + negative: true + words: ["invalid project id", "project id required in the URL", "invalid project id or project secret"] diff --git a/data/rules/instantly.yml b/data/rules/instantly.yml new file mode 100644 index 0000000..955c7cc --- /dev/null +++ b/data/rules/instantly.yml @@ -0,0 +1,41 @@ +rules: + - name: Instantly API Key + id: kingfisher.instantly.1 + pattern: | + (?xi) + \b + instantly + (?:\.ai)? + (?:.|[\n\r]){0,16}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,16}? + \b + ( + [A-Z0-9+/]{66}== + ) + pattern_requirements: + min_digits: 4 + min_entropy: 3.3 + confidence: medium + examples: + - 'INSTANTLY_API_KEY="NmNlMCI1MWUtZDBmMC00NTc4LWE0MDItMDM0NGU0ZWI0MzliOmFzWWtCZUxUY3ZPRg=="' + references: + - https://developer.instantly.ai/api/v2/analytics/getdailyaccountanalytics + validation: + type: Http + content: + request: + method: GET + url: "https://api.instantly.ai/api/v2/accounts/analytics/daily?start_date={{ '' | date: '%Y-%m-01' }}&end_date={{ '' | date: '%Y-%m-%d' }}" + headers: + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200, 401] + - type: WordMatch + negative: true + words: + - '"Invalid authorization header or API key"' + - '"Invalid API key"' + - type: JsonValid diff --git a/data/rules/intercom.yml b/data/rules/intercom.yml index 01fdf0d..0b408ae 100644 --- a/data/rules/intercom.yml +++ b/data/rules/intercom.yml @@ -3,19 +3,22 @@ rules: id: kingfisher.intercom.1 pattern: | (?xi) - (?:intercom|ic) - (?:.|[\n\r]){0,16}? + (?:intercom|ic) + (?:.|[\n\r]){0,16}? (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) - (?:.|[\n\r]){0,16}? + (?:.|[\n\r]){0,16}? + \b ( [0-9A-Z+/]{59}= ) + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: - "intercom_access_token: dG9rOvI0NmJlMTA5XzQwM2NfNDVlM184MjQzXzkwMDnmOTE1NGIyONoxOjA=" - - ic_token = "g1ZsclJXTjNfc1pBSzJDemE0eFVDU0U5c25CeDN4Vm9hQ2Zac0hXemZHNGVDPQ==" + - ic_token = "g1ZsclJXTjNfc1pBSzJDemE0eFVDU0U5c25CeDN4Vm9hQ2Zac0hXemZHNPQ==" references: - https://developers.intercom.com/docs/build-an-integration/learn-more/rest-apis diff --git a/data/rules/intra42.yml b/data/rules/intra42.yml new file mode 100644 index 0000000..7e0a1fd --- /dev/null +++ b/data/rules/intra42.yml @@ -0,0 +1,63 @@ +rules: + - name: Intra42 Client ID + id: kingfisher.intra42.1 + visible: false + pattern: | + (?xi) + \b + ( + u-s4t2(?:ud|af)-[a-f0-9]{64} + ) + \b + min_entropy: 3.0 + confidence: medium + examples: + - 'INTRA42_CLIENT_ID="u-s4t2ud-33ad3d923534cae0bf765b20ac23831a4e35937298f21062a72db03e99de65b7"' + references: + - https://api.intra.42.fr/apidoc/guides/getting_started + + - name: Intra42 Client Secret (s-s4t2ud/af) + id: kingfisher.intra42.2 + pattern: | + (?xi) + \b + ( + s-s4t2(?:ud|af)-[a-f0-9]{64} + ) + \b + min_entropy: 3.6 + confidence: medium + pattern_requirements: + min_digits: 3 + min_lowercase: 2 + examples: + - 'INTRA42_CLIENT_SECRET="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"' + references: + - https://api.intra.42.fr/apidoc/guides/getting_started + - https://api.intra.42.fr/apidoc/guides/web_application_flow + - https://api.intra.42.fr/apidoc/guides/specification + + depends_on_rule: + - rule_id: kingfisher.intra42.1 + variable: CLIENT_ID + + validation: + type: Http + content: + request: + method: POST + url: https://api.intra.42.fr/oauth/token + headers: + Accept: application/json + Content-Type: application/x-www-form-urlencoded + body: > + grant_type=client_credentials&client_id={{ CLIENT_ID | url_encode }}&client_secret={{ TOKEN | url_encode }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: false + words: + - '"access_token"' diff --git a/data/rules/ionic.yml b/data/rules/ionic.yml index 1130a38..b37b83b 100644 --- a/data/rules/ionic.yml +++ b/data/rules/ionic.yml @@ -7,8 +7,10 @@ rules: ( ion_ [a-z0-9]{42} - ) - \b + ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/ipstack.yml b/data/rules/ipstack.yml index a70e43e..1dbb576 100644 --- a/data/rules/ipstack.yml +++ b/data/rules/ipstack.yml @@ -13,6 +13,8 @@ rules: (?:[0-9a-f]{16}){2} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.0 confidence: medium examples: diff --git a/data/rules/jdbc.yml b/data/rules/jdbc.yml new file mode 100644 index 0000000..00fffd5 --- /dev/null +++ b/data/rules/jdbc.yml @@ -0,0 +1,31 @@ +rules: + - name: JDBC connection string with embedded credentials + id: kingfisher.jdbc.1 + pattern: | + (?xi) + ( + jdbc: + [a-z][a-z0-9+.-]{2,32} + (?:[:][a-z0-9+.-]{1,32})* + : + [^\s"'<>,(){}\[\]]{10,448} + ) + pattern_requirements: + min_special_chars: 2 + special_chars: ";=/?@&" + ignore_if_contains: + - "****" + - "xxxx" + - "example" + min_entropy: 3.3 + confidence: medium + validation: + type: Jdbc + examples: + - jdbc:postgresql://db.example.com:5432/app?user=admin&password=s3cr3t + - jdbc:mysql://admin:s3cr3t@prod.internal:3306/inventory + - jdbc:oracle:thin:@ora.example.net:1521/ORCLPDB1 + - jdbc:sqlserver://sql.example.org:1433;databaseName=inventory;user=sa;password=s3cr3t! + references: + - https://docs.oracle.com/javase/8/docs/api/java/sql/DriverManager.html + - https://www.postgresql.org/docs/current/jdbc-use.html diff --git a/data/rules/jenkins.yml b/data/rules/jenkins.yml index f8fbb77..36a1d49 100644 --- a/data/rules/jenkins.yml +++ b/data/rules/jenkins.yml @@ -2,7 +2,8 @@ rules: - name: Jenkins Token or Crumb id: kingfisher.jenkins.1 pattern: '(?i)jenkins.{0,10}(?:crumb)?.{0,10}\b([0-9a-f]{32,36})\b' - categories: [api, fuzzy, secret] + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: @@ -17,8 +18,6 @@ rules: export JENKINS=jenkins-cicd.apps.sno.openshiftlabs.net - | sh "curl -X POST 'http://jenkins.lsfusion.luxsoft.by/job/${Paths.updateParentVersionsJob}/build' --user ${USERPASS} -H 'Jenkins-Crumb:440561953171ba44ace9740562d172bb'" - negative_examples: - - '1. ~~Does not play well with [Build Token Root Plugin](https://wiki.jenkins-ci.org/display/JENKINS/Build+Token+Root+Plugin) URL formats.~~ (added with [this commit](https://github.com/morficus/Parameterized-Remote-Trigger-Plugin/commit/f687dbe75d1c4f39f7e14b68220890384d7c5674) )' references: - https://www.jenkins.io/blog/2018/07/02/new-api-token-system/ - https://www.jenkins.io/doc/book/security/csrf-protection/ \ No newline at end of file diff --git a/data/rules/jina.yml b/data/rules/jina.yml new file mode 100644 index 0000000..807f5a7 --- /dev/null +++ b/data/rules/jina.yml @@ -0,0 +1,35 @@ +rules: + - name: Jina Search Foundation API Key + id: kingfisher.jina.1 + pattern: | + (?x) + \b + ( + jina_[a-zA-Z0-9]{60} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.0 + examples: + - "JINA_KEY = os.getenv('JINA_KEY','jina_c1758c6f49e14ced990ac7776800dc45ShJNTXBCizzwjE6IMFYJ7LD959cG')" + validation: + type: Http + content: + request: + method: GET + url: "https://api.jina.ai/v1/flows" + headers: + Authorization: "Bearer {{TOKEN}}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"data":' + - '"_id":' + match_all_words: true + references: + - https://jina.ai/docs/jina-ai-cloud/api-reference/ \ No newline at end of file diff --git a/data/rules/jira.yml b/data/rules/jira.yml index c0b14a2..2be3a53 100644 --- a/data/rules/jira.yml +++ b/data/rules/jira.yml @@ -2,17 +2,20 @@ rules: - name: Jira Domain id: kingfisher.jira.1 pattern: | - (?xi) + (?xi) + \b ( [a-z][a-z0-9-]{5,24}\.atlassian\.net ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 visible: false confidence: medium examples: - - example-jira.atlassian.net - - jira.sprintUri= https://leakyday.atlassian.net/rest + - examplefoo-jira.atlassian.net + - jira.sprintUri= https://example.atlassian.net/rest - name: Jira Token id: kingfisher.jira.2 @@ -26,8 +29,12 @@ rules: \b ( [a-z0-9-]{24} - ) - \b + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/jotform.yml b/data/rules/jotform.yml new file mode 100644 index 0000000..4b43953 --- /dev/null +++ b/data/rules/jotform.yml @@ -0,0 +1,35 @@ +rules: + - name: Jotform API Key + id: kingfisher.jotform.1 + pattern: | + (?xi) + \b + jotform + (?:.|[\n\r]){0,64}? + (?:api[_-]?key|apikey|token|secret|key) + (?:.|[\n\r]){0,32}? + \b + ( + [0-9A-Z]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - jotform apikey=abcde12345abcde67890abcde12345fg + references: + - https://api.jotform.com/docs/ + validation: + type: Http + content: + request: + method: GET + url: "https://api.jotform.com/user/usage?apiKey={{ TOKEN }}" + headers: + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] diff --git a/data/rules/jumpcloud.yml b/data/rules/jumpcloud.yml new file mode 100644 index 0000000..0cab6eb --- /dev/null +++ b/data/rules/jumpcloud.yml @@ -0,0 +1,40 @@ +rules: + - name: Jumpcloud API Key + id: kingfisher.jumpcloud.1 + pattern: | + (?xi) + \b + jumpcloud + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [a-zA-Z0-9]{40} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - jumpcloud_api_key=1a2b3c4d5e6f7g8h9i0j1a2b3c4d5e6f7g8h9i0j + - JUMPCLOUD_SECRET=k9l8m7n6o5p4q3r2s1t0k9l8m7n6o5p4q3r2s1t0 + references: + - https://docs.jumpcloud.com/api/ + - https://jumpcloud.com/support/retrieve-object-ids-from-the-api + validation: + type: Http + content: + request: + method: GET + url: "https://console.jumpcloud.com/api/systemusers?limit=1&skip=0" + headers: + x-api-key: "{{ TOKEN }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"results"', '"totalCount"'] diff --git a/data/rules/jwt.yml b/data/rules/jwt.yml index cd3f78d..cbd5b46 100644 --- a/data/rules/jwt.yml +++ b/data/rules/jwt.yml @@ -3,15 +3,16 @@ rules: id: kingfisher.jwt.1 pattern: | (?x) - \b ( - ey[A-Za-z0-9_-]{12,} (?# header ) + (?:ey|ewogIC)[A-Za-z0-9_-]{12,} (?# header ) \. ey[A-Za-z0-9_-]{12,} (?# payload ) \. [A-Za-z0-9_-]{12,} (?# signature ) ) - (?:[^A-Z0-9_-]|$) (?# this instead of a \b anchor because that doesn't play nicely with `-` ) + (?:[^A-Z0-9_-]) + pattern_requirements: + min_digits: 4 min_entropy: 3.3 confidence: medium examples: @@ -22,4 +23,6 @@ rules: - https://datatracker.ietf.org/doc/html/rfc7519 - https://en.wikipedia.org/wiki/Base64#URL_applications - https://datatracker.ietf.org/doc/html/rfc4648 - - https://developer.okta.com/blog/2018/06/20/what-happens-if-your-jwt-is-stolen \ No newline at end of file + - https://developer.okta.com/blog/2018/06/20/what-happens-if-your-jwt-is-stolen + validation: + type: JWT \ No newline at end of file diff --git a/data/rules/kagi.yml b/data/rules/kagi.yml new file mode 100644 index 0000000..b1d311c --- /dev/null +++ b/data/rules/kagi.yml @@ -0,0 +1,35 @@ +rules: + - name: Kagi API Key + id: kingfisher.kagi.1 + pattern: | + (?x)(?s) + (?: kagi | KAGI ) + .{0,100} + ( [a-zA-Z0-9_-]{11}\.[a-zA-Z0-9_-]{43} ) + (?: $ | [^a-zA-Z0-9_-] ) + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.0 + references: + - https://help.kagi.com/kagi/api/search.html + examples: + - "KAGI_API_KEY='AQAAUPJ-iQc.yLFDzC5RRHzPNDThhulREdoG0Bn3PiZMwJ6yqC6uJLE'" + - "https://kagi.com/search?token=uwHBLWXZpgY.STzubkAbVXqpfV39Q6TOfzp43KulJeYWK6-963uz1-o" + validation: + type: Http + content: + request: + method: GET + url: "https://kagi.com/api/v0/search?q=test" + headers: + Authorization: "Bot {{TOKEN}}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"data":' + - '"results":' + match_all_words: true \ No newline at end of file diff --git a/data/rules/kickbox.yml b/data/rules/kickbox.yml new file mode 100644 index 0000000..897d25d --- /dev/null +++ b/data/rules/kickbox.yml @@ -0,0 +1,34 @@ +rules: + - name: Kickbox API Key + id: kingfisher.kickbox.1 + pattern: | + (?xi) + \b + kickbox + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9_]+[A-Z0-9]{64} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - kickbox_key=test_abcdefghijklmnopqrstuvwxyzbu9JFVJtII3FINL1rOKcNpveXD4hSMtSDx7opOWd + - kickbox_token=live_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789efgh + validation: + type: Http + content: + request: + method: GET + url: "https://api.kickbox.com/v2/verify?apikey={{ TOKEN }}&email=kingfisher" + response_matcher: + - report_response: true + - type: JsonValid + - type: WordMatch + words: + - '"success":true' diff --git a/data/rules/klaviyo.yml b/data/rules/klaviyo.yml new file mode 100644 index 0000000..c09cb21 --- /dev/null +++ b/data/rules/klaviyo.yml @@ -0,0 +1,33 @@ +rules: + - name: Klaviyo API Key + id: kingfisher.klaviyo.1 + pattern: | + (?xi) + \b + klaviyo + (?:.|[\n\r]){0,16}? + \b + ( + pk_[A-Z0-9]{34} + ) + \b + min_entropy: 3.3 + confidence: medium + examples: + - klaviyo_key = pk_abcd1234fghij5678klmn9012opqr3456s + validation: + type: Http + content: + request: + method: GET + url: https://a.klaviyo.com/api/accounts + headers: + Revision: "2023-02-22" + Authorization: "Klaviyo-API-Key {{ TOKEN }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"data"'] diff --git a/data/rules/klingai.yml b/data/rules/klingai.yml new file mode 100644 index 0000000..b7bbef7 --- /dev/null +++ b/data/rules/klingai.yml @@ -0,0 +1,59 @@ +rules: + - name: Kling AI Secret Key + id: kingfisher.klingai.1 + pattern: | + (?xi) + \b + kling + (?:.|[\n\r]){0,120}? + \b + (?:access[\s_-]*key|accesskeyid|ak) + (?:.|[\n\r]){0,64}? + \b + (?P + [A-Za-z0-9]{32} + ) + \b + (?:.|[\n\r]){0,120}? + \b + (secret[\s_-]*key|accesskeysecret|sk) + \b + (?:.|[\n\r]){0,64}? + \b + (?P + [A-Za-z0-9]{32} + ) + \b + min_entropy: 2.0 + confidence: medium + examples: + - 'kling Access Key: ADaPACHhrBACBMABNbLpyfdQC2aBdf8r kling Secret Key: 8kKdpk9EnNdYJGQ8hRptagCFBmFHDB33"' + references: + - https://docs.qingque.cn/d/home/eZQDkhg4h2Qg8SEVSUTBdzYeY + - https://community.n8n.io/t/authorization-kling-api/112647 + validation: + type: Http + content: + request: + method: GET + + # SINGLE LINE URL (no folded block) so you don't accidentally end up with %20 + url: 'https://api-singapore.klingai.com/account/costs?start_time={{ "" | unix_timestamp | minus: 3600 | times: 1000 }}&end_time={{ "" | unix_timestamp | times: 1000 }}' + + headers: + Content-Type: application/json + Accept: application/json + + # SINGLE LINE Authorization header (no YAML "|" block) so it won't be dropped. + # JWT matches the Python example: HS256 header + {iss,exp,nbf} payload signed with SK. + + Authorization: '{%- assign header = "HS256" | jwt_header -%}{%- assign now = "" | unix_timestamp -%}{%- assign exp = now | plus: 1800 -%}{%- assign nbf = now | minus: 5 -%}{%- assign payload_json = ''{"iss":"'' | append: AKID | append: ''","exp":'' | append: exp | append: '',"nbf":'' | append: nbf | append: ''}'' -%}{%- assign payload = payload_json | b64url_enc -%}{%- assign signing_input = header | append: "." | append: payload -%}{%- assign sig_b64 = signing_input | hmac_sha256: SECRET -%}{%- assign sig = sig_b64 | replace: "+", "-" | replace: "/", "_" | replace: "=", "" -%}Bearer {{ header }}.{{ payload }}.{{ sig }}' + + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: StatusMatch + status: [401, 403, 500] + negative: true + - type: JsonValid \ No newline at end of file diff --git a/data/rules/langchain.yml b/data/rules/langchain.yml new file mode 100644 index 0000000..6f4da05 --- /dev/null +++ b/data/rules/langchain.yml @@ -0,0 +1,56 @@ +rules: + - name: LangSmith Personal Access Token + id: kingfisher.langchain.1 + pattern: | + (?xi) + \b + ( + lsv2_(?:pt)_[0-9a-f]{32}_[0-9a-f]{10} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 4.0 + examples: + - "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" + validation: + type: Http + content: + request: + method: GET + url: "https://api.smith.langchain.com/api/v1/api-key/current" + headers: + X-API-Key: "{{ TOKEN }}" + Accept: "application/json" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - name: LangSmith Service Key + id: kingfisher.langchain.2 + pattern: | + (?xi) + \b + ( + lsv2_sk_[0-9a-f]{32}_[0-9a-f]{10} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 4.0 + examples: + - "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" + validation: + type: Http + content: + request: + method: GET + url: "https://api.smith.langchain.com/api/v1/orgs/current" + headers: + X-API-Key: "{{ TOKEN }}" + Accept: "application/json" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] \ No newline at end of file diff --git a/data/rules/lark.yml b/data/rules/lark.yml new file mode 100644 index 0000000..fb7e0ce --- /dev/null +++ b/data/rules/lark.yml @@ -0,0 +1,142 @@ +rules: + - name: LarkSuite Tenant Access Token + id: kingfisher.lark.1 + pattern: | + (?xi) + (?:lark|larksuite) + (?:.|[\n\r]){0,64}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN|ACCESS_KEY) + (?:.|[\n\r]){0,32}? + \b + ( + t-[A-Z0-9_.]{14,50} + ) + \b + min_entropy: 3.2 + confidence: medium + examples: + - larksuite_tenant_access_token="t-AbCdEfGhIjKlMnOpQrStUvWxYz_1234" + references: + - https://open.larksuite.com/document/home/introduction-to-scope-and-authorization/access-credentials + - https://open.larksuite.com/document/uAjLw4CM/ukTMukTMukTM/reference/im-v1/chat/list + - https://open.larksuite.com/document/faq/trouble-shooting/how-to-fix-the-99991672-error + - https://open.larksuite.com/document/ukTMukTMukTM/ugjM14COyUjL4ITN + validation: + type: Http + content: + request: + method: GET + url: https://open.larksuite.com/open-apis/im/v1/chats?page_size=1 + headers: + Accept: application/json + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200, 400] + - type: JsonValid + # Verified if API says success (code=0) OR "No permission" (code=99991672), + # which still indicates the token is recognized/valid but missing scopes. + - type: WordMatch + match_all_words: false + words: + - '"code":0' + - '"code": 0' + - '"code":99991672' + - '"code": 99991672' + + - name: LarkSuite User Access Token + id: kingfisher.lark.2 + pattern: | + (?xi) + (?:lark|larksuite) + (?:.|[\n\r]){0,64}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN|ACCESS_KEY) + (?:.|[\n\r]){0,32}? + \b + ( + u-[A-Z0-9_.]{14,50} + ) + \b + min_entropy: 3.2 + confidence: medium + examples: + - larksuite_user_access_token="u-ZyXwVuTsRqPoNmLkJiHgFeDcBa_5678" + references: + - https://open.larksuite.com/document/home/introduction-to-scope-and-authorization/access-credentials + - https://open.larksuite.com/document/uAjLw4CM/ukTMukTMukTM/reference/im-v1/chat/list + - https://open.larksuite.com/document/faq/trouble-shooting/how-to-fix-the-99991672-error + - https://open.larksuite.com/document/ukTMukTMukTM/ugjM14COyUjL4ITN + validation: + type: Http + content: + request: + method: GET + url: https://open.larksuite.com/open-apis/im/v1/chats?page_size=1 + headers: + Accept: application/json + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200, 400] + - type: JsonValid + - type: WordMatch + match_all_words: false + words: + - '"code":0' + - '"code": 0' + - '"code":99991672' + - '"code": 99991672' + + - name: LarkSuite App Access Token + id: kingfisher.lark.3 + pattern: | + (?xi) + (?:lark|larksuite) + (?:.|[\n\r]){0,64}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN|ACCESS_KEY) + (?:.|[\n\r]){0,32}? + \b + ( + a-[A-Z0-9_.]{30,35} + ) + \b + min_entropy: 3.2 + confidence: medium + examples: + - larksuite_app_access_token="a-QwBsTyUiOpBsDfGhJnLxYcVbN_9012" + references: + - https://open.larksuite.com/document/home/introduction-to-scope-and-authorization/access-credentials + - https://open.larksuite.com/document/server-docs/getting-started/api-access-token/auth-v3/tenant_access_token + - https://open.larksuite.com/document/server-docs/getting-started/api-access-token/auth-v3/app_access_token + validation: + type: Http + content: + request: + method: POST + # This endpoint checks app_access_token directly; no tenant_key involved. + url: https://open.larksuite.com/open-apis/authen/v1/oidc/refresh_access_token + headers: + Accept: application/json + Content-Type: application/x-www-form-urlencoded + Authorization: "Bearer {{ TOKEN }}" + # Send an intentionally bogus refresh_token. If the app_access_token is valid, + # we should get some error other than "20014" (invalid app_access_token). + body: > + grant_type=refresh_token&refresh_token=kingfisher_invalid_refresh_token + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + + # If the app token is invalid, Lark uses 20014 (and sometimes 99991664 for invalid app token). + - type: WordMatch + negative: true + match_all_words: false + words: + - '"code":20014' + - '"code": 20014' + - '"code":99991664' + - '"code": 99991664' \ No newline at end of file diff --git a/data/rules/launchdarkly.yml b/data/rules/launchdarkly.yml new file mode 100644 index 0000000..26a27d3 --- /dev/null +++ b/data/rules/launchdarkly.yml @@ -0,0 +1,34 @@ +rules: + - name: LaunchDarkly Access Token + id: kingfisher.launchdarkly.1 + pattern: | + (?xi) + launchdarkly + (?:.|[\n\r]){0,32}? + \b + ( + [a-z0-9_\-=]{40} + ) + pattern_requirements: + min_digits: 2 + min_entropy: 3.2 + confidence: medium + examples: + - LAUNCHDARKLY_TOKEN=api-123abc456def789ghi012jkl345mno678pqr + - '"launchdarkly": "ld-abcdefghijklmno1234567890pqrstuvwxzab"' + references: + - https://docs.launchdarkly.com/sdk/api/ + validation: + type: Http + content: + request: + method: GET + url: https://app.launchdarkly.com/api/v2/members + headers: + Authorization: '{{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 diff --git a/data/rules/line.yml b/data/rules/line.yml index c3b43dd..fc6ec4e 100644 --- a/data/rules/line.yml +++ b/data/rules/line.yml @@ -12,6 +12,8 @@ rules: ( (?:[0-9A-Z+/]{57}){3}=? ) + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/linear.yml b/data/rules/linear.yml index fdaa30d..3b73105 100644 --- a/data/rules/linear.yml +++ b/data/rules/linear.yml @@ -7,8 +7,10 @@ rules: ( lin_api_ (?:[0-9A-Z]{8}){5} - ) - \b + ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/linkedin.yml b/data/rules/linkedin.yml index 7663c28..cd293b7 100644 --- a/data/rules/linkedin.yml +++ b/data/rules/linkedin.yml @@ -10,6 +10,8 @@ rules: (?: id | identifier | key ) .{0,2} \s{0,20} .{0,2} \s{0,20} .{0,2} \b ([a-z0-9]{12,14}) \b + pattern_requirements: + min_digits: 2 references: - https://docs.microsoft.com/en-us/linkedin/shared/api-guide/best-practices/secure-applications min_entropy: 2.5 @@ -44,6 +46,8 @@ rules: (?: key | oauth | sec | secret )? .{0,2} \s{0,20} .{0,2} \s{0,20} .{0,2} \b ([a-z0-9]{16}) \b + pattern_requirements: + min_digits: 2 references: - https://docs.microsoft.com/en-us/linkedin/shared/api-guide/best-practices/secure-applications min_entropy: 3.3 diff --git a/data/rules/lob.yml b/data/rules/lob.yml new file mode 100644 index 0000000..deb8800 --- /dev/null +++ b/data/rules/lob.yml @@ -0,0 +1,69 @@ +rules: + - name: Lob API Key + id: kingfisher.lob.1 + pattern: | + (?xi) + lob + (?:.|[\n\r]){0,24}? + \b + ( + (?:live|test)_[a-f0-9]{35} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - export LOB_API_KEY=live_9f8e7d6c5b4a3210fedcba09876543210ab + - LOB_KEY="test_abcdefabcdefabcdefabcdefabcdefabcde" + references: + - https://docs.lob.com/#section/Authentication + validation: + type: Http + content: + request: + method: GET + url: https://api.lob.com/v1/addresses?limit=1 + headers: + Authorization: "Basic {{ TOKEN | append: ':' | b64enc }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 + - name: Lob Publishable API Key + id: kingfisher.lob.2 + pattern: | + (?xi) + lob + (?:.|[\n\r]){0,24}? + \b + ( + (?:test|live)_pub_[a-f0-9]{31} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + examples: + - const LOB_PUB_KEY = "test_pub_abcdefabcdefabcdefabcdefabcdefa"; + - LOB_PUBLISHABLE="live_pub_1234567890abcdef1234567890abcde" + references: + - https://docs.lob.com/#section/Authentication + validation: + type: Http + content: + request: + method: GET + url: https://api.lob.com/v1/addresses?limit=1 + headers: + Authorization: "Basic {{ TOKEN | append: ':' | b64enc }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 diff --git a/data/rules/looker.yml b/data/rules/looker.yml new file mode 100644 index 0000000..70621e0 --- /dev/null +++ b/data/rules/looker.yml @@ -0,0 +1,85 @@ +rules: + - name: Looker Base URL + id: kingfisher.looker.1 + visible: false + confidence: low + min_entropy: 2.0 + pattern: | + (?xi) + \b + ( + https?://[a-z0-9.-]+(?::\d{2,5})? + ) + (?:/api/(?:4\.0|3\.1))? + \b + examples: + - https://example.cloud.looker.com + - https://example.cloud.looker.com:19999 + - https://example.cloud.looker.com:19999/api/4.0 + + - name: Looker Client ID + id: kingfisher.looker.2 + confidence: medium + min_entropy: 3.0 + pattern: | + (?xi) + \blooker + (?:.|[\n\r]){0,64}? + \b + ( + [a-z0-9]{20} + ) + \b + pattern_requirements: + min_digits: 2 + examples: + - LOOKER_CLIENT_ID=1a2b3c4d5e6f7g8h9i0j + - 'looker client_id: "0a1b2c3d4e5f6g7h8i9j"' + references: + - https://docs.cloud.google.com/looker/docs/api-auth + + - name: Looker Client Secret + id: kingfisher.looker.3 + confidence: medium + min_entropy: 3.5 + pattern: | + (?xi) + \b + looker + (?:.|[\n\r]){0,64}? + \b + ( + [a-z0-9]{24} + ) + \b + pattern_requirements: + min_digits: 2 + examples: + - LOOKER_CLIENT_SECRET=1a2b3c4d5e6f7g8h9i0j1k2l + - 'looker client_secret: "0a1b2c3d4e5f6g7h8i9j0k1l"' + references: + - https://docs.cloud.google.com/looker/docs/api-auth + - https://docs.cloud.google.com/looker/docs/reference/looker-api/latest/methods/ApiAuth/login + depends_on_rule: + - rule_id: kingfisher.looker.1 + variable: LOOKER_BASE_URL + - rule_id: kingfisher.looker.2 + variable: LOOKER_CLIENT_ID + validation: + type: Http + content: + request: + method: POST + url: "{{ LOOKER_BASE_URL }}/api/4.0/login" + headers: + Content-Type: application/x-www-form-urlencoded + Accept: application/json + body: | + client_id={{ LOOKER_CLIENT_ID }}&client_secret={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + words: ['"access_token"'] diff --git a/data/rules/mailchimp.yml b/data/rules/mailchimp.yml index a28eaae..c3f6962 100644 --- a/data/rules/mailchimp.yml +++ b/data/rules/mailchimp.yml @@ -3,8 +3,9 @@ rules: id: kingfisher.mailchimp.1 pattern: | (?xi) + \b mailchimp - (?:.|[\n\r]){0,32}? + (?:.|[\n\r]){0,128}? (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) (?:.|[\n\r]){0,32}? \b @@ -12,6 +13,9 @@ rules: (?:[0-9a-f]{8}){4} -us\d{1,2} ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/mailgun.yml b/data/rules/mailgun.yml index c17c40b..5181f3b 100644 --- a/data/rules/mailgun.yml +++ b/data/rules/mailgun.yml @@ -2,8 +2,8 @@ rules: - name: MailGun Token id: kingfisher.mailgun.1 pattern: | - (?xi) - \b + (?xi) + \b mailgun (?:.|[\n\r]){0,32}? (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) @@ -11,7 +11,11 @@ rules: \b ( (?:[0-9A-Z-]{24}){3} - ) + ) + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.5 confidence: medium examples: @@ -41,6 +45,8 @@ rules: key-(?:[0-9a-f]{8}){4} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/mailjet.yml b/data/rules/mailjet.yml new file mode 100644 index 0000000..69161b4 --- /dev/null +++ b/data/rules/mailjet.yml @@ -0,0 +1,75 @@ +rules: + - name: MailJetSMS API Key + id: kingfisher.mailjet.1 + pattern: | + (?xi) + \b + mailjet + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - mailjet ABCDEFGHIJKLMNOPQRSTUVWXYZ012345 + - mailjet-token 9A1B2C3D4E5F6G7H8I9J0K1L2M3N4O5P + references: + - https://dev.mailjet.com/sms/reference/overview/authentication/ + - https://www.postman.com/mailjet-api/mailjet-s-public-workspace/request/velnqvd/retrieve-a-count-of-all-sms-messages + validation: + type: Http + content: + request: + method: GET + url: "https://api.mailjet.com/v4/sms/count" + headers: + Accept: "application/vnd.mailjetsms+json; version=3" + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"Count"'] + - name: MailJet Basic Auth + id: kingfisher.mailjet.2 + pattern: | + (?xi) + \b + mailjet + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9]{87}= + ) + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - mailjet_token = neno01fy530zukbtvq8xunwec74b7m7lsmzha8su93zdvy4mp4dc5gctfa2rcwetllcjzncirjv58se7iwkehhh= + references: + - https://dev.mailjet.com/email/reference/overview/authentication/ + - https://www.postman.com/mailjet-api/mailjet-s-public-workspace/request/5pnoxig/retrieve-all-api-keys + validation: + type: Http + content: + request: + method: GET + url: "https://api.mailjet.com/v3/REST/apikey?Limit=1" + headers: + Authorization: "Basic {{ TOKEN }}" + Accept: "application/json" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"Data"', '"Count"'] diff --git a/data/rules/mandrill.yml b/data/rules/mandrill.yml index 4abd63d..857fbeb 100644 --- a/data/rules/mandrill.yml +++ b/data/rules/mandrill.yml @@ -11,7 +11,11 @@ rules: \b ( (?:[0-9A-Z_-]{11}){2} - ) + ) + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/mapbox.yml b/data/rules/mapbox.yml index d0253d6..313eb9e 100644 --- a/data/rules/mapbox.yml +++ b/data/rules/mapbox.yml @@ -1,7 +1,9 @@ rules: - name: Mapbox Public Access Token id: kingfisher.mapbox.1 - pattern: '(?i)(?s)mapbox.{0,30}(pk\.[a-z0-9\-+/=]{32,128}\.[a-z0-9\-+/=]{20,30})(?:[^a-z0-9\-+/=]|$)' + pattern: '(?i)(?s)mapbox.{0,30}(pk\.[a-z0-9\-+/=]{32,128}\.[a-z0-9\-+/=]{20,30})\b' + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: @@ -26,7 +28,15 @@ rules: - name: Mapbox Secret Access Token id: kingfisher.mapbox.2 - pattern: '(?i)(?s)mapbox.{0,30}(sk\.[a-z0-9\-+/=]{32,128}\.[a-z0-9\-+/=]{20,30})(?:[^a-z0-9\-+/=]|$)' + pattern: | + (?xi)(?s) + mapbox.{0,30} + ( + sk\.[a-z0-9\-+/=]{32,128}\.[a-z0-9\-+/=]{20,30} + ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: @@ -50,7 +60,9 @@ rules: - name: Mapbox Temporary Access Token id: kingfisher.mapbox.3 - pattern: '(?i)(?s)mapbox.{0,30}(tk\.[a-z0-9\-+/=]{32,128}\.[a-z0-9\-+/=]{20,30})(?:[^a-z0-9\-+/=]|$)' + pattern: '(?i)(?s)mapbox.{0,30}(tk\.[a-z0-9\-+/=]{32,128}\.[a-z0-9\-+/=]{20,30})\b' + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/mattermost.yml b/data/rules/mattermost.yml new file mode 100644 index 0000000..ef69c78 --- /dev/null +++ b/data/rules/mattermost.yml @@ -0,0 +1,68 @@ +rules: + - name: Mattermost URL + id: kingfisher.mattermost.1 + pattern: | + (?xi) + \b + mattermost + (?:.|[\n\r]){0,32}? + ( + https?:\/\/[a-z0-9.-]+ + (?::\d{2,5})? + (?:\/[A-Za-z0-9._~\-\/]*)? + ) + \b + confidence: medium + visible: false + min_entropy: 2.0 + examples: + - mattermost_url = "https://community.mattermost.com" + - mattermost_url='http://localhost:8065' + - 'mattermost_url: https://intra.example.com/mattermost' + + - name: Mattermost Access Token + id: kingfisher.mattermost.2 + pattern: | + (?xi) + \b + mattermost + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9]{26} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 4.0 + examples: + - "mattermost_token: abcde12345fghij67890klmno1" + validation: + type: Http + content: + request: + method: GET + # Normalize any captured base that already includes /api/v4 + url: > + {%- assign base = MATTERMOST_URL | replace: "/api/v4/", "/" | replace: "/api/v4", "" -%} + {{ base }}/api/v4/users/me + headers: + Authorization: "Bearer {{ TOKEN }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + words: ['"id"', '"username"'] + match_all_words: true + depends_on_rule: + - rule_id: "kingfisher.mattermost.1" + variable: MATTERMOST_URL + references: + - https://developers.mattermost.com/api-documentation/ + - https://developers.mattermost.com/integrate/faq/ diff --git a/data/rules/maxmind.yml b/data/rules/maxmind.yml new file mode 100644 index 0000000..9952118 --- /dev/null +++ b/data/rules/maxmind.yml @@ -0,0 +1,58 @@ +rules: + - name: MaxMind License Key + id: kingfisher.maxmind.1 + pattern: | + (?xi) + \b + ( + [a-z0-9]{6}_[a-z0-9]{29}_mmk + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.8 + confidence: medium + examples: + - MAXMIND_LICENSE=AB12CD_1234567890abcdef1234567890abc_mmk + - license_key="ZXCVBN_0987654321abcdef1234567890abc_mmk" + references: + - https://dev.maxmind.com/geoip/docs/web-services + depends_on_rule: + - rule_id: kingfisher.maxmind.2 + variable: ACCOUNT_ID + validation: + type: Http + content: + request: + method: GET + url: https://geoip.maxmind.com/geoip/v2.1/city/me + headers: + Authorization: "Basic {{ ACCOUNT_ID | append: ':' | append: TOKEN | b64enc }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 + - name: MaxMind Account ID + id: kingfisher.maxmind.2 + pattern: | + (?xi) + (?:maxmind|geoip|geolite) + (?:.|[\n\r]){0,40}? + (?:account|user) + (?:.|[\n\r]){0,10}? + (?:id|number) + (?:.|[\n\r]){0,16}? + ( + \d{4,8} + ) + min_entropy: 2.0 + confidence: medium + visible: false + examples: + - MAXMIND_ACCOUNT_ID=123456 + - '"maxmind": {"account_id": "654321", "license_key": "..."}' + - 'geoip_account_number: 456789' + references: + - https://dev.maxmind.com/geoip/docs/web-services \ No newline at end of file diff --git a/data/rules/mercury.yml b/data/rules/mercury.yml new file mode 100644 index 0000000..51eaacf --- /dev/null +++ b/data/rules/mercury.yml @@ -0,0 +1,67 @@ +rules: + - name: Mercury Production API Token + id: kingfisher.mercury.1 + pattern: | + (?x) + \b + ( + mercury_production_ + [a-z]{3,6} + _ + [a-zA-Z0-9]{40,50} + _yrucrem + ) + \b + min_entropy: 3.5 + confidence: medium + examples: + - Bearer secret-token:mercury_production_wma_24SCp4G81X3yHL4Wq8FgzuaP9ye3VKf2mgTDctXyRg5HY_yrucrem + references: + - https://docs.mercury.com/docs/api-token-security-policies + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + Accept: application/json + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://api.mercury.com/api/v1/accounts + - name: Mercury Non-Production API Token + id: kingfisher.mercury.2 + pattern: | + (?x) + \b + ( + mercury_sandbox_ + [a-z]{3,6} + _ + [a-zA-Z0-9]{40,50} + _yrucrem + ) + \b + min_entropy: 3.5 + confidence: medium + examples: + - Bearer secret-token:mercury_sandbox_rma_24pnbcT7NygLbpJPr4xBuSuBDpo6tK89S8u3ERYn3FXVz_yrucrem + references: + - https://docs.mercury.com/docs/api-token-security-policies + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + Accept: application/json + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://api-sandbox.mercury.com/api/v1/accounts diff --git a/data/rules/mergify.yml b/data/rules/mergify.yml new file mode 100644 index 0000000..85673d0 --- /dev/null +++ b/data/rules/mergify.yml @@ -0,0 +1,39 @@ +rules: + - name: Mergify Application API Key + id: kingfisher.mergify.1 + pattern: | + (?x) + \b + ( + mergify_application_key_ + [A-Za-z0-9_-]{40,200} + ) + \b + min_entropy: 3.2 + confidence: high + examples: + - mergify_application_key_A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8S9t0U1v2W3x4Y5z6_-AbCdEfGhIj + - mergify_application_key_ZxYwVuTsRqPoNmLkJiHgFeDcBa9876543210_-__aBcDeFgHiJkLmNoPqRsTuVwXyZ + references: + - https://docs.mergify.com/api/ + - https://docs.mergify.com/api-usage/ + validation: + type: Http + content: + request: + method: GET + url: https://api.mergify.com/v1/application + headers: + Accept: application/json + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: HeaderMatch + header: content-type + expected: ["application/json"] + - type: JsonValid + - type: WordMatch + words: ['"id"', '"name"', '"scope"'] + match_all_words: true diff --git a/data/rules/messagebird.yml b/data/rules/messagebird.yml new file mode 100644 index 0000000..4b89531 --- /dev/null +++ b/data/rules/messagebird.yml @@ -0,0 +1,36 @@ +rules: + - name: MessageBird API Token + id: kingfisher.messagebird.1 + pattern: | + (?xi) + \b + message[_-]?bird + (?:.|[\n\r]){0,32}? + \b + ( + [a-z0-9]{25} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.4 + confidence: medium + examples: + - MESSAGEBIRD_API_KEY=abcdefghijklmnopqrstuvwxy + - "messagebird_token: 'abcde12345fghij67890klmno'" + references: + - https://developers.messagebird.com/api/#authentication + validation: + type: Http + content: + request: + method: GET + url: https://rest.messagebird.com/balance + headers: + Authorization: 'AccessKey {{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid \ No newline at end of file diff --git a/data/rules/microsoft_teams.yml b/data/rules/microsoft_teams.yml index 37e4030..98c9068 100644 --- a/data/rules/microsoft_teams.yml +++ b/data/rules/microsoft_teams.yml @@ -5,27 +5,29 @@ rules: (?xi) ( https:// - outlook\.office\.com/webhook/ - [0-9a-f]{8}- - [0-9a-f]{4}- - [0-9a-f]{4}- - [0-9a-f]{4}- - [0-9a-f]{12} + .*\.office\.com/webhook/ + [0-9a-z]{8}- + [0-9a-z]{4}- + [0-9a-z]{4}- + [0-9a-z]{4}- + [0-9a-z]{12} @ - [0-9a-f]{8}- - [0-9a-f]{4}- - [0-9a-f]{4}- - [0-9a-f]{4}- - [0-9a-f]{12} + [0-9a-z]{8}- + [0-9a-z]{4}- + [0-9a-z]{4}- + [0-9a-z]{4}- + [0-9a-z]{12} /IncomingWebhook/ - [0-9a-f]{32} + [0-9a-z]{32} / - [0-9a-f]{8}- - [0-9a-f]{4}- - [0-9a-f]{4}- - [0-9a-f]{4}- - [0-9a-f]{12} + [0-9a-z]{8}- + [0-9a-z]{4}- + [0-9a-z]{4}- + [0-9a-z]{4}- + [0-9a-z]{12} ) + pattern_requirements: + min_digits: 8 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/microsoftteamswebhook.yml b/data/rules/microsoftteamswebhook.yml index 82fbb19..952f1b9 100644 --- a/data/rules/microsoftteamswebhook.yml +++ b/data/rules/microsoftteamswebhook.yml @@ -3,17 +3,23 @@ rules: id: kingfisher.microsoftteamswebhook.1 pattern: | (?xi) - https://[A-Z0-9]+\.webhook\.office\.com/webhookb2 - / - [A-Z0-9]{8}-[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{12} - @ - [A-Z0-9]{8}-[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{12} - / - IncomingWebhook - / - [A-Z0-9]{32} - / - [A-Z0-9]{8}-[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{12} + \b + ( + https://[A-Z0-9]+\.webhook\.office\.com/webhookb2 + / + [A-Z0-9]{8}-[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{12} + @ + [A-Z0-9]{8}-[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{12} + / + IncomingWebhook + / + [A-Z0-9]{32} + / + [A-Z0-9]{8}-[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{4}-[A-Z0-9]{12} + ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: @@ -22,15 +28,14 @@ rules: type: Http content: request: - body: | - {'text':''} + body: '{"text":""}' headers: Content-Type: application/json method: POST response_matcher: - type: StatusMatch status: - - 200 + - 400 - report_response: true type: WordMatch words: diff --git a/data/rules/mistral.yml b/data/rules/mistral.yml index 07b1af8..3e451da 100644 --- a/data/rules/mistral.yml +++ b/data/rules/mistral.yml @@ -13,6 +13,8 @@ rules: [A-Z0-9]{32} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.0 confidence: medium examples: diff --git a/data/rules/monday.yml b/data/rules/monday.yml new file mode 100644 index 0000000..8459472 --- /dev/null +++ b/data/rules/monday.yml @@ -0,0 +1,37 @@ +rules: + - name: Monday.com API Key + id: kingfisher.monday.1 + pattern: | + (?xi) + \b + monday + (?:.|[\n\r]){0,40}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,40}? + \b + ( + eyJ[A-Za-z0-9-_]{10,200}\.eyJ[A-Za-z0-9-_]{50,1000}\.[A-Za-z0-9-_]{20,500} + ) + \b + min_entropy: 3.3 + confidence: medium + examples: + - monday SECRET_TOKEN=eyJhbGciOiJIUzI1TiJ9.eyJ0aWQiOjU7OTC4MzIwMywiYWFpIjoxMSwidWlkIjo5NjYwMzk5MCwiaWBkIjoiMjAyNS0xMS0yMVQwMDoyNjoxMy43OCVaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MzI2MDI5MTIsInJnbiI6InVzZTEifQ.wQtV6psL1JqFHdXgRB2J7-qslSyS2I4TYJHtkX9ofvk + validation: + type: Http + content: + request: + url: https://api.monday.com/v2 + method: POST + headers: + Content-Type: application/json + Authorization: '{{ TOKEN }}' + body: | + {"query": "query { me { id name } }"} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ["data", "me", "id"] + match_all_words: true \ No newline at end of file diff --git a/data/rules/mongodb.yml b/data/rules/mongodb.yml index 63e4775..bdca97f 100644 --- a/data/rules/mongodb.yml +++ b/data/rules/mongodb.yml @@ -11,6 +11,7 @@ rules: .{0,1000}? (?:private|priv|secret|auth|pass|key) (?:.|[\n\r]){0,32}? + \b ( [a-fA-F0-9]{8} - @@ -21,7 +22,12 @@ rules: [a-fA-F0-9]{4} - [a-fA-F0-9]{12} - ) + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.7 examples: - ATLAS_PRIVATE_KEY=4b18315e-6b7d-4337-b449-5d38f5a189ec @@ -75,11 +81,18 @@ rules: mongodb(?:\+srv)?://[\S]{3,50}:(?:[\S]{3,88})@[-.%\w/:]+ ) \b + pattern_requirements: + ignore_if_contains: + - "****" + - "xxxx" + - "example" min_entropy: 3 examples: - client = mongoc_client_new ("mongodb+srv://someuser:hunter2@my-atlas-rd941.mongodb.net/test?retryWrites=true&w=majority"); - "mongodb+srv://user:passw0rd@cluster0.something.mongodb.net/" - "mongodb://mongoadmin:contoso@something.foo.mongodb.net/myFirstDatabase" + validation: + type: MongoDB - name: MongoDB Atlas Service Account Token id: kingfisher.mongodb.4 pattern: | @@ -88,7 +101,8 @@ rules: ( mdb_sa_sk_[0-9A-Z_-]{6}[0-9A-Z]{34} ) - \b min_entropy: 3.5 examples: - - mdb_sa_sk_BdIX_jLzut2WTgglKzKvSgWMDDj5hEoTqdwOyLOL \ No newline at end of file + - mdb_sa_sk_BdIX_jLzut2WTgglKzKvSgWMDDj5hEoTqdwOyLOL + validation: + type: MongoDB \ No newline at end of file diff --git a/data/rules/mysql.yaml b/data/rules/mysql.yaml deleted file mode 100644 index 3a3a0ab..0000000 --- a/data/rules/mysql.yaml +++ /dev/null @@ -1,7 +0,0 @@ -rules: - - name: MySQL URI with Credentials - id: kingfisher.mysql.1 - pattern: (?xi)\bmysql:\/\/[a-z0-9]+:([a-z0-9!@\#$%^&*()_+{}|:<>?=\\-]+)@[a-z0-9.]+:[0-9]+\/[a-z0-9]+\b - min_entropy: 3.5 - examples: - - CONNECTION_URI="mysql://nimda:m42p!o@2wd@google.com:5434/elephant" diff --git a/data/rules/mysql.yml b/data/rules/mysql.yml new file mode 100644 index 0000000..88e3154 --- /dev/null +++ b/data/rules/mysql.yml @@ -0,0 +1,46 @@ +rules: + - name: MySQL URI with Credentials + id: kingfisher.mysql.1 + pattern: | + (?xi) + ( + mysql:\/\/ + (?: + [a-z0-9._%+\-]+ + ) + : + (?: + [^\s:@]+ + ) + @ + (?: + \[ + [0-9a-f:.]+ + \] + | + [a-z0-9.-]+ + ) + (?:: + \d{2,5} + )? + (?: + \/ + [^\s"'?:]+ + )? + (?: + \? + [^\s"']* + )? + ) + pattern_requirements: + ignore_if_contains: + - "****" + - "xxxx" + - "example" + min_entropy: 3.3 + confidence: medium + examples: + - CONNECTION_URI="mysql://nimda:m42p!o@2wd@google.com:3306/elephant" + - mysql://user:pass@example.com:4406/app_db?ssl-mode=REQUIRED + validation: + type: MySQL diff --git a/data/rules/nasa.yml b/data/rules/nasa.yml index efbcc42..1509b2e 100644 --- a/data/rules/nasa.yml +++ b/data/rules/nasa.yml @@ -8,10 +8,13 @@ rules: (?:.|[\n\r]){0,32}? (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) (?:.|[\n\r]){0,32}? + \b ( [A-Z0-9]{40} ) \b + pattern_requirements: + min_digits: 2 examples: - | get('https://api.nasa.gov/planetary/earth/imagery?api_key=fWfSMcDzyHfMuH8BW6jiIUBYaj0hKRyKBRTBqgEQ') diff --git a/data/rules/neon.yml b/data/rules/neon.yml new file mode 100644 index 0000000..2c7ee79 --- /dev/null +++ b/data/rules/neon.yml @@ -0,0 +1,31 @@ +rules: + - name: Neon API Key + id: kingfisher.neon.1 + pattern: | + (?x) + \b + ( + napi_ + [a-zA-Z0-9]{64} + ) + \b + min_entropy: 3.5 + confidence: high + examples: + - napi_f6n4wv0d0nzglfk64c1bnzrc5ug82tmrmekh8h4hsxeq8zd0p5ii234bdkah71kw + references: + - https://neon.com/docs/manage/api-keys + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + Accept: application/json + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://console.neon.tech/api/v2/auth diff --git a/data/rules/netlify.yml b/data/rules/netlify.yml index 0a29d25..7fae1eb 100644 --- a/data/rules/netlify.yml +++ b/data/rules/netlify.yml @@ -8,8 +8,12 @@ rules: (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) (?:.|[\n\r]){0,32}? \b - ([a-f0-9]{60,64}) + ( + [a-f0-9]{60,64} + ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 examples: - netlify_token=3cdfad7b885a6daceff3fb820389115750b373763fb30b10ca0382648b55872d @@ -41,6 +45,8 @@ rules: [A-Z0-9_-]{43,45} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/netrc.yml b/data/rules/netrc.yml index 6b6be12..950e1da 100644 --- a/data/rules/netrc.yml +++ b/data/rules/netrc.yml @@ -10,6 +10,10 @@ rules: password \s+ ([^\s]+) min_entropy: 3.3 confidence: medium + pattern_requirements: + ignore_if_contains: + - '${' + - '$(' examples: - 'machine api.github.com login ziggy^stardust password 012345abcdef' - | diff --git a/data/rules/newrelic.yml b/data/rules/newrelic.yml index d9a582c..c208aa0 100644 --- a/data/rules/newrelic.yml +++ b/data/rules/newrelic.yml @@ -14,6 +14,9 @@ rules: - [A-Z0-9_.]{42} ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/ngrok.yml b/data/rules/ngrok.yml index 56f1b62..711648c 100644 --- a/data/rules/ngrok.yml +++ b/data/rules/ngrok.yml @@ -3,12 +3,16 @@ rules: id: kingfisher.ngrok.1 pattern: | (?xi) + \b ngrok (?:.|[\\n\r]){0,32}? (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) (?:.|[\n\r]){0,32}? - \b - (?:[a-z0-9]{25,30}_\d[a-z0-9]{20}|(?:cr_|ak_)[a-z0-9]{25,30}) + ( + (?:[a-z0-9]{25,30}_\d[a-z0-9]{20} + | + (?:cr_|ak_)[a-z0-9]{25,30}) + ) \b min_entropy: 4 examples: diff --git a/data/rules/notion.yml b/data/rules/notion.yml new file mode 100644 index 0000000..8b3f798 --- /dev/null +++ b/data/rules/notion.yml @@ -0,0 +1,82 @@ +rules: + - name: Notion Legacy Token + id: kingfisher.notion.1 + pattern: | + (?xi) + notion + (?:.|[\\n\r]){0,32}? + \b + ( + secret_[A-Z0-9]{43} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 4.0 + confidence: medium + examples: + - "notion secret_efky1RtWsI0CB1Sn4TRRBLpemW1V11XwPRX3lzUKc5Q" + validation: + type: Http + content: + request: + headers: + Notion-Version: "2022-06-28" + Authorization: "Bearer {{ TOKEN }}" + method: GET + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"object":"user"' + - '"type":"bot"' + match_all_words: true + url: https://api.notion.com/v1/users/me + - name: Notion Token + id: kingfisher.notion.2 + pattern: | + (?xi) + \b + ( + ntn_[0-9]{11}[A-Za-z0-9]{35} + ) + min_entropy: 4.0 + confidence: medium + references: + - https://developers.notion.com/page/changelog#september-11-2024 + examples: + - "notion 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" + validation: + type: Http + content: + request: + headers: + Notion-Version: "2022-06-28" + Authorization: "Bearer {{ TOKEN }}" + method: GET + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"object":"user"' + - '"type":"bot"' + match_all_words: true + url: https://api.notion.com/v1/users/me + + - name: Notion OAuth Refresh Token + id: kingfisher.notion.3 + pattern: | + (?xi) + notion + (?:.|[\\n\r]){0,32}? + ( + nrt_[A-Z0-9_]{40,55} + ) + min_entropy: 3.5 + confidence: medium + examples: + - "notion refresh token = nrt_4Y29zY29vbF9leGFtcGxlX3JlZnJlc2hfdG9rZW4xMjM0NQ" \ No newline at end of file diff --git a/data/rules/npm.yml b/data/rules/npm.yml index 5d2e8c8..6132d80 100644 --- a/data/rules/npm.yml +++ b/data/rules/npm.yml @@ -5,9 +5,17 @@ rules: (?xi) \b ( - npm_[A-Z0-9]{36} + npm_(?P[A-Za-z0-9]{30})(?P[A-Za-z0-9]{6}) ) \b + pattern_requirements: + min_digits: 2 + checksum: + actual: + template: "{{ MATCH | suffix: 6 }}" + requires_capture: checksum + expected: "{{ BODY | crc32 | base62: 6 }}" + skip_if_missing: true references: - https://docs.npmjs.com/about-access-tokens - https://github.com/github/roadmap/issues/557 @@ -15,7 +23,7 @@ rules: min_entropy: 3.3 confidence: medium examples: - - 'npm_TCllNwh2WLQlMWVhybM1iQrsTj6rMQ0BOh6d' + - "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" validation: type: Http content: @@ -35,7 +43,6 @@ rules: id: kingfisher.npm.2 pattern: | (?xi) - \b (?:_authToken|NPM_TOKEN) (?:.|[\n\r]){0,16}? ( diff --git a/data/rules/nuget.yml b/data/rules/nuget.yml index 663a415..3ba909e 100644 --- a/data/rules/nuget.yml +++ b/data/rules/nuget.yml @@ -8,6 +8,8 @@ rules: oy2[a-z0-9]{43} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: @@ -40,11 +42,9 @@ rules: (?:.|[\n\r]){0,32}? (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) (?:.|[\n\r]){0,32}? - \b ( [a-z0-9]{46} ) - \b min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/nvidia.yml b/data/rules/nvidia.yml new file mode 100644 index 0000000..1329314 --- /dev/null +++ b/data/rules/nvidia.yml @@ -0,0 +1,31 @@ +rules: + - name: NVIDIA NIM API Key + id: kingfisher.nvidia.nim.1 + pattern: | + (?xi) + ( + nvapi-[A-Z0-9_-]{60,70} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.5 + examples: + - "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" + - "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" + - "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" + validation: + type: Http + content: + request: + method: GET + url: "https://api.nvcf.nvidia.com/v2/nvcf/functions" + headers: + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ["id", "versionId"] diff --git a/data/rules/nylas.yml b/data/rules/nylas.yml new file mode 100644 index 0000000..5acc8d8 --- /dev/null +++ b/data/rules/nylas.yml @@ -0,0 +1,62 @@ +rules: + # Helper: capture the Nylas API base URI (data residency) from config/env so validation hits the right region. + - name: Nylas API URI + id: kingfisher.nylas.api_uri.1 + visible: false + confidence: medium + min_entropy: 2.0 + pattern: | + (?xi) + \b + ( + https://api\.(?:us|eu)\.nylas\.com + ) + \b + examples: + - https://api.us.nylas.com + - https://api.eu.nylas.com + + - name: Nylas API Key + id: kingfisher.nylas.1 + pattern: | + (?xi) + \b + nylas + (?:.|[\n\r]){0,64}? + (?:api[_-]?key|apikey|secret|private|access|token) + (?:.|[\n\r]){0,64}? + \b + ( + nyk_[A-Z0-9]{67} # common v3 API key format (71 chars total) + | + [0-9A-Z]{30} # legacy/older patterns seen in repos + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.3 + confidence: medium + examples: + - NYLAS_API_KEY=nyk_0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234 + - nylas_token = 2temab2qpfioneggb01j2dhfllqgiu + references: + - https://developer.nylas.com/docs/v3/auth/hosted-oauth-apikey/ + - https://developer.nylas.com/docs/v3/notifications/ + depends_on_rule: + - rule_id: kingfisher.nylas.api_uri.1 + variable: NYLAS_API_URI + validation: + type: Http + content: + request: + method: GET + url: "{{ NYLAS_API_URI }}/v3/webhooks" + headers: + Authorization: "Bearer {{ TOKEN }}" + Accept: "application/json" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"request_id"', '"data"'] diff --git a/data/rules/nytimes.yml b/data/rules/nytimes.yml new file mode 100644 index 0000000..391dd3f --- /dev/null +++ b/data/rules/nytimes.yml @@ -0,0 +1,34 @@ +rules: + - name: New York Times API Key + id: kingfisher.nytimes.1 + pattern: | + (?xi) + (?:nytimes|new[- ]?york[- ]?times) + (?:.|[\n\r]){0,32}? + \b + ( + [a-z0-9_\-=]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.2 + confidence: medium + examples: + - NYTIMES_API_KEY=abcd1234efgh5678ijkl9012mnop3456 + - '"new-york-times": "zyxw9876vuts5432rqpo1098nmlk7654"' + references: + - https://developer.nytimes.com/ + validation: + type: Http + content: + request: + method: GET + url: https://api.nytimes.com/svc/topstories/v2/home.json?api-key={{ TOKEN }} + headers: + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 diff --git a/data/rules/odbc.yml b/data/rules/odbc.yml index 6593f92..6f60c6a 100644 --- a/data/rules/odbc.yml +++ b/data/rules/odbc.yml @@ -6,6 +6,10 @@ rules: (?: User | User\ Id | UserId | Uid) \s*=\s* ([^\s;]{3,100}) \s* ; [\ \t]* .{0,10} [\ \t]* (?: Password | Pwd) \s*=\s* ([^\t\ ;]{3,100}) \s* (?: [;] | $) + pattern_requirements: + ignore_if_contains: + - "localhost" + - "127.0.0.1" min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/okta.yml b/data/rules/okta.yml index 65490b4..16511f4 100644 --- a/data/rules/okta.yml +++ b/data/rules/okta.yml @@ -10,6 +10,9 @@ rules: ( 00[a-z0-9_-]{39}[a-z0-9_] ) + \b + pattern_requirements: + min_digits: 4 min_entropy: 3.3 examples: - okta_api_token=00hqNORUpnTcdFWA5WEM4YwOkw6RXeFw21lFDRKmY1 @@ -44,7 +47,6 @@ rules: id: kingfisher.okta.2 pattern: | (?xi) - \b ( [a-z0-9-]{1,40}\.okta(?:preview|-emea)?\.com ) diff --git a/data/rules/ollama.yml b/data/rules/ollama.yml new file mode 100644 index 0000000..ba686c6 --- /dev/null +++ b/data/rules/ollama.yml @@ -0,0 +1,49 @@ +rules: + - name: Ollama API Key + id: kingfisher.ollama.1 + pattern: | + (?xi) + \b + ollama + (?:.|[\n\r]){0,32}? + \b + ( + [a-f0-9]{32}\.[a-zA-Z0-9_-]{24} + ) + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.5 + validation: + type: Http + content: + request: + method: POST + url: https://ollama.com/api/generate + headers: + Content-Type: application/json + # Turbo keys are sent as the raw value in Authorization (no "Bearer " prefix) + # per working client behavior. + Authorization: "{{ TOKEN }}" + body: | + { + "model": "gpt-oss:20b", + "prompt": "ping", + "stream": false + } + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"response":' + - '"done":true' + references: + - https://ollama.com/blog/turbo + examples: + - "ollama key = 8bcdd9b4e28e4e1b8bf14a2eb8701220.QH5p5TU2BDwzHu5_RCtvJXsj" + - "ollama key = e56714bd7c1146e4b4801244bc2bc67a.3GAswjZGZ5YY6Qdgt0xg56vM" + - "ollama key = 872658d00c284033a707abf1725d4b6c.-4JpTp0dQHmf0nb89xI-wgP-" + - "ollama key = 0c4e6bf1222c4ffc87025a7a9ffd5cac.z-fgt1JO9-LadzA2cL23qLH3" + - "ollama key = dae874a007d442cdb807910c4c57c6f5.B_aHUSdeAe42UR-X41StUFJq" \ No newline at end of file diff --git a/data/rules/onepassword.yml b/data/rules/onepassword.yml index 57f0980..e7f6183 100644 --- a/data/rules/onepassword.yml +++ b/data/rules/onepassword.yml @@ -1,12 +1,15 @@ rules: - name: 1Password Service-Account Token - id: kingfisher.1password.2 + id: kingfisher.1password.1 pattern: | (?xi) \b ( ops_eyj[A-Za-z0-9_-]{80,500} - )\b + ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 4.0 confidence: medium examples: @@ -37,14 +40,13 @@ rules: id: kingfisher.1password.2 pattern: | (?xi) - \b ( A[0-9]-[A-Z0-9]{6}-[A-Z0-9]{6}-[A-Z0-9]{5}(?:-[A-Z0-9]{5}){3} ) - \b + pattern_requirements: + min_digits: 2 min_entropy: 3.8 confidence: medium - prevalidated: true examples: - A3-R69SQK-TZ9KPW-8MXYD-6W373-V7GHJ-EDJQW - A3-ASWWYB-798JRY-LJVD4-23DC2-86TVM-H43EB diff --git a/data/rules/openai.yml b/data/rules/openai.yml index ef78a3f..b8c5933 100644 --- a/data/rules/openai.yml +++ b/data/rules/openai.yml @@ -8,6 +8,8 @@ rules: sk-[A-Z0-9]{48} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: @@ -20,11 +22,75 @@ rules: content: request: headers: - Authorization: "Bearer {{ TOKEN }}" + Authorization: 'Bearer {{ TOKEN }}' method: GET response_matcher: - report_response: true - status: - 200 type: StatusMatch - url: https://api.openai.com/v1/me \ No newline at end of file + url: https://api.openai.com/v1/me + + - name: OpenAI API Key + id: kingfisher.openai.2 + pattern: | + (?xi) + \b + ( + (sk-(?:proj|svcacct|None)-[A-Z0-9_-]{100,}) + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 4.0 + confidence: medium + examples: + - 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 + - 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 + - sk-None-JEBiV9H-bLEZoOhNvLWmCNR74dIbql-p3yWFmWpdYCjeR1PWM_PS40yTLowkF3VzXHYJ3VbFarT3BlbkFJYznRo8bADhczK0Ca7t-WRbdwRlC1DPc8P2EaJm03OIg01Uj0cQxRAPO-4Rjs_TNyKXnVePtkcA + references: + - https://help.openai.com/en/articles/9132009-how-can-i-view-the-users-or-organizations-associated-with-an-api-key + validation: + type: Http + content: + request: + headers: + Authorization: 'Bearer {{ TOKEN }}' + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://api.openai.com/v1/models + + - name: OpenAI API Key (Short Prefixed) + id: kingfisher.openai.3 + pattern: | + (?xi) + \b + ( + sk-None-[A-Z0-9]{48} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.3 + confidence: medium + examples: + - sk-None-abcdefghij1234567890ABCDEFGHIJ1234567890abcdefgh + references: + - https://help.openai.com/en/articles/9132009-how-can-i-view-the-users-or-organizations-associated-with-an-api-key + validation: + type: Http + content: + request: + headers: + Authorization: 'Bearer {{ TOKEN }}' + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://api.openai.com/v1/me diff --git a/data/rules/openrouter.yml b/data/rules/openrouter.yml new file mode 100644 index 0000000..a75653a --- /dev/null +++ b/data/rules/openrouter.yml @@ -0,0 +1,38 @@ +rules: + - name: OpenRouter API Key + id: kingfisher.openrouter.1 + pattern: | + (?xi) + \b + ( + sk-or-v1-[0-9a-f]{64} + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.5 + confidence: medium + examples: + - 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 + - 'Authorization: Bearer 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' + references: + - https://openrouter.ai/docs/api/reference/authentication + - https://openrouter.ai/docs/api/api-reference/credits/get-credits + validation: + type: Http + content: + request: + method: GET + url: https://openrouter.ai/api/v1/key + headers: + Authorization: "Bearer {{ TOKEN }}" + Accept: "application/json" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"data"' + - '"label"' + match_all_words: true \ No newline at end of file diff --git a/data/rules/openweather.yml b/data/rules/openweathermap.yml similarity index 55% rename from data/rules/openweather.yml rename to data/rules/openweathermap.yml index 2153e64..f2b2743 100644 --- a/data/rules/openweather.yml +++ b/data/rules/openweathermap.yml @@ -7,17 +7,16 @@ rules: (?:.|[\n\r]){0,64}? \b ( - (?: - [a-z0-9]{32} - ) - \b - |APPID= - (?: - [a-z0-9]{32} - ) + [a-z0-9]{32} + | + APPID= + [a-z0-9]{32} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 + confidence: medium examples: - pyowm = '3k144a5af729351d0fc58bdrj9a21mkr' - owm = '3k144a5af729351d0fc58bdrj9a21mkr' @@ -28,10 +27,14 @@ rules: content: request: method: GET + url: https://api.openweathermap.org/data/2.5/forecast?q=London&appid={{ TOKEN }} response_matcher: - report_response: true - - match_all_status: true - status: - - 200 - type: StatusMatch - url: https://api.openweathermap.org/geo/1.0/reverse?lat=0&lon=0&limit=1&appid={{ TOKEN }} \ No newline at end of file + - type: StatusMatch + status: [200] + words: ['"cod":"200"'] + references: + - https://openweathermap.org/forecast5 + - https://openweathermap.org/appid + - https://publicapi.dev/open-weather-map-api + diff --git a/data/rules/opsgenie.yml b/data/rules/opsgenie.yml index 012f296..4369d2f 100644 --- a/data/rules/opsgenie.yml +++ b/data/rules/opsgenie.yml @@ -12,20 +12,29 @@ rules: ( [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 examples: - opsgenie_api_key = '12345678-9abc-def0-1234-56789abcdef0' + references: + - https://docs.opsgenie.com/docs/api-overview + - https://support.atlassian.com/security-and-access-policies/docs/send-alerts-to-opsgenie/ + - https://support.atlassian.com/opsgenie/docs/european-service-region/ validation: type: Http content: request: - headers: - Authorization: GenieKey {{ TOKEN }} method: GET url: https://api.opsgenie.com/v2/alerts + headers: + Authorization: "GenieKey {{ TOKEN }}" response_matcher: - report_response: true - - type: WordMatch - words: - - "Could not authenticate" + - type: StatusMatch + status: [401, 403] negative: true + - type: WordMatch + words: ["Could not authenticate", "is not valid"] + negative: true \ No newline at end of file diff --git a/data/rules/optimizely.yml b/data/rules/optimizely.yml new file mode 100644 index 0000000..46f3016 --- /dev/null +++ b/data/rules/optimizely.yml @@ -0,0 +1,38 @@ +rules: + - name: Optimizely Personal Access Token + id: kingfisher.optimizely.1 + pattern: | + (?xi) + \b + optimizely + (?:.|[\n\r]){0,64}? + \b + ( + [0-9A-Z-:]{54} + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.6 + confidence: medium + examples: + - OPTIMIZELY_TOKEN=AbCDefGhijKlmnOpqrStuvWxYz01-23:45AbCDefGhijKlmnOpqrSt + - 'Optimizely Authorization: Bearer AbCDefGhijKlmnOpqrStuvWxYz01-23:45AbCDefGhijKlmnOpqrSt' + references: + - https://docs.developers.optimizely.com/web-experimentation/docs/rest-api-getting-started + - https://docs.developers.optimizely.com/feature-experimentation/reference/get_me + - https://docs.developers.optimizely.com/web-experimentation/docs/api-conventions + validation: + type: Http + content: + request: + method: GET + url: https://api.optimizely.com/v2/me + headers: + Authorization: "Bearer {{ TOKEN }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid diff --git a/data/rules/owlbot.yml b/data/rules/owlbot.yml new file mode 100644 index 0000000..1f1d4a0 --- /dev/null +++ b/data/rules/owlbot.yml @@ -0,0 +1,39 @@ +rules: + - name: Owlbot API Key + id: kingfisher.owlbot.1 + pattern: | + (?xi) + \b + owlbot + (?:.|[\n\r]){0,64}? + (?:api[_-]?key|secret|private|access|token|key) + (?:.|[\n\r]){0,64}? + \b + ( + [a-f0-9]{40} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - "owlbot SECRET b7d21c0e88e9a3c5938fb045b2b6a5e693eaf9d1" + - "owlbot TOKEN 8a5de3a89b7e4f29bf728b45adcdea6ea3410c78" + references: + - https://owlbot.info/ + validation: + type: Http + content: + request: + method: GET + url: "https://owlbot.info/api/v4/dictionary/owl?format=json" + headers: + Authorization: "Token {{ TOKEN }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"word"', '"definitions"'] diff --git a/data/rules/packagecloud.yml b/data/rules/packagecloud.yml new file mode 100644 index 0000000..6dd636f --- /dev/null +++ b/data/rules/packagecloud.yml @@ -0,0 +1,44 @@ +rules: + - name: PackageCloud API Key + id: kingfisher.packagecloud.1 + pattern: | + (?xi) + \b + packagecloud + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN|API[_-]?TOKEN|AUTH) + (?:.|[\n\r]){0,32}? + \b + ( + [0-9a-f]{48} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - packagecloud accessKEY 1234567890abcdef1234567890abcdef1234567890abcdef + - "packagecloud:token=1234567890abcdef1234567890abcdef1234567890abcdef" + - | + "config": { + "packagecloud_secret": "1234567890abcdef1234567890abcdef1234567890abcdef" + } + - packagecloudPRIVATEkey 1234567890abcdef1234567890abcdef1234567890abcdef + references: + - https://packagecloud.io/docs/api + validation: + type: Http + content: + request: + method: GET + url: "https://packagecloud.io/api/v1/distributions.json" + headers: + Authorization: "Basic {{ TOKEN | append: ':' | b64enc }}" + Accept: "application/json" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"deb"', '"rpm"'] diff --git a/data/rules/pagerdutyapikey.yml b/data/rules/pagerdutyapikey.yml index d65bced..3c65ca9 100644 --- a/data/rules/pagerdutyapikey.yml +++ b/data/rules/pagerdutyapikey.yml @@ -3,11 +3,7 @@ rules: id: kingfisher.pagerduty.1 pattern: | (?xi) - \b - (?: - Token | - Authorization | - pd[_-]? | + (?: pd[_-]? | pagerduty[_-]? | pagerduty @@ -19,29 +15,30 @@ rules: ( u\+[A-Z0-9_+-]{18} | # personal user token (20 chars) [A-Z0-9_-]{20} | # legacy PAT (20 chars, mixed case) - [a-f0-9]{32} # integration / routing key (32 hex, lower case) + [a-f0-9]{32} # integration / routing key (32 hex, lower case) ) - \b + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: - - "Authorization: Token token=u+Lyhd2_N2MCy+ZoH-S5" + - "pagerduty: Token token=u+Lyhd2_N2MCy+ZoH-S5" - pd_key = u+3xVszZ-b4m+T6d23KA - - Token token=ABCDEF1234567890ABCDEF1234567890 + - pagerduty token=ABCDEF1234567890ABCDEF1234567890 references: - - https://developer.pagerduty.com/api-reference/4555ca1c983d0-get-the-current-user + - https://developer.pagerduty.com/api-reference/4555ca1c983d0-get-the-current-user validation: type: Http content: request: method: GET - url: https://api.pagerduty.com/users + url: https://api.pagerduty.com/users headers: Authorization: Token token={{ TOKEN }} Accept: application/json response_matcher: - report_response: true - - type: JsonValid - - type: WordMatch - words: + - type: WordMatch + words: - '"users":' diff --git a/data/rules/particle.io.yml b/data/rules/particle.io.yml index c4fdfaf..eb8dd6b 100644 --- a/data/rules/particle.io.yml +++ b/data/rules/particle.io.yml @@ -5,9 +5,12 @@ rules: (?xi) https://api\.particle\.io/v1/[A-Z0-9_\-\s/"\\?]* (?:access_token=|Authorization:\s*Bearer\s*) + ( + [A-Z0-9]{40} + ) \b - ([A-Z0-9]{40}) - \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: @@ -42,7 +45,6 @@ rules: pattern: | (?xi) (?:access_token=|Authorization:\s*Bearer\s*) - \b ([A-Z0-9]{40}) \b [\s"\\]*https://api\.particle\.io/v1 diff --git a/data/rules/pastebin.yml b/data/rules/pastebin.yml index 1ae302e..f19536c 100644 --- a/data/rules/pastebin.yml +++ b/data/rules/pastebin.yml @@ -13,6 +13,8 @@ rules: [A-Z0-9_]{32} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/paypal.yml b/data/rules/paypal.yml index ef65d2d..47fddb1 100644 --- a/data/rules/paypal.yml +++ b/data/rules/paypal.yml @@ -1,56 +1,59 @@ rules: -- name: PayPal OAuth Client ID - id: kingfisher.paypal.1 - pattern: | - (?xi) - paypal - (?:.|[\n\r]){0,8}? - (?:CLIENT|ID|USER) - (?:.|[\n\r]){0,16}? - \b - ( - A[A-Z0-9_-]{78,99} - ) - \b - min_entropy: 3.5 - visible: false - examples: - - paypal_client_id=AZJ6y8Dpr1TYbqAIdhkPzyhjXoY6mIdhkPzyhjXoY6m8GplL7C3zZ3lPrkTIdhkPzyhjXo_Dx3IdhkPzyhjXoY6m + - name: PayPal OAuth Client ID + id: kingfisher.paypal.1 + pattern: | + (?xi) + paypal + (?:.|[\n\r]){0,8}? + (?:CLIENT|ID|USER) + (?:.|[\n\r]){0,16}? + \b + ( + A[A-Z0-9_-]{78,99} + ) + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + visible: false + examples: + - paypal_client_id=AZJ6y8Dpr1TYbqAIdhkPzyhjXoY6mIdhkPzyhjXoY6m8GplL7C3zZ3lPrkTIdhkPzyhjXo_Dx3IdhkPzyhjXoY6m -- name: PayPal OAuth Secret - id: kingfisher.paypal.2 - pattern: | - (?xi) - paypal - (?:.|[\n\r]){0,16}? - (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) - (?:.|[\n\r]){0,32}? - \b - ( - [A-Z0-9_.-]{78,120} - ) - \b - min_entropy: 3.5 - examples: - - paypal_secret=EP0uwUsACKVPcbDRaXFYerX2ij6nbsha71cSdynuQWoSt1pIy4qtIs7gJQRmHwKXu5Icv3g1YQZzAywf + - name: PayPal OAuth Secret + id: kingfisher.paypal.2 + pattern: | + (?xi) + paypal + (?:.|[\n\r]){0,16}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9_.-]{78,120} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + examples: + - paypal_secret=EP0uwUsACKVPcbDRaXFYerX2ij6nbsha71cSdynuQWoSt1pIy4qtIs7gJQRmHwKXu5Icv3g1YQZzAywf - validation: - type: Http - content: - request: - method: POST - url: https://api-m.paypal.com/v1/oauth2/token - headers: - Accept: application/json - Accept-Language: en_US - Content-Type: application/x-www-form-urlencoded - Authorization: | - Basic {{ CLIENTID | append: ':' | append: TOKEN | b64enc }} - body: grant_type=client_credentials - response_matcher: - - report_response: true - - type: StatusMatch - status: [200] - depends_on_rule: - - rule_id: kingfisher.paypal.1 - variable: CLIENTID + validation: + type: Http + content: + request: + method: POST + url: https://api-m.paypal.com/v1/oauth2/token + headers: + Accept: application/json + Accept-Language: en_US + Content-Type: application/x-www-form-urlencoded + Authorization: | + Basic {{ CLIENTID | append: ':' | append: TOKEN | b64enc }} + body: grant_type=client_credentials + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + depends_on_rule: + - rule_id: kingfisher.paypal.1 + variable: CLIENTID diff --git a/data/rules/paystack.yml b/data/rules/paystack.yml new file mode 100644 index 0000000..3841a8f --- /dev/null +++ b/data/rules/paystack.yml @@ -0,0 +1,40 @@ +rules: + - name: Paystack API Key + id: kingfisher.paystack.1 + pattern: | + (?xi) + \b + ( + sk_ + [a-z]{1,} + _ + [A-Z0-9]{40} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.3 + confidence: medium + examples: + - sk_test_abcdef1234567890abcdef1234567890abcdef12 + - sk_live_gwjaoi1234567890abcdef1234567890abcdef12 + references: + - https://paystack.com/docs/api/authentication/ + - https://paystack.com/docs/api/transfer-control/ + validation: + type: Http + content: + request: + method: GET + # Different endpoint than /customer: Check Balance + url: https://api.paystack.co/balance + headers: + Authorization: "Bearer {{ TOKEN }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + words: ['"message":"Balances retrieved"', '"data"'] diff --git a/data/rules/pdflayer.yml b/data/rules/pdflayer.yml new file mode 100644 index 0000000..3d3c830 --- /dev/null +++ b/data/rules/pdflayer.yml @@ -0,0 +1,47 @@ +rules: + - name: PdfLayer API Key + id: kingfisher.pdflayer.1 + pattern: | + (?xi) + (?: + \b + pdflayer + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN|ACCESS_KEY) + (?:.|[\n\r]){0,32}? + | + \bapi\.pdflayer\.com/api/convert\?[^ \t\r\n"'<>]*\baccess_key\s*=\s* + ) + \b + ( + [a-z0-9]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - pdflayer_key=1234567890abcdef1234567890abcdef + - PDFLAYER_ACCESS_TOKEN=abcdef1234567890abcdef1234567890 + - pdflayer_secret=0123456789abcdef0123456789abcdef + references: + - https://pdflayer.com/documentation + validation: + type: Http + content: + request: + method: GET + # Use Sandbox Mode (test=1) and intentionally omit document_url/document_html. + # This yields a JSON error response (instead of generating a PDF) and should not count + # toward monthly API volume per docs. + url: "https://api.pdflayer.com/api/convert?access_key={{ TOKEN }}&test=1" + headers: + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + negative: true + words: ['"invalid_access_key"', '"missing_access_key"'] diff --git a/data/rules/pem.yml b/data/rules/pem.yml index dc8e558..0c0d921 100644 --- a/data/rules/pem.yml +++ b/data/rules/pem.yml @@ -8,9 +8,10 @@ rules: ( (?: [a-zA-Z0-9+/=\s"',] | \\r | \\n ) {50,} ) \s* -----END\ .{0,20}\ ?PRIVATE\ KEY\ ?.{0,20}----- + pattern_requirements: + min_digits: 4 min_entropy: 4.5 confidence: high - prevalidated: true examples: - | -----BEGIN RSA PRIVATE KEY----- @@ -56,13 +57,15 @@ rules: (?: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0t (?# prefix of base64 encoding of `-----BEGIN RSA PRIVATE KEY-----` ) | LS0tLS1CRUdJTiBEU0EgUFJJVkFURSBLRVktLS0t (?# prefix of base64 encoding of `-----BEGIN DSA PRIVATE KEY-----` ) | LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0t (?# prefix of base64 encoding of `-----BEGIN EC PRIVATE KEY-----` ) + | LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0t (?# prefix of base64 encoding of `-----BEGIN PRIVATE KEY-----` ) ) [a-zA-Z0-9+/=]{50,} ) (?: [^a-zA-Z0-9+/=] | $ ) + pattern_requirements: + min_digits: 4 min_entropy: 4.5 confidence: high - prevalidated: true examples: - 'PRIVATE_KEY_B64=LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBb3kxWFh1VkFRcHFIYlFFMDVta2hyTmcvMTI0Ri8ySzlPYW5pelpUWlVVaEswOFU4CkxhaC9SbVVsWHFRMDEvU255aktGOWZqUDhFcU1OZ1dpamUzYmVwL3RPOVpTMEFUMi9PVlJXeS9TOG52RDQ5WTMKenMxMktSbERhR2lZc0RsYUZrbHJkeDQ4RWhRVmdHN3hmWE1jaC9OejJzc2FEby9kRkNBOW80TkZZQWUzM2UveApWNVo1UHNkWkl6dkNZQVlCNDRoUEtpN3JXRE1IbFdzM1kvVkVtQXMzSzVNK2QvL3QzRHB4WnBEbWJERGdYa2w2CjZUdDh3VXloUVZ3MkZpMStobTF1T2QwYjFkaW9aNko2OXNTT2JOZXpSR3YxYjdZaFltT0JKL1JBbHN5ZHoxTmgKVXpXT1lYV0Z1OGJrOU9JM3lQMEc0TE84QjhtbWRldE1RVVoyelFJREFRQUJBb0lCQUN2ckhUUHVVZ0JiSlE0QwpvQ0ZQdEgrWDZIN3NIdk1ndVR0VzdUTlYxN1BYMkVQdE53ZzI3S0tld0pNYmNSbWF3THBjSk5BU09xMDY4MGZxCjlsaHE1NEsybnB4WFVBeXErV3NSc1hid2hUODhibm5aQTBaRzZJR2hTaEpFN0t1cGxBU2htQ29FV2ppbmJTNFgKTGlvTW5HWSs4VFMzSzNrMTRWUDBaWUtuNXprMERHZnFBMEo0VTRXSmxUeGwrTWZxd0pJOTlrcTdHbFVlZkdncQpuK3Q1d2NrV3BPbTd5TUJjZTlTSXlmTm54bnU3TkZYQm50VTN5RGxSUThWUWZmNEtRMzJCaWNiYlJWemR1TThNCnNxMU5CZWNzL0EzUXRvdG1nWUc4d094ZXpNS3Iyays2QzB2NmlFc0h5T0lmR25GWktSZDJFd0dnWlo3aytURHUKUUYrcjd1VUNnWUVBMkRqNUJoYmpybDFRNTZya3BhTGFvVldRV1Y5YUYzUUJtNlNZM2VQYmlvY2JNR2k1ak1ESQpkSjdJVXlLYUljK3BNV1RQYlBmVUd2WmNENlczZDFBNUNUSnFuWHVuVlY3czRqaWJ6WDZUbjhNM3IrMHZTZnNZCmdPMHBtRFpndlNqaVZTRUNBQTZFOFUxQ1lFZU5KUDFDOW12cGJVNzJRTEpndWp3M3JMb2oyYmNDZ1lFQXdUSXYKOUNSeWNOQXRBbDcvUHdWZGh5eXRvVHBSRnZDSU1HSVk5SjMxZ3lva0ZlaFQvWjQ4WkF6anl6ZTBSUXYzdGUxTQoveVJMQkVETGkwbEtrZFVXckVkaVR3dm1KdkpwMDZ0OEdCbERsK25ycXVLWTFxVThDbTR5cis4QzZtRThkVnZrClNINXBhRXptOERFTE1wSjhGVTZFYnhmZHZjRzZmSGx6dnVnZmc1c0NnWUFFQ1BRa3QvS2h3MTRLSkxkRm5BZG0KY1ZsVFFhTkZ3c1Z3NlI1dExaNWdOR3MrZVFYVmFaZVVEWTZCZHFqWHJxOWltNVgvVzVTYXVEUTVtb2NVOCt0TQpqNk5Mc3c0SldzOGkzWm1TdVNUNkcwT0R4ZkpXK0JlWitGTUpZeUpsQlVsTCsyUzFLWkF6akpTTGhXcE40V2dKCmZ6UUk5U3RGUTg3b1NzMWpMTW9VZXdLQmdGOE9CMlFURHErTTdhaE4vejROc0wvU2JyZDJEdkcvZFBLQlFaQVIKcS90V0g1MGJ5ejlzdkgvcGk2YXdDS1UwUnpPZXh4UjkwZDhNMWxqNHZaVFZDQ3ZKajRnZTdhVlovbEdqL1JHSwpWS1NJOW1nRXgzaE1vaWJybzByR3lXTnlaaUhFRGFUUmRhRll2UU9PemRpYkZDd1RqcnR1UGE2Z2c5VzhtQU5sCkNDUmpBb0dBSTRIbnpyV3kzaU5kR2xqVnh4bW1DN1V0c0MvajJBUEZpcHc0ZHJ0U2NsMDFRZzF5WkowbDNBTk4KOU5lTmVSUUFzN3pFTng2T1B1SzlxYy83T1ROMTJKaHdoUTIzdXZwNjZjV0krdTRjcVpOZTJyZVFVVWVmM3psbQpMcXRmOU50VHp5M3pjMGZQcGoxQnBlRmxHSG9SVDhjVHpBWjFTeGwyZWChazlqS2RVeDQ9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0t' - ' "privateKey": "LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBbUhKOEJHdTFYZUZ4aENVQXBrNHNSTVI4RnRTdGtyMEx0OWtWTGNSUjRFWitiOWhHCmR0blJpOFhqV3d5MU5zMHliMkJMdHBpVHZKSFVKTUphWXluZ2ZkZnZhcWhocm1yYm5vV0pLQkxmeUxwTXFNS1EKQ3RialFxbnVrQURJUWVQd2ZGeTNpVHkxd1JkRC9zTUs1U0VtV0Fxb0pZQk50eTFZZzA2UzVkYVlPM2xjY3hrYQpQWjRjcm9McWF6Ny9tU3dDVTR5VWRSb3h4WVF4VG1MZXg5M2tqU09TTmdpK0FXc0lCbjV3UHI0VHNuVHFSeWpIClN2aEdMdk9YREpRYWZRdk56WjFSL1FYMzlOQk9xOEVKZW5pWXdaUm9uNVcvNVhMYW94MFFyUGhrY1BES3A5SVUKeHpJakUwWlNmMStUK1FFbTQ3TkFtSnhvZjFhdGRFVzZDTCtheHdJREFRQUJBb0lCQUQ3enI4REhsWnFSK1NWZgpmbGd1bWRzLzVCb3Rjd3ZRWXlGbFZIaVV4RmEvNVlCY0tDVDJKN0QzWTc1NmplNTJaK2hVTkkvUGk5cG53ZG40CkpBa2xCdDRRcUg0NzBES05UK216TFFOT1gvanM3YkVXdnhLcTBDZjhNbFptN0V0QlRGS2VtdS9pRVJBT2duYVcKcGs0ZUZVNXdBQ1dVU1FObWgxR1p4ZEdCZjFXM1VjUnQxcFRvOEtQTDluZm4vSGJiRFNsQkNVL3VIcWd2TSt2cApmTE03bzRIVDZ1K1ZzU00rWGZqeDhpeE5ZRHdoalNuKzQyZm13d1d3ZzJISHUrdUozZ1pUSWQwRUI1VW9hdUNjCjZUTlVtcEJscjU5UGFmVkZRWUY1S3VxaHJXKzVQaWpHcHBZcXg4Ynl6aFpOQzkwZnl5V0NXcXg2eGFZVm5OdzgKNkJmUXM2a0NnWUVBeVlyRVg1NU1RTzJnWDY2TGwxaGJDMzNzWk1OZzloVG1SK1doSTFjNksvbFZ1TFoyL0RPdwpsYTZ6eHdBU204Z0ZyVUFYbUljV2h2b3FwWGVzNWZzOVZKeDlNT0ZVYVBrckRPQllnY1laMUR6VVNVOHc3SSttCnlyV3hRUkRNajhvSGpRbHVpM0s2MzZucm5RajhxOGkvQ2dranVPcHJGZnliMzVEMFlDdjVXZzBDZ1lFQXdhT3cKRWFhN0l1MjFGa08vbmFjdVhjSnBhNkVlUTNqZFNlNlRQaXZ6bVVXU0haeGJuUy9XSnJaRjQwSExzUWxOZHl0ZgpNTTBKZFU0VmMyR0NVc1pMYjdQSmJwdVRqRERSSHJXV1pCMnhiemF0K3A3N2RzNWlOcXFRcTZ6M0syUVh4Y3ZTCis5am5VZXpDU2Y0N1R1OWNTTW96V3hTMW82b1BPSFdHVFRvdHR5TUNnWUFQdWc1Y3o4TnZoWnR3Ry9TMG1LWnkKSFI5bk5YL0pkQlFNSkRVUXh1dTVKcm16c2psU3NNM2t3RDh6RmlSZGw1d3B5c2lNbEc0RGxsM2hqNWNrVXhpVQpFNm9KT0d3WHpPbTVGWUNTajl6UUhQY0x5V3d0NlgvQWJiRXBQS0JaMEJBS3gyT2k2ZzcvQ1FsanRhSFIzZFphCmVDQWJlOTlqVmRUcit5bTJuM2ZUdVFLQmdBMm5TZ25rbEx0Z3dXMEJkK2hZMm1jWUJ6RGttbXF0Z2dUdGdvcFcKdFFWd3AxM1pJWWlTeituSTNtS295QUVDbytpc01Ua1NyQUVPY1dyQ1RGc2p5anZsRkdYdEtGa3hNLzJUVmpoVwo4NlRnMlNHYnhpVlpaZ2x1dTJhdmVub2Z3NkZadnRXdE5KcE5OR0hkUURkUG4xVXVsTEp1WW1SWTRGdmR4WXQ2CmQ3QzdBb0dBRUsvalFiZ0l3OXFLQUNOZ0JySnB1cU5Ham9JajFoQTRlb29DMXp1bFEyZUpnZ2J5OTBpSDg2VzEKM0xyOVZMVFkyc2JKTzlqekZVR0lOL01BOEhYQTE1a2grZHRibkRsdFRFZGNnenBCRzhCQUZRQ3hQWnBGWHhtZgpDUmhXN1l6RW1IeWJ4R0toR3NOK2M3NUhKTHZFSWwrRTh6eitXRk9xT240dkJXU1ZwSnc9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==",' \ No newline at end of file diff --git a/data/rules/perplexity.yml b/data/rules/perplexity.yml index 6629372..7942b3e 100644 --- a/data/rules/perplexity.yml +++ b/data/rules/perplexity.yml @@ -2,12 +2,14 @@ rules: - name: Perplexity AI API Key id: kingfisher.perplexity.1 pattern: | - (?xi) + (?x) \b ( pplx-[A-Za-z0-9]{48} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.8 confidence: medium examples: diff --git a/data/rules/phpmailer.yml b/data/rules/phpmailer.yml new file mode 100644 index 0000000..2811163 --- /dev/null +++ b/data/rules/phpmailer.yml @@ -0,0 +1,45 @@ +rules: + - name: PHPMailer Credentials + id: kingfisher.phpmailer.1 + + pattern: | + (?x) + \$mail->Host \s* = \s* '([^'\n]{5,})'; \s* (?: //.* )? + (?: \s* .* \s* ){0,3} + \$mail->Username \s* = \s* '([^'\n]{5,})'; \s* (?: //.* )? + (?: \s* .* \s* ){0,3} + \$mail->Password \s* = \s* '([^'\n]{5,})'; + confidence: medium + min_entropy: 3.0 + examples: + - | + //Server settings + $mail->SMTPDebug = SMTP::DEBUG_SERVER; //Enable verbose debug output + $mail->isSMTP(); //Send using SMTP + $mail->Host = 'smtp.example.com'; //Set the SMTP server to send through + $mail->SMTPAuth = true; //Enable SMTP authentication + $mail->Username = 'user@example.com'; //SMTP username + $mail->Password = 'secret'; //SMTP password + $mail->SMTPSecure = PHPMailer::ENCRYPTION_SMTPS; //Enable implicit TLS encryption + $mail->Port = 465; //TCP port to connect to; use 587 if you have set `SMTPSecure = PHPMailer::ENCRYPTION_STARTTLS` + + - | + require 'PHPMailerAutoload.php'; + + function SendMail($sub,$to,$msg) + { + $mail = new PHPMailer; + $mail->isSMTP(); // Set mailer to use SMTP + $mail->Host = 'smtp.gmail.com'; // Specify main and backup SMTP servers + $mail->SMTPAuth = true; // Enable SMTP authentication + $mail->SMTPSecure = 'tls'; // Enable encryption, 'ssl' also accepted + $mail->Username = 'ersatz.technologies@example.com'; // SMTP username + + + + $mail->Password = 'un!techwhooah'; // SMTP password + $mail->From = 'from@example.com'; + $mail->FromName = 'Admin'; + + references: + - https://github.com/PHPMailer/PHPMailer diff --git a/data/rules/plaid.yml b/data/rules/plaid.yml new file mode 100644 index 0000000..91167e7 --- /dev/null +++ b/data/rules/plaid.yml @@ -0,0 +1,264 @@ +rules: + - name: Plaid Client ID (helper) + id: kingfisher.plaid.1 + visible: false + pattern: | + (?xi) + \b + (?:plaid[\w-]{0,32})? + (?:client[_-]?id|plaid[_-]?client[_-]?id) + \b + (?:\s*[:=]\s*|["']\s*:\s*["']|=\s*["']) + \s* + \b + ( + [a-z0-9]{24} + ) + \b + min_entropy: 2.8 + confidence: medium + examples: + - 'plaid_client_id="sd479fjropblyr5b4m2dutha"' + references: + - https://plaid.com/docs/api/institutions/ + + # ------------------------- + - name: Plaid Secret (Production) + id: kingfisher.plaid.2 + pattern: | + (?xi) + \b + plaid + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN|ACCESS_KEY) + (?:.|[\n\r]){0,32}? + \b + ( + [a-z0-9]{30} + ) + \b + min_entropy: 3.2 + confidence: medium + pattern_requirements: + min_digits: 2 + min_lowercase: 6 + ignore_if_contains: + - changeme + - example + - test + examples: + - 'plaid_secret_key="wuxd6sw7ma4lv10xremyhz7ulf9owc"' + references: + - https://plaid.com/docs/api/institutions/ + - https://plaid.com/docs/quickstart/glossary/ + - https://plaid.com/docs/errors/invalid-input/ + depends_on_rule: + - rule_id: kingfisher.plaid.1 + variable: CLIENT_ID + validation: + type: Http + content: + request: + method: POST + url: https://production.plaid.com/institutions/get + headers: + Accept: application/json + Content-Type: application/json + body: | + { + "client_id": "{{ CLIENT_ID | json_escape }}", + "secret": "{{ TOKEN | json_escape }}", + "count": 1, + "offset": 0, + "country_codes": ["US"] + } + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + # If keys are invalid, Plaid returns INVALID_API_KEYS (HTTP 400). + - type: WordMatch + negative: true + words: + - '"INVALID_API_KEYS"' + + - name: Plaid Secret (Sandbox) + id: kingfisher.plaid.3 + pattern: | + (?xi) + \b + plaid + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN|ACCESS_KEY) + (?:.|[\n\r]){0,32}? + \b + ( + [a-z0-9]{30} + ) + \b + min_entropy: 3.2 + confidence: medium + pattern_requirements: + min_digits: 2 + min_lowercase: 6 + ignore_if_contains: + - changeme + - example + - test + examples: + - 'PLAID_SECRET="wuxd6sw7ma4lv10xremyhz7ulf9owc"' + references: + - https://plaid.com/docs/api/institutions/ + - https://plaid.com/docs/quickstart/glossary/ + - https://plaid.com/docs/errors/invalid-input/ + depends_on_rule: + - rule_id: kingfisher.plaid.1 + variable: CLIENT_ID + validation: + type: Http + content: + request: + method: POST + url: https://sandbox.plaid.com/institutions/get + headers: + Accept: application/json + Content-Type: application/json + body: | + { + "client_id": "{{ CLIENT_ID | json_escape }}", + "secret": "{{ TOKEN | json_escape }}", + "count": 1, + "offset": 0, + "country_codes": ["US"] + } + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + # If keys are invalid, Plaid returns INVALID_API_KEYS (HTTP 400). + - type: WordMatch + negative: true + words: + - '"INVALID_API_KEYS"' + + # ------------------------- + # Plaid access tokens (env-specific) + # Validate using /accounts/get (requires client_id + secret + access_token) + # ------------------------- + - name: Plaid Access Token (Production) + id: kingfisher.plaid.4 + pattern: | + (?xi) + \b + ( + access-production- + [0-9a-f]{8}- + [0-9a-f]{4}- + [0-9a-f]{4}- + [0-9a-f]{4}- + [0-9a-f]{12} + ) + \b + min_entropy: 3.4 + confidence: medium + examples: + - 'plaid_api_token="access-production-822f6ce0-ee1a-221e-3cd8-f3ce5094b3e2"' + references: + - https://plaid.com/docs/quickstart/glossary/ + - https://plaid.com/docs/api/accounts/ + - https://plaid.com/docs/errors/invalid-input/ + depends_on_rule: + - rule_id: kingfisher.plaid.1 + variable: CLIENT_ID + - rule_id: kingfisher.plaid.2 + variable: PLAID_SECRET + validation: + type: Http + content: + request: + method: POST + url: https://production.plaid.com/accounts/get + headers: + Accept: application/json + Content-Type: application/json + body: | + { + "client_id": "{{ CLIENT_ID | json_escape }}", + "secret": "{{ PLAID_SECRET | json_escape }}", + "access_token": "{{ TOKEN | json_escape }}" + } + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: + - '"accounts"' + - type: WordMatch + negative: true + match_all_words: false + words: + - '"INVALID_ACCESS_TOKEN"' + - '"INVALID_API_KEYS"' + + - name: Plaid Access Token (Sandbox) + id: kingfisher.plaid.5 + pattern: | + (?xi) + \b + ( + access-sandbox- + [0-9a-f]{8}- + [0-9a-f]{4}- + [0-9a-f]{4}- + [0-9a-f]{4}- + [0-9a-f]{12} + ) + \b + min_entropy: 3.4 + confidence: medium + examples: + - 'PLAID_ACCESS_TOKEN="access-sandbox-822f6ce0-ee1a-221e-3cd8-f3ce5094b3e2"' + references: + - https://plaid.com/docs/quickstart/glossary/ + - https://plaid.com/docs/api/accounts/ + - https://plaid.com/docs/errors/invalid-input/ + depends_on_rule: + - rule_id: kingfisher.plaid.1 + variable: CLIENT_ID + - rule_id: kingfisher.plaid.3 + variable: PLAID_SECRET + validation: + type: Http + content: + request: + method: POST + url: https://sandbox.plaid.com/accounts/get + headers: + Accept: application/json + Content-Type: application/json + body: | + { + "client_id": "{{ CLIENT_ID | json_escape }}", + "secret": "{{ PLAID_SECRET | json_escape }}", + "access_token": "{{ TOKEN | json_escape }}" + } + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: + - '"accounts"' + - type: WordMatch + negative: true + match_all_words: false + words: + - '"INVALID_ACCESS_TOKEN"' + - '"INVALID_API_KEYS"' \ No newline at end of file diff --git a/data/rules/planetscale.yml b/data/rules/planetscale.yml index ecbee4f..d37aa89 100644 --- a/data/rules/planetscale.yml +++ b/data/rules/planetscale.yml @@ -5,9 +5,11 @@ rules: (?xi) \b ( - pscale_tkn_[a-z0-9-_]{43} + pscale_tkn_[a-z0-9-_]{32,64} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 4 examples: - pscale_tkn_abcdefghijklmnopqrstuvwxyZ1234567890_ABCDEF @@ -41,11 +43,11 @@ rules: (?:.|[\n\r]){0,16}? (?:USER|ID|NAME) (?:.|[\n\r]){0,16}? - \b ( [a-z0-9]{12} ) - \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 visible: false examples: diff --git a/data/rules/postgres.yml b/data/rules/postgres.yml index 5cbb4ba..1102636 100644 --- a/data/rules/postgres.yml +++ b/data/rules/postgres.yml @@ -24,6 +24,11 @@ rules: (?: \d+ ) + pattern_requirements: + ignore_if_contains: + - "****" + - "xxxx" + - "example" min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/posthog.yml b/data/rules/posthog.yml new file mode 100644 index 0000000..961be77 --- /dev/null +++ b/data/rules/posthog.yml @@ -0,0 +1,55 @@ +rules: + - name: PostHog Project API Key + id: kingfisher.posthog.1 + pattern: | + (?x) + ( + phc_[a-zA-Z0-9_\-]{43} + ) + min_entropy: 3.0 + confidence: high + examples: + - "phc_E123456789012345678901234567890123456789012" + validation: + type: Http + content: + request: + method: POST + url: https://app.posthog.com/decide/?v=3 + headers: + Content-Type: "application/json" + body: | + {"token": "{{ TOKEN }}", "distinct_id": "validation_check"} + response_matcher: + - type: WordMatch + words: + - '"isAuthenticated":false' + negative: true + - type: StatusMatch + status: [200] + - name: PostHog Personal API Key + id: kingfisher.posthog.2 + pattern: | + (?x) + ( + phx_[a-zA-Z0-9_\-]{47} + ) + min_entropy: 3.0 + confidence: high + examples: + - "phx_FNKCx83Ko0JQMuZH1zz94xgK798TCUybkf79ZKYKwKQWbEw" + validation: + type: Http + content: + request: + method: GET + url: https://app.posthog.com/api/users/@me/ + headers: + Authorization: "Bearer {{ TOKEN }}" + Content-Type: "application/json" + response_matcher: + - report_response: true + - type: WordMatch + words: + - "authentication_failed" + negative: true \ No newline at end of file diff --git a/data/rules/postman.yml b/data/rules/postman.yml index ae7ae7c..8b98b2e 100644 --- a/data/rules/postman.yml +++ b/data/rules/postman.yml @@ -8,6 +8,8 @@ rules: PMAK-[A-Z0-9]{24}-[A-Z0-9]{34} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/postmark.yml b/data/rules/postmark.yml new file mode 100644 index 0000000..f48aa57 --- /dev/null +++ b/data/rules/postmark.yml @@ -0,0 +1,51 @@ +rules: + - name: Postmark API Token + id: kingfisher.postmark.1 + pattern: | + (?xi) + postmark [a-z0-9_-]{0,20} + .{0,10} + \b + ( + [a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.0 + examples: + - | + postmark: "f59dffd2-83ea-47c7-ba8f-95e053a6d0ae", + + - | + postmark: { + fromEmail: '...@....com', // must be verified with postmarkapp.com + postmarkApiToken: '00917922-dbe7-4882-bedc-10b93fc4c4c5', // our account token + }, + + - | + headers: { + Accept: "application/json", + "Content-Type": "application/json", + "X-Postmark-Server-Token": "c2321bd2-5a8f-46f2-b8fb-dbc706bd6923", + }, + references: + - https://postmarkapp.com/developer/api/overview + validation: + type: Http + content: + request: + method: GET + url: "https://api.postmarkapp.com/server" + headers: + X-Postmark-Server-Token: "{{TOKEN}}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"ID":' + - '"Name":' + match_all_words: true \ No newline at end of file diff --git a/data/rules/prefect.yml b/data/rules/prefect.yml new file mode 100644 index 0000000..beba175 --- /dev/null +++ b/data/rules/prefect.yml @@ -0,0 +1,33 @@ +rules: + - name: Prefect API Token + id: kingfisher.prefect.1 + pattern: | + (?xi) + \b + ( + pnu_[a-z0-9]{36} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + examples: + - PREFECT_API_TOKEN=pnu_1234567890abcdef1234567890abcdef1234 + - '"prefectToken": "pnu_abcdefabcdefabcdefabcdefabcdefabcdef"' + references: + - https://docs.prefect.io/latest/concepts/api_keys/ + validation: + type: Http + content: + request: + method: GET + url: https://api.prefect.cloud/api/me/workspaces + headers: + Authorization: 'Bearer {{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 diff --git a/data/rules/privkey.yml b/data/rules/privkey.yml index b0e25b5..a936514 100644 --- a/data/rules/privkey.yml +++ b/data/rules/privkey.yml @@ -22,9 +22,10 @@ rules: PRIVATE\sKEY (\sBLOCK)? ----- + pattern_requirements: + min_digits: 2 min_entropy: 4.5 confidence: high - prevalidated: true examples: - |- -----BEGIN RSA PRIVATE KEY----- @@ -45,8 +46,7 @@ rules: - name: Contains Private Key id: kingfisher.privkey.2 pattern: | - (?xi) - (?ims) + (?xims) ( -----BEGIN\s (?: @@ -68,16 +68,20 @@ rules: PGP | DSA | OPENSSH | - ENCRYPTED + ENCRYPTED | + EC )? \s{0,1} PRIVATE\sKEY (\sBLOCK)? ----- - ) + ) + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 4.5 confidence: high - prevalidated: true examples: - | -----BEGIN PRIVATE KEY----- diff --git a/data/rules/psexec.yml b/data/rules/psexec.yml index 57cfd66..3cca3d3 100644 --- a/data/rules/psexec.yml +++ b/data/rules/psexec.yml @@ -8,7 +8,6 @@ rules: -p \s* (\S+) (?# password ) min_entropy: 3.3 confidence: medium - categories: [fuzzy, secret] examples: - 'cmd.exe /C PSEXEC \\10.0.94.120 -u Administrator -p dev_admin CMD /C ECHO' - 'PSEXEC.EXE \\LocalComputerIPAddress -u DOMAIN\my-user -p mypass CMD' diff --git a/data/rules/pubnub.yml b/data/rules/pubnub.yml index f29310a..f759ed7 100644 --- a/data/rules/pubnub.yml +++ b/data/rules/pubnub.yml @@ -8,6 +8,8 @@ rules: pub-c-[a-z0-9]{8}(?:-[a-z0-9]{4}){3}-[a-z0-9]{12} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 examples: - pub-c-12345678-1234-1234-1234-123456789012 @@ -34,6 +36,8 @@ rules: sub-c-[a-z0-9]{8}(?:-[a-z0-9]{4}){3}-[a-z0-9]{12} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/pulumi.yml b/data/rules/pulumi.yml index 2298d1d..81980a1 100644 --- a/data/rules/pulumi.yml +++ b/data/rules/pulumi.yml @@ -8,6 +8,8 @@ rules: pul-[a-f0-9]{40} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 examples: - pul-18e13e3eebebeb94eac318d421ca8ecc5ca78d5f diff --git a/data/rules/pypi.yml b/data/rules/pypi.yml index 85884ed..1b441f9 100644 --- a/data/rules/pypi.yml +++ b/data/rules/pypi.yml @@ -3,11 +3,12 @@ rules: id: kingfisher.pypi.1 pattern: | (?xi) - \b ( pypi-AgEIcHlwaS5vcmc[A-Z0-9_-]{50,} ) - (?:[^A-Z0-9_-]|$) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/rabbitmq.yml b/data/rules/rabbitmq.yml index 953f10e..607d5f6 100644 --- a/data/rules/rabbitmq.yml +++ b/data/rules/rabbitmq.yml @@ -3,7 +3,6 @@ rules: id: kingfisher.rabbitmq.1 pattern: | (?xi) - \b (?: amqps? ) @@ -16,6 +15,8 @@ rules: @ [-.%\w\/:]+ \b + pattern_requirements: + min_special_chars: 1 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/rapidapi.yml b/data/rules/rapidapi.yml new file mode 100644 index 0000000..b80d1e7 --- /dev/null +++ b/data/rules/rapidapi.yml @@ -0,0 +1,41 @@ +rules: + - name: RapidAPI Key + id: kingfisher.rapidapi.1 + pattern: | + (?xi) + \b + rapidapi + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Za-z0-9_-]{50} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 4.0 + confidence: medium + examples: + - rapidapi_key=abcdefghij1234567890ABCDEFGHIJ1234567890abcdefghij + - '"rapidapiKey":"ABCDEFGHIJ1234567890abcdefghij1234567890ABCDEFGHIJ"' + references: + - https://docs.rapidapi.com/docs/configuring-api-security + - https://docs.rapidapi.com/docs/keys-and-key-rotation + validation: + type: Http + content: + request: + method: GET + url: "https://weatherapi-com.p.rapidapi.com/current.json?q=London" + headers: + x-rapidapi-key: "{{ TOKEN }}" + x-rapidapi-host: "weatherapi-com.p.rapidapi.com" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"country"'] diff --git a/data/rules/react.yml b/data/rules/react.yml index c7d5a40..de22892 100644 --- a/data/rules/react.yml +++ b/data/rules/react.yml @@ -18,9 +18,6 @@ rules: - 'REACT_APP_AUTH_USERNAME=bowie' - ' REACT_APP_AUTH_USERNAME=bowie # some comment' - 'REACT_APP_MAILER_USERNAME=smtp_username # Enter your SMTP email username' - negative_examples: - - 'REACT_APP_FRONTEND_LOGIN_FORGOT_USERNAME=$REACT_APP_MATRIX_BASE_URL/classroom/#/forgot_username' - categories: [fuzzy, identifier] references: - https://create-react-app.dev/docs/adding-custom-environment-variables/ - https://stackoverflow.com/questions/48699820/how-do-i-hide-an-api-key-in-create-react-app @@ -44,10 +41,6 @@ rules: - 'REACT_APP_AUTH_PASSWORD=whiteduke' - ' REACT_APP_AUTH_PASSWORD=whiteduke # some comment' - 'REACT_APP_MAILER_PASSWORD=smtp_password # Enter your SMTP email password' - negative_examples: - - ' const password = process.env.REACT_APP_FIREBASE_DEV_PASSWORD || "not-set"' - - 'REACT_APP_FRONTEND_LOGIN_FORGOT_PASSWORD=$REACT_APP_MATRIX_BASE_URL/classroom/#/forgot_password' - categories: [fuzzy, secret] references: - https://create-react-app.dev/docs/adding-custom-environment-variables/ - https://stackoverflow.com/questions/48699820/how-do-i-hide-an-api-key-in-create-react-app \ No newline at end of file diff --git a/data/rules/readme.yml b/data/rules/readme.yml index 0e415ec..6a4b95d 100644 --- a/data/rules/readme.yml +++ b/data/rules/readme.yml @@ -2,11 +2,14 @@ rules: - name: ReadMe API Key id: kingfisher.readme.1 pattern: | - (?x)(?i) + (?xi) \b ( rdme_(?P[a-z0-9]{70}) ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/recaptcha.yml b/data/rules/recaptcha.yml index 4966857..0c40e91 100644 --- a/data/rules/recaptcha.yml +++ b/data/rules/recaptcha.yml @@ -5,12 +5,12 @@ rules: (?xi) recaptcha (?:.|[\n\r]){0,16}? - (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) - (?:.|[\n\r]){0,32}? \b ( 6l[c-f][a-z0-9_-].{36} ) + pattern_requirements: + min_digits: 3 min_entropy: 3 confidence: medium examples: diff --git a/data/rules/replicate.yml b/data/rules/replicate.yml new file mode 100644 index 0000000..6eb6ffd --- /dev/null +++ b/data/rules/replicate.yml @@ -0,0 +1,41 @@ +rules: + - name: Replicate API Token + id: kingfisher.replicate.1 + pattern: | + (?x) + \b + ( + r8_ + [A-Za-z0-9]{37} + ) + \b + pattern_requirements: + min_digits: 3 + min_entropy: 3.0 + confidence: medium + examples: + - r8_WesXNvqsCpq7r1gpQABpB3NJvdR21nb2s7HVy + - r8_Lvn3Tsrs8H2wCYSEPDiUfyePqWpBOWi0vQTtN + - r8_XOpqpi4q9UADwsgrbEjCpT9p1cDldUu3t1D8R + - r8_ap8Mo5iTbW01FHJtElPrBUqf7fjz1r40EVrJu + references: + - https://replicate.com/docs/reference/http + validation: + type: Http + content: + request: + method: GET + url: https://api.replicate.com/v1/account + headers: + Authorization: Bearer {{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: + - '"type"' + - '"username"' + - '"name"' diff --git a/data/rules/resend.yml b/data/rules/resend.yml new file mode 100644 index 0000000..a889842 --- /dev/null +++ b/data/rules/resend.yml @@ -0,0 +1,43 @@ +rules: + - name: Resend API Key + id: kingfisher.resend.api_key.1 + pattern: | + (?x) + \b + ( + re_ + [A-Za-z0-9]{8} + _ + [A-Za-z0-9]{24} + ) + \b + min_entropy: 3.2 + confidence: high + categories: + - api + - secret + examples: + - 'RESEND_API_KEY="re_EbtXGAbq_2E1LZ8WYqYsrrDfjEHf6DxEK"' + - "Authorization: Bearer re_jZmz3GSH_MqwC1vjBjZpQH88W4dLsTPpu" + references: + - https://resend.com/docs/api-reference/introduction + - https://resend.com/docs/api-reference/domains/list-domains + - https://resend.com/docs/api-reference/errors + - https://resend.com/docs/knowledge-base/how-to-handle-api-keys + validation: + type: Http + content: + request: + method: GET + url: https://api.resend.com/domains + headers: + Authorization: "Bearer {{ TOKEN }}" + Accept: "application/json" + response_matcher: + - report_response: true + # Valid keys: + # - 200 => full_access works for /domains + # - 401 => restricted_api_key (sending-only), still a real key + - type: StatusMatch + status: [200, 401] + - type: JsonValid diff --git a/data/rules/retellai.yml b/data/rules/retellai.yml new file mode 100644 index 0000000..8417cbb --- /dev/null +++ b/data/rules/retellai.yml @@ -0,0 +1,42 @@ +rules: + - name: Retell AI API Key + id: kingfisher.retellai.api_key.1 + pattern: | + (?xi) + \b + ( + key_[0-9a-f]{28} + ) + \b + min_entropy: 3.0 + confidence: medium + examples: + - key_8c08a175a1bf29f79d6bd4b125bc + - key_08f00bd7c1fec315592048630f87 + - key_b15e562e497dce7ea90d1ba6a016 + categories: + - api + - secret + references: + - https://docs.retellai.com/accounts/api-keys-overview + - https://docs.retellai.com/api-references/get-concurrency + + validation: + type: Http + content: + request: + method: GET + url: https://api.retellai.com/get-concurrency + headers: + Accept: application/json + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + words: + - '"current_concurrency"' + - '"concurrency_limit"' + match_all_words: true diff --git a/data/rules/riot.yml b/data/rules/riot.yml new file mode 100644 index 0000000..7ecec86 --- /dev/null +++ b/data/rules/riot.yml @@ -0,0 +1,55 @@ +rules: + - name: Riot Platform Host + id: kingfisher.riot.1 + visible: false + confidence: medium + min_entropy: 1.0 + pattern: | + (?xi) + \b + ( + (?:br1|eun1|euw1|jp1|kr|la1|la2|na1|oc1|ru|tr1 + |ph2|sg2|th2|tw2|vn2 + |americas|europe|asia) + \.api\.riotgames\.com + ) + \b + examples: + - na1.api.riotgames.com + - euw1.api.riotgames.com + - americas.api.riotgames.com + + - name: Riot Games API Key + id: kingfisher.riot.2 + pattern: | + (?xi) + \b + ( + RGAPI-[a-z0-9_-]{36} + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.0 + confidence: medium + examples: + - RGAPI-4sb3f6a1-2941-5a81-9c23-4bf3a83c14f3 + references: + - https://developer.riotgames.com/docs/lol + - https://developer.riotgames.com/apis + depends_on_rule: + - rule_id: kingfisher.riot.1 + variable: RIOT_PLATFORM_HOST + validation: + type: Http + content: + request: + method: GET + url: "https://{{ RIOT_PLATFORM_HOST }}/lol/status/v4/platform-data" + headers: + X-Riot-Token: "{{ TOKEN }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] diff --git a/data/rules/rubygems.yml b/data/rules/rubygems.yml index 93ace54..bb9487c 100644 --- a/data/rules/rubygems.yml +++ b/data/rules/rubygems.yml @@ -8,8 +8,10 @@ rules: ( rubygems_ [a-z0-9]{42,52} - ) - \b + ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium categories: [api, secret] diff --git a/data/rules/runway.yml b/data/rules/runway.yml new file mode 100644 index 0000000..8e899f7 --- /dev/null +++ b/data/rules/runway.yml @@ -0,0 +1,42 @@ +rules: + - name: Runway API Key + id: kingfisher.runway.1 + pattern: | + (?x) + \b + ( + key_ + [A-Fa-f0-9]{128} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 4.0 + confidence: medium + examples: + - key_efef51067da4873eeefa5e3daeeff61537885e52e20e053a824bb5e564d3d707367d005e7d48dbe473de287383356a5abd77421703e1a3e52a27d17f703abe20 + - key_7ab2eab6623761354e72f7de76ea041ee3fd73db8e20b5e1173f1e7537ae5ad3e503267e4da374e650715e457e558724bf7bcb170e723bb3fff6445aa2830784 + - key_5a37de4ea80f355afa4aa653d67e8b6db08aaefd0704a773f1584e40236e4d54bf7f78974e8ed07f72e6a6787e66872127a577046743e3e4117ed7b14adeeeb8 + - key_10eb4a74b40672ddee1716ed008637a7aed5176b70eee76017b4e8e5713b8ab12720a8e4768dfe3e47073f86f718286eee814ffea20e271dd5d87ee8d367f8aa + references: + - https://docs.dev.runwayml.com/guides/setup + - https://api.useapi.net/v1/runwayml/accounts + validation: + type: Http + content: + request: + method: GET + url: https://api.dev.runwayml.com/v1/organization + headers: + Authorization: "Bearer {{ TOKEN }}" + Accept: application/json + X-Runway-Version: "2024-11-06" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: + - '"usage"' diff --git a/data/rules/salesforce.yml b/data/rules/salesforce.yml index d532346..282a936 100644 --- a/data/rules/salesforce.yml +++ b/data/rules/salesforce.yml @@ -2,14 +2,16 @@ rules: - name: Salesforce Access / Refresh Token id: kingfisher.salesforce.1 pattern: | - (?xi) + (?xi) \b - ( + ( 00 [A-Z0-9]{13} ! [A-Z0-9._-]{90,120} ) + pattern_requirements: + min_digits: 6 min_entropy: 3.3 confidence: medium examples: @@ -54,10 +56,192 @@ rules: ) \. my\.salesforce\.com - \b + \b min_entropy: 2.5 confidence: medium visible: false examples: - https://example123.my.salesforce.com - - mydomainname.my.salesforce.com \ No newline at end of file + - mydomainname.my.salesforce.com + - name: Salesforce Consumer Key + id: kingfisher.salesforce.3 + pattern: | + (?x)(?s) + \bconsumerKey\b + (?:.|[\n\r]){0,32}? + \b + ( + [A-Za-z0-9+/=._-]{16,256} + ) + \b + min_entropy: 3.3 + pattern_requirements: + min_digits: 3 + confidence: medium + examples: + - | + + + https://login.example.com/oauth/login/v2/authorize?authHint=SALESFORCE_OAUTH2&authType=oauth2&prompt=login + 012cbddfa6b05ec1941143c0d37a036291492be9f2df0b42c5c0c220198185de + 7TVG9nQ8gW5RaRxV8i1SaI7vwa0xtQQoejTa48AR5QR6HBYV9YBKPnAzPU7bs6QxOgdjJy9TPabQYVTZtgT83 + ExampleProviderOne + false + OpenIdConnect + true + false + true + https://login.example.com/oauth/login/v2/token + + - | + + + https://api.example.net/oauth/authorize + 012cbddfa6b05ec1941143c0d37a036291492be9f2df0b42c5c0c220198185de + 7TVG9nQ8gW5RaRxV8i1SaI7vwa0xtQQoejTa48AR5QR6HBYV9YBKPnAzPU7bs6QxOgdjJy9TPabQYVTZtgT83 + ExampleBatchConnect + false + OpenIdConnect + true + false + true + https://api.example.net/oauth/token + + - | + + + https://api.example.net/oauth/authorize + 012cbddfa6b05ec1941143c0d37a036291492be9f2df0b42c5c0c220198185de + 7TVG9nQ8gW5RaRxV8i1SaI7vwa0xtQQoejTa48AR5QR6HBYV9YBKPnAzPU7bs6QxOgdjJy9TPabQYVTZtgT83 + ExampleConnect + false + OpenIdConnect + true + false + true + https://api.example.net/oauth/token + + - name: Salesforce Consumer Secret + id: kingfisher.salesforce.4 + pattern: | + (?xi)(?s) + consumerSecret\b + (?:.|[\n\r]){0,32}? + \b + ( + [A-Za-z0-9+/=._-]{16,256} + ) + min_entropy: 3.3 + pattern_requirements: + min_digits: 6 + confidence: medium + examples: + - | + + + https://login.example.com/oauth/login/v2/authorize?authHint=SALESFORCE_OAUTH2&authType=oauth2&prompt=login + 012cbddfa6b05ec1941143c0d37a036291492be9f2df0b42c5c0c220198185de + 7TVG9nQ8gW5RaRxV8i1SaI7vwa0xtQQoejTa48AR5QR6HBYV9YBKPnAzPU7bs6QxOgdjJy9TPabQYVTZtgT83 + ExampleProviderOne + false + OpenIdConnect + true + false + true + https://login.example.com/oauth/login/v2/token + + - | + + + https://api.example.net/oauth/authorize + 012cbddfa6b05ec1941143c0d37a036291492be9f2df0b42c5c0c220198185de + 7TVG9nQ8gW5RaRxV8i1SaI7vwa0xtQQoejTa48AR5QR6HBYV9YBKPnAzPU7bs6QxOgdjJy9TPabQYVTZtgT83 + ExampleBatchConnect + false + OpenIdConnect + true + false + true + https://api.example.net/oauth/token + + - | + + + https://api.example.net/oauth/authorize + 012cbddfa6b05ec1941143c0d37a036291492be9f2df0b42c5c0c220198185de + 7TVG9nQ8gW5RaRxV8i1SaI7vwa0xtQQoejTa48AR5QR6HBYV9YBKPnAzPU7bs6QxOgdjJy9TPabQYVTZtgT83 + ExampleConnect + false + OpenIdConnect + true + false + true + https://api.example.net/oauth/token + + - name: Salesforce Consumer Key and Secret + id: kingfisher.salesforce.5 + pattern: | + (?xi)(?s) + (?:salesforce|sforce) + (?:.|[\n\r]){0,256}? + \bconsumerKey\b + (?:.|[\n\r]){0,32}? + \b + (?P + [A-Z0-9+/=._-]{16,256} + ) + \b.*? + (?:.|[\n\r]){0,256}? + \bconsumer\s{0,8}secret\b + (?:.|[\n\r]){0,32}? + \b + (?P + [A-Za-z0-9+/=._-]{16,256} + ) + min_entropy: 3.5 + pattern_requirements: + min_digits: 3 + confidence: medium + examples: + - | + + + https://login.example.com/oauth/login/v2/authorize?authHint=SALESFORCE_OAUTH2&authType=oauth2&prompt=login + 012cbddfa6b05ec1941143c0d37a036291492be9f2df0b42c5c0c220198185de + 7TVG9nQ8gW5RaRxV8i1SaI7vwa0xtQQoejTa48AR5QR6HBYV9YBKPnAzPU7bs6QxOgdjJy9TPabQYVTZtgT83 + ExampleProviderOne + false + OpenIdConnect + true + false + true + https://login.example.com/oauth/login/v2/token + + - | + + + https://api.example.net/oauth/authorize + 012cbddfa6b05ec1941143c0d37a036291492be9f2df0b42c5c0c220198185de + 7TVG9nQ8gW5RaRxV8i1SaI7vwa0xtQQoejTa48AR5QR6HBYV9YBKPnAzPU7bs6QxOgdjJy9TPabQYVTZtgT83 + ExampleBatchConnect + false + OpenIdConnect + true + false + true + https://api.example.net/oauth/token + + - | + + + https://api.example.net/oauth/authorize + 012cbddfa6b05ec1941143c0d37a036291492be9f2df0b42c5c0c220198185de + 7TVG9nQ8gW5RaRxV8i1SaI7vwa0xtQQoejTa48AR5QR6HBYV9YBKPnAzPU7bs6QxOgdjJy9TPabQYVTZtgT83 + ExampleConnect + false + OpenIdConnect + true + false + true + https://api.example.net/oauth/token + \ No newline at end of file diff --git a/data/rules/sauce.yml b/data/rules/sauce.yml new file mode 100644 index 0000000..009f33a --- /dev/null +++ b/data/rules/sauce.yml @@ -0,0 +1,82 @@ +rules: + - name: Sauce Labs Username + id: kingfisher.saucelabs.1 + pattern: | + (?xi) + \b + sauce + (?:.|[\n\r]){0,16}? + (?:USER|ID|NAME|CLIENT|OAUTH) + (?:.|[\n\r]){0,16}? + \b + ( + [A-Z0-9_.-]{2,70} + ) + \b + confidence: medium + visible: false + min_entropy: 1.0 + examples: + - "SAUCE_USERNAME=oauth-someusername-487ea" + - SAUCE_USERNAME="oauth-ci-bot-487ea" + - '"sauce_username":"build-user"' + - 'saucelabs user oauth-release-bot' + - name: Sauce Labs API Endpoint + id: kingfisher.saucelabs.2 + pattern: | + (?xi) + ( + (?:api|ondemand)\.(?:us|eu)-(?:west|east|central)-[0-9]\.saucelabs\.com + ) + \b + confidence: medium + visible: false + min_entropy: 2.0 + examples: + - "api.us-west-1.saucelabs.com" + - "api.eu-central-1.saucelabs.com" + - "ondemand.eu-central-1.saucelabs.com" + + - name: Sauce Labs Access Key + id: kingfisher.saucelabs.3 + pattern: | + (?xi) + \b + sauce + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + ( + [a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12} + ) + \b + pattern_requirements: + min_digits: 4 + confidence: medium + min_entropy: 3.0 + depends_on_rule: + - rule_id: "kingfisher.saucelabs.1" + variable: SAUCE_USERNAME + - rule_id: "kingfisher.saucelabs.2" + variable: SAUCE_URL + validation: + type: Http + content: + request: + method: GET + url: "https://{{ SAUCE_URL | default: 'api.us-west-1.saucelabs.com' | replace: 'ondemand.', 'api.' }}/rest/v1/users/{{SAUCE_USERNAME}}" + headers: + Authorization: "Basic {{ SAUCE_USERNAME | append: ':' | append: TOKEN | b64enc }}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"username":' + references: + - https://docs.saucelabs.com/dev/api/ + - https://docs.saucelabs.com/dev/api/#authentication + examples: + - "SAUCE_ACCESS_KEY=1736468d-b178-39cd-bfde-30fabdc371e4" + diff --git a/data/rules/scale.yml b/data/rules/scale.yml new file mode 100644 index 0000000..81148f2 --- /dev/null +++ b/data/rules/scale.yml @@ -0,0 +1,64 @@ +rules: + - name: Scale API Key + id: kingfisher.scale.1 + pattern: | + (?x) + \b + ( + live_ + [0-9a-f]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.1 + confidence: medium + examples: + - live_8df31399ec4a4755a7cf9e0fb59f967a + - live_54d1bd2d1e62430bb2d521d298ec4231 + - live_1b9fc721a4624a478211ce613c674a03 + references: + - https://scale.com/docs/api-reference/authentication + - https://scale.com/docs/api-reference/studio#list-all-teammates + - https://scale.com/docs/api-reference/authentication#test-and-live-modes + + validation: + type: Http + content: + request: + method: GET + url: https://api.scale.com/v1/teams + headers: + Authorization: 'Basic {{ TOKEN | append: ":" | b64enc }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: + - '"email"' + - '"role"' + + - name: Scale Callback Auth Key + id: kingfisher.scale.2 + pattern: | + (?x) + \b + ( + live_auth_ + [0-9a-f]{32} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.1 + confidence: medium + examples: + - live_auth_250ae896ada542c08a95734f935c871a + references: + - https://scale.com/docs/api-reference/authentication#callback-authentication + # Callback keys are *only* echoed by Scale in webhook headers and + # can’t be validated via an API call, so no `validation:` block. diff --git a/data/rules/scalingo.yml b/data/rules/scalingo.yml new file mode 100644 index 0000000..c297526 --- /dev/null +++ b/data/rules/scalingo.yml @@ -0,0 +1,33 @@ +rules: + - name: Scalingo API Token + id: kingfisher.scalingo.1 + pattern: | + (?xi) + \b + ( + tk-us-[\w-]{48} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + examples: + - SCALINGO_TOKEN=tk-us-abcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdef + - '"scalingo": "tk-us-1234567890abcdef1234567890abcdef1234567890abcdef"' + references: + - https://developers.scalingo.com/apps/api/authentication + validation: + type: Http + content: + request: + method: GET + url: https://api.scalingo.com/v1/users/self + headers: + Authorization: 'Bearer {{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 diff --git a/data/rules/scraperapi.yml b/data/rules/scraperapi.yml new file mode 100644 index 0000000..2bca1ac --- /dev/null +++ b/data/rules/scraperapi.yml @@ -0,0 +1,35 @@ +rules: + - name: ScraperAPI Key + id: kingfisher.scraperapi.1 + pattern: | + (?xi) + \b(?:scraper(?:\s|[_-])?api|scraperapi) + (?:.|[\n\r]){0,32}? + (?:key|token|api[_-]?key) + (?:.|[\n\r]){0,16}? + ([a-z0-9]{32}) + \b + pattern_requirements: + min_digits: 2 + min_lowercase: 10 + min_entropy: 3.5 + confidence: medium + examples: + - 'SCRAPERAPI_KEY=a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6' + - 'scraper_api_token: "9f8e7d6c5b4a3029182736455463728a"' + references: + - https://www.scraperapi.com/documentation/ + validation: + type: Http + content: + request: + method: GET + url: "https://api.scraperapi.com?api_key={{ TOKEN }}&url=http://httpbin.org/ip" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + words: + - '"origin"' diff --git a/data/rules/segment.yml b/data/rules/segment.yml index 1a2518f..234955d 100644 --- a/data/rules/segment.yml +++ b/data/rules/segment.yml @@ -8,6 +8,8 @@ rules: sgp_[A-Z0-9_-]{60,70} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: @@ -33,7 +35,6 @@ rules: id: kingfisher.segment.2 pattern: | (?xi) - \b (?:segment|sgmt) (?:.|[\n\r]){0,16}? (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) @@ -42,7 +43,8 @@ rules: ( [A-Z0-9_-]{40,50}\.[A-Z0-9_-]{40,50} ) - \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/sendbird.yml b/data/rules/sendbird.yml new file mode 100644 index 0000000..d82ec94 --- /dev/null +++ b/data/rules/sendbird.yml @@ -0,0 +1,61 @@ +rules: + - name: Sendbird Application ID + id: kingfisher.sendbird.1 + pattern: | + (?xi) + sendbird + (?:.|[\\n\r]){0,32}? + (?:APPLICATION|APP_ID|APP|CLIENT|ID) + (?:.|[\n\r]){0,32}? + \b + ( + [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + visible: false + min_entropy: 3.0 + examples: + - "sendbird_app_id: 12345678-1234-1234-1234-1234567890ab" + + - name: Sendbird API Token + id: kingfisher.sendbird.2 + pattern: | + (?xi) + sendbird + (?:.|[\\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [a-f0-9]{40} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 4.0 + examples: + - "sendbird_api_token: 1234567890abcdef1234567890abcdef12345678" + validation: + type: Http + content: + request: + method: GET + url: "https://api-{{SENDBIRD_APP_ID}}.sendbird.com/v3/users" + headers: + "Api-Token": "{{TOKEN}}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"users":' + depends_on_rule: + - rule_id: "kingfisher.sendbird.1" + variable: SENDBIRD_APP_ID + references: + - https://sendbird.com/docs/chat/platform-api/v3/prepare-to-use-api#2-authentication \ No newline at end of file diff --git a/data/rules/sendgrid.yml b/data/rules/sendgrid.yml index 89145fc..122c222 100644 --- a/data/rules/sendgrid.yml +++ b/data/rules/sendgrid.yml @@ -12,12 +12,14 @@ rules: [0-9A-Z_-]{39,47} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: - " 'SENDGRID_API_KEYSID': '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'" - "var sendgrid = require('sendgrid')('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');" - - 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 + - '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' references: - https://docs.sendgrid.com/ui/account-and-settings/api-keys validation: diff --git a/data/rules/sendinblue.yml b/data/rules/sendinblue.yml new file mode 100644 index 0000000..f57a613 --- /dev/null +++ b/data/rules/sendinblue.yml @@ -0,0 +1,33 @@ +rules: + - name: Sendinblue API Token + id: kingfisher.sendinblue.1 + pattern: | + (?xi) + \b + ( + xkeysib-[a-f0-9]{64}-[a-z0-9]{16} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.2 + confidence: medium + examples: + - XKEYSIB_TOKEN=xkeysib-abcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdefabcd-1234567890abcd12 + - '"sendinblue": "xkeysib-1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef-ab12cd34ef56gh78"' + references: + - https://developers.sendinblue.com/docs/authentication + validation: + type: Http + content: + request: + method: GET + url: https://api.sendinblue.com/v3/account + headers: + api-key: '{{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 diff --git a/data/rules/sentry.yml b/data/rules/sentry.yml new file mode 100644 index 0000000..d0f4685 --- /dev/null +++ b/data/rules/sentry.yml @@ -0,0 +1,115 @@ +rules: + - name: Sentry Access Token + id: kingfisher.sentry.1 + pattern: | + (?xi) + \b + sentry + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [a-f0-9]{64} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + examples: + - SENTRY_TOKEN=cbadefcbadefcbadefcbadefcbadefcbadefcbadefcbadefcbadefcbadefcbad + - '"sentry-key": "3214567890cbadef3214567890cbadef3214567890cbadef3214567890cbadef"' + references: + - https://docs.sentry.io/api/auth/ + validation: + type: Http + content: + request: + method: GET + url: https://sentry.io/api/0/projects/ + headers: + Authorization: 'Bearer {{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 + + - name: Sentry Organization Token + id: kingfisher.sentry.2 + pattern: | + (?x) + \b + ( + sntrys_eyJpYXQiO + [a-zA-Z0-9+/]{10,192} + (?: + LCJyZWdpb25fdXJs + | InJlZ2lvbl91cmwi + | cmVnaW9uX3VybCI6 + ) + [a-zA-Z0-9+/]{10,192} + ={0,2} + _ + [a-zA-Z0-9+/]{43} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 4.5 + confidence: medium + examples: + - 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 + - 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 + references: + - https://docs.sentry.io/api/auth/ + - https://github.com/getsentry/rfcs/blob/main/text/0091-ci-upload-tokens.md + validation: + type: Http + content: + request: + method: GET + url: https://sentry.io/api/0/projects/ + headers: + Authorization: 'Bearer {{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 + + - name: Sentry User Token + id: kingfisher.sentry.3 + pattern: | + (?xi) + \b + ( + sntryu_[a-f0-9]{64} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - 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 + - SNTRY_USER="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" + references: + - https://docs.sentry.io/api/auth/ + validation: + type: Http + content: + request: + method: GET + url: https://sentry.io/api/0/projects/ + headers: + Authorization: 'Bearer {{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 diff --git a/data/rules/shippo.yml b/data/rules/shippo.yml new file mode 100644 index 0000000..e1bae07 --- /dev/null +++ b/data/rules/shippo.yml @@ -0,0 +1,33 @@ +rules: + - name: Shippo API Token + id: kingfisher.shippo.1 + pattern: | + (?xi) + \b + ( + shippo_(?:live|test)_[a-f0-9]{40} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.3 + confidence: medium + examples: + - SHIPPO_TOKEN=shippo_test_1234567890abcdef1234567890abcdef12345678 + - 'Authorization: "ShippoToken shippo_live_abcdefabcdefabcdefabcdefabcdefabcdefabcd"' + references: + - https://goshippo.com/docs/reference + validation: + type: Http + content: + request: + method: GET + url: https://api.goshippo.com/shipments/ + headers: + Authorization: 'ShippoToken {{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 diff --git a/data/rules/shodan.yml b/data/rules/shodan.yml new file mode 100644 index 0000000..76f946c --- /dev/null +++ b/data/rules/shodan.yml @@ -0,0 +1,36 @@ +rules: + - name: SHODAN API Key + id: kingfisher.shodan.1 + pattern: | + (?xi) + \b + shodan + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9]{32} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 4.0 + validation: + type: Http + content: + request: + method: GET + url: "https://api.shodan.io/api-info?key={{TOKEN}}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"scan_credits"' + references: + - https://developer.shodan.io/api + examples: + - "shodan_api_key = dqlblS2CmTOc5zYn4nZkJljYsXRnNuiq" diff --git a/data/rules/shopify.yml b/data/rules/shopify.yml index da340e5..7f70dd4 100644 --- a/data/rules/shopify.yml +++ b/data/rules/shopify.yml @@ -5,9 +5,11 @@ rules: (?xi) \b ( - (?:shpat|shpca|shppa|shpss)_[a-f0-9]{30,34} + (?:shpat|shpca|shppa|shpss)_[a-f0-9]{32} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 examples: - 'shopify_app_secret: "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"' diff --git a/data/rules/slack.yml b/data/rules/slack.yml index 9544d10..0e61776 100644 --- a/data/rules/slack.yml +++ b/data/rules/slack.yml @@ -15,6 +15,8 @@ rules: [0-9a-z]{10,66} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 examples: - xapp-1-A05V64V7F2B-5062360157732-9f01726eebe77df2c096a65e95acdd02107b2c1e92ca341cff27ca271b7251b4 @@ -55,7 +57,10 @@ rules: ( xoxe-\d- [A-Z0-9]{140,150} - )\b + ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 examples: - xoxb-853BAAEE-1B2eDb6A4c75-01bB6Da1CE3E98f6fED5AeC07Dc3E94C @@ -89,7 +94,8 @@ rules: B[a-z0-9_-]{8,12}/ [a-z0-9_-]{20,30} ) - \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 examples: - https://hooks.slack.com/services/TY40v9sZ9/BxIqhIXIi/NGUyXK6nK7HMAqd0ASzXluoV diff --git a/data/rules/snyk.yml b/data/rules/snyk.yml index e6237e3..3ba783f 100644 --- a/data/rules/snyk.yml +++ b/data/rules/snyk.yml @@ -8,10 +8,12 @@ rules: (?:.|[\n\r]){0,32}? (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) (?:.|[\n\r]){0,32}? - \b ( [A-Z0-9]{8}-(?:[A-Z0-9]{4}-){3}[A-Z0-9]{12} ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 examples: - snyk_token = 123e4567-e89b-12d3-a456-426614174000 diff --git a/data/rules/sonarcloud.yml b/data/rules/sonarcloud.yml index f0b874d..c3307e9 100644 --- a/data/rules/sonarcloud.yml +++ b/data/rules/sonarcloud.yml @@ -13,6 +13,8 @@ rules: [0-9a-z]{40} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 2.5 examples: - sonar_api_token=abcdef0123456789abcdef0123456789abcdef23 diff --git a/data/rules/sonarqube.yml b/data/rules/sonarqube.yml index ceddbee..63be8fe 100644 --- a/data/rules/sonarqube.yml +++ b/data/rules/sonarqube.yml @@ -8,6 +8,9 @@ rules: (?:sq[pua]) _[a-z0-9]{40} ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 examples: - sonar.login=sqp_4b78f8494075e310d62dfdcaeb14be2c78fca2fc @@ -35,11 +38,9 @@ rules: (?xi) sonar.{0,8}host (?:.|[\n\r]){0,64}? - \b ( https?://.*?:\d{2,6} ) - \b min_entropy: 3.5 visible: false examples: diff --git a/data/rules/sourcegraph.yml b/data/rules/sourcegraph.yml index 3aed539..a8838de 100644 --- a/data/rules/sourcegraph.yml +++ b/data/rules/sourcegraph.yml @@ -4,8 +4,12 @@ rules: pattern: | (?xi) \b - sgp_(?:[a-f0-9]{16}_local_)?[a-f0-9]{40} + ( + sgp_(?:[A-F0-9]{16}|local)_[A-F0-9]{40}|sgp_[A-F0-9]{40} + ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 examples: - sgp_210f1131b08e93adcfc3f05faa2d768ff883a61f @@ -41,6 +45,8 @@ rules: (?:sgp_(?:[a-f0-9]{16}_local_)?[a-f0-9]{40}|[a-f0-9]{40}) ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.5 confidence: medium examples: diff --git a/data/rules/square.yml b/data/rules/square.yml index d3fd851..24a635a 100644 --- a/data/rules/square.yml +++ b/data/rules/square.yml @@ -10,10 +10,13 @@ rules: ( EAAA[a-z0-9\-\+=]{60} ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: - - square EAAA7h9fL9zQJR8P0eAioAf9239345rDA2349bQ8edUA9FgA5JojdsF3A9f6nKLmn + - square EAAA7h9fL9zQJR8P0eAioAf9239345rDA2349bQ8edUA9FgA5JojdsF3A9f6nKLm - square EAAAvlYh9H7dZwC9ash2hrHjtlL5D2srERGK5OM6F2nvle23he3NzA60PAeFXNHj validation: type: Http @@ -33,7 +36,15 @@ rules: - name: Square Access Token id: kingfisher.square.2 - pattern: '(?i)\b(sq0atp-[a-z0-9_-]{22})\b' + pattern: | + (?xi) + \b + ( + sq0atp-[a-z0-9_-]{22} + ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: @@ -57,7 +68,15 @@ rules: - name: Square OAuth Secret id: kingfisher.square.3 - pattern: '(?i)\b(sq0csp-[a-z0-9_-]{43})\b' + pattern: | + (?xi) + \b + ( + sq0csp-[a-z0-9_-]{43} + ) + \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/sslmate.yml b/data/rules/sslmate.yml new file mode 100644 index 0000000..937c5de --- /dev/null +++ b/data/rules/sslmate.yml @@ -0,0 +1,39 @@ +rules: + - name: SslMate API Key + id: kingfisher.sslmate.1 + pattern: | + (?xi) + \b + sslmate + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9]{36} + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.5 + confidence: medium + examples: + - sslmate_key="10000r90kriAAAAAseZJwsawemws03jdlmZY" + - SSLMATE_SECRET_KEY=ABCDEFGHIJ1234567890ABCDEFGHIJ123456 + references: + - https://sslmate.com/help/reference/apiv2 + validation: + type: Http + content: + request: + method: GET + url: "https://sslmate.com/api/v2/certs/example.com" + headers: + Authorization: "Basic {{ TOKEN | append: ':' | b64enc }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"cn"', '"exists"'] diff --git a/data/rules/stabilityai.yml b/data/rules/stabilityai.yml new file mode 100644 index 0000000..8848ce2 --- /dev/null +++ b/data/rules/stabilityai.yml @@ -0,0 +1,40 @@ +rules: + - name: Stability AI API Key + id: kingfisher.stabilityai.1 + pattern: | + (?x) + \b + ( + sk- + [A-Za-z0-9]{48} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 4.0 + confidence: medium + examples: + - sk-AnmgropvAII5XEoxVPjbnSMG3XhacEwhJlLh8ossXh7K1iLP + - sk-gQHyuK4k6Vw2viJRaAnLh6zAULaWtUg40ZHWcYjw7JGutlW6 + - sk-nwvJypEMFNASJLiPBgNnzJj1xsDwlHChbFRMNwVkzy3e4UJg + references: + - https://platform.stability.ai/docs/api-reference#v1-user-account + validation: + type: Http + content: + request: + method: GET + url: https://api.stability.ai/v1/user/account + headers: + Authorization: "Bearer {{ TOKEN }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: + - '"id"' + - '"email"' diff --git a/data/rules/stackhawk.yml b/data/rules/stackhawk.yml new file mode 100644 index 0000000..d001e40 --- /dev/null +++ b/data/rules/stackhawk.yml @@ -0,0 +1,34 @@ +rules: + - name: StackHawk API Key + id: kingfisher.stackhawk.1 + pattern: | + (?xi) + \b + ( + hawk\.[0-9A-Z_-]{20}\.[0-9A-Z_-]{20} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.0 + examples: + - 'HAWK_API_KEY="hawk.nHAOHdJjXoNyzAcTDC5M.R2gqQh2aCesrh0yCGB7q"' + references: + - https://docs.stackhawk.com/apidocs.html + - https://apidocs.stackhawk.com/reference/getuser + validation: + type: Http + content: + request: + method: GET + url: "https://api.stackhawk.com/api/v1/auth/user" + headers: + X-Api-Key: "{{TOKEN}}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"user":' \ No newline at end of file diff --git a/data/rules/statuspage.yml b/data/rules/statuspage.yml new file mode 100644 index 0000000..16294a1 --- /dev/null +++ b/data/rules/statuspage.yml @@ -0,0 +1,42 @@ +rules: + - name: Statuspage API Key + id: kingfisher.statuspage.1 + pattern: | + (?xi) + \b + statuspage + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN|API) + (?:.|[\n\r]){0,48}? + \b + ( + [0-9a-f]{64} + | + # Legacy UUID-ish keys (seen in older configs) + [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - 'statuspage_api: OAuth 89a229ce1a8dbcf9ff30430fbe35eb4c0426574bca932061892cefd2138aa4b1' + - statuspage_key = 123e4567-e89b-12d3-a456-426614174000 + references: + - https://developer.statuspage.io/ + validation: + type: Http + content: + request: + method: GET + url: "https://api.statuspage.io/v1/pages" + headers: + Authorization: "OAuth {{ TOKEN }}" + Accept: "application/json" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"id"', '"name"'] diff --git a/data/rules/stripe.yml b/data/rules/stripe.yml index faaa1ab..9d48341 100644 --- a/data/rules/stripe.yml +++ b/data/rules/stripe.yml @@ -11,7 +11,11 @@ rules: ( pk_live_ (?:[0-9A-Z]{6}){4,30} - ) + ) + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.3 confidence: medium categories: [api, key] @@ -33,6 +37,8 @@ rules: )_live_ (?:[0-9A-Z]{8}){3,25} ) + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/supabase.yml b/data/rules/supabase.yml new file mode 100644 index 0000000..6232195 --- /dev/null +++ b/data/rules/supabase.yml @@ -0,0 +1,81 @@ +rules: + - name: Supabase Management Token + id: kingfisher.supabase.1 + pattern: | + (?xi) + ( + sbp_[a-z0-9_-]{40} + ) + pattern_requirements: + min_digits: 2 + min_entropy: 3.5 + confidence: medium + examples: + - sbp_abcd1234efgh5678ijkl9012mnop3456qrst7890 + - sbp_1234567890abcdefghij1234567890klmnopqrst + references: + - https://supabase.com/docs/reference/api/v1-get-an-organization + validation: + type: Http + content: + request: + headers: + Authorization: "Bearer {{ TOKEN }}" + method: GET + url: https://api.supabase.com/v1/organizations + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 + - name: Supabase Project API Key + id: kingfisher.supabase.2 + pattern: | + (?xi) + ( + sb_secret_[a-z0-9_-]{31} + ) + pattern_requirements: + min_digits: 2 + min_entropy: 4.0 + confidence: medium + validation: + type: Http + content: + request: + method: GET + url: "{{SBPROJECTURL}}/rest/v1/?select=*" + headers: + Apikey: "{{TOKEN}}" + User-Agent: "" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"host":' + references: + - https://supabase.com/docs/reference/api/v1-get-an-organization + examples: + - "sb_secret_9uM4GhB0STF5R4K3HxQtlg_bzWW6DRj" + - "sb_secret_szE_jsbktD3pWgnfUjgahw_hcHEIOBH" + depends_on_rule: + - rule_id: "kingfisher.supabase.3" + variable: SBPROJECTURL + - name: Supabase Project URL + id: kingfisher.supabase.3 + pattern: + (?xi) + \b + ( + https:\/\/[a-z0-9]{16,32}\.supabase\.co + ) + \b + confidence: medium + min_entropy: 3.0 + visible: false + references: + - https://supabase.com/docs/guides/api + examples: + - "https://ejcvydfyxzmbtfbfstnq.supabase.co" diff --git a/data/rules/supabasetoken.yml b/data/rules/supabasetoken.yml deleted file mode 100644 index d244d3b..0000000 --- a/data/rules/supabasetoken.yml +++ /dev/null @@ -1,31 +0,0 @@ -rules: - - name: Supabase API Key - id: kingfisher.supabase.1 - pattern: | - (?xi) - \b - sbp_ - ( - [a-z0-9_-]{40} - ) - \b - min_entropy: 3.5 - confidence: medium - examples: - - sbp_abcd1234efgh5678ijkl9012mnop3456qrst7890 - - sbp_1234567890abcdefghij1234567890klmnopqrst - references: - - https://supabase.com/docs/reference/api/v1-get-an-organization - validation: - type: Http - content: - request: - headers: - Authorization: "Bearer {{ TOKEN }}" - method: GET - url: https://api.supabase.com/v1/organizations - response_matcher: - - report_response: true - - type: StatusMatch - status: - - 200 \ No newline at end of file diff --git a/data/rules/tailscale.yml b/data/rules/tailscale.yml index f393da7..08fa9eb 100644 --- a/data/rules/tailscale.yml +++ b/data/rules/tailscale.yml @@ -5,14 +5,16 @@ rules: (?xi) \b ( - tskey-[a-z]+-[A-Z0-9_-]{20,24} + tskey-[a-z]{3,10}-[A-Z0-9_-]{20,36} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.0 confidence: medium examples: - - tskey-secret-12345678-abcdefghijkl - - tskey-api-abcdefg-1234567890123 + - tskey-secret-weRTWSfoeFKI-3480754342kDSFelW3 + - tskey-api-weRTWSfoeFKI-3480754342kDSFelW3 references: - https://tailscale.com/kb/1215/oauth-clients validation: diff --git a/data/rules/tavily.yml b/data/rules/tavily.yml new file mode 100644 index 0000000..1775eb4 --- /dev/null +++ b/data/rules/tavily.yml @@ -0,0 +1,40 @@ +rules: + - name: Tavily API Key + + id: kingfisher.tavily.1 + pattern: | + (?x) + \b + ( + tvly-[a-zA-Z0-9]{32} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.0 + references: + - https://docs.tavily.com/api-reference + examples: + - "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" + - "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" + - 'TAVILY_API_KEY = "tvly-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"' + validation: + type: Http + content: + request: + method: POST + url: "https://api.tavily.com/search" + headers: + Authorization: "Bearer {{TOKEN}}" + Content-Type: "application/json" + body: '{"query": "test"}' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"query":' + - '"results":' + match_all_words: true \ No newline at end of file diff --git a/data/rules/teamcity.yml b/data/rules/teamcity.yml index c3bf68b..b475c32 100644 --- a/data/rules/teamcity.yml +++ b/data/rules/teamcity.yml @@ -12,6 +12,8 @@ rules: \. [A-Za-z0-9_-]{48} ) + pattern_requirements: + min_digits: 2 examples: - '' references: diff --git a/data/rules/telegram.yml b/data/rules/telegram.yml new file mode 100644 index 0000000..3f628a6 --- /dev/null +++ b/data/rules/telegram.yml @@ -0,0 +1,34 @@ +rules: + - name: Telegram Bot Token + id: kingfisher.telegram.1 + pattern: | + (?xi) + \b + (?:telegram|tgram:) + (?:.|[\n\r]){0,32}? + ( + [0-9]{7,10} + : + [A-Z0-9_-]{35} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.5 + validation: + type: Http + content: + request: + method: GET + url: "https://api.telegram.org/bot{{TOKEN}}/getMe" + response_matcher: + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"ok":true' + examples: + - "tgram://110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsawd" + - "telegram: 508627689:AAEuLPKs-EhrjrYGnz60bnYNZqakf6HJxc0" + - "telegram token is 3628091811:BAG9RuJiqgOGIfFbOPBpAo6QhIJoD9mCdDs" \ No newline at end of file diff --git a/data/rules/thingsboard.yml b/data/rules/thingsboard.yml new file mode 100644 index 0000000..ce5ed78 --- /dev/null +++ b/data/rules/thingsboard.yml @@ -0,0 +1,55 @@ +rules: + - name: ThingsBoard Access Token + id: kingfisher.thingsboard.1 + pattern: | + (?x) + \b + thingsboard\.cloud/api/v1/ + ( + [a-z0-9]{20} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.0 + examples: + - http://thingsboard.cloud/api/v1/354u1g321kcqc1oad3w7/telemetry + - https://thingsboard.cloud/api/v1/354u1g321kcqc1oad3w7/telemetry + - coap://coap.thingsboard.cloud/api/v1/354u1g321kcqc1oad3w7/telemetry + references: + - https://thingsboard.io/docs/paas/reference/http-api/ + - https://thingsboard.io/docs/paas/reference/coap-api/ + + - name: ThingsBoard Provision Device Key + id: kingfisher.thingsboard.2 + pattern: | + (?x) + "provisionDeviceKey"\s*:\s*" + ( + [a-z0-9]{20} + ) + \b + confidence: medium + min_entropy: 3.0 + examples: + - '"{"deviceName": "DEVICE_NAME", "provisionDeviceKey": "s2s1gfcuatgbi61n8h5s", "provisionDeviceSecret": "xbzsovaw9ix4qfhi14an"}"' + references: + - https://thingsboard.io/docs/paas/reference/http-api/ + - https://thingsboard.io/docs/paas/user-guide/device-provisioning/ + - name: ThingsBoard Provision Device Secret + id: kingfisher.thingsboard.3 + pattern: | + (?x) + "provisionDeviceSecret"\s*:\s*" + ( + [a-z0-9]{20} + ) + \b + confidence: medium + min_entropy: 3.0 + examples: + - '"{"deviceName": "DEVICE_NAME", "provisionDeviceKey": "s2s1gfcuatgbi61n8h5s", "provisionDeviceSecret": "xbzsovaw9ix4qfhi14an"}"' + references: + - https://thingsboard.io/docs/paas/reference/http-api/ + - https://thingsboard.io/docs/paas/user-guide/device-provisioning/ \ No newline at end of file diff --git a/data/rules/togetherai.yml b/data/rules/togetherai.yml new file mode 100644 index 0000000..e3d9b04 --- /dev/null +++ b/data/rules/togetherai.yml @@ -0,0 +1,39 @@ +rules: + - name: Together.ai API Key + id: kingfisher.together.1 + pattern: | + (?xi) + \b + ( + tgp_v1_[A-Z0-9_-]{43} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.0 + examples: + - tgp_v1_Tctm6OfOeNkwLIKkyxJxUHIqNKx2AvFr65tQRIOMgzY + - tgp_v1_HgWU7iym2128y2Pdj-7-9kX4W_MSCcIT5EhuY_SmNqc + - tgp_v1_xeybrcbPy2c10JR9eAlkOq1qvPaBXT3ZbXp8yKq1VME + - tgp_v1_yanBH3171P6HAZ01LbzSDlnOiXM3lo_89kG2Gg5yzko + validation: + type: Http + content: + request: + method: GET + url: "https://api.together.xyz/v1/models" + headers: + Authorization: "Bearer {{ TOKEN }}" + Accept: "application/json" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"id":' + - '"object":' + references: + - https://docs.together.ai/reference/authentication + - https://docs.together.ai/reference/models-list diff --git a/data/rules/travisci.yml b/data/rules/travisci.yml index 3461fbb..1d4b188 100644 --- a/data/rules/travisci.yml +++ b/data/rules/travisci.yml @@ -13,6 +13,8 @@ rules: [A-Z-_0-9]{22} ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.0 confidence: medium examples: @@ -31,3 +33,22 @@ rules: - report_response: true - type: StatusMatch status: [200] + - name: Travis CI Encrypted Variable + id: kingfisher.travisci.2 + pattern: | + (?xis) + (?:language|env|deploy|script):[\r\n] + (?:.|[\\n\r]){0,256}? + ( + secure:\s*"?[A-Za-z0-9+/=\\]+"?\s* + ) + \b + min_entropy: 3.0 + confidence: medium + examples: + - | + env: + global: + # This sets FOO=super-secret, but the plaintext never appears here. + - secure: "VJh0l9gOb+6AVNDk6cziZSs1AqVM8CqtZU6ot9ZQeJ+KfL1pxnGQ4qQF8Cz9\M1q85c3l1N1+qkQ0uV12QG6O6ylq6Qq1l3VjAJM3h2pY3jdmrA8kX2ZIxRjC/\8+Xj1wVtKQ0R+owM/6i5Y6cyx4hRb3VvSeYlC0lD1iTzQ2vgMyE=" + diff --git a/data/rules/truenas.yml b/data/rules/truenas.yml index 9e0067a..57ffcff 100644 --- a/data/rules/truenas.yml +++ b/data/rules/truenas.yml @@ -6,6 +6,8 @@ rules: "params"\s*:\s*\[\s*" (\d+-[a-zA-Z0-9]{64}) "\s*\] + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: @@ -36,6 +38,8 @@ rules: Bearer\s* (\d+-[a-zA-Z0-9]{64}) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: @@ -46,4 +50,4 @@ rules: - https://www.truenas.com/docs/scale/scaletutorials/toptoolbar/managingapikeys/ - https://www.truenas.com/docs/scale/scaleclireference/auth/cliapikey/ - https://www.truenas.com/docs/scale/api/ - - https://www.truenas.com/community/threads/api-examples-in-perl-python.108053/ \ No newline at end of file + - https://www.truenas.com/community/threads/api-examples-in-perl-python.108053/ diff --git a/data/rules/twilio.yml b/data/rules/twilio.yml index b3b2d32..14c5b0e 100644 --- a/data/rules/twilio.yml +++ b/data/rules/twilio.yml @@ -8,6 +8,9 @@ rules: (?:SK|AC)[A-F0-9]{32} ) \b + pattern_requirements: + min_digits: 2 + visible: false min_entropy: 3.5 examples: - | @@ -31,7 +34,10 @@ rules: ( [a-z0-9]{32} ) - \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 3.0 examples: - Twilio_key=Cd2Bd1dE1201aE2DFFEcfeBafCc3c31D diff --git a/data/rules/twitch.yml b/data/rules/twitch.yml new file mode 100644 index 0000000..d9ecc55 --- /dev/null +++ b/data/rules/twitch.yml @@ -0,0 +1,37 @@ +rules: + - name: Twitch API Token + id: kingfisher.twitch.1 + pattern: | + (?xi) + twitch + (?:.|[\n\r]){0,32}? + \b + ( + [a-z0-9]{30} + ) + \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 + min_entropy: 3.5 + confidence: medium + examples: + - TWITCH_TOKEN=abcdefghijklmnopqrstuvwx123456 + - "twitch_api_token: '0123456789abcdefghijklmnopqrst'" + references: + - https://dev.twitch.tv/docs/authentication/validate-tokens/ + validation: + type: Http + content: + request: + method: GET + url: https://id.twitch.tv/oauth2/validate + headers: + Authorization: 'OAuth {{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid \ No newline at end of file diff --git a/data/rules/twitter.yml b/data/rules/twitter.yml index 2722fc1..7952f50 100644 --- a/data/rules/twitter.yml +++ b/data/rules/twitter.yml @@ -1,6 +1,6 @@ rules: - name: X / Twitter Bearer Token (App-only) - id: kingfisher.twitter.bearer.1 + id: kingfisher.twitter.1 pattern: | (?xi) \b @@ -13,6 +13,10 @@ rules: A{10,}[A-Za-z0-9_\-]{40,200} ) \b + pattern_requirements: + min_digits: 2 + min_uppercase: 1 + min_lowercase: 1 min_entropy: 4.0 confidence: medium examples: @@ -36,3 +40,57 @@ rules: match_all_words: true references: - https://developer.x.com/en/docs/x-api/v1/developer-utilities/rate-limit-status/api-reference/get-application-rate_limit_status + - name: Twitter Consumer Key + id: kingfisher.twitter.2 + pattern: | + (?xi) + \b + twitter + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + ( + [A-Z0-9]{25} + ) + min_entropy: 3.5 + visible: false + examples: + - "TWITTER_KEY=4RTBCyG2TbvL407A1lWxQFKCC" + - name: X / Twitter Consumer Secret + id: kingfisher.twitter.3 + pattern: | + (?xi) + \b + twitter + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + ( + [A-Z0-9]{50} + ) + min_entropy: 4.0 + examples: + - "TWITTER_SECRET=ZGwXeK2DNCqv49Z9ofwYdqlBgeoHDyh8uoAgHju6OeYC7wTQJq" + references: + - https://developer.x.com/en/docs/authentication/oauth-2-0/application-only + validation: + type: Http + content: + request: + method: POST + url: https://api.twitter.com/oauth2/token + headers: + Authorization: "Basic {{ TWITTER_KEY | append: ':' | append: TOKEN | b64enc }}" + Content-Type: "application/x-www-form-urlencoded;charset=UTF-8" + body: "grant_type=client_credentials" + response_matcher: + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"token_type":"bearer"' + - '"access_token":' + match_all_words: true + depends_on_rule: + - rule_id: "kingfisher.twitter.2" + variable: TWITTER_KEY diff --git a/data/rules/typeform.yml b/data/rules/typeform.yml new file mode 100644 index 0000000..cfbc3ee --- /dev/null +++ b/data/rules/typeform.yml @@ -0,0 +1,34 @@ +rules: + - name: Typeform API Token + id: kingfisher.typeform.1 + pattern: | + (?xi) + \b + typeform + (?:.|[\n\r]){0,32}? + ( + tfp_[a-z0-9_\-=\.]{59} + ) + pattern_requirements: + min_digits: 2 + min_entropy: 4.0 + confidence: medium + examples: + - TYPEFORM_TOKEN=tfp_abcd1234efgh5678ijkl9012mnop3456qrst7890uvwx_yzABCD1234efgh + - "typeform_api_key: 'tfp_qwerty1234567890asdfgh0987654321zxcvbnmPOIU_0987654321lkjhi'" + references: + - https://developer.typeform.com/get-started/personal-access-token/ + validation: + type: Http + content: + request: + method: GET + url: https://api.typeform.com/forms + headers: + Authorization: 'Bearer {{ TOKEN }}' + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid \ No newline at end of file diff --git a/data/rules/uri.yml b/data/rules/uri.yml index e5542be..e3e3a7d 100644 --- a/data/rules/uri.yml +++ b/data/rules/uri.yml @@ -3,17 +3,28 @@ rules: id: kingfisher.uri.1 pattern: | (?xi) - (https?):// - [a-z][a-z0-9+\-.]* - : - [a-z0-9\-._~%!$&'()*,;=]+ - @ - (?:[a-z0-9\-._~%]+|\[[a-f0-9:.]+\]|\[v[a-f0-9][a-z0-9\-._~%!$&'()*,;=:]+\]) - (:?[0-9]+)? - (?:/[a-z0-9\-._~%!$&'()*,;=:@]*)* # Match path - /? - (?:\?[a-z0-9\-._~%!$&'()*,;=:@/?]*)? - (?:\#[a-z0-9\-._~%!$&'()*,;=:@/?]*)? + ( + (?:https?):// + [A-Za-z](?:[A-Za-z0-9+\-._~!$&'()*+,;=]|%[0-9A-Fa-f]{2})* + : + (?:[A-Za-z0-9\-._~!$&'()*+,;=]|%[0-9A-Fa-f]{2})+ + @ + (?:[A-Za-z0-9\-._~%]+|\[[A-Fa-f0-9:.]+\]|\[v[A-Fa-f0-9][A-Za-z0-9\-._~%!$&'()*,;=:]+\]) + (:?[0-9]+)? + (?:/[A-Za-z0-9\-._~%!$&'()*,;=:@%]*)* + /? + (?:\?[A-Za-z0-9\-._~%!$&'()*,;=:@/?%]*)? + (?:\#[A-Za-z0-9\-._~%!$&'()*,;=:@/?%]*)? + ) + pattern_requirements: + ignore_if_contains: + - "****" + - "xxxx" + - "username:" + - "user:" + - ":password" + - ":pass" + - ">:<" min_entropy: 4.0 confidence: medium examples: @@ -25,7 +36,7 @@ rules: method: GET url: '{{ TOKEN }}' response_matcher: - - report_response: true + - report_response: false type: StatusMatch status: - 200 diff --git a/data/rules/vastai.yml b/data/rules/vastai.yml new file mode 100644 index 0000000..0adc145 --- /dev/null +++ b/data/rules/vastai.yml @@ -0,0 +1,44 @@ +rules: + - name: Vast.ai API Key + id: kingfisher.vastai.1 + pattern: | + (?xi) + \b + vast(?:\.ai)? + (?:.|[\n\r]){0,16}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,16}? + \b + ( + [a-f0-9]{64} + ) + \b + pattern_requirements: + min_digits: 8 + min_entropy: 3.5 + confidence: medium + examples: + - VAST_API_KEY=c218521a7eaf108227795ae059866db8fdd7be16348f67ec48e66af4b2df4a76 + - 'vastai_access_key: c218521a7eaf108227795ae059866db8fdd7be16348f67ec48e66af4b2df4a76' + references: + - https://docs.vast.ai/api-reference/accounts/show-user + validation: + type: Http + content: + request: + method: GET + url: https://console.vast.ai/api/v0/users/current/ + headers: + Authorization: "Bearer {{ TOKEN }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + - type: WordMatch + match_all_words: true + words: + - '"id"' + - '"email"' + - '"balance"' diff --git a/data/rules/vercel.yml b/data/rules/vercel.yml new file mode 100644 index 0000000..3ace650 --- /dev/null +++ b/data/rules/vercel.yml @@ -0,0 +1,43 @@ +rules: + - name: Vercel API Token + id: kingfisher.vercel.1 + pattern: | + (?xi) + \b + vercel + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9]{24} + ) + \b + pattern_requirements: + min_digits: 6 + min_uppercase: 1 + min_lowercase: 1 + confidence: medium + min_entropy: 3.5 + validation: + type: Http + content: + request: + method: GET + url: https://api.vercel.com/v2/user + headers: + Authorization: "Bearer {{TOKEN}}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"user":' + - '"email":' + match_all_words: true + references: + - https://vercel.com/docs/rest-api#authentication + examples: + - "vercel-key = DdZV6ZDZW6Vpl7n7JqtrCE5i" + - "vercel_token = zyMBA1qVEMAf4UNNZtCAbg6u" + - "vercel_api_key = MTg0AW799OY1HmyDdn84or3C" + - "vercel_secret = A7n9Xfp3tBz7D0XpOTMWpiOM" diff --git a/data/rules/vmware.yml b/data/rules/vmware.yml new file mode 100644 index 0000000..c623ebf --- /dev/null +++ b/data/rules/vmware.yml @@ -0,0 +1,20 @@ +rules: + - name: Credentials in Connect-VIServer Invocation + id: kingfisher.vmware.1 + pattern: | + (?xi) + Connect-VIServer + .{0,50} + -User \s+ (\S{3,30}) \s+ (?# username ) + .{0,50} + -Password \s+ (\S{3,30}) (?# password ) + examples: + - 'Connect-VIServer -Server 192.168.1.51 -User administrator@vSphere.local -Password VMware1!' + - | + #Set-PowerCLIConfiguration -InvalidCertificateAction:Ignore + Connect-VIServer "$endpoint" -User "$username" -Password "$password" | Out-Null + - 'Connect-VIServer $ESXiHost.EsxiHost -user $ESXiUser -password $ESXipass' + - '$null = connect-viserver vc.lab.local -user administrator@vsphere.local -password VMware1!' + + references: + - https://developer.broadcom.com/powercli/latest/vmware.vimautomation.core/commands/connect-viserver \ No newline at end of file diff --git a/data/rules/voyageai.yml b/data/rules/voyageai.yml new file mode 100644 index 0000000..942ab40 --- /dev/null +++ b/data/rules/voyageai.yml @@ -0,0 +1,26 @@ +rules: + - name: Voyage AI API Key + id: kingfisher.voyageai.api_key + description: Detects Voyage AI API keys used for embedding and retrieval models. + # Matches keys starting with 'pa-' followed by 43 URL-safe base64 characters + pattern: | + (?x) + ( + pa-[a-zA-Z0-9\-_]{43} + ) + \b + min_entropy: 4.0 + confidence: high + examples: + - pa-r4yuCYCuPhNO-10Lu9aO7dR4jxUWlLmlUjm_NOVVdSs + validation: + type: Http + content: + request: + method: GET + url: https://api.voyageai.com/v1/files + headers: + Authorization: "Bearer {{ TOKEN }}" + response_matcher: + - type: StatusMatch + status: [200] \ No newline at end of file diff --git a/data/rules/weightsandbiases.yml b/data/rules/weightsandbiases.yml new file mode 100644 index 0000000..88bb37b --- /dev/null +++ b/data/rules/weightsandbiases.yml @@ -0,0 +1,37 @@ +rules: + - name: Weights and Biases API Key + id: kingfisher.wandb.1 + pattern: | + (?xi) + (?:wandb|weightsandbiases) + (?:.|[\n\r]){0,16}? + \b + ( + [a-f0-9]{40} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 3.5 + examples: + - "export WANDB_API_KEY=872ab943740b34157041da2529fb160d89632710" + - "wandb_api_key: 1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b" + - "WeightsandBiases token => 7f9d2e34c1a0b5d6e7f81234abcd5678ef901234" + validation: + type: Http + content: + request: + method: POST + url: "https://api.wandb.ai/graphql" + headers: + Authorization: "Basic {{ 'api:' | append: TOKEN | b64enc }}" + Content-Type: "application/json" + body: | + {"query":"query { viewer { email username } }"} + response_matcher: + - report_response: true + - type: JsonValid + - type: WordMatch + words: + - '"username"' diff --git a/data/rules/wireguard.yml b/data/rules/wireguard.yml index 70aa1ae..47223c1 100644 --- a/data/rules/wireguard.yml +++ b/data/rules/wireguard.yml @@ -2,6 +2,8 @@ rules: - name: WireGuard Private Key id: kingfisher.wireguard.1 pattern: PrivateKey\s*=\s*([A-Za-z0-9+/]{43}=) + pattern_requirements: + min_digits: 2 min_entropy: 3.3 confidence: medium examples: diff --git a/data/rules/xAI.yml b/data/rules/xAI.yml index 398ac03..67a04ae 100644 --- a/data/rules/xAI.yml +++ b/data/rules/xAI.yml @@ -6,8 +6,10 @@ rules: \b ( xai-[A-Za-z0-9_-]{70,120} - ) + ) \b + pattern_requirements: + min_digits: 2 min_entropy: 3.8 confidence: medium examples: diff --git a/data/rules/yandex.yml b/data/rules/yandex.yml new file mode 100644 index 0000000..60ebeb5 --- /dev/null +++ b/data/rules/yandex.yml @@ -0,0 +1,39 @@ +rules: + - name: Yandex API Key + id: kingfisher.yandex.1 + pattern: | + (?xi) + \b + yandex + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9.]{83} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.3 + confidence: medium + examples: + - "yandex_api_key= 'pdct.1.1.20218925T124723Z.07193b9c567c0c90.ebba3042fcf1acfc4d682db12c01a5289f9769c0'" + - "yandex_secret=1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTU" + references: + - https://yandex.com/dev/dictionary + - https://pkg.go.dev/github.com/unitrans/unitrans/src/translator/backend_particular + validation: + type: Http + content: + request: + method: GET + url: "https://dictionary.yandex.net/api/v1/dicservice.json/lookup?key={{ TOKEN }}&lang=en-ru&text=time&ui=en" + headers: + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"def"'] diff --git a/data/rules/yelp.yml b/data/rules/yelp.yml new file mode 100644 index 0000000..6e4e9d3 --- /dev/null +++ b/data/rules/yelp.yml @@ -0,0 +1,39 @@ +rules: + - name: Yelp API Key + id: kingfisher.yelp.1 + pattern: | + (?xi) + \b + yelp + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [a-zA-Z0-9_\\=.\\-]{128} + ) + \b + pattern_requirements: + min_digits: 6 + min_entropy: 3.8 + confidence: medium + examples: + - yelp_token = wiuck20l8j-oWwCd9r53FqpN6ELB7K03zGw-ccUQR7uLHc9NaWubovOMdGdyFqIGGM4aVK6nxQ1DreDZn_qBYU4jky_5kQRVkiIDPSheCPggY3WzyRzi27kxoOpoYAYx + references: + - https://docs.developer.yelp.com/docs/places-authentication + - https://docs.developer.yelp.com/reference/v3_all_categories + validation: + type: Http + content: + request: + method: GET + url: "https://api.yelp.com/v3/categories?locale=en_US" + headers: + Authorization: "Bearer {{ TOKEN }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"categories"'] diff --git a/data/rules/youtube.yml b/data/rules/youtube.yml new file mode 100644 index 0000000..96b82fb --- /dev/null +++ b/data/rules/youtube.yml @@ -0,0 +1,31 @@ +rules: + - name: YouTube API Key + id: kingfisher.youtube.1 + pattern: | + (?xi) + \b + ( + AIza[a-zA-Z0-9_\-\\]{35} + ) + \b + min_entropy: 2.0 + confidence: medium + examples: + - '"youtube":{"api_key":"AIzaSyCKHtojSg-UNteBWlLUR2kHSgjGpUScYjk"' + validation: + type: Http + content: + request: + method: GET + url: https://www.googleapis.com/youtube/v3/videos?part=id&id=dummy&key={{ TOKEN }} + response_matcher: + - report_response: true + - type: WordMatch + words: + - "API key not valid" + - "keyInvalid" + - "API_KEY_INVALID" + - "forbidden" + match_all_words: false + negative: true + - type: JsonValid diff --git a/data/rules/zhipu.yml b/data/rules/zhipu.yml new file mode 100644 index 0000000..3d7ddab --- /dev/null +++ b/data/rules/zhipu.yml @@ -0,0 +1,36 @@ +rules: + - name: Zhipu (BigModel) API Key + id: kingfisher.zhipu.1 + pattern: | + (?xi) + \b + ( + [A-F0-9]{32} + \. + [A-Z0-9]{16} + ) + \b + pattern_requirements: + min_digits: 2 + confidence: medium + min_entropy: 4.0 + examples: + - "3494c505cf244a3fb17417d6894d404c.LLSZ2InjarUXEhNr" + - "a64cb6a9b4e840919351d041dbe65654.eh1YZt0SAhSTOsNR" + - "4d140d7d21c4477ab20d5090e530496c.A5pEbmgcid2deKNA" + validation: + type: Http + content: + request: + method: GET + url: "https://open.bigmodel.cn/api/paas/v4/files" + headers: + Authorization: "Bearer {{ TOKEN }}" + Accept: "application/json" + timeout_seconds: 12 + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ["object", "data"] diff --git a/data/rules/zohocrm.yml b/data/rules/zohocrm.yml new file mode 100644 index 0000000..f0ab57b --- /dev/null +++ b/data/rules/zohocrm.yml @@ -0,0 +1,35 @@ +rules: + - name: Zoho CRM API Access Token + id: kingfisher.zohocrm.1 + pattern: | + (?xi) + \b + ( + 1000\.[a-f0-9]{32}\.[a-f0-9]{32} + ) + \b + pattern_requirements: + min_digits: 4 + min_entropy: 3.5 + confidence: medium + examples: + - 1000.a23f12b4c5d6e7f8901234567890abc1.23d4e5f67890abcdef1234567890abcd + - 1000.123fa4b5c678d90eabcdef1234567890.ab12c3d4e5f6a7890bcd12ef345678ab + references: + - https://www.zoho.com/crm/developer/docs/api/v8/access-refresh.html + - https://www.zoho.com/crm/developer/docs/api/v8/get-users.html + validation: + type: Http + content: + request: + method: GET + url: "https://www.zohoapis.com/crm/v7/users?type=CurrentUser" + headers: + Authorization: "Zoho-oauthtoken {{ TOKEN }}" + Accept: application/json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"users"'] diff --git a/data/rules/zuplo.yml b/data/rules/zuplo.yml new file mode 100644 index 0000000..bbfdb7b --- /dev/null +++ b/data/rules/zuplo.yml @@ -0,0 +1,36 @@ +rules: + - name: Zuplo API Key + id: kingfisher.zuplo.1 + pattern: | + (?xi) + \b + ( + zpka_(?P[a-z0-9]{32})_(?P[0-9a-f]{8}) + ) + pattern_requirements: + checksum: + actual: + template: "{{ CHECKSUM | downcase }}" + requires_capture: checksum + expected: "{{ BODY | crc32_hex }}" + min_entropy: 3.3 + confidence: medium + examples: + - zpka_3e6c4f7d39954ca29353b7ab88589b64_de26cd55 + - zpka_b3f94d8d3d4d4a6ea5c5b20d0a5bb407_18eb262b + references: + - https://zuplo.com/blog/api-key-authentication + validation: + type: Http + content: + request: + headers: + authorization: "Bearer {{ TOKEN }}" + x-api-key: "{{ TOKEN }}" + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://dev.zuplo.com/v1/who-am-i \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..54f0959 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,38 @@ +# syntax=docker/dockerfile:1 +FROM alpine:latest + +RUN apk add --no-cache curl tar git + +ARG TARGETARCH +ENV TARGETARCH=${TARGETARCH} + +WORKDIR /app + +RUN set -eux; \ + # choose the right asset for this build platform + case "${TARGETARCH}" in \ + amd64) SUFFIX="linux-x64.tgz" ;; \ + arm64) SUFFIX="linux-arm64.tgz" ;; \ + *) echo "unsupported arch ${TARGETARCH}" >&2; exit 1 ;; \ + esac; \ + # download & unpack + LATEST_URL=$(curl -fsSL https://api.github.com/repos/mongodb/kingfisher/releases/latest \ + | grep -Eo "https://[^\"]*${SUFFIX}"); \ + if [ -z "$LATEST_URL" ]; then \ + echo "Failed to fetch the latest release URL for ${SUFFIX}" >&2; \ + exit 1; \ + fi; \ + curl -fsSL "$LATEST_URL" -o kingfisher.tgz; \ + tar -xzf kingfisher.tgz; \ + rm -f kingfisher.tgz CHECKSUM-*.txt; \ + # locate the binary (pattern covers kingfisher-linux-x64 / kingfisher-linux-arm64) + KF_PATH=$(find . -type f -name 'kingfisher*' -executable -print -quit); \ + if [ -z "$KF_PATH" ]; then echo "No executable kingfisher binary found" >&2; exit 1; fi; \ + install -m 0755 "$KF_PATH" /usr/local/bin/kingfisher; \ + # optional cleanup to keep the image small + rm -rf /app/* + +# quick smoke-test so the build fails early if something’s wrong +RUN kingfisher --version + +ENTRYPOINT ["kingfisher"] diff --git a/docs/BASELINE.md b/docs/BASELINE.md new file mode 100644 index 0000000..558e819 --- /dev/null +++ b/docs/BASELINE.md @@ -0,0 +1,55 @@ + +# Build a Baseline / Detect Only New Secrets + +There are situations where a repository already contains checked‑in secrets, but you want to ensure no **new** secrets are introduced. A baseline file lets you document the known findings so future scans only report anything that is not already in that list. + +The easiest way to create a baseline is to run a normal scan with the `--manage-baseline` flag (typically at a low confidence level to capture all potential matches): + +```bash +kingfisher scan /path/to/code \ + --confidence low \ + --manage-baseline \ + --baseline-file ./baseline-file.yml +``` + +This generates a YAML file named `baseline-file.yml` in the current directory. The file tracks each finding under an `ExactFindings` section: + +```yaml +ExactFindings: + matches: + - filepath: ruby_vulnerable.rb/ + fingerprint: 056876f00ffd0622 + linenum: 52 + lastupdated: Mon, 14 Jul 2025 10:17:56 -0700 + - filepath: ruby_vulnerable.rb/ + fingerprint: ce41d19b83b2b1b0 + linenum: 53 + lastupdated: Mon, 14 Jul 2025 10:17:56 -0700 + - filepath: ruby_vulnerable.rb/ + fingerprint: e8644d91fa6654f5 + linenum: 40 + lastupdated: Mon, 14 Jul 2025 10:17:56 -0700 +``` + +`fingerprint` reuses Kingfisher's 64-bit *finding fingerprint* algorithm with offsets set to zero. It hashes the secret value together with the normalized filepath, so moving a secret around does not create a new entry. + +Running another scan with `--manage-baseline` rewrites the file so it only contains findings that still exist in the repository. Use the same YAML file with the `--baseline-file` option on future scans to hide all recorded findings: + +```bash +kingfisher scan /path/to/code \ + --baseline-file /path/to/baseline-file.yaml +``` + +If you intentionally add a new secret that should be ignored later, rerun the scan with both `--manage-baseline` and `--baseline-file` to refresh the baseline. New matches are appended and entries for secrets that no longer appear (for example, because files were removed or excluded) are pruned: + +```bash +kingfisher scan /path/to/code \ + --manage-baseline \ + --baseline-file /path/to/baseline-file.yml +``` + +If you want to know which files are being skipped, enable verbose debugging (-v) when scanning, which will report any files being skipped by the baseline file (or via `--exclude`): + +```bash +kingfisher scan /path/to/project -v +``` \ No newline at end of file diff --git a/docs/COMPARISON.md b/docs/COMPARISON.md index ecb8ebf..a0eb4c2 100644 --- a/docs/COMPARISON.md +++ b/docs/COMPARISON.md @@ -12,6 +12,10 @@ | linux | 205.19 | 597.51 | 548.96 | 5.49 | | typescript | 64.99 | 183.04 | 232.34 | 4.23 | +

+ Kingfisher Runtime Comparison +

+ ### Validated/Verified Findings Comparison Note: For GitLeaks and detect-secrets, validated/verified counts are not available. @@ -42,19 +46,6 @@ Note: For GitLeaks and detect-secrets, validated/verified counts are not availab | mongodb | 1 | 191 | 0 | 0 | | linux | 0 | 287 | 0 | 0 | | typescript | 0 | 10 | 0 | 0 | -### QuickChart.io Visualizations - -#### Runtime Chart -*Lower runtimes are better* - -![Runtime Comparison](https://quickchart.io/chart?c=%7B%22type%22%3A%22bar%22%2C%22data%22%3A%7B%22labels%22%3A%5B%22croc%22%2C%22rails%22%2C%22ruby%22%2C%22gitlab%22%2C%22django%22%2C%22lucene%22%2C%22mongodb%22%2C%22linux%22%2C%22typescript%22%5D%2C%22datasets%22%3A%5B%7B%22label%22%3A%22Kingfisher%22%2C%22data%22%3A%5B3.087692041%2C9.816560542%2C22.222204459%2C129.921919875%2C6.748027708%2C18.650581459%2C27.47587625%2C204.192040875%2C62.877494792%5D%7D%2C%7B%22label%22%3A%22TruffleHog%22%2C%22data%22%3A%5B17.667027792%2C24.4969155%2C133.286264708%2C335.819256375%2C248.135664708%2C91.367231833%2C180.311266375%2C585.00584475%2C182.478392708%5D%7D%2C%7B%22label%22%3A%22GitLeaks%22%2C%22data%22%3A%5B2.845539417%2C19.704876208%2C46.658975%2C285.6701695%2C22.446593958%2C53.793195375%2C174.406220375%2C517.420016958%2C164.260176625%5D%7D%2C%7B%22label%22%3A%22detect-secrets%22%2C%22data%22%3A%5B0.703465916%2C0.783118209%2C1.231432834%2C8.751082041%2C1.120182458%2C1.019824708%2C4.737797875%2C8.402164%2C7.170617042%5D%7D%5D%7D%2C%22options%22%3A%7B%22scales%22%3A%7B%22yAxes%22%3A%5B%7B%22ticks%22%3A%7B%22beginAtZero%22%3Atrue%7D%7D%5D%7D%2C%22title%22%3A%7B%22display%22%3A%22true%22%2C%22text%22%3A%22Runtime+Comparison+%28seconds%29%22%7D%7D%7D) - - -#### Validated/Verified Findings Chart -*Validated/Verified counts are reported where available* - -![Findings Comparison](https://quickchart.io/chart?c=%7B%22type%22%3A%22bar%22%2C%22data%22%3A%7B%22labels%22%3A%5B%22croc%22%2C%22rails%22%2C%22ruby%22%2C%22gitlab%22%2C%22django%22%2C%22lucene%22%2C%22mongodb%22%2C%22linux%22%2C%22typescript%22%5D%2C%22datasets%22%3A%5B%7B%22label%22%3A%22detect-secrets%22%2C%22data%22%3A%5B0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%5D%7D%2C%7B%22label%22%3A%22Kingfisher%22%2C%22data%22%3A%5B0%2C0%2C0%2C6%2C0%2C0%2C0%2C0%2C0%5D%7D%2C%7B%22label%22%3A%22TruffleHog%22%2C%22data%22%3A%5B0%2C0%2C0%2C6%2C0%2C0%2C0%2C0%2C0%5D%7D%2C%7B%22label%22%3A%22GitLeaks%22%2C%22data%22%3A%5B0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%2C0%5D%7D%5D%7D%2C%22options%22%3A%7B%22scales%22%3A%7B%22yAxes%22%3A%5B%7B%22ticks%22%3A%7B%22beginAtZero%22%3Atrue%7D%7D%5D%7D%2C%22title%22%3A%7B%22display%22%3A%22true%22%2C%22text%22%3A%22Validated%2FVerified+Findings%22%7D%7D%7D) - *Lower runtimes are better. Validated/Verified counts are reported where available. 'Network Requests' indicates the number of HTTP requests made during scanning.* diff --git a/docs/RULES.md b/docs/RULES.md index 01ae9ec..4a3dd8d 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -38,6 +38,15 @@ rules: - rule_id: kingfisher.aws.id variable: AKID # referenced as {{ AKID }} + pattern_requirements: # (optional) character/word requirements + min_digits: 1 # require at least 1 digit + min_uppercase: 1 # require at least 1 uppercase letter + min_lowercase: 1 # require at least 1 lowercase letter + min_special_chars: 1 # require at least 1 special character + special_chars: "!@#$%^&*()" # optional: custom special character set + ignore_if_contains: # optional: drop matches containing these words + - test + validation: # (optional) live validation type: Http content: @@ -61,17 +70,18 @@ rules: - type: JsonValid ``` -| Field | What it does | -| ----------------- | -------------------------------------------------------------------- | -| name | Friendly name shown in reports | -| id | Unique text ID (namespace.v#) used internally | -| pattern | Regex used to spot secrets (free‑spacing & flags allowed) | -| min_entropy | Threshold to guard against low‑complexity false positives | -| confidence | Suggests severity: low → high | -| examples | Good matches; used for testing | -| visible | false to hide non‑secret captures (e.g. IDs) | -| depends_on_rule | Chain rules: use captures from one rule in another’s validation | -| validation | Configure HTTP, AWS, GCP, etc. checks to verify live validity | +| Field | What it does | +| ----------------------- | -------------------------------------------------------------------- | +| name | Friendly name shown in reports | +| id | Unique text ID (namespace.v#) used internally | +| pattern | Regex used to spot secrets (free‑spacing & flags allowed) | +| min_entropy | Threshold to guard against low‑complexity false positives | +| confidence | Suggests severity: low → high | +| examples | Good matches; used for testing | +| visible | false to hide non‑secret captures (e.g. IDs) | +| depends_on_rule | Chain rules: use captures from one rule in another's validation | +| pattern_requirements | Require character types and/or exclude placeholder words from matches | +| validation | Configure HTTP, AWS, GCP, etc. checks to verify live validity | *responser_matcher* variants. Multiple can be used @@ -107,11 +117,19 @@ Below is the complete list of Liquid filters available in Kingfisher, along with | --------------------- | -------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------- | | `b64enc` | – | Base64-encodes the input using the standard alphabet. | `{{ TOKEN \| b64enc }}` | | `b64url_enc` | – | URL-safe Base64 (no padding). Useful for JWT headers & payloads. | `{{ TOKEN \| b64url_enc }}` | +| `b64dec` | – | Decodes a Base64 string. | `{{ "aGVsbG8=" \| b64dec }}` | | `sha256` | – | Computes the SHA-256 hex digest of the input. | `{{ TOKEN \| sha256 }}` | +| `crc32` | – | Computes the CRC32 checksum of the input and returns a decimal value. | `{{ TOKEN \| crc32 }}` | +| `crc32_dec` | `digits` (integer, optional) | Computes the CRC32 checksum and returns the last `digits` decimal characters (zero-padded). Defaults to the full value when omitted. | `{{ TOKEN \| crc32_dec: 6 }}` | +| `crc32_hex` | `digits` (integer, optional) | Computes the CRC32 checksum and returns the last `digits` hexadecimal characters (zero-padded). Defaults to the full value when omitted. | `{{ TOKEN \| crc32_hex: 8 }}` | +| `crc32_le_b64` | `len` (integer, optional) | Computes the CRC32 checksum, encodes the little-endian bytes using Base64, and optionally truncates to the first `len` characters. | `{{ TOKEN \| crc32_le_b64: 6 }}` | | `hmac_sha1` | `key` (string) | Computes HMAC-SHA1 over the input, returns Base64-encoded result. | `{{ TOKEN \| hmac_sha1: "secret-key" }}` | | `hmac_sha256` | `key` (string) | Computes HMAC-SHA256 over the input, returns Base64-encoded result. | `{{ TOKEN \| hmac_sha256: "secret-key" }}` | | `hmac_sha384` | `key` (string) | Computes HMAC-SHA384 over the input, returns Base64-encoded result. | `{{ TOKEN \| hmac_sha384: "secret-key" }}` | | `random_string` | `len` (integer, optional) | Generates a cryptographically-secure random alphanumeric string of the specified length (default: 32). | `{{ "" \| random_string: 16 }}` | +| `prefix` | `len` (integer, optional) | Returns the first `len` characters from the string (default: full). | `{{ TOKEN \| prefix: 6 }}` | +| `suffix` | `len` (integer, optional) | Returns the last `len` characters from the string (default: full). | `{{ TOKEN \| suffix: 6 }}` | +| `base62` | `width` (integer, optional) | Encodes the input number as Base62, left-padding with zeros as needed. | `{{ TOKEN \| crc32 \| base62: 6 }}` | | `url_encode` | – | Percent-encodes the input according to RFC 3986. | `{{ TOKEN \| url_encode }}` | | `json_escape` | – | Escapes special characters so a string can be safely injected into JSON contexts. | `{{ TOKEN \| json_escape }}` | | `unix_timestamp` | – | Returns the current Unix epoch time in seconds (UTC). | `{{ "" \| unix_timestamp }}` | @@ -236,6 +254,138 @@ For example, a rule might match a username, an email address, an AWS Access Key `visible: false` helps keep the scan output focused on actual secrets while still capturing important contextual data needed for comprehensive validation. +## Character Requirements + +The `pattern_requirements` field allows you to specify data type requirements for matched secrets. This is particularly useful when: + +- Your regex pattern must be permissive (due to Hyperscan limitations) +- You want to enforce password complexity requirements +- You need to filter out low-quality matches that lack certain character types + +Kingfisher's regex engine (Hyperscan) does not support lookahead assertions like `(?=.*\d)` to require specific character types. Instead, use the `pattern_requirements` field to filter matches post-detection. + +### Available Requirements + +```yaml +pattern_requirements: + min_digits: 1 # Require at least 1 digit (0-9) + min_uppercase: 1 # Require at least 1 uppercase letter (A-Z) + min_lowercase: 1 # Require at least 1 lowercase letter (a-z) + min_special_chars: 1 # Require at least 1 special character + special_chars: "!@#$%^&*" # Optional: define which characters are "special" + ignore_if_contains: # Optional: reject matches containing any of these (case-insensitive) + - test + - demo + checksum: # Optional: compare rendered values to drop mismatched formats + actual: + template: "{{ MATCH | suffix: 6 }}" # Liquid template for the observed checksum + requires_capture: checksum # (optional) skip unless this capture is present + expected: "{{ BODY | crc32 | base62: 6 }}" # Liquid template to render the expected checksum + skip_if_missing: true # (optional) treat missing captures as legacy tokens +``` + +All fields are optional. If `special_chars` is not specified, the default set includes: `!@#$%^&*()_+-=[]{}|;:'",.<>?/\`~` + +`ignore_if_contains` performs a case-insensitive substring check. If any entry (after trimming whitespace) appears within the match, the match is discarded. This is helpful for dropping known dummy tokens such as "test" or "demo" that otherwise satisfy the regex. + +The optional `checksum` block renders Liquid templates against the match to determine whether the captured checksum matches your expectation. Both templates gain access to `{{ MATCH }}`, `{{ FULL_MATCH }}`, and every named capture in two forms: the original capture name and its uppercase alias (e.g. `{{ body }}` and `{{ BODY }}`). Use helper filters like `suffix`, `crc32`, and `base62` to mirror provider-specific checksum pipelines. If a required capture is missing or the rendered values differ, Kingfisher skips the finding—logging the reason, including checksum lengths, at the `DEBUG` level. Set `skip_if_missing` to `true` to treat absent captures as legacy matches. + +When any of these filters remove a match it is logged at the `DEBUG` level so you can see exactly why the skip occurred. If you need to keep every match even when one of these substrings appears, pass `--no-ignore-if-contains` to `kingfisher scan`. The flag disables this post-processing step without changing the rule definitions. + +### Are `requires_capture` and `skip_if_missing` equivalent? + +`requires_capture` + - Optional field that names a specific regex capture that must be present before the checksum templates are evaluated. + - In the engine, Kingfisher checks whether that capture exists in the match context. If it’s missing, the behavior falls back to whatever `skip_if_missing` dictates (fail or treat as a legacy match). + +`skip_if_missing` + - Boolean switch that controls what happens when Kingfisher can’t render the checksum—because there’s no match context or a required capture is absent. + - `true`: silently skip (pass) the match so legacy, non-checksum tokens are still accepted. + - `false`: treat the situation as a validation failure. + +In short, `requires_capture` identifies which capture must exist, while `skip_if_missing` determines whether missing data is a hard failure or an allowed legacy case. + +### Example: Secure API Key + +```yaml +rules: + - name: Secure API Key + id: custom.secure_api.1 + pattern: | + (?xi) + api[_-]?key + (?:.|[\n\r]){0,32}? + \b + ([A-Za-z0-9!@#$%^&*]{20,}) + \b + min_entropy: 4.0 + confidence: high + pattern_requirements: + min_digits: 1 # Must contain at least 1 digit + min_uppercase: 1 # Must contain at least 1 uppercase letter + min_lowercase: 1 # Must contain at least 1 lowercase letter + min_special_chars: 1 # Must contain at least 1 special character + ignore_if_contains: + - test + examples: + - api_key = "MyS3cur3K3y!2024" + - 'api-key: "Abc123!@#Token"' +``` + +In this example: +- The regex pattern is permissive: `[A-Za-z0-9!@#$%^&*]{20,}` matches any combination of those characters +- The `pattern_requirements` filters out matches that don't have at least one of each required type +- A match like `"abcdefghijklmnopqrst"` would be rejected (no uppercase, no digit, no special) +- A match like `"Abc123!SecureToken"` would be accepted (has all required types) +- A match like `"Test123!SecureToken"` would be rejected because it contains the `ignore_if_contains` term `test` + +### Example: Excluding Dummy Values + +```yaml +rules: + - name: Token without placeholders + id: custom.token.2 + pattern: |- + (?i)token[:=]\s*([A-Za-z0-9]{12,}) + pattern_requirements: + ignore_if_contains: + - placeholder + - sample + examples: + - token: "REALVALUE1234" + negative_examples: + - token = "SAMPLETOKEN9999" # dropped by ignore_if_contains +``` + +### Example: Custom Special Characters + +```yaml +rules: + - name: Token with Custom Special Chars + id: custom.token.1 + pattern: | + (?xi) + token + (?:.|[\n\r]){0,16}? + \b([A-Za-z0-9$%^]{16,})\b + min_entropy: 3.5 + confidence: medium + pattern_requirements: + min_special_chars: 2 + special_chars: "$%^" # Only these characters count as "special" + examples: + - token = "abc$%defgh123456" +``` + +### How It Works + +1. Hyperscan regex matches a pattern in the input +2. Entropy check filters low-complexity matches (if `min_entropy` is set) +3. **Character requirements check filters matches that don't meet the criteria** +4. Validation checks verify the secret is live (if `validation` is configured) + +Matches that fail the character requirements check are silently dropped with a debug log message. + ## Writing Custom Rules diff --git a/docs/access-map-viewer/app.js b/docs/access-map-viewer/app.js new file mode 100644 index 0000000..d49f4f3 --- /dev/null +++ b/docs/access-map-viewer/app.js @@ -0,0 +1,574 @@ +let rawData = null; +let findings = []; +let accessMap = []; +let currentFilter = ""; +let validationFilter = "all"; +let pageSize = 10; +let currentPage = 1; +let sortField = "rule"; +let sortDirection = "asc"; + +const dropZone = document.getElementById("drop-zone"); +const fileInput = document.getElementById("file-input"); +const loader = document.getElementById("loader"); +const loaderText = document.getElementById("loader-text"); +const errorMsg = document.getElementById("error-msg"); +const uploadSection = document.getElementById("upload-section"); +const dashboard = document.getElementById("dashboard"); + +const searchInput = document.getElementById("search-input"); +const validationSelect = document.getElementById("validation-filter"); +const rowsSelect = document.getElementById("rows-select"); +const pagePrev = document.getElementById("page-prev"); +const pageNext = document.getElementById("page-next"); +const pageInfo = document.getElementById("page-info"); +const findingsBody = document.getElementById("findings-body"); + +const resetButton = document.getElementById("reset-btn"); + +const treeSearch = document.getElementById("tree-search"); +const amContainer = document.getElementById("am-container"); +const amToggle = document.getElementById("am-toggle"); + +dropZone.addEventListener("click", () => fileInput.click()); +fileInput.addEventListener("change", (e) => { + if (e.target.files.length) processFile(e.target.files[0]); +}); +dropZone.addEventListener("dragover", (e) => { + e.preventDefault(); + dropZone.classList.add("active"); +}); +dropZone.addEventListener("dragleave", (e) => { + e.preventDefault(); + dropZone.classList.remove("active"); +}); +dropZone.addEventListener("drop", (e) => { + e.preventDefault(); + dropZone.classList.remove("active"); + if (e.dataTransfer.files.length) processFile(e.dataTransfer.files[0]); +}); + +if (resetButton) { + resetButton.addEventListener("click", () => { + rawData = null; + findings = []; + accessMap = []; + currentFilter = ""; + validationFilter = "all"; + pageSize = 10; + currentPage = 1; + sortField = "rule"; + sortDirection = "asc"; + + searchInput.value = ""; + validationSelect.value = "all"; + rowsSelect.value = "10"; + treeSearch.value = ""; + amContainer.classList.remove("hidden"); + amToggle.textContent = "Collapse"; + + findingsBody.innerHTML = ""; + document.getElementById("access-tree").innerHTML = + '
No access map data found in report.
'; + + resetError(); + uploadSection.classList.remove("hidden"); + dashboard.classList.add("hidden"); + setLoading(false); + }); +} + +searchInput.addEventListener("input", () => { + currentFilter = searchInput.value.trim().toLowerCase(); + currentPage = 1; + renderTable(); +}); + +validationSelect.addEventListener("change", () => { + validationFilter = validationSelect.value; + currentPage = 1; + renderTable(); +}); + +rowsSelect.addEventListener("change", () => { + pageSize = parseInt(rowsSelect.value, 10); + currentPage = 1; + renderTable(); +}); + +pagePrev.addEventListener("click", () => { + if (currentPage > 1) { + currentPage--; + renderTable(); + } +}); + +pageNext.addEventListener("click", () => { + const totalPages = Math.max(1, Math.ceil(filteredFindings().length / pageSize)); + if (currentPage < totalPages) { + currentPage++; + renderTable(); + } +}); + +amToggle.addEventListener("click", () => { + amContainer.classList.toggle("hidden"); + amToggle.textContent = amContainer.classList.contains("hidden") ? "Show Access Map" : "Hide Access Map"; +}); + +function processFile(file) { + resetError(); + setLoading(true, `Reading ${file.name}...`); + + const reader = new FileReader(); + reader.onload = (e) => { + try { + rawData = e.target.result; + parseData(rawData); + } catch (err) { + setError(`Failed to read file: ${err.message}`); + } finally { + setLoading(false); + } + }; + reader.onerror = () => { + setError("Error reading file. Please try again."); + setLoading(false); + }; + reader.readAsText(file); +} + +async function loadEmbeddedReport() { + try { + setLoading(true, "Loading CLI report..."); + const response = await fetch("/report", { cache: "no-store" }); + + if (response.status === 404) { + setLoading(false); + return; + } + + if (!response.ok) { + throw new Error(`server returned ${response.status}`); + } + + const text = await response.text(); + rawData = text; + await parseData(text); + } catch (err) { + setLoading(false); + setError(`Failed to load report from CLI: ${err.message}`); + } +} + +async function parseData(text) { + const parsed = parsePayload(text); + findings = parsed.findings.map(normalizeFinding); + accessMap = flattenAccessMap(normalizeAccessMap(parsed.access_map)); + + renderStats(); + buildAccessTree(accessMap); + renderTable(); + showDashboard(); +} + +function parsePayload(text) { + try { + const parsed = JSON.parse(text); + return collectReportData(parsed); + } catch (_) { + return collectReportDataFromJsonl(text); + } +} + +function collectReportData(root) { + const findings = []; + const accessMap = []; + + const visit = (node) => { + if (node === null || node === undefined) return; + + if (Array.isArray(node)) { + for (const item of node) { + visit(item); + } + return; + } + + if (typeof node !== "object") return; + + if (node.rule && node.finding) { + findings.push(node); + } + + if (Array.isArray(node.findings)) { + findings.push(...node.findings); + } + + if (Array.isArray(node.access_map)) { + accessMap.push(...node.access_map); + } + + Object.values(node).forEach(visit); + }; + + visit(root); + + if (!findings.length && Array.isArray(root)) { + findings.push(...root); + } + + return { findings, access_map: accessMap }; +} + +function collectReportDataFromJsonl(text) { + const findings = []; + const accessMap = []; + const lines = text.split(/\r?\n/).filter(Boolean); + + for (const line of lines) { + try { + const obj = JSON.parse(line); + const parsed = collectReportData(obj); + findings.push(...parsed.findings); + accessMap.push(...parsed.access_map); + } catch (_) { + /* ignore invalid lines */ + } + } + + return { findings, access_map: accessMap }; +} + +function normalizeFinding(row) { + const validation = row.finding?.validation || row.validation || {}; + return { + ruleId: `${row.rule?.id ?? ""}`, + ruleName: `${row.rule?.name ?? ""}`, + findingType: `${row.finding?.type ?? row.finding?.category ?? ""}`, + severity: `${row.finding?.severity ?? row.severity ?? ""}`, + message: `${row.finding?.message ?? row.finding?.snippet ?? ""}`, + path: `${row.finding?.path ?? row.path ?? ""}`, + line: `${row.finding?.line ?? row.finding?.start?.line ?? ""}`, + validationStatus: `${validation.status ?? ""}`, + validationConfidence: `${validation.confidence ?? ""}`, + validationResponse: `${validation.response ?? ""}`, + confidence: `${row.finding?.confidence ?? ""}`, + snippet: `${row.finding?.snippet ?? ""}`, + fingerprint: `${row.finding?.fingerprint ?? ""}`, + raw: row, + }; +} + +function normalizeAccessMap(entries = []) { + if (!Array.isArray(entries)) return []; + + if (entries.some((entry) => Array.isArray(entry.groups))) { + return entries.map((entry) => ({ + provider: entry.provider, + account: entry.account, + fingerprint: entry.fingerprint, + groups: (entry.groups || []).map((group) => ({ + resources: Array.isArray(group.resources) ? group.resources : [], + permissions: Array.isArray(group.permissions) ? group.permissions : [], + })), + })); + } + + return entries.map((entry) => ({ + provider: entry.provider, + account: entry.account, + fingerprint: entry.fingerprint, + groups: [ + { + resources: entry.resource ? [entry.resource] : [], + permissions: Array.isArray(entry.permissions) + ? entry.permissions + : entry.permission + ? String(entry.permission) + .split(",") + .map((p) => p.trim()) + .filter(Boolean) + : [], + }, + ], + })); +} + +function flattenAccessMap(entries = []) { + const rows = []; + entries.forEach((entry) => { + (entry.groups || []).forEach((group) => { + (group.resources || []).forEach((resource) => { + rows.push({ + provider: entry.provider, + account: entry.account, + fingerprint: entry.fingerprint, + resource, + permissions: group.permissions || [], + }); + }); + }); + }); + return rows; +} + +function renderStats() { + const totalFindings = findings.length; + const criticalCount = findings.filter((f) => f.severity.toLowerCase() === "critical").length; + const highCount = findings.filter((f) => f.severity.toLowerCase() === "high").length; + const mediumCount = findings.filter((f) => f.severity.toLowerCase() === "medium").length; + const validatedCount = findings.filter((f) => !!f.validationStatus).length; + + document.getElementById("stat-findings").textContent = totalFindings; + document.getElementById("stat-critical").textContent = criticalCount; + document.getElementById("stat-high").textContent = highCount; + document.getElementById("stat-medium").textContent = mediumCount; + document.getElementById("stat-validated").textContent = validatedCount; + document.getElementById("stat-access-map").textContent = accessMap.length; +} + +function renderTable() { + const rows = filteredFindings(); + + const totalPages = Math.max(1, Math.ceil(rows.length / pageSize)); + currentPage = Math.min(currentPage, totalPages); + + pageInfo.textContent = `Page ${currentPage} of ${totalPages}`; + pagePrev.disabled = currentPage === 1; + pageNext.disabled = currentPage === totalPages; + + const start = (currentPage - 1) * pageSize; + const end = start + pageSize; + const pageRows = rows.slice(start, end); + + findingsBody.innerHTML = pageRows + .map((f) => ` + + ${escapeHtml(f.ruleId)} + ${escapeHtml(f.ruleName)} + ${escapeHtml(f.findingType)} + ${escapeHtml(f.severity)} + ${escapeHtml(f.message)} + ${escapeHtml(f.path)} + ${escapeHtml(f.line)} + ${escapeHtml(f.validationStatus)} + ${escapeHtml(f.validationConfidence)} + + `) + .join(""); +} + +function filteredFindings() { + return findings + .filter((f) => { + if (validationFilter === "active" && f.validationStatus.toLowerCase() !== "active credential") return false; + if (validationFilter === "inactive" && f.validationStatus.toLowerCase() !== "inactive credential") return false; + if (validationFilter === "not_attempted" && f.validationStatus.toLowerCase() !== "not attempted") return false; + + if (!currentFilter) return true; + const haystack = `${f.ruleId} ${f.ruleName} ${f.findingType} ${f.message} ${f.path} ${f.validationStatus} ${f.fingerprint}`.toLowerCase(); + return haystack.includes(currentFilter); + }) + .sort((a, b) => { + const av = getSortValue(a, sortField); + const bv = getSortValue(b, sortField); + if (av === bv) return 0; + return sortDirection === "asc" ? (av > bv ? 1 : -1) : av < bv ? 1 : -1; + }); +} + +function getSortValue(obj, field) { + switch (field) { + case "rule": + return `${obj.ruleId}`.toLowerCase(); + case "location": + return `${obj.path}`.toLowerCase(); + case "severity": + return `${obj.severity}`.toLowerCase(); + case "validation": + return `${obj.validationStatus}`.toLowerCase(); + case "confidence": + return `${obj.confidence}`.toLowerCase(); + case "line": + return `${obj.line}`.toLowerCase(); + default: + return `${obj.path}`.toLowerCase(); + } +} + +document.querySelectorAll("th.sortable").forEach((th) => { + th.addEventListener("click", () => { + const field = th.dataset.sort; + if (sortField === field) { + sortDirection = sortDirection === "asc" ? "desc" : "asc"; + } else { + sortField = field; + sortDirection = "asc"; + } + document + .querySelectorAll("th.sortable") + .forEach((el) => el.classList.toggle("sorted", el.dataset.sort === sortField)); + renderTable(); + }); +}); + +treeSearch.addEventListener("input", () => buildAccessTree(accessMap)); + +function buildAccessTree(entries) { + const search = treeSearch.value.trim().toLowerCase(); + const tree = document.getElementById("access-tree"); + tree.innerHTML = ""; + + if (!entries.length) { + tree.innerHTML = "

No access map entries.

"; + return; + } + + const filtered = entries.filter((entry) => + [entry.provider, entry.account, entry.resource, ...(entry.permissions || [])] + .join(" ") + .toLowerCase() + .includes(search) + ); + + const grouped = {}; + for (const entry of filtered) { + const provider = entry.provider || "Unknown"; + const account = entry.account || "Unknown"; + grouped[provider] = grouped[provider] || {}; + grouped[provider][account] = grouped[provider][account] || []; + grouped[provider][account].push(entry); + } + + Object.entries(grouped).forEach(([provider, accounts]) => { + const providerEl = document.createElement("div"); + providerEl.className = "tree-node"; + providerEl.innerHTML = `
${escapeHtml(provider)}
`; + + Object.entries(accounts).forEach(([account, resources]) => { + const accountEl = document.createElement("div"); + accountEl.className = "tree-node tree-node--child"; + accountEl.innerHTML = `
${escapeHtml(account)}
`; + + resources.forEach((r) => { + const resEl = document.createElement("div"); + resEl.className = "tree-node tree-node--grandchild"; + resEl.innerHTML = ` +
${escapeHtml(r.resource || "(resource)")}
+
${escapeHtml((r.permissions || []).join(", "))}
+ `; + accountEl.appendChild(resEl); + }); + + providerEl.appendChild(accountEl); + }); + + tree.appendChild(providerEl); + }); +} + +function setLoading(enabled, message = "Loading...") { + loader.classList.toggle("hidden", !enabled); + loaderText.textContent = message; +} + +function setError(message) { + errorMsg.textContent = message; + errorMsg.classList.remove("hidden"); + uploadSection.classList.add("error"); +} + +function resetError() { + errorMsg.textContent = ""; + errorMsg.classList.add("hidden"); + uploadSection.classList.remove("error"); +} + +function showDashboard() { + uploadSection.classList.add("hidden"); + dashboard.classList.remove("hidden"); + setLoading(false); +} + +function escapeHtml(str) { + return (str || "") + .toString() + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); +} + +function downloadJson() { + if (!rawData) return; + const blob = new Blob([rawData], { type: "application/json" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = "access-map-report.json"; + a.click(); + URL.revokeObjectURL(url); +} + +function copyAccessMap() { + if (!accessMap.length) return; + const text = JSON.stringify(accessMap, null, 2); + navigator.clipboard.writeText(text).catch(() => { }); +} + +function exportCsv() { + const rows = filteredFindings(); + if (!rows.length) return; + + const header = [ + "rule_id", + "rule_name", + "finding_type", + "severity", + "message", + "path", + "line", + "validation_status", + "validation_confidence", + ]; + + const csv = [header.join(",")] + .concat( + rows.map((f) => + [ + f.ruleId, + f.ruleName, + f.findingType, + f.severity, + escapeCsv(f.message), + f.path, + f.line, + f.validationStatus, + f.validationConfidence, + ].join(",") + ) + ) + .join("\n"); + + const blob = new Blob([csv], { type: "text/csv" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = "access-map-findings.csv"; + a.click(); + URL.revokeObjectURL(url); +} + +function escapeCsv(value) { + const str = value.replace(/"/g, '""'); + return `"${str}"`; +} + +document.getElementById("download-json").addEventListener("click", downloadJson); +document.getElementById("download-csv").addEventListener("click", exportCsv); +document.getElementById("copy-access-map").addEventListener("click", copyAccessMap); + +loadEmbeddedReport(); + diff --git a/docs/access-map-viewer/index.html b/docs/access-map-viewer/index.html new file mode 100644 index 0000000..c6c61fa --- /dev/null +++ b/docs/access-map-viewer/index.html @@ -0,0 +1,2951 @@ + + + + + + Kingfisher Access Map Viewer + + + + + + +
+
+
K
+
+ Kingfisher Viewer + Access Map & Findings +
+
+
+ + +
+
+ +
Client-Side Only. No data is uploaded anywhere.
+ +
+
+
+
+

Upload Report

+

Analyze Kingfisher JSON / JSONL output (auto-loads if provided on the CLI)

+
+
+
+
+
📄
+
Drag & drop a report here
+
…or click to choose a file
+
Your file stays in the browser—load JSON or JSONL reports locally.
+ +
+ +
+
+ + +
+ + + + \ No newline at end of file diff --git a/docs/access-map-viewer/sample-report.json b/docs/access-map-viewer/sample-report.json new file mode 100644 index 0000000..4818fcd --- /dev/null +++ b/docs/access-map-viewer/sample-report.json @@ -0,0 +1,97 @@ +{ + "findings": [ + { + "rule": { + "name": "Alibaba Access Key Secret", + "id": "kingfisher.alibabacloud.2" + }, + "finding": { + "snippet": "m0qx7h2v4n8c9t3b6p1r5w0kzsdjf", + "fingerprint": "13778709639383676952", + "confidence": "medium", + "entropy": "4.55", + "validation": { + "status": "Inactive Credential", + "response": "{\"RequestId\":\"48F0D2A0-7C0E-5DE2-BC89-39811315C04A\",\"Message\":\"Specified access key is not found.\",\"Recommend\":\"https://api.aliyun.com/troubleshoot?q=InvalidAccessKeyId.NotFound&product=Sts&requestId=48F0D2A0-7C0E-5DE2-BC89-39811315C04A\",\"HostId\":\"sts.aliyuncs.com\",\"Code\":\"InvalidAccessKeyId.NotFound\"}" + }, + "language": "Plain Text", + "line": 5, + "column_start": 0, + "column_end": 29, + "path": "/tmp/repo/tmp/secretstuff/alibaba-test.txt" + } + }, + { + "rule": { + "name": "Alibaba Access Key Secret", + "id": "kingfisher.alibabacloud.2" + }, + "finding": { + "snippet": "z91trw6fap8kq2xmd4s7h3b0vnclpf", + "fingerprint": "8292190854848911527", + "confidence": "medium", + "entropy": "4.44", + "validation": { + "status": "Inactive Credential", + "response": "Validation skipped - missing dependent rules: kingfisher.alibabacloud.1, kingfisher.alibabacloud.1" + }, + "language": "Unknown", + "line": 8, + "column_start": 39, + "column_end": 68, + "path": "/tmp/repo/tmp/secretstuff/alibaba/alibaba-validator/.venv/lib/python3.13/site-packages/alibabacloud_tea_util-0.3.13.dist-info/RECORD" + } + }, + { + "rule": { + "name": "AWS Secret Access Key", + "id": "kingfisher.aws.2" + }, + "finding": { + "snippet": "dB9PyxlN/qa8sF0tJ4uM2qZr7eVw6TgHkC0nBpZq", + "fingerprint": "17034522315778178539", + "confidence": "medium", + "entropy": "4.67", + "validation": { + "status": "Active Credential", + "response": "AKIAFAKEKEY123456789 --- ARN: arn:aws:iam::000000000000:user/example_user --- AWS Account Number: 000000000000" + }, + "language": "Unknown", + "line": 1, + "column_start": 65, + "column_end": 104, + "path": "/tmp/repo/tmp/secretstuff/utf8.txt " + } + } + ], + "access_map": [ + { + "provider": "aws", + "account": "prod", + "groups": [ + { "resources": ["arn:aws:s3:::prod-bucket"], "permissions": ["s3:GetObject", "s3:ListBucket"] }, + { "resources": ["arn:aws:iam::123456789012:role/Admin"], "permissions": ["iam:AssumeRole"] } + ] + }, + { + "provider": "gcp", + "account": "test-project", + "groups": [ + { "resources": ["projects/test/instances/primary"], "permissions": ["compute.instances.get", "compute.instances.list"] } + ] + } + ], + "stats": { + "total": 259, + "critical": 37, + "validated": 167, + "unique_paths": 21, + "confidence_buckets": { + "High": 37, + "Medium": 222 + }, + "confidence_order": ["High", "Medium"], + "scan_date": "2025-11-25T15:37:41.863868-08:00", + "kingfisher_version": "1.68.0" + } +} diff --git a/docs/access-map-viewer/viewer.css b/docs/access-map-viewer/viewer.css new file mode 100644 index 0000000..0b4a82f --- /dev/null +++ b/docs/access-map-viewer/viewer.css @@ -0,0 +1,106 @@ +:root { + --bg: #0f172a; + --panel: #111827; + --text: #e5e7eb; + --muted: #9ca3af; + --accent: #38bdf8; + --border: #1f2937; + --good: #34d399; + --warn: #f59e0b; + font-family: "Inter", system-ui, -apple-system, sans-serif; +} + +* { box-sizing: border-box; } + +body { + margin: 0; + background: radial-gradient(circle at 20% 20%, rgba(56,189,248,0.08), transparent 25%), + radial-gradient(circle at 80% 0%, rgba(52,211,153,0.12), transparent 30%), + var(--bg); + color: var(--text); + min-height: 100vh; + padding: 24px; +} + +header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + margin-bottom: 18px; +} + +h1 { margin: 0; font-size: 22px; } +h1 span { color: var(--accent); } + +.page { + display: grid; + grid-template-columns: 1fr 1.2fr; + gap: 16px; +} + +.panel { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 14px; + padding: 16px; + box-shadow: 0 15px 30px rgba(0,0,0,0.35); +} + +.controls { display: flex; gap: 10px; align-items: center; flex-wrap: wrap; } +input[type="file"] { display: none; } + +.btn { + border: 1px solid var(--border); + background: rgba(255,255,255,0.04); + color: var(--text); + padding: 10px 14px; + border-radius: 10px; + cursor: pointer; + font-weight: 700; +} + +.btn-primary { + background: linear-gradient(135deg, #0ea5e9, #22d3ee); + color: #0b1224; + border: none; + box-shadow: 0 12px 28px rgba(56,189,248,0.35); +} + +.stat-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(140px,1fr)); + gap: 10px; +} + +.stat { + background: rgba(255,255,255,0.02); + border: 1px solid var(--border); + padding: 12px; + border-radius: 12px; +} + +.stat-label { color: var(--muted); font-size: 13px; } +.stat-value { font-size: 24px; font-weight: 800; } + +.table { width: 100%; border-collapse: collapse; margin-top: 10px; } +.table th, .table td { padding: 8px; border-bottom: 1px solid var(--border); text-align: left; } +.table th { color: var(--muted); font-weight: 700; } +.table tbody tr:hover { background: rgba(255,255,255,0.03); } + +.badge { padding: 4px 8px; border-radius: 999px; font-weight: 700; font-size: 12px; } +.badge-good { background: rgba(52,211,153,0.16); color: #6ee7b7; } +.badge-warn { background: rgba(245,158,11,0.15); color: #fbbf24; } + +pre { + background: rgba(255,255,255,0.02); + border: 1px dashed var(--border); + padding: 12px; + border-radius: 12px; + color: var(--muted); + overflow: auto; +} + +@media (max-width: 1000px) { + .page { grid-template-columns: 1fr; } +} diff --git a/docs/access-map-viewer/viewer.js b/docs/access-map-viewer/viewer.js new file mode 100644 index 0000000..5d97ed7 --- /dev/null +++ b/docs/access-map-viewer/viewer.js @@ -0,0 +1,195 @@ +const state = { + findings: [], + accessMap: [], +}; + +const fileInput = document.getElementById('file'); +const uploadBtn = document.getElementById('upload-btn'); +const sampleBtn = document.getElementById('sample-btn'); +const stats = { + findings: document.getElementById('stat-findings'), + access: document.getElementById('stat-access'), + providers: document.getElementById('stat-providers'), +}; +const findingsTable = document.getElementById('findings'); +const accessTable = document.getElementById('access-map'); +const payloadPreview = document.getElementById('payload-preview'); + +uploadBtn.addEventListener('click', () => fileInput.click()); +fileInput.addEventListener('change', () => { + if (fileInput.files?.[0]) { + loadFile(fileInput.files[0]); + } +}); + +sampleBtn.addEventListener('click', async () => { + const resp = await fetch('sample-report.json'); + const data = await resp.json(); + render(normalizePayload(data)); +}); + +async function loadFile(file) { + const text = await file.text(); + render(parseReport(text)); +} + +function parseReport(text) { + try { + const parsed = JSON.parse(text); + return normalizePayload(parsed); + } catch (_) { + return parseJsonl(text); + } +} + +function parseJsonl(text) { + const lines = text.split(/\r?\n/).filter(Boolean); + const findings = []; + let accessMap = []; + lines.forEach((line) => { + try { + const row = JSON.parse(line); + if (row.rule && row.finding) { + findings.push(row); + } + if (row.access_map) { + accessMap = accessMap.concat(row.access_map); + } + } catch (_) { + /* ignore */ + } + }); + return { findings, access_map: accessMap }; +} + +function normalizePayload(data) { + if (Array.isArray(data)) { + return { findings: data, access_map: [] }; + } + return { + findings: data.findings || [], + access_map: normalizeAccessMap(data.access_map || []), + }; +} + +function render(payload) { + state.findings = payload.findings || []; + state.accessMap = normalizeAccessMap(payload.access_map || []); + + const flattened = flattenAccessMap(state.accessMap); + + stats.findings.textContent = state.findings.length; + stats.access.textContent = flattened.length; + stats.providers.textContent = new Set(state.accessMap.map((e) => e.provider || '')).size; + + renderFindings(); + renderAccessMap(flattened); + payloadPreview.textContent = JSON.stringify({ ...payload, access_map: state.accessMap }, null, 2); +} + +function renderFindings() { + const tbody = findingsTable.querySelector('tbody'); + tbody.innerHTML = ''; + if (state.findings.length === 0) { + tbody.innerHTML = 'No findings yet.'; + return; + } + + state.findings.slice(0, 50).forEach((f) => { + const tr = document.createElement('tr'); + tr.innerHTML = ` + ${escapeHtml(f.rule?.name || '')} + ${escapeHtml(f.rule?.id || '')} + ${escapeHtml(f.finding?.path || '')} + ${escapeHtml( + f.finding?.confidence || '' + )} + `; + tbody.appendChild(tr); + }); +} + +function renderAccessMap(rows) { + const tbody = accessTable.querySelector('tbody'); + tbody.innerHTML = ''; + if (rows.length === 0) { + tbody.innerHTML = 'No access-map entries yet.'; + return; + } + + rows.forEach((row) => { + const tr = document.createElement('tr'); + tr.innerHTML = ` + ${escapeHtml(row.provider || '')} + ${escapeHtml(row.account || '')} + ${escapeHtml(row.resource || '')} + ${escapeHtml(row.permissions.join(', ') || '')} + `; + tbody.appendChild(tr); + }); +} + +function escapeHtml(str = '') { + const div = document.createElement('div'); + div.textContent = str; + return div.innerHTML; +} + +function classForConfidence(conf = '') { + const c = conf.toLowerCase(); + if (c === 'high') return 'badge-warn'; + if (c === 'medium') return 'badge'; + if (c === 'low') return 'badge-good'; + return 'badge'; +} + +function normalizeAccessMap(entries = []) { + if (!Array.isArray(entries)) return []; + + // Already in new schema + if (entries.some((e) => Array.isArray(e.groups))) { + return entries.map((entry) => ({ + provider: entry.provider, + account: entry.account, + groups: (entry.groups || []).map((group) => ({ + resources: Array.isArray(group.resources) ? group.resources : [], + permissions: Array.isArray(group.permissions) ? group.permissions : [], + })), + })); + } + + // Fallback for legacy flat entries + return entries.map((entry) => { + const permissions = Array.isArray(entry.permissions) + ? entry.permissions + : entry.permission + ? String(entry.permission) + .split(',') + .map((p) => p.trim()) + .filter(Boolean) + : []; + const resource = entry.resource ? [entry.resource] : []; + return { + provider: entry.provider, + account: entry.account, + groups: [{ resources: resource, permissions }], + }; + }); +} + +function flattenAccessMap(entries = []) { + const rows = []; + entries.forEach((entry) => { + (entry.groups || []).forEach((group) => { + (group.resources || []).forEach((resource) => { + rows.push({ + provider: entry.provider, + account: entry.account, + resource, + permissions: group.permissions || [], + }); + }); + }); + }); + return rows; +} diff --git a/docs/assets/icons/aws-s3.svg b/docs/assets/icons/aws-s3.svg new file mode 100644 index 0000000..3f63be5 --- /dev/null +++ b/docs/assets/icons/aws-s3.svg @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + diff --git a/docs/assets/icons/azure-devops.svg b/docs/assets/icons/azure-devops.svg new file mode 100644 index 0000000..d5db277 --- /dev/null +++ b/docs/assets/icons/azure-devops.svg @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/docs/assets/icons/bitbucket.svg b/docs/assets/icons/bitbucket.svg new file mode 100644 index 0000000..38af1ce --- /dev/null +++ b/docs/assets/icons/bitbucket.svg @@ -0,0 +1,15 @@ + + + + + + + + Bitbucket-blue + + + + + + + \ No newline at end of file diff --git a/docs/assets/icons/confluence.svg b/docs/assets/icons/confluence.svg new file mode 100644 index 0000000..22249e1 --- /dev/null +++ b/docs/assets/icons/confluence.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/icons/docker.svg b/docs/assets/icons/docker.svg new file mode 100644 index 0000000..0a9c6b0 --- /dev/null +++ b/docs/assets/icons/docker.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/icons/files.svg b/docs/assets/icons/files.svg new file mode 100644 index 0000000..1ebd008 --- /dev/null +++ b/docs/assets/icons/files.svg @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/assets/icons/gcs.svg b/docs/assets/icons/gcs.svg new file mode 100644 index 0000000..842c121 --- /dev/null +++ b/docs/assets/icons/gcs.svg @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/assets/icons/gitea.svg b/docs/assets/icons/gitea.svg new file mode 100644 index 0000000..7ed0012 --- /dev/null +++ b/docs/assets/icons/gitea.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/icons/github.svg b/docs/assets/icons/github.svg new file mode 100644 index 0000000..a8d1174 --- /dev/null +++ b/docs/assets/icons/github.svg @@ -0,0 +1,3 @@ + + + diff --git a/docs/assets/icons/gitlab.svg b/docs/assets/icons/gitlab.svg new file mode 100644 index 0000000..abe3f37 --- /dev/null +++ b/docs/assets/icons/gitlab.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/icons/huggingface.svg b/docs/assets/icons/huggingface.svg new file mode 100644 index 0000000..43711df --- /dev/null +++ b/docs/assets/icons/huggingface.svg @@ -0,0 +1,41 @@ + + + + + + + + + + + + diff --git a/docs/assets/icons/jira.svg b/docs/assets/icons/jira.svg new file mode 100644 index 0000000..57a68f0 --- /dev/null +++ b/docs/assets/icons/jira.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/icons/local-git.svg b/docs/assets/icons/local-git.svg new file mode 100644 index 0000000..994fb2c --- /dev/null +++ b/docs/assets/icons/local-git.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/assets/icons/slack.svg b/docs/assets/icons/slack.svg new file mode 100644 index 0000000..fb55f72 --- /dev/null +++ b/docs/assets/icons/slack.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/docs/demos/.gitignore b/docs/demos/.gitignore new file mode 100644 index 0000000..d8d07cb --- /dev/null +++ b/docs/demos/.gitignore @@ -0,0 +1,142 @@ +### Node ### +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Diagnostic reports (https://nodejs.org/api/report.html) +report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage +*.lcov + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) +web_modules/ + +# TypeScript cache +*.tsbuildinfo + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional stylelint cache +.stylelintcache + +# Microbundle cache +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) +.cache +.parcel-cache + +# Next.js build output +.next +out + +# Nuxt.js build / generate output +.nuxt +dist + +# Gatsby files +.cache/ +# Comment in the public line in if your project uses Gatsby and not Next.js +# https://nextjs.org/blog/next-9-1#public-directory-support +# public + +# vuepress build output +.vuepress/dist + +# vuepress v2.x temp and cache directory +.temp + +# Docusaurus cache and generated files +.docusaurus + +# Serverless directories +.serverless/ + +# FuseBox cache +.fusebox/ + +# DynamoDB Local files +.dynamodb/ + +# TernJS port file +.tern-port + +# Stores VSCode versions used for testing VSCode extensions +.vscode-test + +# yarn v2 +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +### Node Patch ### +# Serverless Webpack directories +.webpack/ + +# Optional stylelint cache + +# SvelteKit build / generate output +.svelte-kit + + +pw-out/* diff --git a/docs/demos/findings-thumbnail.png b/docs/demos/findings-thumbnail.png new file mode 100644 index 0000000..574f04a Binary files /dev/null and b/docs/demos/findings-thumbnail.png differ diff --git a/docs/demos/kingfisher-usage-01.tape b/docs/demos/kingfisher-usage-01.tape new file mode 100644 index 0000000..1dadb55 --- /dev/null +++ b/docs/demos/kingfisher-usage-01.tape @@ -0,0 +1,18 @@ +Output ../kingfisher-usage-01.gif +Set Width 1200 +Set Height 700 +Set FontSize 16 +Set TypingSpeed 60ms +Set Framerate 60 +Set PlaybackSpeed 1.0 + +Type "kingfisher scan ~/tmp/secretsdemo | less -R" +Enter + +Wait+Screen@30s /(report|findings|summary|kingfisher)/ + +Sleep 1200ms +Space@1000ms 8 +Sleep 800ms +Sleep 1200ms +Type "q" diff --git a/docs/demos/kingfisher-usage-access-map-01.tape b/docs/demos/kingfisher-usage-access-map-01.tape new file mode 100644 index 0000000..61e38a0 --- /dev/null +++ b/docs/demos/kingfisher-usage-access-map-01.tape @@ -0,0 +1,14 @@ +Output ../kingfisher-usage-access-map-01.gif +Set Width 1200 +Set Height 700 +Set FontSize 16 +Set TypingSpeed 60ms +Set Framerate 60 +Set PlaybackSpeed 1.3 + +Type "kingfisher scan --git-url https://github.com/leaktk/fake-leaks.git --access-map --view-report" +Enter + +Wait+Screen@30s /(report|findings|summary|kingfisher)/ + +Sleep 14000ms diff --git a/docs/demos/make_demos.sh b/docs/demos/make_demos.sh new file mode 100755 index 0000000..8a1d9b4 --- /dev/null +++ b/docs/demos/make_demos.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +vhs ./kingfisher-usage-01.tape +vhs ./kingfisher-usage-access-map-01.tape +echo "Demos generated. Preparing browser recording in 5 seconds..." +sleep 5 +#npm i -D playwright +#npx playwright install +node ./record.mjs \ No newline at end of file diff --git a/docs/demos/record.mjs b/docs/demos/record.mjs new file mode 100644 index 0000000..5defc84 --- /dev/null +++ b/docs/demos/record.mjs @@ -0,0 +1,150 @@ +import { chromium } from "playwright"; +import fs from "node:fs"; + +const outDir = "pw-out"; +fs.mkdirSync(outDir, { recursive: true }); + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +const cursorOverlayScript = ` +(() => { + const style = document.createElement('style'); + style.textContent = \` + #__pw_cursor { + position: fixed; + top: 0; left: 0; + width: 18px; height: 18px; + transform: translate(-100px, -100px); + z-index: 2147483647; + pointer-events: none; + } + #__pw_cursor svg { width: 18px; height: 18px; } + #__pw_cursor .dot { + fill: rgba(255,255,255,0.9); + stroke: rgba(0,0,0,0.85); + stroke-width: 2; + } + #__pw_click { + position: fixed; + width: 8px; height: 8px; + border-radius: 50%; + transform: translate(-100px, -100px); + z-index: 2147483646; + pointer-events: none; + opacity: 0; + border: 2px solid rgba(0,0,0,0.6); + } + \`; + document.documentElement.appendChild(style); + + const cursor = document.createElement('div'); + cursor.id = '__pw_cursor'; + cursor.innerHTML = \` + + \`; + document.documentElement.appendChild(cursor); + + const click = document.createElement('div'); + click.id = '__pw_click'; + document.documentElement.appendChild(click); + + let x = -100, y = -100; + const move = (nx, ny) => { + x = nx; y = ny; + cursor.style.transform = \`translate(\${x}px, \${y}px)\`; + click.style.transform = \`translate(\${x}px, \${y}px)\`; + }; + + window.addEventListener('pointermove', (e) => move(e.clientX, e.clientY), { passive: true }); + window.addEventListener('mousemove', (e) => move(e.clientX, e.clientY), { passive: true }); + + window.addEventListener('pointerdown', () => { + click.style.transition = 'none'; + click.style.opacity = '0.9'; + click.style.width = '8px'; + click.style.height = '8px'; + requestAnimationFrame(() => { + click.style.transition = 'all 250ms ease-out'; + click.style.opacity = '0'; + click.style.width = '28px'; + click.style.height = '28px'; + }); + }, { passive: true }); +})(); +`; + +// Converts native