diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8cf2a45..cc46915 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -80,6 +80,8 @@ jobs: - name: Build Darwin x64 run: make darwin-x64 + - name: Run tests + run: make tests - name: Move artifacts to dist shell: bash @@ -109,6 +111,8 @@ jobs: - name: Build Darwin arm64 run: make darwin-arm64 + - name: Run tests + run: make tests - name: Move artifacts to dist shell: bash @@ -171,13 +175,12 @@ jobs: name: kingfisher-windows-x64 path: dist/kingfisher-*windows-x64*.* - # ──────────────── Draft public release ──────────────── release: name: Public GitHub Release - needs: [linux-x64, linux-arm64, windows, macos-x64, macos-arm64] # wait for all builds to finish + needs: [linux-x64, linux-arm64, windows, macos-x64, macos-arm64] runs-on: ubuntu-latest permissions: - contents: write # allow release upload + contents: write steps: - uses: actions/checkout@v4 - name: Read version from Cargo.toml @@ -189,11 +192,23 @@ jobs: with: path: target/release/kingfisher-* merge-multiple: true + - name: Extract latest changelog section + run: | + awk ' + BEGIN { grabbing = 0 } + /^## \[/ { + if (grabbing) exit; # already grabbed latest entry + grabbing = 1 + } + grabbing { print } + ' CHANGELOG.md > .latest_changelog.md + + # ── create the release using just that snippet ───────────────────── - name: Create release & upload assets uses: ncipollo/release-action@v1 with: tag: v${{ steps.version.outputs.version }} name: "Kingfisher v${{ steps.version.outputs.version }}" - bodyFile: CHANGELOG.md # use existing changelog - generateReleaseNotes: false # turn off auto-notes - artifacts: target/release/** + bodyFile: .latest_changelog.md # ← only the most-recent entry + generateReleaseNotes: false + artifacts: target/release/** \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 342e0c8..689cb46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. +## [1.13.0] +- Added new rules for Planetscale, Postman, Openweather, opsgenie, pagerduty, pastebin, paypal, netlify, netrc, newrelic, ngrok, npm, nuget, mandrill, mapbox, microsoft teams, stripe, linkedin, mailchimp, mailgun, linear, line, huggingface, ibm cloud, intercom, ipstack, heroku, gradle, grafana +- Added `--rule-stats` command-line flag that will display rule performance statistics during a scan. Useful when creating or debugging rules + + ## [1.12.0] - Added automatic update checks using GitHub releases. - New `--self-update` flag installs updates when available diff --git a/Cargo.toml b/Cargo.toml index 4d9d103..2b30bee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ publish = false [package] name = "kingfisher" -version = "1.12.0" +version = "1.13.0" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/README.md b/README.md index 0a24069..67098a3 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,11 @@ cat /path/to/file.py | kingfisher scan - kingfisher scan /path/to/repo --rule kingfisher.aws ``` +### Display rule performance statistics +```bash +kingfisher scan /path/to/repo --rule-stats +``` + --- ## Scanning GitHub @@ -244,6 +249,10 @@ The document below details the four-field formula (rule SHA-1, origin label, st See ([docs/FINGERPRINT.md](docs/FINGERPRINT.md)) +## Rule Performance Profiling +Use `--rule-stats` to collect timing information for every rule. After scanning, the summary prints a **Rule Performance Stats** section showing how many matches each rule produced along with its slowest and average match times. Useful when creating rules or debugging rules. + + ## CLI Options ```bash kingfisher scan --help diff --git a/data/rules/adafruitio.yml b/data/rules/adafruitio.yml index ee622f9..17b271f 100644 --- a/data/rules/adafruitio.yml +++ b/data/rules/adafruitio.yml @@ -28,4 +28,4 @@ rules: type: StatusMatch - type: WordMatch words: - - '"username":"kingfishermdb"' \ No newline at end of file + - '"username"' \ No newline at end of file diff --git a/data/rules/azuredevops.yml b/data/rules/azuredevops.yml index 2597a77..4188999 100644 --- a/data/rules/azuredevops.yml +++ b/data/rules/azuredevops.yml @@ -13,7 +13,7 @@ rules: min_entropy: 3 confidence: medium examples: - - azure devops pat = FBdFol081crwkIHWJH2yiqDDyrFjVSi7HWl22hN2hTYfsB8NlGDpJQQJ77BAACAAAAAAAAAAAAASAZDOBucT + - azure devops pat = FBdFol081crwkIHWJH2yiqDDyrFjVSi7HWl22hN2hTYfsB8NlGDpJQQJ77BAACAAAAAAAAAAAAASAZDOBucTj references: - https://learn.microsoft.com/en-us/rest/api/azure/devops/profile/profiles/get?view=azure-devops-rest-7.1&tabs=HTTP - https://learn.microsoft.com/en-us/azure/devops/release-notes/2024/general/sprint-241-update diff --git a/data/rules/azuresearchquery.yml b/data/rules/azuresearchquery.yml index 6cfdbf7..87272fb 100644 --- a/data/rules/azuresearchquery.yml +++ b/data/rules/azuresearchquery.yml @@ -9,7 +9,7 @@ rules: (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) (?:.|[\n\r]){0,32}? ( - [0-9a-zA-Z]{52} + [0-9A-Z]{52} ) \b min_entropy: 3.3 diff --git a/data/rules/fileio.yml b/data/rules/fileio.yml index c6413a1..6bdd8dd 100644 --- a/data/rules/fileio.yml +++ b/data/rules/fileio.yml @@ -10,9 +10,8 @@ rules: (?:.|[\n\r]){0,16}? \b ( - [A-Z0-9]{16} - (?:\.[A-Z0-9]{7}){2} - \.[A-Z0-9]{8} + [A-Z0-9]{20} + \.[A-Z0-9]{20} ) \b min_entropy: 3.3 diff --git a/data/rules/frame.io.yml b/data/rules/frame.io.yml index 8b3b562..a5ef94b 100644 --- a/data/rules/frame.io.yml +++ b/data/rules/frame.io.yml @@ -12,7 +12,6 @@ rules: confidence: medium examples: - fio-u-TaWoPIBovaGCbBkUtGPKWS0D3cu254VA33IFCCrtwl8J2Dtq2pMJ9MvNHmNoL2XX - - ffio-u-TaWoPIBovaGCbBkUtGPKWS0D3cu254VA33IFCCrtwl8J2Dtq2pMJ9MvNHmNoL2XX references: - https://developer.frame.io/api/reference/operation/getMe/ validation: diff --git a/data/rules/gradle.yml b/data/rules/gradle.yml new file mode 100644 index 0000000..42d27ea --- /dev/null +++ b/data/rules/gradle.yml @@ -0,0 +1,33 @@ +rules: + - name: Hardcoded Gradle Credentials + id: kingfisher.gradle.1 + pattern: | + (?xi) + credentials \s* \{ + (?:\s*//.*)* + \s* (?:username|password) \s ['"]([^'"]{1,60})['"] + (?:\s*//.*)* + \s* (?:username|password) \s ['"]([^'"]{1,60})['"] + min_entropy: 3.3 + confidence: medium + examples: + - | + credentials { + username 'user' + password 'password' + } + - | + publishing { + repositories { + maven { + url "http://us01cmsysart01.example.com:8081/artifactory/Mobile-Libs-Internal" + credentials { + // your password here + + username "SOME_USERNAME" + password "SOME_PASSWORD" + } + } + } + - "credentials {\n username 'user'\n password 'password'\n}" + - "credentials {\n username \"user\"\n password \"password\"\n}" \ No newline at end of file diff --git a/data/rules/grafana.yml b/data/rules/grafana.yml new file mode 100644 index 0000000..8a3b859 --- /dev/null +++ b/data/rules/grafana.yml @@ -0,0 +1,107 @@ +rules: + - name: Grafana API Token + id: kingfisher.grafana.1 + pattern: | + (?xi) + \b + ( + eyJrIjoi[a-z0-9]{60,100} + ) + \b + min_entropy: 3.3 + confidence: medium + examples: + - 'Authorization: Bearer eyJrIjoiWHZiSWd5NzdCYUZnNUtibE8obUpESmE2bzJYNDRIc1UiLCJuIjoibXlrZXkiLCJpZCI7MX1' + - 'admin_client = GrafanaClient("eyJrIjoiY21sM1JRYjB6RnVYSTNLenRWQkFEaWN2bXI2V202U2IiLCJuIjoiYWRtaW5rZXkiLCJpZCI6MX0=", host=grafana_host, port=3000, protocol="http")' + references: + - https://grafana.com/docs/grafana/latest/developers/http_api/auth/ + + - name: Grafana Cloud API Token + id: kingfisher.grafana.2 + pattern: | + (?xi) + \b + ( + glc_ + [a-z0-9+/]{40,150} + ={0,2} + ) + min_entropy: 3.3 + confidence: medium + examples: + - ' "token": "glc_eyJrIjoiZjI0YzZkNGEwZDBmZmZjMmUzNTU3ODcxMmY0ZWZlNTQ1NTljMDFjOCIsIm6iOiJteXRva3VuIiwiaWQiOjF8"' + - 'grafana = glc_etLvNLoNMLt7MTczNNwNbN6Nm1ldGEtbW9paxRvcmlpZt14ZXN4NNwNatN6NLCxdKeH7KTUvWpNqCrHlMKE9EhLcZH7to' + references: + - https://grafana.com/docs/grafana-cloud/developer-resources/api-reference/cloud-api/#regions + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://grafana.com/api/stack-regions + + - name: Grafana Service Account Token + id: kingfisher.grafana.3 + pattern: | + (?x) + \b + (glsa_[a-zA-Z0-9]{32}_[a-fA-F0-9]{8}) + \b + min_entropy: 3.3 + confidence: medium + examples: + - | + curl -H "Authorization: Bearer glsa_HOruNAb7SOiCdshU7algkrq7FDsNSLAa_55e2f8be" -X GET '/api/access-control/user/permissions' | jq + - | + // getData() + // { + // let url="http://localhost:4200/api/search" + // const headers = new HttpHeaders({ + // 'Content-Type': 'application/json', + // 'Authorization': `Bearer glsa_Sof0HKi3agxrQP9qm5r2G98VacBNwV5P_9b638c45` + // }) + // return this.http.get(url, {headers: headers}); + // } + references: + - https://grafana.com/docs/grafana/latest/administration/service-accounts/ + validation: + type: Http + content: + request: + method: GET + headers: + Authorization: Bearer {{ TOKEN }} + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: "{{ GRAFANADOMAIN }}/api/access-control/me" + depends_on_rule: + - rule_id: kingfisher.grafana.4 + variable: GRAFANADOMAIN + + - name: Grafana Domain + id: kingfisher.grafana.4 + pattern: | + (?xi) + (?:https?://)? + (?:[A-Za-z0-9-]+\.)* + grafana\.[A-Za-z0-9.-]+ + (?::\d{2,5})? + (?:[/?\#]\S*)? + min_entropy: 3.0 + visible: false + confidence: medium + examples: + - https://grafana.example.com + - http://grafana.prod.eu-west.mycorp.internal:3000/login + - https://api.team1.grafana.services.cluster.local/health + - grafana.dev.foo-bar.co.uk diff --git a/data/rules/hashes.yml b/data/rules/hashes.yml new file mode 100644 index 0000000..ab7cdeb --- /dev/null +++ b/data/rules/hashes.yml @@ -0,0 +1,109 @@ +rules: + - name: Password Hash (md5crypt) + id: kingfisher.pwhash.1 + pattern: '(\$1\$[./A-Za-z0-9]{8}\$[./A-Za-z0-9]{22})' + references: + - https://en.wikipedia.org/wiki/Crypt_(C)#MD5-based_scheme + - https://unix.stackexchange.com/a/511017 + - https://hashcat.net/wiki/doku.php?id=example_hashes + - https://passwordvillage.org/salted.html#md5crypt + min_entropy: 3.3 + confidence: medium + examples: # generated with `openssl passwd -1 -salt 'OKgLCmVl' 'a'` + - '$1$OKgLCmVl$d02jECa4DXn/oXX0R.MoQ/' + - '$1$28772684$iEwNOgGugqO9.bIz5sk8k/' + - name: Password Hash (bcrypt) + id: kingfisher.pwhash.2 + # Format from Wikipedia: + # $2$[cost]$[22 character salt][31 character hash] + pattern: '(\$2[abxy]\$\d+\$[./A-Za-z0-9]{53})' + references: + - https://en.wikipedia.org/wiki/Bcrypt + - https://hashcat.net/wiki/doku.php?id=example_hashes + min_entropy: 3.3 + confidence: medium + examples: + - '$2a$12$R9h/cIPz0gi.URNNX3kh2OPST9/PgBkqquzi.Ss7KIUgO2t0jWMUW' + - '$2a$05$/VT2Xs2dMd8GJKfrXhjYP.DkTjOVrY12yDN7/6I8ZV0q/1lEohLru' + - '$2a$05$Uo385Fa0g86uUXHwZxB90.qMMdRFExaXePGka4WGFv.86I45AEjmO' + - '$2a$05$LhayLxezLhK1LhWvKxCyLOj0j1u.Kj0jZ0pEmm134uzrQlFvQJLF6' + - '$2y$12$atWJ1Nx6ep65tNx0YIJ4I.jzgI86znQbNRI3lF0qIt/XCYnEPxSc2' + - name: Password Hash (sha256crypt) + id: kingfisher.pwhash.3 + pattern: | + (?x) + ( + \$ 5 + (?: \$ rounds=\d+ )? + \$ [./A-Za-z0-9]{8,16} + \$ [./A-Za-z0-9]{43} + ) + references: + - https://en.wikipedia.org/wiki/Crypt_(C)#Key_derivation_functions_supported_by_crypt + - https://hashcat.net/wiki/doku.php?id=example_hashes + - https://passwordvillage.org/salted.html#sha256crypt + min_entropy: 3.3 + confidence: medium + examples: + - '$5$rounds=5000$GX7BopJZJxPc/KEK$le16UF8I2Anb.rOrn22AUPWvzUETDGefUmAV8AZkGcD' + - '$5$9ks3nNEqv31FX.F$gdEoLFsCRsn/WRN3wxUnzfeZLoooVlzeF4WjLomTRFD' + - '$5$KAlz5SULZNybHwil$3UgmS1pmo2r5HG.tjbjzoVxISBh8IH81d.bJh4MCC19' + - name: Password Hash (sha512crypt) + id: kingfisher.pwhash.4 + pattern: | + (?x) + ( + \$ 6 + (?: \$ rounds=\d+ )? + \$ [./A-Za-z0-9]{8,16} + \$ [./A-Za-z0-9]{86} + ) + references: + - https://en.wikipedia.org/wiki/Crypt_(C)#Key_derivation_functions_supported_by_crypt + - https://hashcat.net/wiki/doku.php?id=example_hashes + - https://passwordvillage.org/salted.html#sha512crypt + min_entropy: 3.3 + confidence: medium + examples: + - '$6$52450745$k5ka2p8bFuSmoVT1tzOyyuaREkkKBcCNqoDKzYiJL9RaE8yMnPgh2XzzF0NDrUhgrcLwg78xs1w5pJiypEdFX/' + - '$6$qoE2letU$wWPRl.PVczjzeMVgjiA8LLy2nOyZbf7Amj3qLIL978o18gbMySdKZ7uepq9tmMQXxyTIrS12Pln.2Q/6Xscao0' + - name: Password Hash (Cisco IOS PBKDF2 with SHA256) + id: kingfisher.pwhash.5 + pattern: | + (?x) + ( + \$ 8 + \$ [./A-Za-z0-9]{8,16} + \$ [./A-Za-z0-9]{43} + ) + references: + - https://en.wikipedia.org/wiki/Crypt_(C)#Key_derivation_functions_supported_by_crypt + - https://hashcat.net/wiki/doku.php?id=example_hashes + min_entropy: 3.3 + confidence: medium + examples: + - '$8$TnGX/fE4KGHOVU$pEhnEvxrvaynpi8j4f.EMHr6M.FzU8xnZnBr/tJdFWk' + - '$8$mTj4RZG8N9ZDOk$elY/asfm8kD3iDmkBe3hD2r4xcA/0oWS5V3os.O91u.' + - name: Password Hash (Kerberos 5, etype 23, AS-REP) + id: kingfisher.krb5.asrep.23.1 + pattern: | + (?x) + ( + \$ krb5asrep + \$ 23 + \$ + (?: [^:]+ : )? + [0-9a-f]{32} + \$ [0-9a-f]{64,} + ) + \b + references: + - https://hashcat.net/wiki/doku.php?id=example_hashes + min_entropy: 3.3 + confidence: medium + examples: # Kerberos 5, etype 23, AS-REP + - '$krb5asrep$23$user@domain.com:3e156ada591263b8aab0965f5aebd837$007497cb51b6c8116d6407a782ea0e1c5402b17db7afa6b05a6d30ed164a9933c754d720e279c6c573679bd27128fe77e5fea1f72334c1193c8ff0b370fadc6368bf2d49bbfdba4c5dccab95e8c8ebfdc75f438a0797dbfb2f8a1a5f4c423f9bfc1fea483342a11bd56a216f4d5158ccc4b224b52894fadfba3957dfe4b6b8f5f9f9fe422811a314768673e0c924340b8ccb84775ce9defaa3baa0910b676ad0036d13032b0dd94e3b13903cc738a7b6d00b0b3c210d1f972a6c7cae9bd3c959acf7565be528fc179118f28c679f6deeee1456f0781eb8154e18e49cb27b64bf74cd7112a0ebae2102ac' + - '$krb5asrep$23$8cf8eb5287e28a4006c064892150c4fb$3e05ecc13548bec8e1eeb900dea5429cc6931bae9b8524490eb3a8801560871fe44355ed556202afbb39872e1bbb5c3c4f1b37dcd68fda89a23ebad917d4bbb0933edd94331598939e5d0c0c98c7e219a2e9dd6b877280d1bd7c51131413be577a167208bcc21e9fe7ae8f393278d740e72ca5c44c42d5cb0bf6bab0a36f1b88b7ddc4abbc6f152e652f6ba35c2955fb4132e11b7e566f3b422c3740f79847b77783d245a4e570b8a621b4ff6ff4815566446af70313ee78133707a76a4e4424783bd7c04920aa822a1a36b29f7e25cef186e6439fc46e42e23d6bd918969ef49b8388aef158e443b3a57dbde7ada631fbef7326f9046a9b' + - '$krb5asrep$23$c447eddaebf22ebf006a8fc6f986488c$eb3a17eb56287b474cecad5d4e0490d949977ba3f5015220bcd3080444d5601d67b76c5453b678e8527624e40c273bea4cfe4a7303e136b9bc3b9e63b6fb492ee4b4d2f830c5fa5a55466b57a678f708438f6712354a2deb851792b09270f4941966b82a2fd5ad8fa1fbd95a60b0f9bcd57774b3e55467a02ffcb3f1379104c24e468342f83df20b571e6f34f9a9842b43735d58d94514dcefa76719c0f5c7c3a3bfa770380924625aa0a3472d7c02d10dbb278fd946f7efcfe59a4d4cb7bdb9c5dbddc027611fe333d3ac940ec5b4ed43b55ab54b03cd2df0a9a2a7b5d235c226b259bd5ff8e0e49680351d4f0c4d13e258bc8d383cad6fc2711be0' + - '$krb5asrep$23$771adbc2397abddef676742924414f2b$2df6eb2d9c71820dc3fa2c098e071d920f0e412f5f12411632c5ee70e004da1be6f003b78661f8e4507e173552a52da751c45887c19bc1661ed334e0ccb4ef33975d4bd68b3d24746f281b4ca4fdf98fca0e50a8e845ad7d834e020c05b1495bc473b0295c6e9b94963cb912d3ff0f2f48c9075b0f52d9a31e5f4cc67c7af1d816b6ccfda0da5ccf35820a4d7d79073fa404726407ac840910357ef210fcf19ed81660106dfc3f4d9166a89d59d274f31619ddd9a1e2712c879a4e9c471965098842b44fae7ca6dd389d5d98b7fd7aca566ca399d072025e81cf0ef5075447687f80100307145fade7a8' + - '$krb5asrep$23$user@domain.com:3e156ada591263b8aab0965f5aebd837$007497cb51b6c8116d6407a782ea0e1c5402b17db7afa6b05a6d30ed164a9933c754d720e279c6c573679bd27128fe77e5fea1f72334c1193c8ff0b370fadc6368bf2d49bbfdba4c5dccab95e8c8ebfdc75f438a0797dbfb2f8a1a5f4c423f9bfc1fea483342a11bd56a216f4d5158ccc4b224b52894fadfba3957dfe4b6b8f5f9f9fe422811a314768673e0c924340b8ccb84775ce9defaa3baa0910b676ad0036d13032b0dd94e3b13903cc738a7b6d00b0b3c210d1f972a6c7cae9bd3c959acf7565be528fc179118f28c679f6deeee1456f0781eb8154e18e49cb27b64bf74cd7112a0ebae2102ac' diff --git a/data/rules/heroku.yml b/data/rules/heroku.yml new file mode 100644 index 0000000..817a2a4 --- /dev/null +++ b/data/rules/heroku.yml @@ -0,0 +1,34 @@ +rules: + - name: Heroku API Key + id: kingfisher.heroku.1 + pattern: | + (?xi) + heroku + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [0-9a-f]{8}-[0-9a-f]{4}- + [0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} + ) + \b + min_entropy: 3.0 + confidence: medium + examples: + - 'HEROKU_API_KEY: c55dbac4-e0e8-4a06-b892-75cac2387ce5' + references: + - https://devcenter.heroku.com/articles/authentication + validation: + type: Http + content: + request: + method: GET + headers: + Accept: application/vnd.heroku+json; version=3 + Authorization: Bearer {{ TOKEN }} + url: https://api.heroku.com/apps + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] diff --git a/data/rules/huggingface.yml b/data/rules/huggingface.yml new file mode 100644 index 0000000..d103438 --- /dev/null +++ b/data/rules/huggingface.yml @@ -0,0 +1,39 @@ +rules: + - name: HuggingFace User Access Token + id: kingfisher.huggingface.1 + pattern: | + (?xi) + \b + (?: + ( + (?:api_org|hf)_ + (?:[0-9A-Z]{17}){2} + ) + ) + \b + references: + - https://huggingface.co/docs/hub/security-tokens + min_entropy: 3.3 + confidence: medium + examples: + - 'HF_TOKEN:"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"' + - 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 + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + Content-Type: application/json + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + - match_all_words: true + type: WordMatch + words: + - '"name":' + - '"id":' + url: https://huggingface.co/api/whoami-v2 \ No newline at end of file diff --git a/data/rules/ibm.yml b/data/rules/ibm.yml new file mode 100644 index 0000000..0cf2e24 --- /dev/null +++ b/data/rules/ibm.yml @@ -0,0 +1,35 @@ +rules: + - name: IBM Cloud User API Key + id: kingfisher.ibm.1 + pattern: | + (?xi) + (?:ibm(?:cloud)?|bx) + (?:.|[\n\r]){0,32}? + \b + ( + [0-9A-Z_-]{42,44} + ) + min_entropy: 3.5 + confidence: medium + + examples: + - ibmcloud_apikey = abcdef0123_56789abcdef0123456789abcdef01234 + - ibm_platform_key="f-_RrJDVnuVh07HNTcmnQx_b6CbcQsxmEarVm9P_RWtF" + + references: + - https://cloud.ibm.com/docs/account?topic=account-userapikey + - https://cloud.ibm.com/apidocs/iam-identity-token-api + + validation: + type: Http + content: + request: + method: GET + headers: + Authorization: Basic Yng6Yng= # public “bx:bx” client credentials + Accept: application/json + url: https://iam.cloud.ibm.com/v1/apikeys/details?apikey={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] diff --git a/data/rules/intercom.yml b/data/rules/intercom.yml new file mode 100644 index 0000000..0c75e33 --- /dev/null +++ b/data/rules/intercom.yml @@ -0,0 +1,35 @@ +rules: + - name: Intercom API Token + id: kingfisher.intercom.1 + pattern: | + (?xi) + (?:intercom(?:_access)?|ic) + (?:.|[\n\r]){0,16}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,16}? + ( + [0-9A-Z+/]{59}= + ) + min_entropy: 3.5 + confidence: medium + + examples: + - "intercom_access_token: dG9rOvI0NmJlMTA5XzQwM2NfNDVlM184MjQzXzkwMDnmOTE1NGIyONoxOjA=" + - ic_token = "g1ZsclJXTjNfc1pBSzJDemE0eFVDU0U5c25CeDN4Vm9hQ2Zac0hXemZHNGVDPQ==" + + references: + - https://developers.intercom.com/docs/build-an-integration/learn-more/rest-apis + validation: + type: Http + content: + request: + method: GET + headers: + Accept: application/json + Authorization: Bearer {{ TOKEN }} + url: https://api.intercom.io/me + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 diff --git a/data/rules/ionic.yml b/data/rules/ionic.yml new file mode 100644 index 0000000..1130a38 --- /dev/null +++ b/data/rules/ionic.yml @@ -0,0 +1,28 @@ +rules: + - name: Ionic API token + id: kingfisher.ionic.1 + pattern: | + (?xi) + \b + ( + ion_ + [a-z0-9]{42} + ) + \b + min_entropy: 3.5 + confidence: medium + examples: + - ion_VNR17uGgdxr9P2aOrCulvSLTFDqijIV2ImQsOUhDEI + validation: + type: Http + content: + request: + headers: + Authorization: 'Bearer {{ TOKEN }}' + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://api.ionic.io/v1/auth/status \ No newline at end of file diff --git a/data/rules/ipstack.yml b/data/rules/ipstack.yml new file mode 100644 index 0000000..a70e43e --- /dev/null +++ b/data/rules/ipstack.yml @@ -0,0 +1,33 @@ +rules: + - name: IpStack API Key + id: kingfisher.ipstack.1 + pattern: | + (?xi) + \b + ipstack + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + (?:[0-9a-f]{16}){2} + ) + \b + min_entropy: 3.0 + confidence: medium + examples: + - "ipstack_token=123e4567e89b12d3a456426614174000" + - "ipstack_key=abcdefabcdefabcdefabcdefabcdef12" + references: + - https://ipstack.com/documentation + validation: + type: Http + content: + request: + method: GET + url: http://api.ipstack.com/1.1.1.1?access_key={{ TOKEN }} + response_matcher: + - type: WordMatch + words: + - '"ip":"1.1.1.1"' + - '"continent_code"' \ No newline at end of file diff --git a/data/rules/jenkins.yml b/data/rules/jenkins.yml new file mode 100644 index 0000000..f8fbb77 --- /dev/null +++ b/data/rules/jenkins.yml @@ -0,0 +1,24 @@ +rules: + - name: Jenkins Token or Crumb + id: kingfisher.jenkins.1 + pattern: '(?i)jenkins.{0,10}(?:crumb)?.{0,10}\b([0-9a-f]{32,36})\b' + categories: [api, fuzzy, secret] + min_entropy: 3.3 + confidence: medium + examples: + - | + jenkins_user = 'root' + # jenkins_passwd = '116365fd86d63bf507aba962606a5c8956' Pre token + jenkins_passwd = '11811f784531053132519844d047186074' # Dev Token + jenkins_url = 'http://10.1.188.121' + - | + export JENKINS_USER=justin-admin-edit-view + export JENKINS_TOKEN=11f4274ec59be12eace9a08b08ee13d54b + export JENKINS=jenkins-cicd.apps.sno.openshiftlabs.net + - | + sh "curl -X POST 'http://jenkins.lsfusion.luxsoft.by/job/${Paths.updateParentVersionsJob}/build' --user ${USERPASS} -H 'Jenkins-Crumb:440561953171ba44ace9740562d172bb'" + negative_examples: + - '1. ~~Does not play well with [Build Token Root Plugin](https://wiki.jenkins-ci.org/display/JENKINS/Build+Token+Root+Plugin) URL formats.~~ (added with [this commit](https://github.com/morficus/Parameterized-Remote-Trigger-Plugin/commit/f687dbe75d1c4f39f7e14b68220890384d7c5674) )' + references: + - https://www.jenkins.io/blog/2018/07/02/new-api-token-system/ + - https://www.jenkins.io/doc/book/security/csrf-protection/ \ No newline at end of file diff --git a/data/rules/jira.yml b/data/rules/jira.yml new file mode 100644 index 0000000..e106e77 --- /dev/null +++ b/data/rules/jira.yml @@ -0,0 +1,54 @@ +rules: + - name: Jira Domain + id: kingfisher.jira.1 + pattern: | + (?x) + (?i) + ( + [a-z][a-z0-9-]{5,24}\.atlassian\.net + ) + \b + min_entropy: 3.5 + visible: false + confidence: medium + examples: + - example-jira.atlassian.net + - jira.sprintUri= https://leakyday.atlassian.net/rest + + - name: Jira Token + id: kingfisher.jira.2 + pattern: | + (?x) + (?i) + \b + jira + (?:.|[\n\r]){0,8}? + (?:SECRET|PRIVATE|ACCESS|KEY|PASSWORD|TOKEN) + (?:.|[\n\r]){0,16}? + \b + ( + [a-z0-9-]{24} + ) + \b + min_entropy: 3.3 + confidence: medium + examples: + - 'Here is my Jira token: VDOheDe1sSCeGkuTARhkFDE2' + - public static final String JIRA_PASSWORD = "VDOheDe1sSCeGkuTARhkFDE2"; + validation: + type: Http + content: + request: + headers: + Accept: application/json + Authorization: Basic {{ TOKEN }} + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://{{ DOMAIN }}/rest/api/3/dashboard + depends_on_rule: + - rule_id: kingfisher.jira.1 + variable: DOMAIN \ No newline at end of file diff --git a/data/rules/line.yml b/data/rules/line.yml new file mode 100644 index 0000000..3dd453a --- /dev/null +++ b/data/rules/line.yml @@ -0,0 +1,36 @@ +rules: + - name: Line Messaging API Token + id: kingfisher.line.1 + pattern: | + (?x) + (?i) + \b + line + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + (?:[0-9A-Za-z+/]{57}){3}=? + ) + min_entropy: 3.5 + confidence: medium + examples: + - line_access_token = 13IRTqF+j0TfDtuJoIWKRBPhpDnqYUaaSlOilnoy0urLE+kbf5hN4HUf5pSPw20ruyO0BFFF1IDjnBojctp5emFw0hZ51WxB8c75qo48upJInfmqDQ1xrFd4yFKBwx4yRBHYXmI/FyrtcWKd0FBoBAdB04t81/1O/w1cDnyilFU= + - linemsg_token:"13IRTqF+j0TfDtuJoIWKRBPhpDnqYUaaSlOilnoy0urLE+kbf5hN4HUf5pSPw20ruyO0BFFF1IDjnBojctp5emFw0hZ51WxB8c75qo48upJInfmqDQ1xrFd4yFKBwx4yRBHYXmI/FyrtcWKd0FBoBAdB04t81/1O/w1cDnyilFU=" + references: + - https://developers.line.biz/en/reference/messaging-api/#get-consumption + validation: + type: Http + content: + request: + headers: + Authorization: 'Bearer {{ TOKEN }}' + Content-Type: application/json + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://api.line.me/v2/bot/message/quota/consumption \ No newline at end of file diff --git a/data/rules/linear.yml b/data/rules/linear.yml new file mode 100644 index 0000000..9adf6e8 --- /dev/null +++ b/data/rules/linear.yml @@ -0,0 +1,38 @@ +rules: + - name: Linear API Key + id: kingfisher.linear.1 + pattern: | + (?x) + (?i) + \b + ( + lin_api_ + (?:[0-9A-Za-z]{8}){5} + ) + \b + min_entropy: 3.5 + confidence: medium + examples: + - linear_api_key = lin_api_2thngjik222gkiihzivh242LU7zvkdvdgB14B41S + - lin_api_token:"lin_api_9A6bCDeF0Gh1Ij2Klm3No4PQr5St6Uv7Wx8YZaBc" + references: + - https://linear.app/developers/graphql + validation: + type: Http + content: + request: + method: POST + headers: + Authorization: '{{ TOKEN }}' + Content-Type: application/json + body: > + { + "query": "query { issues(first: 1) { nodes { id } } }" + } + url: https://api.linear.app/graphql + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"issues":', '"nodes":'] \ No newline at end of file diff --git a/data/rules/linkedin.yml b/data/rules/linkedin.yml new file mode 100644 index 0000000..97b4865 --- /dev/null +++ b/data/rules/linkedin.yml @@ -0,0 +1,67 @@ +rules: + - name: LinkedIn Client ID + id: kingfisher.linkedin.1 + pattern: | + (?x)(?i) + linkedin + .? + (?: api | app | application | client | consumer | customer )? + .? + (?: id | identifier | key ) + .{0,2} \s{0,20} .{0,2} \s{0,20} .{0,2} + \b ([a-z0-9]{12,14}) \b + references: + - https://docs.microsoft.com/en-us/linkedin/shared/api-guide/best-practices/secure-applications + min_entropy: 2.5 + confidence: medium + examples: + - 'Email ID Last 5 Digits of your SSN LinkedIn ID Availability' + - | + LINKEDIN_KEY = "77yg7tx91p4lag" + LINKEDIN_SECRET = "zt7GeN6IH911xvRj" + validation: + type: Http + content: + request: + headers: + Authorization: "Bearer {{ TOKEN }}" + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://api.linkedin.com/v2/me + + - name: LinkedIn Secret Key + id: kingfisher.linkedin.2 + pattern: | + (?x)(?i) + linkedin + .? + (?: api | app | application | client | consumer | customer | secret | key ) + .? + (?: key | oauth | sec | secret )? + .{0,2} \s{0,20} .{0,2} \s{0,20} .{0,2} + \b ([a-z0-9]{16}) \b + references: + - https://docs.microsoft.com/en-us/linkedin/shared/api-guide/best-practices/secure-applications + min_entropy: 3.3 + confidence: medium + examples: + - | + LINKEDIN_KEY = "77yg7tx91p4lag" + LINKEDIN_SECRET = "zt7GeN6IH911xvRj" + validation: + type: Http + content: + request: + headers: + Authorization: "Bearer {{ TOKEN }}" + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://api.linkedin.com/v2/me \ No newline at end of file diff --git a/data/rules/mailchimp.yml b/data/rules/mailchimp.yml new file mode 100644 index 0000000..a28eaae --- /dev/null +++ b/data/rules/mailchimp.yml @@ -0,0 +1,36 @@ +rules: + - name: Mailchimp API Key + id: kingfisher.mailchimp.1 + pattern: | + (?xi) + mailchimp + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + (?:[0-9a-f]{8}){4} + -us\d{1,2} + ) + min_entropy: 3.5 + confidence: medium + examples: + - 'mailchimp_token = abcdef1234567890abcdef1234567890-us1' + - 'mailchimp_key = "0123456789abcdeffedcba9876543210-us20"' + validation: + type: Http + content: + request: + headers: + Authorization: 'Basic {{ "x:" | append: TOKEN | b64enc }}' + Accept: application/json + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: 'https://{{ TOKEN | split: "-" | last }}.api.mailchimp.com/3.0/ping' + references: + - https://mailchimp.com/developer/marketing/api/root/ + - https://mailchimp.com/developer/guides/marketing-api-conventions/ \ No newline at end of file diff --git a/data/rules/mailgun.yml b/data/rules/mailgun.yml new file mode 100644 index 0000000..4bac90e --- /dev/null +++ b/data/rules/mailgun.yml @@ -0,0 +1,63 @@ +rules: + - name: MailGun Token + id: kingfisher.mailgun.1 + pattern: | + (?x) + (?i) + \b + mailgun + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + (?:[0-9A-Za-z-]{24}){3} + ) + min_entropy: 3.5 + confidence: medium + examples: + - mailgun_api_key=abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456 + validation: + type: Http + content: + request: + headers: + Authorization: Basic {{ TOKEN | b64enc }} + Accept: application/json + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://api.mailgun.net/v3/address/validate?address=test@example.com + - name: MailGun Primary Key + id: kingfisher.mailgun.2 + pattern: | + (?x) + (?i) + (?:mailgun|mg) + (?:.|[\n\r]){0,64}? + \b + ( + key-(?:[0-9a-f]{8}){4} + ) + \b + min_entropy: 3.5 + confidence: medium + examples: + - key-mailgun_token= key-ad13dfc23adf55fa404a91e76d96f472 + validation: + type: Http + content: + request: + headers: + Authorization: 'Basic {{ "api:" | append: TOKEN | b64enc }}' + Accept: application/json + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://api.mailgun.net/v3/address/validate?address=test@example.com \ No newline at end of file diff --git a/data/rules/mandrill.yml b/data/rules/mandrill.yml new file mode 100644 index 0000000..4f0228c --- /dev/null +++ b/data/rules/mandrill.yml @@ -0,0 +1,38 @@ +rules: + - name: Mandrill API Key + id: kingfisher.mandrill.1 + pattern: | + (?x) + (?i) + \b + mandrill + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + (?:[0-9A-Za-z_-]{11}){2} + ) + min_entropy: 3.5 + confidence: medium + examples: + - mandrill_token = taqnVL1P5AJrM4oU4opSqQ + categories: + - api + - identifier + validation: + type: Http + content: + request: + method: POST + headers: + Content-Type: application/json + body: | + { "key": "{{ TOKEN }}" } + url: https://mandrillapp.com/api/1.0/users/ping.json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"PONG!"'] \ No newline at end of file diff --git a/data/rules/mapbox.yml b/data/rules/mapbox.yml new file mode 100644 index 0000000..d0253d6 --- /dev/null +++ b/data/rules/mapbox.yml @@ -0,0 +1,73 @@ +rules: + - name: Mapbox Public Access Token + id: kingfisher.mapbox.1 + pattern: '(?i)(?s)mapbox.{0,30}(pk\.[a-z0-9\-+/=]{32,128}\.[a-z0-9\-+/=]{20,30})(?:[^a-z0-9\-+/=]|$)' + min_entropy: 3.3 + confidence: medium + examples: + - | + mapboxApiKey: + 'pk.eyJ1Ijoia3Jpc3R3IiwiYSI6ImNqbGg1N242NTFlczczdnBcf99iMjgzZ2sifQ.lUneM-o3NucXN189EYyXxQ' + references: + - https://docs.mapbox.com/api/accounts/tokens/#token-format + - https://docs.mapbox.com/help/getting-started/access-tokens/ + - https://docs.mapbox.com/help/troubleshooting/how-to-use-mapbox-securely + validation: + type: Http + content: + request: + method: GET + url: https://api.mapbox.com/styles/v1/mapbox/streets-v11?access_token={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + + - name: Mapbox Secret Access Token + id: kingfisher.mapbox.2 + pattern: '(?i)(?s)mapbox.{0,30}(sk\.[a-z0-9\-+/=]{32,128}\.[a-z0-9\-+/=]{20,30})(?:[^a-z0-9\-+/=]|$)' + min_entropy: 3.3 + confidence: medium + examples: + - " //mapboxgl.accessToken = 'sk.eyJ1Ijoic2hlbmdsaWgiLCJhIjCf99ttaWF5bDBsMGNlaDJubGZyMGUwZXNmaCJ9.eI8KXNm5zKZXOKh0c8u9vg';" + - 'export MAPBOX_SECRET_TOKEN=sk.eyJ1IjoiY2FwcGVsYWVyZSIsImEicf99c1BaTkZnIn0.P4lD1eHeSEx7AsBq1zbJ4g' + references: + - https://docs.mapbox.com/api/accounts/tokens/#token-format + - https://docs.mapbox.com/help/getting-started/access-tokens/ + - https://docs.mapbox.com/help/troubleshooting/how-to-use-mapbox-securely + validation: + type: Http + content: + request: + method: GET + url: https://api.mapbox.com/styles/v1/mapbox/streets-v11?access_token={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + + - name: Mapbox Temporary Access Token + id: kingfisher.mapbox.3 + pattern: '(?i)(?s)mapbox.{0,30}(tk\.[a-z0-9\-+/=]{32,128}\.[a-z0-9\-+/=]{20,30})(?:[^a-z0-9\-+/=]|$)' + min_entropy: 3.3 + confidence: medium + examples: + - " //mapboxgl.accessToken = 'tk.eyJ1Ijoic2hlbmdsaWgiLCJhIjCf99ttaWF5bDBsMGNlaDJubGZyMGUwZXNmaCJ9.eI8KXNm5zKZXOKh0c8u9vg';" + - 'export MAPBOX_TEMP_TOKEN=tk.eyJ1IjoiY2FwcGVsYWVyZSIsImEicf99c1BaTkZnIn0.P4lD1eHeSEx7AsBq1zbJ4g' + references: + - https://docs.mapbox.com/api/accounts/tokens/#token-format + - https://docs.mapbox.com/help/getting-started/access-tokens/ + - https://docs.mapbox.com/help/troubleshooting/how-to-use-mapbox-securely + validation: + type: Http + content: + request: + method: GET + url: https://api.mapbox.com/styles/v1/mapbox/streets-v11?access_token={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid diff --git a/data/rules/microsoft_teams.yml b/data/rules/microsoft_teams.yml new file mode 100644 index 0000000..13cc044 --- /dev/null +++ b/data/rules/microsoft_teams.yml @@ -0,0 +1,52 @@ +rules: + - name: Microsoft Teams Webhook + id: kingfisher.msteams.1 + pattern: | + (?xi) + ( + https:// + outlook\.office\.com/webhook/ + [0-9a-f]{8}- + [0-9a-f]{4}- + [0-9a-f]{4}- + [0-9a-f]{4}- + [0-9a-f]{12} + @ + [0-9a-f]{8}- + [0-9a-f]{4}- + [0-9a-f]{4}- + [0-9a-f]{4}- + [0-9a-f]{12} + /IncomingWebhook/ + [0-9a-f]{32} + / + [0-9a-f]{8}- + [0-9a-f]{4}- + [0-9a-f]{4}- + [0-9a-f]{4}- + [0-9a-f]{12} + ) + min_entropy: 3.3 + confidence: medium + examples: + - 'https://outlook.office.com/webhook/9da5da9c-4218-4c22-aed6-b5c8baebfff5@2f2b54b7-0141-4ba7-8fcd-ab7d17a60547/IncomingWebhook/1bf66ccbb8e745e791fa6e6de0cf465b/4361420b-8fde-48eb-b62a-0e34fec63f5c' + - 'https://outlook.office.com/webhook/fa4983ab-49ea-4c1b-9297-2658ea56164c@f784fbed-7fc7-4c7a-aae9-d2f387b67c5d/IncomingWebhook/4d2b3a16113d47b080b7a083b5a5e533/74f315eb-1dde-4731-b6b5-2524b77f2acd' + - 'https://outlook.office.com/webhook/555aa7fc-ea71-4fb7-ae9e-755caa4404ed@72f988bf-86f1-41af-91ab-2d7cd011db47/IncomingWebhook/16085df23e564bb9076842605ede3af2/51dab674-ad95-4f0a-8964-8bdefc25b6d9' + - 'https://outlook.office.com/webhook/2f92c502-7feb-4a6c-86f1-477271ae576f@990414fa-d0a3-42f5-b740-21d865a44a28/IncomingWebhook/54e43eb586f14aa9984d5c0bec3d5050/539ce6fa-e9aa-413f-a79b-fb7e8998fcac' + validation: + type: Http + content: + request: + method: POST + url: '{{ TOKEN }}' + headers: + Content-Type: application/json + body: '{"text":""}' + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 400 + - type: WordMatch + words: + - 'Text is required' \ No newline at end of file diff --git a/data/rules/microsoftteamswebhook.yml b/data/rules/microsoftteamswebhook.yml new file mode 100644 index 0000000..3fc349c --- /dev/null +++ b/data/rules/microsoftteamswebhook.yml @@ -0,0 +1,38 @@ +rules: + - name: Microsoft Teams Webhook + id: kingfisher.microsoftteamswebhook.1 + pattern: | + (?x) + https://[a-zA-Z0-9]+\.webhook\.office\.com/webhookb2 + / + [a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12} + @ + [a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12} + / + IncomingWebhook + / + [a-zA-Z0-9]{32} + / + [a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12} + min_entropy: 3.3 + confidence: medium + examples: + - "https://contoso.webhook.office.com/webhookb2/12345678-abcd-1234-efgh-56789abcdef0@12345678-abcd-1234-efgh-56789abcdef0/IncomingWebhook/abcdefgh12345678abcdefgh12345678/12345678-abcd-1234-efgh-56789abcdef0" + validation: + type: Http + content: + request: + body: | + {'text':''} + headers: + Content-Type: application/json + method: POST + response_matcher: + - type: StatusMatch + status: + - 200 + - report_response: true + type: WordMatch + words: + - "Text is required" + url: '{{ TOKEN }}' \ No newline at end of file diff --git a/data/rules/netlify.yml b/data/rules/netlify.yml new file mode 100644 index 0000000..f7736a3 --- /dev/null +++ b/data/rules/netlify.yml @@ -0,0 +1,62 @@ +rules: + - name: Netlify API Key + id: kingfisher.netlify.1 + pattern: | + (?xi) + netlify + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ([a-f0-9]{60,64}) + \b + min_entropy: 3.3 + examples: + - netlify_token=3cdfad7b885a6daceff3fb820389115750b373763fb30b10ca0382648b55872d + - netlify_secret=7a9ef2c84d6b3e5f1c8a0b9d2e4f6a8c7b3d5e9f2a1c8b4d6e3f5a9c7b2d8e4 + references: + - https://howtorotate.com/docs/tutorials/netlify/ + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + method: GET + url: https://api.netlify.com/api/v1/user + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + + - name: Netlify API Key + id: kingfisher.netlify.2 + pattern: | + (?xi) + \b + netlify + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ([A-Za-z0-9_-]{43,45}) + \b + min_entropy: 3.5 + confidence: medium + examples: + - netlify_token=G5yT54abRasekrOpe7SaArsowiuHTeR45sfEhsH-K1L2 + - netlify_key=H7xZ98cdWbsemqNpv8UaXtsnyjKgVeQ34rsDkpM-N5P6 + references: + - https://howtorotate.com/docs/tutorials/netlify/ + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + method: GET + url: https://api.netlify.com/api/v1/user + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] diff --git a/data/rules/netrc.yml b/data/rules/netrc.yml new file mode 100644 index 0000000..35ed66e --- /dev/null +++ b/data/rules/netrc.yml @@ -0,0 +1,29 @@ +rules: + - name: netrc Credentials + id: kingfisher.netrc.1 + pattern: | + (?x) + ( machine \s+ [^\s]+ | default ) + \s+ + login \s+ ([^\s]+) + \s+ + password \s+ ([^\s]+) + min_entropy: 3.3 + confidence: medium + examples: + - 'machine api.github.com login ziggy^stardust password 012345abcdef' + - | + ``` + machine raw.github.com + login visionmedia + password pass123 + ``` + - | + """ + machine api.wandb.ai + login user + password 7cc938e45e63e9014f88f811be240ba0395c02dd + """ + references: + - https://everything.curl.dev/usingcurl/netrc + - https://devcenter.heroku.com/articles/authentication#api-token-storage \ No newline at end of file diff --git a/data/rules/newrelic.yml b/data/rules/newrelic.yml new file mode 100644 index 0000000..d9a582c --- /dev/null +++ b/data/rules/newrelic.yml @@ -0,0 +1,33 @@ +rules: + - name: New Relic Personal API Key + id: kingfisher.newrelic.1 + pattern: | + (?xi) + \b + newrelic + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9_.]{4} + - + [A-Z0-9_.]{42} + ) + min_entropy: 3.3 + confidence: medium + examples: + - newrelic_key = abcd-1234567890abcdef1234567890abcdef1234dd5678 + - newrelic_token = 1234-abcdefghijklmnopqrstuvwxyzABCD2342EFGHIJKL + validation: + type: Http + content: + request: + method: GET + headers: + X-Api-Key: '{{ TOKEN }}' + url: https://api.newrelic.com/v2/servers.json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] diff --git a/data/rules/ngrok.yml b/data/rules/ngrok.yml new file mode 100644 index 0000000..3635d08 --- /dev/null +++ b/data/rules/ngrok.yml @@ -0,0 +1,31 @@ +rules: + - name: Ngrok API Key + id: kingfisher.ngrok.1 + pattern: | + (?x)(?i) + ngrok + (?:.|[\\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + (?:[a-z0-9]{25,30}_\d[a-z0-9]{20}|(?:cr_|ak_)[a-z0-9]{25,30}) + \b + min_entropy: 4 + examples: + - 'ngrok authtoken: 2Ot3hdNCKF3JRJDCioqNV2J0PPc_6th2CSUm9KsjfXdtRDvzT' + validation: + type: Http + content: + request: + method: GET + headers: + Authorization: Bearer {{ TOKEN }} + ngrok-version: 2 + url: https://api.ngrok.com/endpoints + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"endpoints":' diff --git a/data/rules/npm.yml b/data/rules/npm.yml new file mode 100644 index 0000000..af94dfe --- /dev/null +++ b/data/rules/npm.yml @@ -0,0 +1,65 @@ +rules: + - name: NPM Access Token (fine-grained) + id: kingfisher.npm.1 + pattern: | + (?x) + \b + ( + npm_[A-Za-z0-9]{36} + ) + \b + references: + - https://docs.npmjs.com/about-access-tokens + - https://github.com/github/roadmap/issues/557 + - https://github.blog/changelog/2022-12-06-limit-scope-of-npm-tokens-with-the-new-granular-access-tokens/ + min_entropy: 3.3 + confidence: medium + examples: + - 'npm_TCllNwh2WLQlMWVhybM1iQrsTj6rMQ0BOh6d' + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + method: GET + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"name":'] + url: https://registry.npmjs.org/-/npm/v1/user + + - name: NPM Access Token (old format) + id: kingfisher.npm.2 + pattern: | + (?xi) + \b + (?:_authToken|NPM_TOKEN) + (?:.|[\n\r]){0,16}? + ( + [0-9A-F]{8} + (?:-[0-9A-F]{4}){3} + -[0-9A-F]{12} + ) + \b + min_entropy: 3.5 + confidence: medium + examples: + - '"_authToken": "b98ec224-cdb2-4340-b7bd-9617fc719d1d"' + - '-export NPM_TOKEN="007e64c7-635d-4d54-8295-f364cd8e0e0f"' + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + method: GET + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"name":'] + url: https://registry.npmjs.org/-/npm/v1/user \ No newline at end of file diff --git a/data/rules/nuget.yml b/data/rules/nuget.yml new file mode 100644 index 0000000..8d39cce --- /dev/null +++ b/data/rules/nuget.yml @@ -0,0 +1,68 @@ +rules: + - name: NuGet API Key + id: kingfisher.nuget.1 + pattern: | + (?x) + \b + ( + oy2[a-z0-9]{43} + ) + \b + min_entropy: 3.3 + confidence: medium + examples: + - oy2er16dp0r068m6p36u4bvr9nmkescfm1pf9lek1bgn3n + - oy2gdbfofub9ecpohsfdndem7nr8sui1g8le3ptnljqhlu + validation: + type: Http + content: + request: + method: POST + url: https://www.nuget.org/api/v2/package/create-verification-key/Newtonsoft.Json + headers: + X-NuGet-ApiKey: '{{ TOKEN }}' + X-NuGet-Protocol-Version: '4.1.0' + Content-Length: '0' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"Key":'] + + + - name: NuGet API Key + id: kingfisher.nuget.2 + pattern: | + (?xi) + \b + nuget + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [a-z0-9]{46} + ) + \b + min_entropy: 3.5 + confidence: medium + examples: + - nuget key = af7e33adc0rq7ls6ijcfmb5rgf1gp2o0tqk6d8s5p21k6t + - nuget secret dfaru1u13sd58q0kd0edvs4276p2mb6o31eljkvjh8t30u + validation: + type: Http + content: + request: + method: POST + url: https://www.nuget.org/api/v2/package/create-verification-key/Newtonsoft.Json + headers: + X-NuGet-ApiKey: '{{ TOKEN }}' + X-NuGet-Protocol-Version: '4.1.0' + Content-Length: '0' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"Key":'] \ No newline at end of file diff --git a/data/rules/openweather.yml b/data/rules/openweather.yml new file mode 100644 index 0000000..2153e64 --- /dev/null +++ b/data/rules/openweather.yml @@ -0,0 +1,37 @@ +rules: + - name: OpenWeather Map API Key + id: kingfisher.openweather.1 + pattern: | + (?xi) + (?:pyowm|openweather|\bowm\b) + (?:.|[\n\r]){0,64}? + \b + ( + (?: + [a-z0-9]{32} + ) + \b + |APPID= + (?: + [a-z0-9]{32} + ) + ) + \b + min_entropy: 3.5 + examples: + - pyowm = '3k144a5af729351d0fc58bdrj9a21mkr' + - owm = '3k144a5af729351d0fc58bdrj9a21mkr' + - openweatherapikey=cd2b1d12d01ae2deffecfebafcc3c31d + - apikey=openweather:cd2b1d12d01ae2deffecfebafcc3c31d + validation: + type: Http + content: + request: + method: GET + response_matcher: + - report_response: true + - match_all_status: true + status: + - 200 + type: StatusMatch + url: https://api.openweathermap.org/geo/1.0/reverse?lat=0&lon=0&limit=1&appid={{ TOKEN }} \ No newline at end of file diff --git a/data/rules/opsgenie.yml b/data/rules/opsgenie.yml new file mode 100644 index 0000000..fafef06 --- /dev/null +++ b/data/rules/opsgenie.yml @@ -0,0 +1,32 @@ +rules: + - name: OpsGenie API Key + id: kingfisher.opsgenie.1 + pattern: | + (?x) + (?i) + \b + opsgenie + (?:.|[\\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} + ) + min_entropy: 3.5 + examples: + - opsgenie_api_key = '12345678-9abc-def0-1234-56789abcdef0' + validation: + type: Http + content: + request: + headers: + Authorization: GenieKey {{ TOKEN }} + method: GET + url: https://api.opsgenie.com/v2/alerts + response_matcher: + - report_response: true + - type: WordMatch + words: + - "Could not authenticate" + negative: true diff --git a/data/rules/pagerdutyapikey.yml b/data/rules/pagerdutyapikey.yml new file mode 100644 index 0000000..be4b24f --- /dev/null +++ b/data/rules/pagerdutyapikey.yml @@ -0,0 +1,36 @@ +rules: + - name: PagerDuty API Key + id: kingfisher.pagerduty.1 + pattern: | + (?xi) + \b + (?:pagerduty|pager[_-]duty|pd[-_\]=\)]|pd\.webhook?) + (?:.|[\n\r]){0,16}? + ( + u\+[A-Z0-9_+-]{18} # new personal tokens + | + [A-Z0-9_-]{20} # legacy personal tokens + | + [A-F0-9]{32} # integration keys / routing keys + ) + \b + min_entropy: 3.3 + confidence: medium + examples: + - pagerduty_key = u+Lyhd2_N2MCy+ZoH-S5 + - pd_key = u+3xVszZ-b4m+T6d23KA + validation: + type: Http + content: + request: + method: GET + url: https://api.pagerduty.com/abilities + headers: + Authorization: Token token={{ TOKEN }} + Accept: application/vnd.pagerduty+json;version=2 + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"abilities":'] \ No newline at end of file diff --git a/data/rules/particle.io.yml b/data/rules/particle.io.yml new file mode 100644 index 0000000..8237ebd --- /dev/null +++ b/data/rules/particle.io.yml @@ -0,0 +1,74 @@ +rules: + - name: particle.io Access Token + id: kingfisher.particleio.1 + pattern: | + (?x) + https://api\.particle\.io/v1/[a-zA-Z0-9_\-\s/"\\?]* + (?:access_token=|Authorization:\s*Bearer\s*) + \b + ([a-zA-Z0-9]{40}) + \b + min_entropy: 3.3 + confidence: medium + examples: + - | + curl https://api.particle.io/v1/devices \ + -H "Authorization: Bearer 38bb7b318cc6898c80317decb34525844bc9db55" + - | + curl https://api.particle.io/v1/devices \ + -d access_token=38bb7b318cc6898c80317decb34525844bc9db55 + - 'curl https://api.particle.io/v1/devices -H "Authorization: Bearer 38bb7b318cc6898c80317decb34525844bc9db55"' + - 'curl https://api.particle.io/v1/devices -d access_token=38bb7b318cc6898c80317decb34525844bc9db55' + - 'curl "https://api.particle.io/v1/devices/events?access_token=38bb7b318cc6898c80317decb34525844bc9db55"' + - 'curl "https://api.particle.io/v1/access_tokens/current?access_token=38bb7b318cc6898c80317decb34525844bc9db55"' + references: + - https://docs.particle.io/reference/cloud-apis/api/ + validation: + type: Http + content: + request: + method: GET + url: https://api.particle.io/v1/user?access_token={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + match_all_words: true + words: ['"username":'] + + - name: particle.io Access Token + id: kingfisher.particleio.2 + pattern: | + (?x) + (?:access_token=|Authorization:\s*Bearer\s*) + \b + ([a-zA-Z0-9]{40}) + \b + [\s"\\]*https://api\.particle\.io/v1 + min_entropy: 3.3 + confidence: medium + examples: + - | + curl -H "Authorization: Bearer 38bb7b318cc6898c80317decb34525844bc9db55" \ + https://api.particle.io/v1/devices + - | + curl -d access_token=38bb7b318cc6898c80317decb34525844bc9db55 \ + https://api.particle.io/v1/devices + - 'curl -H "Authorization: Bearer 38bb7b318cc6898c80317decb34525844bc9db55" https://api.particle.io/v1/devices' + - 'curl -d access_token=38bb7b318cc6898c80317decb34525844bc9db55 https://api.particle.io/v1/devices' + references: + - https://docs.particle.io/reference/cloud-apis/api/ + validation: + type: Http + content: + request: + method: GET + url: https://api.particle.io/v1/user?access_token={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + match_all_words: true + words: ['"username":'] \ No newline at end of file diff --git a/data/rules/pastebin.yml b/data/rules/pastebin.yml new file mode 100644 index 0000000..26a55dd --- /dev/null +++ b/data/rules/pastebin.yml @@ -0,0 +1,37 @@ +rules: + - name: Pastebin API Key + id: kingfisher.pastebin.1 + pattern: | + (?xi) + \b + pastebin + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [a-zA-Z0-9_]{32} + ) + \b + min_entropy: 3.5 + confidence: medium + examples: + - pastebin_key=zwD26NeyMCvBsR9nxfaybLHD7TcLh22O + - pastebin_api_token=zwD26NeyMCvBsR9n_faybLHD7TcLh22O + validation: + type: Http + content: + request: + method: POST + url: https://pastebin.com/api/api_login.php + headers: + Content-Type: application/x-www-form-urlencoded + body: | + api_dev_key={{ TOKEN }}&api_user_name=dummy&api_user_password=dummy + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['invalid api_dev_key'] + negative: true \ No newline at end of file diff --git a/data/rules/paypal.yml b/data/rules/paypal.yml new file mode 100644 index 0000000..2441a68 --- /dev/null +++ b/data/rules/paypal.yml @@ -0,0 +1,56 @@ +rules: +- name: PayPal OAuth Client ID + id: kingfisher.paypal.1 + pattern: | + (?xi) + paypal + (?:.|[\n\r]){0,8}? + (?:CLIENT|ID|USER) + (?:.|[\n\r]){0,16}? + \b + ( + A[A-Z0-9_-]{79,99} + ) + \b + min_entropy: 3.5 + visible: false + examples: + - paypal_client_id=AZJ6y8Dpr1TYbqAIdhkPzyhjXoY6m8GplL7C3zZ3lPrkTIdhkPzyhjXo_Dx3 + +- name: PayPal OAuth Secret + id: kingfisher.paypal.2 + pattern: | + (?xi) + paypal + (?:.|[\n\r]){0,16}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9_.-]{80,120} + ) + \b + min_entropy: 3.5 + examples: + - paypal_secret=EDe5J6y8Dpr1TYbqAIdhkPzyhjXoY6m8GplL7C3zZ3lPrkT1XlV6hYPSeJL5b1T1 + + validation: + type: Http + content: + request: + method: POST + url: https://api-m.paypal.com/v1/oauth2/token + headers: + Accept: application/json + Accept-Language: en_US + Content-Type: application/x-www-form-urlencoded + Authorization: | + Basic {{ CLIENTID | append: ':' | append: TOKEN | b64enc }} + body: grant_type=client_credentials + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + depends_on_rule: + - rule_id: kingfisher.paypal.1 + variable: CLIENTID diff --git a/data/rules/planetscale.yml b/data/rules/planetscale.yml new file mode 100644 index 0000000..119957c --- /dev/null +++ b/data/rules/planetscale.yml @@ -0,0 +1,55 @@ +rules: + - name: PlanetScale API Token + id: kingfisher.planetscale.1 + pattern: | + (?x) + (?i) + \b + ( + pscale_tkn_[a-z0-9-_]{43} + ) + \b + min_entropy: 4 + examples: + - pscale_tkn_abcdefghijklmnopqrstuvwxyZ1234567890_ABCDEF + validation: + type: Http + content: + request: + headers: + Accept: application/json + Authorization: '{{ USERNAME | append: ":" | append: TOKEN }}' + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + - type: WordMatch + words: + - '"id":' + - '"username":' + url: https://api.planetscale.com/v1/user + depends_on_rule: + - rule_id: kingfisher.planetscale.2 + variable: USERNAME + + - name: PlanetScale Username + id: kingfisher.planetscale.2 + pattern: | + (?x) + (?i) + (?:pscale|planetscale) + (?:.|[\n\r]){0,16}? + (?:USER|ID|NAME) + (?:.|[\n\r]){0,16}? + \b + ( + [a-z0-9]{12} + ) + \b + min_entropy: 3.5 + visible: false + examples: + - pscale_user = abcdefghijkl + - 'planetscale_id: hgtmrnzlv1t7' diff --git a/data/rules/postman.yml b/data/rules/postman.yml new file mode 100644 index 0000000..707b4e2 --- /dev/null +++ b/data/rules/postman.yml @@ -0,0 +1,37 @@ +rules: + - name: Postman API Key + id: kingfisher.postman.1 + pattern: | + (?x) + \b + ( + PMAK-[A-Z0-9]{24}-[A-Z0-9]{34} + ) + \b + min_entropy: 3.3 + confidence: medium + examples: + - PMAK-5dd543842789bd0036bf98c1-a5a9b8f1dfda8fbf18a4664ebe558b04ed + - PMAK-642a58a823faa300316566d1-6715a3a826ce5d5d62be8539d6ac357146 + - PMAK-642a9b9084d6110029e75d7d-09efdcb872587f6f67696f02929647d9c6 + - "// ('x-api-key', 'PMAK-629c73facbc064567cbf6970-f56e8b4cd0bb14d00962f17afc158dc2a2')" + references: + - https://learning.postman.com/docs/developer/intro-api/ + - https://learning.postman.com/docs/developer/postman-api/authentication/ + - https://learning.postman.com/docs/administration/managing-your-team/managing-api-keys/ + validation: + type: Http + content: + request: + headers: + x-api-key: '{{ TOKEN }}' + method: GET + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 + - type: WordMatch + words: + - '"user":' + url: https://api.getpostman.com/me \ No newline at end of file diff --git a/data/rules/stripe.yml b/data/rules/stripe.yml new file mode 100644 index 0000000..0cb2072 --- /dev/null +++ b/data/rules/stripe.yml @@ -0,0 +1,56 @@ +rules: + - name: Stripe Publishable Key + id: kingfisher.stripe.1 + + pattern: | + (?xi) + (?:stripe|strp) + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + ( + pk_live_ + (?:[0-9A-Za-z]{6}){4,30} + ) + min_entropy: 3.3 + confidence: medium + categories: [api, key] + examples: + - stripe_pub_key = pk_live_HQS0j4H75XpthOW87eY1sXa2BYz3Ab + + - name: Stripe Secret / Restricted Key + id: kingfisher.stripe.2 + + pattern: | + (?ix) + (?:^|[\s"'=]) + (?:stripe|strp) + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + ( + (?: + sk|rk + )_live_ + (?:[0-9A-Za-z]{8}){3,25} + ) + min_entropy: 3.3 + confidence: medium + examples: + - stripe_secret_key = sk_live_f01c79xuuug7yodgzj5ws0h1x2kyvho3 + - "strp_sec_key: rk_live_4haG9YwGkL2hXqTj5pSzo8FzB3uCwE7n" + + validation: + type: Http + content: + request: + method: GET + headers: + Authorization: Bearer {{ TOKEN }} + Accept: application/json + url: https://api.stripe.com/v1/account + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + match_all_words: true + words: ['"object":"account"'] diff --git a/data/rules/tailscale.yml b/data/rules/tailscale.yml index 4e442c8..8ac7e50 100644 --- a/data/rules/tailscale.yml +++ b/data/rules/tailscale.yml @@ -12,8 +12,8 @@ rules: min_entropy: 3.0 confidence: medium examples: - - tskey-secret-12345678-abcd - - tskey-api-abcdefg-123456789 + - tskey-secret-12345678-abcdefghijkl + - tskey-api-abcdefg-1234567890123 references: - https://tailscale.com/kb/1215/oauth-clients validation: diff --git a/src/matcher.rs b/src/matcher.rs index 9048206..89aed1b 100644 --- a/src/matcher.rs +++ b/src/matcher.rs @@ -25,6 +25,7 @@ use smallvec::SmallVec; use tracing::debug; use xxhash_rust::xxh3::xxh3_64; +use crate::rule_profiling::RuleTimer; use crate::{ blob::{Blob, BlobId, BlobIdMap}, entropy::calculate_shannon_entropy, @@ -386,6 +387,8 @@ impl<'a> Matcher<'a> { origin, None, redact, + &filename, + self.profiler.as_ref(), ); } // If tree-sitter produced base64-decoded matches, try them against all rules @@ -407,6 +410,8 @@ impl<'a> Matcher<'a> { origin, Some(ts_match.clone()), redact, + &filename, + self.profiler.as_ref(), ); } } @@ -456,7 +461,21 @@ fn filter_match<'b>( _origin: &OriginSet, ts_match: Option, redact: bool, + filename: &str, + profiler: Option<&Arc>, ) { + let mut timer = profiler.map(|p| { + RuleTimer::new( + p, + rule.id(), + rule.name(), + &rule.syntax.pattern, + filename, + ) + }); + + let initial_len = matches.len(); + // Use Cow to avoid unnecessary copying when ts_match is None let byte_slice: Cow<[u8]> = match ts_match { Some(ts_match_value) => Cow::Owned(ts_match_value.into_bytes()), @@ -515,6 +534,10 @@ fn filter_match<'b>( }); previous_matches.push((rule_id, matching_input_offset_span)); } + if let Some(t) = timer.take() { + let new_count = (matches.len() - initial_len) as u64; + t.end(new_count > 0, new_count, 0); + } } fn get_language_and_queries(lang: &str) -> Option<(Language, FxHashMap)> { match lang.to_lowercase().as_str() { diff --git a/src/scanner/runner.rs b/src/scanner/runner.rs index 86165d4..112a04e 100644 --- a/src/scanner/runner.rs +++ b/src/scanner/runner.rs @@ -75,7 +75,7 @@ pub async fn run_async_scan( progress_enabled, rules_db, enable_profiling, - shared_profiler, + Arc::clone(&shared_profiler), &matcher_stats, )?; @@ -117,7 +117,15 @@ pub async fn run_async_scan( // // Call cmd_report here crate::reporter::run(global_args, Arc::clone(&datastore), args) .context("Failed to run report command")?; - print_scan_summary(start_time, &datastore, global_args, args, rules_db, &matcher_stats); + print_scan_summary( + start_time, + &datastore, + global_args, + args, + rules_db, + &matcher_stats, + if enable_profiling { Some(shared_profiler.as_ref()) } else { None }, + ); Ok(()) } diff --git a/src/scanner/summary.rs b/src/scanner/summary.rs index ff331c4..11a45f9 100644 --- a/src/scanner/summary.rs +++ b/src/scanner/summary.rs @@ -17,6 +17,7 @@ use crate::{ }, findings_store, matcher::MatcherStats, + rule_profiling::ConcurrentRuleProfiler, rules_database::RulesDatabase, }; @@ -42,6 +43,7 @@ pub fn print_scan_summary( // inputs: &FilesystemEnumeratorResult, rules_db: &RulesDatabase, matcher_stats: &Mutex, + profiler: Option<&ConcurrentRuleProfiler>, ) { // let duration = start_time.elapsed(); let ds = datastore.lock().unwrap(); @@ -152,6 +154,47 @@ pub fn print_scan_summary( humantime::format_duration(duration) ); } + + if args.rule_stats { + if let Some(prof) = profiler { + let stats = prof.generate_report(); + if !stats.is_empty() { + // Calculate dynamic column widths + let name_w = stats.iter().map(|s| s.rule_name.len()).max().unwrap_or(4); + let id_w = stats.iter().map(|s| s.rule_id.len()).max().unwrap_or(2); + + // Header + safe_println!("\n{:-^1$}", " Rule Performance Stats ", name_w + id_w + 47); + safe_println!( + "{: 8} {: >15} {: >15}", + "Rule", + "ID", + "Matches", + "Slowest", + "Average", + name_w = name_w, + id_w = id_w + ); + safe_println!("{:-8} {: >15?} {: >15?}", + rs.rule_name, + rs.rule_id, + rs.total_matches, + rs.slowest_match_time, + rs.average_match_time, + name_w = name_w, + id_w = id_w + ); + } + } + } + } + + debug!("\nAll Rules with Matches:"); debug!("======================="); let max_rule_length = sorted_findings.iter().map(|(rule, _)| rule.len()).max().unwrap_or(0); diff --git a/tests/update.rs b/tests/smoke_update.rs similarity index 100% rename from tests/update.rs rename to tests/smoke_update.rs