From 9208589d24ebc0e320b3ec53d251562e40f830cc Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Wed, 25 Jun 2025 08:39:10 -0700 Subject: [PATCH 01/11] Added gradle and grafana rules --- .github/workflows/release.yml | 4 ++ data/rules/adafruitio.yml | 2 +- data/rules/azuresearchquery.yml | 2 +- data/rules/gradle.yml | 33 +++++++++ data/rules/grafana.yml | 114 ++++++++++++++++++++++++++++++++ 5 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 data/rules/gradle.yml create mode 100644 data/rules/grafana.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8cf2a45..5f625a6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -80,6 +80,8 @@ jobs: - name: Build Darwin x64 run: make darwin-x64 + - name: Run tests + run: make tests - name: Move artifacts to dist shell: bash @@ -109,6 +111,8 @@ jobs: - name: Build Darwin arm64 run: make darwin-arm64 + - name: Run tests + run: make tests - name: Move artifacts to dist shell: bash diff --git a/data/rules/adafruitio.yml b/data/rules/adafruitio.yml index ee622f9..17b271f 100644 --- a/data/rules/adafruitio.yml +++ b/data/rules/adafruitio.yml @@ -28,4 +28,4 @@ rules: type: StatusMatch - type: WordMatch words: - - '"username":"kingfishermdb"' \ No newline at end of file + - '"username"' \ No newline at end of file diff --git a/data/rules/azuresearchquery.yml b/data/rules/azuresearchquery.yml index 6cfdbf7..87272fb 100644 --- a/data/rules/azuresearchquery.yml +++ b/data/rules/azuresearchquery.yml @@ -9,7 +9,7 @@ rules: (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) (?:.|[\n\r]){0,32}? ( - [0-9a-zA-Z]{52} + [0-9A-Z]{52} ) \b min_entropy: 3.3 diff --git a/data/rules/gradle.yml b/data/rules/gradle.yml new file mode 100644 index 0000000..42d27ea --- /dev/null +++ b/data/rules/gradle.yml @@ -0,0 +1,33 @@ +rules: + - name: Hardcoded Gradle Credentials + id: kingfisher.gradle.1 + pattern: | + (?xi) + credentials \s* \{ + (?:\s*//.*)* + \s* (?:username|password) \s ['"]([^'"]{1,60})['"] + (?:\s*//.*)* + \s* (?:username|password) \s ['"]([^'"]{1,60})['"] + min_entropy: 3.3 + confidence: medium + examples: + - | + credentials { + username 'user' + password 'password' + } + - | + publishing { + repositories { + maven { + url "http://us01cmsysart01.example.com:8081/artifactory/Mobile-Libs-Internal" + credentials { + // your password here + + username "SOME_USERNAME" + password "SOME_PASSWORD" + } + } + } + - "credentials {\n username 'user'\n password 'password'\n}" + - "credentials {\n username \"user\"\n password \"password\"\n}" \ No newline at end of file diff --git a/data/rules/grafana.yml b/data/rules/grafana.yml new file mode 100644 index 0000000..354d530 --- /dev/null +++ b/data/rules/grafana.yml @@ -0,0 +1,114 @@ +rules: + - name: Grafana API Token + id: kingfisher.grafana.1 + pattern: | + (?xi) + \b + ( + eyJrIjoi[a-z0-9]{60,100} + ) + \b + min_entropy: 3.3 + confidence: medium + examples: + - 'Authorization: Bearer eyJrIjoiWHZiSWd5NzdCYUZnNUtibE8obUpESmE2bzJYNDRIc1UiLCJuIjoibXlrZXkiLCJpZCI7MX1' + - 'admin_client = GrafanaClient("eyJrIjoiY21sM1JRYjB6RnVYSTNLenRWQkFEaWN2bXI2V202U2IiLCJuIjoiYWRtaW5rZXkiLCJpZCI6MX0=", host=grafana_host, port=3000, protocol="http")' + references: + - https://grafana.com/docs/grafana/latest/developers/http_api/auth/ + + - name: Grafana Cloud API Token + id: kingfisher.grafana.2 + pattern: | + (?xi) + \b + ( + glc_ + [a-z0-9+/]{40,150} + ={0,2} + ) + min_entropy: 3.3 + confidence: medium + examples: + - ' "token": "glc_eyJrIjoiZjI0YzZkNGEwZDBmZmZjMmUzNTU3ODcxMmY0ZWZlNTQ1NTljMDFjOCIsIm6iOiJteXRva3VuIiwiaWQiOjF8"' + - 'grafana = glc_etLvNLoNMLt7MTczNNwNbN6Nm1ldGEtbW9paxRvcmlpZt14ZXN4NNwNatN6NLCxdKeH7KTUvWpNqCrHlMKE9EhLcZH7to' + references: + - https://grafana.com/docs/grafana-cloud/developer-resources/api-reference/cloud-api/#regions + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://grafana.com/api/stack-regions + + - name: Grafana Service Account Token + id: kingfisher.grafana.3 + pattern: | + (?x) + \b + (glsa_[a-zA-Z0-9]{32}_[a-fA-F0-9]{8}) + \b + min_entropy: 3.3 + confidence: medium + examples: + - | + curl -H "Authorization: Bearer glsa_HOruNAb7SOiCdshU7algkrq7FDsNSLAa_55e2f8be" -X GET '/api/access-control/user/permissions' | jq + - | + // getData() + // { + // let url="http://localhost:4200/api/search" + // const headers = new HttpHeaders({ + // 'Content-Type': 'application/json', + // 'Authorization': `Bearer glsa_Sof0HKi3agxrQP9qm5r2G98VacBNwV5P_9b638c45` + // }) + // return this.http.get(url, {headers: headers}); + // } + references: + - https://grafana.com/docs/grafana/latest/administration/service-accounts/ + validation: + type: Http + content: + request: + method: GET + headers: + Authorization: Bearer {{ TOKEN }} + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: "{{ GRAFANADOMAIN }}/api/access-control/me" + depends_on_rule: + - rule_id: kingfisher.grafana.4 + variable: GRAFANADOMAIN + + - name: Grafana Domain + id: kingfisher.grafana.4 + pattern: | + (?xi) + \b + ( + (?:https?://)? + (?: + (?:[A-Za-z0-9-]+\.)* + grafana + (?:\.[A-Za-z0-9-]+)+ + ) + (?:\:\d{2,5})? + (?:[/?#][^\s]*)? + ) + \b + min_entropy: 3.0 + visible: false + confidence: medium + examples: + - https://grafana.example.com + - http://grafana.prod.eu-west.mycorp.internal:3000/login + - https://api.team1.grafana.services.cluster.local/health + - grafana.dev.foo-bar.co.uk From 581ef3c9409812069d104044fa81e99badb77bb7 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Wed, 25 Jun 2025 08:53:22 -0700 Subject: [PATCH 02/11] Added generic hashes detection rule and heroku rule --- data/rules/hashes.yml | 109 ++++++++++++++++++++++++++++++++++++++++++ data/rules/heroku.yml | 36 ++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 data/rules/hashes.yml create mode 100644 data/rules/heroku.yml diff --git a/data/rules/hashes.yml b/data/rules/hashes.yml new file mode 100644 index 0000000..ab7cdeb --- /dev/null +++ b/data/rules/hashes.yml @@ -0,0 +1,109 @@ +rules: + - name: Password Hash (md5crypt) + id: kingfisher.pwhash.1 + pattern: '(\$1\$[./A-Za-z0-9]{8}\$[./A-Za-z0-9]{22})' + references: + - https://en.wikipedia.org/wiki/Crypt_(C)#MD5-based_scheme + - https://unix.stackexchange.com/a/511017 + - https://hashcat.net/wiki/doku.php?id=example_hashes + - https://passwordvillage.org/salted.html#md5crypt + min_entropy: 3.3 + confidence: medium + examples: # generated with `openssl passwd -1 -salt 'OKgLCmVl' 'a'` + - '$1$OKgLCmVl$d02jECa4DXn/oXX0R.MoQ/' + - '$1$28772684$iEwNOgGugqO9.bIz5sk8k/' + - name: Password Hash (bcrypt) + id: kingfisher.pwhash.2 + # Format from Wikipedia: + # $2$[cost]$[22 character salt][31 character hash] + pattern: '(\$2[abxy]\$\d+\$[./A-Za-z0-9]{53})' + references: + - https://en.wikipedia.org/wiki/Bcrypt + - https://hashcat.net/wiki/doku.php?id=example_hashes + min_entropy: 3.3 + confidence: medium + examples: + - '$2a$12$R9h/cIPz0gi.URNNX3kh2OPST9/PgBkqquzi.Ss7KIUgO2t0jWMUW' + - '$2a$05$/VT2Xs2dMd8GJKfrXhjYP.DkTjOVrY12yDN7/6I8ZV0q/1lEohLru' + - '$2a$05$Uo385Fa0g86uUXHwZxB90.qMMdRFExaXePGka4WGFv.86I45AEjmO' + - '$2a$05$LhayLxezLhK1LhWvKxCyLOj0j1u.Kj0jZ0pEmm134uzrQlFvQJLF6' + - '$2y$12$atWJ1Nx6ep65tNx0YIJ4I.jzgI86znQbNRI3lF0qIt/XCYnEPxSc2' + - name: Password Hash (sha256crypt) + id: kingfisher.pwhash.3 + pattern: | + (?x) + ( + \$ 5 + (?: \$ rounds=\d+ )? + \$ [./A-Za-z0-9]{8,16} + \$ [./A-Za-z0-9]{43} + ) + references: + - https://en.wikipedia.org/wiki/Crypt_(C)#Key_derivation_functions_supported_by_crypt + - https://hashcat.net/wiki/doku.php?id=example_hashes + - https://passwordvillage.org/salted.html#sha256crypt + min_entropy: 3.3 + confidence: medium + examples: + - '$5$rounds=5000$GX7BopJZJxPc/KEK$le16UF8I2Anb.rOrn22AUPWvzUETDGefUmAV8AZkGcD' + - '$5$9ks3nNEqv31FX.F$gdEoLFsCRsn/WRN3wxUnzfeZLoooVlzeF4WjLomTRFD' + - '$5$KAlz5SULZNybHwil$3UgmS1pmo2r5HG.tjbjzoVxISBh8IH81d.bJh4MCC19' + - name: Password Hash (sha512crypt) + id: kingfisher.pwhash.4 + pattern: | + (?x) + ( + \$ 6 + (?: \$ rounds=\d+ )? + \$ [./A-Za-z0-9]{8,16} + \$ [./A-Za-z0-9]{86} + ) + references: + - https://en.wikipedia.org/wiki/Crypt_(C)#Key_derivation_functions_supported_by_crypt + - https://hashcat.net/wiki/doku.php?id=example_hashes + - https://passwordvillage.org/salted.html#sha512crypt + min_entropy: 3.3 + confidence: medium + examples: + - '$6$52450745$k5ka2p8bFuSmoVT1tzOyyuaREkkKBcCNqoDKzYiJL9RaE8yMnPgh2XzzF0NDrUhgrcLwg78xs1w5pJiypEdFX/' + - '$6$qoE2letU$wWPRl.PVczjzeMVgjiA8LLy2nOyZbf7Amj3qLIL978o18gbMySdKZ7uepq9tmMQXxyTIrS12Pln.2Q/6Xscao0' + - name: Password Hash (Cisco IOS PBKDF2 with SHA256) + id: kingfisher.pwhash.5 + pattern: | + (?x) + ( + \$ 8 + \$ [./A-Za-z0-9]{8,16} + \$ [./A-Za-z0-9]{43} + ) + references: + - https://en.wikipedia.org/wiki/Crypt_(C)#Key_derivation_functions_supported_by_crypt + - https://hashcat.net/wiki/doku.php?id=example_hashes + min_entropy: 3.3 + confidence: medium + examples: + - '$8$TnGX/fE4KGHOVU$pEhnEvxrvaynpi8j4f.EMHr6M.FzU8xnZnBr/tJdFWk' + - '$8$mTj4RZG8N9ZDOk$elY/asfm8kD3iDmkBe3hD2r4xcA/0oWS5V3os.O91u.' + - name: Password Hash (Kerberos 5, etype 23, AS-REP) + id: kingfisher.krb5.asrep.23.1 + pattern: | + (?x) + ( + \$ krb5asrep + \$ 23 + \$ + (?: [^:]+ : )? + [0-9a-f]{32} + \$ [0-9a-f]{64,} + ) + \b + references: + - https://hashcat.net/wiki/doku.php?id=example_hashes + min_entropy: 3.3 + confidence: medium + examples: # Kerberos 5, etype 23, AS-REP + - '$krb5asrep$23$user@domain.com:3e156ada591263b8aab0965f5aebd837$007497cb51b6c8116d6407a782ea0e1c5402b17db7afa6b05a6d30ed164a9933c754d720e279c6c573679bd27128fe77e5fea1f72334c1193c8ff0b370fadc6368bf2d49bbfdba4c5dccab95e8c8ebfdc75f438a0797dbfb2f8a1a5f4c423f9bfc1fea483342a11bd56a216f4d5158ccc4b224b52894fadfba3957dfe4b6b8f5f9f9fe422811a314768673e0c924340b8ccb84775ce9defaa3baa0910b676ad0036d13032b0dd94e3b13903cc738a7b6d00b0b3c210d1f972a6c7cae9bd3c959acf7565be528fc179118f28c679f6deeee1456f0781eb8154e18e49cb27b64bf74cd7112a0ebae2102ac' + - '$krb5asrep$23$8cf8eb5287e28a4006c064892150c4fb$3e05ecc13548bec8e1eeb900dea5429cc6931bae9b8524490eb3a8801560871fe44355ed556202afbb39872e1bbb5c3c4f1b37dcd68fda89a23ebad917d4bbb0933edd94331598939e5d0c0c98c7e219a2e9dd6b877280d1bd7c51131413be577a167208bcc21e9fe7ae8f393278d740e72ca5c44c42d5cb0bf6bab0a36f1b88b7ddc4abbc6f152e652f6ba35c2955fb4132e11b7e566f3b422c3740f79847b77783d245a4e570b8a621b4ff6ff4815566446af70313ee78133707a76a4e4424783bd7c04920aa822a1a36b29f7e25cef186e6439fc46e42e23d6bd918969ef49b8388aef158e443b3a57dbde7ada631fbef7326f9046a9b' + - '$krb5asrep$23$c447eddaebf22ebf006a8fc6f986488c$eb3a17eb56287b474cecad5d4e0490d949977ba3f5015220bcd3080444d5601d67b76c5453b678e8527624e40c273bea4cfe4a7303e136b9bc3b9e63b6fb492ee4b4d2f830c5fa5a55466b57a678f708438f6712354a2deb851792b09270f4941966b82a2fd5ad8fa1fbd95a60b0f9bcd57774b3e55467a02ffcb3f1379104c24e468342f83df20b571e6f34f9a9842b43735d58d94514dcefa76719c0f5c7c3a3bfa770380924625aa0a3472d7c02d10dbb278fd946f7efcfe59a4d4cb7bdb9c5dbddc027611fe333d3ac940ec5b4ed43b55ab54b03cd2df0a9a2a7b5d235c226b259bd5ff8e0e49680351d4f0c4d13e258bc8d383cad6fc2711be0' + - '$krb5asrep$23$771adbc2397abddef676742924414f2b$2df6eb2d9c71820dc3fa2c098e071d920f0e412f5f12411632c5ee70e004da1be6f003b78661f8e4507e173552a52da751c45887c19bc1661ed334e0ccb4ef33975d4bd68b3d24746f281b4ca4fdf98fca0e50a8e845ad7d834e020c05b1495bc473b0295c6e9b94963cb912d3ff0f2f48c9075b0f52d9a31e5f4cc67c7af1d816b6ccfda0da5ccf35820a4d7d79073fa404726407ac840910357ef210fcf19ed81660106dfc3f4d9166a89d59d274f31619ddd9a1e2712c879a4e9c471965098842b44fae7ca6dd389d5d98b7fd7aca566ca399d072025e81cf0ef5075447687f80100307145fade7a8' + - '$krb5asrep$23$user@domain.com:3e156ada591263b8aab0965f5aebd837$007497cb51b6c8116d6407a782ea0e1c5402b17db7afa6b05a6d30ed164a9933c754d720e279c6c573679bd27128fe77e5fea1f72334c1193c8ff0b370fadc6368bf2d49bbfdba4c5dccab95e8c8ebfdc75f438a0797dbfb2f8a1a5f4c423f9bfc1fea483342a11bd56a216f4d5158ccc4b224b52894fadfba3957dfe4b6b8f5f9f9fe422811a314768673e0c924340b8ccb84775ce9defaa3baa0910b676ad0036d13032b0dd94e3b13903cc738a7b6d00b0b3c210d1f972a6c7cae9bd3c959acf7565be528fc179118f28c679f6deeee1456f0781eb8154e18e49cb27b64bf74cd7112a0ebae2102ac' diff --git a/data/rules/heroku.yml b/data/rules/heroku.yml new file mode 100644 index 0000000..000cb05 --- /dev/null +++ b/data/rules/heroku.yml @@ -0,0 +1,36 @@ + - name: Heroku API Key + id: kingfisher.heroku.1 + pattern: | + (?xi) + heroku + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [0-9a-f]{8}-[0-9a-f]{4}- + [0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} + ) + \b + min_entropy: 3.0 + confidence: medium + examples: + - 'HEROKU_API_KEY: c55dbac4-e0e8-4a06-b892-75cac2387ce5' + negative_examples: + - 'curl https://kolkrabbi.heroku.com/apps/98fc74a8-ff56-4a21-85f6-7a1fcac895c9/github/push \\' + references: + - https://devcenter.heroku.com/articles/authentication + # ─── validation ──────────────────────────────────────────── + validation: + type: Http + content: + request: + method: GET + headers: + Accept: application/vnd.heroku+json; version=3 + Authorization: Bearer {{ TOKEN }} + url: https://api.heroku.com/apps + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] From 3ab80a2e29f96e0450c1e2031182244b69b2e193 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Wed, 25 Jun 2025 14:07:11 -0700 Subject: [PATCH 03/11] Added rules for huggingface, ibm cloud, intercom, and ipstack --- data/rules/grafana.yml | 33 +++++++++++++------------------- data/rules/heroku.yml | 14 ++++++-------- data/rules/huggingface.yml | 39 ++++++++++++++++++++++++++++++++++++++ data/rules/ibm.yml | 35 ++++++++++++++++++++++++++++++++++ data/rules/intercom.yml | 35 ++++++++++++++++++++++++++++++++++ data/rules/ionic.yml | 28 +++++++++++++++++++++++++++ data/rules/ipstack.yml | 33 ++++++++++++++++++++++++++++++++ 7 files changed, 189 insertions(+), 28 deletions(-) create mode 100644 data/rules/huggingface.yml create mode 100644 data/rules/ibm.yml create mode 100644 data/rules/intercom.yml create mode 100644 data/rules/ionic.yml create mode 100644 data/rules/ipstack.yml diff --git a/data/rules/grafana.yml b/data/rules/grafana.yml index 354d530..8a3b859 100644 --- a/data/rules/grafana.yml +++ b/data/rules/grafana.yml @@ -78,32 +78,25 @@ rules: method: GET headers: Authorization: Bearer {{ TOKEN }} - response_matcher: - - report_response: true - - status: - - 200 - type: StatusMatch - url: "{{ GRAFANADOMAIN }}/api/access-control/me" + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: "{{ GRAFANADOMAIN }}/api/access-control/me" depends_on_rule: - rule_id: kingfisher.grafana.4 variable: GRAFANADOMAIN - + - name: Grafana Domain id: kingfisher.grafana.4 pattern: | - (?xi) - \b - ( - (?:https?://)? - (?: - (?:[A-Za-z0-9-]+\.)* - grafana - (?:\.[A-Za-z0-9-]+)+ - ) - (?:\:\d{2,5})? - (?:[/?#][^\s]*)? - ) - \b + (?xi) + (?:https?://)? + (?:[A-Za-z0-9-]+\.)* + grafana\.[A-Za-z0-9.-]+ + (?::\d{2,5})? + (?:[/?\#]\S*)? min_entropy: 3.0 visible: false confidence: medium diff --git a/data/rules/heroku.yml b/data/rules/heroku.yml index 000cb05..817a2a4 100644 --- a/data/rules/heroku.yml +++ b/data/rules/heroku.yml @@ -1,3 +1,4 @@ +rules: - name: Heroku API Key id: kingfisher.heroku.1 pattern: | @@ -16,11 +17,8 @@ confidence: medium examples: - 'HEROKU_API_KEY: c55dbac4-e0e8-4a06-b892-75cac2387ce5' - negative_examples: - - 'curl https://kolkrabbi.heroku.com/apps/98fc74a8-ff56-4a21-85f6-7a1fcac895c9/github/push \\' references: - https://devcenter.heroku.com/articles/authentication - # ─── validation ──────────────────────────────────────────── validation: type: Http content: @@ -29,8 +27,8 @@ headers: Accept: application/vnd.heroku+json; version=3 Authorization: Bearer {{ TOKEN }} - url: https://api.heroku.com/apps - response_matcher: - - report_response: true - - type: StatusMatch - status: [200] + url: https://api.heroku.com/apps + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] diff --git a/data/rules/huggingface.yml b/data/rules/huggingface.yml new file mode 100644 index 0000000..d103438 --- /dev/null +++ b/data/rules/huggingface.yml @@ -0,0 +1,39 @@ +rules: + - name: HuggingFace User Access Token + id: kingfisher.huggingface.1 + pattern: | + (?xi) + \b + (?: + ( + (?:api_org|hf)_ + (?:[0-9A-Z]{17}){2} + ) + ) + \b + references: + - https://huggingface.co/docs/hub/security-tokens + min_entropy: 3.3 + confidence: medium + examples: + - 'HF_TOKEN:"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"' + - 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 + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + Content-Type: application/json + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + - match_all_words: true + type: WordMatch + words: + - '"name":' + - '"id":' + url: https://huggingface.co/api/whoami-v2 \ No newline at end of file diff --git a/data/rules/ibm.yml b/data/rules/ibm.yml new file mode 100644 index 0000000..0cf2e24 --- /dev/null +++ b/data/rules/ibm.yml @@ -0,0 +1,35 @@ +rules: + - name: IBM Cloud User API Key + id: kingfisher.ibm.1 + pattern: | + (?xi) + (?:ibm(?:cloud)?|bx) + (?:.|[\n\r]){0,32}? + \b + ( + [0-9A-Z_-]{42,44} + ) + min_entropy: 3.5 + confidence: medium + + examples: + - ibmcloud_apikey = abcdef0123_56789abcdef0123456789abcdef01234 + - ibm_platform_key="f-_RrJDVnuVh07HNTcmnQx_b6CbcQsxmEarVm9P_RWtF" + + references: + - https://cloud.ibm.com/docs/account?topic=account-userapikey + - https://cloud.ibm.com/apidocs/iam-identity-token-api + + validation: + type: Http + content: + request: + method: GET + headers: + Authorization: Basic Yng6Yng= # public “bx:bx” client credentials + Accept: application/json + url: https://iam.cloud.ibm.com/v1/apikeys/details?apikey={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] diff --git a/data/rules/intercom.yml b/data/rules/intercom.yml new file mode 100644 index 0000000..8c16153 --- /dev/null +++ b/data/rules/intercom.yml @@ -0,0 +1,35 @@ +rules: + - name: Intercom API Token + id: kingfisher.intercom.1 + pattern: | + (?xi) + (?:^|[\s"'=]) + (?:intercom(?:_access)?|ic) + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,16}? + ( + [0-9A-Z+/]{59}= + ) + min_entropy: 3.5 + confidence: medium + + examples: + - "intercom_access_token: dG9rOvI0NmJlMTA5XzQwM2NfNDVlM184MjQzXzkwMDnmOTE1NGIyONoxOjA=" + - ic_token = "g1ZsclJXTjNfc1pBSzJDemE0eFVDU0U5c25CeDN4Vm9hQ2Zac0hXemZHNGVDPQ==" + + references: + - https://developers.intercom.com/docs/build-an-integration/learn-more/rest-apis + validation: + type: Http + content: + request: + method: GET + headers: + Accept: application/json + Authorization: Bearer {{ TOKEN }} + url: https://api.intercom.io/me + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 diff --git a/data/rules/ionic.yml b/data/rules/ionic.yml new file mode 100644 index 0000000..1130a38 --- /dev/null +++ b/data/rules/ionic.yml @@ -0,0 +1,28 @@ +rules: + - name: Ionic API token + id: kingfisher.ionic.1 + pattern: | + (?xi) + \b + ( + ion_ + [a-z0-9]{42} + ) + \b + min_entropy: 3.5 + confidence: medium + examples: + - ion_VNR17uGgdxr9P2aOrCulvSLTFDqijIV2ImQsOUhDEI + validation: + type: Http + content: + request: + headers: + Authorization: 'Bearer {{ TOKEN }}' + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://api.ionic.io/v1/auth/status \ No newline at end of file diff --git a/data/rules/ipstack.yml b/data/rules/ipstack.yml new file mode 100644 index 0000000..a70e43e --- /dev/null +++ b/data/rules/ipstack.yml @@ -0,0 +1,33 @@ +rules: + - name: IpStack API Key + id: kingfisher.ipstack.1 + pattern: | + (?xi) + \b + ipstack + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + (?:[0-9a-f]{16}){2} + ) + \b + min_entropy: 3.0 + confidence: medium + examples: + - "ipstack_token=123e4567e89b12d3a456426614174000" + - "ipstack_key=abcdefabcdefabcdefabcdefabcdef12" + references: + - https://ipstack.com/documentation + validation: + type: Http + content: + request: + method: GET + url: http://api.ipstack.com/1.1.1.1?access_key={{ TOKEN }} + response_matcher: + - type: WordMatch + words: + - '"ip":"1.1.1.1"' + - '"continent_code"' \ No newline at end of file From cc75be309b953da97fd3e29fd5dc874aff94e3a7 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Wed, 25 Jun 2025 14:22:37 -0700 Subject: [PATCH 04/11] Added rules for line --- data/rules/jenkins.yml | 24 +++++++++++++++++++ data/rules/jira.yml | 54 ++++++++++++++++++++++++++++++++++++++++++ data/rules/line.yml | 36 ++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+) create mode 100644 data/rules/jenkins.yml create mode 100644 data/rules/jira.yml create mode 100644 data/rules/line.yml diff --git a/data/rules/jenkins.yml b/data/rules/jenkins.yml new file mode 100644 index 0000000..f8fbb77 --- /dev/null +++ b/data/rules/jenkins.yml @@ -0,0 +1,24 @@ +rules: + - name: Jenkins Token or Crumb + id: kingfisher.jenkins.1 + pattern: '(?i)jenkins.{0,10}(?:crumb)?.{0,10}\b([0-9a-f]{32,36})\b' + categories: [api, fuzzy, secret] + min_entropy: 3.3 + confidence: medium + examples: + - | + jenkins_user = 'root' + # jenkins_passwd = '116365fd86d63bf507aba962606a5c8956' Pre token + jenkins_passwd = '11811f784531053132519844d047186074' # Dev Token + jenkins_url = 'http://10.1.188.121' + - | + export JENKINS_USER=justin-admin-edit-view + export JENKINS_TOKEN=11f4274ec59be12eace9a08b08ee13d54b + export JENKINS=jenkins-cicd.apps.sno.openshiftlabs.net + - | + sh "curl -X POST 'http://jenkins.lsfusion.luxsoft.by/job/${Paths.updateParentVersionsJob}/build' --user ${USERPASS} -H 'Jenkins-Crumb:440561953171ba44ace9740562d172bb'" + negative_examples: + - '1. ~~Does not play well with [Build Token Root Plugin](https://wiki.jenkins-ci.org/display/JENKINS/Build+Token+Root+Plugin) URL formats.~~ (added with [this commit](https://github.com/morficus/Parameterized-Remote-Trigger-Plugin/commit/f687dbe75d1c4f39f7e14b68220890384d7c5674) )' + references: + - https://www.jenkins.io/blog/2018/07/02/new-api-token-system/ + - https://www.jenkins.io/doc/book/security/csrf-protection/ \ No newline at end of file diff --git a/data/rules/jira.yml b/data/rules/jira.yml new file mode 100644 index 0000000..e106e77 --- /dev/null +++ b/data/rules/jira.yml @@ -0,0 +1,54 @@ +rules: + - name: Jira Domain + id: kingfisher.jira.1 + pattern: | + (?x) + (?i) + ( + [a-z][a-z0-9-]{5,24}\.atlassian\.net + ) + \b + min_entropy: 3.5 + visible: false + confidence: medium + examples: + - example-jira.atlassian.net + - jira.sprintUri= https://leakyday.atlassian.net/rest + + - name: Jira Token + id: kingfisher.jira.2 + pattern: | + (?x) + (?i) + \b + jira + (?:.|[\n\r]){0,8}? + (?:SECRET|PRIVATE|ACCESS|KEY|PASSWORD|TOKEN) + (?:.|[\n\r]){0,16}? + \b + ( + [a-z0-9-]{24} + ) + \b + min_entropy: 3.3 + confidence: medium + examples: + - 'Here is my Jira token: VDOheDe1sSCeGkuTARhkFDE2' + - public static final String JIRA_PASSWORD = "VDOheDe1sSCeGkuTARhkFDE2"; + validation: + type: Http + content: + request: + headers: + Accept: application/json + Authorization: Basic {{ TOKEN }} + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://{{ DOMAIN }}/rest/api/3/dashboard + depends_on_rule: + - rule_id: kingfisher.jira.1 + variable: DOMAIN \ No newline at end of file diff --git a/data/rules/line.yml b/data/rules/line.yml new file mode 100644 index 0000000..3dd453a --- /dev/null +++ b/data/rules/line.yml @@ -0,0 +1,36 @@ +rules: + - name: Line Messaging API Token + id: kingfisher.line.1 + pattern: | + (?x) + (?i) + \b + line + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + (?:[0-9A-Za-z+/]{57}){3}=? + ) + min_entropy: 3.5 + confidence: medium + examples: + - line_access_token = 13IRTqF+j0TfDtuJoIWKRBPhpDnqYUaaSlOilnoy0urLE+kbf5hN4HUf5pSPw20ruyO0BFFF1IDjnBojctp5emFw0hZ51WxB8c75qo48upJInfmqDQ1xrFd4yFKBwx4yRBHYXmI/FyrtcWKd0FBoBAdB04t81/1O/w1cDnyilFU= + - linemsg_token:"13IRTqF+j0TfDtuJoIWKRBPhpDnqYUaaSlOilnoy0urLE+kbf5hN4HUf5pSPw20ruyO0BFFF1IDjnBojctp5emFw0hZ51WxB8c75qo48upJInfmqDQ1xrFd4yFKBwx4yRBHYXmI/FyrtcWKd0FBoBAdB04t81/1O/w1cDnyilFU=" + references: + - https://developers.line.biz/en/reference/messaging-api/#get-consumption + validation: + type: Http + content: + request: + headers: + Authorization: 'Bearer {{ TOKEN }}' + Content-Type: application/json + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://api.line.me/v2/bot/message/quota/consumption \ No newline at end of file From 83e37fa9f49e65a7b13c79e6243e34d9773a7de3 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Wed, 25 Jun 2025 14:27:10 -0700 Subject: [PATCH 05/11] Added rules for linear --- data/rules/linear.yml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 data/rules/linear.yml diff --git a/data/rules/linear.yml b/data/rules/linear.yml new file mode 100644 index 0000000..da4330b --- /dev/null +++ b/data/rules/linear.yml @@ -0,0 +1,38 @@ +rules: + - name: Linear API Key + id: kingfisher.linear.1 + pattern: | + (?x) + (?i) + \b + ( + lin_api_ + (?:[0-9A-Za-z]{8}){5} + ) + \b + min_entropy: 3.5 + confidence: medium + examples: + - linear_api_key = lin_api_2thngjik222gkiihzivh242LU7zvkdvdgB14B41S + - lin_api_token:"lin_api_9A6bCDeF0Gh1Ij2Klm3No4PQr5St6Uv7Wx8YZaBc" + references: + - https://linear.app/developers/graphql + validation: + type: Http + content: + request: + method: POST + headers: + Authorization: '{{ TOKEN }}' + Content-Type: application/json + body: > + { + "query": "query { issues(first: 1) { nodes { id } } }" + } + url: https://api.linear.app/graphql + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"issues":', '"nodes":'] \ No newline at end of file From 71eb6b20705f26d8dadf80c62a664162e0d6cd21 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Wed, 25 Jun 2025 14:40:15 -0700 Subject: [PATCH 06/11] Added rules for linkedin, mailchimp, mailgun --- data/rules/linkedin.yml | 67 ++++++++++++++++++++++++++++++++++++++++ data/rules/mailchimp.yml | 36 +++++++++++++++++++++ data/rules/mailgun.yml | 63 +++++++++++++++++++++++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 data/rules/linkedin.yml create mode 100644 data/rules/mailchimp.yml create mode 100644 data/rules/mailgun.yml diff --git a/data/rules/linkedin.yml b/data/rules/linkedin.yml new file mode 100644 index 0000000..97b4865 --- /dev/null +++ b/data/rules/linkedin.yml @@ -0,0 +1,67 @@ +rules: + - name: LinkedIn Client ID + id: kingfisher.linkedin.1 + pattern: | + (?x)(?i) + linkedin + .? + (?: api | app | application | client | consumer | customer )? + .? + (?: id | identifier | key ) + .{0,2} \s{0,20} .{0,2} \s{0,20} .{0,2} + \b ([a-z0-9]{12,14}) \b + references: + - https://docs.microsoft.com/en-us/linkedin/shared/api-guide/best-practices/secure-applications + min_entropy: 2.5 + confidence: medium + examples: + - 'Email ID Last 5 Digits of your SSN LinkedIn ID Availability' + - | + LINKEDIN_KEY = "77yg7tx91p4lag" + LINKEDIN_SECRET = "zt7GeN6IH911xvRj" + validation: + type: Http + content: + request: + headers: + Authorization: "Bearer {{ TOKEN }}" + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://api.linkedin.com/v2/me + + - name: LinkedIn Secret Key + id: kingfisher.linkedin.2 + pattern: | + (?x)(?i) + linkedin + .? + (?: api | app | application | client | consumer | customer | secret | key ) + .? + (?: key | oauth | sec | secret )? + .{0,2} \s{0,20} .{0,2} \s{0,20} .{0,2} + \b ([a-z0-9]{16}) \b + references: + - https://docs.microsoft.com/en-us/linkedin/shared/api-guide/best-practices/secure-applications + min_entropy: 3.3 + confidence: medium + examples: + - | + LINKEDIN_KEY = "77yg7tx91p4lag" + LINKEDIN_SECRET = "zt7GeN6IH911xvRj" + validation: + type: Http + content: + request: + headers: + Authorization: "Bearer {{ TOKEN }}" + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://api.linkedin.com/v2/me \ No newline at end of file diff --git a/data/rules/mailchimp.yml b/data/rules/mailchimp.yml new file mode 100644 index 0000000..a28eaae --- /dev/null +++ b/data/rules/mailchimp.yml @@ -0,0 +1,36 @@ +rules: + - name: Mailchimp API Key + id: kingfisher.mailchimp.1 + pattern: | + (?xi) + mailchimp + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + (?:[0-9a-f]{8}){4} + -us\d{1,2} + ) + min_entropy: 3.5 + confidence: medium + examples: + - 'mailchimp_token = abcdef1234567890abcdef1234567890-us1' + - 'mailchimp_key = "0123456789abcdeffedcba9876543210-us20"' + validation: + type: Http + content: + request: + headers: + Authorization: 'Basic {{ "x:" | append: TOKEN | b64enc }}' + Accept: application/json + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: 'https://{{ TOKEN | split: "-" | last }}.api.mailchimp.com/3.0/ping' + references: + - https://mailchimp.com/developer/marketing/api/root/ + - https://mailchimp.com/developer/guides/marketing-api-conventions/ \ No newline at end of file diff --git a/data/rules/mailgun.yml b/data/rules/mailgun.yml new file mode 100644 index 0000000..4bac90e --- /dev/null +++ b/data/rules/mailgun.yml @@ -0,0 +1,63 @@ +rules: + - name: MailGun Token + id: kingfisher.mailgun.1 + pattern: | + (?x) + (?i) + \b + mailgun + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + (?:[0-9A-Za-z-]{24}){3} + ) + min_entropy: 3.5 + confidence: medium + examples: + - mailgun_api_key=abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456abcdef123456 + validation: + type: Http + content: + request: + headers: + Authorization: Basic {{ TOKEN | b64enc }} + Accept: application/json + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://api.mailgun.net/v3/address/validate?address=test@example.com + - name: MailGun Primary Key + id: kingfisher.mailgun.2 + pattern: | + (?x) + (?i) + (?:mailgun|mg) + (?:.|[\n\r]){0,64}? + \b + ( + key-(?:[0-9a-f]{8}){4} + ) + \b + min_entropy: 3.5 + confidence: medium + examples: + - key-mailgun_token= key-ad13dfc23adf55fa404a91e76d96f472 + validation: + type: Http + content: + request: + headers: + Authorization: 'Basic {{ "api:" | append: TOKEN | b64enc }}' + Accept: application/json + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + url: https://api.mailgun.net/v3/address/validate?address=test@example.com \ No newline at end of file From 3fcb1c75d63f542617fdb25a293750dbd4efde81 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Wed, 25 Jun 2025 15:12:42 -0700 Subject: [PATCH 07/11] Fixed some rules. Added rules for mandrill, mapbox, microsoft teams, stripe --- data/rules/azuredevops.yml | 2 +- data/rules/fileio.yml | 5 +- data/rules/frame.io.yml | 1 - data/rules/intercom.yml | 4 +- data/rules/linear.yml | 2 +- data/rules/mandrill.yml | 38 ++++++++++++++ data/rules/mapbox.yml | 74 ++++++++++++++++++++++++++++ data/rules/microsoft_teams.yml | 52 +++++++++++++++++++ data/rules/microsoftteamswebhook.yml | 38 ++++++++++++++ data/rules/stripe.yml | 60 ++++++++++++++++++++++ data/rules/tailscale.yml | 4 +- 11 files changed, 270 insertions(+), 10 deletions(-) create mode 100644 data/rules/mandrill.yml create mode 100644 data/rules/mapbox.yml create mode 100644 data/rules/microsoft_teams.yml create mode 100644 data/rules/microsoftteamswebhook.yml create mode 100644 data/rules/stripe.yml diff --git a/data/rules/azuredevops.yml b/data/rules/azuredevops.yml index 2597a77..4188999 100644 --- a/data/rules/azuredevops.yml +++ b/data/rules/azuredevops.yml @@ -13,7 +13,7 @@ rules: min_entropy: 3 confidence: medium examples: - - azure devops pat = FBdFol081crwkIHWJH2yiqDDyrFjVSi7HWl22hN2hTYfsB8NlGDpJQQJ77BAACAAAAAAAAAAAAASAZDOBucT + - azure devops pat = FBdFol081crwkIHWJH2yiqDDyrFjVSi7HWl22hN2hTYfsB8NlGDpJQQJ77BAACAAAAAAAAAAAAASAZDOBucTj references: - https://learn.microsoft.com/en-us/rest/api/azure/devops/profile/profiles/get?view=azure-devops-rest-7.1&tabs=HTTP - https://learn.microsoft.com/en-us/azure/devops/release-notes/2024/general/sprint-241-update diff --git a/data/rules/fileio.yml b/data/rules/fileio.yml index c6413a1..6bdd8dd 100644 --- a/data/rules/fileio.yml +++ b/data/rules/fileio.yml @@ -10,9 +10,8 @@ rules: (?:.|[\n\r]){0,16}? \b ( - [A-Z0-9]{16} - (?:\.[A-Z0-9]{7}){2} - \.[A-Z0-9]{8} + [A-Z0-9]{20} + \.[A-Z0-9]{20} ) \b min_entropy: 3.3 diff --git a/data/rules/frame.io.yml b/data/rules/frame.io.yml index 8b3b562..a5ef94b 100644 --- a/data/rules/frame.io.yml +++ b/data/rules/frame.io.yml @@ -12,7 +12,6 @@ rules: confidence: medium examples: - fio-u-TaWoPIBovaGCbBkUtGPKWS0D3cu254VA33IFCCrtwl8J2Dtq2pMJ9MvNHmNoL2XX - - ffio-u-TaWoPIBovaGCbBkUtGPKWS0D3cu254VA33IFCCrtwl8J2Dtq2pMJ9MvNHmNoL2XX references: - https://developer.frame.io/api/reference/operation/getMe/ validation: diff --git a/data/rules/intercom.yml b/data/rules/intercom.yml index 8c16153..0c75e33 100644 --- a/data/rules/intercom.yml +++ b/data/rules/intercom.yml @@ -2,9 +2,9 @@ rules: - name: Intercom API Token id: kingfisher.intercom.1 pattern: | - (?xi) - (?:^|[\s"'=]) + (?xi) (?:intercom(?:_access)?|ic) + (?:.|[\n\r]){0,16}? (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) (?:.|[\n\r]){0,16}? ( diff --git a/data/rules/linear.yml b/data/rules/linear.yml index da4330b..9adf6e8 100644 --- a/data/rules/linear.yml +++ b/data/rules/linear.yml @@ -29,7 +29,7 @@ rules: { "query": "query { issues(first: 1) { nodes { id } } }" } - url: https://api.linear.app/graphql + url: https://api.linear.app/graphql response_matcher: - report_response: true - type: StatusMatch diff --git a/data/rules/mandrill.yml b/data/rules/mandrill.yml new file mode 100644 index 0000000..4f0228c --- /dev/null +++ b/data/rules/mandrill.yml @@ -0,0 +1,38 @@ +rules: + - name: Mandrill API Key + id: kingfisher.mandrill.1 + pattern: | + (?x) + (?i) + \b + mandrill + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + (?:[0-9A-Za-z_-]{11}){2} + ) + min_entropy: 3.5 + confidence: medium + examples: + - mandrill_token = taqnVL1P5AJrM4oU4opSqQ + categories: + - api + - identifier + validation: + type: Http + content: + request: + method: POST + headers: + Content-Type: application/json + body: | + { "key": "{{ TOKEN }}" } + url: https://mandrillapp.com/api/1.0/users/ping.json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"PONG!"'] \ No newline at end of file diff --git a/data/rules/mapbox.yml b/data/rules/mapbox.yml new file mode 100644 index 0000000..d29ffdf --- /dev/null +++ b/data/rules/mapbox.yml @@ -0,0 +1,74 @@ +rules: + - name: Mapbox Public Access Token + id: kingfisher.mapbox.1 + pattern: '(?i)(?s)mapbox.{0,30}(pk\.[a-z0-9\-+/=]{32,128}\.[a-z0-9\-+/=]{20,30})(?:[^a-z0-9\-+/=]|$)' + min_entropy: 3.3 + confidence: medium + examples: + - | + mapboxApiKey: + 'pk.eyJ1Ijoia3Jpc3R3IiwiYSI6ImNqbGg1N242NTFlczczdnBcf99iMjgzZ2sifQ.lUneM-o3NucXN189EYyXxQ' + references: + - https://docs.mapbox.com/api/accounts/tokens/#token-format + - https://docs.mapbox.com/help/getting-started/access-tokens/ + - https://docs.mapbox.com/help/troubleshooting/how-to-use-mapbox-securely + validation: + type: Http + content: + request: + method: GET + # smallest public-data endpoint: returns JSON style definition + url: https://api.mapbox.com/styles/v1/mapbox/streets-v11?access_token={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + + - name: Mapbox Secret Access Token + id: kingfisher.mapbox.2 + pattern: '(?i)(?s)mapbox.{0,30}(sk\.[a-z0-9\-+/=]{32,128}\.[a-z0-9\-+/=]{20,30})(?:[^a-z0-9\-+/=]|$)' + min_entropy: 3.3 + confidence: medium + examples: + - " //mapboxgl.accessToken = 'sk.eyJ1Ijoic2hlbmdsaWgiLCJhIjCf99ttaWF5bDBsMGNlaDJubGZyMGUwZXNmaCJ9.eI8KXNm5zKZXOKh0c8u9vg';" + - 'export MAPBOX_SECRET_TOKEN=sk.eyJ1IjoiY2FwcGVsYWVyZSIsImEicf99c1BaTkZnIn0.P4lD1eHeSEx7AsBq1zbJ4g' + references: + - https://docs.mapbox.com/api/accounts/tokens/#token-format + - https://docs.mapbox.com/help/getting-started/access-tokens/ + - https://docs.mapbox.com/help/troubleshooting/how-to-use-mapbox-securely + validation: + type: Http + content: + request: + method: GET + url: https://api.mapbox.com/styles/v1/mapbox/streets-v11?access_token={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid + + - name: Mapbox Temporary Access Token + id: kingfisher.mapbox.3 + pattern: '(?i)(?s)mapbox.{0,30}(tk\.[a-z0-9\-+/=]{32,128}\.[a-z0-9\-+/=]{20,30})(?:[^a-z0-9\-+/=]|$)' + min_entropy: 3.3 + confidence: medium + examples: + - " //mapboxgl.accessToken = 'tk.eyJ1Ijoic2hlbmdsaWgiLCJhIjCf99ttaWF5bDBsMGNlaDJubGZyMGUwZXNmaCJ9.eI8KXNm5zKZXOKh0c8u9vg';" + - 'export MAPBOX_TEMP_TOKEN=tk.eyJ1IjoiY2FwcGVsYWVyZSIsImEicf99c1BaTkZnIn0.P4lD1eHeSEx7AsBq1zbJ4g' + references: + - https://docs.mapbox.com/api/accounts/tokens/#token-format + - https://docs.mapbox.com/help/getting-started/access-tokens/ + - https://docs.mapbox.com/help/troubleshooting/how-to-use-mapbox-securely + validation: + type: Http + content: + request: + method: GET + url: https://api.mapbox.com/styles/v1/mapbox/streets-v11?access_token={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid diff --git a/data/rules/microsoft_teams.yml b/data/rules/microsoft_teams.yml new file mode 100644 index 0000000..13cc044 --- /dev/null +++ b/data/rules/microsoft_teams.yml @@ -0,0 +1,52 @@ +rules: + - name: Microsoft Teams Webhook + id: kingfisher.msteams.1 + pattern: | + (?xi) + ( + https:// + outlook\.office\.com/webhook/ + [0-9a-f]{8}- + [0-9a-f]{4}- + [0-9a-f]{4}- + [0-9a-f]{4}- + [0-9a-f]{12} + @ + [0-9a-f]{8}- + [0-9a-f]{4}- + [0-9a-f]{4}- + [0-9a-f]{4}- + [0-9a-f]{12} + /IncomingWebhook/ + [0-9a-f]{32} + / + [0-9a-f]{8}- + [0-9a-f]{4}- + [0-9a-f]{4}- + [0-9a-f]{4}- + [0-9a-f]{12} + ) + min_entropy: 3.3 + confidence: medium + examples: + - 'https://outlook.office.com/webhook/9da5da9c-4218-4c22-aed6-b5c8baebfff5@2f2b54b7-0141-4ba7-8fcd-ab7d17a60547/IncomingWebhook/1bf66ccbb8e745e791fa6e6de0cf465b/4361420b-8fde-48eb-b62a-0e34fec63f5c' + - 'https://outlook.office.com/webhook/fa4983ab-49ea-4c1b-9297-2658ea56164c@f784fbed-7fc7-4c7a-aae9-d2f387b67c5d/IncomingWebhook/4d2b3a16113d47b080b7a083b5a5e533/74f315eb-1dde-4731-b6b5-2524b77f2acd' + - 'https://outlook.office.com/webhook/555aa7fc-ea71-4fb7-ae9e-755caa4404ed@72f988bf-86f1-41af-91ab-2d7cd011db47/IncomingWebhook/16085df23e564bb9076842605ede3af2/51dab674-ad95-4f0a-8964-8bdefc25b6d9' + - 'https://outlook.office.com/webhook/2f92c502-7feb-4a6c-86f1-477271ae576f@990414fa-d0a3-42f5-b740-21d865a44a28/IncomingWebhook/54e43eb586f14aa9984d5c0bec3d5050/539ce6fa-e9aa-413f-a79b-fb7e8998fcac' + validation: + type: Http + content: + request: + method: POST + url: '{{ TOKEN }}' + headers: + Content-Type: application/json + body: '{"text":""}' + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 400 + - type: WordMatch + words: + - 'Text is required' \ No newline at end of file diff --git a/data/rules/microsoftteamswebhook.yml b/data/rules/microsoftteamswebhook.yml new file mode 100644 index 0000000..3fc349c --- /dev/null +++ b/data/rules/microsoftteamswebhook.yml @@ -0,0 +1,38 @@ +rules: + - name: Microsoft Teams Webhook + id: kingfisher.microsoftteamswebhook.1 + pattern: | + (?x) + https://[a-zA-Z0-9]+\.webhook\.office\.com/webhookb2 + / + [a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12} + @ + [a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12} + / + IncomingWebhook + / + [a-zA-Z0-9]{32} + / + [a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12} + min_entropy: 3.3 + confidence: medium + examples: + - "https://contoso.webhook.office.com/webhookb2/12345678-abcd-1234-efgh-56789abcdef0@12345678-abcd-1234-efgh-56789abcdef0/IncomingWebhook/abcdefgh12345678abcdefgh12345678/12345678-abcd-1234-efgh-56789abcdef0" + validation: + type: Http + content: + request: + body: | + {'text':''} + headers: + Content-Type: application/json + method: POST + response_matcher: + - type: StatusMatch + status: + - 200 + - report_response: true + type: WordMatch + words: + - "Text is required" + url: '{{ TOKEN }}' \ No newline at end of file diff --git a/data/rules/stripe.yml b/data/rules/stripe.yml new file mode 100644 index 0000000..92a9d2e --- /dev/null +++ b/data/rules/stripe.yml @@ -0,0 +1,60 @@ +rules: + - name: Stripe Publishable Key + id: kingfisher.stripe.1 + + pattern: | + (?xi) + (?:stripe|strp) + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + ( + pk_live_ + (?:[0-9A-Za-z]{6}){4,30} + ) + (?=$|[\s"']) + min_entropy: 3.3 + confidence: medium + categories: [api, key] + examples: + - stripe_pub_key = pk_live_HQS0j4H75XpthOW87eY1sXa2BYz3Ab + + - name: Stripe Secret / Restricted Key + id: kingfisher.stripe.2 + + pattern: | + (?ix) + (?:^|[\s"'=]) + (?:stripe|strp) + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + ( + (?: + sk|rk + )_live_ + (?:[0-9A-Za-z]{8}){3,25} + ) + (?=$|[\s"']) + + min_entropy: 3.3 + confidence: medium + categories: [api, key] + examples: + - stripe_secret_key = sk_live_f01c79xuuug7yodgzj5ws0h1x2kyvho3 + - "strp_sec_key: rk_live_4haG9YwGkL2hXqTj5pSzo8FzB3uCwE7n" + + validation: + type: Http + content: + request: + method: GET + headers: + Authorization: Bearer {{ TOKEN }} + Accept: application/json + url: https://api.stripe.com/v1/account + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + match_all_words: true + words: ['"object":"account"'] diff --git a/data/rules/tailscale.yml b/data/rules/tailscale.yml index 4e442c8..8ac7e50 100644 --- a/data/rules/tailscale.yml +++ b/data/rules/tailscale.yml @@ -12,8 +12,8 @@ rules: min_entropy: 3.0 confidence: medium examples: - - tskey-secret-12345678-abcd - - tskey-api-abcdefg-123456789 + - tskey-secret-12345678-abcdefghijkl + - tskey-api-abcdefg-1234567890123 references: - https://tailscale.com/kb/1215/oauth-clients validation: From a1cecef6856057c4aeb5d77a70296cc5150bd21a Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Wed, 25 Jun 2025 15:46:33 -0700 Subject: [PATCH 08/11] Added netlify, netrc, newrelic, ngrok, npm, and nuget rules --- data/rules/mapbox.yml | 37 +++++++++++----------- data/rules/netlify.yml | 62 +++++++++++++++++++++++++++++++++++++ data/rules/netrc.yml | 29 ++++++++++++++++++ data/rules/newrelic.yml | 33 ++++++++++++++++++++ data/rules/ngrok.yml | 31 +++++++++++++++++++ data/rules/npm.yml | 65 +++++++++++++++++++++++++++++++++++++++ data/rules/nuget.yml | 68 +++++++++++++++++++++++++++++++++++++++++ data/rules/stripe.yml | 22 ++++++------- 8 files changed, 315 insertions(+), 32 deletions(-) create mode 100644 data/rules/netlify.yml create mode 100644 data/rules/netrc.yml create mode 100644 data/rules/newrelic.yml create mode 100644 data/rules/ngrok.yml create mode 100644 data/rules/npm.yml create mode 100644 data/rules/nuget.yml diff --git a/data/rules/mapbox.yml b/data/rules/mapbox.yml index d29ffdf..d0253d6 100644 --- a/data/rules/mapbox.yml +++ b/data/rules/mapbox.yml @@ -17,13 +17,12 @@ rules: content: request: method: GET - # smallest public-data endpoint: returns JSON style definition - url: https://api.mapbox.com/styles/v1/mapbox/streets-v11?access_token={{ TOKEN }} - response_matcher: - - report_response: true - - type: StatusMatch - status: [200] - - type: JsonValid + url: https://api.mapbox.com/styles/v1/mapbox/streets-v11?access_token={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid - name: Mapbox Secret Access Token id: kingfisher.mapbox.2 @@ -42,12 +41,12 @@ rules: content: request: method: GET - url: https://api.mapbox.com/styles/v1/mapbox/streets-v11?access_token={{ TOKEN }} - response_matcher: - - report_response: true - - type: StatusMatch - status: [200] - - type: JsonValid + url: https://api.mapbox.com/styles/v1/mapbox/streets-v11?access_token={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid - name: Mapbox Temporary Access Token id: kingfisher.mapbox.3 @@ -66,9 +65,9 @@ rules: content: request: method: GET - url: https://api.mapbox.com/styles/v1/mapbox/streets-v11?access_token={{ TOKEN }} - response_matcher: - - report_response: true - - type: StatusMatch - status: [200] - - type: JsonValid + url: https://api.mapbox.com/styles/v1/mapbox/streets-v11?access_token={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: JsonValid diff --git a/data/rules/netlify.yml b/data/rules/netlify.yml new file mode 100644 index 0000000..f7736a3 --- /dev/null +++ b/data/rules/netlify.yml @@ -0,0 +1,62 @@ +rules: + - name: Netlify API Key + id: kingfisher.netlify.1 + pattern: | + (?xi) + netlify + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ([a-f0-9]{60,64}) + \b + min_entropy: 3.3 + examples: + - netlify_token=3cdfad7b885a6daceff3fb820389115750b373763fb30b10ca0382648b55872d + - netlify_secret=7a9ef2c84d6b3e5f1c8a0b9d2e4f6a8c7b3d5e9f2a1c8b4d6e3f5a9c7b2d8e4 + references: + - https://howtorotate.com/docs/tutorials/netlify/ + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + method: GET + url: https://api.netlify.com/api/v1/user + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + + - name: Netlify API Key + id: kingfisher.netlify.2 + pattern: | + (?xi) + \b + netlify + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ([A-Za-z0-9_-]{43,45}) + \b + min_entropy: 3.5 + confidence: medium + examples: + - netlify_token=G5yT54abRasekrOpe7SaArsowiuHTeR45sfEhsH-K1L2 + - netlify_key=H7xZ98cdWbsemqNpv8UaXtsnyjKgVeQ34rsDkpM-N5P6 + references: + - https://howtorotate.com/docs/tutorials/netlify/ + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + method: GET + url: https://api.netlify.com/api/v1/user + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] diff --git a/data/rules/netrc.yml b/data/rules/netrc.yml new file mode 100644 index 0000000..35ed66e --- /dev/null +++ b/data/rules/netrc.yml @@ -0,0 +1,29 @@ +rules: + - name: netrc Credentials + id: kingfisher.netrc.1 + pattern: | + (?x) + ( machine \s+ [^\s]+ | default ) + \s+ + login \s+ ([^\s]+) + \s+ + password \s+ ([^\s]+) + min_entropy: 3.3 + confidence: medium + examples: + - 'machine api.github.com login ziggy^stardust password 012345abcdef' + - | + ``` + machine raw.github.com + login visionmedia + password pass123 + ``` + - | + """ + machine api.wandb.ai + login user + password 7cc938e45e63e9014f88f811be240ba0395c02dd + """ + references: + - https://everything.curl.dev/usingcurl/netrc + - https://devcenter.heroku.com/articles/authentication#api-token-storage \ No newline at end of file diff --git a/data/rules/newrelic.yml b/data/rules/newrelic.yml new file mode 100644 index 0000000..d9a582c --- /dev/null +++ b/data/rules/newrelic.yml @@ -0,0 +1,33 @@ +rules: + - name: New Relic Personal API Key + id: kingfisher.newrelic.1 + pattern: | + (?xi) + \b + newrelic + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9_.]{4} + - + [A-Z0-9_.]{42} + ) + min_entropy: 3.3 + confidence: medium + examples: + - newrelic_key = abcd-1234567890abcdef1234567890abcdef1234dd5678 + - newrelic_token = 1234-abcdefghijklmnopqrstuvwxyzABCD2342EFGHIJKL + validation: + type: Http + content: + request: + method: GET + headers: + X-Api-Key: '{{ TOKEN }}' + url: https://api.newrelic.com/v2/servers.json + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] diff --git a/data/rules/ngrok.yml b/data/rules/ngrok.yml new file mode 100644 index 0000000..3635d08 --- /dev/null +++ b/data/rules/ngrok.yml @@ -0,0 +1,31 @@ +rules: + - name: Ngrok API Key + id: kingfisher.ngrok.1 + pattern: | + (?x)(?i) + ngrok + (?:.|[\\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + (?:[a-z0-9]{25,30}_\d[a-z0-9]{20}|(?:cr_|ak_)[a-z0-9]{25,30}) + \b + min_entropy: 4 + examples: + - 'ngrok authtoken: 2Ot3hdNCKF3JRJDCioqNV2J0PPc_6th2CSUm9KsjfXdtRDvzT' + validation: + type: Http + content: + request: + method: GET + headers: + Authorization: Bearer {{ TOKEN }} + ngrok-version: 2 + url: https://api.ngrok.com/endpoints + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"endpoints":' diff --git a/data/rules/npm.yml b/data/rules/npm.yml new file mode 100644 index 0000000..af94dfe --- /dev/null +++ b/data/rules/npm.yml @@ -0,0 +1,65 @@ +rules: + - name: NPM Access Token (fine-grained) + id: kingfisher.npm.1 + pattern: | + (?x) + \b + ( + npm_[A-Za-z0-9]{36} + ) + \b + references: + - https://docs.npmjs.com/about-access-tokens + - https://github.com/github/roadmap/issues/557 + - https://github.blog/changelog/2022-12-06-limit-scope-of-npm-tokens-with-the-new-granular-access-tokens/ + min_entropy: 3.3 + confidence: medium + examples: + - 'npm_TCllNwh2WLQlMWVhybM1iQrsTj6rMQ0BOh6d' + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + method: GET + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"name":'] + url: https://registry.npmjs.org/-/npm/v1/user + + - name: NPM Access Token (old format) + id: kingfisher.npm.2 + pattern: | + (?xi) + \b + (?:_authToken|NPM_TOKEN) + (?:.|[\n\r]){0,16}? + ( + [0-9A-F]{8} + (?:-[0-9A-F]{4}){3} + -[0-9A-F]{12} + ) + \b + min_entropy: 3.5 + confidence: medium + examples: + - '"_authToken": "b98ec224-cdb2-4340-b7bd-9617fc719d1d"' + - '-export NPM_TOKEN="007e64c7-635d-4d54-8295-f364cd8e0e0f"' + validation: + type: Http + content: + request: + headers: + Authorization: Bearer {{ TOKEN }} + method: GET + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"name":'] + url: https://registry.npmjs.org/-/npm/v1/user \ No newline at end of file diff --git a/data/rules/nuget.yml b/data/rules/nuget.yml new file mode 100644 index 0000000..8d39cce --- /dev/null +++ b/data/rules/nuget.yml @@ -0,0 +1,68 @@ +rules: + - name: NuGet API Key + id: kingfisher.nuget.1 + pattern: | + (?x) + \b + ( + oy2[a-z0-9]{43} + ) + \b + min_entropy: 3.3 + confidence: medium + examples: + - oy2er16dp0r068m6p36u4bvr9nmkescfm1pf9lek1bgn3n + - oy2gdbfofub9ecpohsfdndem7nr8sui1g8le3ptnljqhlu + validation: + type: Http + content: + request: + method: POST + url: https://www.nuget.org/api/v2/package/create-verification-key/Newtonsoft.Json + headers: + X-NuGet-ApiKey: '{{ TOKEN }}' + X-NuGet-Protocol-Version: '4.1.0' + Content-Length: '0' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"Key":'] + + + - name: NuGet API Key + id: kingfisher.nuget.2 + pattern: | + (?xi) + \b + nuget + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [a-z0-9]{46} + ) + \b + min_entropy: 3.5 + confidence: medium + examples: + - nuget key = af7e33adc0rq7ls6ijcfmb5rgf1gp2o0tqk6d8s5p21k6t + - nuget secret dfaru1u13sd58q0kd0edvs4276p2mb6o31eljkvjh8t30u + validation: + type: Http + content: + request: + method: POST + url: https://www.nuget.org/api/v2/package/create-verification-key/Newtonsoft.Json + headers: + X-NuGet-ApiKey: '{{ TOKEN }}' + X-NuGet-Protocol-Version: '4.1.0' + Content-Length: '0' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"Key":'] \ No newline at end of file diff --git a/data/rules/stripe.yml b/data/rules/stripe.yml index 92a9d2e..0cb2072 100644 --- a/data/rules/stripe.yml +++ b/data/rules/stripe.yml @@ -10,8 +10,7 @@ rules: ( pk_live_ (?:[0-9A-Za-z]{6}){4,30} - ) - (?=$|[\s"']) + ) min_entropy: 3.3 confidence: medium categories: [api, key] @@ -33,11 +32,8 @@ rules: )_live_ (?:[0-9A-Za-z]{8}){3,25} ) - (?=$|[\s"']) - min_entropy: 3.3 confidence: medium - categories: [api, key] examples: - stripe_secret_key = sk_live_f01c79xuuug7yodgzj5ws0h1x2kyvho3 - "strp_sec_key: rk_live_4haG9YwGkL2hXqTj5pSzo8FzB3uCwE7n" @@ -50,11 +46,11 @@ rules: headers: Authorization: Bearer {{ TOKEN }} Accept: application/json - url: https://api.stripe.com/v1/account - response_matcher: - - report_response: true - - type: StatusMatch - status: [200] - - type: WordMatch - match_all_words: true - words: ['"object":"account"'] + url: https://api.stripe.com/v1/account + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + match_all_words: true + words: ['"object":"account"'] From 627f8a8ff6d9bd420b487dab04429e3fa82b30b0 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Wed, 25 Jun 2025 16:28:34 -0700 Subject: [PATCH 09/11] Added openweather, opsgeneie, pagerduty, particle.io, psatebin, and paypal rules --- data/rules/openweather.yml | 37 +++++++++++++++++ data/rules/opsgenie.yml | 32 +++++++++++++++ data/rules/pagerdutyapikey.yml | 36 +++++++++++++++++ data/rules/particle.io.yml | 74 ++++++++++++++++++++++++++++++++++ data/rules/pastebin.yml | 37 +++++++++++++++++ data/rules/paypal.yml | 56 +++++++++++++++++++++++++ 6 files changed, 272 insertions(+) create mode 100644 data/rules/openweather.yml create mode 100644 data/rules/opsgenie.yml create mode 100644 data/rules/pagerdutyapikey.yml create mode 100644 data/rules/particle.io.yml create mode 100644 data/rules/pastebin.yml create mode 100644 data/rules/paypal.yml diff --git a/data/rules/openweather.yml b/data/rules/openweather.yml new file mode 100644 index 0000000..2153e64 --- /dev/null +++ b/data/rules/openweather.yml @@ -0,0 +1,37 @@ +rules: + - name: OpenWeather Map API Key + id: kingfisher.openweather.1 + pattern: | + (?xi) + (?:pyowm|openweather|\bowm\b) + (?:.|[\n\r]){0,64}? + \b + ( + (?: + [a-z0-9]{32} + ) + \b + |APPID= + (?: + [a-z0-9]{32} + ) + ) + \b + min_entropy: 3.5 + examples: + - pyowm = '3k144a5af729351d0fc58bdrj9a21mkr' + - owm = '3k144a5af729351d0fc58bdrj9a21mkr' + - openweatherapikey=cd2b1d12d01ae2deffecfebafcc3c31d + - apikey=openweather:cd2b1d12d01ae2deffecfebafcc3c31d + validation: + type: Http + content: + request: + method: GET + response_matcher: + - report_response: true + - match_all_status: true + status: + - 200 + type: StatusMatch + url: https://api.openweathermap.org/geo/1.0/reverse?lat=0&lon=0&limit=1&appid={{ TOKEN }} \ No newline at end of file diff --git a/data/rules/opsgenie.yml b/data/rules/opsgenie.yml new file mode 100644 index 0000000..fafef06 --- /dev/null +++ b/data/rules/opsgenie.yml @@ -0,0 +1,32 @@ +rules: + - name: OpsGenie API Key + id: kingfisher.opsgenie.1 + pattern: | + (?x) + (?i) + \b + opsgenie + (?:.|[\\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} + ) + min_entropy: 3.5 + examples: + - opsgenie_api_key = '12345678-9abc-def0-1234-56789abcdef0' + validation: + type: Http + content: + request: + headers: + Authorization: GenieKey {{ TOKEN }} + method: GET + url: https://api.opsgenie.com/v2/alerts + response_matcher: + - report_response: true + - type: WordMatch + words: + - "Could not authenticate" + negative: true diff --git a/data/rules/pagerdutyapikey.yml b/data/rules/pagerdutyapikey.yml new file mode 100644 index 0000000..be4b24f --- /dev/null +++ b/data/rules/pagerdutyapikey.yml @@ -0,0 +1,36 @@ +rules: + - name: PagerDuty API Key + id: kingfisher.pagerduty.1 + pattern: | + (?xi) + \b + (?:pagerduty|pager[_-]duty|pd[-_\]=\)]|pd\.webhook?) + (?:.|[\n\r]){0,16}? + ( + u\+[A-Z0-9_+-]{18} # new personal tokens + | + [A-Z0-9_-]{20} # legacy personal tokens + | + [A-F0-9]{32} # integration keys / routing keys + ) + \b + min_entropy: 3.3 + confidence: medium + examples: + - pagerduty_key = u+Lyhd2_N2MCy+ZoH-S5 + - pd_key = u+3xVszZ-b4m+T6d23KA + validation: + type: Http + content: + request: + method: GET + url: https://api.pagerduty.com/abilities + headers: + Authorization: Token token={{ TOKEN }} + Accept: application/vnd.pagerduty+json;version=2 + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"abilities":'] \ No newline at end of file diff --git a/data/rules/particle.io.yml b/data/rules/particle.io.yml new file mode 100644 index 0000000..8237ebd --- /dev/null +++ b/data/rules/particle.io.yml @@ -0,0 +1,74 @@ +rules: + - name: particle.io Access Token + id: kingfisher.particleio.1 + pattern: | + (?x) + https://api\.particle\.io/v1/[a-zA-Z0-9_\-\s/"\\?]* + (?:access_token=|Authorization:\s*Bearer\s*) + \b + ([a-zA-Z0-9]{40}) + \b + min_entropy: 3.3 + confidence: medium + examples: + - | + curl https://api.particle.io/v1/devices \ + -H "Authorization: Bearer 38bb7b318cc6898c80317decb34525844bc9db55" + - | + curl https://api.particle.io/v1/devices \ + -d access_token=38bb7b318cc6898c80317decb34525844bc9db55 + - 'curl https://api.particle.io/v1/devices -H "Authorization: Bearer 38bb7b318cc6898c80317decb34525844bc9db55"' + - 'curl https://api.particle.io/v1/devices -d access_token=38bb7b318cc6898c80317decb34525844bc9db55' + - 'curl "https://api.particle.io/v1/devices/events?access_token=38bb7b318cc6898c80317decb34525844bc9db55"' + - 'curl "https://api.particle.io/v1/access_tokens/current?access_token=38bb7b318cc6898c80317decb34525844bc9db55"' + references: + - https://docs.particle.io/reference/cloud-apis/api/ + validation: + type: Http + content: + request: + method: GET + url: https://api.particle.io/v1/user?access_token={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + match_all_words: true + words: ['"username":'] + + - name: particle.io Access Token + id: kingfisher.particleio.2 + pattern: | + (?x) + (?:access_token=|Authorization:\s*Bearer\s*) + \b + ([a-zA-Z0-9]{40}) + \b + [\s"\\]*https://api\.particle\.io/v1 + min_entropy: 3.3 + confidence: medium + examples: + - | + curl -H "Authorization: Bearer 38bb7b318cc6898c80317decb34525844bc9db55" \ + https://api.particle.io/v1/devices + - | + curl -d access_token=38bb7b318cc6898c80317decb34525844bc9db55 \ + https://api.particle.io/v1/devices + - 'curl -H "Authorization: Bearer 38bb7b318cc6898c80317decb34525844bc9db55" https://api.particle.io/v1/devices' + - 'curl -d access_token=38bb7b318cc6898c80317decb34525844bc9db55 https://api.particle.io/v1/devices' + references: + - https://docs.particle.io/reference/cloud-apis/api/ + validation: + type: Http + content: + request: + method: GET + url: https://api.particle.io/v1/user?access_token={{ TOKEN }} + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + match_all_words: true + words: ['"username":'] \ No newline at end of file diff --git a/data/rules/pastebin.yml b/data/rules/pastebin.yml new file mode 100644 index 0000000..26a55dd --- /dev/null +++ b/data/rules/pastebin.yml @@ -0,0 +1,37 @@ +rules: + - name: Pastebin API Key + id: kingfisher.pastebin.1 + pattern: | + (?xi) + \b + pastebin + (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [a-zA-Z0-9_]{32} + ) + \b + min_entropy: 3.5 + confidence: medium + examples: + - pastebin_key=zwD26NeyMCvBsR9nxfaybLHD7TcLh22O + - pastebin_api_token=zwD26NeyMCvBsR9n_faybLHD7TcLh22O + validation: + type: Http + content: + request: + method: POST + url: https://pastebin.com/api/api_login.php + headers: + Content-Type: application/x-www-form-urlencoded + body: | + api_dev_key={{ TOKEN }}&api_user_name=dummy&api_user_password=dummy + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['invalid api_dev_key'] + negative: true \ No newline at end of file diff --git a/data/rules/paypal.yml b/data/rules/paypal.yml new file mode 100644 index 0000000..2441a68 --- /dev/null +++ b/data/rules/paypal.yml @@ -0,0 +1,56 @@ +rules: +- name: PayPal OAuth Client ID + id: kingfisher.paypal.1 + pattern: | + (?xi) + paypal + (?:.|[\n\r]){0,8}? + (?:CLIENT|ID|USER) + (?:.|[\n\r]){0,16}? + \b + ( + A[A-Z0-9_-]{79,99} + ) + \b + min_entropy: 3.5 + visible: false + examples: + - paypal_client_id=AZJ6y8Dpr1TYbqAIdhkPzyhjXoY6m8GplL7C3zZ3lPrkTIdhkPzyhjXo_Dx3 + +- name: PayPal OAuth Secret + id: kingfisher.paypal.2 + pattern: | + (?xi) + paypal + (?:.|[\n\r]){0,16}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? + \b + ( + [A-Z0-9_.-]{80,120} + ) + \b + min_entropy: 3.5 + examples: + - paypal_secret=EDe5J6y8Dpr1TYbqAIdhkPzyhjXoY6m8GplL7C3zZ3lPrkT1XlV6hYPSeJL5b1T1 + + validation: + type: Http + content: + request: + method: POST + url: https://api-m.paypal.com/v1/oauth2/token + headers: + Accept: application/json + Accept-Language: en_US + Content-Type: application/x-www-form-urlencoded + Authorization: | + Basic {{ CLIENTID | append: ':' | append: TOKEN | b64enc }} + body: grant_type=client_credentials + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + depends_on_rule: + - rule_id: kingfisher.paypal.1 + variable: CLIENTID From 4f2585e35c70384cfd833b1b8e27a2c253b67511 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Wed, 25 Jun 2025 17:02:44 -0700 Subject: [PATCH 10/11] Added planetscal and postman rules --- data/rules/planetscale.yml | 55 ++++++++++++++++++++++++++++ data/rules/postman.yml | 37 +++++++++++++++++++ tests/{update.rs => smoke_update.rs} | 0 3 files changed, 92 insertions(+) create mode 100644 data/rules/planetscale.yml create mode 100644 data/rules/postman.yml rename tests/{update.rs => smoke_update.rs} (100%) diff --git a/data/rules/planetscale.yml b/data/rules/planetscale.yml new file mode 100644 index 0000000..119957c --- /dev/null +++ b/data/rules/planetscale.yml @@ -0,0 +1,55 @@ +rules: + - name: PlanetScale API Token + id: kingfisher.planetscale.1 + pattern: | + (?x) + (?i) + \b + ( + pscale_tkn_[a-z0-9-_]{43} + ) + \b + min_entropy: 4 + examples: + - pscale_tkn_abcdefghijklmnopqrstuvwxyZ1234567890_ABCDEF + validation: + type: Http + content: + request: + headers: + Accept: application/json + Authorization: '{{ USERNAME | append: ":" | append: TOKEN }}' + method: GET + response_matcher: + - report_response: true + - status: + - 200 + type: StatusMatch + - type: WordMatch + words: + - '"id":' + - '"username":' + url: https://api.planetscale.com/v1/user + depends_on_rule: + - rule_id: kingfisher.planetscale.2 + variable: USERNAME + + - name: PlanetScale Username + id: kingfisher.planetscale.2 + pattern: | + (?x) + (?i) + (?:pscale|planetscale) + (?:.|[\n\r]){0,16}? + (?:USER|ID|NAME) + (?:.|[\n\r]){0,16}? + \b + ( + [a-z0-9]{12} + ) + \b + min_entropy: 3.5 + visible: false + examples: + - pscale_user = abcdefghijkl + - 'planetscale_id: hgtmrnzlv1t7' diff --git a/data/rules/postman.yml b/data/rules/postman.yml new file mode 100644 index 0000000..707b4e2 --- /dev/null +++ b/data/rules/postman.yml @@ -0,0 +1,37 @@ +rules: + - name: Postman API Key + id: kingfisher.postman.1 + pattern: | + (?x) + \b + ( + PMAK-[A-Z0-9]{24}-[A-Z0-9]{34} + ) + \b + min_entropy: 3.3 + confidence: medium + examples: + - PMAK-5dd543842789bd0036bf98c1-a5a9b8f1dfda8fbf18a4664ebe558b04ed + - PMAK-642a58a823faa300316566d1-6715a3a826ce5d5d62be8539d6ac357146 + - PMAK-642a9b9084d6110029e75d7d-09efdcb872587f6f67696f02929647d9c6 + - "// ('x-api-key', 'PMAK-629c73facbc064567cbf6970-f56e8b4cd0bb14d00962f17afc158dc2a2')" + references: + - https://learning.postman.com/docs/developer/intro-api/ + - https://learning.postman.com/docs/developer/postman-api/authentication/ + - https://learning.postman.com/docs/administration/managing-your-team/managing-api-keys/ + validation: + type: Http + content: + request: + headers: + x-api-key: '{{ TOKEN }}' + method: GET + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 + - type: WordMatch + words: + - '"user":' + url: https://api.getpostman.com/me \ No newline at end of file diff --git a/tests/update.rs b/tests/smoke_update.rs similarity index 100% rename from tests/update.rs rename to tests/smoke_update.rs From 9b5c220182c96d5d95be6319ff43d9350ba039d3 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Wed, 25 Jun 2025 17:23:35 -0700 Subject: [PATCH 11/11] Updated github actions to extract only the latest changelog entry. Added --rule-stats flag to display rule performance --- .github/workflows/release.yml | 23 ++++++++++++++----- CHANGELOG.md | 5 ++++ Cargo.toml | 2 +- README.md | 9 ++++++++ src/matcher.rs | 23 +++++++++++++++++++ src/scanner/runner.rs | 12 ++++++++-- src/scanner/summary.rs | 43 +++++++++++++++++++++++++++++++++++ 7 files changed, 108 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5f625a6..cc46915 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -175,13 +175,12 @@ jobs: name: kingfisher-windows-x64 path: dist/kingfisher-*windows-x64*.* - # ──────────────── Draft public release ──────────────── release: name: Public GitHub Release - needs: [linux-x64, linux-arm64, windows, macos-x64, macos-arm64] # wait for all builds to finish + needs: [linux-x64, linux-arm64, windows, macos-x64, macos-arm64] runs-on: ubuntu-latest permissions: - contents: write # allow release upload + contents: write steps: - uses: actions/checkout@v4 - name: Read version from Cargo.toml @@ -193,11 +192,23 @@ jobs: with: path: target/release/kingfisher-* merge-multiple: true + - name: Extract latest changelog section + run: | + awk ' + BEGIN { grabbing = 0 } + /^## \[/ { + if (grabbing) exit; # already grabbed latest entry + grabbing = 1 + } + grabbing { print } + ' CHANGELOG.md > .latest_changelog.md + + # ── create the release using just that snippet ───────────────────── - name: Create release & upload assets uses: ncipollo/release-action@v1 with: tag: v${{ steps.version.outputs.version }} name: "Kingfisher v${{ steps.version.outputs.version }}" - bodyFile: CHANGELOG.md # use existing changelog - generateReleaseNotes: false # turn off auto-notes - artifacts: target/release/** + bodyFile: .latest_changelog.md # ← only the most-recent entry + generateReleaseNotes: false + artifacts: target/release/** \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 342e0c8..689cb46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ All notable changes to this project will be documented in this file. +## [1.13.0] +- Added new rules for Planetscale, Postman, Openweather, opsgenie, pagerduty, pastebin, paypal, netlify, netrc, newrelic, ngrok, npm, nuget, mandrill, mapbox, microsoft teams, stripe, linkedin, mailchimp, mailgun, linear, line, huggingface, ibm cloud, intercom, ipstack, heroku, gradle, grafana +- Added `--rule-stats` command-line flag that will display rule performance statistics during a scan. Useful when creating or debugging rules + + ## [1.12.0] - Added automatic update checks using GitHub releases. - New `--self-update` flag installs updates when available diff --git a/Cargo.toml b/Cargo.toml index 4d9d103..2b30bee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ publish = false [package] name = "kingfisher" -version = "1.12.0" +version = "1.13.0" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/README.md b/README.md index 0a24069..67098a3 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,11 @@ cat /path/to/file.py | kingfisher scan - kingfisher scan /path/to/repo --rule kingfisher.aws ``` +### Display rule performance statistics +```bash +kingfisher scan /path/to/repo --rule-stats +``` + --- ## Scanning GitHub @@ -244,6 +249,10 @@ The document below details the four-field formula (rule SHA-1, origin label, st See ([docs/FINGERPRINT.md](docs/FINGERPRINT.md)) +## Rule Performance Profiling +Use `--rule-stats` to collect timing information for every rule. After scanning, the summary prints a **Rule Performance Stats** section showing how many matches each rule produced along with its slowest and average match times. Useful when creating rules or debugging rules. + + ## CLI Options ```bash kingfisher scan --help diff --git a/src/matcher.rs b/src/matcher.rs index 9048206..89aed1b 100644 --- a/src/matcher.rs +++ b/src/matcher.rs @@ -25,6 +25,7 @@ use smallvec::SmallVec; use tracing::debug; use xxhash_rust::xxh3::xxh3_64; +use crate::rule_profiling::RuleTimer; use crate::{ blob::{Blob, BlobId, BlobIdMap}, entropy::calculate_shannon_entropy, @@ -386,6 +387,8 @@ impl<'a> Matcher<'a> { origin, None, redact, + &filename, + self.profiler.as_ref(), ); } // If tree-sitter produced base64-decoded matches, try them against all rules @@ -407,6 +410,8 @@ impl<'a> Matcher<'a> { origin, Some(ts_match.clone()), redact, + &filename, + self.profiler.as_ref(), ); } } @@ -456,7 +461,21 @@ fn filter_match<'b>( _origin: &OriginSet, ts_match: Option, redact: bool, + filename: &str, + profiler: Option<&Arc>, ) { + let mut timer = profiler.map(|p| { + RuleTimer::new( + p, + rule.id(), + rule.name(), + &rule.syntax.pattern, + filename, + ) + }); + + let initial_len = matches.len(); + // Use Cow to avoid unnecessary copying when ts_match is None let byte_slice: Cow<[u8]> = match ts_match { Some(ts_match_value) => Cow::Owned(ts_match_value.into_bytes()), @@ -515,6 +534,10 @@ fn filter_match<'b>( }); previous_matches.push((rule_id, matching_input_offset_span)); } + if let Some(t) = timer.take() { + let new_count = (matches.len() - initial_len) as u64; + t.end(new_count > 0, new_count, 0); + } } fn get_language_and_queries(lang: &str) -> Option<(Language, FxHashMap)> { match lang.to_lowercase().as_str() { diff --git a/src/scanner/runner.rs b/src/scanner/runner.rs index 86165d4..112a04e 100644 --- a/src/scanner/runner.rs +++ b/src/scanner/runner.rs @@ -75,7 +75,7 @@ pub async fn run_async_scan( progress_enabled, rules_db, enable_profiling, - shared_profiler, + Arc::clone(&shared_profiler), &matcher_stats, )?; @@ -117,7 +117,15 @@ pub async fn run_async_scan( // // Call cmd_report here crate::reporter::run(global_args, Arc::clone(&datastore), args) .context("Failed to run report command")?; - print_scan_summary(start_time, &datastore, global_args, args, rules_db, &matcher_stats); + print_scan_summary( + start_time, + &datastore, + global_args, + args, + rules_db, + &matcher_stats, + if enable_profiling { Some(shared_profiler.as_ref()) } else { None }, + ); Ok(()) } diff --git a/src/scanner/summary.rs b/src/scanner/summary.rs index ff331c4..11a45f9 100644 --- a/src/scanner/summary.rs +++ b/src/scanner/summary.rs @@ -17,6 +17,7 @@ use crate::{ }, findings_store, matcher::MatcherStats, + rule_profiling::ConcurrentRuleProfiler, rules_database::RulesDatabase, }; @@ -42,6 +43,7 @@ pub fn print_scan_summary( // inputs: &FilesystemEnumeratorResult, rules_db: &RulesDatabase, matcher_stats: &Mutex, + profiler: Option<&ConcurrentRuleProfiler>, ) { // let duration = start_time.elapsed(); let ds = datastore.lock().unwrap(); @@ -152,6 +154,47 @@ pub fn print_scan_summary( humantime::format_duration(duration) ); } + + if args.rule_stats { + if let Some(prof) = profiler { + let stats = prof.generate_report(); + if !stats.is_empty() { + // Calculate dynamic column widths + let name_w = stats.iter().map(|s| s.rule_name.len()).max().unwrap_or(4); + let id_w = stats.iter().map(|s| s.rule_id.len()).max().unwrap_or(2); + + // Header + safe_println!("\n{:-^1$}", " Rule Performance Stats ", name_w + id_w + 47); + safe_println!( + "{: 8} {: >15} {: >15}", + "Rule", + "ID", + "Matches", + "Slowest", + "Average", + name_w = name_w, + id_w = id_w + ); + safe_println!("{:-8} {: >15?} {: >15?}", + rs.rule_name, + rs.rule_id, + rs.total_matches, + rs.slowest_match_time, + rs.average_match_time, + name_w = name_w, + id_w = id_w + ); + } + } + } + } + + debug!("\nAll Rules with Matches:"); debug!("======================="); let max_rule_length = sorted_findings.iter().map(|(rule, _)| rule.len()).max().unwrap_or(0);