From 6d669b4bb741aa04a280ea5ea4dce63c6d30123c Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sat, 16 Aug 2025 20:23:27 -0700 Subject: [PATCH] added more rules --- CHANGELOG.md | 2 +- README.md | 2 +- data/rules/firecrawl.yml | 32 +++++++++ data/rules/gitalk.yml | 21 ++++++ data/rules/hashicorp.yml | 133 +++++++++++++++++++++++++++++++++++++ data/rules/http.yml | 36 ++++++++++ data/rules/hubspot.yml | 31 +++++++++ data/rules/jina.yml | 12 ++++ data/rules/kagi.yml | 35 ++++++++++ data/rules/phpmailer.yml | 45 +++++++++++++ data/rules/postmark.yml | 49 ++++++++++++++ data/rules/sauce.yml | 23 +++++++ data/rules/stackhawk.yml | 26 ++++++++ data/rules/tavily.yml | 38 +++++++++++ data/rules/thingsboard.yml | 45 +++++++++++++ data/rules/truenas.yml | 2 +- 16 files changed, 529 insertions(+), 3 deletions(-) create mode 100644 data/rules/firecrawl.yml create mode 100644 data/rules/gitalk.yml create mode 100644 data/rules/hashicorp.yml create mode 100644 data/rules/http.yml create mode 100644 data/rules/hubspot.yml create mode 100644 data/rules/jina.yml create mode 100644 data/rules/kagi.yml create mode 100644 data/rules/phpmailer.yml create mode 100644 data/rules/postmark.yml create mode 100644 data/rules/sauce.yml create mode 100644 data/rules/stackhawk.yml create mode 100644 data/rules/tavily.yml create mode 100644 data/rules/thingsboard.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 64aa8a1..b467a7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ All notable changes to this project will be documented in this file. ## [1.42.0] -- Added rules for authress, clickhouse, codecov, contentful, curl, dropbox, fly.io +- Added rules for authress, clickhouse, codecov, contentful, curl, dropbox, fly.io, hubspot, firecrawl - Internal refactoring of rule loader, git enumerator, and filetype guesser - Improved language detection diff --git a/README.md b/README.md index 6873a6a..bdd05ba 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Kingfisher is a blazingly fast secret‑scanning and live validation tool built in Rust. It combines Intel’s hardware‑accelerated Hyperscan regex engine with language‑aware parsing via Tree‑Sitter, and **ships with hundreds of built‑in rules** to detect, validate, and triage secrets before they ever reach production

-Kingfisher originated as a fork of Praetorian's [Nosey Parker](https://github.com/praetorian-inc/noseyparker), and is built atop their incredible work and the work contributed by the Nosey Parker community. +Kingfisher originated as a fork of Praetorian's Nosey Parker, and is built atop their incredible work and the work contributed by the Nosey Parker community. ## What Kingfisher Adds - **Live validation** via cloud-provider APIs diff --git a/data/rules/firecrawl.yml b/data/rules/firecrawl.yml new file mode 100644 index 0000000..fc729c6 --- /dev/null +++ b/data/rules/firecrawl.yml @@ -0,0 +1,32 @@ +rules: + - name: Firecrawl API Key + id: kingfisher.firecrawl.1 + pattern: | + (?xi) + \b + ( + fc-[a-f0-9]{32} + ) + \b + confidence: medium + min_entropy: 3.0 + validation: + type: Http + content: + request: + method: GET + url: "https://api.firecrawl.dev/v1/crawl/active" + headers: + Authorization: "Bearer {{TOKEN}}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"success"' + references: + - https://www.firecrawl.dev + - https://docs.firecrawl.dev/api-reference/introduction#authentication + examples: + - 'app = FirecrawlApp(api_key="fc-7da8b1ca1d2150c496e91440d777fea8")' diff --git a/data/rules/gitalk.yml b/data/rules/gitalk.yml new file mode 100644 index 0000000..d0e36d4 --- /dev/null +++ b/data/rules/gitalk.yml @@ -0,0 +1,21 @@ +rules: + - name: Gitalk OAuth Credentials + id: kingfisher.gitalk.1 + + pattern: | + (?x) + \b + new \s+ Gitalk \s* \( \s* \{ \s* + clientID: \s* '([a-f0-9]{20})', \s* + clientSecret: \s* '([a-f0-9]{40})', + confidence: medium + min_entropy: 3.0 + references: + - https://gitalk.github.io + - https://github.com/gitalk/gitalk + + examples: + - | + new Gitalk({ + clientID: 'd17d49be2e680b78a83d', + clientSecret:'9363cb456dda6402cb71d65092490e75c9f11873', diff --git a/data/rules/hashicorp.yml b/data/rules/hashicorp.yml new file mode 100644 index 0000000..20adfc1 --- /dev/null +++ b/data/rules/hashicorp.yml @@ -0,0 +1,133 @@ +rules: + - name: Hashicorp Vault Service Token (< v1.10) + id: kingfisher.hashicorp.1 + + pattern: | + (?x) + (?i: hashicorp | vault | token | key | secret ) + ["':=\ ]{0,5} + \b + (s\.[A-Za-z0-9_-]{24,128}) + (?: [^A-Za-z0-9_-] | $ ) + confidence: medium + min_entropy: 3.0 + + examples: + - 'VAULT_CLIENT_TOKEN="s.Z4bTMtngfLeQ18AqVoBBkUAOD1"' + - '`-vaultToken s.CAESIP2jTxc9S3K7Z6CtcFWQv7-044m_oSsxkingfisher.0H3nF89l3GiYKHGh3cy5sQmlIZVNyTWJNcDRsYWJpQjlhYjVlb2cQh6PL8wEYAg"`' + + references: + - https://developer.hashicorp.com/vault/docs/concepts/tokens + + - name: Hashicorp Vault Batch Token (< v1.10) + id: kingfisher.hashicorp.2 + + pattern: | + (?x) + (?i: hashicorp | vault | token | key | secret ) + ["':=\ ]{0,5} + \b + (b\.[A-Za-z0-9_-]{24,500}) + (?: [^A-Za-z0-9_-] | $ ) + + examples: + - 'VAULT_CLIENT_TOKEN="b.Z4bTMtngfLeQ18AqVoBBkUAOD1"' + confidence: medium + min_entropy: 3.0 + references: + - https://developer.hashicorp.com/vault/docs/concepts/tokens + + - name: Hashicorp Vault Recovery Token (< v1.10) + id: kingfisher.hashicorp.3 + + pattern: | + (?x) + (?i: hashicorp | vault | token | key | secret ) + ["':=\ ]{0,5} + \b + (r\.[A-Za-z0-9_-]{24,500}) + (?: [^A-Za-z0-9_-] | $ ) + + examples: + - 'VAULT_CLIENT_TOKEN="r.Z4bTMtngfLeQ18AqVoBBkUAOD1"' + + confidence: medium + min_entropy: 3.0 + references: + - https://developer.hashicorp.com/vault/docs/concepts/tokens + - https://developer.hashicorp.com/vault/docs/concepts/recovery-mode + + - name: Hashicorp Vault Service Token (>= v1.10) + id: kingfisher.hashicorp.4 + + pattern: | + (?x) + \b + (hvs\.[A-Za-z0-9]{24,130}) + (?: [^A-Za-z0-9_-] | $ ) + + examples: + - "apikey: hvs.JGbZZaCkOSgsZ56uhGlTK2zyC1j2mwhy0VLp4" + + confidence: medium + min_entropy: 3.0 + references: + - https://developer.hashicorp.com/vault/docs/concepts/tokens + + - name: Hashicorp Vault Batch Token (>= v1.10) + id: kingfisher.hashicorp.5 + + pattern: | + (?x) + \b + (hvb\.[A-Za-z0-9_-]{24,500}) + (?: [^A-Za-z0-9_-] | $ ) + + examples: + - "apikey: hvb.JGbZZaCkOSgsZ56uhGlTK2zyC1j2mwhy0VLp4" + - "hvb.AAAAAQJgxDgqsGNorpoOR8hPZ5SU-ynBvCl764jyRP_fnX8WvkdkDzGjbLNGdPdtlY32Als2P36yDZueqzfdGw9RsaTeaYXSH5E4RYSWuRoQ9YRKIw9o7mDDY2ZcT3KOB7RwtW2w1FN2eDqcy_sbCjXPaM1iBVH-mqMSYRmRd2nb5D1SJPeBzIYRqSglLc32wUGN7xEzyrKUczqOKsIcybQA" + + confidence: medium + min_entropy: 3.0 + + references: + - https://developer.hashicorp.com/vault/docs/concepts/tokens + + - name: Hashicorp Vault Recovery Token (>= v1.10) + id: kingfisher.hashicorp.6 + + pattern: | + (?x) + \b + (hvr\.[A-Za-z0-9]{24,130}) + (?: [^A-Za-z0-9_-] | $ ) + + examples: + - "apikey: hvr.JGbZZaCkOSgsZ56uhGlTK2zyC1j2mwhy0VLp4" + + confidence: medium + min_entropy: 3.0 + references: + - https://developer.hashicorp.com/vault/docs/concepts/tokens + - https://developer.hashicorp.com/vault/docs/concepts/recovery-mode + + - name: Hashicorp Vault Unseal Key + id: kingfisher.hashicorp.7 + + pattern: | + (?x) + (?i: unseal ) + \b + .{1,10} + \b + ([a-zA-Z0-9+/]{44}) + (?: [^a-zA-Z0-9+/] | $ ) + + examples: + - "Unseal Key 2: 0tZn+7QQCxphpHwTm7/dC3LpP5JGIbYl3PK8Sy81R+P2" + - "oc -n vault exec -ti vault-0 -- vault operator unseal 98m+o2ylRhVbOi+3o5ub6PbP343ocFUVORgSYeypMDjh" + + confidence: medium + min_entropy: 3.0 + references: + - https://developer.hashicorp.com/vault/docs/concepts/seal diff --git a/data/rules/http.yml b/data/rules/http.yml new file mode 100644 index 0000000..a3b523b --- /dev/null +++ b/data/rules/http.yml @@ -0,0 +1,36 @@ +rules: + - name: HTTP Basic Authentication + id: kingfisher.http.1 + + pattern: | + (?x)(?i) + Authorization (?: :\s+ | \s*.{1, 5}\s*) Basic \s+ + ( [A-Za-z0-9+/]{6,} ={0,2} ) + (?: [^A-Za-z0-9+/=] | $ ) + confidence: low + min_entropy: 3.0 + examples: + - "Authorization: Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==" + + references: + - https://datatracker.ietf.org/doc/html/rfc7617 + - https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication + + - name: HTTP Bearer Token + id: kingfisher.http.2 + pattern: | + (?x)(?i) + Authorization (?: :\s+ | \s*.{1, 5}\s*) Bearer \s+ + ([a-zA-z0-9._~+/-]{6,} =*) + (?: [^a-zA-z0-9._~+/=-] | $ ) + confidence: low + min_entropy: 3.0 + examples: + - | + GET /resource HTTP/1.1 + Host: server.example.com + Authorization: Bearer mF_9.B5f-4.1JqM + + references: + - https://datatracker.ietf.org/doc/html/rfc6750 + - https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication diff --git a/data/rules/hubspot.yml b/data/rules/hubspot.yml new file mode 100644 index 0000000..2840203 --- /dev/null +++ b/data/rules/hubspot.yml @@ -0,0 +1,31 @@ +rules: + - name: HubSpot Private App Token + id: kingfisher.hubspot.1 + pattern: | + (?xi) + \b + ( + pat-[a-z0-9]{2,3}-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} + ) + \b + confidence: medium + min_entropy: 3.0 + validation: + type: Http + content: + request: + method: GET + url: "https://api.hubapi.com/crm/v3/owners/" + headers: + Authorization: "Bearer {{TOKEN}}" + response_matcher: + - report_response: true + - type: JsonValid + - type: WordMatch + words: + - '"INVALID_AUTHENTICATION"' + negative: true + references: + - https://developers.hubspot.com/docs/api/private-apps + examples: + - "pat-na2-3b124f92-f4cb-4d1d-8d1c-7fc3f3512dba" diff --git a/data/rules/jina.yml b/data/rules/jina.yml new file mode 100644 index 0000000..05f9613 --- /dev/null +++ b/data/rules/jina.yml @@ -0,0 +1,12 @@ +rules: + - name: Jina Search Foundation API Key + id: kingfisher.jina.1 + pattern: | + (?x) + \b + (jina_[a-zA-Z0-9]{60}) + \b + confidence: medium + min_entropy: 3.0 + examples: + - "JINA_KEY = os.getenv('JINA_KEY','jina_c1758c6f49e14ced990ac7776800dc45ShJNTXBCizzwjE6IMFYJ7LD959cG')" diff --git a/data/rules/kagi.yml b/data/rules/kagi.yml new file mode 100644 index 0000000..17732af --- /dev/null +++ b/data/rules/kagi.yml @@ -0,0 +1,35 @@ +rules: + - name: Kagi API Key + id: kingfisher.kagi.1 + pattern: | + (?x)(?s) + \b + (?: kagi | KAGI ) + .{0,100} + \b + ( [a-zA-Z0-9_-]{11}\.[a-zA-Z0-9_-]{43} ) + (?: $ | [^a-zA-Z0-9_-] ) + confidence: medium + min_entropy: 3.0 + references: + - https://help.kagi.com/kagi/api/search.html + examples: + - "KAGI_API_KEY='AQAAUPJ-iQc.yLFDzC5RRHzPNDThhulREdoG0Bn3PiZMwJ6yqC6uJLE'" + - "https://kagi.com/search?token=uwHBLWXZpgY.STzubkAbVXqpfV39Q6TOfzp43KulJeYWK6-963uz1-o" + validation: + type: Http + content: + request: + method: GET + url: "https://kagi.com/api/v0/search?q=test" + headers: + Authorization: "Bot {{TOKEN}}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"data":' + - '"results":' + match_all_words: true \ No newline at end of file diff --git a/data/rules/phpmailer.yml b/data/rules/phpmailer.yml new file mode 100644 index 0000000..2811163 --- /dev/null +++ b/data/rules/phpmailer.yml @@ -0,0 +1,45 @@ +rules: + - name: PHPMailer Credentials + id: kingfisher.phpmailer.1 + + pattern: | + (?x) + \$mail->Host \s* = \s* '([^'\n]{5,})'; \s* (?: //.* )? + (?: \s* .* \s* ){0,3} + \$mail->Username \s* = \s* '([^'\n]{5,})'; \s* (?: //.* )? + (?: \s* .* \s* ){0,3} + \$mail->Password \s* = \s* '([^'\n]{5,})'; + confidence: medium + min_entropy: 3.0 + examples: + - | + //Server settings + $mail->SMTPDebug = SMTP::DEBUG_SERVER; //Enable verbose debug output + $mail->isSMTP(); //Send using SMTP + $mail->Host = 'smtp.example.com'; //Set the SMTP server to send through + $mail->SMTPAuth = true; //Enable SMTP authentication + $mail->Username = 'user@example.com'; //SMTP username + $mail->Password = 'secret'; //SMTP password + $mail->SMTPSecure = PHPMailer::ENCRYPTION_SMTPS; //Enable implicit TLS encryption + $mail->Port = 465; //TCP port to connect to; use 587 if you have set `SMTPSecure = PHPMailer::ENCRYPTION_STARTTLS` + + - | + require 'PHPMailerAutoload.php'; + + function SendMail($sub,$to,$msg) + { + $mail = new PHPMailer; + $mail->isSMTP(); // Set mailer to use SMTP + $mail->Host = 'smtp.gmail.com'; // Specify main and backup SMTP servers + $mail->SMTPAuth = true; // Enable SMTP authentication + $mail->SMTPSecure = 'tls'; // Enable encryption, 'ssl' also accepted + $mail->Username = 'ersatz.technologies@example.com'; // SMTP username + + + + $mail->Password = 'un!techwhooah'; // SMTP password + $mail->From = 'from@example.com'; + $mail->FromName = 'Admin'; + + references: + - https://github.com/PHPMailer/PHPMailer diff --git a/data/rules/postmark.yml b/data/rules/postmark.yml new file mode 100644 index 0000000..6e31071 --- /dev/null +++ b/data/rules/postmark.yml @@ -0,0 +1,49 @@ +rules: + - name: Postmark API Token + id: kingfisher.postmark.1 + pattern: | + (?xi) + postmark [a-z0-9_-]{0,20} + .{0,10} + \b + ( + [a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12} + ) + \b + confidence: medium + min_entropy: 3.0 + examples: + - | + postmark: "f59dffd2-83ea-47c7-ba8f-95e053a6d0ae", + + - | + postmark: { + fromEmail: '...@....com', // must be verified with postmarkapp.com + postmarkApiToken: '00917922-dbe7-4882-bedc-10b93fc4c4c5', // our account token + }, + + - | + headers: { + Accept: "application/json", + "Content-Type": "application/json", + "X-Postmark-Server-Token": "c2321bd2-5a8f-46f2-b8fb-dbc706bd6923", + }, + references: + - https://postmarkapp.com/developer/api/overview + validation: + type: Http + content: + request: + method: GET + url: "https://api.postmarkapp.com/server" + headers: + X-Postmark-Server-Token: "{{TOKEN}}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"ID":' + - '"Name":' + match_all_words: true \ No newline at end of file diff --git a/data/rules/sauce.yml b/data/rules/sauce.yml new file mode 100644 index 0000000..761bdd2 --- /dev/null +++ b/data/rules/sauce.yml @@ -0,0 +1,23 @@ +rules: + - name: Sauce Token + id: kingfisher.sauce.1 + + pattern: | + (?x)(?i) + sauce .{0,50} + \b + ([a-f0-9-]{36}) + (?: [^a-f0-9-] | $ ) + + confidence: medium + min_entropy: 3.0 + + examples: + - | + - SAUCE_USERNAME=vitess + - SAUCE_ACCESS_KEY=2397f603-c2c4-4897-a8ca-587ace5dc8dd + - SAUCE_ACCESS_KEY=2397f603-c2c4-4897-a8ca-587ace5dc8d- + + references: + - https://docs.saucelabs.com/dev/api/ + - https://docs.saucelabs.com/dev/api/#authentication diff --git a/data/rules/stackhawk.yml b/data/rules/stackhawk.yml new file mode 100644 index 0000000..fc62628 --- /dev/null +++ b/data/rules/stackhawk.yml @@ -0,0 +1,26 @@ +rules: + - name: StackHawk API Key + id: kingfisher.stackhawk.1 + pattern: '\b(hawk\.[0-9A-Za-z_-]{20}\.[0-9A-Za-z_-]{20})\b' + confidence: medium + min_entropy: 3.0 + examples: + - 'HAWK_API_KEY="hawk.nHAOHdJjXoNyzAcTDC5M.R2gqQh2aCesrh0yCGB7q"' + references: + - https://docs.stackhawk.com/apidocs.html + - https://apidocs.stackhawk.com/reference/getuser + validation: + type: Http + content: + request: + method: GET + url: "https://api.stackhawk.com/api/v1/auth/user" + headers: + X-Api-Key: "{{TOKEN}}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"user":' \ No newline at end of file diff --git a/data/rules/tavily.yml b/data/rules/tavily.yml new file mode 100644 index 0000000..a908a21 --- /dev/null +++ b/data/rules/tavily.yml @@ -0,0 +1,38 @@ +rules: + - name: Tavily API Key + + id: kingfisher.tavily.1 + pattern: | + (?x) + \b + ( + tvly-[a-zA-Z0-9]{32} + ) + \b + confidence: medium + min_entropy: 3.0 + references: + - https://docs.tavily.com/api-reference + examples: + - "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" + - "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" + - 'TAVILY_API_KEY = "tvly-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"' + validation: + type: Http + content: + request: + method: POST + url: "https://api.tavily.com/search" + headers: + Authorization: "Bearer {{TOKEN}}" + Content-Type: "application/json" + body: '{"query": "test"}' + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"query":' + - '"results":' + match_all_words: true \ No newline at end of file diff --git a/data/rules/thingsboard.yml b/data/rules/thingsboard.yml new file mode 100644 index 0000000..562ee25 --- /dev/null +++ b/data/rules/thingsboard.yml @@ -0,0 +1,45 @@ +rules: + - name: ThingsBoard Access Token + id: kingfisher.thingsboard.1 + pattern: | + (?x) + thingsboard\.cloud/api/v1/ + ([a-z0-9]{20}) + confidence: medium + min_entropy: 3.0 + examples: + - http://thingsboard.cloud/api/v1/354u1g321kcqc1oad3w7/telemetry + - https://thingsboard.cloud/api/v1/354u1g321kcqc1oad3w7/telemetry + - coap://coap.thingsboard.cloud/api/v1/354u1g321kcqc1oad3w7/telemetry + references: + - https://thingsboard.io/docs/paas/reference/http-api/ + - https://thingsboard.io/docs/paas/reference/coap-api/ + + - name: ThingsBoard Provision Device Key + id: kingfisher.thingsboard.2 + pattern: | + (?x) + "provisionDeviceKey"\s*:\s*" + ([a-z0-9]{20}) + " + confidence: medium + min_entropy: 3.0 + examples: + - '"{"deviceName": "DEVICE_NAME", "provisionDeviceKey": "s2s1gfcuatgbi61n8h5s", "provisionDeviceSecret": "xbzsovaw9ix4qfhi14an"}"' + references: + - https://thingsboard.io/docs/paas/reference/http-api/ + - https://thingsboard.io/docs/paas/user-guide/device-provisioning/ + - name: ThingsBoard Provision Device Secret + id: kingfisher.thingsboard.3 + pattern: | + (?x) + "provisionDeviceSecret"\s*:\s*" + ([a-z0-9]{20}) + " + confidence: medium + min_entropy: 3.0 + examples: + - '"{"deviceName": "DEVICE_NAME", "provisionDeviceKey": "s2s1gfcuatgbi61n8h5s", "provisionDeviceSecret": "xbzsovaw9ix4qfhi14an"}"' + references: + - https://thingsboard.io/docs/paas/reference/http-api/ + - https://thingsboard.io/docs/paas/user-guide/device-provisioning/ \ No newline at end of file diff --git a/data/rules/truenas.yml b/data/rules/truenas.yml index 9e0067a..a0a6d96 100644 --- a/data/rules/truenas.yml +++ b/data/rules/truenas.yml @@ -46,4 +46,4 @@ rules: - https://www.truenas.com/docs/scale/scaletutorials/toptoolbar/managingapikeys/ - https://www.truenas.com/docs/scale/scaleclireference/auth/cliapikey/ - https://www.truenas.com/docs/scale/api/ - - https://www.truenas.com/community/threads/api-examples-in-perl-python.108053/ \ No newline at end of file + - https://www.truenas.com/community/threads/api-examples-in-perl-python.108053/