From e518fb30f2b87f2ff7a61e125b4162e042fac413 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Tue, 10 Feb 2026 19:24:19 -0800 Subject: [PATCH] v1.81.0 --- .vscode/settings.json | 3 + CHANGELOG.md | 6 + Cargo.toml | 5 +- README.md | 2 +- __pycache__/check_references.cpython-314.pyc | Bin 0 -> 10522 bytes .../data/rules/adafruitio.yml | 2 + crates/kingfisher-rules/data/rules/ai21.yml | 1 - .../kingfisher-rules/data/rules/algolia.yml | 4 + .../kingfisher-rules/data/rules/alibaba.yml | 6 + .../kingfisher-rules/data/rules/anypoint.yml | 1 - .../data/rules/artifactory.yml | 8 + crates/kingfisher-rules/data/rules/asana.yml | 1 - .../kingfisher-rules/data/rules/atlassian.yml | 1 - crates/kingfisher-rules/data/rules/aws.yml | 2 +- crates/kingfisher-rules/data/rules/azure.yml | 2 - .../data/rules/azurestorage.yml | 6 + crates/kingfisher-rules/data/rules/beamer.yml | 4 +- .../kingfisher-rules/data/rules/bitbucket.yml | 8 +- crates/kingfisher-rules/data/rules/blynk.yml | 12 +- .../data/rules/ciscomeraki.yml | 2 +- .../kingfisher-rules/data/rules/clearbit.yml | 4 +- .../data/rules/clickhouse.yml | 2 +- .../kingfisher-rules/data/rules/clojars.yml | 2 - .../data/rules/cloudflare.yml | 10 +- crates/kingfisher-rules/data/rules/codacy.yml | 4 +- .../data/rules/codeclimate.yml | 2 +- .../kingfisher-rules/data/rules/codecov.yml | 2 +- .../data/rules/coderabbit.yml | 2 +- .../kingfisher-rules/data/rules/confluent.yml | 6 +- .../kingfisher-rules/data/rules/crates.io.yml | 4 +- .../data/rules/credentials.yml | 4 +- .../data/rules/databricks.yml | 1 - .../kingfisher-rules/data/rules/deepgram.yml | 2 +- .../data/rules/dependency_track.yml | 2 + .../data/rules/digitalocean.yml | 4 + .../kingfisher-rules/data/rules/discord.yml | 6 + crates/kingfisher-rules/data/rules/django.yml | 2 + crates/kingfisher-rules/data/rules/docker.yml | 6 +- .../kingfisher-rules/data/rules/dropbox.yml | 4 +- .../kingfisher-rules/data/rules/easypost.yml | 4 +- .../kingfisher-rules/data/rules/eraserio.yml | 2 +- .../kingfisher-rules/data/rules/facebook.yml | 6 + crates/kingfisher-rules/data/rules/fileio.yml | 2 + .../kingfisher-rules/data/rules/finicity.yml | 6 +- crates/kingfisher-rules/data/rules/flyio.yml | 2 +- .../kingfisher-rules/data/rules/frameio.yml | 2 +- .../data/rules/freshbooks.yml | 1 - crates/kingfisher-rules/data/rules/gcp.yml | 7 +- .../kingfisher-rules/data/rules/generic.yml | 2 +- crates/kingfisher-rules/data/rules/github.yml | 18 +- crates/kingfisher-rules/data/rules/gitlab.yml | 6 +- crates/kingfisher-rules/data/rules/gitter.yml | 2 +- .../data/rules/gocardless.yml | 6 +- crates/kingfisher-rules/data/rules/google.yml | 17 +- .../data/rules/googleoauth2.yml | 4 +- crates/kingfisher-rules/data/rules/gradle.yml | 4 +- crates/kingfisher-rules/data/rules/groq.yml | 2 +- .../data/rules/huggingface.yml | 2 - .../kingfisher-rules/data/rules/imagekit.yml | 2 - .../kingfisher-rules/data/rules/infracost.yml | 4 +- crates/kingfisher-rules/data/rules/infura.yml | 2 +- crates/kingfisher-rules/data/rules/ionic.yml | 4 +- crates/kingfisher-rules/data/rules/jdbc.yml | 2 +- crates/kingfisher-rules/data/rules/jina.yml | 2 +- crates/kingfisher-rules/data/rules/jira.yml | 6 +- .../kingfisher-rules/data/rules/kickbox.yml | 2 + .../kingfisher-rules/data/rules/klaviyo.yml | 2 + .../kingfisher-rules/data/rules/langchain.yml | 6 +- .../data/rules/launchdarkly.yml | 2 +- .../kingfisher-rules/data/rules/linkedin.yml | 1 - crates/kingfisher-rules/data/rules/lob.yml | 4 +- .../kingfisher-rules/data/rules/mailgun.yml | 7 +- .../kingfisher-rules/data/rules/mandrill.yml | 4 +- .../data/rules/messagebird.yml | 2 +- .../data/rules/microsoft_teams.yml | 4 +- .../data/rules/microsoftteamswebhook.yml | 4 +- .../kingfisher-rules/data/rules/mistral.yml | 2 +- crates/kingfisher-rules/data/rules/modal.yml | 68 ++++++ crates/kingfisher-rules/data/rules/monday.yml | 4 +- .../kingfisher-rules/data/rules/mongodb.yml | 11 +- crates/kingfisher-rules/data/rules/mysql.yml | 2 + .../kingfisher-rules/data/rules/netlify.yml | 4 + .../kingfisher-rules/data/rules/newrelic.yml | 2 + crates/kingfisher-rules/data/rules/ngrok.yml | 2 + crates/kingfisher-rules/data/rules/npm.yml | 6 +- crates/kingfisher-rules/data/rules/nuget.yml | 6 +- crates/kingfisher-rules/data/rules/nvidia.yml | 2 + crates/kingfisher-rules/data/rules/okta.yml | 8 +- crates/kingfisher-rules/data/rules/ollama.yml | 2 +- .../data/rules/onepassword.yml | 2 +- crates/kingfisher-rules/data/rules/owlbot.yml | 2 +- .../kingfisher-rules/data/rules/pastebin.yml | 4 +- crates/kingfisher-rules/data/rules/paypal.yml | 4 + crates/kingfisher-rules/data/rules/pem.yml | 6 +- .../data/rules/planetscale.yml | 10 +- .../kingfisher-rules/data/rules/postgres.yml | 4 +- .../kingfisher-rules/data/rules/posthog.yml | 6 +- .../kingfisher-rules/data/rules/prefect.yml | 2 +- .../kingfisher-rules/data/rules/privkey.yml | 8 +- crates/kingfisher-rules/data/rules/pubnub.yml | 6 +- crates/kingfisher-rules/data/rules/pypi.yml | 5 +- .../kingfisher-rules/data/rules/rabbitmq.yml | 21 +- .../kingfisher-rules/data/rules/recaptcha.yml | 6 +- .../data/rules/salesforce.yml | 12 +- .../kingfisher-rules/data/rules/scalingo.yml | 4 +- .../data/rules/scraperapi.yml | 2 +- crates/kingfisher-rules/data/rules/sentry.yml | 4 +- crates/kingfisher-rules/data/rules/shippo.yml | 2 +- .../kingfisher-rules/data/rules/shopify.yml | 6 +- crates/kingfisher-rules/data/rules/snyk.yml | 4 +- .../data/rules/sonarcloud.yml | 2 + .../data/rules/sourcegraph.yml | 6 + crates/kingfisher-rules/data/rules/square.yml | 8 +- crates/kingfisher-rules/data/rules/stripe.yml | 4 + .../kingfisher-rules/data/rules/supabase.yml | 3 +- crates/kingfisher-rules/data/rules/tavily.yml | 1 - .../kingfisher-rules/data/rules/telegram.yml | 4 +- .../kingfisher-rules/data/rules/travisci.yml | 4 + crates/kingfisher-rules/data/rules/twilio.yml | 4 + crates/kingfisher-rules/data/rules/twitch.yml | 4 +- crates/kingfisher-rules/data/rules/uri.yml | 2 + crates/kingfisher-rules/data/rules/vercel.yml | 2 +- .../kingfisher-rules/data/rules/voyageai.yml | 4 +- .../data/rules/weightsandbiases.yml | 2 + .../kingfisher-rules/data/rules/youtube.yml | 2 + crates/kingfisher-rules/data/rules/zhipu.yml | 2 + crates/kingfisher-rules/src/lib.rs | 10 +- crates/kingfisher-rules/src/rule.rs | 28 +++ .../{main.py => check_endpoints.py} | 0 data/default/rule_cleanup/check_references.py | 168 +++++++++++++++ docs/RULES.md | 48 +++++ docs/USAGE.md | 2 +- src/direct_validate.rs | 77 ++++++- src/grpc_validation.rs | 202 ++++++++++++++++++ src/lib.rs | 1 + src/main.rs | 136 ++++++++---- src/reporter.rs | 37 +++- src/rules.rs | 8 +- src/validation.rs | 92 ++++++++ 139 files changed, 1183 insertions(+), 219 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 __pycache__/check_references.cpython-314.pyc create mode 100644 crates/kingfisher-rules/data/rules/modal.yml rename data/default/rule_cleanup/{main.py => check_endpoints.py} (100%) create mode 100644 data/default/rule_cleanup/check_references.py create mode 100644 src/grpc_validation.rs diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..5dabb89 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "sarif-viewer.connectToGithubCodeScanning": "off" +} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 0792454..f32f641 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [v1.81.0] +- Fixed checksum-template evaluation for prefixed tokens by using explicit checksum/body captures in NPM, GitHub, Confluent, and GitLab rules. +- Updated references sections to rules with API documentation links. +- Updated Google OAuth credentials rule requirements so bundled client-id/client-secret examples pass `rules check` consistently. +- Added gRPC validation support for gRPC-only APIs via `validation: type: Grpc` (e.g., Modal administrative keys). + ## [v1.80.0] - Added `--full-validation-response` flag to include complete validation response bodies without truncation. By default, validation responses are still truncated to 512 characters for readability. When enabled, users can parse and present full validation responses as needed (e.g., for GitHub token validation responses that include user metadata beyond the first 512 characters). - Improved AWS rule. diff --git a/Cargo.toml b/Cargo.toml index 8f2487c..617aba9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ publish = false [package] name = "kingfisher" -version = "1.80.0" +version = "1.81.0" description = "MongoDB's blazingly fast and accurate secret scanning and validation tool" edition.workspace = true rust-version.workspace = true @@ -221,6 +221,9 @@ gcloud-storage = { version = "1.1.1", default-features = false, features = [ ] } tokei = "12.1.2" crc32fast = "1.5.0" +bytes = "1.11.1" +tokio-rustls = "0.26.4" +h2 = "0.4.13" [target.'cfg(not(windows))'.dependencies] sha1 = { version = "0.10.6", features = ["asm"] } diff --git a/README.md b/README.md index 7c3aa29..d3364ab 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Designed for offensive security engineers and blue-teamers alike, Kingfisher hel ### Performance, Accuracy, and Hundreds of Rules - **Performance**: multithreaded, Hyperscan‑powered scanning built for huge codebases - **Extensible rules**: hundreds of built-in detectors plus YAML-defined custom rules ([docs/RULES.md](/docs/RULES.md)) -- **Validate & Revoke**: live validation of discovered secrets, plus direct revocation for supported platforms (GitHub, GitLab, Slack, AWS, GCP, and more) ([docs/USAGE.md](/docs/USAGE.md)) +- **Validate & Revoke**: live validation of discovered secrets, plus direct revocation for supported platforms (GitHub, GitLab, Slack, AWS, GCP, and more)[docs/USAGE.md](/docs/USAGE.md)) - **Blast Radius Mapping**: instantly map leaked keys to their effective cloud identities and exposed resources with `--access-map`. Supports AWS, GCP, Azure, GitHub, Gitlab, and more token support coming. - **Broad AI SaaS coverage**: finds and validates tokens for OpenAI, Anthropic, Google Gemini, Cohere, AWS Bedrock, Voyage AI, Mistral, Stability AI, Replicate, xAI (Grok), Ollama, Langchain, Perplexity, Weights & Biases, Cerebras, Friendli, Fireworks.ai, NVIDIA NIM, Together.ai, Zhipu, and many more - **Compressed Files**: Supports extracting and scanning compressed files for secrets diff --git a/__pycache__/check_references.cpython-314.pyc b/__pycache__/check_references.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7766438eceb6c62c9a9b0ad26da51f6a710b2610 GIT binary patch literal 10522 zcmb_CZBSd+mG|jAeSaV$@nyishG1l{F!(co?O+TxaUjYwmXijf%IE<~1XA9UVq4Qp z)alL!x0@Q*X-y`b37PFo?b+^x{Mbo4v%AGjQpf$Xj|v;*g`H-%+1cqoTql#+v_JNo z`}8CkLy~6qV&8k-z31M0?!D)pdp^|cuv-YEe|_o)k-Kg}{(v7^(yIV-?-_=WG~r2t zj1ZpbXO8OnKi8{_z_+P_@a8=_b(}qFK@ywR#kByg^#R;c z-Ujvd3KDfDT)YF|5(>L{C%~I1T*j9I?4ocv?*_Px!WDcuz!em(JX6J29(aIyhqSD%WN)kk=buu#g?0Bmr zj<6@bH z^tCp1^Ix^8X4p$!kdn~^B|~9>K7~6Qi$xU!-s<9{S-}@86i^)qL4_6Q(u%uBU0k&VlFvDiKJW^6DLCn31{3U zBE5L{xzhz!5Vt|qR`@SfLzX6=lWN9!7qhvVRV?K8qx6%mfD#XQG@&r0#Lod)NW5QL zRt^CR9@tzHG=9Kdo3v{kXM7~C8zFHvzz*kxDIozyyY2^oMS7WV6>iLFR|{@+F2%WF zWG0&YW(_KofxLK(>jV1jxrT8Yjuc==m=@&XT#B`b4O1&y=rg8b#LWSdwkKiO zJ=oo!9O%$k@Wcmj28H&=v3=O7jqS&~yCwzlbU8`iUP9uQfXx{cv%%Ttq2NS~igkq86(FhfflEOxXDaFqB_Vx0;gFU?`gM6=| zhZ{CJq3FlOSTe5YK>|nOiXoaS;blT3<5*-W9EzfNiN}Shuqt*8Aa?=VYo!DzdX&ME z!htZ76ec24f_4BwiuG(r5`r*<0HWSFniSz}ODHDVJ1&G3gW5!?myl^2J^7^G`*K){ z2%@bZiG%P=NaDkQOeMfy07w%MT9*bd4$d8$9?aNUmTmhNZTmB}hts{;x~=J>S(7>4 zowZp1Zgl3MA3XE2FI(b%`TR7u;;dbE)-O8i=Zu<;$_1$lXaB5Qa>YH+3~}vYxW)F zbZz^c>h_tQth@5+iOVN`W}9zV*z_-_K4AaNa?2v0d@6JNTT7m&@3@EWvZQABs)5)Y zNaOdrKem>nRUv(!d`7`SMO*{jnvLZ%ujdV|EXZi1xCKz@DX8yt9 zjDt_rnsmC2*YD5G*~tW%q%O$@^D16kKb)820od4)5BB74d>r5*LsHkNI{=rWlgUdi zIH(#@M|k4_7>UFh* zx5SNyjrFjqFT;&<8s@-0wMT?0$ZMdgQKf1)FS&>E^4jlSLnM3x1-2n0r2|enKekC5 zJ3`bw`U(zMhm)St@%AE4GJO*+dg{Ahmy3q}4_q{aT;%wsT*O=Ib9?j+`DpA*`N;J( z`ADU|h>wOijq}5xO92;_3~{hd=Z4(nXy@P_6^+1`d>h^~IOQoAIy>_w&Q5Cbfa%lb+of>F zd%KT%b&7EU6bdks6vL#Dn23cHa-9)7;g%@|G_I0TWCkREN`HK4$nQxRJx#RV;804} z;ZfLQy+bL3rwIygU&`3&QL%KLo!lzEnv9iBs@`H&?MEO^JPob%CF}-q4XR2>wOo5xc07_kw&HMKnz}f(>}XhYG|Zh`Xp|id z8ApHmShm!iKC!B^>bq9#u1g~qN0#mNi}w1tW3s(|*?w@zesH03rL1Q5$+`3Mu1wk9 z>AtMHVs_uMd;5}m`&blsq zY~8wO-TEov^t(Q`IxZc!ctEabUa~f4T@|x?=UhLpe4}#Ca&5a@)h@f*Z`<0lm9?{@ zuTQ)>G5f8#b8>m>%ho@ySlyozoxbanDrkDGYPqU;v8q|FY`J#insiN=-!GSU0@kV? zP`>!oL|j!@eV2X9rA>>aP1jrtf?V2^DSZqVYu9()zt|IWvE57d?u946)?&-$9goPa zM{u!6m}Pr6{L9$8A6%_o-L@~o?J_UYMyYZddZ zw=3VOoVUEwC~w9My>06TC304u-;L+xa-Uq{yKVBRJHqNai~5)T;^@5VANrSD4lTAE zlA8~|BQDf0lr4_I&Paf{=900LBEg|ft{{59jPQY zHhGWe^*1^!0RLKVJyK@=wUa}*jK#dN?&x;**Za5i>|x)nVgP=(&WZ3I8^G@|So5B) z`sgb?c>9|4zp1qV{F^3g-^1qL?B)=Dn8m!S?$KKIX7#q?o$Sqn z48R|-Mu0!Cx-j2DYZ^8oyqChAHo&=MLim<>`=gcYtz8t}R|D{GOA-EUr5^JC9|aao zFx5#zuX7iEqUx%8JJ3k=G8=2FA!wJjLWbI^j_JCK*(^8|pqSUw;vFxod{9|6KsS(F z%)LR{e;6d0n!v%3QyH}yECb+C(^ScG&@R>&4C2$e01NI1l!K5!JJ*Rv003)`E)P8( zVvSmXl0_9?=%n#-Dh6G99!)|?i^55jOC@XkTH?2$xoG7pWm{&Y*U!CrZr-?5cHp~% z={~sNv#$BRC2Ob5bru{d+6r-ax^fQy>96z#QAXaqyrDV!gJ*wN58on#e@|p8!m(HEeRP4Ot^i6YFlkJlA zqIJo=@wS~svIk$36oBFAlXx~;e>R<$AVx}{1(;&?nHN`h|?+_+La zG%9@M$xcE?3AK5=edSEuUCd;6BQ4bC0PTjv{Xl==hx4y>6q~48wW5j5qS8uhGLioVVIW++Y{pKgT*=G|dMG z$74-3_t207ICIpnLu5*gIpp;}DIam5L58p(je?U0^`(?`=S~rv@X}7WvZ-yOVhCL6 zMb2`yu@%D*trRb{_h>|f@I_3#;5ipbOyp*QAv|Ld7{}8T7REy0#_sSuq=8<^3rxhr zt|`V`tHMnP(Rj+db7#&UF|O{Z?~&9#s!HY*Fi)=818ah)No*>-(F{1&xtRDYXtb$4 z1EKSg$>gL5I^I zK@X9>MyM$&7J4@jQ9@@o{SqYWc=*fdNC&+Yi*ljRDsG&^mZVzjh==jT%pAV}KTrqXy#G@TXA;4+IOVHsw|gn%r~K z{-&aVa-cSa8fQBJQL%stRtS5Innm0kHwDc8<_G)nMu<|Iv|X<&<_dEWSL7eBqB^ri zdvWju`BrBb;IpionPB#tpZB_{-Tc@ zsG(_;$1O#-!y2%_U9fFH(}4#^18n;S*d6O)I{+IWfHeumDQVB|GH}RDPER!7f7&CU zVd0HN7jaS-*GI^gyMcR|$C_0qXRe1SLcGq@R_Yn)@W3mcdz`~Dc#Pr41Rmi$L}NQV z@E&paLZ==YjHznESPXujfDh|>dP2(3N1IcwLUUfbfU&HTq4UAqdpk&NJZ%F&G!u~J z%)OrgP%NG|5Xzj^u~2G9xj_$9`q!ilFxS!bc4JtC8T9IDJVdpt5KWD3;oE{c6-fw_ z^xbY$jVn`EG&FfO96F?wPlh5>zOHC&G!&H%0if2u1|225+i3#fA?{7nb<>;n>-J2; zzIV7^nSN>dmHn6Yg~ZMCH_qSu&W-OZ)(_El4W?qB6)vPq-S{IAp~I81^iQebet^yW zgWWwt{efPO!gPozo>Dr`(+YFOTSs*WhzX+V7KB4b_n8!nJ};oD0p~3#`nZU{!eFHf z;Ng=J;OP^oAt54>ilv|#K_oBg@togjJbn`fs$t@ zBME9+s=h*7ND7X|CLw$T){eLf$A+H)(BCnjp{j6b2NiZ%v5G<=DGq+J_CC zdAjeF-7|?R56?8qwY%o4=B=`&Yl%CE#@w+_%&ui~^`g0Y+3Z_1`~G?NJI^gN+-$kg zlIeV0Hv5*%{w1?t9(wY_K|WJ@@>Y03y4rcU^QSM)vu_W-HGIc)UX zH%`y)dwbxY2Cf=r3~!uHAD2sxWI5YQ1K%B3EFVe_EODoB4aZ+OnG1lL{(j(gS)1%` zmo2-OxIHV-_~hpMyL#fP{@g@1RZX|aT-|I4vj6#0EpZJopR>3;*gDzl z|I42&#M#40_=djg()WAy?^ZVTH<;cd-A4F$-{tB*$i3fefc#biHsA7E`#a3|YbE59 zV9;Bw7=yuZY*hW>lFDuHYZX-L#BpT0(!z!E_jSUqHa7db@p8T#Nl! z<;Yd#EFQ(0|IL>mi6~4Vybc!(M7M}oNxy__m1P*_Kgi+#BHR8zN + (?:ak|as)-[A-Za-z0-9]{22} + ) + \b + (?:.|[\n\r]){0,80}? + \b + ( + (?:ak|as)-[A-Za-z0-9]{22} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 2.8 + confidence: high + examples: + - "modal token set --token-id ak-BJbwFRtNnI4Y11oxC4hngY --token-secret as-sRul9S1EAi9qNlq3G6NTIb" + references: + - https://modal.com/docs/reference/cli/token + - https://modal.com/docs/reference/modal.Client + - https://modal.com/docs/reference/modal.App + validation: + type: Grpc + content: + request: + # Use the same handshake call as the Modal SDK (`client.hello()`). + url: https://api.modal.com/modal.client.ModalClient/ClientHello + headers: + content-type: application/grpc + te: trailers + x-modal-token-id: "{{ TOKEN_ID }}" + x-modal-token-secret: "{{ TOKEN }}" + x-modal-client-type: "1" + # Modal uses this for compatibility checks; "0" is rejected as deprecated. + x-modal-client-version: "1.0.0" + x-modal-python-version: "3.11.0" + x-modal-platform: kingfisher + x-modal-node: kingfisher + body: "\u0000\u0000\u0000\u0000\u0000" + response_matcher: + - report_response: true + - type: HeaderMatch + header: grpc-status + expected: ["0"] + + - name: Modal Token Secret + id: kingfisher.modal.2 + pattern: | + (?x) + \b + ( + as-[A-Za-z0-9]{22} + ) + \b + pattern_requirements: + min_digits: 2 + min_entropy: 3.0 + confidence: medium + examples: + - "as-aB1cD2eF3gH4iJ5kL6mN7P" + references: + - https://modal.com/docs/reference/cli/token + - https://modal.com/docs/reference/modal.Client + - https://modal.com/docs/reference/modal.App \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/monday.yml b/crates/kingfisher-rules/data/rules/monday.yml index 8459472..7051eb1 100644 --- a/crates/kingfisher-rules/data/rules/monday.yml +++ b/crates/kingfisher-rules/data/rules/monday.yml @@ -34,4 +34,6 @@ rules: status: [200] - type: WordMatch words: ["data", "me", "id"] - match_all_words: true \ No newline at end of file + match_all_words: true + references: + - https://developer.monday.com/api-reference/docs/authentication \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/mongodb.yml b/crates/kingfisher-rules/data/rules/mongodb.yml index fb10f65..624cfe0 100644 --- a/crates/kingfisher-rules/data/rules/mongodb.yml +++ b/crates/kingfisher-rules/data/rules/mongodb.yml @@ -26,7 +26,6 @@ rules: \b pattern_requirements: min_digits: 2 - min_uppercase: 1 min_lowercase: 1 min_entropy: 3.7 examples: @@ -48,6 +47,8 @@ rules: - '"orgId":' - '"id":' url: https://cloud.mongodb.com/api/atlas/v2/groups + references: + - https://www.mongodb.com/docs/atlas/api/ depends_on_rule: - rule_id: "kingfisher.mongodb.2" variable: PUBKEY @@ -108,6 +109,8 @@ rules: visible: false examples: - 'mongodb-public: qj4Zrh8e6A' + references: + - https://www.mongodb.com/docs/atlas/api/ - name: MongoDB URI Connection String id: kingfisher.mongodb.3 pattern: | @@ -130,6 +133,8 @@ rules: validation: type: MongoDB tls_mode: lax + references: + - https://www.mongodb.com/docs/manual/reference/connection-string/ - name: MongoDB Atlas Service Account Token id: kingfisher.mongodb.4 pattern: | @@ -143,4 +148,6 @@ rules: - mdb_sa_sk_BdIX_jLzut2WTgglKzKvSgWMDDj5hEoTqdwOyLOL validation: type: MongoDB - tls_mode: lax \ No newline at end of file + tls_mode: lax + references: + - https://www.mongodb.com/docs/atlas/api/service-accounts-overview/ \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/mysql.yml b/crates/kingfisher-rules/data/rules/mysql.yml index 8f1c2c3..ecb9c62 100644 --- a/crates/kingfisher-rules/data/rules/mysql.yml +++ b/crates/kingfisher-rules/data/rules/mysql.yml @@ -45,3 +45,5 @@ rules: validation: type: MySQL tls_mode: lax + references: + - https://dev.mysql.com/doc/refman/8.0/en/connecting.html diff --git a/crates/kingfisher-rules/data/rules/netlify.yml b/crates/kingfisher-rules/data/rules/netlify.yml index 7fae1eb..828d03c 100644 --- a/crates/kingfisher-rules/data/rules/netlify.yml +++ b/crates/kingfisher-rules/data/rules/netlify.yml @@ -30,6 +30,8 @@ rules: - report_response: true - type: StatusMatch status: [200] + references: + - https://docs.netlify.com/api/get-started/#authentication - name: Netlify API Key id: kingfisher.netlify.2 @@ -64,3 +66,5 @@ rules: - report_response: true - type: StatusMatch status: [200] + references: + - https://docs.netlify.com/api/get-started/#authentication diff --git a/crates/kingfisher-rules/data/rules/newrelic.yml b/crates/kingfisher-rules/data/rules/newrelic.yml index c208aa0..e4953c2 100644 --- a/crates/kingfisher-rules/data/rules/newrelic.yml +++ b/crates/kingfisher-rules/data/rules/newrelic.yml @@ -34,3 +34,5 @@ rules: - report_response: true - type: StatusMatch status: [200] + references: + - https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/ diff --git a/crates/kingfisher-rules/data/rules/ngrok.yml b/crates/kingfisher-rules/data/rules/ngrok.yml index 711648c..3f4053a 100644 --- a/crates/kingfisher-rules/data/rules/ngrok.yml +++ b/crates/kingfisher-rules/data/rules/ngrok.yml @@ -33,3 +33,5 @@ rules: - type: WordMatch words: - '"endpoints":' + references: + - https://ngrok.com/docs/api#authentication diff --git a/crates/kingfisher-rules/data/rules/npm.yml b/crates/kingfisher-rules/data/rules/npm.yml index 0d0a6da..9c57abb 100644 --- a/crates/kingfisher-rules/data/rules/npm.yml +++ b/crates/kingfisher-rules/data/rules/npm.yml @@ -12,9 +12,9 @@ rules: min_digits: 2 checksum: actual: - template: "{{ MATCH | suffix: 6 }}" + template: "{{ checksum }}" requires_capture: checksum - expected: "{{ BODY | crc32 | base62: 6 }}" + expected: "{{ body | crc32 | base62: 6 }}" skip_if_missing: true references: - https://docs.npmjs.com/about-access-tokens @@ -23,7 +23,7 @@ rules: min_entropy: 3.3 confidence: medium examples: - - "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" + - "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" validation: type: Http content: diff --git a/crates/kingfisher-rules/data/rules/nuget.yml b/crates/kingfisher-rules/data/rules/nuget.yml index 3ba909e..b358e68 100644 --- a/crates/kingfisher-rules/data/rules/nuget.yml +++ b/crates/kingfisher-rules/data/rules/nuget.yml @@ -31,6 +31,8 @@ rules: status: [200] - type: WordMatch words: ['"Key":'] + references: + - https://learn.microsoft.com/en-us/nuget/api/overview#authentication - name: NuGet API Key @@ -65,4 +67,6 @@ rules: - type: StatusMatch status: [200] - type: WordMatch - words: ['"Key":'] \ No newline at end of file + words: ['"Key":'] + references: + - https://learn.microsoft.com/en-us/nuget/api/overview#authentication \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/nvidia.yml b/crates/kingfisher-rules/data/rules/nvidia.yml index 1329314..47b7bb2 100644 --- a/crates/kingfisher-rules/data/rules/nvidia.yml +++ b/crates/kingfisher-rules/data/rules/nvidia.yml @@ -29,3 +29,5 @@ rules: status: [200] - type: WordMatch words: ["id", "versionId"] + references: + - https://docs.nvidia.com/cloud-functions/index.html diff --git a/crates/kingfisher-rules/data/rules/okta.yml b/crates/kingfisher-rules/data/rules/okta.yml index 16511f4..243eda0 100644 --- a/crates/kingfisher-rules/data/rules/okta.yml +++ b/crates/kingfisher-rules/data/rules/okta.yml @@ -16,8 +16,6 @@ rules: min_entropy: 3.3 examples: - okta_api_token=00hqNORUpnTcdFWA5WEM4YwOkw6RXeFw21lFDRKmY1 - - 'okta_api_token = 00aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' - - 'OKTA_API_KEY = "00-aaaaaaaaaaaaa-aaaaaaaaaaaaaaaaaaaaaaaaa"' - 'okta_secret: 00QCjAl4MlV-WPXM-ABCDEFGHIJKL-0HmjFx-vbGua' - 'Authorization: SSWS 00QCjAl4MlV-WPXM-ABCDEFGHIJKL-0HmjFx-vbGua' - | @@ -40,6 +38,8 @@ rules: words: - activated url: https://{{ DOMAIN }}/api/v1/users/me + references: + - https://developer.okta.com/docs/reference/core-okta-api/#authentication depends_on_rule: - rule_id: "kingfisher.okta.2" variable: DOMAIN @@ -54,4 +54,6 @@ rules: min_entropy: 3 visible: false examples: - - company-name.okta.com \ No newline at end of file + - company-name.okta.com + references: + - https://developer.okta.com/docs/concepts/okta-organizations/ \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/ollama.yml b/crates/kingfisher-rules/data/rules/ollama.yml index ba686c6..55887d4 100644 --- a/crates/kingfisher-rules/data/rules/ollama.yml +++ b/crates/kingfisher-rules/data/rules/ollama.yml @@ -40,7 +40,7 @@ rules: - '"response":' - '"done":true' references: - - https://ollama.com/blog/turbo + - https://ollama.com/blog examples: - "ollama key = 8bcdd9b4e28e4e1b8bf14a2eb8701220.QH5p5TU2BDwzHu5_RCtvJXsj" - "ollama key = e56714bd7c1146e4b4801244bc2bc67a.3GAswjZGZ5YY6Qdgt0xg56vM" diff --git a/crates/kingfisher-rules/data/rules/onepassword.yml b/crates/kingfisher-rules/data/rules/onepassword.yml index e7f6183..ebc4f87 100644 --- a/crates/kingfisher-rules/data/rules/onepassword.yml +++ b/crates/kingfisher-rules/data/rules/onepassword.yml @@ -52,4 +52,4 @@ rules: - A3-ASWWYB-798JRY-LJVD4-23DC2-86TVM-H43EB references: - https://support.1password.com/secret-key-security/ - - https://developer.1password.com/files/1password-white-paper.pdf + - https://1passwordstatic.com/files/security/1password-white-paper.pdf diff --git a/crates/kingfisher-rules/data/rules/owlbot.yml b/crates/kingfisher-rules/data/rules/owlbot.yml index 1f1d4a0..6fafe10 100644 --- a/crates/kingfisher-rules/data/rules/owlbot.yml +++ b/crates/kingfisher-rules/data/rules/owlbot.yml @@ -21,7 +21,7 @@ rules: - "owlbot SECRET b7d21c0e88e9a3c5938fb045b2b6a5e693eaf9d1" - "owlbot TOKEN 8a5de3a89b7e4f29bf728b45adcdea6ea3410c78" references: - - https://owlbot.info/ + - https://documentation.owlbot.ai/ validation: type: Http content: diff --git a/crates/kingfisher-rules/data/rules/pastebin.yml b/crates/kingfisher-rules/data/rules/pastebin.yml index f19536c..4a82cbc 100644 --- a/crates/kingfisher-rules/data/rules/pastebin.yml +++ b/crates/kingfisher-rules/data/rules/pastebin.yml @@ -36,4 +36,6 @@ rules: status: [200] - type: WordMatch words: ['invalid api_dev_key'] - negative: true \ No newline at end of file + negative: true + references: + - https://pastebin.com/doc_api \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/paypal.yml b/crates/kingfisher-rules/data/rules/paypal.yml index 47fddb1..f3366a4 100644 --- a/crates/kingfisher-rules/data/rules/paypal.yml +++ b/crates/kingfisher-rules/data/rules/paypal.yml @@ -17,6 +17,8 @@ rules: visible: false examples: - paypal_client_id=AZJ6y8Dpr1TYbqAIdhkPzyhjXoY6mIdhkPzyhjXoY6m8GplL7C3zZ3lPrkTIdhkPzyhjXo_Dx3IdhkPzyhjXoY6m + references: + - https://developer.paypal.com/api/rest/authentication/ - name: PayPal OAuth Secret id: kingfisher.paypal.2 @@ -57,3 +59,5 @@ rules: depends_on_rule: - rule_id: kingfisher.paypal.1 variable: CLIENTID + references: + - https://developer.paypal.com/api/rest/authentication/ diff --git a/crates/kingfisher-rules/data/rules/pem.yml b/crates/kingfisher-rules/data/rules/pem.yml index 0c0d921..1e99aec 100644 --- a/crates/kingfisher-rules/data/rules/pem.yml +++ b/crates/kingfisher-rules/data/rules/pem.yml @@ -49,6 +49,8 @@ rules: -----END RSA PRIVATE KEY----- - | "-----BEGIN RSA PRIVATE KEY-----\r\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAlwAAAAdzc2gtcn\r\nNhAAAAAwEAAQAAAIEAtDSHFO5tfN+jYMJuiNvBaplkSI3eFqKMLOvXyVu+dmSEic6xyKWQ\r\nqjQiFpXogArvAq2tBxWOq7F+a6rNhDKdICD2amRwDHqKD1bzXVSZ5c1XnpCFsBiQaEyX2i\r\nqyjScnntFHIpTCVHNxILDxsStocj6Cf99C7hfCGVhft/Ts/O0AAAIQJOKnUyTip1MAAAAH\r\nc3NoLXJzYQAAAIEAtDSHFO5tfN+jYMJuiNvBaplkSI3eFqKMLOvXyVu+dmSEic6xyKWQqj\r\nQiFpXogArvAq2tBxWOq7F+a6rNhDKdICD2amRwDHqKD1bzXVSZ5c1XnpCFsBiQaEyX2iqy\r\njScnntFHIpTCVHNxILDxsStocj6Cf99C7hfCGVhft/Ts/O0AAAADAQABAAAAgBcaTN8gGi\r\nVSPo3fH3CoS8mw1KyAk6JvQG1Z5xZHjsl65YsNVrmUkFFh0aT3nxEbVb0QKwineN0GKmD/\r\nSs3R91a573gzli7TJPFCHhhBbE7FRC4KQMTc1/UANwFYQVcfZ4n9IVHr3jiWToSY3XbC66\r\nZcd0sg+d+YRjIxUktuNFHBAAAAQQCOOKbSUJAWzcTDbxImwDCAfBMlEeMAnJrwobL/zxbT\r\nGhKdnqnomoreFdYL8vOcOlwZG0hUKIA6AM1GsMzp6aCwAAAAQQDmAABpOQnkDy8v8kTDhP\r\ndW3lAqRGOU4WRWj7WystQv/VjuJpceekhOyhNJBuNHDKZ3IT1agAZHIhhL+webE2S1AAAA\r\nQQDIk4H1agCohlHUg50PcyKzE/zZ85Gw0ErTmgqIIGd4B1AqUtjwVe1qFoqHuZPtq2cbVF\r\n1HTHh6GX//J6rKWVJZAAAAGWJsYXJzZW5AYnJhZGZvcmRzLW1icC5sYW4B\r\n-----END RSA PRIVATE KEY-----" + references: + - https://www.rfc-editor.org/rfc/rfc7468 - name: Base64-PEM-Encoded Private Key id: kingfisher.pem.2 pattern: | @@ -68,4 +70,6 @@ rules: confidence: high examples: - 'PRIVATE_KEY_B64=LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBb3kxWFh1VkFRcHFIYlFFMDVta2hyTmcvMTI0Ri8ySzlPYW5pelpUWlVVaEswOFU4CkxhaC9SbVVsWHFRMDEvU255aktGOWZqUDhFcU1OZ1dpamUzYmVwL3RPOVpTMEFUMi9PVlJXeS9TOG52RDQ5WTMKenMxMktSbERhR2lZc0RsYUZrbHJkeDQ4RWhRVmdHN3hmWE1jaC9OejJzc2FEby9kRkNBOW80TkZZQWUzM2UveApWNVo1UHNkWkl6dkNZQVlCNDRoUEtpN3JXRE1IbFdzM1kvVkVtQXMzSzVNK2QvL3QzRHB4WnBEbWJERGdYa2w2CjZUdDh3VXloUVZ3MkZpMStobTF1T2QwYjFkaW9aNko2OXNTT2JOZXpSR3YxYjdZaFltT0JKL1JBbHN5ZHoxTmgKVXpXT1lYV0Z1OGJrOU9JM3lQMEc0TE84QjhtbWRldE1RVVoyelFJREFRQUJBb0lCQUN2ckhUUHVVZ0JiSlE0QwpvQ0ZQdEgrWDZIN3NIdk1ndVR0VzdUTlYxN1BYMkVQdE53ZzI3S0tld0pNYmNSbWF3THBjSk5BU09xMDY4MGZxCjlsaHE1NEsybnB4WFVBeXErV3NSc1hid2hUODhibm5aQTBaRzZJR2hTaEpFN0t1cGxBU2htQ29FV2ppbmJTNFgKTGlvTW5HWSs4VFMzSzNrMTRWUDBaWUtuNXprMERHZnFBMEo0VTRXSmxUeGwrTWZxd0pJOTlrcTdHbFVlZkdncQpuK3Q1d2NrV3BPbTd5TUJjZTlTSXlmTm54bnU3TkZYQm50VTN5RGxSUThWUWZmNEtRMzJCaWNiYlJWemR1TThNCnNxMU5CZWNzL0EzUXRvdG1nWUc4d094ZXpNS3Iyays2QzB2NmlFc0h5T0lmR25GWktSZDJFd0dnWlo3aytURHUKUUYrcjd1VUNnWUVBMkRqNUJoYmpybDFRNTZya3BhTGFvVldRV1Y5YUYzUUJtNlNZM2VQYmlvY2JNR2k1ak1ESQpkSjdJVXlLYUljK3BNV1RQYlBmVUd2WmNENlczZDFBNUNUSnFuWHVuVlY3czRqaWJ6WDZUbjhNM3IrMHZTZnNZCmdPMHBtRFpndlNqaVZTRUNBQTZFOFUxQ1lFZU5KUDFDOW12cGJVNzJRTEpndWp3M3JMb2oyYmNDZ1lFQXdUSXYKOUNSeWNOQXRBbDcvUHdWZGh5eXRvVHBSRnZDSU1HSVk5SjMxZ3lva0ZlaFQvWjQ4WkF6anl6ZTBSUXYzdGUxTQoveVJMQkVETGkwbEtrZFVXckVkaVR3dm1KdkpwMDZ0OEdCbERsK25ycXVLWTFxVThDbTR5cis4QzZtRThkVnZrClNINXBhRXptOERFTE1wSjhGVTZFYnhmZHZjRzZmSGx6dnVnZmc1c0NnWUFFQ1BRa3QvS2h3MTRLSkxkRm5BZG0KY1ZsVFFhTkZ3c1Z3NlI1dExaNWdOR3MrZVFYVmFaZVVEWTZCZHFqWHJxOWltNVgvVzVTYXVEUTVtb2NVOCt0TQpqNk5Mc3c0SldzOGkzWm1TdVNUNkcwT0R4ZkpXK0JlWitGTUpZeUpsQlVsTCsyUzFLWkF6akpTTGhXcE40V2dKCmZ6UUk5U3RGUTg3b1NzMWpMTW9VZXdLQmdGOE9CMlFURHErTTdhaE4vejROc0wvU2JyZDJEdkcvZFBLQlFaQVIKcS90V0g1MGJ5ejlzdkgvcGk2YXdDS1UwUnpPZXh4UjkwZDhNMWxqNHZaVFZDQ3ZKajRnZTdhVlovbEdqL1JHSwpWS1NJOW1nRXgzaE1vaWJybzByR3lXTnlaaUhFRGFUUmRhRll2UU9PemRpYkZDd1RqcnR1UGE2Z2c5VzhtQU5sCkNDUmpBb0dBSTRIbnpyV3kzaU5kR2xqVnh4bW1DN1V0c0MvajJBUEZpcHc0ZHJ0U2NsMDFRZzF5WkowbDNBTk4KOU5lTmVSUUFzN3pFTng2T1B1SzlxYy83T1ROMTJKaHdoUTIzdXZwNjZjV0krdTRjcVpOZTJyZVFVVWVmM3psbQpMcXRmOU50VHp5M3pjMGZQcGoxQnBlRmxHSG9SVDhjVHpBWjFTeGwyZWChazlqS2RVeDQ9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0t' - - ' "privateKey": "LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBbUhKOEJHdTFYZUZ4aENVQXBrNHNSTVI4RnRTdGtyMEx0OWtWTGNSUjRFWitiOWhHCmR0blJpOFhqV3d5MU5zMHliMkJMdHBpVHZKSFVKTUphWXluZ2ZkZnZhcWhocm1yYm5vV0pLQkxmeUxwTXFNS1EKQ3RialFxbnVrQURJUWVQd2ZGeTNpVHkxd1JkRC9zTUs1U0VtV0Fxb0pZQk50eTFZZzA2UzVkYVlPM2xjY3hrYQpQWjRjcm9McWF6Ny9tU3dDVTR5VWRSb3h4WVF4VG1MZXg5M2tqU09TTmdpK0FXc0lCbjV3UHI0VHNuVHFSeWpIClN2aEdMdk9YREpRYWZRdk56WjFSL1FYMzlOQk9xOEVKZW5pWXdaUm9uNVcvNVhMYW94MFFyUGhrY1BES3A5SVUKeHpJakUwWlNmMStUK1FFbTQ3TkFtSnhvZjFhdGRFVzZDTCtheHdJREFRQUJBb0lCQUQ3enI4REhsWnFSK1NWZgpmbGd1bWRzLzVCb3Rjd3ZRWXlGbFZIaVV4RmEvNVlCY0tDVDJKN0QzWTc1NmplNTJaK2hVTkkvUGk5cG53ZG40CkpBa2xCdDRRcUg0NzBES05UK216TFFOT1gvanM3YkVXdnhLcTBDZjhNbFptN0V0QlRGS2VtdS9pRVJBT2duYVcKcGs0ZUZVNXdBQ1dVU1FObWgxR1p4ZEdCZjFXM1VjUnQxcFRvOEtQTDluZm4vSGJiRFNsQkNVL3VIcWd2TSt2cApmTE03bzRIVDZ1K1ZzU00rWGZqeDhpeE5ZRHdoalNuKzQyZm13d1d3ZzJISHUrdUozZ1pUSWQwRUI1VW9hdUNjCjZUTlVtcEJscjU5UGFmVkZRWUY1S3VxaHJXKzVQaWpHcHBZcXg4Ynl6aFpOQzkwZnl5V0NXcXg2eGFZVm5OdzgKNkJmUXM2a0NnWUVBeVlyRVg1NU1RTzJnWDY2TGwxaGJDMzNzWk1OZzloVG1SK1doSTFjNksvbFZ1TFoyL0RPdwpsYTZ6eHdBU204Z0ZyVUFYbUljV2h2b3FwWGVzNWZzOVZKeDlNT0ZVYVBrckRPQllnY1laMUR6VVNVOHc3SSttCnlyV3hRUkRNajhvSGpRbHVpM0s2MzZucm5RajhxOGkvQ2dranVPcHJGZnliMzVEMFlDdjVXZzBDZ1lFQXdhT3cKRWFhN0l1MjFGa08vbmFjdVhjSnBhNkVlUTNqZFNlNlRQaXZ6bVVXU0haeGJuUy9XSnJaRjQwSExzUWxOZHl0ZgpNTTBKZFU0VmMyR0NVc1pMYjdQSmJwdVRqRERSSHJXV1pCMnhiemF0K3A3N2RzNWlOcXFRcTZ6M0syUVh4Y3ZTCis5am5VZXpDU2Y0N1R1OWNTTW96V3hTMW82b1BPSFdHVFRvdHR5TUNnWUFQdWc1Y3o4TnZoWnR3Ry9TMG1LWnkKSFI5bk5YL0pkQlFNSkRVUXh1dTVKcm16c2psU3NNM2t3RDh6RmlSZGw1d3B5c2lNbEc0RGxsM2hqNWNrVXhpVQpFNm9KT0d3WHpPbTVGWUNTajl6UUhQY0x5V3d0NlgvQWJiRXBQS0JaMEJBS3gyT2k2ZzcvQ1FsanRhSFIzZFphCmVDQWJlOTlqVmRUcit5bTJuM2ZUdVFLQmdBMm5TZ25rbEx0Z3dXMEJkK2hZMm1jWUJ6RGttbXF0Z2dUdGdvcFcKdFFWd3AxM1pJWWlTeituSTNtS295QUVDbytpc01Ua1NyQUVPY1dyQ1RGc2p5anZsRkdYdEtGa3hNLzJUVmpoVwo4NlRnMlNHYnhpVlpaZ2x1dTJhdmVub2Z3NkZadnRXdE5KcE5OR0hkUURkUG4xVXVsTEp1WW1SWTRGdmR4WXQ2CmQ3QzdBb0dBRUsvalFiZ0l3OXFLQUNOZ0JySnB1cU5Ham9JajFoQTRlb29DMXp1bFEyZUpnZ2J5OTBpSDg2VzEKM0xyOVZMVFkyc2JKTzlqekZVR0lOL01BOEhYQTE1a2grZHRibkRsdFRFZGNnenBCRzhCQUZRQ3hQWnBGWHhtZgpDUmhXN1l6RW1IeWJ4R0toR3NOK2M3NUhKTHZFSWwrRTh6eitXRk9xT240dkJXU1ZwSnc9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==",' \ No newline at end of file + - ' "privateKey": "LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBbUhKOEJHdTFYZUZ4aENVQXBrNHNSTVI4RnRTdGtyMEx0OWtWTGNSUjRFWitiOWhHCmR0blJpOFhqV3d5MU5zMHliMkJMdHBpVHZKSFVKTUphWXluZ2ZkZnZhcWhocm1yYm5vV0pLQkxmeUxwTXFNS1EKQ3RialFxbnVrQURJUWVQd2ZGeTNpVHkxd1JkRC9zTUs1U0VtV0Fxb0pZQk50eTFZZzA2UzVkYVlPM2xjY3hrYQpQWjRjcm9McWF6Ny9tU3dDVTR5VWRSb3h4WVF4VG1MZXg5M2tqU09TTmdpK0FXc0lCbjV3UHI0VHNuVHFSeWpIClN2aEdMdk9YREpRYWZRdk56WjFSL1FYMzlOQk9xOEVKZW5pWXdaUm9uNVcvNVhMYW94MFFyUGhrY1BES3A5SVUKeHpJakUwWlNmMStUK1FFbTQ3TkFtSnhvZjFhdGRFVzZDTCtheHdJREFRQUJBb0lCQUQ3enI4REhsWnFSK1NWZgpmbGd1bWRzLzVCb3Rjd3ZRWXlGbFZIaVV4RmEvNVlCY0tDVDJKN0QzWTc1NmplNTJaK2hVTkkvUGk5cG53ZG40CkpBa2xCdDRRcUg0NzBES05UK216TFFOT1gvanM3YkVXdnhLcTBDZjhNbFptN0V0QlRGS2VtdS9pRVJBT2duYVcKcGs0ZUZVNXdBQ1dVU1FObWgxR1p4ZEdCZjFXM1VjUnQxcFRvOEtQTDluZm4vSGJiRFNsQkNVL3VIcWd2TSt2cApmTE03bzRIVDZ1K1ZzU00rWGZqeDhpeE5ZRHdoalNuKzQyZm13d1d3ZzJISHUrdUozZ1pUSWQwRUI1VW9hdUNjCjZUTlVtcEJscjU5UGFmVkZRWUY1S3VxaHJXKzVQaWpHcHBZcXg4Ynl6aFpOQzkwZnl5V0NXcXg2eGFZVm5OdzgKNkJmUXM2a0NnWUVBeVlyRVg1NU1RTzJnWDY2TGwxaGJDMzNzWk1OZzloVG1SK1doSTFjNksvbFZ1TFoyL0RPdwpsYTZ6eHdBU204Z0ZyVUFYbUljV2h2b3FwWGVzNWZzOVZKeDlNT0ZVYVBrckRPQllnY1laMUR6VVNVOHc3SSttCnlyV3hRUkRNajhvSGpRbHVpM0s2MzZucm5RajhxOGkvQ2dranVPcHJGZnliMzVEMFlDdjVXZzBDZ1lFQXdhT3cKRWFhN0l1MjFGa08vbmFjdVhjSnBhNkVlUTNqZFNlNlRQaXZ6bVVXU0haeGJuUy9XSnJaRjQwSExzUWxOZHl0ZgpNTTBKZFU0VmMyR0NVc1pMYjdQSmJwdVRqRERSSHJXV1pCMnhiemF0K3A3N2RzNWlOcXFRcTZ6M0syUVh4Y3ZTCis5am5VZXpDU2Y0N1R1OWNTTW96V3hTMW82b1BPSFdHVFRvdHR5TUNnWUFQdWc1Y3o4TnZoWnR3Ry9TMG1LWnkKSFI5bk5YL0pkQlFNSkRVUXh1dTVKcm16c2psU3NNM2t3RDh6RmlSZGw1d3B5c2lNbEc0RGxsM2hqNWNrVXhpVQpFNm9KT0d3WHpPbTVGWUNTajl6UUhQY0x5V3d0NlgvQWJiRXBQS0JaMEJBS3gyT2k2ZzcvQ1FsanRhSFIzZFphCmVDQWJlOTlqVmRUcit5bTJuM2ZUdVFLQmdBMm5TZ25rbEx0Z3dXMEJkK2hZMm1jWUJ6RGttbXF0Z2dUdGdvcFcKdFFWd3AxM1pJWWlTeituSTNtS295QUVDbytpc01Ua1NyQUVPY1dyQ1RGc2p5anZsRkdYdEtGa3hNLzJUVmpoVwo4NlRnMlNHYnhpVlpaZ2x1dTJhdmVub2Z3NkZadnRXdE5KcE5OR0hkUURkUG4xVXVsTEp1WW1SWTRGdmR4WXQ2CmQ3QzdBb0dBRUsvalFiZ0l3OXFLQUNOZ0JySnB1cU5Ham9JajFoQTRlb29DMXp1bFEyZUpnZ2J5OTBpSDg2VzEKM0xyOVZMVFkyc2JKTzlqekZVR0lOL01BOEhYQTE1a2grZHRibkRsdFRFZGNnenBCRzhCQUZRQ3hQWnBGWHhtZgpDUmhXN1l6RW1IeWJ4R0toR3NOK2M3NUhKTHZFSWwrRTh6eitXRk9xT240dkJXU1ZwSnc9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==",' + references: + - https://www.rfc-editor.org/rfc/rfc7468 \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/planetscale.yml b/crates/kingfisher-rules/data/rules/planetscale.yml index d37aa89..8130fa6 100644 --- a/crates/kingfisher-rules/data/rules/planetscale.yml +++ b/crates/kingfisher-rules/data/rules/planetscale.yml @@ -12,7 +12,7 @@ rules: min_digits: 2 min_entropy: 4 examples: - - pscale_tkn_abcdefghijklmnopqrstuvwxyZ1234567890_ABCDEF + - pscale_tkn_abcdefghi12lmnopqrstuvwxyZ1234567890_ABCDEF validation: type: Http content: @@ -31,6 +31,8 @@ rules: - '"id":' - '"username":' url: https://api.planetscale.com/v1/user + references: + - https://planetscale.com/docs/api depends_on_rule: - rule_id: kingfisher.planetscale.2 variable: USERNAME @@ -51,5 +53,7 @@ rules: min_entropy: 3.5 visible: false examples: - - pscale_user = abcdefghijkl - - 'planetscale_id: hgtmrnzlv1t7' + - pscale_user = 0dm7fw8prpel + - 'planetscale_id: 0dm7fw8prpel' + references: + - https://planetscale.com/docs/api diff --git a/crates/kingfisher-rules/data/rules/postgres.yml b/crates/kingfisher-rules/data/rules/postgres.yml index 7ab0007..1fb850c 100644 --- a/crates/kingfisher-rules/data/rules/postgres.yml +++ b/crates/kingfisher-rules/data/rules/postgres.yml @@ -39,4 +39,6 @@ rules: - CONNECTION_URI="postgis://postgres:s2Tf2k@rLMy@google.com:5434/elephant" validation: type: Postgres - tls_mode: lax \ No newline at end of file + tls_mode: lax + references: + - https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/posthog.yml b/crates/kingfisher-rules/data/rules/posthog.yml index 961be77..28fc6f5 100644 --- a/crates/kingfisher-rules/data/rules/posthog.yml +++ b/crates/kingfisher-rules/data/rules/posthog.yml @@ -27,6 +27,8 @@ rules: negative: true - type: StatusMatch status: [200] + references: + - https://posthog.com/docs/api/overview#authentication - name: PostHog Personal API Key id: kingfisher.posthog.2 pattern: | @@ -52,4 +54,6 @@ rules: - type: WordMatch words: - "authentication_failed" - negative: true \ No newline at end of file + negative: true + references: + - https://posthog.com/docs/api/overview#authentication \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/prefect.yml b/crates/kingfisher-rules/data/rules/prefect.yml index beba175..7f1b6fd 100644 --- a/crates/kingfisher-rules/data/rules/prefect.yml +++ b/crates/kingfisher-rules/data/rules/prefect.yml @@ -14,7 +14,7 @@ rules: confidence: medium examples: - PREFECT_API_TOKEN=pnu_1234567890abcdef1234567890abcdef1234 - - '"prefectToken": "pnu_abcdefabcdefabcdefabcdefabcdefabcdef"' + - '"prefectToken": "pnu_abcdefabcdef12cdefabcdefabcdefabcdef"' references: - https://docs.prefect.io/latest/concepts/api_keys/ validation: diff --git a/crates/kingfisher-rules/data/rules/privkey.yml b/crates/kingfisher-rules/data/rules/privkey.yml index a936514..e6fa5e9 100644 --- a/crates/kingfisher-rules/data/rules/privkey.yml +++ b/crates/kingfisher-rules/data/rules/privkey.yml @@ -22,12 +22,10 @@ rules: PRIVATE\sKEY (\sBLOCK)? ----- - pattern_requirements: - min_digits: 2 min_entropy: 4.5 confidence: high examples: - - |- + - | -----BEGIN RSA PRIVATE KEY----- Proc-Type: 4,ENCRYPTED DEK-Info: AES-256-CBC,1F77CE6d2Bb6B18537633Ec3aD093b9C @@ -42,6 +40,8 @@ rules: +ril frnc129xvp11ndqbyjqlg3jf9ovlb1qula84ftj8m -----END RSA PRIVATE KEY----- + references: + - https://www.rfc-editor.org/rfc/rfc7468 - name: Contains Private Key id: kingfisher.privkey.2 @@ -106,3 +106,5 @@ rules: -----BEGIN ENCRYPTED PRIVATE KEY BLOCK----- V75NeIrlsI80Gf0aTS2RZQvEcUQ3n6XwFnOvB/O5rRv3HGqvptc3P3n0bxfEg5KA -----END ENCRYPTED PRIVATE KEY BLOCK----- + references: + - https://www.rfc-editor.org/rfc/rfc7468 diff --git a/crates/kingfisher-rules/data/rules/pubnub.yml b/crates/kingfisher-rules/data/rules/pubnub.yml index f759ed7..5397ca1 100644 --- a/crates/kingfisher-rules/data/rules/pubnub.yml +++ b/crates/kingfisher-rules/data/rules/pubnub.yml @@ -24,6 +24,8 @@ rules: - 200 type: StatusMatch url: "https://ps.pndsn.com/publish/{{ TOKEN }}/{{ SUBSCRIPTIONTOKEN }}/0/kingfisher/0/%22ping%22?uuid=kingfisher_validate" + references: + - https://www.pubnub.com/docs/sdks/rest-api depends_on_rule: - rule_id: "kingfisher.pubnub.2" variable: SUBSCRIPTIONTOKEN @@ -52,4 +54,6 @@ rules: - report_response: true - status: - 200 - type: StatusMatch \ No newline at end of file + type: StatusMatch + references: + - https://www.pubnub.com/docs/sdks/rest-api \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/pypi.yml b/crates/kingfisher-rules/data/rules/pypi.yml index 1b441f9..46a38ba 100644 --- a/crates/kingfisher-rules/data/rules/pypi.yml +++ b/crates/kingfisher-rules/data/rules/pypi.yml @@ -62,4 +62,7 @@ rules: - name: content type: file content: "path/to/my_package-0.0.1.tar.gz" - content_type: "application/octet-stream" \ No newline at end of file + content_type: "application/octet-stream" + references: + - https://pypi.org/help/#apitoken + - https://warehouse.pypa.io/api-reference/ \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/rabbitmq.yml b/crates/kingfisher-rules/data/rules/rabbitmq.yml index 607d5f6..293ec81 100644 --- a/crates/kingfisher-rules/data/rules/rabbitmq.yml +++ b/crates/kingfisher-rules/data/rules/rabbitmq.yml @@ -15,27 +15,10 @@ rules: @ [-.%\w\/:]+ \b - pattern_requirements: - min_special_chars: 1 min_entropy: 3.5 confidence: medium examples: - amqp://user:password@rabbitmq.example.com/queue - amqps://admin:3eCa3P@192.168.1.10:5671/vhost - # validation: - # type: Http - # content: - # request: - # url: '{{ URL }}' - # headers: - # Custom-header: '{{ TOKEN }}' - # method: GET - # response_matcher: - # - report_response: true - # - status: - # - 200 - # type: StatusMatch - # - report_response: true - # - type: WordMatch - # words: - # - '"connected":true' \ No newline at end of file + references: + - https://www.rabbitmq.com/uri-spec.html diff --git a/crates/kingfisher-rules/data/rules/recaptcha.yml b/crates/kingfisher-rules/data/rules/recaptcha.yml index 0c40e91..dcc7a16 100644 --- a/crates/kingfisher-rules/data/rules/recaptcha.yml +++ b/crates/kingfisher-rules/data/rules/recaptcha.yml @@ -10,7 +10,7 @@ rules: 6l[c-f][a-z0-9_-].{36} ) pattern_requirements: - min_digits: 3 + min_digits: 1 min_entropy: 3 confidence: medium examples: @@ -32,4 +32,6 @@ rules: type: WordMatch words: - '"success": true' - url: https://www.google.com/recaptcha/api/siteverify \ No newline at end of file + url: https://www.google.com/recaptcha/api/siteverify + references: + - https://developers.google.com/recaptcha/docs/verify \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/salesforce.yml b/crates/kingfisher-rules/data/rules/salesforce.yml index 282a936..3da3a6a 100644 --- a/crates/kingfisher-rules/data/rules/salesforce.yml +++ b/crates/kingfisher-rules/data/rules/salesforce.yml @@ -41,6 +41,8 @@ rules: words: ["DailyApiRequests"] match_all_words: true url: "https://{{ INSTANCE }}.my.salesforce.com/services/data/v60.0/limits" + references: + - https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/intro_authentication.htm depends_on_rule: - rule_id: "kingfisher.salesforce.2" variable: INSTANCE @@ -63,6 +65,8 @@ rules: examples: - https://example123.my.salesforce.com - mydomainname.my.salesforce.com + references: + - https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/intro_authentication.htm - name: Salesforce Consumer Key id: kingfisher.salesforce.3 pattern: | @@ -121,6 +125,8 @@ rules: true https://api.example.net/oauth/token + references: + - https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/intro_oauth_and_connected_apps.htm - name: Salesforce Consumer Secret id: kingfisher.salesforce.4 pattern: | @@ -178,6 +184,8 @@ rules: true https://api.example.net/oauth/token + references: + - https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/intro_oauth_and_connected_apps.htm - name: Salesforce Consumer Key and Secret id: kingfisher.salesforce.5 pattern: | @@ -244,4 +252,6 @@ rules: false true https://api.example.net/oauth/token - \ No newline at end of file + + references: + - https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/intro_oauth_and_connected_apps.htm \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/scalingo.yml b/crates/kingfisher-rules/data/rules/scalingo.yml index c297526..131d14b 100644 --- a/crates/kingfisher-rules/data/rules/scalingo.yml +++ b/crates/kingfisher-rules/data/rules/scalingo.yml @@ -13,10 +13,10 @@ rules: min_entropy: 3.0 confidence: medium examples: - - SCALINGO_TOKEN=tk-us-abcdefabcdefabcdefabcdefabcdefabcdefabcdefabcdef + - SCALINGO_TOKEN=tk-us-abcdefabcde12bcdefabcdefabcdefabcdefabcdefabcdef - '"scalingo": "tk-us-1234567890abcdef1234567890abcdef1234567890abcdef"' references: - - https://developers.scalingo.com/apps/api/authentication + - https://developers.scalingo.com/index#authentication validation: type: Http content: diff --git a/crates/kingfisher-rules/data/rules/scraperapi.yml b/crates/kingfisher-rules/data/rules/scraperapi.yml index 2bca1ac..d44adb9 100644 --- a/crates/kingfisher-rules/data/rules/scraperapi.yml +++ b/crates/kingfisher-rules/data/rules/scraperapi.yml @@ -11,7 +11,7 @@ rules: \b pattern_requirements: min_digits: 2 - min_lowercase: 10 + min_lowercase: 5 min_entropy: 3.5 confidence: medium examples: diff --git a/crates/kingfisher-rules/data/rules/sentry.yml b/crates/kingfisher-rules/data/rules/sentry.yml index d0f4685..9d02ba3 100644 --- a/crates/kingfisher-rules/data/rules/sentry.yml +++ b/crates/kingfisher-rules/data/rules/sentry.yml @@ -18,7 +18,7 @@ rules: min_entropy: 3.0 confidence: medium examples: - - SENTRY_TOKEN=cbadefcbadefcbadefcbadefcbadefcbadefcbadefcbadefcbadefcbadefcbad + - SENTRY_TOKEN=cbadefcbadefc12defcbadefcbadefcbadefcbadefcbadefcbadefcbadefcbad - '"sentry-key": "3214567890cbadef3214567890cbadef3214567890cbadef3214567890cbadef"' references: - https://docs.sentry.io/api/auth/ @@ -95,7 +95,7 @@ rules: min_entropy: 3.5 confidence: medium examples: - - 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 + - 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 - SNTRY_USER="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" references: - https://docs.sentry.io/api/auth/ diff --git a/crates/kingfisher-rules/data/rules/shippo.yml b/crates/kingfisher-rules/data/rules/shippo.yml index e1bae07..34d1bd8 100644 --- a/crates/kingfisher-rules/data/rules/shippo.yml +++ b/crates/kingfisher-rules/data/rules/shippo.yml @@ -14,7 +14,7 @@ rules: confidence: medium examples: - SHIPPO_TOKEN=shippo_test_1234567890abcdef1234567890abcdef12345678 - - 'Authorization: "ShippoToken shippo_live_abcdefabcdefabcdefabcdefabcdefabcdefabcd"' + - 'Authorization: "ShippoToken shippo_live_abc12fabcdefabcdefabcdefabcdefabcdefabcd"' references: - https://goshippo.com/docs/reference validation: diff --git a/crates/kingfisher-rules/data/rules/shopify.yml b/crates/kingfisher-rules/data/rules/shopify.yml index 7f70dd4..daaa6ed 100644 --- a/crates/kingfisher-rules/data/rules/shopify.yml +++ b/crates/kingfisher-rules/data/rules/shopify.yml @@ -29,6 +29,8 @@ rules: match_all_words: true words: ['"shop":'] url: https://{{ DOMAIN }}/admin/api/2024-10/shop.json + references: + - https://shopify.dev/docs/api/admin-rest#authentication depends_on_rule: - rule_id: "kingfisher.shopify.2" variable: DOMAIN @@ -39,4 +41,6 @@ rules: min_entropy: 3.0 visible: false examples: - - example.myshopify.com \ No newline at end of file + - example.myshopify.com + references: + - https://shopify.dev/docs/api/admin-rest#authentication \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/snyk.yml b/crates/kingfisher-rules/data/rules/snyk.yml index 3ba783f..d8af8be 100644 --- a/crates/kingfisher-rules/data/rules/snyk.yml +++ b/crates/kingfisher-rules/data/rules/snyk.yml @@ -33,4 +33,6 @@ rules: status: [200] - type: WordMatch words: ['"username"'] - match_all_words: true \ No newline at end of file + match_all_words: true + references: + - https://docs.snyk.io/snyk-api/authentication-for-api \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/sonarcloud.yml b/crates/kingfisher-rules/data/rules/sonarcloud.yml index c3307e9..cac46e6 100644 --- a/crates/kingfisher-rules/data/rules/sonarcloud.yml +++ b/crates/kingfisher-rules/data/rules/sonarcloud.yml @@ -35,3 +35,5 @@ rules: match_all_words: true words: - '"tokens":' + references: + - https://sonarcloud.io/web_api diff --git a/crates/kingfisher-rules/data/rules/sourcegraph.yml b/crates/kingfisher-rules/data/rules/sourcegraph.yml index 431e7c6..54b3f49 100644 --- a/crates/kingfisher-rules/data/rules/sourcegraph.yml +++ b/crates/kingfisher-rules/data/rules/sourcegraph.yml @@ -31,6 +31,8 @@ rules: - type: WordMatch words: ['"site":{'] match_all_words: true + references: + - https://docs.sourcegraph.com/admin/api#authentication - name: Sourcegraph _Legacy_ API Key id: kingfisher.sourcegraph.2 @@ -68,6 +70,8 @@ rules: status: [200] - type: WordMatch words: ['"site":{'] + references: + - https://docs.sourcegraph.com/admin/api#authentication - name: Sourcegraph Cody Gateway Key id: kingfisher.sourcegraph.3 @@ -95,3 +99,5 @@ rules: - type: WordMatch words: ['"token"', '"limit"'] match_all_words: true + references: + - https://docs.sourcegraph.com/cody/cody_gateway diff --git a/crates/kingfisher-rules/data/rules/square.yml b/crates/kingfisher-rules/data/rules/square.yml index 24a635a..8789530 100644 --- a/crates/kingfisher-rules/data/rules/square.yml +++ b/crates/kingfisher-rules/data/rules/square.yml @@ -33,6 +33,8 @@ rules: status: [200] - type: WordMatch words: ['"locations":'] + references: + - https://developer.squareup.com/ - name: Square Access Token id: kingfisher.square.2 @@ -65,6 +67,8 @@ rules: status: [200] - type: WordMatch words: ['"locations":'] + references: + - https://developer.squareup.com/ - name: Square OAuth Secret id: kingfisher.square.3 @@ -96,4 +100,6 @@ rules: - type: StatusMatch status: [200] - type: WordMatch - words: ['"locations":'] \ No newline at end of file + words: ['"locations":'] + references: + - https://developer.squareup.com/ \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/stripe.yml b/crates/kingfisher-rules/data/rules/stripe.yml index 9d48341..9d223ae 100644 --- a/crates/kingfisher-rules/data/rules/stripe.yml +++ b/crates/kingfisher-rules/data/rules/stripe.yml @@ -21,6 +21,8 @@ rules: categories: [api, key] examples: - stripe_pub_key = pk_live_HQS0j4H75XpthOW87eY1sXa2BYz3Ab + references: + - https://stripe.com/docs/api/authentication - name: Stripe Secret / Restricted Key id: kingfisher.stripe.2 @@ -61,3 +63,5 @@ rules: - type: WordMatch match_all_words: true words: ['"object":"account"'] + references: + - https://stripe.com/docs/api/authentication diff --git a/crates/kingfisher-rules/data/rules/supabase.yml b/crates/kingfisher-rules/data/rules/supabase.yml index 6232195..a2af991 100644 --- a/crates/kingfisher-rules/data/rules/supabase.yml +++ b/crates/kingfisher-rules/data/rules/supabase.yml @@ -36,7 +36,8 @@ rules: sb_secret_[a-z0-9_-]{31} ) pattern_requirements: - min_digits: 2 + min_uppercase: 3 + min_lowercase: 3 min_entropy: 4.0 confidence: medium validation: diff --git a/crates/kingfisher-rules/data/rules/tavily.yml b/crates/kingfisher-rules/data/rules/tavily.yml index 1775eb4..d6af170 100644 --- a/crates/kingfisher-rules/data/rules/tavily.yml +++ b/crates/kingfisher-rules/data/rules/tavily.yml @@ -18,7 +18,6 @@ rules: examples: - "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" - "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" - - 'TAVILY_API_KEY = "tvly-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"' validation: type: Http content: diff --git a/crates/kingfisher-rules/data/rules/telegram.yml b/crates/kingfisher-rules/data/rules/telegram.yml index 3f628a6..3c03c6a 100644 --- a/crates/kingfisher-rules/data/rules/telegram.yml +++ b/crates/kingfisher-rules/data/rules/telegram.yml @@ -31,4 +31,6 @@ rules: examples: - "tgram://110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsawd" - "telegram: 508627689:AAEuLPKs-EhrjrYGnz60bnYNZqakf6HJxc0" - - "telegram token is 3628091811:BAG9RuJiqgOGIfFbOPBpAo6QhIJoD9mCdDs" \ No newline at end of file + - "telegram token is 3628091811:BAG9RuJiqgOGIfFbOPBpAo6QhIJoD9mCdDs" + references: + - https://core.telegram.org/bots/api#authorizing-your-bot \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/travisci.yml b/crates/kingfisher-rules/data/rules/travisci.yml index 1d4b188..4eb8c45 100644 --- a/crates/kingfisher-rules/data/rules/travisci.yml +++ b/crates/kingfisher-rules/data/rules/travisci.yml @@ -33,6 +33,8 @@ rules: - report_response: true - type: StatusMatch status: [200] + references: + - https://developer.travis-ci.com/authentication - name: Travis CI Encrypted Variable id: kingfisher.travisci.2 pattern: | @@ -51,4 +53,6 @@ rules: global: # This sets FOO=super-secret, but the plaintext never appears here. - secure: "VJh0l9gOb+6AVNDk6cziZSs1AqVM8CqtZU6ot9ZQeJ+KfL1pxnGQ4qQF8Cz9\M1q85c3l1N1+qkQ0uV12QG6O6ylq6Qq1l3VjAJM3h2pY3jdmrA8kX2ZIxRjC/\8+Xj1wVtKQ0R+owM/6i5Y6cyx4hRb3VvSeYlC0lD1iTzQ2vgMyE=" + references: + - https://docs.travis-ci.com/user/encryption-keys/ diff --git a/crates/kingfisher-rules/data/rules/twilio.yml b/crates/kingfisher-rules/data/rules/twilio.yml index b626499..219bde1 100644 --- a/crates/kingfisher-rules/data/rules/twilio.yml +++ b/crates/kingfisher-rules/data/rules/twilio.yml @@ -21,6 +21,8 @@ rules: // https://www.twilio.com/console/video/dev-tools/api-keys 'API' => env('TWILIO_API','SK6e84981d07ace5c9df33e1ab043a2fb2'), 'API_KEY' => env('TWILIO_API_KEY', 'wbTs1SUt6Aace5eKeNCxuYvJa6PhaRd0') + references: + - https://www.twilio.com/docs/iam/api - name: Twilio API Key id: kingfisher.twilio.2 pattern: | @@ -58,6 +60,8 @@ rules: - '"first_page_uri":' - '"accounts":' url: https://api.twilio.com/2010-04-01/Accounts.json + references: + - https://www.twilio.com/docs/usage/api#authentication depends_on_rule: - rule_id: "kingfisher.twilio.1" variable: TWILIOID diff --git a/crates/kingfisher-rules/data/rules/twitch.yml b/crates/kingfisher-rules/data/rules/twitch.yml index d9ecc55..9f7cf50 100644 --- a/crates/kingfisher-rules/data/rules/twitch.yml +++ b/crates/kingfisher-rules/data/rules/twitch.yml @@ -17,8 +17,8 @@ rules: min_entropy: 3.5 confidence: medium examples: - - TWITCH_TOKEN=abcdefghijklmnopqrstuvwx123456 - - "twitch_api_token: '0123456789abcdefghijklmnopqrst'" + - TWITCH_TOKEN=abCDefghijklmnopqrstuvwx123456 + - "twitch_api_token: '0123456789ABcdefghijklmnopqrst'" references: - https://dev.twitch.tv/docs/authentication/validate-tokens/ validation: diff --git a/crates/kingfisher-rules/data/rules/uri.yml b/crates/kingfisher-rules/data/rules/uri.yml index e3e3a7d..a214645 100644 --- a/crates/kingfisher-rules/data/rules/uri.yml +++ b/crates/kingfisher-rules/data/rules/uri.yml @@ -40,3 +40,5 @@ rules: type: StatusMatch status: - 200 + references: + - https://www.rfc-editor.org/rfc/rfc3986#section-3.2.1 diff --git a/crates/kingfisher-rules/data/rules/vercel.yml b/crates/kingfisher-rules/data/rules/vercel.yml index 3ace650..97646a8 100644 --- a/crates/kingfisher-rules/data/rules/vercel.yml +++ b/crates/kingfisher-rules/data/rules/vercel.yml @@ -12,7 +12,7 @@ rules: ) \b pattern_requirements: - min_digits: 6 + min_digits: 3 min_uppercase: 1 min_lowercase: 1 confidence: medium diff --git a/crates/kingfisher-rules/data/rules/voyageai.yml b/crates/kingfisher-rules/data/rules/voyageai.yml index 942ab40..ce2a0a8 100644 --- a/crates/kingfisher-rules/data/rules/voyageai.yml +++ b/crates/kingfisher-rules/data/rules/voyageai.yml @@ -23,4 +23,6 @@ rules: Authorization: "Bearer {{ TOKEN }}" response_matcher: - type: StatusMatch - status: [200] \ No newline at end of file + status: [200] + references: + - https://docs.voyageai.com/reference \ No newline at end of file diff --git a/crates/kingfisher-rules/data/rules/weightsandbiases.yml b/crates/kingfisher-rules/data/rules/weightsandbiases.yml index 88bb37b..b60f491 100644 --- a/crates/kingfisher-rules/data/rules/weightsandbiases.yml +++ b/crates/kingfisher-rules/data/rules/weightsandbiases.yml @@ -35,3 +35,5 @@ rules: - type: WordMatch words: - '"username"' + references: + - https://docs.wandb.ai/ref/cli/wandb-login diff --git a/crates/kingfisher-rules/data/rules/youtube.yml b/crates/kingfisher-rules/data/rules/youtube.yml index 96b82fb..ebdb851 100644 --- a/crates/kingfisher-rules/data/rules/youtube.yml +++ b/crates/kingfisher-rules/data/rules/youtube.yml @@ -29,3 +29,5 @@ rules: match_all_words: false negative: true - type: JsonValid + references: + - https://developers.google.com/youtube/v3/getting-started#before-you-start diff --git a/crates/kingfisher-rules/data/rules/zhipu.yml b/crates/kingfisher-rules/data/rules/zhipu.yml index 3d7ddab..19a86e6 100644 --- a/crates/kingfisher-rules/data/rules/zhipu.yml +++ b/crates/kingfisher-rules/data/rules/zhipu.yml @@ -34,3 +34,5 @@ rules: status: [200] - type: WordMatch words: ["object", "data"] + references: + - https://open.bigmodel.cn/dev/api diff --git a/crates/kingfisher-rules/src/lib.rs b/crates/kingfisher-rules/src/lib.rs index cd6ecb9..b464638 100644 --- a/crates/kingfisher-rules/src/lib.rs +++ b/crates/kingfisher-rules/src/lib.rs @@ -16,11 +16,11 @@ pub mod rules_database; // Re-export rule types pub use rule::{ - ChecksumActual, ChecksumRequirement, Confidence, DependsOnRule, HttpMultiStepRevocation, - HttpRequest, HttpValidation, MultipartConfig, MultipartPart, PatternRequirementContext, - PatternRequirements, PatternValidationResult, ReportResponseData, ResponseExtractor, - ResponseMatcher, Revocation, RevocationStep, Rule, RuleSyntax, TlsMode, Validation, - RULE_COMMENTS_PATTERN, + ChecksumActual, ChecksumRequirement, Confidence, DependsOnRule, GrpcRequest, GrpcValidation, + HttpMultiStepRevocation, HttpRequest, HttpValidation, MultipartConfig, MultipartPart, + PatternRequirementContext, PatternRequirements, PatternValidationResult, ReportResponseData, + ResponseExtractor, ResponseMatcher, Revocation, RevocationStep, Rule, RuleSyntax, TlsMode, + Validation, RULE_COMMENTS_PATTERN, }; // Re-export Rules collection diff --git a/crates/kingfisher-rules/src/rule.rs b/crates/kingfisher-rules/src/rule.rs index 1d22899..07eec09 100644 --- a/crates/kingfisher-rules/src/rule.rs +++ b/crates/kingfisher-rules/src/rule.rs @@ -79,6 +79,7 @@ pub enum Validation { JWT, Raw(String), Http(HttpValidation), + Grpc(GrpcValidation), } /// Represents revocation actions that a rule can perform. @@ -387,6 +388,33 @@ pub struct HttpValidation { pub multipart: Option, } +/// Configuration for gRPC validation. +/// +/// This is intended for services that are gRPC-only (HTTP/2 + trailers), +/// such as Modal's control plane. +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash, Clone)] +pub struct GrpcValidation { + pub request: GrpcRequest, +} + +/// Configuration for a single gRPC unary request. +/// +/// Notes: +/// - `url` should include the full gRPC method path, e.g. +/// `https://api.modal.com/modal.client.ModalClientModal/ContainerHello` +/// - `headers` can include custom auth headers, content-type, etc. +/// - `body` is sent as raw bytes; YAML `\u0000` escapes are supported. +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash, Clone)] +pub struct GrpcRequest { + pub url: String, + #[serde(default)] + pub headers: BTreeMap, + #[serde(default)] + pub body: Option, + #[serde(default)] + pub response_matcher: Option>, +} + /// Configuration for an HTTP request used for validation. #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash, Clone)] pub struct HttpRequest { diff --git a/data/default/rule_cleanup/main.py b/data/default/rule_cleanup/check_endpoints.py similarity index 100% rename from data/default/rule_cleanup/main.py rename to data/default/rule_cleanup/check_endpoints.py diff --git a/data/default/rule_cleanup/check_references.py b/data/default/rule_cleanup/check_references.py new file mode 100644 index 0000000..c044beb --- /dev/null +++ b/data/default/rule_cleanup/check_references.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Check URLs under YAML `references` sections for activity.""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import pathlib +import re +import sys +import urllib.error +import urllib.request +from dataclasses import dataclass + +URL_RE = re.compile(r"https?://[^\s\"'<>]+") +REFERENCES_RE = re.compile(r"^(\s*)references:\s*$") + + +@dataclass(frozen=True) +class ReferenceUrl: + path: pathlib.Path + line: int + url: str + + +@dataclass(frozen=True) +class UrlResult: + ref: ReferenceUrl + active: bool + detail: str + + +def extract_reference_urls(path: pathlib.Path) -> list[ReferenceUrl]: + lines = path.read_text(encoding="utf-8").splitlines() + refs: list[ReferenceUrl] = [] + i = 0 + while i < len(lines): + line = lines[i] + match = REFERENCES_RE.match(line) + if not match: + i += 1 + continue + + base_indent = len(match.group(1)) + i += 1 + while i < len(lines): + current = lines[i] + stripped = current.strip() + indent = len(current) - len(current.lstrip(" ")) + + if stripped and indent <= base_indent: + break + + if stripped: + for url in URL_RE.findall(current): + refs.append(ReferenceUrl(path=path, line=i + 1, url=url.rstrip(",.)]"))) + + i += 1 + + return refs + + +def check_url(url: str, timeout: float) -> tuple[bool, str]: + headers = {"User-Agent": "kingfisher-reference-checker/1.0"} + request = urllib.request.Request(url, headers=headers, method="HEAD") + head_detail = "" + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + status = getattr(response, "status", 200) + return (200 <= status < 400), f"HTTP {status} (HEAD)" + except urllib.error.HTTPError as exc: + # Many docs hosts block HEAD. Retry with GET. + if exc.code in {401, 403, 405, 429}: + return True, f"HTTP {exc.code} (HEAD)" + head_detail = f"HTTP {exc.code} (HEAD)" + except Exception as exc: # noqa: BLE001 + # Retry with GET for transient/protocol issues. + head_detail = f"{type(exc).__name__}: {exc} (HEAD)" + + get_request = urllib.request.Request(url, headers=headers, method="GET") + try: + with urllib.request.urlopen(get_request, timeout=timeout) as response: + status = getattr(response, "status", 200) + return (200 <= status < 400), f"HTTP {status} (GET)" + except urllib.error.HTTPError as exc: + if exc.code in {401, 403, 429}: + return True, f"HTTP {exc.code} (GET)" + if head_detail: + return False, f"{head_detail}; HTTP {exc.code} (GET)" + return False, f"HTTP {exc.code} (GET)" + except Exception as exc: # noqa: BLE001 + if head_detail: + return False, f"{head_detail}; {type(exc).__name__}: {exc} (GET)" + return False, f"{type(exc).__name__}: {exc} (GET)" + + +def check_reference(ref: ReferenceUrl, timeout: float) -> UrlResult: + active, detail = check_url(ref.url, timeout=timeout) + return UrlResult(ref=ref, active=active, detail=detail) + + +def gather_references(base_dir: pathlib.Path) -> list[ReferenceUrl]: + refs: list[ReferenceUrl] = [] + for path in sorted(base_dir.glob("*.yml")): + refs.extend(extract_reference_urls(path)) + return refs + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Check all URLs in YAML references sections." + ) + parser.add_argument( + "--rules-dir", + type=pathlib.Path, + default=pathlib.Path("../../crates/kingfisher-rules/data/rules"), + help="Directory with YAML rule files (default: %(default)s)", + ) + parser.add_argument( + "--timeout", + type=float, + default=15.0, + help="HTTP request timeout in seconds (default: %(default)s)", + ) + parser.add_argument( + "--workers", + type=int, + default=20, + help="Maximum concurrent URL checks (default: %(default)s)", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + rules_dir = args.rules_dir.resolve() + if not rules_dir.exists(): + print(f"error: directory does not exist: {rules_dir}", file=sys.stderr) + return 2 + + refs = gather_references(rules_dir) + if not refs: + print("No URLs found in references sections.") + return 0 + + print(f"Found {len(refs)} reference URLs in {rules_dir}") + results: list[UrlResult] = [] + + with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, args.workers)) as pool: + futures = [pool.submit(check_reference, ref, args.timeout) for ref in refs] + for future in concurrent.futures.as_completed(futures): + results.append(future.result()) + + inactive = [result for result in results if not result.active] + inactive.sort(key=lambda item: (str(item.ref.path), item.ref.line, item.ref.url)) + + print(f"Active: {len(results) - len(inactive)}") + print(f"Inactive: {len(inactive)}") + + for result in inactive: + rel = result.ref.path.relative_to(pathlib.Path.cwd()) + print(f"INACTIVE {rel}:{result.ref.line} {result.ref.url} [{result.detail}]") + + return 1 if inactive else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/RULES.md b/docs/RULES.md index 072a500..c806aa2 100644 --- a/docs/RULES.md +++ b/docs/RULES.md @@ -71,6 +71,24 @@ rules: expected: ["application/json"] - type: JsonValid + # NOTE: Some providers are gRPC-only (no REST endpoint). For those, use Grpc validation. + validation: + type: Grpc + content: + request: + url: https://api.example.com/./ + headers: + content-type: application/grpc + te: trailers + Authorization: "Bearer {{ TOKEN }}" + # Raw bytes are allowed (YAML \\u0000 escapes become NUL bytes). + body: "\\u0000\\u0000\\u0000\\u0000\\u0000" + response_matcher: + - report_response: true + - type: HeaderMatch + header: grpc-status + expected: ["0"] + revocation: # (optional) revoke a secret type: Http content: @@ -153,6 +171,36 @@ revocation: | validation | Configure HTTP, AWS, GCP, etc. checks to verify live validity | | revocation | Configure HTTP, AWS, or multi-step revocation for a detected secret | +## gRPC Validation (Grpc) + +Some services (notably CLI/SDK control planes) are **gRPC-only**. For these, `validation: type: Http` +is not sufficient because gRPC status is typically returned via HTTP/2 trailers (`grpc-status`, +`grpc-message`). Kingfisher’s `Grpc` validator performs an HTTP/2 request and evaluates matchers +against the merged headers+trailers. + +`Grpc` is currently intended for unary requests and expects you to provide a fully-qualified method URL: + +```yaml +validation: + type: Grpc + content: + request: + url: https://api.modal.com/modal.client.ModalClient/ClientHello + headers: + content-type: application/grpc + te: trailers + x-modal-token-id: "{{ TOKEN_ID }}" + x-modal-token-secret: "{{ TOKEN }}" + x-modal-client-type: "1" + x-modal-client-version: "1.0.0" + body: "\u0000\u0000\u0000\u0000\u0000" # Empty protobuf frame + response_matcher: + - report_response: true + - type: HeaderMatch + header: grpc-status + expected: ["0"] +``` + *responser_matcher* variants. Multiple can be used | Variant | Required keys | Behavior | diff --git a/docs/USAGE.md b/docs/USAGE.md index 1fd3970..8622331 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -150,7 +150,7 @@ kingfisher validate --rule jwt \ "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9..." ``` -**Supported validators:** HTTP, AWS, GCP, MongoDB, MySQL, Postgres, JDBC, JWT, Azure Storage, and Coinbase. +**Supported validators:** HTTP, Grpc, AWS, GCP, MongoDB, MySQL, Postgres, JDBC, JWT, Azure Storage, and Coinbase. **Exit codes:** Returns `0` if any matching rule validates the secret as valid, `1` if all are invalid or an error occurred. diff --git a/src/direct_validate.rs b/src/direct_validate.rs index f32bbfb..d05640d 100644 --- a/src/direct_validate.rs +++ b/src/direct_validate.rs @@ -29,7 +29,8 @@ use crate::{ azure::validate_azure_storage_credentials, coinbase::validate_cdp_api_key, gcp::GcpValidator, - httpvalidation::{build_request_builder, retry_request, validate_response}, + httpvalidation::validate_response, + httpvalidation::{build_request_builder, retry_request}, jdbc::validate_jdbc, jwt::validate_jwt, mongodb::validate_mongodb, @@ -40,6 +41,8 @@ use crate::{ validation_body, }; +use crate::grpc_validation; + /// Result of a direct validation attempt. #[derive(Debug, Clone, Serialize)] pub struct DirectValidationResult { @@ -135,6 +138,21 @@ fn extract_validation_vars(validation: &Validation) -> BTreeSet { vars.extend(extract_template_vars(body)); } } + Validation::Grpc(grpc) => { + // Extract from URL + vars.extend(extract_template_vars(&grpc.request.url)); + + // Extract from headers + for (key, value) in &grpc.request.headers { + vars.extend(extract_template_vars(key)); + vars.extend(extract_template_vars(value)); + } + + // Extract from body + if let Some(body) = &grpc.request.body { + vars.extend(extract_template_vars(body)); + } + } // Non-HTTP validators typically use fixed variable names Validation::AWS => { vars.insert("AKID".to_string()); @@ -312,6 +330,60 @@ async fn execute_http_validation( }) } +/// Execute gRPC validation against the provided rule. +async fn execute_grpc_validation( + grpc_validation_cfg: &kingfisher_rules::GrpcValidation, + globals: &Object, + parser: &liquid::Parser, + timeout: Duration, +) -> Result { + // Render the URL + let url = render_and_parse_url(parser, globals, &grpc_validation_cfg.request.url).await?; + + debug!("Validating against gRPC URL: {}", url); + + let res = grpc_validation::grpc_unary_call_from_rule( + &url, + &grpc_validation_cfg.request.headers, + &grpc_validation_cfg.request.body, + parser, + globals, + timeout, + ) + .await + .map_err(|e| anyhow!("gRPC request failed: {e}"))?; + + let status = res.http_status; + let headers = res.headers; + let mut body = String::from_utf8_lossy(&res.body_bytes).to_string(); + let grpc_status = + headers.get("grpc-status").and_then(|v| v.to_str().ok()).unwrap_or("").to_string(); + let grpc_message = + headers.get("grpc-message").and_then(|v| v.to_str().ok()).unwrap_or("").to_string(); + if grpc_status == "0" { + body = "grpc-status=0".to_string(); + } else if body.trim().is_empty() && (!grpc_status.is_empty() || !grpc_message.is_empty()) { + body = format!("grpc-status={grpc_status} grpc-message={grpc_message}"); + } else if body.as_bytes().contains(&0) { + body = format!("grpc-status={grpc_status} grpc-message={grpc_message}"); + } + + // Truncate body for display if too long + let display_body = if body.len() > 500 { format!("{}...", &body[..500]) } else { body.clone() }; + + // Validate the response + let matchers = grpc_validation_cfg.request.response_matcher.as_deref().unwrap_or(&[]); + let is_valid = validate_response(matchers, &body, &status, &headers, false); + + Ok(DirectValidationResult { + rule_id: String::new(), // Will be filled in by caller + rule_name: String::new(), + is_valid, + status_code: Some(status.as_u16()), + message: display_body, + }) +} + /// Run direct validation of a secret against one or more rules. /// /// If the rule selector matches multiple rules, all matching rules are tried. @@ -476,6 +548,9 @@ pub async fn run_direct_validation( ) .await? } + Validation::Grpc(grpc_validation_cfg) => { + execute_grpc_validation(grpc_validation_cfg, &globals, &parser, timeout).await? + } Validation::AWS => { // AWS needs AKID and TOKEN (secret access key) diff --git a/src/grpc_validation.rs b/src/grpc_validation.rs new file mode 100644 index 0000000..8d20b2f --- /dev/null +++ b/src/grpc_validation.rs @@ -0,0 +1,202 @@ +use std::{collections::BTreeMap, sync::Arc, time::Duration}; + +use anyhow::{anyhow, Context, Result}; +use bytes::Bytes; +use h2::client; +use http::{header::HeaderName, HeaderMap, HeaderValue, Request, Uri}; +use liquid::Object; +use reqwest::Url; +use rustls::{ClientConfig, RootCertStore}; +use tokio::net::TcpStream; +use tokio_rustls::TlsConnector; + +/// Result of a gRPC unary call over HTTP/2. +pub struct GrpcCallResult { + pub http_status: http::StatusCode, + /// Response headers + trailers merged into one map. + pub headers: HeaderMap, + pub body_bytes: Vec, +} + +fn build_root_store() -> Result { + let mut roots = RootCertStore::empty(); + let native = rustls_native_certs::load_native_certs(); + if !native.errors.is_empty() { + // Best-effort: still proceed if we got any certs. + // (Some platforms may have a few unparsable roots.) + } + for cert in native.certs { + roots.add(cert).map_err(|e| anyhow!("Failed to add native root cert: {e:?}"))?; + } + Ok(roots) +} + +fn url_to_h2_uri(url: &Url) -> Result { + let scheme = url.scheme(); + if scheme != "https" { + return Err(anyhow!("gRPC validation only supports https URLs, got: {scheme}")); + } + let host = url.host_str().ok_or_else(|| anyhow!("URL is missing host: {url}"))?; + let authority = match url.port() { + Some(p) => format!("{host}:{p}"), + None => host.to_string(), + }; + let path_and_query = &url[url::Position::BeforePath..]; + Uri::builder() + .scheme("https") + .authority(authority) + .path_and_query(path_and_query) + .build() + .context("Failed to build HTTP/2 URI for gRPC request") +} + +fn header_map_from_templates( + templates: &BTreeMap, + parser: &liquid::Parser, + globals: &Object, +) -> Result { + let mut out = HeaderMap::new(); + for (k, v_template) in templates { + // Header names in YAML are expected to be static. + let name = HeaderName::from_bytes(k.as_bytes()) + .with_context(|| format!("Invalid header name in GrpcValidation: '{k}'"))?; + + let tmpl = parser + .parse(v_template) + .map_err(|e| anyhow!("Failed to parse header template '{k}': {e}"))?; + let rendered = tmpl + .render(globals) + .map_err(|e| anyhow!("Failed to render header template '{k}': {e}"))?; + + let value = HeaderValue::from_str(&rendered) + .with_context(|| format!("Invalid header value for '{k}'"))?; + out.append(name, value); + } + Ok(out) +} + +/// Execute a single unary gRPC request over HTTP/2 and return headers + trailers. +/// +/// This is intentionally low-level so that rules can validate gRPC-only APIs +/// without pretending they are REST endpoints. +pub async fn grpc_unary_call( + url: &Url, + headers: HeaderMap, + body: Vec, + timeout: Duration, +) -> Result { + let host = url.host_str().ok_or_else(|| anyhow!("URL is missing host: {url}"))?; + let port = url.port_or_known_default().unwrap_or(443); + + let addr = format!("{host}:{port}"); + let tcp = tokio::time::timeout(timeout, TcpStream::connect(addr)) + .await + .context("Timed out connecting to gRPC host")? + .context("Failed to connect to gRPC host")?; + + let mut tls_config = + ClientConfig::builder().with_root_certificates(build_root_store()?).with_no_client_auth(); + tls_config.alpn_protocols = vec![b"h2".to_vec()]; + + let connector = TlsConnector::from(Arc::new(tls_config)); + let server_name = rustls::pki_types::ServerName::try_from(host.to_string()) + .map_err(|_| anyhow!("Invalid TLS server name: {host}"))?; + + let tls = tokio::time::timeout(timeout, connector.connect(server_name, tcp)) + .await + .context("Timed out during TLS handshake")? + .context("TLS handshake failed")?; + + let (mut h2_client, connection) = tokio::time::timeout(timeout, client::handshake(tls)) + .await + .context("Timed out during HTTP/2 handshake")? + .context("HTTP/2 handshake failed")?; + + // Drive the HTTP/2 connection in the background. + tokio::spawn(async move { + let _ = connection.await; + }); + + let uri = url_to_h2_uri(url)?; + + let mut req_builder = Request::builder().method("POST").uri(uri); + { + let hdrs = req_builder.headers_mut().expect("headers_mut should exist"); + for (k, v) in headers.iter() { + hdrs.append(k, v.clone()); + } + } + + let request = req_builder.body(()).context("Failed to build HTTP/2 request")?; + + let (response_future, mut send_stream) = + h2_client.send_request(request, false).context("Failed to send gRPC request headers")?; + + // Send gRPC request bytes (including the 5-byte gRPC frame prefix). + send_stream.send_data(Bytes::from(body), true).context("Failed to send gRPC request body")?; + + let response = tokio::time::timeout(timeout, response_future) + .await + .context("Timed out waiting for gRPC response headers")? + .context("Failed to receive gRPC response headers")?; + + let http_status = response.status(); + let (parts, mut recv_stream) = response.into_parts(); + let mut merged_headers = parts.headers; + + // Read data frames (may be empty). + let mut body_bytes: Vec = Vec::new(); + loop { + // h2 returns `Option>` here: + // - None => end of stream + // - Some(Ok(bytes)) => a data chunk + // - Some(Err(err)) => stream error + let next_opt = tokio::time::timeout(timeout, recv_stream.data()) + .await + .context("Timed out reading gRPC response data")?; + + match next_opt { + Some(Ok(b)) => body_bytes.extend_from_slice(b.as_ref()), + Some(Err(e)) => return Err(anyhow!("Error reading gRPC response data: {e}")), + None => break, + } + } + + // Read trailers (where grpc-status is typically reported). + if let Some(trailers) = tokio::time::timeout(timeout, recv_stream.trailers()) + .await + .context("Timed out reading gRPC response trailers")? + .context("Error reading gRPC response trailers")? + { + for (k, v) in trailers.iter() { + merged_headers.append(k, v.clone()); + } + } + + Ok(GrpcCallResult { http_status, headers: merged_headers, body_bytes }) +} + +/// Helper to render & execute a gRPC request from rule templates. +pub async fn grpc_unary_call_from_rule( + url: &Url, + header_templates: &BTreeMap, + body_template: &Option, + parser: &liquid::Parser, + globals: &Object, + timeout: Duration, +) -> Result { + let headers = header_map_from_templates(header_templates, parser, globals)?; + let body = match body_template { + Some(t) => { + let tmpl = + parser.parse(t).map_err(|e| anyhow!("Failed to parse gRPC body template: {e}"))?; + let rendered = tmpl + .render(globals) + .map_err(|e| anyhow!("Failed to render gRPC body template: {e}"))?; + rendered.into_bytes() + } + None => Vec::new(), + }; + + grpc_unary_call(url, headers, body, timeout).await +} diff --git a/src/lib.rs b/src/lib.rs index 3cab0ea..5c2e716 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,6 +30,7 @@ pub mod git_url; pub mod gitea; pub mod github; pub mod gitlab; +pub mod grpc_validation; pub mod huggingface; pub mod inline_ignore; pub mod jira; diff --git a/src/main.rs b/src/main.rs index 04c1dd3..08f6580 100644 --- a/src/main.rs +++ b/src/main.rs @@ -80,6 +80,9 @@ use crate::cli::commands::{ fn main() -> anyhow::Result<()> { color_backtrace::install(); + // Rustls 0.23 requires an explicit crypto provider selection when multiple + // providers are present in the dependency graph. + let _ = rustls::crypto::ring::default_provider().install_default(); // Parse command-line arguments let CommandLineArgs { command, global_args } = CommandLineArgs::parse_args(); @@ -583,55 +586,112 @@ pub fn run_rules_check(args: &RulesCheckArgs) -> Result<()> { num_errors += 1; continue; } - // Test each example against both vectorscan and regex + // Test each example against regex and pattern_requirements for (example_index, example) in rule_syntax.examples.iter().enumerate() { - // Create a test blob from the example - // let blob = Blob::new(BlobId::new(example.as_bytes()), - // example.as_bytes().to_vec()); let origin = OriginSet::new( - // Origin::from_file(PathBuf::from("test_example")), - // Vec::new(), - // ); - // // Check vectorscan match - // let vectorscan_matched = match matcher.scan_blob(&blob, &origin, None)? { - // ScanResult::New(matches) => !matches.is_empty(), - // _ => false, - // }; - // Check regex match // Get the regex using the public method let re = rules_db.get_regex_by_rule_id(rule.id()).expect("Failed to get regex for rule"); - let regex_matched = re.is_match(example.as_bytes()); + + // Check if the example matches the pattern + let example_bytes = example.as_bytes(); + let regex_matched = re.is_match(example_bytes); + if !regex_matched { - // ||!vectorscan_matched { println!("\nTesting rule {} - {}", rule_index + 1, rule_syntax.name); println!(" Processing example {}", example_index + 1); - println!(" [!] Mismatch detected for example: {}", example); - // if !vectorscan_matched { - // println!(" Vectorscan match: {}", vectorscan_matched); - // num_errors += 1; - // } - if !regex_matched { - println!(" Regex match: {}", regex_matched); - num_errors += 1; - } + println!(" [!] Pattern mismatch detected for example: {}", example); + println!(" Regex match: {}", regex_matched); + num_errors += 1; + continue; } - // // Report any mismatches - // if !vectorscan_matched || !regex_matched { - // error!("Rule '{}' example {} failed validation:", - // rule.name(), example_index + 1); println!(" - // Example text: {}", example); + // If the rule has pattern_requirements, validate them against the match + if let Some(pattern_reqs) = rule.pattern_requirements() { + // Get the captures from the match + if let Some(captures) = re.captures(example_bytes) { + // Get the full match (group 0) + let full_capture = captures.get(0).expect("Group 0 should always exist"); + let full_bytes = full_capture.as_bytes(); - // if !vectorscan_matched { - // error!(" - Vectorscan pattern did not match example"); - // num_errors += 1; - // } + // Determine which bytes to validate (same logic as in matcher.rs) + // Find the primary capture group for validation + let matching_input_for_validation = 'block: { + // 1. Look for a named capture "secret" (case-insensitive). + if let Some(secret_cap) = + captures.name("secret").or_else(|| captures.name("SECRET")) + { + break 'block secret_cap; + } - // if !regex_matched { - // error!(" - Regex pattern did not match example"); - // num_errors += 1; - // } - // } + // 2. Look for any other named capture. + if let Some(named_cap) = (1..captures.len()).find_map(|i| { + let name_opt = re.capture_names().nth(i).and_then(|n| n); + name_opt.and_then(|_| captures.get(i)) + }) { + break 'block named_cap; + } + + // 3. Fall back to first positional capture (group 1) if it exists. + if let Some(pos_cap) = captures.get(1) { + break 'block pos_cap; + } + + // 4. Finally, fall back to the full match (group 0). + break 'block full_capture; + }; + + let validation_bytes = matching_input_for_validation.as_bytes(); + + // Create context for pattern requirements validation + use kingfisher_rules::PatternRequirementContext; + let context = PatternRequirementContext { + regex: re, + captures: &captures, + full_match: full_bytes, + }; + + // Validate pattern requirements (without respect_ignore_if_contains for examples) + use kingfisher_rules::PatternValidationResult; + match pattern_reqs.validate(validation_bytes, Some(context), false) { + PatternValidationResult::Passed => { + // All requirements met + } + PatternValidationResult::Failed => { + println!("\nTesting rule {} - {}", rule_index + 1, rule_syntax.name); + println!(" Processing example {}", example_index + 1); + println!( + " [!] Pattern requirements not met for example: {}", + example + ); + println!(" The match does not satisfy the character requirements (min_digits, min_uppercase, etc.)"); + num_errors += 1; + } + PatternValidationResult::FailedChecksum { actual_len, expected_len } => { + println!("\nTesting rule {} - {}", rule_index + 1, rule_syntax.name); + println!(" Processing example {}", example_index + 1); + println!(" [!] Checksum validation failed for example: {}", example); + println!( + " Actual checksum length: {}, Expected checksum length: {}", + actual_len, expected_len + ); + num_errors += 1; + } + PatternValidationResult::IgnoredBySubstring { matched_term } => { + // For examples, we don't want to treat this as an error in check mode + // since ignore_if_contains is meant for runtime filtering + // But we can warn about it + println!("\nTesting rule {} - {}", rule_index + 1, rule_syntax.name); + println!(" Processing example {}", example_index + 1); + println!( + " [!] Example would be ignored due to containing term: {}", + matched_term + ); + println!(" Example: {}", example); + num_errors += 1; + } + } + } + } } } // Print summary diff --git a/src/reporter.rs b/src/reporter.rs index 0c0b56b..77b3709 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -181,6 +181,15 @@ fn build_validate_command( escape_for_shell(snippet) )) } + Validation::Grpc(_) => { + // gRPC-based validation with dependent variables + Some(format!( + "kingfisher validate --rule {} {}{}", + rule_id, + var_args, + escape_for_shell(snippet) + )) + } Validation::MongoDB | Validation::MySQL | Validation::Postgres @@ -645,8 +654,17 @@ impl DetailsReporter { let source_span = rm.m.location.resolved_source_span(); let line_num = source_span.start.line; - // Get raw snippet value (for revoke command) and display snippet (for output) - let (raw_snippet, snippet) = if let Some(capture) = rm.m.groups.captures.get(0) { + // Prefer the named TOKEN capture (when present) for display + validate/revoke commands. + // This avoids cases like Modal CLI pairs where capture(0) is an ID and TOKEN is the secret. + let snippet_capture = + rm.m.groups + .captures + .iter() + .find(|c| c.name.map(|n| n.eq_ignore_ascii_case("TOKEN")).unwrap_or(false)) + .or_else(|| rm.m.groups.captures.get(0)); + + // Get raw snippet value (for revoke/validate command) and display snippet (for output) + let (raw_snippet, snippet) = if let Some(capture) = snippet_capture { let raw = capture.raw_value().to_string(); let displayed = capture.display_value(); (raw, Escaped(displayed.as_ref().as_bytes()).to_string()) @@ -718,11 +736,24 @@ impl DetailsReporter { // Generate validate command for findings with validation support let validate_cmd = if let Some(validation) = &rm.m.rule.syntax().validation { + // Merge dependent captures with named regex captures so the generated command is runnable. + // (E.g., Modal needs TOKEN_ID, which is a named capture on the same rule.) + let mut merged_vars = rm.m.dependent_captures.clone(); + for cap in rm.m.groups.captures.iter() { + let Some(name) = cap.name else { continue }; + if name.eq_ignore_ascii_case("TOKEN") { + continue; + } + merged_vars + .entry(name.to_uppercase()) + .or_insert_with(|| cap.raw_value().to_string()); + } + build_validate_command( rm.m.rule.id(), validation, &raw_snippet, - &rm.m.dependent_captures, + &merged_vars, akid_from_captures.as_deref(), akid_from_body.as_deref(), ) diff --git a/src/rules.rs b/src/rules.rs index 1b1a7ac..85ff0f4 100644 --- a/src/rules.rs +++ b/src/rules.rs @@ -11,8 +11,8 @@ pub mod rule { pub use kingfisher_rules::rule::Revocation; pub use kingfisher_rules::rules::{Rules, RulesError}; pub use kingfisher_rules::{ - ChecksumActual, ChecksumRequirement, Confidence, DependsOnRule, HttpRequest, HttpValidation, - MultipartConfig, MultipartPart, PatternRequirementContext, PatternRequirements, - PatternValidationResult, ReportResponseData, ResponseMatcher, Rule, RuleSyntax, Validation, - RULE_COMMENTS_PATTERN, + ChecksumActual, ChecksumRequirement, Confidence, DependsOnRule, GrpcRequest, GrpcValidation, + HttpRequest, HttpValidation, MultipartConfig, MultipartPart, PatternRequirementContext, + PatternRequirements, PatternValidationResult, ReportResponseData, ResponseMatcher, Rule, + RuleSyntax, Validation, RULE_COMMENTS_PATTERN, }; diff --git a/src/validation.rs b/src/validation.rs index 0671e3f..80dbccd 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -25,6 +25,8 @@ use crate::{ validation_body::{self}, }; +use crate::grpc_validation; + // Re-export TlsMode from kingfisher_rules for use in client_for_rule pub use kingfisher_rules::TlsMode as RuleTlsMode; @@ -464,6 +466,15 @@ async fn timed_validate_single_match<'a>( let mut globals = Object::new(); populate_globals_from_captures(&mut globals, &captured_values); + // Persist named captures (non-TOKEN) for validate/revoke command generation. + // This is especially important for gRPC validators like Modal where TOKEN_ID is required. + for (k, v, ..) in &captured_values { + if k.eq_ignore_ascii_case("TOKEN") { + continue; + } + m.dependent_captures.entry(k.to_uppercase()).or_insert_with(|| v.clone()); + } + let rule_syntax = m.rule.syntax(); // ────────────────────────────────────────────────────────── @@ -717,6 +728,87 @@ async fn timed_validate_single_match<'a>( } } + // ---------------------------------------------------- gRPC validator + Some(Validation::Grpc(grpc_validation_cfg)) => { + let request_timeout = validation_timeout; + + // Render URL + let url = match render_and_parse_url( + parser, + &globals, + &rule_syntax.name, + &grpc_validation_cfg.request.url, + ) + .await + { + Ok(u) => u, + Err(e) => { + m.validation_success = false; + m.validation_response_body = validation_body::from_string(e); + m.validation_response_status = StatusCode::BAD_REQUEST; + commit_and_return(m); + return; + } + }; + + // Execute gRPC unary call (HTTP/2 + trailers). + let res = match grpc_validation::grpc_unary_call_from_rule( + &url, + &grpc_validation_cfg.request.headers, + &grpc_validation_cfg.request.body, + parser, + &globals, + request_timeout, + ) + .await + { + Ok(r) => r, + Err(e) => { + m.validation_success = false; + m.validation_response_body = + validation_body::from_string(format!("gRPC error: {}", e)); + m.validation_response_status = StatusCode::BAD_GATEWAY; + commit_and_return(m); + return; + } + }; + + let status = StatusCode::from_u16(res.http_status.as_u16()).unwrap_or(StatusCode::OK); + let headers = res.headers; + let mut body = String::from_utf8_lossy(&res.body_bytes).to_string(); + + // gRPC errors are typically reported in trailers, not the body. + // Surface them for debugging and for `--full-validation-response` output. + let grpc_status = + headers.get("grpc-status").and_then(|v| v.to_str().ok()).unwrap_or("").to_string(); + let grpc_message = + headers.get("grpc-message").and_then(|v| v.to_str().ok()).unwrap_or("").to_string(); + // Avoid storing raw protobuf bytes in the report (they often contain NULs and make + // output logs non-UTF8). Prefer a compact status/message string. + if grpc_status == "0" { + body = "grpc-status=0".to_string(); + } else if body.trim().is_empty() + && (!grpc_status.is_empty() || !grpc_message.is_empty()) + { + body = format!("grpc-status={grpc_status} grpc-message={grpc_message}"); + } else if body.as_bytes().contains(&0) { + body = format!("grpc-status={grpc_status} grpc-message={grpc_message}"); + } + truncate_to_char_boundary(&mut body, MAX_VALIDATION_BODY_LEN); + + m.validation_response_status = status; + m.validation_response_body = validation_body::from_string(body.clone()); + + let matchers = grpc_validation_cfg + .request + .response_matcher + .as_ref() + .expect("missing response_matcher"); + + m.validation_success = + httpvalidation::validate_response(matchers, &body, &status, &headers, false); + } + // ---------------------------------------------------- MongoDB validator Some(Validation::MongoDB) => { let uri = globals