From e83b1716943f55f2f03a574d3ff0466d948d6baa Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Wed, 13 Aug 2025 15:35:04 -0700 Subject: [PATCH 1/4] added rule for Vercel --- CHANGELOG.md | 3 +++ Cargo.toml | 2 +- data/rules/airbrake.yml | 4 +--- data/rules/aiven.yml | 5 ++--- data/rules/asana.yml | 2 -- data/rules/atlassian.yml | 2 -- data/rules/baremetrics.yml | 2 -- data/rules/fastly.yml | 4 +++- data/rules/heroku.yml | 5 ++--- data/rules/vercel.yml | 39 ++++++++++++++++++++++++++++++++++++++ 10 files changed, 51 insertions(+), 17 deletions(-) create mode 100644 data/rules/vercel.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 6526ee6..aec63c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All notable changes to this project will be documented in this file. +## [1.41.0] +- Added rules for: Vercel + ## [1.40.0] - Dropped the “prevalidated” flag from rule definitions and validation logic so every finding now flows through the standard active/inactive/unknown pipeline, simplifying rule configuration and preventing special‑case bypasses - Improved Tailscale api key detectors diff --git a/Cargo.toml b/Cargo.toml index 15ad49a..db985c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ publish = false [package] name = "kingfisher" -version = "1.40.0" +version = "1.41.0" description = "MongoDB's blazingly fast secret scanning and validation tool" edition.workspace = true rust-version.workspace = true diff --git a/data/rules/airbrake.yml b/data/rules/airbrake.yml index 9d55e24..d54d261 100644 --- a/data/rules/airbrake.yml +++ b/data/rules/airbrake.yml @@ -5,9 +5,7 @@ rules: (?xi) \b airbrake - (?:.|[\n\r]){0,16}? - (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) - (?:.|[\n\r]){0,16}? + (?:.|[\n\r]){0,32}? ( [A-Z0-9-]{40} ) diff --git a/data/rules/aiven.yml b/data/rules/aiven.yml index 0f4c727..19f449a 100644 --- a/data/rules/aiven.yml +++ b/data/rules/aiven.yml @@ -2,11 +2,10 @@ rules: - name: Aiven API Key id: kingfisher.aiven.1 pattern: | - (?xi) + (?xi) + \b aiven (?:.|[\n\r]){0,32}? - (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) - (?:.|[\n\r]){0,32}? \b ( [a-z0-9/+=]{372} diff --git a/data/rules/asana.yml b/data/rules/asana.yml index 0824711..85e30c5 100644 --- a/data/rules/asana.yml +++ b/data/rules/asana.yml @@ -6,8 +6,6 @@ rules: \b asana (?:.|[\n\r]){0,32}? - (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) - (?:.|[\n\r]){0,32}? \b ( [0-9]{16} diff --git a/data/rules/atlassian.yml b/data/rules/atlassian.yml index 6d4ac9c..32dccb2 100644 --- a/data/rules/atlassian.yml +++ b/data/rules/atlassian.yml @@ -6,8 +6,6 @@ rules: \b atlassian (?:.|[\n\r]){0,32}? - (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) - (?:.|[\n\r]){0,32}? \b ( [a-z0-9]{24} diff --git a/data/rules/baremetrics.yml b/data/rules/baremetrics.yml index 415731e..2036d18 100644 --- a/data/rules/baremetrics.yml +++ b/data/rules/baremetrics.yml @@ -6,8 +6,6 @@ rules: \b baremetrics (?:.|[\n\r]){0,32}? - (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) - (?:.|[\n\r]){0,32}? \b ( [a-z0-9_-]{25} diff --git a/data/rules/fastly.yml b/data/rules/fastly.yml index c5d4fcb..77618a6 100644 --- a/data/rules/fastly.yml +++ b/data/rules/fastly.yml @@ -9,7 +9,9 @@ rules: (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) (?:.|[\n\r]){0,32}? \b - ([a-z0-9_-]{32}) + ( + [a-z0-9_-]{32} + ) \b min_entropy: 3.5 confidence: medium diff --git a/data/rules/heroku.yml b/data/rules/heroku.yml index 3686e7a..b06d58e 100644 --- a/data/rules/heroku.yml +++ b/data/rules/heroku.yml @@ -2,10 +2,9 @@ rules: - name: Heroku API Key id: kingfisher.heroku.1 pattern: | - (?xi) + (?xi) + \b heroku - (?:.|[\n\r]){0,32}? - (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) (?:.|[\n\r]){0,32}? \b ( diff --git a/data/rules/vercel.yml b/data/rules/vercel.yml new file mode 100644 index 0000000..d649b00 --- /dev/null +++ b/data/rules/vercel.yml @@ -0,0 +1,39 @@ +rules: + - name: Vercel API Token + id: kingfisher.vercel.1 + pattern: | + (?xi) + \b + vercel + (?:.|[\n\r]){0,32}? + \b + ( + [a-zA-Z0-9]{24} + ) + \b + confidence: medium + min_entropy: 3.5 + validation: + type: Http + content: + request: + method: GET + url: https://api.vercel.com/v2/user + headers: + Authorization: "Bearer {{TOKEN}}" + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: + - '"user":' + - '"email":' + match_all_words: true + references: + - https://vercel.com/docs/rest-api#authentication + examples: + - "vercel-key = DdZV6ZDZW6Vpl7n7JqtrCE5i" + - "vercel_token = zyMBA1qVEMAf4UNNZtCAbg6u" + - "vercel_api_key = MTg0AW799OY1HmyDdn84or3C" + - "vercel_secret = A7n9Xfp3tBz7D0XpOTMWpiOM" From 14fccc9cc64916946b0f7d30caedefe2e99a9978 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Thu, 14 Aug 2025 09:25:18 -0700 Subject: [PATCH 2/4] - Added support for scanning gitlab subgroups, with 'kingfisher scan --gitlab-group my-group --gitlab-include-subgroups' --- README.md | 4 ++++ src/cli/commands/gitlab.rs | 4 ++++ src/cli/commands/inputs.rs | 4 ++++ src/gitlab.rs | 14 ++++++++++++-- src/main.rs | 2 ++ src/reporter/json_format.rs | 1 + src/scanner/repos.rs | 1 + tests/int_dedup.rs | 1 + tests/int_github.rs | 1 + tests/int_gitlab.rs | 2 ++ tests/int_redact.rs | 1 + tests/int_slack.rs | 2 ++ tests/int_validation_cache.rs | 1 + tests/int_vulnerable_files.rs | 2 ++ 14 files changed, 38 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d51f0b3..6873a6a 100644 --- a/README.md +++ b/README.md @@ -384,6 +384,8 @@ KF_GITHUB_TOKEN="ghp_…" kingfisher scan --git-url https://github.com/org/priva ```bash kingfisher scan --gitlab-group my-group +# include repositories from all nested subgroups +kingfisher scan --gitlab-group my-group --gitlab-include-subgroups ``` ### Scan GitLab user @@ -402,6 +404,8 @@ kingfisher scan --git-url https://gitlab.com/group/project.git ```bash kingfisher gitlab repos list --group my-group +# include repositories from all nested subgroups +kingfisher gitlab repos list --group my-group --include-subgroups ``` ## Scanning Jira diff --git a/src/cli/commands/gitlab.rs b/src/cli/commands/gitlab.rs index e1bbdc3..8765c87 100644 --- a/src/cli/commands/gitlab.rs +++ b/src/cli/commands/gitlab.rs @@ -56,6 +56,10 @@ pub struct GitLabRepoSpecifiers { /// Filter by repository type #[arg(long, default_value_t = GitLabRepoType::All, alias = "gitlab-repo-type")] pub repo_type: GitLabRepoType, + + /// Include repositories from subgroups of the specified groups + #[arg(long, alias = "gitlab-include-subgroups")] + pub include_subgroups: bool, } impl GitLabRepoSpecifiers { diff --git a/src/cli/commands/inputs.rs b/src/cli/commands/inputs.rs index e6f9168..7836d79 100644 --- a/src/cli/commands/inputs.rs +++ b/src/cli/commands/inputs.rs @@ -89,6 +89,10 @@ pub struct InputSpecifierArgs { #[arg(long, default_value_t = GitLabRepoType::All)] pub gitlab_repo_type: GitLabRepoType, + /// Include projects from GitLab subgroups when scanning groups + #[arg(long, alias = "include-subgroups")] + pub gitlab_include_subgroups: bool, + /// Jira base URL (e.g. https://jira.example.com) #[arg(long, value_hint = ValueHint::Url, requires = "jql")] pub jira_url: Option, diff --git a/src/gitlab.rs b/src/gitlab.rs index e7df15e..80b7751 100644 --- a/src/gitlab.rs +++ b/src/gitlab.rs @@ -42,6 +42,7 @@ pub struct RepoSpecifiers { pub user: Vec, pub group: Vec, pub all_groups: bool, + pub include_subgroups: bool, pub repo_filter: RepoType, } @@ -137,6 +138,9 @@ pub async fn enumerate_repo_urls( if matches!(repo_specifiers.repo_filter, RepoType::Owner) { gp_builder.owned(true); } + if repo_specifiers.include_subgroups { + gp_builder.include_subgroups(true); + } let gp_ep = gp_builder.build()?; let projects: Vec = gp_ep.query(&client)?; @@ -162,10 +166,16 @@ pub async fn list_repositories( users: &[String], groups: &[String], all_groups: bool, + include_subgroups: bool, repo_filter: RepoType, ) -> Result<()> { - let repo_specifiers = - RepoSpecifiers { user: users.to_vec(), group: groups.to_vec(), all_groups, repo_filter }; + let repo_specifiers = RepoSpecifiers { + user: users.to_vec(), + group: groups.to_vec(), + all_groups, + include_subgroups, + repo_filter, + }; // Create a progress bar for displaying status let mut progress = if progress_enabled { diff --git a/src/main.rs b/src/main.rs index 3943b1e..2e9b6e2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -245,6 +245,7 @@ async fn async_main(args: CommandLineArgs) -> Result<()> { &list_args.repo_specifiers.user, &list_args.repo_specifiers.group, list_args.repo_specifiers.all_groups, + list_args.repo_specifiers.include_subgroups, list_args.repo_specifiers.repo_type.into(), ) .await?; @@ -282,6 +283,7 @@ fn create_default_scan_args() -> cli::commands::scan::ScanArgs { all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::All, + gitlab_include_subgroups: false, jira_url: None, jql: None, diff --git a/src/reporter/json_format.rs b/src/reporter/json_format.rs index 10d7aee..4020cee 100644 --- a/src/reporter/json_format.rs +++ b/src/reporter/json_format.rs @@ -84,6 +84,7 @@ mod tests { all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::All, + gitlab_include_subgroups: false, // Jira options jira_url: None, jql: None, diff --git a/src/scanner/repos.rs b/src/scanner/repos.rs index a249f02..e5052c1 100644 --- a/src/scanner/repos.rs +++ b/src/scanner/repos.rs @@ -182,6 +182,7 @@ pub async fn enumerate_gitlab_repos( user: args.input_specifier_args.gitlab_user.clone(), group: args.input_specifier_args.gitlab_group.clone(), all_groups: args.input_specifier_args.all_gitlab_groups, + include_subgroups: args.input_specifier_args.gitlab_include_subgroups, repo_filter: args.input_specifier_args.gitlab_repo_type.into(), }; diff --git a/tests/int_dedup.rs b/tests/int_dedup.rs index 68b9663..090effa 100644 --- a/tests/int_dedup.rs +++ b/tests/int_dedup.rs @@ -78,6 +78,7 @@ rules: all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::Owner, + gitlab_include_subgroups: false, jira_url: None, jql: None, diff --git a/tests/int_github.rs b/tests/int_github.rs index 0bae089..dbedcb5 100644 --- a/tests/int_github.rs +++ b/tests/int_github.rs @@ -65,6 +65,7 @@ fn test_github_remote_scan() -> Result<()> { all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::Owner, + gitlab_include_subgroups: false, jira_url: None, jql: None, diff --git a/tests/int_gitlab.rs b/tests/int_gitlab.rs index 7e48f60..5a72ce5 100644 --- a/tests/int_gitlab.rs +++ b/tests/int_gitlab.rs @@ -64,6 +64,7 @@ fn test_gitlab_remote_scan() -> Result<()> { all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/")?, gitlab_repo_type: GitLabRepoType::Owner, + gitlab_include_subgroups: false, jira_url: None, jql: None, @@ -169,6 +170,7 @@ fn test_gitlab_remote_scan_no_history() -> Result<()> { all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/")?, gitlab_repo_type: GitLabRepoType::Owner, + gitlab_include_subgroups: false, jira_url: None, jql: None, diff --git a/tests/int_redact.rs b/tests/int_redact.rs index 69a1061..6bd97c7 100644 --- a/tests/int_redact.rs +++ b/tests/int_redact.rs @@ -49,6 +49,7 @@ async fn test_redact_hashes_finding_values() -> Result<()> { all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::Owner, + gitlab_include_subgroups: false, jira_url: None, jql: None, confluence_url: None, diff --git a/tests/int_slack.rs b/tests/int_slack.rs index e94607f..86cea6f 100644 --- a/tests/int_slack.rs +++ b/tests/int_slack.rs @@ -55,6 +55,7 @@ impl TestContext { all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::Owner, + gitlab_include_subgroups: false, jira_url: None, jql: None, confluence_url: None, @@ -147,6 +148,7 @@ async fn test_scan_slack_messages() -> Result<()> { all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::Owner, + gitlab_include_subgroups: false, jira_url: None, jql: None, confluence_url: None, diff --git a/tests/int_validation_cache.rs b/tests/int_validation_cache.rs index 2caa10a..6e2cc6a 100644 --- a/tests/int_validation_cache.rs +++ b/tests/int_validation_cache.rs @@ -121,6 +121,7 @@ async fn test_validation_cache_and_depvars() -> Result<()> { all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::Owner, + gitlab_include_subgroups: false, jira_url: None, jql: None, diff --git a/tests/int_vulnerable_files.rs b/tests/int_vulnerable_files.rs index 2478170..31a74ac 100644 --- a/tests/int_vulnerable_files.rs +++ b/tests/int_vulnerable_files.rs @@ -64,6 +64,7 @@ impl TestContext { all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::Owner, + gitlab_include_subgroups: false, jira_url: None, jql: None, @@ -142,6 +143,7 @@ impl TestContext { all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::Owner, + gitlab_include_subgroups: false, jira_url: None, jql: None, From 737e4829acdaa6ed3d64c6652dd43c330f9735f4 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Thu, 14 Aug 2025 09:26:44 -0700 Subject: [PATCH 3/4] - Added support for scanning gitlab subgroups, with 'kingfisher scan --gitlab-group my-group --gitlab-include-subgroups' --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aec63c5..cb5b3e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,8 @@ All notable changes to this project will be documented in this file. ## [1.41.0] -- Added rules for: Vercel +- Added support for scanning gitlab subgroups, with `kingfisher scan --gitlab-group my-group --gitlab-include-subgroups` +- Added rule for Vercel ## [1.40.0] - Dropped the “prevalidated” flag from rule definitions and validation logic so every finding now flows through the standard active/inactive/unknown pipeline, simplifying rule configuration and preventing special‑case bypasses From a3a7efb96e9eaddb117a0908b1a760dc8ab75b7d Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Thu, 14 Aug 2025 10:17:16 -0700 Subject: [PATCH 4/4] fixed aiven regex to pass test --- data/rules/aiven.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/data/rules/aiven.yml b/data/rules/aiven.yml index 19f449a..09a7269 100644 --- a/data/rules/aiven.yml +++ b/data/rules/aiven.yml @@ -2,8 +2,7 @@ rules: - name: Aiven API Key id: kingfisher.aiven.1 pattern: | - (?xi) - \b + (?xi) aiven (?:.|[\n\r]){0,32}? \b