From 563fa66d4648cb1b4a2bbea2b688c75e365bd7a5 Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Mon, 15 Sep 2025 21:26:51 -0700 Subject: [PATCH] Added --github-exclude and --gitlab-exclude options to skip specific repositories when scanning or listing GitHub and GitLab sources, including support for gitignore-style glob patterns --- CHANGELOG.md | 3 + Cargo.toml | 2 +- README.md | 32 ++++++ src/cli/commands/github.rs | 4 + src/cli/commands/gitlab.rs | 9 ++ src/cli/commands/inputs.rs | 13 +++ src/github.rs | 199 +++++++++++++++++++++++++++++++++- src/gitlab.rs | 184 +++++++++++++++++++++++++++++++ src/main.rs | 4 + src/reporter/json_format.rs | 2 + src/scanner/repos.rs | 2 + tests/int_allowlist.rs | 2 + tests/int_dedup.rs | 2 + tests/int_github.rs | 2 + tests/int_gitlab.rs | 4 + tests/int_redact.rs | 2 + tests/int_slack.rs | 4 + tests/int_validation_cache.rs | 2 + tests/int_vulnerable_files.rs | 4 + 19 files changed, 473 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5355560..2e99583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All notable changes to this project will be documented in this file. +## [1.50.0] +- Added `--github-exclude` and `--gitlab-exclude` options to skip specific repositories when scanning or listing GitHub and GitLab sources, including support for gitignore-style glob patterns + ## [1.49.0] - Enabled MongoDB URI validation - AWS + GCP validators now respect HTTPS_PROXY and share a consistent user agent across AWS, GCP, and HTTP validation diff --git a/Cargo.toml b/Cargo.toml index 4d810df..fd82763 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ publish = false [package] name = "kingfisher" -version = "1.49.0" +version = "1.50.0" description = "MongoDB's blazingly fast secret scanning and validation tool" edition.workspace = true rust-version.workspace = true diff --git a/README.md b/README.md index bea6c11..2338597 100644 --- a/README.md +++ b/README.md @@ -62,10 +62,12 @@ See ([docs/COMPARISON.md](docs/COMPARISON.md)) - [Scanning Docker Images](#scanning-docker-images) - [Scanning GitHub](#scanning-github) - [Scan GitHub organisation (requires `KF_GITHUB_TOKEN`)](#scan-github-organisation-requires-kf_github_token) + - [Skip specific GitHub repositories during enumeration](#skip-specific-github-repositories-during-enumeration) - [Scan remote GitHub repository](#scan-remote-github-repository) - [Scanning GitLab](#scanning-gitlab) - [Scan GitLab group (requires `KF_GITLAB_TOKEN`)](#scan-gitlab-group-requires-kf_gitlab_token) - [Scan GitLab user](#scan-gitlab-user) + - [Skip specific GitLab projects during enumeration](#skip-specific-gitlab-projects-during-enumeration) - [Scan remote GitLab repository by URL](#scan-remote-gitlab-repository-by-url) - [List GitLab repositories](#list-gitlab-repositories) - [Scanning Jira](#scanning-jira) @@ -427,6 +429,19 @@ kingfisher scan --docker-image private.registry.example.com/my-image:tag kingfisher scan --github-organization my-org ``` +### Skip specific GitHub repositories during enumeration + +Repeat `--github-exclude` for every repository you want to ignore when scanning +users or organizations. You can provide exact repositories like +`OWNER/REPO` or gitignore-style glob patterns such as `owner/*-archive` +(matching is case-insensitive). + +```bash +kingfisher scan --github-organization my-org \ + --github-exclude my-org/huge-repo \ + --github-exclude my-org/*-archive +``` + ### Scan remote GitHub repository `--git-url` clones the repository and scans its files and history. To also inspect @@ -464,6 +479,19 @@ kingfisher scan --gitlab-group my-group --gitlab-include-subgroups kingfisher scan --gitlab-user johndoe ``` +### Skip specific GitLab projects during enumeration + +Repeat `--gitlab-exclude` for every project path you want to ignore when scanning +users or groups. Specify project paths as `group/project` (case-insensitive) or +use gitignore-style glob patterns like `group/**/archive-*` to drop families of +projects across nested subgroups. + +```bash +kingfisher scan --gitlab-group my-group \ + --gitlab-exclude my-group/huge-project \ + --gitlab-exclude my-group/**/archive-* +``` + ### Scan remote GitLab repository by URL `--git-url` by itself clones the project repository. To include server-side @@ -488,6 +516,8 @@ KF_GITLAB_TOKEN="glpat-…" kingfisher scan --git-url https://gitlab.com/group/p kingfisher gitlab repos list --group my-group # include repositories from all nested subgroups kingfisher gitlab repos list --group my-group --include-subgroups +# skip specific projects when listing or scanning (supports glob patterns) +kingfisher gitlab repos list --group my-group --gitlab-exclude my-group/**/legacy-* ``` ## Scanning Jira @@ -666,6 +696,8 @@ kingfisher rules check --rules-path ./my_rules.yml # List GitHub repos kingfisher github repos list --user my-user kingfisher github repos list --organization my-org +# Skip specific repositories when listing or scanning (supports glob patterns) +kingfisher github repos list --organization my-org --github-exclude my-org/*-archive ``` diff --git a/src/cli/commands/github.rs b/src/cli/commands/github.rs index 7537c2e..62194f5 100644 --- a/src/cli/commands/github.rs +++ b/src/cli/commands/github.rs @@ -49,6 +49,10 @@ pub struct GitHubRepoSpecifiers { #[arg(long, alias = "org", alias = "github-organization", alias = "github-org")] pub organization: Vec, + /// Skip specific repositories when enumerating GitHub sources (format: owner/repo) + #[arg(long = "github-exclude", alias = "github-exclude-repo", value_name = "OWNER/REPO")] + pub exclude_repos: Vec, + /// Repositories for all organizations (Enterprise only) #[arg( long, diff --git a/src/cli/commands/gitlab.rs b/src/cli/commands/gitlab.rs index 8765c87..73f44a4 100644 --- a/src/cli/commands/gitlab.rs +++ b/src/cli/commands/gitlab.rs @@ -49,6 +49,15 @@ pub struct GitLabRepoSpecifiers { #[arg(long, alias = "gitlab-group")] pub group: Vec, + /// Skip specific repositories when enumerating GitLab sources (format: group/project) + #[arg( + long = "gitlab-exclude", + alias = "gitlab-exclude-project", + alias = "gitlab-exclude-repo", + value_name = "GROUP/PROJECT" + )] + pub exclude_repos: Vec, + /// Repositories for all groups (Enterprise only) #[arg(long, alias = "all-groups", alias = "all-gitlab-groups", requires = "gitlab_api_url")] pub all_groups: bool, diff --git a/src/cli/commands/inputs.rs b/src/cli/commands/inputs.rs index 60bfba8..77fc192 100644 --- a/src/cli/commands/inputs.rs +++ b/src/cli/commands/inputs.rs @@ -48,6 +48,10 @@ pub struct InputSpecifierArgs { #[arg(long, alias = "github-org")] pub github_organization: Vec, + /// Skip repositories when enumerating GitHub users or organizations (format: owner/repo) + #[arg(long = "github-exclude", alias = "github-exclude-repo", value_name = "OWNER/REPO")] + pub github_exclude: Vec, + /// Scan repositories from all GitHub organizations (requires non-default --github-api-url) #[arg(long, alias = "all-github-orgs", requires = "github_api_url")] pub all_github_organizations: bool, @@ -73,6 +77,15 @@ pub struct InputSpecifierArgs { #[arg(long, alias = "gitlab-group")] pub gitlab_group: Vec, + /// Skip repositories when enumerating GitLab users or groups (format: group/project) + #[arg( + long = "gitlab-exclude", + alias = "gitlab-exclude-project", + alias = "gitlab-exclude-repo", + value_name = "GROUP/PROJECT" + )] + pub gitlab_exclude: Vec, + /// Scan repositories from all GitLab groups (requires non-default --gitlab-api-url) #[arg(long, alias = "all-gitlab-groups", requires = "gitlab_api_url")] pub all_gitlab_groups: bool, diff --git a/src/github.rs b/src/github.rs index 599db85..56b7c3e 100644 --- a/src/github.rs +++ b/src/github.rs @@ -7,6 +7,7 @@ use std::{ }; use anyhow::{Context, Result}; +use globset::{Glob, GlobSet, GlobSetBuilder}; use indicatif::{ProgressBar, ProgressStyle}; use octorust::{ auth::Credentials, @@ -14,6 +15,7 @@ use octorust::{ Client, }; use serde_json::Value; +use tracing::warn; use url::Url; use crate::{findings_store, git_url::GitUrl}; @@ -25,6 +27,7 @@ pub struct RepoSpecifiers { pub organization: Vec, pub all_organizations: bool, pub repo_filter: RepoType, + pub exclude_repos: Vec, } impl RepoSpecifiers { pub fn is_empty(&self) -> bool { @@ -55,6 +58,133 @@ impl From for ReposListOrgType { } } } + +fn normalize_repo_identifier(owner: &str, repo: &str) -> Option { + let owner = owner.trim().trim_matches('/'); + let repo = repo.trim().trim_matches('/'); + let repo = repo.strip_suffix(".git").unwrap_or(repo); + if owner.is_empty() || repo.is_empty() { + return None; + } + Some(format!("{}/{}", owner.to_lowercase(), repo.to_lowercase())) +} + +fn parse_repo_name_from_path(path: &str) -> Option { + let trimmed = path.trim().trim_matches('/'); + if trimmed.is_empty() { + return None; + } + let mut parts = trimmed.split('/'); + let owner = parts.next()?; + let repo = parts.next()?; + if parts.next().is_some() { + return None; + } + normalize_repo_identifier(owner, repo) +} + +fn parse_repo_name_from_url(repo_url: &str) -> Option { + let url = Url::parse(repo_url).ok()?; + parse_repo_name_from_path(url.path()) +} + +fn parse_excluded_repo(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + + if let Some(name) = parse_repo_name_from_url(trimmed) { + return Some(name); + } + + if let Some(idx) = trimmed.rfind(':') { + if let Some(name) = parse_repo_name_from_path(&trimmed[idx + 1..]) { + return Some(name); + } + } + + parse_repo_name_from_path(trimmed) +} + +struct ExcludeMatcher { + exact: HashSet, + globs: Option, +} + +impl ExcludeMatcher { + fn is_empty(&self) -> bool { + self.exact.is_empty() && self.globs.is_none() + } + + fn matches(&self, name: &str) -> bool { + if self.exact.contains(name) { + return true; + } + if let Some(globs) = &self.globs { + return globs.is_match(name); + } + false + } +} + +fn looks_like_glob(pattern: &str) -> bool { + pattern.contains('*') || pattern.contains('?') || pattern.contains('[') +} + +fn build_exclude_matcher(exclude_repos: &[String]) -> ExcludeMatcher { + let mut exact = HashSet::new(); + let mut glob_builder = GlobSetBuilder::new(); + let mut has_glob = false; + + for raw in exclude_repos { + match parse_excluded_repo(raw) { + Some(name) => { + if looks_like_glob(&name) { + match Glob::new(&name) { + Ok(glob) => { + glob_builder.add(glob); + has_glob = true; + } + Err(err) => { + warn!("Ignoring invalid GitHub exclusion pattern '{raw}': {err}"); + exact.insert(name); + } + } + } else { + exact.insert(name); + } + } + None => { + warn!("Ignoring invalid GitHub exclusion '{raw}' (expected owner/repo)"); + } + } + } + + let globs = if has_glob { + match glob_builder.build() { + Ok(set) => Some(set), + Err(err) => { + warn!("Failed to build GitHub exclusion patterns: {err}"); + None + } + } + } else { + None + }; + + ExcludeMatcher { exact, globs } +} + +fn should_exclude_repo(clone_url: &str, excludes: &ExcludeMatcher) -> bool { + if excludes.is_empty() { + return false; + } + if let Some(name) = parse_repo_name_from_url(clone_url) { + return excludes.matches(&name); + } + false +} fn create_github_client(github_url: &url::Url, ignore_certs: bool) -> Result> { // Try personal access token let credentials = if let Ok(token) = env::var("KF_GITHUB_TOKEN") { @@ -92,6 +222,7 @@ pub async fn enumerate_repo_urls( ) -> Result> { let client = create_github_client(&github_url, ignore_certs)?; let mut repo_urls = Vec::new(); + let exclude_set = build_exclude_matcher(&repo_specifiers.exclude_repos); let user_repo_type: ReposListUserType = repo_specifiers.repo_filter.clone().into(); let org_repo_type: ReposListOrgType = repo_specifiers.repo_filter.clone().into(); for username in &repo_specifiers.user { @@ -104,7 +235,14 @@ pub async fn enumerate_repo_urls( Order::Desc, ) .await?; - repo_urls.extend(repos.body.into_iter().filter_map(|repo| Some(repo.clone_url))); + repo_urls.extend(repos.body.into_iter().filter_map(|repo| { + let clone_url = repo.clone_url; + if should_exclude_repo(&clone_url, &exclude_set) { + None + } else { + Some(clone_url) + } + })); if let Some(progress) = progress.as_mut() { progress.inc(1); } @@ -127,7 +265,14 @@ pub async fn enumerate_repo_urls( Order::Desc, ) .await?; - repo_urls.extend(repos.body.into_iter().filter_map(|repo| Some(repo.clone_url))); + repo_urls.extend(repos.body.into_iter().filter_map(|repo| { + let clone_url = repo.clone_url; + if should_exclude_repo(&clone_url, &exclude_set) { + None + } else { + Some(clone_url) + } + })); if let Some(progress) = progress.as_mut() { progress.inc(1); } @@ -143,6 +288,7 @@ pub async fn list_repositories( users: &[String], orgs: &[String], all_orgs: bool, + exclude_repos: &[String], repo_filter: RepoType, ) -> Result<()> { let repo_specifiers = RepoSpecifiers { @@ -150,6 +296,7 @@ pub async fn list_repositories( organization: orgs.to_vec(), all_organizations: all_orgs, repo_filter, + exclude_repos: exclude_repos.to_vec(), }; // Create a progress bar just for displaying status // let mut progress = ProgressBar::new_spinner("Fetching repositories...", @@ -358,3 +505,51 @@ pub async fn fetch_repo_items( Ok(dirs) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_excluded_repo_variants() { + assert_eq!(parse_excluded_repo("Owner/Repo").as_deref(), Some("owner/repo")); + assert_eq!(parse_excluded_repo("owner/repo.git").as_deref(), Some("owner/repo")); + assert_eq!( + parse_excluded_repo("https://github.com/Owner/Repo.git").as_deref(), + Some("owner/repo") + ); + assert_eq!( + parse_excluded_repo("git@github.com:Owner/Repo.git").as_deref(), + Some("owner/repo") + ); + assert_eq!( + parse_excluded_repo("ssh://git@github.example.com/Owner/Repo.git").as_deref(), + Some("owner/repo") + ); + assert_eq!( + parse_excluded_repo(" https://github.com/Owner/Repo ").as_deref(), + Some("owner/repo") + ); + assert_eq!(parse_excluded_repo("not-a-repo"), None); + } + + #[test] + fn should_exclude_repo_matches_normalized_names() { + let excludes = build_exclude_matcher(&vec!["Owner/Repo".to_string()]); + assert!(should_exclude_repo("https://github.com/owner/repo.git", &excludes)); + assert!(!should_exclude_repo("https://github.com/owner/other.git", &excludes)); + } + + #[test] + fn should_exclude_repo_matches_ssh_urls() { + let excludes = build_exclude_matcher(&vec!["owner/repo".to_string()]); + assert!(should_exclude_repo("ssh://git@github.example.com/owner/repo.git", &excludes)); + } + + #[test] + fn should_exclude_repo_matches_globs() { + let excludes = build_exclude_matcher(&vec!["owner/*-archive".to_string()]); + assert!(should_exclude_repo("https://github.com/owner/project-archive.git", &excludes)); + assert!(!should_exclude_repo("https://github.com/owner/project.git", &excludes)); + } +} diff --git a/src/gitlab.rs b/src/gitlab.rs index e311c5b..17926e3 100644 --- a/src/gitlab.rs +++ b/src/gitlab.rs @@ -1,4 +1,5 @@ use std::{ + collections::HashSet, env, fs, path::{Path, PathBuf}, sync::{Arc, Mutex}, @@ -15,9 +16,11 @@ use gitlab::{ }, Gitlab, GitlabBuilder, }; +use globset::{Glob, GlobSet, GlobSetBuilder}; use indicatif::{ProgressBar, ProgressStyle}; use serde::Deserialize; use serde_json::Value; +use tracing::warn; use url::{form_urlencoded, Url}; use crate::{findings_store, git_url::GitUrl}; @@ -54,6 +57,7 @@ pub struct RepoSpecifiers { pub all_groups: bool, pub include_subgroups: bool, pub repo_filter: RepoType, + pub exclude_repos: Vec, } impl RepoSpecifiers { @@ -62,6 +66,126 @@ impl RepoSpecifiers { } } +fn normalize_project_path(path: &str) -> Option { + let trimmed = path.trim().trim_matches('/'); + if trimmed.is_empty() { + return None; + } + let without_git = trimmed.strip_suffix(".git").unwrap_or(trimmed); + let segments: Vec<&str> = without_git.split('/').filter(|s| !s.is_empty()).collect(); + if segments.len() < 2 { + return None; + } + Some(segments.join("/").to_lowercase()) +} + +fn parse_project_path_from_url(repo_url: &str) -> Option { + let url = Url::parse(repo_url).ok()?; + normalize_project_path(url.path()) +} + +fn parse_project_path(raw: &str) -> Option { + normalize_project_path(raw) +} + +fn parse_excluded_project(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + + if let Some(name) = parse_project_path_from_url(trimmed) { + return Some(name); + } + + if let Some(idx) = trimmed.rfind(':') { + if let Some(name) = parse_project_path(&trimmed[idx + 1..]) { + return Some(name); + } + } + + parse_project_path(trimmed) +} + +struct ExcludeMatcher { + exact: HashSet, + globs: Option, +} + +impl ExcludeMatcher { + fn is_empty(&self) -> bool { + self.exact.is_empty() && self.globs.is_none() + } + + fn matches(&self, name: &str) -> bool { + if self.exact.contains(name) { + return true; + } + if let Some(globs) = &self.globs { + return globs.is_match(name); + } + false + } +} + +fn looks_like_glob(pattern: &str) -> bool { + pattern.contains('*') || pattern.contains('?') || pattern.contains('[') +} + +fn build_exclude_matcher(exclude_repos: &[String]) -> ExcludeMatcher { + let mut exact = HashSet::new(); + let mut glob_builder = GlobSetBuilder::new(); + let mut has_glob = false; + + for raw in exclude_repos { + match parse_excluded_project(raw) { + Some(name) => { + if looks_like_glob(&name) { + match Glob::new(&name) { + Ok(glob) => { + glob_builder.add(glob); + has_glob = true; + } + Err(err) => { + warn!("Ignoring invalid GitLab exclusion pattern '{raw}': {err}"); + exact.insert(name); + } + } + } else { + exact.insert(name); + } + } + None => { + warn!("Ignoring invalid GitLab exclusion '{raw}' (expected group/project)"); + } + } + } + + let globs = if has_glob { + match glob_builder.build() { + Ok(set) => Some(set), + Err(err) => { + warn!("Failed to build GitLab exclusion patterns: {err}"); + None + } + } + } else { + None + }; + + ExcludeMatcher { exact, globs } +} + +fn should_exclude_repo(clone_url: &str, excludes: &ExcludeMatcher) -> bool { + if excludes.is_empty() { + return false; + } + if let Some(name) = parse_project_path_from_url(clone_url) { + return excludes.matches(&name); + } + false +} + fn create_gitlab_client(gitlab_url: &Url, ignore_certs: bool) -> Result { let host = gitlab_url.host_str().context("GitLab URL must contain a host")?; @@ -89,6 +213,7 @@ pub async fn enumerate_repo_urls( ) -> Result> { let client = create_gitlab_client(&gitlab_url, ignore_certs)?; let mut repo_urls = Vec::new(); + let exclude_set = build_exclude_matcher(&repo_specifiers.exclude_repos); // 1) Process each GitLab username for username in &repo_specifiers.user { @@ -118,6 +243,9 @@ pub async fn enumerate_repo_urls( let projects_ep = builder.build()?; let projects: Vec = paged(projects_ep, Pagination::All).query(&client)?; for proj in projects { + if should_exclude_repo(&proj.http_url_to_repo, &exclude_set) { + continue; + } repo_urls.push(proj.http_url_to_repo); } @@ -153,6 +281,9 @@ pub async fn enumerate_repo_urls( let gp_ep = gp_builder.build()?; let projects: Vec = paged(gp_ep, Pagination::All).query(&client)?; for proj in projects { + if should_exclude_repo(&proj.http_url_to_repo, &exclude_set) { + continue; + } repo_urls.push(proj.http_url_to_repo); } if let Some(pb) = progress.as_mut() { @@ -175,6 +306,7 @@ pub async fn list_repositories( groups: &[String], all_groups: bool, include_subgroups: bool, + exclude_repos: &[String], repo_filter: RepoType, ) -> Result<()> { let repo_specifiers = RepoSpecifiers { @@ -183,6 +315,7 @@ pub async fn list_repositories( all_groups, include_subgroups, repo_filter, + exclude_repos: exclude_repos.to_vec(), }; // Create a progress bar for displaying status @@ -320,3 +453,54 @@ pub async fn fetch_repo_items( Ok(dirs) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_excluded_project_variants() { + assert_eq!(parse_excluded_project("Group/Project").as_deref(), Some("group/project")); + assert_eq!(parse_excluded_project("group/project.git").as_deref(), Some("group/project")); + assert_eq!( + parse_excluded_project("https://gitlab.com/Group/Project.git").as_deref(), + Some("group/project") + ); + assert_eq!( + parse_excluded_project("git@gitlab.com:Group/Sub/Project.git").as_deref(), + Some("group/sub/project") + ); + assert_eq!( + parse_excluded_project("ssh://git@gitlab.example.com/Group/Sub/Project.git").as_deref(), + Some("group/sub/project") + ); + assert_eq!( + parse_excluded_project(" group/sub/project ").as_deref(), + Some("group/sub/project") + ); + assert_eq!(parse_excluded_project("not-a-project"), None); + } + + #[test] + fn should_exclude_repo_matches_normalized_paths() { + let excludes = build_exclude_matcher(&vec!["Group/Sub/Project".to_string()]); + assert!(should_exclude_repo("https://gitlab.com/group/sub/project.git", &excludes)); + assert!(!should_exclude_repo("https://gitlab.com/group/other/project.git", &excludes)); + } + + #[test] + fn should_exclude_repo_matches_ssh_urls() { + let excludes = build_exclude_matcher(&vec!["group/sub/project".to_string()]); + assert!(should_exclude_repo( + "ssh://git@gitlab.example.com/group/sub/project.git", + &excludes + )); + } + + #[test] + fn should_exclude_repo_matches_globs() { + let excludes = build_exclude_matcher(&vec!["group/**/archive-*".to_string()]); + assert!(should_exclude_repo("https://gitlab.com/group/sub/archive-2023.git", &excludes)); + assert!(!should_exclude_repo("https://gitlab.com/group/sub/project.git", &excludes)); + } +} diff --git a/src/main.rs b/src/main.rs index bfd16f8..cc15c6c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -232,6 +232,7 @@ async fn async_main(args: CommandLineArgs) -> Result<()> { &list_args.repo_specifiers.user, &list_args.repo_specifiers.organization, list_args.repo_specifiers.all_organizations, + &list_args.repo_specifiers.exclude_repos, list_args.repo_specifiers.repo_type.into(), ) .await?; @@ -249,6 +250,7 @@ async fn async_main(args: CommandLineArgs) -> Result<()> { &list_args.repo_specifiers.group, list_args.repo_specifiers.all_groups, list_args.repo_specifiers.include_subgroups, + &list_args.repo_specifiers.exclude_repos, list_args.repo_specifiers.repo_type.into(), ) .await?; @@ -282,12 +284,14 @@ fn create_default_scan_args() -> cli::commands::scan::ScanArgs { git_url: Vec::new(), github_user: Vec::new(), github_organization: Vec::new(), + github_exclude: Vec::new(), all_github_organizations: false, github_api_url: url::Url::parse("https://api.github.com/").unwrap(), github_repo_type: GitHubRepoType::Source, // new GitLab defaults gitlab_user: Vec::new(), gitlab_group: Vec::new(), + gitlab_exclude: Vec::new(), all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::All, diff --git a/src/reporter/json_format.rs b/src/reporter/json_format.rs index 1ee7fb7..e6074be 100644 --- a/src/reporter/json_format.rs +++ b/src/reporter/json_format.rs @@ -76,6 +76,7 @@ mod tests { // GitHub github_user: Vec::new(), github_organization: Vec::new(), + github_exclude: Vec::new(), all_github_organizations: false, github_api_url: Url::parse("https://api.github.com/").unwrap(), github_repo_type: GitHubRepoType::Source, @@ -83,6 +84,7 @@ mod tests { // GitLab gitlab_user: Vec::new(), gitlab_group: Vec::new(), + gitlab_exclude: Vec::new(), all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::All, diff --git a/src/scanner/repos.rs b/src/scanner/repos.rs index 6770434..c05e2c6 100644 --- a/src/scanner/repos.rs +++ b/src/scanner/repos.rs @@ -130,6 +130,7 @@ pub async fn enumerate_github_repos( organization: args.input_specifier_args.github_organization.clone(), all_organizations: args.input_specifier_args.all_github_organizations, repo_filter: args.input_specifier_args.github_repo_type.into(), + exclude_repos: args.input_specifier_args.github_exclude.clone(), }; let mut repo_urls = args.input_specifier_args.git_url.clone(); if !repo_specifiers.is_empty() { @@ -188,6 +189,7 @@ pub async fn enumerate_gitlab_repos( all_groups: args.input_specifier_args.all_gitlab_groups, include_subgroups: args.input_specifier_args.gitlab_include_subgroups, repo_filter: args.input_specifier_args.gitlab_repo_type.into(), + exclude_repos: args.input_specifier_args.gitlab_exclude.clone(), }; let mut repo_urls = args.input_specifier_args.git_url.clone(); diff --git a/tests/int_allowlist.rs b/tests/int_allowlist.rs index 0370755..ecc53ec 100644 --- a/tests/int_allowlist.rs +++ b/tests/int_allowlist.rs @@ -58,11 +58,13 @@ fn run_skiplist(skip_regex: Vec, skip_skipword: Vec) -> Result Result<()> { git_url: vec![git_url], github_user: Vec::new(), github_organization: Vec::new(), + github_exclude: Vec::new(), all_github_organizations: false, github_api_url: Url::parse("https://api.github.com/").unwrap(), github_repo_type: GitHubRepoType::Source, // new GitLab defaults gitlab_user: Vec::new(), gitlab_group: Vec::new(), + gitlab_exclude: Vec::new(), all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::Owner, diff --git a/tests/int_gitlab.rs b/tests/int_gitlab.rs index 9cfde7c..b048864 100644 --- a/tests/int_gitlab.rs +++ b/tests/int_gitlab.rs @@ -56,11 +56,13 @@ fn test_gitlab_remote_scan() -> Result<()> { git_url: vec![git_url], github_user: Vec::new(), github_organization: Vec::new(), + github_exclude: Vec::new(), all_github_organizations: false, github_api_url: Url::parse("https://api.github.com/")?, github_repo_type: GitHubRepoType::Source, gitlab_user: Vec::new(), gitlab_group: Vec::new(), + gitlab_exclude: Vec::new(), all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/")?, gitlab_repo_type: GitLabRepoType::Owner, @@ -166,11 +168,13 @@ fn test_gitlab_remote_scan_no_history() -> Result<()> { git_url: vec![git_url], github_user: Vec::new(), github_organization: Vec::new(), + github_exclude: Vec::new(), all_github_organizations: false, github_api_url: Url::parse("https://api.github.com/")?, github_repo_type: GitHubRepoType::Source, gitlab_user: Vec::new(), gitlab_group: Vec::new(), + gitlab_exclude: Vec::new(), all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/")?, gitlab_repo_type: GitLabRepoType::Owner, diff --git a/tests/int_redact.rs b/tests/int_redact.rs index 9be8c4a..2248d2d 100644 --- a/tests/int_redact.rs +++ b/tests/int_redact.rs @@ -41,11 +41,13 @@ async fn test_redact_hashes_finding_values() -> Result<()> { git_url: Vec::new(), github_user: Vec::new(), github_organization: Vec::new(), + github_exclude: Vec::new(), all_github_organizations: false, github_api_url: Url::parse("https://api.github.com/").unwrap(), github_repo_type: GitHubRepoType::Source, gitlab_user: Vec::new(), gitlab_group: Vec::new(), + gitlab_exclude: Vec::new(), all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::Owner, diff --git a/tests/int_slack.rs b/tests/int_slack.rs index 0bcae59..265fe06 100644 --- a/tests/int_slack.rs +++ b/tests/int_slack.rs @@ -47,11 +47,13 @@ impl TestContext { git_url: Vec::new(), github_user: Vec::new(), github_organization: Vec::new(), + github_exclude: Vec::new(), all_github_organizations: false, github_api_url: Url::parse("https://api.github.com/").unwrap(), github_repo_type: GitHubRepoType::Source, gitlab_user: Vec::new(), gitlab_group: Vec::new(), + gitlab_exclude: Vec::new(), all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::Owner, @@ -144,11 +146,13 @@ async fn test_scan_slack_messages() -> Result<()> { git_url: Vec::new(), github_user: Vec::new(), github_organization: Vec::new(), + github_exclude: Vec::new(), all_github_organizations: false, github_api_url: Url::parse("https://api.github.com/").unwrap(), github_repo_type: GitHubRepoType::Source, gitlab_user: Vec::new(), gitlab_group: Vec::new(), + gitlab_exclude: Vec::new(), all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::Owner, diff --git a/tests/int_validation_cache.rs b/tests/int_validation_cache.rs index 0cf2a08..ba1f5a7 100644 --- a/tests/int_validation_cache.rs +++ b/tests/int_validation_cache.rs @@ -111,6 +111,7 @@ async fn test_validation_cache_and_depvars() -> Result<()> { git_url: Vec::new(), github_user: Vec::new(), github_organization: Vec::new(), + github_exclude: Vec::new(), all_github_organizations: false, github_api_url: Url::parse("https://api.github.com/").unwrap(), github_repo_type: GitHubRepoType::Source, @@ -118,6 +119,7 @@ async fn test_validation_cache_and_depvars() -> Result<()> { // new GitLab defaults gitlab_user: Vec::new(), gitlab_group: Vec::new(), + gitlab_exclude: Vec::new(), all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::Owner, diff --git a/tests/int_vulnerable_files.rs b/tests/int_vulnerable_files.rs index c53b8fc..7eaa147 100644 --- a/tests/int_vulnerable_files.rs +++ b/tests/int_vulnerable_files.rs @@ -55,12 +55,14 @@ impl TestContext { git_url: Vec::new(), github_user: Vec::new(), github_organization: Vec::new(), + github_exclude: Vec::new(), all_github_organizations: false, github_api_url: Url::parse("https://api.github.com/").unwrap(), github_repo_type: GitHubRepoType::Source, // new GitLab defaults gitlab_user: Vec::new(), gitlab_group: Vec::new(), + gitlab_exclude: Vec::new(), all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::Owner, @@ -138,12 +140,14 @@ impl TestContext { git_url: Vec::new(), github_user: Vec::new(), github_organization: Vec::new(), + github_exclude: Vec::new(), all_github_organizations: false, github_api_url: Url::parse("https://api.github.com/").unwrap(), github_repo_type: GitHubRepoType::Source, // new GitLab defaults gitlab_user: Vec::new(), gitlab_group: Vec::new(), + gitlab_exclude: Vec::new(), all_gitlab_groups: false, gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), gitlab_repo_type: GitLabRepoType::Owner,