From cf45930e2cb233cb36a2b7f8e201a3fa994ba24e Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 5 Oct 2025 10:48:57 -0700 Subject: [PATCH] Added first-class Azure Repos support, including CLI commands, enumeration, and documentation updates. Fixed a few bugs. --- Cargo.toml | 6 +-- README.md | 5 +- src/azure.rs | 125 +++++++++++++++++++++++++++++++++++++++++----- src/git_binary.rs | 24 ++++++++- src/reporter.rs | 51 +++++++++++++++++++ 5 files changed, 192 insertions(+), 19 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 579d4e5..b8dd860 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ publish = false [package] name = "kingfisher" version = "1.55.0" -description = "MongoDB's blazingly fast secret scanning and validation tool" +description = "MongoDB's blazingly fast and accurate secret scanning and validation tool" edition.workspace = true rust-version.workspace = true license.workspace = true @@ -32,7 +32,7 @@ assets = [ [package.metadata.generate-rpm] package = "kingfisher" -summary = "MongoDB's blazingly fast secret scanning and validation tool" +summary = "MongoDB's blazingly fast and accurate secret scanning and validation tool" license = "Apache-2.0" url = "https://github.com/mongodb/kingfisher" assets = [ @@ -229,7 +229,7 @@ incremental = false [profile.dev] opt-level = 0 -debug = true +# debug = true incremental = true codegen-units = 256 diff --git a/README.md b/README.md index 4a5e62b..55707a0 100644 --- a/README.md +++ b/README.md @@ -596,8 +596,9 @@ kingfisher scan --azure-project my-org/payments --azure-project my-org/core-plat ### Skip specific Azure repositories during enumeration Repeat `--azure-exclude` to ignore repositories when scanning organizations or projects. -Use identifiers like `ORGANIZATION/PROJECT/REPOSITORY` or gitignore-style patterns such as -`my-org/*/archive-*`. +Use identifiers like `ORGANIZATION/PROJECT/REPOSITORY`. Repositories that share the same +name as their project can be excluded with `ORGANIZATION/PROJECT`, and gitignore-style +patterns such as `my-org/*/archive-*` are also supported. ```bash kingfisher scan --azure-organization my-org \ diff --git a/src/azure.rs b/src/azure.rs index e9dea55..b0d40d1 100644 --- a/src/azure.rs +++ b/src/azure.rs @@ -63,11 +63,12 @@ struct ExcludeMatcher { impl ExcludeMatcher { fn matches(&self, name: &str) -> bool { - if self.exact.contains(name) { + let candidate = name.to_lowercase(); + if self.exact.contains(&candidate) { return true; } if let Some(globs) = &self.globs { - return globs.is_match(name); + return globs.is_match(&candidate); } false } @@ -108,13 +109,36 @@ fn normalize_repo_identifier(parts: &[String]) -> Option { } fn parse_repo_identifier_from_path(path: &str) -> Option { - let segments: Vec = path + let mut segments: Vec = path .trim_matches('/') .split('/') .filter(|s| !s.is_empty()) .map(|s| s.to_string()) .collect(); + if segments.is_empty() { + return None; + } + + if segments.len() == 2 { + let org = segments.first()?.trim().trim_matches('/'); + let project = segments.last()?.trim().trim_matches('/'); + if org.is_empty() || project.is_empty() { + return None; + } + + let org = org.to_lowercase(); + let project_raw = project.to_string(); + if looks_like_glob(&project_raw) { + let pattern = format!("{org}/{}/**", project_raw.to_lowercase()); + return Some(pattern); + } + + let project_normalized = project_raw.trim_end_matches(".git").to_lowercase(); + let repo = project_normalized.clone(); + return Some(format!("{org}/{project_normalized}/{repo}")); + } + if segments.len() < 3 { return None; } @@ -181,23 +205,24 @@ fn build_exclude_matcher(exclude_repos: &[String]) -> ExcludeMatcher { for raw in exclude_repos { match parse_excluded_repo(raw) { Some(name) => { - if looks_like_glob(&name) { - match Glob::new(&name) { + let normalized = name.to_lowercase(); + if looks_like_glob(&normalized) { + match Glob::new(&normalized) { Ok(glob) => { glob_builder.add(glob); has_glob = true; } Err(err) => { warn!("Ignoring invalid Azure exclusion pattern '{raw}': {err}"); - exact.insert(name); + exact.insert(normalized); } } } else { - exact.insert(name); + exact.insert(normalized); } } None => { - warn!("Ignoring invalid Azure exclusion '{raw}' (expected organization/project/repository)"); + warn!("Ignoring invalid Azure exclusion '{raw}' (expected organization/project[/repository])"); } } } @@ -293,14 +318,50 @@ async fn fetch_repositories_for_org( let url = format!("{base}/{encoded_org}/_apis/git/repositories?api-version={API_VERSION}"); let request = auth.apply(client.get(&url)); let response = request.send().await?; - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - return Err(anyhow!( + let status = response.status(); + let headers = response.headers().clone(); + let body_bytes = response.bytes().await?; + + if !status.is_success() { + let body = String::from_utf8_lossy(&body_bytes).trim().to_string(); + let auth_hint = if matches!( + status, + reqwest::StatusCode::UNAUTHORIZED | reqwest::StatusCode::FORBIDDEN + ) { + if auth.token.is_some() { + "Verify that the Azure token or PAT has access to the requested organization and has not expired." + } else { + "Set KF_AZURE_TOKEN or KF_AZURE_PAT with an Azure DevOps Personal Access Token that can read repositories." + } + } else { + "" + }; + + let mut message = format!( "Azure Repos API request failed for organization '{organization}' ({status}): {body}" + ); + if !auth_hint.is_empty() { + message.push_str(&format!("\n{auth_hint}")); + } + return Err(anyhow!(message)); + } + + let is_json = headers + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(|value| { + value.split(';').next().unwrap_or("").trim().eq_ignore_ascii_case("application/json") + }) + .unwrap_or(false); + + if !is_json { + let body = String::from_utf8_lossy(&body_bytes); + return Err(anyhow!( + "Azure Repos API response for organization '{organization}' did not include JSON: {body}" )); } - let payload: AzureListResponse = response.json().await?; + + let payload: AzureListResponse = serde_json::from_slice(&body_bytes)?; Ok(payload.value) } @@ -561,12 +622,50 @@ mod tests { assert_eq!(ident, "org/project/repo"); } + #[test] + fn parse_excluded_repo_allows_project_alias() { + let raw = "Org/Project"; + let ident = parse_excluded_repo(raw).expect("identifier"); + assert_eq!(ident, "org/project/project"); + } + + #[test] + fn parse_excluded_repo_allows_project_glob() { + let raw = "org/*"; + let ident = parse_excluded_repo(raw).expect("identifier"); + assert_eq!(ident, "org/*/**"); + } + #[test] fn exclude_matcher_matches_glob() { let matcher = build_exclude_matcher(&["org/*/repo".to_string()]); assert!(should_exclude_repo("https://dev.azure.com/org/project/_git/repo", &matcher)); } + #[test] + fn exclude_matcher_matches_project_alias() { + let matcher = build_exclude_matcher(&["org/project".to_string()]); + assert!(should_exclude_repo("https://dev.azure.com/org/project/_git/project", &matcher)); + } + + #[test] + fn exclude_matcher_matches_project_glob() { + let matcher = build_exclude_matcher(&["org/*".to_string()]); + assert!(should_exclude_repo("https://dev.azure.com/org/project/_git/repo", &matcher)); + } + + #[test] + fn exclude_matcher_is_case_insensitive_for_exact_matches() { + let matcher = build_exclude_matcher(&["Org/Project/Repo".to_string()]); + assert!(should_exclude_repo("https://dev.azure.com/org/project/_git/repo", &matcher)); + } + + #[test] + fn exclude_matcher_is_case_insensitive_for_globs() { + let matcher = build_exclude_matcher(&["ORG/*".to_string()]); + assert!(should_exclude_repo("https://dev.azure.com/org/project/_git/repo", &matcher)); + } + #[test] fn wiki_url_appends_suffix() { let url = GitUrl::from_str("https://dev.azure.com/org/project/_git/repo").unwrap(); diff --git a/src/git_binary.rs b/src/git_binary.rs index 09f6658..82fd990 100644 --- a/src/git_binary.rs +++ b/src/git_binary.rs @@ -31,6 +31,15 @@ const GITEA_CREDENTIAL_HELPER: &str = r#"credential.helper=!_gteacreds() { fi }; _gteacreds"#; +const AZURE_CREDENTIAL_HELPER: &str = r#"credential.helper=!_azcreds() { + token="${KF_AZURE_TOKEN:-${KF_AZURE_PAT:-}}"; + if [ -n "$token" ]; then + user="${KF_AZURE_USERNAME:-pat}"; + echo username="$user"; + echo password="$token"; + fi +}; _azcreds"#; + /// Represents errors that can occur when interacting with the `git` CLI. #[derive(Debug, thiserror::Error)] pub enum GitError { @@ -79,9 +88,17 @@ impl Git { matches!(std::env::var("KF_BITBUCKET_OAUTH_TOKEN"), Ok(value) if !value.is_empty()); let has_bitbucket_credentials = has_bitbucket_oauth_token || (has_bitbucket_username && has_bitbucket_password); + let has_azure_token = ["KF_AZURE_TOKEN", "KF_AZURE_PAT"] + .iter() + .any(|key| matches!(std::env::var(key), Ok(value) if !value.is_empty())); // If credentials are provided via environment variables, clear existing helpers first. - if has_github_token || has_gitlab_token || has_gitea_token || has_bitbucket_credentials { + if has_github_token + || has_gitlab_token + || has_gitea_token + || has_bitbucket_credentials + || has_azure_token + { credentials.push("-c".into()); credentials.push(r#"credential.helper="#.into()); } @@ -114,6 +131,11 @@ impl Git { credentials.push(BITBUCKET_CREDENTIAL_HELPER.into()); } + if has_azure_token { + credentials.push("-c".into()); + credentials.push(AZURE_CREDENTIAL_HELPER.into()); + } + Self { credentials, ignore_certs } } diff --git a/src/reporter.rs b/src/reporter.rs index 764dcfd..3344a11 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -48,6 +48,19 @@ const BITBUCKET_FRAGMENT_ENCODE_SET: &AsciiSet = &CONTROLS .add(b'}') .add(b'|'); +const AZURE_QUERY_ENCODE_SET: &AsciiSet = &CONTROLS + .add(b' ') + .add(b'"') + .add(b'#') + .add(b'%') + .add(b'<') + .add(b'>') + .add(b'?') + .add(b'`') + .add(b'{') + .add(b'}') + .add(b'|'); + fn build_git_urls( repo_url: &str, commit_id: &str, @@ -94,12 +107,50 @@ fn build_git_urls( commit_url = format!("{base}/commits/{commit_id}"); file_url = format!("{base}/commits/{commit_id}#L{anchor}F{line}"); } + } else if host.eq_ignore_ascii_case("dev.azure.com") || host.ends_with(".visualstudio.com") + { + let normalized = file_path.replace('\\', "/"); + let trimmed = normalized.trim_start_matches('/'); + let encoded_path = utf8_percent_encode(trimmed, AZURE_QUERY_ENCODE_SET).to_string(); + repository_url = repo_url.to_string(); + commit_url = format!("{repo_url}/commit/{commit_id}"); + if line > 0 { + file_url = + format!("{repo_url}/commit/{commit_id}?path=/{}&line={line}", encoded_path); + } else { + file_url = format!("{repo_url}/commit/{commit_id}?path=/{}", encoded_path); + } } } (repository_url, commit_url, file_url) } +#[cfg(test)] +mod tests { + use super::build_git_urls; + + #[test] + fn azure_commit_links_use_query_paths() { + let (repo_url, commit_url, file_url) = build_git_urls( + "https://dev.azure.com/org/project/_git/repo", + "0123456789abcdef", + "dir/file.txt", + 7, + ); + + assert_eq!(repo_url, "https://dev.azure.com/org/project/_git/repo"); + assert_eq!( + commit_url, + "https://dev.azure.com/org/project/_git/repo/commit/0123456789abcdef" + ); + assert_eq!( + file_url, + "https://dev.azure.com/org/project/_git/repo/commit/0123456789abcdef?path=/dir/file.txt&line=7" + ); + } +} + pub fn run( global_args: &GlobalArgs, ds: Arc>,