eblume/kingfisher
1
0
Fork 0
forked from mirrors/kingfisher
Code Pull requests Activity Actions

Added first-class Azure Repos support, including CLI commands, enumeration, and documentation updates. Fixed a few bugs.

Browse source
This commit is contained in:
Mick Grove 2025-10-05 10:48:57 -07:00
commit cf45930e2c
5 changed files with 192 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

6
Cargo.toml
View file

@ -11,7 +11,7 @@ publish = false
[package]
name = "kingfisher"
version = "1.55.0"
description = "MongoDB's blazingly fast secret scanning and validation tool"
description = "MongoDB's blazingly fast and accurate secret scanning and validation tool"
edition.workspace = true
rust-version.workspace = true
license.workspace = true
@ -32,7 +32,7 @@ assets = [
[package.metadata.generate-rpm]
package = "kingfisher"
summary = "MongoDB's blazingly fast secret scanning and validation tool"
summary = "MongoDB's blazingly fast and accurate secret scanning and validation tool"
license = "Apache-2.0"
url = "https://github.com/mongodb/kingfisher"
assets = [
@ -229,7 +229,7 @@ incremental = false
[profile.dev]
opt-level = 0
debug = true
# debug = true
incremental = true
codegen-units = 256

5
README.md
View file

@ -596,8 +596,9 @@ kingfisher scan --azure-project my-org/payments --azure-project my-org/core-plat
### Skip specific Azure repositories during enumeration
Repeat `--azure-exclude` to ignore repositories when scanning organizations or projects.
Use identifiers like `ORGANIZATION/PROJECT/REPOSITORY` or gitignore-style patterns such as
`my-org/*/archive-*`.
Use identifiers like `ORGANIZATION/PROJECT/REPOSITORY`. Repositories that share the same
name as their project can be excluded with `ORGANIZATION/PROJECT`, and gitignore-style
patterns such as `my-org/*/archive-*` are also supported.
```bash
kingfisher scan --azure-organization my-org \

125
src/azure.rs
View file

@ -63,11 +63,12 @@ struct ExcludeMatcher {
impl ExcludeMatcher {
fn matches(&self, name: &str) -> bool {
if self.exact.contains(name) {
let candidate = name.to_lowercase();
if self.exact.contains(&candidate) {
return true;
}
if let Some(globs) = &self.globs {
return globs.is_match(name);
return globs.is_match(&candidate);
}
false
}
@ -108,13 +109,36 @@ fn normalize_repo_identifier(parts: &[String]) -> Option<String> {
}
fn parse_repo_identifier_from_path(path: &str) -> Option<String> {
let segments: Vec<String> = path
let mut segments: Vec<String> = path
.trim_matches('/')
.split('/')
.filter(|s| !s.is_empty())
.map(|s| s.to_string())
.collect();
if segments.is_empty() {
return None;
}
if segments.len() == 2 {
let org = segments.first()?.trim().trim_matches('/');
let project = segments.last()?.trim().trim_matches('/');
if org.is_empty() || project.is_empty() {
return None;
}
let org = org.to_lowercase();
let project_raw = project.to_string();
if looks_like_glob(&project_raw) {
let pattern = format!("{org}/{}/**", project_raw.to_lowercase());
return Some(pattern);
}
let project_normalized = project_raw.trim_end_matches(".git").to_lowercase();
let repo = project_normalized.clone();
return Some(format!("{org}/{project_normalized}/{repo}"));
}
if segments.len() < 3 {
return None;
}
@ -181,23 +205,24 @@ fn build_exclude_matcher(exclude_repos: &[String]) -> ExcludeMatcher {
for raw in exclude_repos {
match parse_excluded_repo(raw) {
Some(name) => {
if looks_like_glob(&name) {
match Glob::new(&name) {
let normalized = name.to_lowercase();
if looks_like_glob(&normalized) {
match Glob::new(&normalized) {
Ok(glob) => {
glob_builder.add(glob);
has_glob = true;
}
Err(err) => {
warn!("Ignoring invalid Azure exclusion pattern '{raw}': {err}");
exact.insert(name);
exact.insert(normalized);
}
}
} else {
exact.insert(name);
exact.insert(normalized);
}
}
None => {
warn!("Ignoring invalid Azure exclusion '{raw}' (expected organization/project/repository)");
warn!("Ignoring invalid Azure exclusion '{raw}' (expected organization/project[/repository])");
}
}
}
@ -293,14 +318,50 @@ async fn fetch_repositories_for_org(
let url = format!("{base}/{encoded_org}/_apis/git/repositories?api-version={API_VERSION}");
let request = auth.apply(client.get(&url));
let response = request.send().await?;
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
return Err(anyhow!(
let status = response.status();
let headers = response.headers().clone();
let body_bytes = response.bytes().await?;
if !status.is_success() {
let body = String::from_utf8_lossy(&body_bytes).trim().to_string();
let auth_hint = if matches!(
status,
reqwest::StatusCode::UNAUTHORIZED | reqwest::StatusCode::FORBIDDEN
) {
if auth.token.is_some() {
"Verify that the Azure token or PAT has access to the requested organization and has not expired."
} else {
"Set KF_AZURE_TOKEN or KF_AZURE_PAT with an Azure DevOps Personal Access Token that can read repositories."
}
} else {
""
};
let mut message = format!(
"Azure Repos API request failed for organization '{organization}' ({status}): {body}"
);
if !auth_hint.is_empty() {
message.push_str(&format!("\n{auth_hint}"));
}
return Err(anyhow!(message));
}
let is_json = headers
.get(reqwest::header::CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.map(|value| {
value.split(';').next().unwrap_or("").trim().eq_ignore_ascii_case("application/json")
})
.unwrap_or(false);
if !is_json {
let body = String::from_utf8_lossy(&body_bytes);
return Err(anyhow!(
"Azure Repos API response for organization '{organization}' did not include JSON: {body}"
));
}
let payload: AzureListResponse<AzureRepository> = response.json().await?;
let payload: AzureListResponse<AzureRepository> = serde_json::from_slice(&body_bytes)?;
Ok(payload.value)
}
@ -561,12 +622,50 @@ mod tests {
assert_eq!(ident, "org/project/repo");
}
#[test]
fn parse_excluded_repo_allows_project_alias() {
let raw = "Org/Project";
let ident = parse_excluded_repo(raw).expect("identifier");
assert_eq!(ident, "org/project/project");
}
#[test]
fn parse_excluded_repo_allows_project_glob() {
let raw = "org/*";
let ident = parse_excluded_repo(raw).expect("identifier");
assert_eq!(ident, "org/*/**");
}
#[test]
fn exclude_matcher_matches_glob() {
let matcher = build_exclude_matcher(&["org/*/repo".to_string()]);
assert!(should_exclude_repo("https://dev.azure.com/org/project/_git/repo", &matcher));
}
#[test]
fn exclude_matcher_matches_project_alias() {
let matcher = build_exclude_matcher(&["org/project".to_string()]);
assert!(should_exclude_repo("https://dev.azure.com/org/project/_git/project", &matcher));
}
#[test]
fn exclude_matcher_matches_project_glob() {
let matcher = build_exclude_matcher(&["org/*".to_string()]);
assert!(should_exclude_repo("https://dev.azure.com/org/project/_git/repo", &matcher));
}
#[test]
fn exclude_matcher_is_case_insensitive_for_exact_matches() {
let matcher = build_exclude_matcher(&["Org/Project/Repo".to_string()]);
assert!(should_exclude_repo("https://dev.azure.com/org/project/_git/repo", &matcher));
}
#[test]
fn exclude_matcher_is_case_insensitive_for_globs() {
let matcher = build_exclude_matcher(&["ORG/*".to_string()]);
assert!(should_exclude_repo("https://dev.azure.com/org/project/_git/repo", &matcher));
}
#[test]
fn wiki_url_appends_suffix() {
let url = GitUrl::from_str("https://dev.azure.com/org/project/_git/repo").unwrap();

24
src/git_binary.rs
View file

@ -31,6 +31,15 @@ const GITEA_CREDENTIAL_HELPER: &str = r#"credential.helper=!_gteacreds() {
fi
}; _gteacreds"#;
const AZURE_CREDENTIAL_HELPER: &str = r#"credential.helper=!_azcreds() {
token="${KF_AZURE_TOKEN:-${KF_AZURE_PAT:-}}";
if [ -n "$token" ]; then
user="${KF_AZURE_USERNAME:-pat}";
echo username="$user";
echo password="$token";
fi
}; _azcreds"#;
/// Represents errors that can occur when interacting with the `git` CLI.
#[derive(Debug, thiserror::Error)]
pub enum GitError {
@ -79,9 +88,17 @@ impl Git {
matches!(std::env::var("KF_BITBUCKET_OAUTH_TOKEN"), Ok(value) if !value.is_empty());
let has_bitbucket_credentials =
has_bitbucket_oauth_token || (has_bitbucket_username && has_bitbucket_password);
let has_azure_token = ["KF_AZURE_TOKEN", "KF_AZURE_PAT"]
.iter()
.any(|key| matches!(std::env::var(key), Ok(value) if !value.is_empty()));
// If credentials are provided via environment variables, clear existing helpers first.
if has_github_token || has_gitlab_token || has_gitea_token || has_bitbucket_credentials {
if has_github_token
|| has_gitlab_token
|| has_gitea_token
|| has_bitbucket_credentials
|| has_azure_token
{
credentials.push("-c".into());
credentials.push(r#"credential.helper="#.into());
}
@ -114,6 +131,11 @@ impl Git {
credentials.push(BITBUCKET_CREDENTIAL_HELPER.into());
}
if has_azure_token {
credentials.push("-c".into());
credentials.push(AZURE_CREDENTIAL_HELPER.into());
}
Self { credentials, ignore_certs }
}

51
src/reporter.rs
View file

@ -48,6 +48,19 @@ const BITBUCKET_FRAGMENT_ENCODE_SET: &AsciiSet = &CONTROLS
.add(b'}')
.add(b'|');
const AZURE_QUERY_ENCODE_SET: &AsciiSet = &CONTROLS
.add(b' ')
.add(b'"')
.add(b'#')
.add(b'%')
.add(b'<')
.add(b'>')
.add(b'?')
.add(b'`')
.add(b'{')
.add(b'}')
.add(b'|');
fn build_git_urls(
repo_url: &str,
commit_id: &str,
@ -94,12 +107,50 @@ fn build_git_urls(
commit_url = format!("{base}/commits/{commit_id}");
file_url = format!("{base}/commits/{commit_id}#L{anchor}F{line}");
}
} else if host.eq_ignore_ascii_case("dev.azure.com") || host.ends_with(".visualstudio.com")
{
let normalized = file_path.replace('\\', "/");
let trimmed = normalized.trim_start_matches('/');
let encoded_path = utf8_percent_encode(trimmed, AZURE_QUERY_ENCODE_SET).to_string();
repository_url = repo_url.to_string();
commit_url = format!("{repo_url}/commit/{commit_id}");
if line > 0 {
file_url =
format!("{repo_url}/commit/{commit_id}?path=/{}&line={line}", encoded_path);
} else {
file_url = format!("{repo_url}/commit/{commit_id}?path=/{}", encoded_path);
}
}
}
(repository_url, commit_url, file_url)
}
#[cfg(test)]
mod tests {
use super::build_git_urls;
#[test]
fn azure_commit_links_use_query_paths() {
let (repo_url, commit_url, file_url) = build_git_urls(
"https://dev.azure.com/org/project/_git/repo",
"0123456789abcdef",
"dir/file.txt",
7,
);
assert_eq!(repo_url, "https://dev.azure.com/org/project/_git/repo");
assert_eq!(
commit_url,
"https://dev.azure.com/org/project/_git/repo/commit/0123456789abcdef"
);
assert_eq!(
file_url,
"https://dev.azure.com/org/project/_git/repo/commit/0123456789abcdef?path=/dir/file.txt&line=7"
);
}
}
pub fn run(
global_args: &GlobalArgs,
ds: Arc<Mutex<findings_store::FindingsStore>>,