forked from mirrors/kingfisher
Added support for BitBucket
This commit is contained in:
parent
19cca00c2b
commit
5c70fdc8e5
26 changed files with 1544 additions and 38 deletions
|
|
@ -1,6 +1,10 @@
|
|||
# Changelog
|
||||
|
||||
All notable changes to this project will be documented in this file.
|
||||
|
||||
## [v1.53.0]
|
||||
- Added first-class Bitbucket support, including CLI commands, authentication helpers, documentation, and integration testing.
|
||||
|
||||
## [v1.52.0]
|
||||
- Enabled ANSI formatting in the tracing formatter whenever stderr is attached to a terminal so colorized updater messages render correctly instead of showing escape sequences.
|
||||
- Added a new CLI flag, `--user-agent-suffix` to allow developers to append additional information to the user-agent
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ publish = false
|
|||
|
||||
[package]
|
||||
name = "kingfisher"
|
||||
version = "1.52.0"
|
||||
version = "1.53.0"
|
||||
description = "MongoDB's blazingly fast secret scanning and validation tool"
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
|
|
|||
89
README.md
89
README.md
|
|
@ -15,8 +15,8 @@ Originally forked from Praetorian’s Nosey Parker, Kingfisher adds live cloud-A
|
|||
- **Extensible rules**: hundreds of built-in detectors plus YAML-defined custom rules ([docs/RULES.md](/docs/RULES.md))
|
||||
- **Broad AI SaaS coverage**: finds and validates tokens for OpenAI, Anthropic, Google Gemini, Cohere, Mistral, Stability AI, Replicate, xAI (Grok), Ollama, Langchain, Perplexity, Weights & Biases, Cerebras, Friendli, Fireworks.ai, NVIDIA NIM, Together.ai, Zhipu, and many more
|
||||
- **Multiple targets**:
|
||||
- **Git history**: local repos or GitHub/GitLab orgs/users
|
||||
- **Repository artifacts**: with `--repo-artifacts`, scan GitHub/GitLab repository artifacts such as issues, pull/merge requests, wikis, snippets, and owner gists in addition to code
|
||||
- **Git history**: local repos or GitHub/GitLab/Bitbucket orgs, users, and workspaces
|
||||
- **Repository artifacts**: with `--repo-artifacts`, scan GitHub/GitLab/Bitbucket repository artifacts such as issues, pull/merge requests, wikis, snippets, and owner gists in addition to code
|
||||
- **Docker images**: public or private via `--docker-image`
|
||||
- **Jira issues**: JQL‑driven scans with `--jira-url` and `--jql`
|
||||
- **Confluence pages**: CQL‑driven scans with `--confluence-url` and `--cql`
|
||||
|
|
@ -71,6 +71,14 @@ See ([docs/COMPARISON.md](docs/COMPARISON.md))
|
|||
- [Skip specific GitLab projects during enumeration](#skip-specific-gitlab-projects-during-enumeration)
|
||||
- [Scan remote GitLab repository by URL](#scan-remote-gitlab-repository-by-url)
|
||||
- [List GitLab repositories](#list-gitlab-repositories)
|
||||
- [Scanning Bitbucket](#scanning-bitbucket)
|
||||
- [Scan Bitbucket workspace](#scan-bitbucket-workspace)
|
||||
- [Scan Bitbucket user](#scan-bitbucket-user)
|
||||
- [Skip specific Bitbucket repositories during enumeration](#skip-specific-bitbucket-repositories-during-enumeration)
|
||||
- [Scan remote Bitbucket repository by URL](#scan-remote-bitbucket-repository-by-url)
|
||||
- [List Bitbucket repositories](#list-bitbucket-repositories)
|
||||
- [Authenticate to Bitbucket](#authenticate-to-bitbucket)
|
||||
- [Self-hosted Bitbucket Server](#self-hosted-bitbucket-server)
|
||||
- [Scanning Jira](#scanning-jira)
|
||||
- [Scan Jira issues matching a JQL query](#scan-jira-issues-matching-a-jql-query)
|
||||
- [Scan the last 1,000 Jira issues:](#scan-the-last-1000-jira-issues)
|
||||
|
|
@ -552,6 +560,80 @@ kingfisher gitlab repos list --group my-group --include-subgroups
|
|||
kingfisher gitlab repos list --group my-group --gitlab-exclude my-group/**/legacy-*
|
||||
```
|
||||
|
||||
## Scanning Bitbucket
|
||||
|
||||
### Scan Bitbucket workspace
|
||||
|
||||
```bash
|
||||
kingfisher scan --bitbucket-workspace my-team
|
||||
# include Bitbucket Cloud repositories from every accessible workspace
|
||||
kingfisher scan --all-bitbucket-workspaces --bitbucket-token "$APP_PASSWORD" --bitbucket-username "$USER"
|
||||
```
|
||||
|
||||
### Scan Bitbucket user
|
||||
|
||||
```bash
|
||||
kingfisher scan --bitbucket-user johndoe
|
||||
```
|
||||
|
||||
### Skip specific Bitbucket repositories during enumeration
|
||||
|
||||
Use `--bitbucket-exclude` to ignore repositories while scanning users, workspaces,
|
||||
or projects. Patterns accept either `owner/repo` (case-insensitive) or
|
||||
gitignore-style globs such as `workspace/**/archive-*`.
|
||||
|
||||
```bash
|
||||
kingfisher scan --bitbucket-workspace my-team \
|
||||
--bitbucket-exclude my-team/legacy-repo \
|
||||
--bitbucket-exclude my-team/**/archive-*
|
||||
```
|
||||
|
||||
### Scan remote Bitbucket repository by URL
|
||||
|
||||
`--git-url` clones the repository and scans its files and history. To inspect
|
||||
Bitbucket artifacts such as issues, add `--repo-artifacts`. Private artifacts
|
||||
require credentials (see [Authenticate to Bitbucket](#authenticate-to-bitbucket)).
|
||||
|
||||
```bash
|
||||
# Scan the repository only
|
||||
kingfisher scan --git-url https://bitbucket.org/hashashash/secretstest.git
|
||||
|
||||
# Include repository issues
|
||||
KF_BITBUCKET_USERNAME="user" \
|
||||
KF_BITBUCKET_APP_PASSWORD="app-password" \
|
||||
kingfisher scan --git-url https://bitbucket.org/workspace/project.git --repo-artifacts
|
||||
```
|
||||
|
||||
### List Bitbucket repositories
|
||||
|
||||
```bash
|
||||
kingfisher bitbucket repos list --bitbucket-workspace my-team
|
||||
# enumerate all accessible workspaces or projects
|
||||
kingfisher bitbucket repos list --all-bitbucket-workspaces --bitbucket-token "$APP_PASSWORD" --bitbucket-username "$USER"
|
||||
# filter out repositories using glob patterns
|
||||
kingfisher bitbucket repos list --bitbucket-workspace my-team --bitbucket-exclude my-team/**/experimental-*
|
||||
```
|
||||
|
||||
### Authenticate to Bitbucket
|
||||
|
||||
Kingfisher supports Bitbucket Cloud and Bitbucket Server credentials:
|
||||
|
||||
- **App password or server token** – set `KF_BITBUCKET_USERNAME` and either
|
||||
`KF_BITBUCKET_APP_PASSWORD` or `KF_BITBUCKET_TOKEN`, or pass
|
||||
`--bitbucket-username`/`--bitbucket-token` on the CLI.
|
||||
- **OAuth/PAT token** – set `KF_BITBUCKET_OAUTH_TOKEN` or supply
|
||||
`--bitbucket-oauth-token`.
|
||||
|
||||
These credentials match the options described in the [ghorg setup
|
||||
guide](https://github.com/gabrie30/ghorg/blob/master/README.md#bitbucket-setup).
|
||||
|
||||
### Self-hosted Bitbucket Server
|
||||
|
||||
Use `--bitbucket-api-url` to point Kingfisher at your server's REST endpoint, for example
|
||||
`https://bitbucket.example.com/rest/api/1.0/`. Provide credentials with
|
||||
`--bitbucket-username` and `--bitbucket-token`, and pass `--ignore-certs` when
|
||||
connecting to HTTP or otherwise insecure instances.
|
||||
|
||||
## Scanning Jira
|
||||
|
||||
### Scan Jira issues matching a JQL query
|
||||
|
|
@ -618,6 +700,9 @@ KF_SLACK_TOKEN="xoxp-1234..." kingfisher scan \
|
|||
| ----------------- | ---------------------------- |
|
||||
| `KF_GITHUB_TOKEN` | GitHub Personal Access Token |
|
||||
| `KF_GITLAB_TOKEN` | GitLab Personal Access Token |
|
||||
| `KF_BITBUCKET_USERNAME` | Bitbucket username for basic authentication |
|
||||
| `KF_BITBUCKET_APP_PASSWORD` / `KF_BITBUCKET_TOKEN` | Bitbucket app password or server token |
|
||||
| `KF_BITBUCKET_OAUTH_TOKEN` | Bitbucket OAuth or PAT token |
|
||||
| `KF_JIRA_TOKEN` | Jira API token |
|
||||
| `KF_CONFLUENCE_TOKEN` | Confluence API token |
|
||||
| `KF_SLACK_TOKEN` | Slack API token |
|
||||
|
|
|
|||
708
src/bitbucket.rs
Normal file
708
src/bitbucket.rs
Normal file
|
|
@ -0,0 +1,708 @@
|
|||
use std::{
|
||||
collections::HashSet,
|
||||
env, fs,
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Mutex},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use globset::{Glob, GlobSet, GlobSetBuilder};
|
||||
use indicatif::{ProgressBar, ProgressStyle};
|
||||
use reqwest::Url;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
use tracing::warn;
|
||||
|
||||
use crate::{findings_store, git_url::GitUrl, validation::GLOBAL_USER_AGENT};
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum RepoType {
|
||||
All,
|
||||
Source,
|
||||
Fork,
|
||||
}
|
||||
|
||||
impl RepoType {
|
||||
fn allows(self, is_fork: bool) -> bool {
|
||||
match self {
|
||||
RepoType::All => true,
|
||||
RepoType::Source => !is_fork,
|
||||
RepoType::Fork => is_fork,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct AuthConfig {
|
||||
pub username: Option<String>,
|
||||
pub password: Option<String>,
|
||||
pub bearer_token: Option<String>,
|
||||
}
|
||||
|
||||
impl AuthConfig {
|
||||
pub fn from_options(
|
||||
username: Option<String>,
|
||||
password: Option<String>,
|
||||
bearer_token: Option<String>,
|
||||
) -> Self {
|
||||
let username = username.or_else(|| env::var("KF_BITBUCKET_USERNAME").ok());
|
||||
let password = password
|
||||
.or_else(|| env::var("KF_BITBUCKET_APP_PASSWORD").ok())
|
||||
.or_else(|| env::var("KF_BITBUCKET_TOKEN").ok())
|
||||
.or_else(|| env::var("KF_BITBUCKET_PASSWORD").ok());
|
||||
let bearer_token = bearer_token.or_else(|| env::var("KF_BITBUCKET_OAUTH_TOKEN").ok());
|
||||
Self { username, password, bearer_token }
|
||||
}
|
||||
|
||||
fn apply(&self, request: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
|
||||
if let Some(token) = &self.bearer_token {
|
||||
request.bearer_auth(token)
|
||||
} else if let (Some(username), Some(password)) = (&self.username, &self.password) {
|
||||
request.basic_auth(username, Some(password))
|
||||
} else {
|
||||
request
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct RepoSpecifiers {
|
||||
pub user: Vec<String>,
|
||||
pub workspace: Vec<String>,
|
||||
pub project: Vec<String>,
|
||||
pub all_workspaces: bool,
|
||||
pub repo_filter: RepoType,
|
||||
pub exclude_repos: Vec<String>,
|
||||
}
|
||||
|
||||
impl RepoSpecifiers {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.user.is_empty()
|
||||
&& self.workspace.is_empty()
|
||||
&& self.project.is_empty()
|
||||
&& !self.all_workspaces
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum BitbucketKind {
|
||||
Cloud,
|
||||
Server,
|
||||
}
|
||||
|
||||
impl BitbucketKind {
|
||||
fn from_url(api_url: &Url) -> Self {
|
||||
let host = api_url.host_str().unwrap_or_default();
|
||||
if host.eq_ignore_ascii_case("api.bitbucket.org") || api_url.path().contains("/2.0") {
|
||||
BitbucketKind::Cloud
|
||||
} else {
|
||||
BitbucketKind::Server
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ExcludeMatcher {
|
||||
exact: HashSet<String>,
|
||||
globs: Option<GlobSet>,
|
||||
}
|
||||
|
||||
impl ExcludeMatcher {
|
||||
fn matches(&self, name: &str) -> bool {
|
||||
if self.exact.contains(name) {
|
||||
return true;
|
||||
}
|
||||
if let Some(globs) = &self.globs {
|
||||
return globs.is_match(name);
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn is_empty(&self) -> bool {
|
||||
self.exact.is_empty() && self.globs.is_none()
|
||||
}
|
||||
}
|
||||
|
||||
fn looks_like_glob(pattern: &str) -> bool {
|
||||
pattern.contains('*') || pattern.contains('?') || pattern.contains('[')
|
||||
}
|
||||
|
||||
fn normalize_repo_identifier(owner: &str, repo: &str) -> Option<String> {
|
||||
let owner = owner.trim().trim_matches('/');
|
||||
let repo = repo.trim().trim_matches('/');
|
||||
let repo = repo.strip_suffix(".git").unwrap_or(repo);
|
||||
if owner.is_empty() || repo.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(format!("{}/{}", owner.to_lowercase(), repo.to_lowercase()))
|
||||
}
|
||||
|
||||
fn parse_repo_name_from_path(path: &str) -> Option<String> {
|
||||
let parts: Vec<&str> =
|
||||
path.trim_matches('/').split('/').filter(|segment| !segment.is_empty()).collect();
|
||||
if parts.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
let repo = parts.last().unwrap();
|
||||
let owner = parts.get(parts.len() - 2).unwrap();
|
||||
normalize_repo_identifier(owner, repo)
|
||||
}
|
||||
|
||||
fn parse_repo_name_from_url(repo_url: &str) -> Option<String> {
|
||||
let url = Url::parse(repo_url).ok()?;
|
||||
parse_repo_name_from_path(url.path())
|
||||
}
|
||||
|
||||
fn parse_excluded_repo(raw: &str) -> Option<String> {
|
||||
let trimmed = raw.trim();
|
||||
if trimmed.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(name) = parse_repo_name_from_url(trimmed) {
|
||||
return Some(name);
|
||||
}
|
||||
|
||||
if let Some(idx) = trimmed.rfind(':') {
|
||||
if let Some(name) = parse_repo_name_from_path(&trimmed[idx + 1..]) {
|
||||
return Some(name);
|
||||
}
|
||||
}
|
||||
|
||||
parse_repo_name_from_path(trimmed)
|
||||
}
|
||||
|
||||
fn build_exclude_matcher(exclude_repos: &[String]) -> ExcludeMatcher {
|
||||
let mut exact = HashSet::new();
|
||||
let mut glob_builder = GlobSetBuilder::new();
|
||||
let mut has_glob = false;
|
||||
|
||||
for raw in exclude_repos {
|
||||
match parse_excluded_repo(raw) {
|
||||
Some(name) => {
|
||||
if looks_like_glob(&name) {
|
||||
match Glob::new(&name) {
|
||||
Ok(glob) => {
|
||||
glob_builder.add(glob);
|
||||
has_glob = true;
|
||||
}
|
||||
Err(err) => {
|
||||
warn!("Ignoring invalid Bitbucket exclusion pattern '{raw}': {err}");
|
||||
exact.insert(name);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
exact.insert(name);
|
||||
}
|
||||
}
|
||||
None => {
|
||||
warn!("Ignoring invalid Bitbucket exclusion '{raw}' (expected owner/repo)");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let globs = if has_glob {
|
||||
match glob_builder.build() {
|
||||
Ok(set) => Some(set),
|
||||
Err(err) => {
|
||||
warn!("Failed to build Bitbucket exclusion patterns: {err}");
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
ExcludeMatcher { exact, globs }
|
||||
}
|
||||
|
||||
fn should_exclude_repo(clone_url: &str, excludes: &ExcludeMatcher) -> bool {
|
||||
if excludes.is_empty() {
|
||||
return false;
|
||||
}
|
||||
if let Some(name) = parse_repo_name_from_url(clone_url) {
|
||||
return excludes.matches(&name);
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn repo_clone_url_from_links(links: &[CloneLink]) -> Option<String> {
|
||||
links
|
||||
.iter()
|
||||
.find(|link| link.name.as_deref().map(|n| n.eq_ignore_ascii_case("https")).unwrap_or(false))
|
||||
.or_else(|| links.first())
|
||||
.map(|link| link.href.clone())
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CloneLink {
|
||||
href: String,
|
||||
name: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CloudRepoLinks {
|
||||
#[serde(default)]
|
||||
clone: Vec<CloneLink>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CloudRepo {
|
||||
links: CloudRepoLinks,
|
||||
#[serde(default)]
|
||||
parent: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CloudRepoList {
|
||||
values: Vec<CloudRepo>,
|
||||
#[serde(default)]
|
||||
next: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CloudWorkspaceList {
|
||||
values: Vec<CloudWorkspace>,
|
||||
#[serde(default)]
|
||||
next: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CloudWorkspace {
|
||||
slug: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ServerRepo {
|
||||
links: CloudRepoLinks,
|
||||
#[serde(default)]
|
||||
origin: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ServerRepoList {
|
||||
values: Vec<ServerRepo>,
|
||||
#[serde(default, rename = "isLastPage")]
|
||||
is_last_page: bool,
|
||||
#[serde(default, rename = "nextPageStart")]
|
||||
next_page_start: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ServerProjectList {
|
||||
values: Vec<ServerProject>,
|
||||
#[serde(default, rename = "isLastPage")]
|
||||
is_last_page: bool,
|
||||
#[serde(default, rename = "nextPageStart")]
|
||||
next_page_start: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ServerProject {
|
||||
key: String,
|
||||
}
|
||||
|
||||
async fn fetch_cloud_repositories(
|
||||
client: &reqwest::Client,
|
||||
base: &Url,
|
||||
owner: &str,
|
||||
auth: &AuthConfig,
|
||||
repo_filter: RepoType,
|
||||
excludes: &ExcludeMatcher,
|
||||
results: &mut Vec<String>,
|
||||
) -> Result<()> {
|
||||
let mut next = base
|
||||
.join(&format!("repositories/{owner}?pagelen=100"))
|
||||
.context("failed to construct Bitbucket API URL")?;
|
||||
|
||||
loop {
|
||||
let mut req = client.get(next.clone()).header("User-Agent", GLOBAL_USER_AGENT.as_str());
|
||||
req = auth.apply(req);
|
||||
let resp = req.send().await?;
|
||||
if resp.status() == reqwest::StatusCode::NOT_FOUND {
|
||||
break;
|
||||
}
|
||||
let resp = resp.error_for_status()?;
|
||||
let payload: CloudRepoList = resp.json().await?;
|
||||
for repo in payload.values {
|
||||
let is_fork = repo.parent.is_some();
|
||||
if !repo_filter.allows(is_fork) {
|
||||
continue;
|
||||
}
|
||||
if let Some(clone) = repo_clone_url_from_links(&repo.links.clone) {
|
||||
if should_exclude_repo(&clone, excludes) {
|
||||
continue;
|
||||
}
|
||||
results.push(clone);
|
||||
}
|
||||
}
|
||||
if let Some(next_url) = payload.next {
|
||||
next = Url::parse(&next_url)?;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn fetch_server_repositories(
|
||||
client: &reqwest::Client,
|
||||
base: &Url,
|
||||
path: &str,
|
||||
auth: &AuthConfig,
|
||||
repo_filter: RepoType,
|
||||
excludes: &ExcludeMatcher,
|
||||
results: &mut Vec<String>,
|
||||
) -> Result<()> {
|
||||
let mut start = 0u64;
|
||||
loop {
|
||||
let api_path = if path.contains('?') {
|
||||
format!("{path}&start={start}")
|
||||
} else {
|
||||
format!("{path}?limit=100&start={start}")
|
||||
};
|
||||
let mut req = client
|
||||
.get(base.join(&api_path).context("failed to build Bitbucket Server URL")?)
|
||||
.header("User-Agent", GLOBAL_USER_AGENT.as_str());
|
||||
req = auth.apply(req);
|
||||
let resp = req.send().await?;
|
||||
if resp.status() == reqwest::StatusCode::NOT_FOUND {
|
||||
break;
|
||||
}
|
||||
let resp = resp.error_for_status()?;
|
||||
let payload: ServerRepoList = resp.json().await?;
|
||||
for repo in payload.values {
|
||||
let is_fork = repo.origin.is_some();
|
||||
if !repo_filter.allows(is_fork) {
|
||||
continue;
|
||||
}
|
||||
if let Some(clone) = repo_clone_url_from_links(&repo.links.clone) {
|
||||
if should_exclude_repo(&clone, excludes) {
|
||||
continue;
|
||||
}
|
||||
results.push(clone);
|
||||
}
|
||||
}
|
||||
if payload.is_last_page {
|
||||
break;
|
||||
}
|
||||
start = payload.next_page_start.unwrap_or_else(|| start + 100);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_cloud_workspaces(
|
||||
client: &reqwest::Client,
|
||||
base: &Url,
|
||||
auth: &AuthConfig,
|
||||
) -> Result<Vec<String>> {
|
||||
let mut workspaces = Vec::new();
|
||||
let mut next = base
|
||||
.join("workspaces?role=member&pagelen=100")
|
||||
.context("failed to build Bitbucket workspace URL")?;
|
||||
loop {
|
||||
let mut req = client.get(next.clone()).header("User-Agent", GLOBAL_USER_AGENT.as_str());
|
||||
req = auth.apply(req);
|
||||
let resp = req.send().await?;
|
||||
if resp.status() == reqwest::StatusCode::NOT_FOUND {
|
||||
break;
|
||||
}
|
||||
let resp = resp.error_for_status()?;
|
||||
let payload: CloudWorkspaceList = resp.json().await?;
|
||||
for ws in payload.values {
|
||||
workspaces.push(ws.slug);
|
||||
}
|
||||
if let Some(next_url) = payload.next {
|
||||
next = Url::parse(&next_url)?;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(workspaces)
|
||||
}
|
||||
|
||||
async fn list_server_projects(
|
||||
client: &reqwest::Client,
|
||||
base: &Url,
|
||||
auth: &AuthConfig,
|
||||
) -> Result<Vec<String>> {
|
||||
let mut projects = Vec::new();
|
||||
let mut start = 0u64;
|
||||
loop {
|
||||
let mut req = client
|
||||
.get(
|
||||
base.join(&format!("projects?limit=100&start={start}"))
|
||||
.context("failed to build Bitbucket projects URL")?,
|
||||
)
|
||||
.header("User-Agent", GLOBAL_USER_AGENT.as_str());
|
||||
req = auth.apply(req);
|
||||
let resp = req.send().await?;
|
||||
if resp.status() == reqwest::StatusCode::NOT_FOUND {
|
||||
break;
|
||||
}
|
||||
let resp = resp.error_for_status()?;
|
||||
let payload: ServerProjectList = resp.json().await?;
|
||||
for project in payload.values {
|
||||
projects.push(project.key);
|
||||
}
|
||||
if payload.is_last_page {
|
||||
break;
|
||||
}
|
||||
start = payload.next_page_start.unwrap_or_else(|| start + 100);
|
||||
}
|
||||
Ok(projects)
|
||||
}
|
||||
|
||||
pub async fn enumerate_repo_urls(
|
||||
repo_specifiers: &RepoSpecifiers,
|
||||
api_url: Url,
|
||||
auth: &AuthConfig,
|
||||
ignore_certs: bool,
|
||||
mut progress: Option<&mut ProgressBar>,
|
||||
) -> Result<Vec<String>> {
|
||||
let client = reqwest::Client::builder()
|
||||
.danger_accept_invalid_certs(ignore_certs)
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()?;
|
||||
let kind = BitbucketKind::from_url(&api_url);
|
||||
let excludes = build_exclude_matcher(&repo_specifiers.exclude_repos);
|
||||
let mut repo_urls = Vec::new();
|
||||
|
||||
match kind {
|
||||
BitbucketKind::Cloud => {
|
||||
let mut owners: HashSet<String> = HashSet::new();
|
||||
owners.extend(repo_specifiers.user.iter().cloned());
|
||||
owners.extend(repo_specifiers.workspace.iter().cloned());
|
||||
owners.extend(repo_specifiers.project.iter().cloned());
|
||||
if repo_specifiers.all_workspaces {
|
||||
match list_cloud_workspaces(&client, &api_url, auth).await {
|
||||
Ok(ws) => owners.extend(ws),
|
||||
Err(err) => warn!("Failed to enumerate Bitbucket workspaces: {err:#}"),
|
||||
}
|
||||
}
|
||||
for owner in owners {
|
||||
if let Err(err) = fetch_cloud_repositories(
|
||||
&client,
|
||||
&api_url,
|
||||
&owner,
|
||||
auth,
|
||||
repo_specifiers.repo_filter,
|
||||
&excludes,
|
||||
&mut repo_urls,
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!("Failed to fetch Bitbucket repositories for '{owner}': {err:#}");
|
||||
}
|
||||
if let Some(progress) = progress.as_mut() {
|
||||
progress.inc(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
BitbucketKind::Server => {
|
||||
let mut projects: HashSet<String> = HashSet::new();
|
||||
projects.extend(repo_specifiers.workspace.iter().cloned());
|
||||
projects.extend(repo_specifiers.project.iter().cloned());
|
||||
if repo_specifiers.all_workspaces {
|
||||
match list_server_projects(&client, &api_url, auth).await {
|
||||
Ok(p) => projects.extend(p),
|
||||
Err(err) => warn!("Failed to enumerate Bitbucket projects: {err:#}"),
|
||||
}
|
||||
}
|
||||
for user in &repo_specifiers.user {
|
||||
if let Err(err) = fetch_server_repositories(
|
||||
&client,
|
||||
&api_url,
|
||||
&format!("users/{user}/repos?limit=100"),
|
||||
auth,
|
||||
repo_specifiers.repo_filter,
|
||||
&excludes,
|
||||
&mut repo_urls,
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!("Failed to fetch Bitbucket repositories for user '{user}': {err:#}");
|
||||
}
|
||||
if let Some(progress) = progress.as_mut() {
|
||||
progress.inc(1);
|
||||
}
|
||||
}
|
||||
for project in projects {
|
||||
if let Err(err) = fetch_server_repositories(
|
||||
&client,
|
||||
&api_url,
|
||||
&format!("projects/{project}/repos"),
|
||||
auth,
|
||||
repo_specifiers.repo_filter,
|
||||
&excludes,
|
||||
&mut repo_urls,
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
"Failed to fetch Bitbucket repositories for project '{project}': {err:#}"
|
||||
);
|
||||
}
|
||||
if let Some(progress) = progress.as_mut() {
|
||||
progress.inc(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
repo_urls.sort();
|
||||
repo_urls.dedup();
|
||||
Ok(repo_urls)
|
||||
}
|
||||
|
||||
pub async fn list_repositories(
|
||||
api_url: Url,
|
||||
auth: AuthConfig,
|
||||
ignore_certs: bool,
|
||||
progress_enabled: bool,
|
||||
users: &[String],
|
||||
workspaces: &[String],
|
||||
projects: &[String],
|
||||
all_workspaces: bool,
|
||||
exclude_repos: &[String],
|
||||
repo_filter: RepoType,
|
||||
) -> Result<()> {
|
||||
let mut progress = if progress_enabled {
|
||||
let style = ProgressStyle::with_template("{spinner} {msg} [{elapsed_precise}]")
|
||||
.expect("progress bar style template should compile");
|
||||
let pb = ProgressBar::new_spinner()
|
||||
.with_style(style)
|
||||
.with_message("Fetching Bitbucket repositories");
|
||||
pb.enable_steady_tick(Duration::from_millis(500));
|
||||
pb
|
||||
} else {
|
||||
ProgressBar::hidden()
|
||||
};
|
||||
let repo_specifiers = RepoSpecifiers {
|
||||
user: users.to_vec(),
|
||||
workspace: workspaces.to_vec(),
|
||||
project: projects.to_vec(),
|
||||
all_workspaces,
|
||||
repo_filter,
|
||||
exclude_repos: exclude_repos.to_vec(),
|
||||
};
|
||||
let repos =
|
||||
enumerate_repo_urls(&repo_specifiers, api_url, &auth, ignore_certs, Some(&mut progress))
|
||||
.await?;
|
||||
for repo in repos {
|
||||
println!("{repo}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_repo(repo_url: &GitUrl) -> Option<(String, String, String)> {
|
||||
let url = Url::parse(repo_url.as_str()).ok()?;
|
||||
let host = url.host_str()?.to_string();
|
||||
let parts: Vec<&str> = url
|
||||
.path_segments()
|
||||
.map(|segments| segments.filter(|s| !s.is_empty()).collect::<Vec<_>>())?;
|
||||
if parts.len() < 2 {
|
||||
return None;
|
||||
}
|
||||
let repo = parts.last()?.trim_end_matches(".git").to_string();
|
||||
let owner = parts.get(parts.len() - 2)?.to_string();
|
||||
Some((host, owner, repo))
|
||||
}
|
||||
|
||||
pub fn wiki_url(_repo_url: &GitUrl) -> Option<GitUrl> {
|
||||
None
|
||||
}
|
||||
|
||||
pub async fn fetch_repo_items(
|
||||
repo_url: &GitUrl,
|
||||
api_base: &Url,
|
||||
auth: &AuthConfig,
|
||||
ignore_certs: bool,
|
||||
output_root: &Path,
|
||||
datastore: &Arc<Mutex<findings_store::FindingsStore>>,
|
||||
) -> Result<Vec<PathBuf>> {
|
||||
let (_host, owner, repo) = parse_repo(repo_url).context("invalid Bitbucket repo URL")?;
|
||||
|
||||
let client = reqwest::Client::builder().danger_accept_invalid_certs(ignore_certs).build()?;
|
||||
|
||||
let mut dirs = Vec::new();
|
||||
let issues_dir = output_root.join("bitbucket_issues").join(format!("{owner}_{repo}"));
|
||||
fs::create_dir_all(&issues_dir)?;
|
||||
let kind = BitbucketKind::from_url(api_base);
|
||||
let mut next = match kind {
|
||||
BitbucketKind::Cloud => api_base
|
||||
.join(&format!("repositories/{owner}/{repo}/issues?pagelen=50"))
|
||||
.context("failed to construct Bitbucket Cloud issues URL")?,
|
||||
BitbucketKind::Server => api_base
|
||||
.join(&format!("projects/{owner}/repos/{repo}/issues?limit=50"))
|
||||
.context("failed to construct Bitbucket Server issues URL")?,
|
||||
};
|
||||
let mut any_issue = false;
|
||||
loop {
|
||||
let mut req = client.get(next.clone()).header("User-Agent", GLOBAL_USER_AGENT.as_str());
|
||||
req = auth.apply(req);
|
||||
let resp = req.send().await?;
|
||||
if resp.status().is_client_error() {
|
||||
break;
|
||||
}
|
||||
let payload: Value = resp.json().await?;
|
||||
if payload.get("type").and_then(|v| v.as_str()) == Some("error") {
|
||||
break;
|
||||
}
|
||||
let Some(values) = payload.get("values").and_then(|v| v.as_array()) else {
|
||||
break;
|
||||
};
|
||||
if values.is_empty() {
|
||||
break;
|
||||
}
|
||||
for issue in values {
|
||||
let id = issue.get("id").and_then(|v| v.as_u64()).unwrap_or(0);
|
||||
let title = issue.get("title").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let body = issue
|
||||
.get("content")
|
||||
.and_then(|v| v.get("raw"))
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("");
|
||||
let content = format!("# {title}\n\n{body}");
|
||||
let file_path = issues_dir.join(format!("issue_{id}.md"));
|
||||
fs::write(&file_path, content)?;
|
||||
if matches!(kind, BitbucketKind::Cloud) {
|
||||
let url = format!("https://bitbucket.org/{owner}/{repo}/issues/{id}");
|
||||
let mut ds = datastore.lock().unwrap();
|
||||
ds.register_repo_link(file_path, url);
|
||||
}
|
||||
any_issue = true;
|
||||
}
|
||||
if let Some(next_url) = payload.get("next").and_then(|v| v.as_str()) {
|
||||
next = Url::parse(next_url)?;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if any_issue {
|
||||
dirs.push(issues_dir);
|
||||
}
|
||||
|
||||
Ok(dirs)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_excluded_repo_variants() {
|
||||
assert_eq!(parse_excluded_repo("workspace/repo").as_deref(), Some("workspace/repo"));
|
||||
assert_eq!(parse_excluded_repo("workspace/repo.git").as_deref(), Some("workspace/repo"));
|
||||
assert_eq!(
|
||||
parse_excluded_repo("https://bitbucket.org/workspace/repo.git").as_deref(),
|
||||
Some("workspace/repo")
|
||||
);
|
||||
assert_eq!(
|
||||
parse_excluded_repo("ssh://git@bitbucket.example.com/scm/WS/repo.git").as_deref(),
|
||||
Some("ws/repo")
|
||||
);
|
||||
}
|
||||
}
|
||||
121
src/cli/commands/bitbucket.rs
Normal file
121
src/cli/commands/bitbucket.rs
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
use clap::{Args, Subcommand, ValueEnum, ValueHint};
|
||||
use strum_macros::Display;
|
||||
use url::Url;
|
||||
|
||||
use crate::cli::commands::output::OutputArgs;
|
||||
|
||||
#[derive(Args, Debug, Clone, Default)]
|
||||
pub struct BitbucketAuthArgs {
|
||||
/// Username for Bitbucket basic authentication (app password or server)
|
||||
#[arg(long)]
|
||||
pub bitbucket_username: Option<String>,
|
||||
|
||||
/// Bitbucket app password, PAT, or server token
|
||||
#[arg(long = "bitbucket-token", alias = "bitbucket-password")]
|
||||
pub bitbucket_token: Option<String>,
|
||||
|
||||
/// Bitbucket OAuth token for bearer authentication
|
||||
#[arg(long = "bitbucket-oauth-token", alias = "bitbucket-oauth")]
|
||||
pub bitbucket_oauth_token: Option<String>,
|
||||
}
|
||||
|
||||
/// Top-level Bitbucket command group
|
||||
#[derive(Args, Debug)]
|
||||
pub struct BitbucketArgs {
|
||||
#[command(subcommand)]
|
||||
pub command: BitbucketCommand,
|
||||
|
||||
/// Override Bitbucket API URL (Cloud or self-hosted)
|
||||
#[arg(
|
||||
global = true,
|
||||
long,
|
||||
default_value = "https://api.bitbucket.org/2.0/",
|
||||
value_hint = ValueHint::Url
|
||||
)]
|
||||
pub bitbucket_api_url: Url,
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
pub enum BitbucketCommand {
|
||||
/// Interact with Bitbucket repositories
|
||||
#[command(subcommand)]
|
||||
Repos(BitbucketReposCommand),
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
pub enum BitbucketReposCommand {
|
||||
/// List repositories for users, workspaces, or projects
|
||||
List(BitbucketReposListArgs),
|
||||
}
|
||||
|
||||
#[derive(Args, Debug, Clone)]
|
||||
pub struct BitbucketReposListArgs {
|
||||
#[command(flatten)]
|
||||
pub repo_specifiers: BitbucketRepoSpecifiers,
|
||||
|
||||
#[command(flatten)]
|
||||
pub output_args: OutputArgs<BitbucketOutputFormat>,
|
||||
|
||||
#[command(flatten)]
|
||||
pub auth: BitbucketAuthArgs,
|
||||
}
|
||||
|
||||
#[derive(Args, Debug, Clone)]
|
||||
pub struct BitbucketRepoSpecifiers {
|
||||
/// Repositories belonging to these users
|
||||
#[arg(long, alias = "bitbucket-user")]
|
||||
pub user: Vec<String>,
|
||||
|
||||
/// Repositories belonging to these workspaces or teams
|
||||
#[arg(long, alias = "bitbucket-workspace", alias = "bitbucket-team")]
|
||||
pub workspace: Vec<String>,
|
||||
|
||||
/// Repositories belonging to these Bitbucket Server projects
|
||||
#[arg(long, alias = "bitbucket-project")]
|
||||
pub project: Vec<String>,
|
||||
|
||||
/// Skip specific repositories during enumeration (format: owner/repo)
|
||||
#[arg(long = "bitbucket-exclude", value_name = "OWNER/REPO")]
|
||||
pub exclude_repos: Vec<String>,
|
||||
|
||||
/// Enumerate all accessible workspaces or projects
|
||||
#[arg(long, alias = "all-bitbucket-workspaces", requires = "bitbucket_api_url")]
|
||||
pub all_workspaces: bool,
|
||||
|
||||
/// Filter repositories by type
|
||||
#[arg(long, default_value_t = BitbucketRepoType::Source, alias = "bitbucket-repo-type")]
|
||||
pub repo_type: BitbucketRepoType,
|
||||
}
|
||||
|
||||
impl BitbucketRepoSpecifiers {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.user.is_empty()
|
||||
&& self.workspace.is_empty()
|
||||
&& self.project.is_empty()
|
||||
&& !self.all_workspaces
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, Display, PartialEq, Eq, PartialOrd, Ord, ValueEnum)]
|
||||
#[strum(serialize_all = "kebab-case")]
|
||||
pub enum BitbucketRepoType {
|
||||
/// Source repositories (exclude forks)
|
||||
Source,
|
||||
/// Fork repositories only
|
||||
#[value(alias = "forks")]
|
||||
Fork,
|
||||
/// All repositories (source and forks)
|
||||
All,
|
||||
}
|
||||
|
||||
pub type BitbucketOutputFormat = crate::cli::commands::output::GitHubOutputFormat;
|
||||
|
||||
impl From<BitbucketRepoType> for crate::bitbucket::RepoType {
|
||||
fn from(value: BitbucketRepoType) -> Self {
|
||||
match value {
|
||||
BitbucketRepoType::All => crate::bitbucket::RepoType::All,
|
||||
BitbucketRepoType::Source => crate::bitbucket::RepoType::Source,
|
||||
BitbucketRepoType::Fork => crate::bitbucket::RepoType::Fork,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@ use url::Url;
|
|||
|
||||
use crate::{
|
||||
cli::commands::{
|
||||
bitbucket::{BitbucketAuthArgs, BitbucketRepoType},
|
||||
github::{GitCloneMode, GitHistoryMode, GitHubRepoType},
|
||||
gitlab::GitLabRepoType,
|
||||
},
|
||||
|
|
@ -23,15 +24,20 @@ pub struct InputSpecifierArgs {
|
|||
"github_organization",
|
||||
"gitlab_user",
|
||||
"gitlab_group",
|
||||
"bitbucket_user",
|
||||
"bitbucket_workspace",
|
||||
"bitbucket_project",
|
||||
"git_url",
|
||||
"all_github_organizations",
|
||||
"all_gitlab_groups",
|
||||
"all_bitbucket_workspaces",
|
||||
"jira_url",
|
||||
"confluence_url",
|
||||
"docker_image",
|
||||
"slack_query",
|
||||
"s3_bucket"
|
||||
]),
|
||||
num_args = 0..,
|
||||
value_hint = ValueHint::AnyPath
|
||||
)]
|
||||
pub path_inputs: Vec<PathBuf>,
|
||||
|
|
@ -106,6 +112,37 @@ pub struct InputSpecifierArgs {
|
|||
#[arg(long, alias = "include-subgroups")]
|
||||
pub gitlab_include_subgroups: bool,
|
||||
|
||||
// Bitbucket Options
|
||||
/// Scan repositories belonging to the specified Bitbucket users
|
||||
#[arg(long)]
|
||||
pub bitbucket_user: Vec<String>,
|
||||
|
||||
/// Scan repositories belonging to the specified Bitbucket workspaces or teams
|
||||
#[arg(long, alias = "bitbucket-workspace", alias = "bitbucket-team")]
|
||||
pub bitbucket_workspace: Vec<String>,
|
||||
|
||||
/// Scan repositories belonging to the specified Bitbucket Server projects
|
||||
#[arg(long, alias = "bitbucket-project")]
|
||||
pub bitbucket_project: Vec<String>,
|
||||
|
||||
/// Skip repositories when enumerating Bitbucket sources (format: owner/repo)
|
||||
#[arg(long = "bitbucket-exclude", value_name = "OWNER/REPO")]
|
||||
pub bitbucket_exclude: Vec<String>,
|
||||
|
||||
/// Scan repositories from all accessible Bitbucket workspaces or projects
|
||||
#[arg(long, alias = "all-bitbucket-workspaces", requires = "bitbucket_api_url")]
|
||||
pub all_bitbucket_workspaces: bool,
|
||||
|
||||
/// Use the specified URL for Bitbucket API access (Cloud or self-hosted)
|
||||
#[arg(long, default_value = "https://api.bitbucket.org/2.0/", value_hint = ValueHint::Url)]
|
||||
pub bitbucket_api_url: Url,
|
||||
|
||||
#[arg(long, default_value_t = BitbucketRepoType::Source)]
|
||||
pub bitbucket_repo_type: BitbucketRepoType,
|
||||
|
||||
#[command(flatten)]
|
||||
pub bitbucket_auth: BitbucketAuthArgs,
|
||||
|
||||
/// Jira base URL (e.g. https://jira.example.com)
|
||||
#[arg(long, value_hint = ValueHint::Url, requires = "jql")]
|
||||
pub jira_url: Option<Url>,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
pub mod bitbucket;
|
||||
pub mod github;
|
||||
pub mod gitlab;
|
||||
pub mod inputs;
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ use sysinfo::{MemoryRefreshKind, RefreshKind, System};
|
|||
use tracing::Level;
|
||||
|
||||
use crate::cli::commands::{
|
||||
github::GitHubArgs, gitlab::GitLabArgs, rules::RulesArgs, scan::ScanArgs,
|
||||
bitbucket::BitbucketArgs, github::GitHubArgs, gitlab::GitLabArgs, rules::RulesArgs,
|
||||
scan::ScanArgs,
|
||||
};
|
||||
|
||||
#[deny(missing_docs)]
|
||||
|
|
@ -68,6 +69,10 @@ pub enum Command {
|
|||
#[command(name = "gitlab")]
|
||||
GitLab(GitLabArgs),
|
||||
|
||||
/// Interact with the Bitbucket API
|
||||
#[command(name = "bitbucket")]
|
||||
Bitbucket(BitbucketArgs),
|
||||
|
||||
/// Manage rules
|
||||
#[command(alias = "rule")]
|
||||
Rules(RulesArgs),
|
||||
|
|
|
|||
|
|
@ -7,6 +7,22 @@ use tracing::{debug, debug_span};
|
|||
|
||||
use crate::git_url::GitUrl;
|
||||
|
||||
const BITBUCKET_CREDENTIAL_HELPER: &str = r#"credential.helper=!_bbcreds() {
|
||||
if [ -n "$KF_BITBUCKET_OAUTH_TOKEN" ]; then
|
||||
echo username="x-token-auth";
|
||||
echo password="$KF_BITBUCKET_OAUTH_TOKEN";
|
||||
return;
|
||||
fi
|
||||
if [ -n "$KF_BITBUCKET_USERNAME" ]; then
|
||||
bb_pass="${KF_BITBUCKET_APP_PASSWORD:-${KF_BITBUCKET_TOKEN:-${KF_BITBUCKET_PASSWORD:-}}}";
|
||||
if [ -n "$bb_pass" ]; then
|
||||
echo username="$KF_BITBUCKET_USERNAME";
|
||||
echo password="$bb_pass";
|
||||
return;
|
||||
fi
|
||||
fi
|
||||
}; _bbcreds"#;
|
||||
|
||||
/// Represents errors that can occur when interacting with the `git` CLI.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum GitError {
|
||||
|
|
@ -24,9 +40,9 @@ pub enum GitError {
|
|||
|
||||
/// A helper struct for running `git` commands.
|
||||
///
|
||||
/// It supports optional GitHub credentials passed via the
|
||||
/// `KF_GITHUB_TOKEN` environment variable, and optionally
|
||||
/// ignores TLS certificate validation if requested.
|
||||
/// It supports optional GitHub, GitLab, and Bitbucket credentials passed via
|
||||
/// environment variables and optionally ignores TLS certificate validation if
|
||||
/// requested.
|
||||
pub struct Git {
|
||||
credentials: Vec<String>,
|
||||
ignore_certs: bool,
|
||||
|
|
@ -39,14 +55,29 @@ impl Git {
|
|||
pub fn new(ignore_certs: bool) -> Self {
|
||||
let mut credentials = Vec::new();
|
||||
|
||||
// If either GitHub or GitLab token is set, first clear existing credential.helpers
|
||||
if std::env::var("KF_GITHUB_TOKEN").is_ok() || std::env::var("KF_GITLAB_TOKEN").is_ok() {
|
||||
let has_github_token =
|
||||
matches!(std::env::var("KF_GITHUB_TOKEN"), Ok(token) if !token.is_empty());
|
||||
let has_gitlab_token =
|
||||
matches!(std::env::var("KF_GITLAB_TOKEN"), Ok(token) if !token.is_empty());
|
||||
let has_bitbucket_username =
|
||||
matches!(std::env::var("KF_BITBUCKET_USERNAME"), Ok(value) if !value.is_empty());
|
||||
let has_bitbucket_password =
|
||||
["KF_BITBUCKET_APP_PASSWORD", "KF_BITBUCKET_TOKEN", "KF_BITBUCKET_PASSWORD"]
|
||||
.iter()
|
||||
.any(|key| matches!(std::env::var(key), Ok(value) if !value.is_empty()));
|
||||
let has_bitbucket_oauth_token =
|
||||
matches!(std::env::var("KF_BITBUCKET_OAUTH_TOKEN"), Ok(value) if !value.is_empty());
|
||||
let has_bitbucket_credentials =
|
||||
has_bitbucket_oauth_token || (has_bitbucket_username && has_bitbucket_password);
|
||||
|
||||
// If credentials are provided via environment variables, clear existing helpers first.
|
||||
if has_github_token || has_gitlab_token || has_bitbucket_credentials {
|
||||
credentials.push("-c".into());
|
||||
credentials.push(r#"credential.helper="#.into());
|
||||
}
|
||||
|
||||
// Inject GitHub token helper
|
||||
if std::env::var("KF_GITHUB_TOKEN").is_ok() {
|
||||
if has_github_token {
|
||||
credentials.push("-c".into());
|
||||
credentials.push(
|
||||
r#"credential.helper=!_ghcreds() { echo username="kingfisher"; echo password="$KF_GITHUB_TOKEN"; }; _ghcreds"#.into(),
|
||||
|
|
@ -54,19 +85,25 @@ impl Git {
|
|||
}
|
||||
|
||||
// Inject GitLab token helper
|
||||
if std::env::var("KF_GITLAB_TOKEN").is_ok() {
|
||||
if has_gitlab_token {
|
||||
credentials.push("-c".into());
|
||||
credentials.push(
|
||||
r#"credential.helper=!_glcreds() { echo username="oauth2"; echo password="$KF_GITLAB_TOKEN"; }; _glcreds"#.into(),
|
||||
);
|
||||
}
|
||||
|
||||
// Inject Bitbucket credential helper for OAuth tokens or basic auth.
|
||||
if has_bitbucket_credentials {
|
||||
credentials.push("-c".into());
|
||||
credentials.push(BITBUCKET_CREDENTIAL_HELPER.into());
|
||||
}
|
||||
|
||||
Self { credentials, ignore_certs }
|
||||
}
|
||||
|
||||
/// Create a basic `git` `Command` with environment variables set to
|
||||
/// limit config usage and (optionally) ignore certs. Includes credentials
|
||||
/// if a `KF_GITHUB_TOKEN` is present.
|
||||
/// if GitHub, GitLab, or Bitbucket tokens are present.
|
||||
fn git(&self) -> Command {
|
||||
let mut cmd = Command::new("git");
|
||||
cmd.env("GIT_CONFIG_GLOBAL", "/dev/null");
|
||||
|
|
@ -194,6 +231,30 @@ mod tests {
|
|||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_git_new_bitbucket_oauth() {
|
||||
temp_env::with_var("KF_BITBUCKET_OAUTH_TOKEN", Some("oauth"), || {
|
||||
let git = Git::new(false);
|
||||
assert_eq!(git.credentials.len(), 4);
|
||||
assert!(git.credentials.iter().any(|value| value == BITBUCKET_CREDENTIAL_HELPER));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_git_new_bitbucket_basic_auth() {
|
||||
temp_env::with_vars(
|
||||
&[
|
||||
("KF_BITBUCKET_USERNAME", Some("user")),
|
||||
("KF_BITBUCKET_APP_PASSWORD", Some("password")),
|
||||
],
|
||||
|| {
|
||||
let git = Git::new(false);
|
||||
assert_eq!(git.credentials.len(), 4);
|
||||
assert!(git.credentials.iter().any(|value| value == BITBUCKET_CREDENTIAL_HELPER));
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clone_mode_arg() {
|
||||
assert_eq!(CloneMode::Bare.arg(), Some("--bare"));
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
pub mod baseline;
|
||||
pub mod binary;
|
||||
pub mod bitbucket;
|
||||
pub mod blob;
|
||||
pub mod bstring_escape;
|
||||
pub mod bstring_table;
|
||||
|
|
|
|||
40
src/main.rs
40
src/main.rs
|
|
@ -33,6 +33,7 @@ use std::{
|
|||
|
||||
use anyhow::{Context, Result};
|
||||
use kingfisher::{
|
||||
bitbucket,
|
||||
cli::{
|
||||
self,
|
||||
commands::{
|
||||
|
|
@ -69,7 +70,10 @@ use tracing_subscriber::{
|
|||
};
|
||||
use url::Url;
|
||||
|
||||
use crate::cli::commands::gitlab::{GitLabCommand, GitLabRepoType, GitLabReposCommand};
|
||||
use crate::cli::commands::{
|
||||
bitbucket::{BitbucketAuthArgs, BitbucketCommand, BitbucketRepoType, BitbucketReposCommand},
|
||||
gitlab::{GitLabCommand, GitLabRepoType, GitLabReposCommand},
|
||||
};
|
||||
|
||||
fn main() -> anyhow::Result<()> {
|
||||
color_backtrace::install();
|
||||
|
|
@ -84,6 +88,7 @@ fn main() -> anyhow::Result<()> {
|
|||
Command::SelfUpdate => 1, // Self-update doesn't need a thread pool
|
||||
Command::GitHub(_) => num_cpus::get(), // Default for GitHub commands
|
||||
Command::GitLab(_) => num_cpus::get(), // Default for GitLab commands
|
||||
Command::Bitbucket(_) => num_cpus::get(), // Default for Bitbucket commands
|
||||
Command::Rules(_) => num_cpus::get(), // Default for Rules commands
|
||||
};
|
||||
|
||||
|
|
@ -260,6 +265,30 @@ async fn async_main(args: CommandLineArgs) -> Result<()> {
|
|||
}
|
||||
},
|
||||
},
|
||||
Command::Bitbucket(bitbucket_args) => match bitbucket_args.command {
|
||||
BitbucketCommand::Repos(repos_command) => match repos_command {
|
||||
BitbucketReposCommand::List(list_args) => {
|
||||
let auth = bitbucket::AuthConfig::from_options(
|
||||
list_args.auth.bitbucket_username.clone(),
|
||||
list_args.auth.bitbucket_token.clone(),
|
||||
list_args.auth.bitbucket_oauth_token.clone(),
|
||||
);
|
||||
bitbucket::list_repositories(
|
||||
bitbucket_args.bitbucket_api_url.clone(),
|
||||
auth,
|
||||
global_args.ignore_certs,
|
||||
global_args.use_progress(),
|
||||
&list_args.repo_specifiers.user,
|
||||
&list_args.repo_specifiers.workspace,
|
||||
&list_args.repo_specifiers.project,
|
||||
list_args.repo_specifiers.all_workspaces,
|
||||
&list_args.repo_specifiers.exclude_repos,
|
||||
list_args.repo_specifiers.repo_type.into(),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
},
|
||||
},
|
||||
Command::SelfUpdate => {
|
||||
anyhow::bail!("SelfUpdate command should not reach this branch")
|
||||
}
|
||||
|
|
@ -300,6 +329,15 @@ fn create_default_scan_args() -> cli::commands::scan::ScanArgs {
|
|||
gitlab_repo_type: GitLabRepoType::All,
|
||||
gitlab_include_subgroups: false,
|
||||
|
||||
bitbucket_user: Vec::new(),
|
||||
bitbucket_workspace: Vec::new(),
|
||||
bitbucket_project: Vec::new(),
|
||||
bitbucket_exclude: Vec::new(),
|
||||
all_bitbucket_workspaces: false,
|
||||
bitbucket_api_url: Url::parse("https://api.bitbucket.org/2.0/").unwrap(),
|
||||
bitbucket_repo_type: BitbucketRepoType::Source,
|
||||
bitbucket_auth: BitbucketAuthArgs::default(),
|
||||
|
||||
jira_url: None,
|
||||
jql: None,
|
||||
confluence_url: None,
|
||||
|
|
|
|||
|
|
@ -5,8 +5,10 @@ use std::{
|
|||
|
||||
use anyhow::Result;
|
||||
use http::StatusCode;
|
||||
use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS};
|
||||
use schemars::JsonSchema;
|
||||
use serde::Serialize;
|
||||
use url::Url;
|
||||
|
||||
use crate::{
|
||||
blob::BlobMetadata,
|
||||
|
|
@ -33,6 +35,71 @@ use crate::{
|
|||
origin::{get_repo_url, GitRepoOrigin},
|
||||
};
|
||||
|
||||
const BITBUCKET_FRAGMENT_ENCODE_SET: &AsciiSet = &CONTROLS
|
||||
.add(b' ')
|
||||
.add(b'"')
|
||||
.add(b'#')
|
||||
.add(b'%')
|
||||
.add(b'<')
|
||||
.add(b'>')
|
||||
.add(b'?')
|
||||
.add(b'`')
|
||||
.add(b'{')
|
||||
.add(b'}')
|
||||
.add(b'|');
|
||||
|
||||
fn build_git_urls(
|
||||
repo_url: &str,
|
||||
commit_id: &str,
|
||||
file_path: &str,
|
||||
line: usize,
|
||||
) -> (String, String, String) {
|
||||
let repo_url = repo_url.trim_end_matches('/');
|
||||
let mut repository_url = repo_url.to_string();
|
||||
let mut commit_url = format!("{repo_url}/commit/{commit_id}");
|
||||
let mut file_url = format!("{repo_url}/blob/{commit_id}/{file_path}#L{line}",);
|
||||
|
||||
if let Ok(parsed) = Url::parse(repo_url) {
|
||||
let scheme = parsed.scheme();
|
||||
let host = parsed.host_str().unwrap_or_default();
|
||||
let segments: Vec<&str> = parsed
|
||||
.path_segments()
|
||||
.map(|segments| segments.filter(|s| !s.is_empty()).collect())
|
||||
.unwrap_or_default();
|
||||
|
||||
let format_anchor = |path: &str| {
|
||||
let normalized = path.replace('\\', "/");
|
||||
utf8_percent_encode(normalized.trim_start_matches('/'), BITBUCKET_FRAGMENT_ENCODE_SET)
|
||||
.to_string()
|
||||
};
|
||||
|
||||
if host.eq_ignore_ascii_case("bitbucket.org") {
|
||||
let joined = segments.join("/");
|
||||
let base = if joined.is_empty() {
|
||||
format!("{scheme}://{host}")
|
||||
} else {
|
||||
format!("{scheme}://{host}/{joined}")
|
||||
};
|
||||
let anchor = format_anchor(file_path);
|
||||
repository_url = base.clone();
|
||||
commit_url = format!("{base}/commits/{commit_id}");
|
||||
file_url = format!("{base}/commits/{commit_id}#L{anchor}F{line}");
|
||||
} else if host.contains("bitbucket") {
|
||||
if segments.len() >= 3 && segments[0].eq_ignore_ascii_case("scm") {
|
||||
let project = segments[1];
|
||||
let repo = segments[2];
|
||||
let base = format!("{scheme}://{host}/projects/{project}/repos/{repo}");
|
||||
let anchor = format_anchor(file_path);
|
||||
repository_url = base.clone();
|
||||
commit_url = format!("{base}/commits/{commit_id}");
|
||||
file_url = format!("{base}/commits/{commit_id}#L{anchor}F{line}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(repository_url, commit_url, file_url)
|
||||
}
|
||||
|
||||
pub fn run(
|
||||
global_args: &GlobalArgs,
|
||||
ds: Arc<Mutex<findings_store::FindingsStore>>,
|
||||
|
|
@ -67,6 +134,9 @@ impl DetailsReporter {
|
|||
let repo_url = repo_url.trim_end_matches(".git").to_string();
|
||||
if let Some(cs) = &prov.first_commit {
|
||||
let cmd = &cs.commit_metadata;
|
||||
let commit_id = cmd.commit_id.to_string();
|
||||
let (repository_url, commit_url, file_url) =
|
||||
build_git_urls(&repo_url, &commit_id, &cs.blob_path, source_span.start.line);
|
||||
// let msg =
|
||||
// String::from_utf8_lossy(cmd.message.lines().next().unwrap_or(&[],),).
|
||||
// into_owned();
|
||||
|
|
@ -75,10 +145,10 @@ impl DetailsReporter {
|
|||
cmd.committer_timestamp.format(gix::date::time::format::SHORT.clone()).to_string();
|
||||
|
||||
let git_metadata = serde_json::json!({
|
||||
"repository_url": repo_url,
|
||||
"repository_url": repository_url,
|
||||
"commit": {
|
||||
"id": cmd.commit_id.to_string(),
|
||||
"url": format!("{}/commit/{}", repo_url, cmd.commit_id),
|
||||
"id": commit_id,
|
||||
"url": commit_url,
|
||||
"date": atime,
|
||||
"committer": {
|
||||
"name": &cmd.committer_name,
|
||||
|
|
@ -92,13 +162,7 @@ impl DetailsReporter {
|
|||
},
|
||||
"file": {
|
||||
"path": &cs.blob_path,
|
||||
"url": format!(
|
||||
"{}/blob/{}/{}#L{}",
|
||||
repo_url,
|
||||
cmd.commit_id,
|
||||
&cs.blob_path,
|
||||
source_span.start.line
|
||||
),
|
||||
"url": file_url,
|
||||
"git_command": format!(
|
||||
"git -C {} show {}:{}",
|
||||
prov.repo_path.display(),
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ mod tests {
|
|||
use crate::util::intern;
|
||||
use crate::{
|
||||
blob::BlobId,
|
||||
cli::commands::bitbucket::{BitbucketAuthArgs, BitbucketRepoType},
|
||||
cli::commands::github::GitHubRepoType,
|
||||
cli::commands::inputs::ContentFilteringArgs,
|
||||
cli::commands::inputs::InputSpecifierArgs,
|
||||
|
|
@ -89,6 +90,15 @@ mod tests {
|
|||
gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(),
|
||||
gitlab_repo_type: GitLabRepoType::All,
|
||||
gitlab_include_subgroups: false,
|
||||
// Bitbucket
|
||||
bitbucket_user: Vec::new(),
|
||||
bitbucket_workspace: Vec::new(),
|
||||
bitbucket_project: Vec::new(),
|
||||
bitbucket_exclude: Vec::new(),
|
||||
all_bitbucket_workspaces: false,
|
||||
bitbucket_api_url: Url::parse("https://api.bitbucket.org/2.0/").unwrap(),
|
||||
bitbucket_repo_type: BitbucketRepoType::Source,
|
||||
bitbucket_auth: BitbucketAuthArgs::default(),
|
||||
// Jira options
|
||||
jira_url: None,
|
||||
jql: None,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
//! Public façade for the scanner subsystem.
|
||||
pub(crate) use docker::save_docker_images;
|
||||
pub(crate) use enumerate::enumerate_filesystem_inputs;
|
||||
pub(crate) use repos::{clone_or_update_git_repos, enumerate_github_repos};
|
||||
pub(crate) use repos::{
|
||||
clone_or_update_git_repos, enumerate_bitbucket_repos, enumerate_github_repos,
|
||||
};
|
||||
pub use runner::{load_and_record_rules, run_async_scan, run_scan};
|
||||
pub(crate) use validation::run_secret_validation;
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ use url::Url;
|
|||
|
||||
use crate::blob::BlobIdMap;
|
||||
use crate::{
|
||||
bitbucket,
|
||||
blob::BlobMetadata,
|
||||
cli::{
|
||||
commands::{github::GitCloneMode, github::GitHistoryMode, scan},
|
||||
|
|
@ -242,6 +243,71 @@ pub async fn enumerate_gitlab_repos(
|
|||
Ok(repo_urls)
|
||||
}
|
||||
|
||||
pub async fn enumerate_bitbucket_repos(
|
||||
args: &scan::ScanArgs,
|
||||
global_args: &global::GlobalArgs,
|
||||
) -> Result<Vec<GitUrl>> {
|
||||
let repo_specifiers = bitbucket::RepoSpecifiers {
|
||||
user: args.input_specifier_args.bitbucket_user.clone(),
|
||||
workspace: args.input_specifier_args.bitbucket_workspace.clone(),
|
||||
project: args.input_specifier_args.bitbucket_project.clone(),
|
||||
all_workspaces: args.input_specifier_args.all_bitbucket_workspaces,
|
||||
repo_filter: args.input_specifier_args.bitbucket_repo_type.into(),
|
||||
exclude_repos: args.input_specifier_args.bitbucket_exclude.clone(),
|
||||
};
|
||||
let mut repo_urls = args.input_specifier_args.git_url.clone();
|
||||
if !repo_specifiers.is_empty() {
|
||||
let mut progress = if global_args.use_progress() {
|
||||
let style =
|
||||
ProgressStyle::with_template("{spinner} {msg} {human_len} [{elapsed_precise}]")
|
||||
.expect("progress bar style template should compile");
|
||||
let pb = ProgressBar::new_spinner()
|
||||
.with_style(style)
|
||||
.with_message("Enumerating Bitbucket repositories...");
|
||||
pb.enable_steady_tick(Duration::from_millis(500));
|
||||
pb
|
||||
} else {
|
||||
ProgressBar::hidden()
|
||||
};
|
||||
let mut num_found: u64 = 0;
|
||||
let api_url = args.input_specifier_args.bitbucket_api_url.clone();
|
||||
let auth = bitbucket::AuthConfig::from_options(
|
||||
args.input_specifier_args.bitbucket_auth.bitbucket_username.clone(),
|
||||
args.input_specifier_args.bitbucket_auth.bitbucket_token.clone(),
|
||||
args.input_specifier_args.bitbucket_auth.bitbucket_oauth_token.clone(),
|
||||
);
|
||||
let repo_strings = bitbucket::enumerate_repo_urls(
|
||||
&repo_specifiers,
|
||||
api_url,
|
||||
&auth,
|
||||
global_args.ignore_certs,
|
||||
Some(&mut progress),
|
||||
)
|
||||
.await
|
||||
.context("Failed to enumerate Bitbucket repositories")?;
|
||||
for repo_string in repo_strings {
|
||||
match GitUrl::from_str(&repo_string) {
|
||||
Ok(repo_url) => {
|
||||
repo_urls.push(repo_url);
|
||||
num_found += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
progress.suspend(|| {
|
||||
error!("Failed to parse repo URL from {repo_string}: {e}");
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
progress.finish_with_message(format!(
|
||||
"Found {} repositories from Bitbucket",
|
||||
HumanCount(num_found)
|
||||
));
|
||||
}
|
||||
repo_urls.sort();
|
||||
repo_urls.dedup();
|
||||
Ok(repo_urls)
|
||||
}
|
||||
|
||||
pub async fn fetch_jira_issues(
|
||||
args: &scan::ScanArgs,
|
||||
global_args: &global::GlobalArgs,
|
||||
|
|
@ -338,6 +404,9 @@ pub async fn fetch_slack_messages(
|
|||
|
||||
pub async fn fetch_git_host_artifacts(
|
||||
repo_urls: &[GitUrl],
|
||||
bitbucket_api_url: &Url,
|
||||
bitbucket_auth: &bitbucket::AuthConfig,
|
||||
bitbucket_host: Option<String>,
|
||||
global_args: &global::GlobalArgs,
|
||||
datastore: &Arc<Mutex<findings_store::FindingsStore>>,
|
||||
) -> Result<Vec<PathBuf>> {
|
||||
|
|
@ -371,6 +440,23 @@ pub async fn fetch_git_host_artifacts(
|
|||
)
|
||||
.await?,
|
||||
);
|
||||
} else if host.contains("bitbucket")
|
||||
|| bitbucket_host
|
||||
.as_deref()
|
||||
.map(|expected| expected.eq_ignore_ascii_case(&host))
|
||||
.unwrap_or(false)
|
||||
{
|
||||
dirs.extend(
|
||||
bitbucket::fetch_repo_items(
|
||||
repo_url,
|
||||
bitbucket_api_url,
|
||||
bitbucket_auth,
|
||||
global_args.ignore_certs,
|
||||
&output_root,
|
||||
datastore,
|
||||
)
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(dirs)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ use tokio::time::{Duration, Instant};
|
|||
use tracing::{debug, error, error_span, info, trace};
|
||||
|
||||
use crate::{
|
||||
bitbucket,
|
||||
cli::{commands::scan, global},
|
||||
findings_store,
|
||||
findings_store::{FindingsStore, FindingsStoreMessage},
|
||||
|
|
@ -19,7 +20,8 @@ use crate::{
|
|||
rules_database::RulesDatabase,
|
||||
safe_list,
|
||||
scanner::{
|
||||
clone_or_update_git_repos, enumerate_filesystem_inputs, enumerate_github_repos,
|
||||
clone_or_update_git_repos, enumerate_bitbucket_repos, enumerate_filesystem_inputs,
|
||||
enumerate_github_repos,
|
||||
repos::{
|
||||
enumerate_gitlab_repos, fetch_confluence_pages, fetch_git_host_artifacts,
|
||||
fetch_jira_issues, fetch_s3_objects, fetch_slack_messages,
|
||||
|
|
@ -71,9 +73,11 @@ pub async fn run_async_scan(
|
|||
|
||||
let mut repo_urls = enumerate_github_repos(args, global_args).await?;
|
||||
let gitlab_repo_urls = enumerate_gitlab_repos(args, global_args).await?;
|
||||
let bitbucket_repo_urls = enumerate_bitbucket_repos(args, global_args).await?;
|
||||
|
||||
// Combine repository URLs
|
||||
repo_urls.extend(gitlab_repo_urls);
|
||||
repo_urls.extend(bitbucket_repo_urls);
|
||||
repo_urls.sort();
|
||||
repo_urls.dedup();
|
||||
|
||||
|
|
@ -87,6 +91,9 @@ pub async fn run_async_scan(
|
|||
if let Some(w) = gitlab::wiki_url(url) {
|
||||
wiki_urls.push(w);
|
||||
}
|
||||
if let Some(w) = bitbucket::wiki_url(url) {
|
||||
wiki_urls.push(w);
|
||||
}
|
||||
}
|
||||
repo_urls.extend(wiki_urls);
|
||||
repo_urls.sort();
|
||||
|
|
@ -96,9 +103,24 @@ pub async fn run_async_scan(
|
|||
let mut input_roots = clone_or_update_git_repos(args, global_args, &repo_urls, &datastore)?;
|
||||
|
||||
// Fetch issues, gists, and wikis if enabled
|
||||
let bitbucket_auth = bitbucket::AuthConfig::from_options(
|
||||
args.input_specifier_args.bitbucket_auth.bitbucket_username.clone(),
|
||||
args.input_specifier_args.bitbucket_auth.bitbucket_token.clone(),
|
||||
args.input_specifier_args.bitbucket_auth.bitbucket_oauth_token.clone(),
|
||||
);
|
||||
let bitbucket_host =
|
||||
args.input_specifier_args.bitbucket_api_url.host_str().map(|s| s.to_string());
|
||||
|
||||
if args.input_specifier_args.repo_artifacts {
|
||||
let repo_artifact_dirs =
|
||||
fetch_git_host_artifacts(&repo_urls, global_args, &datastore).await?;
|
||||
let repo_artifact_dirs = fetch_git_host_artifacts(
|
||||
&repo_urls,
|
||||
&args.input_specifier_args.bitbucket_api_url,
|
||||
&bitbucket_auth,
|
||||
bitbucket_host.clone(),
|
||||
global_args,
|
||||
&datastore,
|
||||
)
|
||||
.await?;
|
||||
input_roots.extend(repo_artifact_dirs);
|
||||
}
|
||||
// Fetch Jira issues if requested
|
||||
|
|
|
|||
|
|
@ -23,6 +23,10 @@ use tracing::{error, info, warn};
|
|||
|
||||
use crate::{cli::global::GlobalArgs, reporter::styles::Styles};
|
||||
|
||||
fn styled_heading(styles: &Styles, text: &str) -> String {
|
||||
styles.style_finding_active_heading.apply_to(text).to_string()
|
||||
}
|
||||
|
||||
/// Check GitHub for a newer Kingfisher release and optionally self‑update.
|
||||
///
|
||||
/// * `base_url` lets tests point at a mock server.
|
||||
|
|
@ -98,7 +102,7 @@ pub fn check_for_update(global_args: &GlobalArgs, base_url: Option<&str>) -> Opt
|
|||
// ───────────── Case 1: running == latest ─────────────
|
||||
if release.version == running_v {
|
||||
let plain = format!("Kingfisher {running_v} is up to date");
|
||||
info!("{}", styles.style_finding_active_heading.apply_to(&plain));
|
||||
info!("{}", styled_heading(&styles, plain.as_str()));
|
||||
return Some(plain);
|
||||
}
|
||||
|
||||
|
|
@ -109,7 +113,7 @@ pub fn check_for_update(global_args: &GlobalArgs, base_url: Option<&str>) -> Opt
|
|||
if curr > latest {
|
||||
let plain =
|
||||
format!("Running Kingfisher {curr} which is newer than latest released {latest}");
|
||||
info!("{}", styles.style_finding_active_heading.apply_to(&plain));
|
||||
info!("{}", styled_heading(&styles, plain.as_str()));
|
||||
return Some(plain);
|
||||
}
|
||||
// else fall through to Case 3 (latest > running)
|
||||
|
|
@ -117,23 +121,22 @@ pub fn check_for_update(global_args: &GlobalArgs, base_url: Option<&str>) -> Opt
|
|||
|
||||
// ───────────── Case 3: latest > running ─────────────
|
||||
let plain = format!("New Kingfisher release {} available", release.version);
|
||||
info!("{}", styles.style_finding_active_heading.apply_to(&plain));
|
||||
info!("{}", styled_heading(&styles, plain.as_str()));
|
||||
|
||||
// Attempt self‑update when allowed and feasible.
|
||||
if global_args.self_update {
|
||||
match updater.update() {
|
||||
Ok(status) => info!(
|
||||
"{}",
|
||||
styles
|
||||
.style_finding_active_heading
|
||||
.apply_to(&format!("Updated to version {}", status.version()))
|
||||
),
|
||||
Ok(status) => {
|
||||
let message = format!("Updated to version {}", status.version());
|
||||
info!("{}", styled_heading(&styles, message.as_str()));
|
||||
}
|
||||
Err(e) => match e {
|
||||
UpdError::Io(ref io_err) => match io_err.kind() {
|
||||
ErrorKind::PermissionDenied => {
|
||||
warn!(
|
||||
"{}",
|
||||
styles.style_finding_active_heading.apply_to(
|
||||
styled_heading(
|
||||
&styles,
|
||||
"Cannot replace the current binary - permission denied.\n\
|
||||
If you installed via a package manager, run its upgrade command.\n\
|
||||
Otherwise reinstall to a user-writable directory or re-run with sudo."
|
||||
|
|
@ -143,7 +146,8 @@ pub fn check_for_update(global_args: &GlobalArgs, base_url: Option<&str>) -> Opt
|
|||
ErrorKind::NotFound => {
|
||||
warn!(
|
||||
"{}",
|
||||
styles.style_finding_active_heading.apply_to(
|
||||
styled_heading(
|
||||
&styles,
|
||||
"Cannot replace the current binary - file not found.\n\
|
||||
If you installed via a package manager, run its upgrade command.\n\
|
||||
Otherwise reinstall to a user-writable directory."
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ use anyhow::Result;
|
|||
use kingfisher::{
|
||||
cli::{
|
||||
commands::{
|
||||
bitbucket::{BitbucketAuthArgs, BitbucketRepoType},
|
||||
github::{GitCloneMode, GitHistoryMode, GitHubRepoType},
|
||||
gitlab::GitLabRepoType,
|
||||
inputs::{ContentFilteringArgs, InputSpecifierArgs},
|
||||
|
|
@ -69,6 +70,14 @@ fn run_skiplist(skip_regex: Vec<String>, skip_skipword: Vec<String>) -> Result<u
|
|||
gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(),
|
||||
gitlab_repo_type: GitLabRepoType::Owner,
|
||||
gitlab_include_subgroups: false,
|
||||
bitbucket_user: Vec::new(),
|
||||
bitbucket_workspace: Vec::new(),
|
||||
bitbucket_project: Vec::new(),
|
||||
bitbucket_exclude: Vec::new(),
|
||||
all_bitbucket_workspaces: false,
|
||||
bitbucket_api_url: Url::parse("https://api.bitbucket.org/2.0/").unwrap(),
|
||||
bitbucket_repo_type: BitbucketRepoType::Source,
|
||||
bitbucket_auth: BitbucketAuthArgs::default(),
|
||||
jira_url: None,
|
||||
jql: None,
|
||||
confluence_url: None,
|
||||
|
|
|
|||
155
tests/int_bitbucket.rs
Normal file
155
tests/int_bitbucket.rs
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
use std::{
|
||||
str::FromStr,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use kingfisher::{
|
||||
cli::{
|
||||
commands::{
|
||||
bitbucket::{BitbucketAuthArgs, BitbucketRepoType},
|
||||
github::{GitCloneMode, GitHistoryMode, GitHubRepoType},
|
||||
gitlab::GitLabRepoType,
|
||||
inputs::{ContentFilteringArgs, InputSpecifierArgs},
|
||||
output::{OutputArgs, ReportOutputFormat},
|
||||
rules::RuleSpecifierArgs,
|
||||
scan::{ConfidenceLevel, ScanArgs},
|
||||
},
|
||||
global::Mode,
|
||||
GlobalArgs,
|
||||
},
|
||||
findings_store::FindingsStore,
|
||||
git_url::GitUrl,
|
||||
scanner::{load_and_record_rules, run_scan},
|
||||
};
|
||||
use tempfile::TempDir;
|
||||
use tokio::runtime::Runtime;
|
||||
use url::Url;
|
||||
|
||||
fn determine_exit_code(total: usize, validated: usize) -> i32 {
|
||||
match (total, validated) {
|
||||
(0, _) => 0,
|
||||
(_, v) if v > 0 => 205,
|
||||
_ => 200,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bitbucket_remote_scan() -> Result<()> {
|
||||
let temp_dir = TempDir::new().context("tmp dir")?;
|
||||
let clone_dir = temp_dir.path().to_path_buf();
|
||||
|
||||
let repo_url = "https://bitbucket.org/hashashash/secretstest.git";
|
||||
let git_url = GitUrl::from_str(repo_url).expect("parse Bitbucket URL");
|
||||
|
||||
let scan_args = ScanArgs {
|
||||
num_jobs: 2,
|
||||
rules: RuleSpecifierArgs {
|
||||
rules_path: Vec::new(),
|
||||
rule: vec!["all".into()],
|
||||
load_builtins: true,
|
||||
},
|
||||
input_specifier_args: InputSpecifierArgs {
|
||||
path_inputs: Vec::new(),
|
||||
git_url: vec![git_url],
|
||||
github_user: Vec::new(),
|
||||
github_organization: Vec::new(),
|
||||
github_exclude: Vec::new(),
|
||||
all_github_organizations: false,
|
||||
github_api_url: Url::parse("https://api.github.com/")?,
|
||||
github_repo_type: GitHubRepoType::Source,
|
||||
gitlab_user: Vec::new(),
|
||||
gitlab_group: Vec::new(),
|
||||
gitlab_exclude: Vec::new(),
|
||||
all_gitlab_groups: false,
|
||||
gitlab_api_url: Url::parse("https://gitlab.com/")?,
|
||||
gitlab_repo_type: GitLabRepoType::Owner,
|
||||
gitlab_include_subgroups: false,
|
||||
|
||||
bitbucket_user: Vec::new(),
|
||||
bitbucket_workspace: Vec::new(),
|
||||
bitbucket_project: Vec::new(),
|
||||
bitbucket_exclude: Vec::new(),
|
||||
all_bitbucket_workspaces: false,
|
||||
bitbucket_api_url: Url::parse("https://api.bitbucket.org/2.0/")?,
|
||||
bitbucket_repo_type: BitbucketRepoType::Source,
|
||||
bitbucket_auth: BitbucketAuthArgs::default(),
|
||||
|
||||
jira_url: None,
|
||||
jql: None,
|
||||
confluence_url: None,
|
||||
cql: None,
|
||||
max_results: 100,
|
||||
slack_query: None,
|
||||
slack_api_url: Url::parse("https://slack.com/api/").unwrap(),
|
||||
s3_bucket: None,
|
||||
s3_prefix: None,
|
||||
role_arn: None,
|
||||
aws_local_profile: None,
|
||||
docker_image: Vec::new(),
|
||||
git_clone: GitCloneMode::Bare,
|
||||
git_history: GitHistoryMode::Full,
|
||||
commit_metadata: true,
|
||||
repo_artifacts: false,
|
||||
scan_nested_repos: true,
|
||||
since_commit: None,
|
||||
branch: None,
|
||||
},
|
||||
content_filtering_args: ContentFilteringArgs {
|
||||
max_file_size_mb: 25.0,
|
||||
no_extract_archives: false,
|
||||
extraction_depth: 2,
|
||||
no_binary: true,
|
||||
exclude: Vec::new(),
|
||||
},
|
||||
confidence: ConfidenceLevel::Medium,
|
||||
no_validate: false,
|
||||
rule_stats: false,
|
||||
only_valid: false,
|
||||
min_entropy: None,
|
||||
redact: false,
|
||||
git_repo_timeout: 1800,
|
||||
output_args: OutputArgs { output: None, format: ReportOutputFormat::Pretty },
|
||||
no_dedup: true,
|
||||
baseline_file: None,
|
||||
manage_baseline: false,
|
||||
skip_regex: Vec::new(),
|
||||
skip_word: Vec::new(),
|
||||
no_base64: false,
|
||||
};
|
||||
|
||||
let global_args = GlobalArgs {
|
||||
verbose: 0,
|
||||
quiet: false,
|
||||
color: Mode::Auto,
|
||||
progress: Mode::Auto,
|
||||
no_update_check: false,
|
||||
self_update: false,
|
||||
ignore_certs: false,
|
||||
user_agent_suffix: None,
|
||||
};
|
||||
|
||||
let datastore = Arc::new(Mutex::new(FindingsStore::new(clone_dir)));
|
||||
let runtime = Runtime::new()?;
|
||||
let rules_db = Arc::new(load_and_record_rules(&scan_args, &datastore)?);
|
||||
|
||||
runtime.block_on(async {
|
||||
run_scan(&global_args, &scan_args, &rules_db, Arc::clone(&datastore)).await
|
||||
})?;
|
||||
|
||||
let ds = datastore.lock().unwrap();
|
||||
let findings = ds.get_matches();
|
||||
let total = findings.len();
|
||||
let validated = findings.iter().filter(|m| m.as_ref().2.validation_success).count();
|
||||
|
||||
assert!(total >= 5, "expected at least 5 findings from Bitbucket repo, got {total}");
|
||||
|
||||
let exit_code = determine_exit_code(total, validated);
|
||||
assert!(
|
||||
exit_code >= 200,
|
||||
"expected findings from Bitbucket repo (exit_code >= 200), got {exit_code}"
|
||||
);
|
||||
|
||||
drop(runtime);
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -11,6 +11,7 @@ use anyhow::Result;
|
|||
use kingfisher::{
|
||||
cli::{
|
||||
commands::{
|
||||
bitbucket::{BitbucketAuthArgs, BitbucketRepoType},
|
||||
github::{GitCloneMode, GitHistoryMode, GitHubRepoType},
|
||||
gitlab::GitLabRepoType,
|
||||
inputs::{ContentFilteringArgs, InputSpecifierArgs},
|
||||
|
|
@ -82,6 +83,15 @@ rules:
|
|||
gitlab_repo_type: GitLabRepoType::Owner,
|
||||
gitlab_include_subgroups: false,
|
||||
|
||||
bitbucket_user: Vec::new(),
|
||||
bitbucket_workspace: Vec::new(),
|
||||
bitbucket_project: Vec::new(),
|
||||
bitbucket_exclude: Vec::new(),
|
||||
all_bitbucket_workspaces: false,
|
||||
bitbucket_api_url: Url::parse("https://api.bitbucket.org/2.0/").unwrap(),
|
||||
bitbucket_repo_type: BitbucketRepoType::Source,
|
||||
bitbucket_auth: BitbucketAuthArgs::default(),
|
||||
|
||||
jira_url: None,
|
||||
jql: None,
|
||||
confluence_url: None,
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ use anyhow::{Context, Result};
|
|||
use kingfisher::{
|
||||
cli::{
|
||||
commands::{
|
||||
bitbucket::{BitbucketAuthArgs, BitbucketRepoType},
|
||||
github::{GitCloneMode, GitHistoryMode, GitHubRepoType},
|
||||
gitlab::GitLabRepoType,
|
||||
inputs::{ContentFilteringArgs, InputSpecifierArgs},
|
||||
|
|
@ -69,6 +70,15 @@ fn test_github_remote_scan() -> Result<()> {
|
|||
gitlab_repo_type: GitLabRepoType::Owner,
|
||||
gitlab_include_subgroups: false,
|
||||
|
||||
bitbucket_user: Vec::new(),
|
||||
bitbucket_workspace: Vec::new(),
|
||||
bitbucket_project: Vec::new(),
|
||||
bitbucket_exclude: Vec::new(),
|
||||
all_bitbucket_workspaces: false,
|
||||
bitbucket_api_url: Url::parse("https://api.bitbucket.org/2.0/").unwrap(),
|
||||
bitbucket_repo_type: BitbucketRepoType::Source,
|
||||
bitbucket_auth: BitbucketAuthArgs::default(),
|
||||
|
||||
jira_url: None,
|
||||
jql: None,
|
||||
confluence_url: None,
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ use anyhow::{Context, Result};
|
|||
use kingfisher::{
|
||||
cli::{
|
||||
commands::{
|
||||
bitbucket::{BitbucketAuthArgs, BitbucketRepoType},
|
||||
github::{GitCloneMode, GitHistoryMode, GitHubRepoType},
|
||||
gitlab::GitLabRepoType,
|
||||
inputs::{ContentFilteringArgs, InputSpecifierArgs},
|
||||
|
|
@ -68,6 +69,15 @@ fn test_gitlab_remote_scan() -> Result<()> {
|
|||
gitlab_repo_type: GitLabRepoType::Owner,
|
||||
gitlab_include_subgroups: false,
|
||||
|
||||
bitbucket_user: Vec::new(),
|
||||
bitbucket_workspace: Vec::new(),
|
||||
bitbucket_project: Vec::new(),
|
||||
bitbucket_exclude: Vec::new(),
|
||||
all_bitbucket_workspaces: false,
|
||||
bitbucket_api_url: Url::parse("https://api.bitbucket.org/2.0/")?,
|
||||
bitbucket_repo_type: BitbucketRepoType::Source,
|
||||
bitbucket_auth: BitbucketAuthArgs::default(),
|
||||
|
||||
jira_url: None,
|
||||
jql: None,
|
||||
confluence_url: None,
|
||||
|
|
@ -182,6 +192,15 @@ fn test_gitlab_remote_scan_no_history() -> Result<()> {
|
|||
gitlab_repo_type: GitLabRepoType::Owner,
|
||||
gitlab_include_subgroups: false,
|
||||
|
||||
bitbucket_user: Vec::new(),
|
||||
bitbucket_workspace: Vec::new(),
|
||||
bitbucket_project: Vec::new(),
|
||||
bitbucket_exclude: Vec::new(),
|
||||
all_bitbucket_workspaces: false,
|
||||
bitbucket_api_url: Url::parse("https://api.bitbucket.org/2.0/")?,
|
||||
bitbucket_repo_type: BitbucketRepoType::Source,
|
||||
bitbucket_auth: BitbucketAuthArgs::default(),
|
||||
|
||||
jira_url: None,
|
||||
jql: None,
|
||||
confluence_url: None,
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ use anyhow::Result;
|
|||
use kingfisher::{
|
||||
cli::{
|
||||
commands::{
|
||||
bitbucket::{BitbucketAuthArgs, BitbucketRepoType},
|
||||
github::{GitCloneMode, GitHistoryMode, GitHubRepoType},
|
||||
gitlab::GitLabRepoType,
|
||||
inputs::{ContentFilteringArgs, InputSpecifierArgs},
|
||||
|
|
@ -52,6 +53,14 @@ async fn test_redact_hashes_finding_values() -> Result<()> {
|
|||
gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(),
|
||||
gitlab_repo_type: GitLabRepoType::Owner,
|
||||
gitlab_include_subgroups: false,
|
||||
bitbucket_user: Vec::new(),
|
||||
bitbucket_workspace: Vec::new(),
|
||||
bitbucket_project: Vec::new(),
|
||||
bitbucket_exclude: Vec::new(),
|
||||
all_bitbucket_workspaces: false,
|
||||
bitbucket_api_url: Url::parse("https://api.bitbucket.org/2.0/").unwrap(),
|
||||
bitbucket_repo_type: BitbucketRepoType::Source,
|
||||
bitbucket_auth: BitbucketAuthArgs::default(),
|
||||
jira_url: None,
|
||||
jql: None,
|
||||
confluence_url: None,
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ use anyhow::Result;
|
|||
use kingfisher::{
|
||||
cli::{
|
||||
commands::{
|
||||
bitbucket::{BitbucketAuthArgs, BitbucketRepoType},
|
||||
github::{GitCloneMode, GitHistoryMode, GitHubRepoType},
|
||||
gitlab::GitLabRepoType,
|
||||
inputs::{ContentFilteringArgs, InputSpecifierArgs},
|
||||
|
|
@ -58,6 +59,14 @@ impl TestContext {
|
|||
gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(),
|
||||
gitlab_repo_type: GitLabRepoType::Owner,
|
||||
gitlab_include_subgroups: false,
|
||||
bitbucket_user: Vec::new(),
|
||||
bitbucket_workspace: Vec::new(),
|
||||
bitbucket_project: Vec::new(),
|
||||
bitbucket_exclude: Vec::new(),
|
||||
all_bitbucket_workspaces: false,
|
||||
bitbucket_api_url: Url::parse("https://api.bitbucket.org/2.0/").unwrap(),
|
||||
bitbucket_repo_type: BitbucketRepoType::Source,
|
||||
bitbucket_auth: BitbucketAuthArgs::default(),
|
||||
jira_url: None,
|
||||
jql: None,
|
||||
confluence_url: None,
|
||||
|
|
@ -159,6 +168,14 @@ async fn test_scan_slack_messages() -> Result<()> {
|
|||
gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(),
|
||||
gitlab_repo_type: GitLabRepoType::Owner,
|
||||
gitlab_include_subgroups: false,
|
||||
bitbucket_user: Vec::new(),
|
||||
bitbucket_workspace: Vec::new(),
|
||||
bitbucket_project: Vec::new(),
|
||||
bitbucket_exclude: Vec::new(),
|
||||
all_bitbucket_workspaces: false,
|
||||
bitbucket_api_url: Url::parse("https://api.bitbucket.org/2.0/").unwrap(),
|
||||
bitbucket_repo_type: BitbucketRepoType::Source,
|
||||
bitbucket_auth: BitbucketAuthArgs::default(),
|
||||
jira_url: None,
|
||||
jql: None,
|
||||
confluence_url: None,
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ use anyhow::Result;
|
|||
use kingfisher::{
|
||||
cli::{
|
||||
commands::{
|
||||
bitbucket::{BitbucketAuthArgs, BitbucketRepoType},
|
||||
github::{GitCloneMode, GitHistoryMode, GitHubRepoType},
|
||||
gitlab::GitLabRepoType,
|
||||
inputs::{ContentFilteringArgs, InputSpecifierArgs},
|
||||
|
|
@ -124,6 +125,14 @@ async fn test_validation_cache_and_depvars() -> Result<()> {
|
|||
gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(),
|
||||
gitlab_repo_type: GitLabRepoType::Owner,
|
||||
gitlab_include_subgroups: false,
|
||||
bitbucket_user: Vec::new(),
|
||||
bitbucket_workspace: Vec::new(),
|
||||
bitbucket_project: Vec::new(),
|
||||
bitbucket_exclude: Vec::new(),
|
||||
all_bitbucket_workspaces: false,
|
||||
bitbucket_api_url: Url::parse("https://api.bitbucket.org/2.0/").unwrap(),
|
||||
bitbucket_repo_type: BitbucketRepoType::Source,
|
||||
bitbucket_auth: BitbucketAuthArgs::default(),
|
||||
|
||||
jira_url: None,
|
||||
jql: None,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ use anyhow::{Context, Result};
|
|||
use kingfisher::{
|
||||
cli::{
|
||||
commands::{
|
||||
bitbucket::{BitbucketAuthArgs, BitbucketRepoType},
|
||||
github::{GitCloneMode, GitHistoryMode, GitHubRepoType},
|
||||
gitlab::GitLabRepoType,
|
||||
inputs::{ContentFilteringArgs, InputSpecifierArgs},
|
||||
|
|
@ -68,6 +69,15 @@ impl TestContext {
|
|||
gitlab_repo_type: GitLabRepoType::Owner,
|
||||
gitlab_include_subgroups: false,
|
||||
|
||||
bitbucket_user: Vec::new(),
|
||||
bitbucket_workspace: Vec::new(),
|
||||
bitbucket_project: Vec::new(),
|
||||
bitbucket_exclude: Vec::new(),
|
||||
all_bitbucket_workspaces: false,
|
||||
bitbucket_api_url: Url::parse("https://api.bitbucket.org/2.0/").unwrap(),
|
||||
bitbucket_repo_type: BitbucketRepoType::Source,
|
||||
bitbucket_auth: BitbucketAuthArgs::default(),
|
||||
|
||||
jira_url: None,
|
||||
jql: None,
|
||||
confluence_url: None,
|
||||
|
|
@ -155,6 +165,15 @@ impl TestContext {
|
|||
gitlab_repo_type: GitLabRepoType::Owner,
|
||||
gitlab_include_subgroups: false,
|
||||
|
||||
bitbucket_user: Vec::new(),
|
||||
bitbucket_workspace: Vec::new(),
|
||||
bitbucket_project: Vec::new(),
|
||||
bitbucket_exclude: Vec::new(),
|
||||
all_bitbucket_workspaces: false,
|
||||
bitbucket_api_url: Url::parse("https://api.bitbucket.org/2.0/").unwrap(),
|
||||
bitbucket_repo_type: BitbucketRepoType::Source,
|
||||
bitbucket_auth: BitbucketAuthArgs::default(),
|
||||
|
||||
jira_url: None,
|
||||
jql: None,
|
||||
confluence_url: None,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue