eblume/kingfisher
1
0
Fork 0
forked from mirrors/kingfisher
Code Pull requests Activity Actions

Added support for scanning Confluence pages

Browse source
This commit is contained in:
Mick Grove 2025-08-10 21:54:26 -07:00
commit 94e2e11de3
3 changed files with 0 additions and 701 deletions
Download patch file Download diff file Expand all files Collapse all files

241
tests/int_gitlab.rs.orig
View file

@ -1,241 +0,0 @@
// tests/int_gitlab.rs
use std::{
str::FromStr,
sync::{Arc, Mutex},
};
use anyhow::{Context, Result};
use kingfisher::{
cli::{
commands::{
github::{GitCloneMode, GitHistoryMode, GitHubRepoType},
gitlab::GitLabRepoType,
inputs::{ContentFilteringArgs, InputSpecifierArgs},
output::{OutputArgs, ReportOutputFormat},
rules::RuleSpecifierArgs,
scan::{ConfidenceLevel, ScanArgs},
},
global::{AdvancedArgs, Mode},
GlobalArgs,
},
findings_store::FindingsStore,
git_url::GitUrl,
scanner::{load_and_record_rules, run_scan},
};
use tempfile::TempDir;
use tokio::runtime::Runtime;
use url::Url;
/// Derive process exit-codes from findings
fn determine_exit_code(total: usize, validated: usize) -> i32 {
match (total, validated) {
(0, _) => 0,
(_, v) if v > 0 => 205,
_ => 200,
}
}
#[test]
fn test_gitlab_remote_scan() -> Result<()> {
let temp_dir = TempDir::new().context("tmp dir")?;
let clone_dir = temp_dir.path().to_path_buf();
// Public GitLab repo seeded with test secrets
let test_repo_url = "https://gitlab.com/micksmix/SecretsTest.git";
let git_url = GitUrl::from_str(test_repo_url).expect("parse GitLab URL");
let scan_args = ScanArgs {
num_jobs: 2,
rules: RuleSpecifierArgs {
rules_path: Vec::new(),
rule: vec!["all".into()],
load_builtins: true,
},
input_specifier_args: InputSpecifierArgs {
path_inputs: Vec::new(),
git_url: vec![git_url],
github_user: Vec::new(),
github_organization: Vec::new(),
all_github_organizations: false,
github_api_url: Url::parse("https://api.github.com/")?,
github_repo_type: GitHubRepoType::Source,
gitlab_user: Vec::new(),
gitlab_group: Vec::new(),
all_gitlab_groups: false,
gitlab_api_url: Url::parse("https://gitlab.com/")?,
gitlab_repo_type: GitLabRepoType::Owner,
jira_url: None,
jql: None,
max_results: 100,
slack_query: None,
slack_api_url: Url::parse("https://slack.com/api/").unwrap(),
// s3
s3_bucket: None,
s3_prefix: None,
role_arn: None,
aws_local_profile: None,
// Docker image scanning
docker_image: Vec::new(),
git_clone: GitCloneMode::Bare,
git_history: GitHistoryMode::Full,
scan_nested_repos: true,
commit_metadata: true,
},
content_filtering_args: ContentFilteringArgs {
max_file_size_mb: 25.0,
no_extract_archives: false,
extraction_depth: 2,
no_binary: true,
exclude: Vec::new(), // Exclude patterns
},
confidence: ConfidenceLevel::Medium,
no_validate: false,
rule_stats: false,
only_valid: false,
min_entropy: None,
redact: false,
git_repo_timeout: 1800, // 30 minutes
output_args: OutputArgs { output: None, format: ReportOutputFormat::Pretty },
no_dedup: true,
baseline_file: None,
manage_baseline: false,
};
let global_args = GlobalArgs {
verbose: 0,
quiet: false,
color: Mode::Auto,
progress: Mode::Auto,
no_update_check: false,
self_update: false,
ignore_certs: false,
advanced: AdvancedArgs { rlimit_nofile: 16_384 },
};
let datastore = Arc::new(Mutex::new(FindingsStore::new(clone_dir)));
let rt = Runtime::new()?;
let rules_db = Arc::new(load_and_record_rules(&scan_args, &datastore)?);
rt.block_on(async {
run_scan(&global_args, &scan_args, &rules_db, Arc::clone(&datastore)).await
})?;
let ds = datastore.lock().unwrap();
let findings = ds.get_matches();
let total = findings.len();
let validated = findings.iter().filter(|m| m.as_ref().2.validation_success).count();
assert!(total >= 10, "expected at least 10 findings from GitLab repo, got {total}");
let exit_code = determine_exit_code(total, validated);
assert!(
exit_code >= 200,
"expected kingfisher to report findings (exit_code >= 200), got {exit_code}"
);
drop(rt);
Ok(())
}
#[test]
fn test_gitlab_remote_scan_no_history() -> Result<()> {
let temp_dir = TempDir::new().context("tmp dir")?;
let clone_dir = temp_dir.path().to_path_buf();
let test_repo_url = "https://gitlab.com/micksmix/SecretsTest.git";
let git_url = GitUrl::from_str(test_repo_url).expect("parse GitLab URL");
let scan_args = ScanArgs {
num_jobs: 2,
rules: RuleSpecifierArgs {
rules_path: Vec::new(),
rule: vec!["all".into()],
load_builtins: true,
},
input_specifier_args: InputSpecifierArgs {
path_inputs: Vec::new(),
git_url: vec![git_url],
github_user: Vec::new(),
github_organization: Vec::new(),
all_github_organizations: false,
github_api_url: Url::parse("https://api.github.com/")?,
github_repo_type: GitHubRepoType::Source,
gitlab_user: Vec::new(),
gitlab_group: Vec::new(),
all_gitlab_groups: false,
gitlab_api_url: Url::parse("https://gitlab.com/")?,
gitlab_repo_type: GitLabRepoType::Owner,
jira_url: None,
jql: None,
max_results: 100,
slack_query: None,
slack_api_url: Url::parse("https://slack.com/api/").unwrap(),
s3_bucket: None,
s3_prefix: None,
role_arn: None,
aws_local_profile: None,
docker_image: Vec::new(),
git_clone: GitCloneMode::Bare,
git_history: GitHistoryMode::None,
scan_nested_repos: true,
commit_metadata: true,
},
content_filtering_args: ContentFilteringArgs {
max_file_size_mb: 25.0,
no_extract_archives: false,
extraction_depth: 2,
no_binary: true,
exclude: Vec::new(),
},
confidence: ConfidenceLevel::Medium,
no_validate: false,
rule_stats: false,
only_valid: false,
min_entropy: None,
redact: false,
git_repo_timeout: 1800,
output_args: OutputArgs { output: None, format: ReportOutputFormat::Pretty },
no_dedup: true,
baseline_file: None,
manage_baseline: false,
};
let global_args = GlobalArgs {
verbose: 0,
quiet: false,
color: Mode::Auto,
progress: Mode::Auto,
no_update_check: false,
self_update: false,
ignore_certs: false,
advanced: AdvancedArgs { rlimit_nofile: 16_384 },
};
let datastore = Arc::new(Mutex::new(FindingsStore::new(clone_dir)));
let rt = Runtime::new()?;
let rules_db = Arc::new(load_and_record_rules(&scan_args, &datastore)?);
rt.block_on(async {
run_scan(&global_args, &scan_args, &rules_db, Arc::clone(&datastore)).await
})?;
let ds = datastore.lock().unwrap();
let findings = ds.get_matches();
let total = findings.len();
let validated = findings.iter().filter(|m| m.as_ref().2.validation_success).count();
assert!(total >= 10, "expected at least 10 findings from GitLab repo, got {total}");
let exit_code = determine_exit_code(total, validated);
assert!(
exit_code >= 200,
"expected kingfisher to report findings (exit_code >= 200), got {exit_code}"
);
drop(rt);
Ok(())
}

205
tests/int_slack.rs.orig
View file

@ -1,205 +0,0 @@
use std::{
env,
sync::{Arc, Mutex},
};
use anyhow::Result;
use kingfisher::{
cli::{
commands::{
github::{GitCloneMode, GitHistoryMode, GitHubRepoType},
gitlab::GitLabRepoType,
inputs::{ContentFilteringArgs, InputSpecifierArgs},
output::{OutputArgs, ReportOutputFormat},
rules::RuleSpecifierArgs,
scan::{ConfidenceLevel, ScanArgs},
},
global::{AdvancedArgs, Mode},
GlobalArgs,
},
findings_store::FindingsStore,
rule_loader::RuleLoader,
rules_database::RulesDatabase,
scanner::run_async_scan,
};
use tempfile::TempDir;
use url::Url;
use wiremock::{
matchers::{method, path},
Mock, MockServer, ResponseTemplate,
};
struct TestContext {
rules_db: Arc<RulesDatabase>,
}
impl TestContext {
fn new() -> Result<Self> {
let scan_args = ScanArgs {
num_jobs: 2,
rules: RuleSpecifierArgs {
rules_path: Vec::new(),
rule: vec!["all".into()],
load_builtins: true,
},
input_specifier_args: InputSpecifierArgs {
path_inputs: Vec::new(),
git_url: Vec::new(),
github_user: Vec::new(),
github_organization: Vec::new(),
all_github_organizations: false,
github_api_url: Url::parse("https://api.github.com/").unwrap(),
github_repo_type: GitHubRepoType::Source,
gitlab_user: Vec::new(),
gitlab_group: Vec::new(),
all_gitlab_groups: false,
gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(),
gitlab_repo_type: GitLabRepoType::Owner,
jira_url: None,
jql: None,
slack_query: None,
slack_api_url: Url::parse("https://slack.com/api/").unwrap(),
s3_bucket: None,
s3_prefix: None,
role_arn: None,
aws_local_profile: None,
max_results: 10,
docker_image: Vec::new(),
git_clone: GitCloneMode::Bare,
git_history: GitHistoryMode::Full,
scan_nested_repos: true,
commit_metadata: true,
},
content_filtering_args: ContentFilteringArgs {
max_file_size_mb: 25.0,
extraction_depth: 2,
no_binary: true,
no_extract_archives: false,
exclude: Vec::new(),
},
confidence: ConfidenceLevel::Low,
no_validate: true,
rule_stats: false,
only_valid: false,
min_entropy: Some(0.0),
redact: false,
git_repo_timeout: 1800,
output_args: OutputArgs { output: None, format: ReportOutputFormat::Pretty },
no_dedup: true,
baseline_file: None,
manage_baseline: false,
};
let loaded = RuleLoader::from_rule_specifiers(&scan_args.rules).load(&scan_args)?;
let resolved = loaded.resolve_enabled_rules()?;
let rules_db = RulesDatabase::from_rules(resolved.into_iter().cloned().collect())?;
Ok(Self { rules_db: Arc::new(rules_db) })
}
}
#[tokio::test]
async fn test_scan_slack_messages() -> Result<()> {
let ctx = TestContext::new()?;
let server = MockServer::start().await;
let response = serde_json::json!({
"ok": true,
"messages": {
"matches": [{
"permalink": "https://example.slack.com/archives/C123/p1234",
"text": "This contains a github token ghp_1wuHFikBKQtCcH3EB2FBUkyn8krXhP2qLqPa",
"ts": "1234.56",
"channel": {"id": "C123", "name": "general"}
}],
"pagination": {"page": 1, "page_count": 1}
}
});
Mock::given(method("GET"))
.and(path("/search.messages"))
.respond_with(ResponseTemplate::new(200).set_body_json(response))
.mount(&server)
.await;
env::set_var("KF_SLACK_TOKEN", "xoxp-test");
let temp_dir = TempDir::new()?;
let clone_dir = temp_dir.path().to_path_buf();
let scan_args = ScanArgs {
num_jobs: 2,
rules: RuleSpecifierArgs {
rules_path: Vec::new(),
rule: vec!["all".into()],
load_builtins: true,
},
input_specifier_args: InputSpecifierArgs {
path_inputs: Vec::new(),
git_url: Vec::new(),
github_user: Vec::new(),
github_organization: Vec::new(),
all_github_organizations: false,
github_api_url: Url::parse("https://api.github.com/").unwrap(),
github_repo_type: GitHubRepoType::Source,
gitlab_user: Vec::new(),
gitlab_group: Vec::new(),
all_gitlab_groups: false,
gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(),
gitlab_repo_type: GitLabRepoType::Owner,
jira_url: None,
jql: None,
slack_query: Some("test".into()),
slack_api_url: Url::parse(&format!("{}/", server.uri()))?,
max_results: 10,
// s3
s3_bucket: None,
s3_prefix: None,
role_arn: None,
aws_local_profile: None,
docker_image: Vec::new(),
git_clone: GitCloneMode::Bare,
git_history: GitHistoryMode::Full,
scan_nested_repos: true,
commit_metadata: true,
},
content_filtering_args: ContentFilteringArgs {
max_file_size_mb: 25.0,
extraction_depth: 2,
no_binary: true,
no_extract_archives: false,
exclude: Vec::new(),
},
confidence: ConfidenceLevel::Low,
no_validate: true,
rule_stats: false,
only_valid: false,
min_entropy: Some(0.0),
redact: false,
git_repo_timeout: 1800,
output_args: OutputArgs { output: None, format: ReportOutputFormat::Pretty },
no_dedup: true,
baseline_file: None,
manage_baseline: false,
};
let global_args = GlobalArgs {
verbose: 0,
quiet: true,
color: Mode::Auto,
no_update_check: false,
self_update: false,
progress: Mode::Never,
ignore_certs: false,
advanced: AdvancedArgs { rlimit_nofile: 16384 },
};
let datastore = Arc::new(Mutex::new(FindingsStore::new(clone_dir)));
run_async_scan(&global_args, &scan_args, Arc::clone(&datastore), &ctx.rules_db).await?;
let findings = {
let ds = datastore.lock().unwrap();
ds.get_matches().len()
};
assert!(findings > 0);
Ok(())
}

255
tests/int_vulnerable_files.rs.orig
View file

@ -1,255 +0,0 @@
// tests/integration_scan.rs
use std::{
path::{Path, PathBuf},
sync::{Arc, Mutex},
};
use anyhow::{Context, Result};
use kingfisher::{
cli::{
commands::{
github::{GitCloneMode, GitHistoryMode, GitHubRepoType},
gitlab::GitLabRepoType,
inputs::{ContentFilteringArgs, InputSpecifierArgs},
output::{OutputArgs, ReportOutputFormat},
rules::RuleSpecifierArgs,
scan::{ConfidenceLevel, ScanArgs},
},
global::{AdvancedArgs, Mode},
GlobalArgs,
},
findings_store::FindingsStore,
rule_loader::RuleLoader,
rules_database::RulesDatabase,
scanner::run_async_scan,
};
use tempfile::TempDir;
use url::Url;
#[derive(Debug)]
struct TestCase {
file_name: &'static str,
min_expected_findings: usize,
}
struct TestContext {
rules_db: Arc<RulesDatabase>,
}
fn root_dir() -> PathBuf {
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
}
impl TestContext {
fn new() -> Result<Self> {
let scan_args = ScanArgs {
num_jobs: 2,
rules: RuleSpecifierArgs {
rules_path: Vec::new(),
rule: vec!["all".into()],
load_builtins: true,
},
input_specifier_args: InputSpecifierArgs {
path_inputs: Vec::new(),
git_url: Vec::new(),
github_user: Vec::new(),
github_organization: Vec::new(),
all_github_organizations: false,
github_api_url: Url::parse("https://api.github.com/").unwrap(),
github_repo_type: GitHubRepoType::Source,
// new GitLab defaults
gitlab_user: Vec::new(),
gitlab_group: Vec::new(),
all_gitlab_groups: false,
gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(),
gitlab_repo_type: GitLabRepoType::Owner,
jira_url: None,
jql: None,
max_results: 100,
slack_query: None,
slack_api_url: Url::parse("https://slack.com/api/").unwrap(),
// s3
s3_bucket: None,
s3_prefix: None,
role_arn: None,
aws_local_profile: None,
// Docker image scanning
docker_image: Vec::new(),
// git clone / history options
git_clone: GitCloneMode::Bare,
git_history: GitHistoryMode::Full,
scan_nested_repos: true,
commit_metadata: true,
},
content_filtering_args: ContentFilteringArgs {
max_file_size_mb: 25.0,
extraction_depth: 2,
no_binary: true,
no_extract_archives: false,
exclude: Vec::new(), // Exclude patterns
},
confidence: ConfidenceLevel::Low,
no_validate: true,
rule_stats: false,
only_valid: false,
min_entropy: Some(0.0),
redact: false,
git_repo_timeout: 1800, // 30 minutes
output_args: OutputArgs { output: None, format: ReportOutputFormat::Pretty },
no_dedup: true,
baseline_file: None,
manage_baseline: false,
};
let loaded = RuleLoader::from_rule_specifiers(&scan_args.rules)
.load(&scan_args)
.context("Failed to load rules")?;
let resolved = loaded.resolve_enabled_rules().context("Failed to resolve rules")?;
let rules_db = RulesDatabase::from_rules(resolved.into_iter().cloned().collect())
.context("Failed to compile rules")?;
Ok(Self { rules_db: Arc::new(rules_db) })
}
async fn scan_file(&self, file_path: &Path) -> Result<usize> {
let temp_dir = TempDir::new().context("Failed to create temporary directory")?;
let clone_dir = temp_dir.path().to_path_buf();
let scan_args = ScanArgs {
num_jobs: 2,
rules: RuleSpecifierArgs {
rules_path: Vec::new(),
rule: vec!["all".into()],
load_builtins: true,
},
input_specifier_args: InputSpecifierArgs {
path_inputs: vec![file_path.to_path_buf()],
git_url: Vec::new(),
github_user: Vec::new(),
github_organization: Vec::new(),
all_github_organizations: false,
github_api_url: Url::parse("https://api.github.com/").unwrap(),
github_repo_type: GitHubRepoType::Source,
// new GitLab defaults
gitlab_user: Vec::new(),
gitlab_group: Vec::new(),
all_gitlab_groups: false,
gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(),
gitlab_repo_type: GitLabRepoType::Owner,
jira_url: None,
jql: None,
max_results: 100,
slack_query: None,
slack_api_url: Url::parse("https://slack.com/api/").unwrap(),
// s3
s3_bucket: None,
s3_prefix: None,
role_arn: None,
aws_local_profile: None,
// Docker image scanning
docker_image: Vec::new(),
// git clone / history options
git_clone: GitCloneMode::Bare,
git_history: GitHistoryMode::Full,
scan_nested_repos: true,
commit_metadata: true,
},
content_filtering_args: ContentFilteringArgs {
max_file_size_mb: 25.0,
extraction_depth: 2,
no_binary: true,
no_extract_archives: false,
exclude: Vec::new(), // Exclude patterns
},
confidence: ConfidenceLevel::Low,
no_validate: true,
rule_stats: false,
only_valid: false,
min_entropy: Some(0.0),
redact: false,
git_repo_timeout: 1800, // 30 minutes
output_args: OutputArgs { output: None, format: ReportOutputFormat::Pretty },
no_dedup: true,
baseline_file: None,
manage_baseline: false,
};
let global_args = GlobalArgs {
verbose: 0,
quiet: true,
color: Mode::Auto,
no_update_check: false,
self_update: false,
progress: Mode::Never,
ignore_certs: false,
advanced: AdvancedArgs { rlimit_nofile: 16384 },
};
let datastore = Arc::new(Mutex::new(FindingsStore::new(clone_dir)));
run_async_scan(&global_args, &scan_args, Arc::clone(&datastore), &self.rules_db).await?;
let findings = {
let ds = datastore.lock().unwrap();
ds.get_matches().len()
};
Ok(findings)
}
}
#[tokio::test]
async fn test_scan_vulnerable_files() -> Result<()> {
let test_context = TestContext::new()?;
let test_cases = vec![
TestCase { file_name: "testdata/c_vulnerable.c", min_expected_findings: 3 },
TestCase { file_name: "testdata/cpp_vulnerable.cpp", min_expected_findings: 3 },
TestCase { file_name: "testdata/csharp_vulnerable.cs", min_expected_findings: 4 },
TestCase { file_name: "testdata/elixir_vulnerable.exs", min_expected_findings: 1 },
TestCase { file_name: "testdata/generic_secrets.py", min_expected_findings: 9 },
TestCase { file_name: "testdata/go_vulnerable.go", min_expected_findings: 4 },
TestCase { file_name: "testdata/java_vulnerable.java", min_expected_findings: 4 },
TestCase { file_name: "testdata/javascript_vulnerable.js", min_expected_findings: 4 },
TestCase { file_name: "testdata/json_vulnerable.json", min_expected_findings: 4 },
TestCase { file_name: "testdata/kotlin_vulnerable.kt", min_expected_findings: 7 },
TestCase { file_name: "testdata/objc_vulnerable.m", min_expected_findings: 4 },
TestCase { file_name: "testdata/php_vulnerable.php", min_expected_findings: 5 },
TestCase { file_name: "testdata/python_vulnerable.py", min_expected_findings: 10 },
TestCase { file_name: "testdata/python2_vulnerable.py", min_expected_findings: 4 },
TestCase { file_name: "testdata/ruby_vulnerable.rb", min_expected_findings: 6 },
TestCase { file_name: "testdata/rust_vulnerable.rs", min_expected_findings: 3 },
TestCase { file_name: "testdata/scala_vulnerable.scala", min_expected_findings: 3 },
TestCase { file_name: "testdata/shell_vulnerable.sh", min_expected_findings: 2 },
TestCase { file_name: "testdata/slack_tokens.properties", min_expected_findings: 17 },
TestCase { file_name: "testdata/swift_vulnerable.swift", min_expected_findings: 2 },
TestCase { file_name: "testdata/toml_vulnerable.toml", min_expected_findings: 4 },
TestCase { file_name: "testdata/tsx_vulnerable.tsx", min_expected_findings: 1 },
TestCase { file_name: "testdata/typescript_vulnerable.ts", min_expected_findings: 1 },
TestCase { file_name: "testdata/yaml_vulnerable.yaml", min_expected_findings: 4 },
];
let root = root_dir();
for test_case in test_cases {
let test_file = root.join(test_case.file_name);
println!("Testing file: {}", test_case.file_name);
let findings = test_context.scan_file(&test_file).await?;
assert!(
findings >= test_case.min_expected_findings,
"File: {} - Expected >= {} findings, got {}",
test_case.file_name,
test_case.min_expected_findings,
findings
);
}
Ok(())
}