From 94e2e11de390313cc8f2f417164f3cfd3ce1392e Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Sun, 10 Aug 2025 21:54:26 -0700 Subject: [PATCH] Added support for scanning Confluence pages --- tests/int_gitlab.rs.orig | 241 --------------------------- tests/int_slack.rs.orig | 205 ----------------------- tests/int_vulnerable_files.rs.orig | 255 ----------------------------- 3 files changed, 701 deletions(-) delete mode 100644 tests/int_gitlab.rs.orig delete mode 100644 tests/int_slack.rs.orig delete mode 100644 tests/int_vulnerable_files.rs.orig diff --git a/tests/int_gitlab.rs.orig b/tests/int_gitlab.rs.orig deleted file mode 100644 index 4668439..0000000 --- a/tests/int_gitlab.rs.orig +++ /dev/null @@ -1,241 +0,0 @@ -// tests/int_gitlab.rs -use std::{ - str::FromStr, - sync::{Arc, Mutex}, -}; - -use anyhow::{Context, Result}; -use kingfisher::{ - cli::{ - commands::{ - github::{GitCloneMode, GitHistoryMode, GitHubRepoType}, - gitlab::GitLabRepoType, - inputs::{ContentFilteringArgs, InputSpecifierArgs}, - output::{OutputArgs, ReportOutputFormat}, - rules::RuleSpecifierArgs, - scan::{ConfidenceLevel, ScanArgs}, - }, - global::{AdvancedArgs, Mode}, - GlobalArgs, - }, - findings_store::FindingsStore, - git_url::GitUrl, - scanner::{load_and_record_rules, run_scan}, -}; -use tempfile::TempDir; -use tokio::runtime::Runtime; -use url::Url; - -/// Derive process exit-codes from findings -fn determine_exit_code(total: usize, validated: usize) -> i32 { - match (total, validated) { - (0, _) => 0, - (_, v) if v > 0 => 205, - _ => 200, - } -} - -#[test] -fn test_gitlab_remote_scan() -> Result<()> { - let temp_dir = TempDir::new().context("tmp dir")?; - let clone_dir = temp_dir.path().to_path_buf(); - - // Public GitLab repo seeded with test secrets - let test_repo_url = "https://gitlab.com/micksmix/SecretsTest.git"; - let git_url = GitUrl::from_str(test_repo_url).expect("parse GitLab URL"); - - let scan_args = ScanArgs { - num_jobs: 2, - rules: RuleSpecifierArgs { - rules_path: Vec::new(), - rule: vec!["all".into()], - load_builtins: true, - }, - input_specifier_args: InputSpecifierArgs { - path_inputs: Vec::new(), - git_url: vec![git_url], - github_user: Vec::new(), - github_organization: Vec::new(), - all_github_organizations: false, - github_api_url: Url::parse("https://api.github.com/")?, - github_repo_type: GitHubRepoType::Source, - gitlab_user: Vec::new(), - gitlab_group: Vec::new(), - all_gitlab_groups: false, - gitlab_api_url: Url::parse("https://gitlab.com/")?, - gitlab_repo_type: GitLabRepoType::Owner, - - jira_url: None, - jql: None, - max_results: 100, - slack_query: None, - slack_api_url: Url::parse("https://slack.com/api/").unwrap(), - // s3 - s3_bucket: None, - s3_prefix: None, - role_arn: None, - aws_local_profile: None, - // Docker image scanning - docker_image: Vec::new(), - git_clone: GitCloneMode::Bare, - git_history: GitHistoryMode::Full, - scan_nested_repos: true, - commit_metadata: true, - }, - content_filtering_args: ContentFilteringArgs { - max_file_size_mb: 25.0, - no_extract_archives: false, - extraction_depth: 2, - no_binary: true, - exclude: Vec::new(), // Exclude patterns - }, - confidence: ConfidenceLevel::Medium, - no_validate: false, - rule_stats: false, - only_valid: false, - min_entropy: None, - redact: false, - git_repo_timeout: 1800, // 30 minutes - output_args: OutputArgs { output: None, format: ReportOutputFormat::Pretty }, - no_dedup: true, - baseline_file: None, - manage_baseline: false, - }; - - let global_args = GlobalArgs { - verbose: 0, - quiet: false, - color: Mode::Auto, - progress: Mode::Auto, - no_update_check: false, - self_update: false, - ignore_certs: false, - advanced: AdvancedArgs { rlimit_nofile: 16_384 }, - }; - - let datastore = Arc::new(Mutex::new(FindingsStore::new(clone_dir))); - let rt = Runtime::new()?; - - let rules_db = Arc::new(load_and_record_rules(&scan_args, &datastore)?); - - rt.block_on(async { - run_scan(&global_args, &scan_args, &rules_db, Arc::clone(&datastore)).await - })?; - - let ds = datastore.lock().unwrap(); - let findings = ds.get_matches(); - let total = findings.len(); - let validated = findings.iter().filter(|m| m.as_ref().2.validation_success).count(); - - assert!(total >= 10, "expected at least 10 findings from GitLab repo, got {total}"); - - let exit_code = determine_exit_code(total, validated); - assert!( - exit_code >= 200, - "expected kingfisher to report findings (exit_code >= 200), got {exit_code}" - ); - - drop(rt); - Ok(()) -} - -#[test] -fn test_gitlab_remote_scan_no_history() -> Result<()> { - let temp_dir = TempDir::new().context("tmp dir")?; - let clone_dir = temp_dir.path().to_path_buf(); - - let test_repo_url = "https://gitlab.com/micksmix/SecretsTest.git"; - let git_url = GitUrl::from_str(test_repo_url).expect("parse GitLab URL"); - - let scan_args = ScanArgs { - num_jobs: 2, - rules: RuleSpecifierArgs { - rules_path: Vec::new(), - rule: vec!["all".into()], - load_builtins: true, - }, - input_specifier_args: InputSpecifierArgs { - path_inputs: Vec::new(), - git_url: vec![git_url], - github_user: Vec::new(), - github_organization: Vec::new(), - all_github_organizations: false, - github_api_url: Url::parse("https://api.github.com/")?, - github_repo_type: GitHubRepoType::Source, - gitlab_user: Vec::new(), - gitlab_group: Vec::new(), - all_gitlab_groups: false, - gitlab_api_url: Url::parse("https://gitlab.com/")?, - gitlab_repo_type: GitLabRepoType::Owner, - - jira_url: None, - jql: None, - max_results: 100, - slack_query: None, - slack_api_url: Url::parse("https://slack.com/api/").unwrap(), - s3_bucket: None, - s3_prefix: None, - role_arn: None, - aws_local_profile: None, - docker_image: Vec::new(), - git_clone: GitCloneMode::Bare, - git_history: GitHistoryMode::None, - scan_nested_repos: true, - commit_metadata: true, - }, - content_filtering_args: ContentFilteringArgs { - max_file_size_mb: 25.0, - no_extract_archives: false, - extraction_depth: 2, - no_binary: true, - exclude: Vec::new(), - }, - confidence: ConfidenceLevel::Medium, - no_validate: false, - rule_stats: false, - only_valid: false, - min_entropy: None, - redact: false, - git_repo_timeout: 1800, - output_args: OutputArgs { output: None, format: ReportOutputFormat::Pretty }, - no_dedup: true, - baseline_file: None, - manage_baseline: false, - }; - - let global_args = GlobalArgs { - verbose: 0, - quiet: false, - color: Mode::Auto, - progress: Mode::Auto, - no_update_check: false, - self_update: false, - ignore_certs: false, - advanced: AdvancedArgs { rlimit_nofile: 16_384 }, - }; - - let datastore = Arc::new(Mutex::new(FindingsStore::new(clone_dir))); - let rt = Runtime::new()?; - - let rules_db = Arc::new(load_and_record_rules(&scan_args, &datastore)?); - - rt.block_on(async { - run_scan(&global_args, &scan_args, &rules_db, Arc::clone(&datastore)).await - })?; - - let ds = datastore.lock().unwrap(); - let findings = ds.get_matches(); - let total = findings.len(); - let validated = findings.iter().filter(|m| m.as_ref().2.validation_success).count(); - - assert!(total >= 10, "expected at least 10 findings from GitLab repo, got {total}"); - - let exit_code = determine_exit_code(total, validated); - assert!( - exit_code >= 200, - "expected kingfisher to report findings (exit_code >= 200), got {exit_code}" - ); - - drop(rt); - Ok(()) -} diff --git a/tests/int_slack.rs.orig b/tests/int_slack.rs.orig deleted file mode 100644 index 7284e1d..0000000 --- a/tests/int_slack.rs.orig +++ /dev/null @@ -1,205 +0,0 @@ -use std::{ - env, - sync::{Arc, Mutex}, -}; - -use anyhow::Result; -use kingfisher::{ - cli::{ - commands::{ - github::{GitCloneMode, GitHistoryMode, GitHubRepoType}, - gitlab::GitLabRepoType, - inputs::{ContentFilteringArgs, InputSpecifierArgs}, - output::{OutputArgs, ReportOutputFormat}, - rules::RuleSpecifierArgs, - scan::{ConfidenceLevel, ScanArgs}, - }, - global::{AdvancedArgs, Mode}, - GlobalArgs, - }, - findings_store::FindingsStore, - rule_loader::RuleLoader, - rules_database::RulesDatabase, - scanner::run_async_scan, -}; -use tempfile::TempDir; -use url::Url; -use wiremock::{ - matchers::{method, path}, - Mock, MockServer, ResponseTemplate, -}; - -struct TestContext { - rules_db: Arc, -} - -impl TestContext { - fn new() -> Result { - let scan_args = ScanArgs { - num_jobs: 2, - rules: RuleSpecifierArgs { - rules_path: Vec::new(), - rule: vec!["all".into()], - load_builtins: true, - }, - input_specifier_args: InputSpecifierArgs { - path_inputs: Vec::new(), - git_url: Vec::new(), - github_user: Vec::new(), - github_organization: Vec::new(), - all_github_organizations: false, - github_api_url: Url::parse("https://api.github.com/").unwrap(), - github_repo_type: GitHubRepoType::Source, - gitlab_user: Vec::new(), - gitlab_group: Vec::new(), - all_gitlab_groups: false, - gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), - gitlab_repo_type: GitLabRepoType::Owner, - jira_url: None, - jql: None, - slack_query: None, - slack_api_url: Url::parse("https://slack.com/api/").unwrap(), - s3_bucket: None, - s3_prefix: None, - role_arn: None, - aws_local_profile: None, - max_results: 10, - docker_image: Vec::new(), - git_clone: GitCloneMode::Bare, - git_history: GitHistoryMode::Full, - scan_nested_repos: true, - commit_metadata: true, - }, - content_filtering_args: ContentFilteringArgs { - max_file_size_mb: 25.0, - extraction_depth: 2, - no_binary: true, - no_extract_archives: false, - exclude: Vec::new(), - }, - confidence: ConfidenceLevel::Low, - no_validate: true, - rule_stats: false, - only_valid: false, - min_entropy: Some(0.0), - redact: false, - git_repo_timeout: 1800, - output_args: OutputArgs { output: None, format: ReportOutputFormat::Pretty }, - no_dedup: true, - baseline_file: None, - manage_baseline: false, - }; - - let loaded = RuleLoader::from_rule_specifiers(&scan_args.rules).load(&scan_args)?; - let resolved = loaded.resolve_enabled_rules()?; - let rules_db = RulesDatabase::from_rules(resolved.into_iter().cloned().collect())?; - Ok(Self { rules_db: Arc::new(rules_db) }) - } -} - -#[tokio::test] -async fn test_scan_slack_messages() -> Result<()> { - let ctx = TestContext::new()?; - - let server = MockServer::start().await; - let response = serde_json::json!({ - "ok": true, - "messages": { - "matches": [{ - "permalink": "https://example.slack.com/archives/C123/p1234", - "text": "This contains a github token ghp_1wuHFikBKQtCcH3EB2FBUkyn8krXhP2qLqPa", - "ts": "1234.56", - "channel": {"id": "C123", "name": "general"} - }], - "pagination": {"page": 1, "page_count": 1} - } - }); - Mock::given(method("GET")) - .and(path("/search.messages")) - .respond_with(ResponseTemplate::new(200).set_body_json(response)) - .mount(&server) - .await; - - env::set_var("KF_SLACK_TOKEN", "xoxp-test"); - - let temp_dir = TempDir::new()?; - let clone_dir = temp_dir.path().to_path_buf(); - - let scan_args = ScanArgs { - num_jobs: 2, - rules: RuleSpecifierArgs { - rules_path: Vec::new(), - rule: vec!["all".into()], - load_builtins: true, - }, - input_specifier_args: InputSpecifierArgs { - path_inputs: Vec::new(), - git_url: Vec::new(), - github_user: Vec::new(), - github_organization: Vec::new(), - all_github_organizations: false, - github_api_url: Url::parse("https://api.github.com/").unwrap(), - github_repo_type: GitHubRepoType::Source, - gitlab_user: Vec::new(), - gitlab_group: Vec::new(), - all_gitlab_groups: false, - gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), - gitlab_repo_type: GitLabRepoType::Owner, - jira_url: None, - jql: None, - slack_query: Some("test".into()), - slack_api_url: Url::parse(&format!("{}/", server.uri()))?, - max_results: 10, - // s3 - s3_bucket: None, - s3_prefix: None, - role_arn: None, - aws_local_profile: None, - docker_image: Vec::new(), - git_clone: GitCloneMode::Bare, - git_history: GitHistoryMode::Full, - scan_nested_repos: true, - commit_metadata: true, - }, - content_filtering_args: ContentFilteringArgs { - max_file_size_mb: 25.0, - extraction_depth: 2, - no_binary: true, - no_extract_archives: false, - exclude: Vec::new(), - }, - confidence: ConfidenceLevel::Low, - no_validate: true, - rule_stats: false, - only_valid: false, - min_entropy: Some(0.0), - redact: false, - git_repo_timeout: 1800, - output_args: OutputArgs { output: None, format: ReportOutputFormat::Pretty }, - no_dedup: true, - baseline_file: None, - manage_baseline: false, - }; - - let global_args = GlobalArgs { - verbose: 0, - quiet: true, - color: Mode::Auto, - no_update_check: false, - self_update: false, - progress: Mode::Never, - ignore_certs: false, - advanced: AdvancedArgs { rlimit_nofile: 16384 }, - }; - - let datastore = Arc::new(Mutex::new(FindingsStore::new(clone_dir))); - - run_async_scan(&global_args, &scan_args, Arc::clone(&datastore), &ctx.rules_db).await?; - - let findings = { - let ds = datastore.lock().unwrap(); - ds.get_matches().len() - }; - assert!(findings > 0); - Ok(()) -} diff --git a/tests/int_vulnerable_files.rs.orig b/tests/int_vulnerable_files.rs.orig deleted file mode 100644 index c53adae..0000000 --- a/tests/int_vulnerable_files.rs.orig +++ /dev/null @@ -1,255 +0,0 @@ -// tests/integration_scan.rs - -use std::{ - path::{Path, PathBuf}, - sync::{Arc, Mutex}, -}; - -use anyhow::{Context, Result}; -use kingfisher::{ - cli::{ - commands::{ - github::{GitCloneMode, GitHistoryMode, GitHubRepoType}, - gitlab::GitLabRepoType, - inputs::{ContentFilteringArgs, InputSpecifierArgs}, - output::{OutputArgs, ReportOutputFormat}, - rules::RuleSpecifierArgs, - scan::{ConfidenceLevel, ScanArgs}, - }, - global::{AdvancedArgs, Mode}, - GlobalArgs, - }, - findings_store::FindingsStore, - rule_loader::RuleLoader, - rules_database::RulesDatabase, - scanner::run_async_scan, -}; -use tempfile::TempDir; -use url::Url; - -#[derive(Debug)] -struct TestCase { - file_name: &'static str, - min_expected_findings: usize, -} - -struct TestContext { - rules_db: Arc, -} - -fn root_dir() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) -} - -impl TestContext { - fn new() -> Result { - let scan_args = ScanArgs { - num_jobs: 2, - rules: RuleSpecifierArgs { - rules_path: Vec::new(), - rule: vec!["all".into()], - load_builtins: true, - }, - input_specifier_args: InputSpecifierArgs { - path_inputs: Vec::new(), - git_url: Vec::new(), - github_user: Vec::new(), - github_organization: Vec::new(), - all_github_organizations: false, - github_api_url: Url::parse("https://api.github.com/").unwrap(), - github_repo_type: GitHubRepoType::Source, - // new GitLab defaults - gitlab_user: Vec::new(), - gitlab_group: Vec::new(), - all_gitlab_groups: false, - gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), - gitlab_repo_type: GitLabRepoType::Owner, - - jira_url: None, - jql: None, - max_results: 100, - slack_query: None, - slack_api_url: Url::parse("https://slack.com/api/").unwrap(), - // s3 - s3_bucket: None, - s3_prefix: None, - role_arn: None, - aws_local_profile: None, - // Docker image scanning - docker_image: Vec::new(), - // git clone / history options - git_clone: GitCloneMode::Bare, - git_history: GitHistoryMode::Full, - scan_nested_repos: true, - commit_metadata: true, - }, - content_filtering_args: ContentFilteringArgs { - max_file_size_mb: 25.0, - extraction_depth: 2, - no_binary: true, - no_extract_archives: false, - exclude: Vec::new(), // Exclude patterns - }, - confidence: ConfidenceLevel::Low, - no_validate: true, - rule_stats: false, - only_valid: false, - min_entropy: Some(0.0), - redact: false, - git_repo_timeout: 1800, // 30 minutes - output_args: OutputArgs { output: None, format: ReportOutputFormat::Pretty }, - no_dedup: true, - baseline_file: None, - manage_baseline: false, - }; - - let loaded = RuleLoader::from_rule_specifiers(&scan_args.rules) - .load(&scan_args) - .context("Failed to load rules")?; - - let resolved = loaded.resolve_enabled_rules().context("Failed to resolve rules")?; - - let rules_db = RulesDatabase::from_rules(resolved.into_iter().cloned().collect()) - .context("Failed to compile rules")?; - - Ok(Self { rules_db: Arc::new(rules_db) }) - } - - async fn scan_file(&self, file_path: &Path) -> Result { - let temp_dir = TempDir::new().context("Failed to create temporary directory")?; - let clone_dir = temp_dir.path().to_path_buf(); - - let scan_args = ScanArgs { - num_jobs: 2, - rules: RuleSpecifierArgs { - rules_path: Vec::new(), - rule: vec!["all".into()], - load_builtins: true, - }, - input_specifier_args: InputSpecifierArgs { - path_inputs: vec![file_path.to_path_buf()], - git_url: Vec::new(), - github_user: Vec::new(), - github_organization: Vec::new(), - all_github_organizations: false, - github_api_url: Url::parse("https://api.github.com/").unwrap(), - github_repo_type: GitHubRepoType::Source, - // new GitLab defaults - gitlab_user: Vec::new(), - gitlab_group: Vec::new(), - all_gitlab_groups: false, - gitlab_api_url: Url::parse("https://gitlab.com/").unwrap(), - gitlab_repo_type: GitLabRepoType::Owner, - - jira_url: None, - jql: None, - max_results: 100, - slack_query: None, - slack_api_url: Url::parse("https://slack.com/api/").unwrap(), - // s3 - s3_bucket: None, - s3_prefix: None, - role_arn: None, - aws_local_profile: None, - // Docker image scanning - docker_image: Vec::new(), - // git clone / history options - git_clone: GitCloneMode::Bare, - git_history: GitHistoryMode::Full, - scan_nested_repos: true, - commit_metadata: true, - }, - content_filtering_args: ContentFilteringArgs { - max_file_size_mb: 25.0, - extraction_depth: 2, - no_binary: true, - no_extract_archives: false, - exclude: Vec::new(), // Exclude patterns - }, - confidence: ConfidenceLevel::Low, - no_validate: true, - rule_stats: false, - only_valid: false, - min_entropy: Some(0.0), - redact: false, - git_repo_timeout: 1800, // 30 minutes - output_args: OutputArgs { output: None, format: ReportOutputFormat::Pretty }, - no_dedup: true, - baseline_file: None, - manage_baseline: false, - }; - - let global_args = GlobalArgs { - verbose: 0, - quiet: true, - color: Mode::Auto, - no_update_check: false, - self_update: false, - progress: Mode::Never, - ignore_certs: false, - advanced: AdvancedArgs { rlimit_nofile: 16384 }, - }; - - let datastore = Arc::new(Mutex::new(FindingsStore::new(clone_dir))); - - run_async_scan(&global_args, &scan_args, Arc::clone(&datastore), &self.rules_db).await?; - - let findings = { - let ds = datastore.lock().unwrap(); - ds.get_matches().len() - }; - - Ok(findings) - } -} - -#[tokio::test] -async fn test_scan_vulnerable_files() -> Result<()> { - let test_context = TestContext::new()?; - - let test_cases = vec![ - TestCase { file_name: "testdata/c_vulnerable.c", min_expected_findings: 3 }, - TestCase { file_name: "testdata/cpp_vulnerable.cpp", min_expected_findings: 3 }, - TestCase { file_name: "testdata/csharp_vulnerable.cs", min_expected_findings: 4 }, - TestCase { file_name: "testdata/elixir_vulnerable.exs", min_expected_findings: 1 }, - TestCase { file_name: "testdata/generic_secrets.py", min_expected_findings: 9 }, - TestCase { file_name: "testdata/go_vulnerable.go", min_expected_findings: 4 }, - TestCase { file_name: "testdata/java_vulnerable.java", min_expected_findings: 4 }, - TestCase { file_name: "testdata/javascript_vulnerable.js", min_expected_findings: 4 }, - TestCase { file_name: "testdata/json_vulnerable.json", min_expected_findings: 4 }, - TestCase { file_name: "testdata/kotlin_vulnerable.kt", min_expected_findings: 7 }, - TestCase { file_name: "testdata/objc_vulnerable.m", min_expected_findings: 4 }, - TestCase { file_name: "testdata/php_vulnerable.php", min_expected_findings: 5 }, - TestCase { file_name: "testdata/python_vulnerable.py", min_expected_findings: 10 }, - TestCase { file_name: "testdata/python2_vulnerable.py", min_expected_findings: 4 }, - TestCase { file_name: "testdata/ruby_vulnerable.rb", min_expected_findings: 6 }, - TestCase { file_name: "testdata/rust_vulnerable.rs", min_expected_findings: 3 }, - TestCase { file_name: "testdata/scala_vulnerable.scala", min_expected_findings: 3 }, - TestCase { file_name: "testdata/shell_vulnerable.sh", min_expected_findings: 2 }, - TestCase { file_name: "testdata/slack_tokens.properties", min_expected_findings: 17 }, - TestCase { file_name: "testdata/swift_vulnerable.swift", min_expected_findings: 2 }, - TestCase { file_name: "testdata/toml_vulnerable.toml", min_expected_findings: 4 }, - TestCase { file_name: "testdata/tsx_vulnerable.tsx", min_expected_findings: 1 }, - TestCase { file_name: "testdata/typescript_vulnerable.ts", min_expected_findings: 1 }, - TestCase { file_name: "testdata/yaml_vulnerable.yaml", min_expected_findings: 4 }, - ]; - - let root = root_dir(); - - for test_case in test_cases { - let test_file = root.join(test_case.file_name); - println!("Testing file: {}", test_case.file_name); - - let findings = test_context.scan_file(&test_file).await?; - - assert!( - findings >= test_case.min_expected_findings, - "File: {} - Expected >= {} findings, got {}", - test_case.file_name, - test_case.min_expected_findings, - findings - ); - } - - Ok(()) -}