forked from mirrors/kingfisher
Added provider endpoint overrides for validation and revocation via global --endpoint PROVIDER=URL and --endpoint-config FILE, with built-in support for self-hosted GitHub, GitLab, Gitea, Jira, Confluence, and Artifactory instances.
This commit is contained in:
parent
5465d903cf
commit
19dafa42ea
19 changed files with 790 additions and 141 deletions
|
|
@ -1,4 +1,5 @@
|
|||
use std::io::IsTerminal;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
|
|
@ -144,6 +145,16 @@ pub struct GlobalArgs {
|
|||
#[arg(global = true, long = "user-agent-suffix", value_name = "SUFFIX")]
|
||||
pub user_agent_suffix: Option<String>,
|
||||
|
||||
/// Override provider API endpoints for validation/revocation (PROVIDER=URL), repeatable.
|
||||
///
|
||||
/// Supported providers: github, gitlab, gitea, jira, jira-cloud, confluence, artifactory.
|
||||
#[arg(global = true, long = "endpoint", value_name = "PROVIDER=URL")]
|
||||
pub endpoint: Vec<String>,
|
||||
|
||||
/// YAML file containing provider endpoint overrides.
|
||||
#[arg(global = true, long = "endpoint-config", value_name = "FILE")]
|
||||
pub endpoint_config: Option<PathBuf>,
|
||||
|
||||
// Internal fields (not CLI arguments)
|
||||
#[clap(skip)]
|
||||
pub color: Mode,
|
||||
|
|
@ -163,6 +174,8 @@ impl Default for GlobalArgs {
|
|||
self_update: false,
|
||||
no_update_check: false,
|
||||
user_agent_suffix: None,
|
||||
endpoint: Vec::new(),
|
||||
endpoint_config: None,
|
||||
color: Mode::Auto,
|
||||
progress: Mode::Auto,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ use tracing::debug;
|
|||
use crate::{
|
||||
cli::{commands::revoke::RevokeArgs, global::GlobalArgs},
|
||||
liquid_filters::register_all,
|
||||
provider_endpoints::{ProviderEndpointOverrides, hydrate_endpoint_globals_for_rule},
|
||||
rule_loader::RuleLoader,
|
||||
template_vars::extract_template_vars,
|
||||
validation::GLOBAL_USER_AGENT,
|
||||
|
|
@ -138,15 +139,22 @@ fn get_global_var(globals: &Object, name: &str) -> Option<String> {
|
|||
|
||||
/// Build the globals object for Liquid template rendering.
|
||||
fn build_globals(
|
||||
rule_id: &str,
|
||||
secret: &str,
|
||||
args: &[String],
|
||||
variables: &[String],
|
||||
template_vars: &BTreeSet<String>,
|
||||
endpoint_overrides: &ProviderEndpointOverrides,
|
||||
) -> Result<Object> {
|
||||
let mut globals = Object::new();
|
||||
globals.insert("TOKEN".into(), Value::scalar(secret.to_string()));
|
||||
|
||||
let auto_assign_vars: Vec<&String> = template_vars.iter().filter(|v| *v != "TOKEN").collect();
|
||||
endpoint_overrides.apply_defaults(&mut globals);
|
||||
|
||||
let auto_assign_vars: Vec<&String> = template_vars
|
||||
.iter()
|
||||
.filter(|v| *v != "TOKEN" && !globals.contains_key(v.as_str()))
|
||||
.collect();
|
||||
|
||||
for (i, arg_value) in args.iter().enumerate() {
|
||||
if i < auto_assign_vars.len() {
|
||||
|
|
@ -171,6 +179,8 @@ fn build_globals(
|
|||
globals.insert(name.into(), Value::scalar(value));
|
||||
}
|
||||
|
||||
hydrate_endpoint_globals_for_rule(rule_id, &mut globals);
|
||||
|
||||
Ok(globals)
|
||||
}
|
||||
|
||||
|
|
@ -553,6 +563,7 @@ pub async fn run_direct_revocation(
|
|||
|
||||
let parser = register_all(liquid::ParserBuilder::with_stdlib()).build()?;
|
||||
let timeout = Duration::from_secs(args.timeout);
|
||||
let endpoint_overrides = ProviderEndpointOverrides::from_global_args(global_args)?;
|
||||
|
||||
let mut results = Vec::new();
|
||||
|
||||
|
|
@ -597,7 +608,14 @@ pub async fn run_direct_revocation(
|
|||
}
|
||||
}
|
||||
|
||||
let globals = build_globals(&secret, &args.args, &args.variables, &template_vars)?;
|
||||
let globals = build_globals(
|
||||
&rule_id,
|
||||
&secret,
|
||||
&args.args,
|
||||
&args.variables,
|
||||
&template_vars,
|
||||
&endpoint_overrides,
|
||||
)?;
|
||||
|
||||
if !non_token_vars.is_empty() && !args.args.is_empty() {
|
||||
debug!(
|
||||
|
|
@ -1028,7 +1046,15 @@ mod tests {
|
|||
#[test]
|
||||
fn build_globals_sets_token() {
|
||||
let template_vars = BTreeSet::from(["TOKEN".to_string()]);
|
||||
let globals = build_globals("my-secret", &[], &[], &template_vars).unwrap();
|
||||
let globals = build_globals(
|
||||
"kingfisher.test.1",
|
||||
"my-secret",
|
||||
&[],
|
||||
&[],
|
||||
&template_vars,
|
||||
&ProviderEndpointOverrides::default(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(globals.get("TOKEN"), Some(Value::scalar("my-secret".to_string())).as_ref());
|
||||
}
|
||||
|
||||
|
|
@ -1037,7 +1063,15 @@ mod tests {
|
|||
let template_vars =
|
||||
BTreeSet::from(["TOKEN".to_string(), "AKID".to_string(), "REGION".to_string()]);
|
||||
let args = vec!["my-akid".to_string(), "us-east-1".to_string()];
|
||||
let globals = build_globals("secret", &args, &[], &template_vars).unwrap();
|
||||
let globals = build_globals(
|
||||
"kingfisher.test.1",
|
||||
"secret",
|
||||
&args,
|
||||
&[],
|
||||
&template_vars,
|
||||
&ProviderEndpointOverrides::default(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(globals.get("TOKEN"), Some(Value::scalar("secret".to_string())).as_ref());
|
||||
assert_eq!(globals.get("AKID"), Some(Value::scalar("my-akid".to_string())).as_ref());
|
||||
|
|
@ -1048,7 +1082,15 @@ mod tests {
|
|||
fn build_globals_explicit_variables() {
|
||||
let template_vars = BTreeSet::from(["TOKEN".to_string(), "AKID".to_string()]);
|
||||
let vars = vec!["AKID=explicit-value".to_string()];
|
||||
let globals = build_globals("secret", &[], &vars, &template_vars).unwrap();
|
||||
let globals = build_globals(
|
||||
"kingfisher.test.1",
|
||||
"secret",
|
||||
&[],
|
||||
&vars,
|
||||
&template_vars,
|
||||
&ProviderEndpointOverrides::default(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(globals.get("AKID"), Some(Value::scalar("explicit-value".to_string())).as_ref());
|
||||
}
|
||||
|
|
@ -1057,7 +1099,14 @@ mod tests {
|
|||
fn build_globals_invalid_var_format() {
|
||||
let template_vars = BTreeSet::new();
|
||||
let vars = vec!["NO_EQUALS_SIGN".to_string()];
|
||||
let result = build_globals("secret", &[], &vars, &template_vars);
|
||||
let result = build_globals(
|
||||
"kingfisher.test.1",
|
||||
"secret",
|
||||
&[],
|
||||
&vars,
|
||||
&template_vars,
|
||||
&ProviderEndpointOverrides::default(),
|
||||
);
|
||||
assert!(result.is_err());
|
||||
assert!(result.unwrap_err().to_string().contains("Expected NAME=VALUE"));
|
||||
}
|
||||
|
|
@ -1066,7 +1115,14 @@ mod tests {
|
|||
fn build_globals_empty_var_name() {
|
||||
let template_vars = BTreeSet::new();
|
||||
let vars = vec!["=value".to_string()];
|
||||
let result = build_globals("secret", &[], &vars, &template_vars);
|
||||
let result = build_globals(
|
||||
"kingfisher.test.1",
|
||||
"secret",
|
||||
&[],
|
||||
&vars,
|
||||
&template_vars,
|
||||
&ProviderEndpointOverrides::default(),
|
||||
);
|
||||
assert!(result.is_err());
|
||||
assert!(result.unwrap_err().to_string().contains("cannot be empty"));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ use tracing::debug;
|
|||
use crate::{
|
||||
cli::{commands::validate::ValidateArgs, global::GlobalArgs},
|
||||
liquid_filters::register_all,
|
||||
provider_endpoints::{ProviderEndpointOverrides, hydrate_endpoint_globals_for_rule},
|
||||
rule_loader::RuleLoader,
|
||||
rules::{HttpValidation, Validation, rule::Rule},
|
||||
template_vars::extract_template_vars,
|
||||
|
|
@ -210,18 +211,25 @@ fn extract_validation_vars(validation: &Validation) -> BTreeSet<String> {
|
|||
/// - `variables`: Named variables in NAME=VALUE format (explicit overrides)
|
||||
/// - `template_vars`: Set of variable names used in the validation template
|
||||
fn build_globals(
|
||||
rule_id: &str,
|
||||
secret: &str,
|
||||
args: &[String],
|
||||
variables: &[String],
|
||||
template_vars: &BTreeSet<String>,
|
||||
endpoint_overrides: &ProviderEndpointOverrides,
|
||||
) -> Result<Object> {
|
||||
let mut globals = Object::new();
|
||||
|
||||
// Set TOKEN to the provided secret
|
||||
globals.insert("TOKEN".into(), Value::scalar(secret.to_string()));
|
||||
|
||||
endpoint_overrides.apply_defaults(&mut globals);
|
||||
|
||||
// Get non-TOKEN variables in alphabetical order for auto-assignment
|
||||
let auto_assign_vars: Vec<&String> = template_vars.iter().filter(|v| *v != "TOKEN").collect();
|
||||
let auto_assign_vars: Vec<&String> = template_vars
|
||||
.iter()
|
||||
.filter(|v| *v != "TOKEN" && !globals.contains_key(v.as_str()))
|
||||
.collect();
|
||||
|
||||
// Auto-assign --arg values to template variables
|
||||
for (i, arg_value) in args.iter().enumerate() {
|
||||
|
|
@ -248,6 +256,8 @@ fn build_globals(
|
|||
globals.insert(name.into(), Value::scalar(value));
|
||||
}
|
||||
|
||||
hydrate_endpoint_globals_for_rule(rule_id, &mut globals);
|
||||
|
||||
Ok(globals)
|
||||
}
|
||||
|
||||
|
|
@ -469,6 +479,7 @@ pub async fn run_direct_validation(
|
|||
|
||||
// Build Liquid parser
|
||||
let parser = register_all(liquid::ParserBuilder::with_stdlib()).build()?;
|
||||
let endpoint_overrides = ProviderEndpointOverrides::from_global_args(global_args)?;
|
||||
|
||||
let timeout = Duration::from_secs(args.timeout);
|
||||
let rate_limiter =
|
||||
|
|
@ -525,7 +536,14 @@ pub async fn run_direct_validation(
|
|||
}
|
||||
}
|
||||
|
||||
let globals = build_globals(&secret, &args.args, &args.variables, &template_vars)?;
|
||||
let globals = build_globals(
|
||||
&rule_id,
|
||||
&secret,
|
||||
&args.args,
|
||||
&args.variables,
|
||||
&template_vars,
|
||||
&endpoint_overrides,
|
||||
)?;
|
||||
|
||||
// Log auto-assignment info for debugging
|
||||
if !non_token_vars.is_empty() && !args.args.is_empty() {
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ pub mod location;
|
|||
pub mod matcher;
|
||||
pub mod origin;
|
||||
pub mod parser;
|
||||
pub mod provider_endpoints;
|
||||
pub mod pyc;
|
||||
pub mod reporter;
|
||||
pub mod rule_loader;
|
||||
|
|
|
|||
409
src/provider_endpoints.rs
Normal file
409
src/provider_endpoints.rs
Normal file
|
|
@ -0,0 +1,409 @@
|
|||
use std::{collections::BTreeMap, fs, path::Path};
|
||||
|
||||
use anyhow::{Context, Result, anyhow, bail};
|
||||
use liquid::Object;
|
||||
use liquid_core::{Value, ValueView};
|
||||
use serde::Deserialize;
|
||||
use url::Url;
|
||||
|
||||
use crate::cli::global::GlobalArgs;
|
||||
|
||||
const GITHUB_API_BASE_URL: &str = "GITHUB_API_BASE_URL";
|
||||
const GITHUB_WEB_BASE_URL: &str = "GITHUB_WEB_BASE_URL";
|
||||
const GITLAB_API_BASE_URL: &str = "GITLAB_API_BASE_URL";
|
||||
const GITEA_API_BASE_URL: &str = "GITEA_API_BASE_URL";
|
||||
const JIRA_BASE_URL: &str = "JIRA_BASE_URL";
|
||||
const JIRA_CLOUD_BASE_URL: &str = "JIRA_CLOUD_BASE_URL";
|
||||
const CONFLUENCE_BASE_URL: &str = "CONFLUENCE_BASE_URL";
|
||||
const ARTIFACTORY_BASE_URL: &str = "ARTIFACTORY_BASE_URL";
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ProviderEndpointOverrides {
|
||||
config: EndpointVars,
|
||||
cli: EndpointVars,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
struct EndpointVars {
|
||||
values: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Default)]
|
||||
struct EndpointConfigFile {
|
||||
#[serde(default)]
|
||||
endpoints: BTreeMap<String, String>,
|
||||
#[serde(default)]
|
||||
provider_endpoints: BTreeMap<String, String>,
|
||||
#[serde(default)]
|
||||
providers: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl ProviderEndpointOverrides {
|
||||
pub fn from_global_args(global_args: &GlobalArgs) -> Result<Self> {
|
||||
let config = match &global_args.endpoint_config {
|
||||
Some(path) => EndpointVars::from_config_path(path)?,
|
||||
None => EndpointVars::default(),
|
||||
};
|
||||
let cli = EndpointVars::from_pairs(&global_args.endpoint)?;
|
||||
Ok(Self { config, cli })
|
||||
}
|
||||
|
||||
pub fn apply_defaults(&self, globals: &mut Object) {
|
||||
self.config.apply(globals, false);
|
||||
apply_builtin_defaults(globals);
|
||||
self.cli.apply(globals, true);
|
||||
}
|
||||
|
||||
pub fn apply_scan_overrides(&self, globals: &mut Object) {
|
||||
self.config.apply(globals, false);
|
||||
apply_builtin_defaults(globals);
|
||||
self.cli.apply(globals, true);
|
||||
}
|
||||
}
|
||||
|
||||
impl EndpointVars {
|
||||
fn from_config_path(path: &Path) -> Result<Self> {
|
||||
let raw = fs::read_to_string(path)
|
||||
.with_context(|| format!("Failed to read endpoint config from {}", path.display()))?;
|
||||
let parsed: EndpointConfigFile = serde_yaml::from_str(&raw)
|
||||
.with_context(|| format!("Failed to parse endpoint config {}", path.display()))?;
|
||||
|
||||
let mut merged = parsed.endpoints;
|
||||
merged.extend(parsed.provider_endpoints);
|
||||
merged.extend(parsed.providers);
|
||||
Self::from_map(merged)
|
||||
}
|
||||
|
||||
fn from_pairs(pairs: &[String]) -> Result<Self> {
|
||||
let mut map = BTreeMap::new();
|
||||
for pair in pairs {
|
||||
let (provider, endpoint) = parse_assignment(pair)?;
|
||||
map.insert(provider, endpoint);
|
||||
}
|
||||
Self::from_map(map)
|
||||
}
|
||||
|
||||
fn from_map(map: BTreeMap<String, String>) -> Result<Self> {
|
||||
let mut values = BTreeMap::new();
|
||||
for (provider, endpoint) in map {
|
||||
let normalized = normalize_endpoint_key(&provider);
|
||||
match normalized.as_str() {
|
||||
"github" => {
|
||||
let github = normalize_github_endpoint(&endpoint)?;
|
||||
values.insert(GITHUB_API_BASE_URL.to_string(), github.api_base_url);
|
||||
values.insert(GITHUB_WEB_BASE_URL.to_string(), github.web_base_url);
|
||||
}
|
||||
"gitlab" => {
|
||||
values.insert(
|
||||
GITLAB_API_BASE_URL.to_string(),
|
||||
normalize_api_base_url(&endpoint, "/api/v4")?,
|
||||
);
|
||||
}
|
||||
"gitea" => {
|
||||
values.insert(
|
||||
GITEA_API_BASE_URL.to_string(),
|
||||
normalize_api_base_url(&endpoint, "/api/v1")?,
|
||||
);
|
||||
}
|
||||
"jira" | "jira-dc" => {
|
||||
values.insert(JIRA_BASE_URL.to_string(), normalize_base_url(&endpoint)?);
|
||||
}
|
||||
"jira-cloud" => {
|
||||
values.insert(JIRA_CLOUD_BASE_URL.to_string(), normalize_base_url(&endpoint)?);
|
||||
}
|
||||
"confluence" | "confluence-dc" => {
|
||||
values.insert(CONFLUENCE_BASE_URL.to_string(), normalize_base_url(&endpoint)?);
|
||||
}
|
||||
"artifactory" | "jfrog" => {
|
||||
values.insert(
|
||||
ARTIFACTORY_BASE_URL.to_string(),
|
||||
normalize_artifactory_base_url(&endpoint)?,
|
||||
);
|
||||
}
|
||||
_ => bail!(
|
||||
"Unsupported endpoint provider '{}'. Supported values: github, gitlab, gitea, jira, jira-cloud, confluence, artifactory",
|
||||
provider
|
||||
),
|
||||
}
|
||||
}
|
||||
Ok(Self { values })
|
||||
}
|
||||
|
||||
fn apply(&self, globals: &mut Object, overwrite_existing: bool) {
|
||||
for (name, value) in &self.values {
|
||||
if overwrite_existing || !globals.contains_key(name.as_str()) {
|
||||
globals.insert(name.clone().into(), Value::scalar(value.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct GitHubEndpoint {
|
||||
api_base_url: String,
|
||||
web_base_url: String,
|
||||
}
|
||||
|
||||
pub fn hydrate_endpoint_globals_for_rule(rule_id: &str, globals: &mut Object) {
|
||||
hydrate_github_globals(globals);
|
||||
hydrate_artifactory_globals(globals);
|
||||
hydrate_confluence_globals(globals);
|
||||
hydrate_jira_dc_globals(globals);
|
||||
if rule_id == "kingfisher.jira.2" {
|
||||
hydrate_jira_cloud_globals(globals);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn endpoint_var_names() -> &'static [&'static str] {
|
||||
&[
|
||||
GITHUB_API_BASE_URL,
|
||||
GITHUB_WEB_BASE_URL,
|
||||
GITLAB_API_BASE_URL,
|
||||
GITEA_API_BASE_URL,
|
||||
JIRA_BASE_URL,
|
||||
JIRA_CLOUD_BASE_URL,
|
||||
CONFLUENCE_BASE_URL,
|
||||
ARTIFACTORY_BASE_URL,
|
||||
]
|
||||
}
|
||||
|
||||
fn hydrate_github_globals(globals: &mut Object) {
|
||||
match (string_var(globals, GITHUB_API_BASE_URL), string_var(globals, GITHUB_WEB_BASE_URL)) {
|
||||
(Some(api), None) => {
|
||||
if let Ok(normalized) = normalize_github_endpoint(&api) {
|
||||
globals.insert(GITHUB_API_BASE_URL.into(), Value::scalar(normalized.api_base_url));
|
||||
globals.insert(GITHUB_WEB_BASE_URL.into(), Value::scalar(normalized.web_base_url));
|
||||
}
|
||||
}
|
||||
(None, Some(web)) => {
|
||||
if let Ok(normalized) = normalize_github_endpoint(&web) {
|
||||
globals.insert(GITHUB_API_BASE_URL.into(), Value::scalar(normalized.api_base_url));
|
||||
globals.insert(GITHUB_WEB_BASE_URL.into(), Value::scalar(normalized.web_base_url));
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn hydrate_artifactory_globals(globals: &mut Object) {
|
||||
if globals.contains_key(ARTIFACTORY_BASE_URL) {
|
||||
return;
|
||||
}
|
||||
if let Some(jfrog_url) = string_var(globals, "JFROGURL")
|
||||
&& let Ok(base_url) = normalize_artifactory_base_url(&jfrog_url)
|
||||
{
|
||||
globals.insert(ARTIFACTORY_BASE_URL.into(), Value::scalar(base_url));
|
||||
}
|
||||
}
|
||||
|
||||
fn hydrate_confluence_globals(globals: &mut Object) {
|
||||
if globals.contains_key(CONFLUENCE_BASE_URL) {
|
||||
return;
|
||||
}
|
||||
if let Some(domain) = string_var(globals, "CONFLUENCEDCDOMAIN")
|
||||
&& let Ok(base_url) = normalize_base_url(&domain)
|
||||
{
|
||||
globals.insert(CONFLUENCE_BASE_URL.into(), Value::scalar(base_url));
|
||||
}
|
||||
}
|
||||
|
||||
fn hydrate_jira_dc_globals(globals: &mut Object) {
|
||||
if globals.contains_key(JIRA_BASE_URL) {
|
||||
return;
|
||||
}
|
||||
if let Some(domain) = string_var(globals, "JIRADCDOMAIN")
|
||||
&& let Ok(base_url) = normalize_base_url(&domain)
|
||||
{
|
||||
globals.insert(JIRA_BASE_URL.into(), Value::scalar(base_url));
|
||||
}
|
||||
}
|
||||
|
||||
fn hydrate_jira_cloud_globals(globals: &mut Object) {
|
||||
if globals.contains_key(JIRA_CLOUD_BASE_URL) {
|
||||
return;
|
||||
}
|
||||
if let Some(domain) = string_var(globals, "DOMAIN")
|
||||
&& let Ok(base_url) = normalize_base_url(&domain)
|
||||
{
|
||||
globals.insert(JIRA_CLOUD_BASE_URL.into(), Value::scalar(base_url));
|
||||
}
|
||||
}
|
||||
|
||||
fn string_var(globals: &Object, name: &str) -> Option<String> {
|
||||
globals.get(name).map(|value| value.to_kstr().to_string()).filter(|s| !s.is_empty())
|
||||
}
|
||||
|
||||
fn apply_builtin_defaults(globals: &mut Object) {
|
||||
for (name, value) in [
|
||||
(GITHUB_API_BASE_URL, "https://api.github.com"),
|
||||
(GITHUB_WEB_BASE_URL, "https://github.com"),
|
||||
(GITLAB_API_BASE_URL, "https://gitlab.com/api/v4"),
|
||||
(GITEA_API_BASE_URL, "https://gitea.com/api/v1"),
|
||||
] {
|
||||
if !globals.contains_key(name) {
|
||||
globals.insert(name.into(), Value::scalar(value.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_assignment(raw: &str) -> Result<(String, String)> {
|
||||
let (provider, endpoint) = raw
|
||||
.split_once('=')
|
||||
.ok_or_else(|| anyhow!("Invalid endpoint '{}'. Expected PROVIDER=URL", raw))?;
|
||||
let provider = provider.trim();
|
||||
let endpoint = endpoint.trim();
|
||||
if provider.is_empty() {
|
||||
bail!("Invalid endpoint '{}'. Provider name cannot be empty", raw);
|
||||
}
|
||||
if endpoint.is_empty() {
|
||||
bail!("Invalid endpoint '{}'. URL cannot be empty", raw);
|
||||
}
|
||||
Ok((provider.to_string(), endpoint.to_string()))
|
||||
}
|
||||
|
||||
fn normalize_endpoint_key(key: &str) -> String {
|
||||
key.trim().to_ascii_lowercase().replace('_', "-")
|
||||
}
|
||||
|
||||
fn normalize_base_url(raw: &str) -> Result<String> {
|
||||
let url = parse_url_or_assume_https(raw)?;
|
||||
Ok(url_with_path(&url, url.path().trim_end_matches('/')))
|
||||
}
|
||||
|
||||
fn normalize_api_base_url(raw: &str, api_suffix: &str) -> Result<String> {
|
||||
let url = parse_url_or_assume_https(raw)?;
|
||||
let path = url.path().trim_end_matches('/');
|
||||
let full_path = if path.is_empty() {
|
||||
api_suffix.to_string()
|
||||
} else if path.ends_with(api_suffix) {
|
||||
path.to_string()
|
||||
} else {
|
||||
format!("{path}{api_suffix}")
|
||||
};
|
||||
Ok(url_with_path(&url, &full_path))
|
||||
}
|
||||
|
||||
fn normalize_artifactory_base_url(raw: &str) -> Result<String> {
|
||||
let url = parse_url_or_assume_https(raw)?;
|
||||
let mut path = url.path().trim_end_matches('/').to_string();
|
||||
if let Some(prefix) = path.strip_suffix("/artifactory") {
|
||||
path = prefix.to_string();
|
||||
}
|
||||
Ok(url_with_path(&url, &path))
|
||||
}
|
||||
|
||||
fn normalize_github_endpoint(raw: &str) -> Result<GitHubEndpoint> {
|
||||
let url = parse_url_or_assume_https(raw)?;
|
||||
let host = url
|
||||
.host_str()
|
||||
.ok_or_else(|| anyhow!("Endpoint '{}' is missing a host", raw))?
|
||||
.to_ascii_lowercase();
|
||||
let path = url.path().trim_end_matches('/');
|
||||
|
||||
if host == "api.github.com" {
|
||||
return Ok(GitHubEndpoint {
|
||||
api_base_url: "https://api.github.com".to_string(),
|
||||
web_base_url: "https://github.com".to_string(),
|
||||
});
|
||||
}
|
||||
if host == "github.com" && path.is_empty() {
|
||||
return Ok(GitHubEndpoint {
|
||||
api_base_url: "https://api.github.com".to_string(),
|
||||
web_base_url: "https://github.com".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let (web_path, api_path) = if path.is_empty() {
|
||||
("".to_string(), "/api/v3".to_string())
|
||||
} else if let Some(prefix) = path.strip_suffix("/api/v3") {
|
||||
(prefix.to_string(), path.to_string())
|
||||
} else {
|
||||
(path.to_string(), format!("{path}/api/v3"))
|
||||
};
|
||||
|
||||
Ok(GitHubEndpoint {
|
||||
api_base_url: url_with_path(&url, &api_path),
|
||||
web_base_url: url_with_path(&url, &web_path),
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_url_or_assume_https(raw: &str) -> Result<Url> {
|
||||
match Url::parse(raw.trim()) {
|
||||
Ok(url) => Ok(url),
|
||||
Err(url::ParseError::RelativeUrlWithoutBase) => {
|
||||
Url::parse(&format!("https://{}", raw.trim())).with_context(|| {
|
||||
format!("Invalid endpoint URL '{}'. Use a full URL or hostname", raw)
|
||||
})
|
||||
}
|
||||
Err(err) => Err(anyhow!("Invalid endpoint URL '{}': {}", raw, err)),
|
||||
}
|
||||
}
|
||||
|
||||
fn url_with_path(url: &Url, path: &str) -> String {
|
||||
let mut out = url.clone();
|
||||
out.set_query(None);
|
||||
out.set_fragment(None);
|
||||
if path.is_empty() {
|
||||
out.set_path("");
|
||||
} else {
|
||||
out.set_path(path);
|
||||
}
|
||||
out.to_string().trim_end_matches('/').to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn github_endpoint_normalizes_host_only() {
|
||||
let normalized = normalize_github_endpoint("ghe.corp.example.com").unwrap();
|
||||
assert_eq!(normalized.api_base_url, "https://ghe.corp.example.com/api/v3");
|
||||
assert_eq!(normalized.web_base_url, "https://ghe.corp.example.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn github_endpoint_normalizes_api_path() {
|
||||
let normalized = normalize_github_endpoint("https://ghe.corp.example.com/api/v3").unwrap();
|
||||
assert_eq!(normalized.api_base_url, "https://ghe.corp.example.com/api/v3");
|
||||
assert_eq!(normalized.web_base_url, "https://ghe.corp.example.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gitlab_endpoint_appends_api_path() {
|
||||
assert_eq!(
|
||||
normalize_api_base_url("gitlab.example.com/gitlab", "/api/v4").unwrap(),
|
||||
"https://gitlab.example.com/gitlab/api/v4"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifactory_endpoint_strips_artifactory_suffix() {
|
||||
assert_eq!(
|
||||
normalize_artifactory_base_url("http://localhost:8071/artifactory").unwrap(),
|
||||
"http://localhost:8071"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jira_cloud_hydrates_from_legacy_domain() {
|
||||
let mut globals = Object::new();
|
||||
globals.insert("DOMAIN".into(), Value::scalar("example.atlassian.net"));
|
||||
hydrate_endpoint_globals_for_rule("kingfisher.jira.2", &mut globals);
|
||||
assert_eq!(
|
||||
string_var(&globals, JIRA_CLOUD_BASE_URL).as_deref(),
|
||||
Some("https://example.atlassian.net")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn artifactory_hydrates_from_legacy_host() {
|
||||
let mut globals = Object::new();
|
||||
globals.insert("JFROGURL".into(), Value::scalar("repo.example.com"));
|
||||
hydrate_endpoint_globals_for_rule("kingfisher.artifactory.1", &mut globals);
|
||||
assert_eq!(
|
||||
string_var(&globals, ARTIFACTORY_BASE_URL).as_deref(),
|
||||
Some("https://repo.example.com")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -23,6 +23,7 @@ use crate::{
|
|||
gitea, github, gitlab,
|
||||
liquid_filters::register_all,
|
||||
matcher::MatcherStats,
|
||||
provider_endpoints::ProviderEndpointOverrides,
|
||||
reporter::styles::Styles,
|
||||
rule_loader::RuleLoader,
|
||||
rule_profiling::ConcurrentRuleProfiler,
|
||||
|
|
@ -46,12 +47,14 @@ use crate::{
|
|||
validation_rate_limit::ValidationRateLimiter,
|
||||
};
|
||||
|
||||
/// Shared validation dependencies: (liquid parser, HTTP clients, validation cache, rate limiter).
|
||||
/// Shared validation dependencies:
|
||||
/// (liquid parser, HTTP clients, validation cache, rate limiter, provider endpoint overrides).
|
||||
type ValidationDeps = Arc<(
|
||||
liquid::Parser,
|
||||
crate::validation::ValidationClients,
|
||||
Arc<SkipMap<String, CachedResponse>>,
|
||||
Option<Arc<ValidationRateLimiter>>,
|
||||
Arc<ProviderEndpointOverrides>,
|
||||
)>;
|
||||
|
||||
pub async fn run_scan(
|
||||
|
|
@ -159,6 +162,7 @@ pub async fn run_async_scan(
|
|||
let validation_rate_limiter =
|
||||
ValidationRateLimiter::from_cli(args.validation_rps, &args.validation_rps_rule)?
|
||||
.map(Arc::new);
|
||||
let provider_endpoints = Arc::new(ProviderEndpointOverrides::from_global_args(global_args)?);
|
||||
|
||||
let validation_deps: Option<ValidationDeps> = if !args.no_validate {
|
||||
info!("Starting secret validation phase...");
|
||||
|
|
@ -170,6 +174,7 @@ pub async fn run_async_scan(
|
|||
)?,
|
||||
Arc::new(SkipMap::new()),
|
||||
validation_rate_limiter.clone(),
|
||||
Arc::clone(&provider_endpoints),
|
||||
)))
|
||||
} else {
|
||||
None
|
||||
|
|
@ -517,8 +522,8 @@ async fn run_validation_phase(
|
|||
access_map_collector: Option<AccessMapCollector>,
|
||||
) -> Result<()> {
|
||||
if let Some(validation) = validation_deps {
|
||||
let (parser, clients, cache, rate_limiter) =
|
||||
(&validation.0, &validation.1, &validation.2, &validation.3);
|
||||
let (parser, clients, cache, rate_limiter, provider_endpoints) =
|
||||
(&validation.0, &validation.1, &validation.2, &validation.3, &validation.4);
|
||||
run_secret_validation(
|
||||
Arc::clone(datastore),
|
||||
parser,
|
||||
|
|
@ -528,6 +533,7 @@ async fn run_validation_phase(
|
|||
match_range,
|
||||
access_map_collector,
|
||||
rate_limiter.clone(),
|
||||
provider_endpoints.clone(),
|
||||
Duration::from_secs(args.validation_timeout),
|
||||
args.validation_retries,
|
||||
effective_max_validation_body_len(args),
|
||||
|
|
@ -661,8 +667,8 @@ async fn run_parallel_scan(
|
|||
|
||||
// Validate initial (non-repo) matches
|
||||
if let Some(validation) = validation_deps {
|
||||
let (parser, clients, cache, rate_limiter) =
|
||||
(&validation.0, &validation.1, &validation.2, &validation.3);
|
||||
let (parser, clients, cache, rate_limiter, provider_endpoints) =
|
||||
(&validation.0, &validation.1, &validation.2, &validation.3, &validation.4);
|
||||
let initial_match_count = { datastore.lock().unwrap().get_matches().len() };
|
||||
if initial_match_count > 0 {
|
||||
run_secret_validation(
|
||||
|
|
@ -674,6 +680,7 @@ async fn run_parallel_scan(
|
|||
Some(0..initial_match_count),
|
||||
access_map_collector.clone(),
|
||||
rate_limiter.clone(),
|
||||
provider_endpoints.clone(),
|
||||
Duration::from_secs(args.validation_timeout),
|
||||
args.validation_retries,
|
||||
effective_max_validation_body_len(args),
|
||||
|
|
@ -749,8 +756,13 @@ async fn run_parallel_scan(
|
|||
}
|
||||
|
||||
if let Some(validation) = validation_deps.clone() {
|
||||
let (parser, clients, cache, rate_limiter) =
|
||||
(&validation.0, &validation.1, &validation.2, &validation.3);
|
||||
let (parser, clients, cache, rate_limiter, provider_endpoints) = (
|
||||
&validation.0,
|
||||
&validation.1,
|
||||
&validation.2,
|
||||
&validation.3,
|
||||
&validation.4,
|
||||
);
|
||||
let match_count =
|
||||
{ repo_datastore.lock().unwrap().get_matches().len() };
|
||||
if match_count > 0 {
|
||||
|
|
@ -763,6 +775,7 @@ async fn run_parallel_scan(
|
|||
Some(0..match_count),
|
||||
access_map.clone(),
|
||||
rate_limiter.clone(),
|
||||
provider_endpoints.clone(),
|
||||
Duration::from_secs(args.validation_timeout),
|
||||
args.validation_retries,
|
||||
effective_max_validation_body_len(&args),
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ use crate::{
|
|||
findings_store::{FindingsStore, FindingsStoreMessage},
|
||||
location::OffsetSpan,
|
||||
matcher::OwnedBlobMatch,
|
||||
provider_endpoints::ProviderEndpointOverrides,
|
||||
rules::rule::Validation,
|
||||
validation::{
|
||||
CachedResponse, collect_variables_and_dependencies, utils, validate_single_match,
|
||||
|
|
@ -421,6 +422,7 @@ pub async fn run_secret_validation(
|
|||
range: Option<std::ops::Range<usize>>,
|
||||
access_map: Option<AccessMapCollector>,
|
||||
rate_limiter: Option<Arc<ValidationRateLimiter>>,
|
||||
provider_endpoints: Arc<ProviderEndpointOverrides>,
|
||||
validation_timeout: Duration,
|
||||
validation_retries: u32,
|
||||
max_body_len: usize,
|
||||
|
|
@ -536,6 +538,7 @@ pub async fn run_secret_validation(
|
|||
let pb = pb.clone();
|
||||
let access_map = access_map.clone();
|
||||
let rate_limiter = rate_limiter.clone();
|
||||
let provider_endpoints = provider_endpoints.clone();
|
||||
let empty_dep_vars = &empty_dep_vars;
|
||||
let empty_missing = &empty_missing;
|
||||
let empty_cache = empty_cache.clone();
|
||||
|
|
@ -577,6 +580,7 @@ pub async fn run_secret_validation(
|
|||
&cache_glob,
|
||||
access_map.as_ref(),
|
||||
rate_limiter.as_deref(),
|
||||
&provider_endpoints,
|
||||
validation_timeout,
|
||||
validation_retries,
|
||||
max_body_len,
|
||||
|
|
@ -690,6 +694,7 @@ pub async fn run_secret_validation(
|
|||
let cache_glob = cache.clone();
|
||||
let access_map = access_map.clone();
|
||||
let rate_limiter = rate_limiter.clone();
|
||||
let provider_endpoints = provider_endpoints.clone();
|
||||
let validation_timeout = validation_timeout;
|
||||
let validation_retries = validation_retries;
|
||||
|
||||
|
|
@ -730,6 +735,7 @@ pub async fn run_secret_validation(
|
|||
let cache_glob = cache_glob.clone();
|
||||
let access_map = access_map.clone();
|
||||
let rate_limiter = rate_limiter.clone();
|
||||
let provider_endpoints = provider_endpoints.clone();
|
||||
async move {
|
||||
validate_single(
|
||||
&mut rep,
|
||||
|
|
@ -744,6 +750,7 @@ pub async fn run_secret_validation(
|
|||
&cache_glob,
|
||||
access_map.as_ref(),
|
||||
rate_limiter.as_deref(),
|
||||
&provider_endpoints,
|
||||
validation_timeout,
|
||||
validation_retries,
|
||||
max_body_len,
|
||||
|
|
@ -839,6 +846,7 @@ async fn validate_single(
|
|||
cache2: &Arc<SkipMap<String, CachedResponse>>,
|
||||
access_map: Option<&AccessMapCollector>,
|
||||
rate_limiter: Option<&ValidationRateLimiter>,
|
||||
provider_endpoints: &Arc<ProviderEndpointOverrides>,
|
||||
validation_timeout: Duration,
|
||||
validation_retries: u32,
|
||||
max_body_len: usize,
|
||||
|
|
@ -905,6 +913,7 @@ async fn validate_single(
|
|||
validation_timeout,
|
||||
validation_retries,
|
||||
rate_limiter,
|
||||
provider_endpoints.as_ref(),
|
||||
max_body_len,
|
||||
)
|
||||
.boxed(),
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@ use crate::{
|
|||
cli::global::TlsMode,
|
||||
location::OffsetSpan,
|
||||
matcher::{OwnedBlobMatch, SerializableCaptures},
|
||||
provider_endpoints::{
|
||||
ProviderEndpointOverrides, endpoint_var_names, hydrate_endpoint_globals_for_rule,
|
||||
},
|
||||
rules::rule::Validation,
|
||||
validation_body::{self},
|
||||
};
|
||||
|
|
@ -441,6 +444,7 @@ pub async fn validate_single_match(
|
|||
validation_timeout: Duration,
|
||||
validation_retries: u32,
|
||||
rate_limiter: Option<&crate::validation_rate_limit::ValidationRateLimiter>,
|
||||
provider_endpoints: &ProviderEndpointOverrides,
|
||||
max_body_len: usize,
|
||||
) {
|
||||
let fp = validation_dedup_key(m);
|
||||
|
|
@ -456,6 +460,7 @@ pub async fn validate_single_match(
|
|||
validation_timeout,
|
||||
validation_retries,
|
||||
rate_limiter,
|
||||
provider_endpoints,
|
||||
max_body_len,
|
||||
)
|
||||
.boxed(),
|
||||
|
|
@ -499,6 +504,7 @@ async fn timed_validate_single_match<'a>(
|
|||
validation_timeout: Duration,
|
||||
validation_retries: u32,
|
||||
rate_limiter: Option<&crate::validation_rate_limit::ValidationRateLimiter>,
|
||||
provider_endpoints: &ProviderEndpointOverrides,
|
||||
max_body_len: usize,
|
||||
) {
|
||||
// Select the appropriate HTTP client based on rule's TLS mode preference
|
||||
|
|
@ -595,6 +601,8 @@ async fn timed_validate_single_match<'a>(
|
|||
|
||||
let mut globals = Object::new();
|
||||
populate_globals_from_captures(&mut globals, &captured_values);
|
||||
hydrate_endpoint_globals_for_rule(m.rule.id(), &mut globals);
|
||||
provider_endpoints.apply_scan_overrides(&mut globals);
|
||||
|
||||
// Persist named captures (non-TOKEN) for validate/revoke command generation.
|
||||
// This is especially important for gRPC validators like Modal where TOKEN_ID is required.
|
||||
|
|
@ -604,6 +612,13 @@ async fn timed_validate_single_match<'a>(
|
|||
}
|
||||
m.dependent_captures.entry(k.to_uppercase()).or_insert_with(|| v.clone());
|
||||
}
|
||||
for endpoint_var in endpoint_var_names() {
|
||||
if let Some(value) = globals.get(*endpoint_var).and_then(|v| v.as_scalar()) {
|
||||
m.dependent_captures
|
||||
.entry((*endpoint_var).to_string())
|
||||
.or_insert_with(|| value.to_kstr().to_string());
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let rule_syntax = m.rule.syntax();
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue