diff --git a/CHANGELOG.md b/CHANGELOG.md index 453c0d1..5ba1012 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,9 @@ All notable changes to this project will be documented in this file. -## [1.13.1] -- Fixed broken pagerduty rule +## [1.14.0] +- Fixed several malformed rules +- Now validating that response_matcher is present in validation section of all rules ## [1.13.0] - Added new rules for Planetscale, Postman, Openweather, opsgenie, pagerduty, pastebin, paypal, netlify, netrc, newrelic, ngrok, npm, nuget, mandrill, mapbox, microsoft teams, stripe, linkedin, mailchimp, mailgun, linear, line, huggingface, ibm cloud, intercom, ipstack, heroku, gradle, grafana diff --git a/Cargo.toml b/Cargo.toml index 13e750d..7ced54e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ publish = false [package] name = "kingfisher" -version = "1.13.1" +version = "1.14.0" edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/data/rules/digitalocean.yml b/data/rules/digitalocean.yml index 3206ab7..194d09c 100644 --- a/data/rules/digitalocean.yml +++ b/data/rules/digitalocean.yml @@ -60,9 +60,9 @@ rules: "grant_type": "refresh_token", "refresh_token": "{{ TOKEN }}" } - response_matcher: - - report_response: true - - type: StatusMatch - status: - - 200 - - type: JsonValid + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 + - type: JsonValid diff --git a/data/rules/doppler.yml b/data/rules/doppler.yml index fe21244..0afdba5 100644 --- a/data/rules/doppler.yml +++ b/data/rules/doppler.yml @@ -23,11 +23,11 @@ rules: headers: Authorization: Bearer {{ TOKEN }} Accept: application/json - response_matcher: - - report_response: true - - type: StatusMatch - status: - - 200 + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 - name: Doppler Personal Token id: kingfisher.doppler.2 pattern: | @@ -52,11 +52,11 @@ rules: headers: Authorization: Bearer {{ TOKEN }} Accept: application/json - response_matcher: - - report_response: true - - type: StatusMatch - status: - - 200 + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 - name: Doppler Service Token id: kingfisher.doppler.3 @@ -82,11 +82,11 @@ rules: headers: Authorization: Bearer {{ TOKEN }} Accept: application/json - response_matcher: - - report_response: true - - type: StatusMatch - status: - - 200 + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 - name: Doppler Service Account Token id: kingfisher.doppler.4 @@ -112,11 +112,11 @@ rules: headers: Authorization: Bearer {{ TOKEN }} Accept: application/json - response_matcher: - - report_response: true - - type: StatusMatch - status: - - 200 + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 - name: Doppler SCIM Token id: kingfisher.doppler.5 @@ -142,11 +142,11 @@ rules: headers: Authorization: Bearer {{ TOKEN }} Accept: application/json - response_matcher: - - report_response: true - - type: StatusMatch - status: - - 200 + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 - name: Doppler Audit Token id: kingfisher.doppler.6 @@ -172,8 +172,8 @@ rules: headers: Authorization: Bearer {{ TOKEN }} Accept: application/json - response_matcher: - - report_response: true - - type: StatusMatch - status: - - 200 \ No newline at end of file + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 200 \ No newline at end of file diff --git a/data/rules/figma.yml b/data/rules/figma.yml index 444a003..9863b6f 100644 --- a/data/rules/figma.yml +++ b/data/rules/figma.yml @@ -23,6 +23,12 @@ rules: X-Figma-Token: '{{ TOKEN }}' method: GET url: https://api.figma.com/v1/me + response_matcher: + - report_response: true + - type: WordMatch + words: + - "Invalid token" + negative: true - name: Figma Personal Access Header Token id: kingfisher.figma.2 diff --git a/data/rules/ibm.yml b/data/rules/ibm.yml index 0cf2e24..55a33be 100644 --- a/data/rules/ibm.yml +++ b/data/rules/ibm.yml @@ -3,8 +3,11 @@ rules: id: kingfisher.ibm.1 pattern: | (?xi) + \b (?:ibm(?:cloud)?|bx) (?:.|[\n\r]){0,32}? + (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) + (?:.|[\n\r]){0,32}? \b ( [0-9A-Z_-]{42,44} diff --git a/data/rules/linear.yml b/data/rules/linear.yml index 9adf6e8..62749ea 100644 --- a/data/rules/linear.yml +++ b/data/rules/linear.yml @@ -30,9 +30,9 @@ rules: "query": "query { issues(first: 1) { nodes { id } } }" } url: https://api.linear.app/graphql - response_matcher: - - report_response: true - - type: StatusMatch - status: [200] - - type: WordMatch - words: ['"issues":', '"nodes":'] \ No newline at end of file + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['"issues":', '"nodes":'] \ No newline at end of file diff --git a/data/rules/microsoft_teams.yml b/data/rules/microsoft_teams.yml index 13cc044..37e4030 100644 --- a/data/rules/microsoft_teams.yml +++ b/data/rules/microsoft_teams.yml @@ -42,11 +42,11 @@ rules: headers: Content-Type: application/json body: '{"text":""}' - response_matcher: - - report_response: true - - type: StatusMatch - status: - - 400 - - type: WordMatch - words: - - 'Text is required' \ No newline at end of file + response_matcher: + - report_response: true + - type: StatusMatch + status: + - 400 + - type: WordMatch + words: + - 'Text is required' \ No newline at end of file diff --git a/data/rules/pagerdutyapikey.yml b/data/rules/pagerdutyapikey.yml index f37be3f..d65bced 100644 --- a/data/rules/pagerdutyapikey.yml +++ b/data/rules/pagerdutyapikey.yml @@ -12,9 +12,10 @@ rules: pagerduty[_-]? | pagerduty ) - (?:.|[\n\r]){0,16}? + \W{0,20} (?:SECRET|PRIVATE|ACCESS|KEY|TOKEN) - (?:.|[\n\r]){0,16}? + (?:.|[\n\r]){0,16}? + \b ( u\+[A-Z0-9_+-]{18} | # personal user token (20 chars) [A-Z0-9_-]{20} | # legacy PAT (20 chars, mixed case) @@ -28,7 +29,7 @@ rules: - pd_key = u+3xVszZ-b4m+T6d23KA - Token token=ABCDEF1234567890ABCDEF1234567890 references: - - https://developer.pagerduty.com/api-reference/c96e889522dd6-list-users + - https://developer.pagerduty.com/api-reference/4555ca1c983d0-get-the-current-user validation: type: Http content: @@ -37,11 +38,10 @@ rules: url: https://api.pagerduty.com/users headers: Authorization: Token token={{ TOKEN }} - Accept: application/vnd.pagerduty+json;version=2 - Content-Type: application/json - response_matcher: - - report_response: true - - type: StatusMatch - status: [200] - - type: WordMatch - words: ['"user":'] + Accept: application/json + response_matcher: + - report_response: true + - type: JsonValid + - type: WordMatch + words: + - '"users":' diff --git a/data/rules/particle.io.yml b/data/rules/particle.io.yml index 8237ebd..76d6f27 100644 --- a/data/rules/particle.io.yml +++ b/data/rules/particle.io.yml @@ -29,13 +29,13 @@ rules: request: method: GET url: https://api.particle.io/v1/user?access_token={{ TOKEN }} - response_matcher: - - report_response: true - - type: StatusMatch - status: [200] - - type: WordMatch - match_all_words: true - words: ['"username":'] + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + match_all_words: true + words: ['"username":'] - name: particle.io Access Token id: kingfisher.particleio.2 @@ -65,10 +65,10 @@ rules: request: method: GET url: https://api.particle.io/v1/user?access_token={{ TOKEN }} - response_matcher: - - report_response: true - - type: StatusMatch - status: [200] - - type: WordMatch - match_all_words: true - words: ['"username":'] \ No newline at end of file + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + match_all_words: true + words: ['"username":'] \ No newline at end of file diff --git a/data/rules/pastebin.yml b/data/rules/pastebin.yml index 26a55dd..2173766 100644 --- a/data/rules/pastebin.yml +++ b/data/rules/pastebin.yml @@ -28,10 +28,10 @@ rules: Content-Type: application/x-www-form-urlencoded body: | api_dev_key={{ TOKEN }}&api_user_name=dummy&api_user_password=dummy - response_matcher: - - report_response: true - - type: StatusMatch - status: [200] - - type: WordMatch - words: ['invalid api_dev_key'] - negative: true \ No newline at end of file + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] + - type: WordMatch + words: ['invalid api_dev_key'] + negative: true \ No newline at end of file diff --git a/data/rules/paypal.yml b/data/rules/paypal.yml index 2441a68..4011eee 100644 --- a/data/rules/paypal.yml +++ b/data/rules/paypal.yml @@ -47,10 +47,10 @@ rules: Authorization: | Basic {{ CLIENTID | append: ':' | append: TOKEN | b64enc }} body: grant_type=client_credentials - response_matcher: - - report_response: true - - type: StatusMatch - status: [200] + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] depends_on_rule: - rule_id: kingfisher.paypal.1 variable: CLIENTID diff --git a/data/rules/tailscale.yml b/data/rules/tailscale.yml index 8ac7e50..b06f16b 100644 --- a/data/rules/tailscale.yml +++ b/data/rules/tailscale.yml @@ -25,7 +25,7 @@ rules: headers: Authorization: "Bearer {{ TOKEN }}" Accept: application/json - response_matcher: - - report_response: true - - type: StatusMatch - status: [200] + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] diff --git a/data/rules/travisci.yml b/data/rules/travisci.yml index 4299937..dd0bd05 100644 --- a/data/rules/travisci.yml +++ b/data/rules/travisci.yml @@ -28,7 +28,7 @@ rules: Authorization: token {{ TOKEN }} Accept: application/vnd.travis-ci.3+json Travis-API-Version: "3" - response_matcher: - - report_response: true - - type: StatusMatch - status: [200] + response_matcher: + - report_response: true + - type: StatusMatch + status: [200] diff --git a/src/matcher.rs b/src/matcher.rs index 1a35b5f..d7ae76d 100644 --- a/src/matcher.rs +++ b/src/matcher.rs @@ -981,7 +981,7 @@ mod test { method: "GET".to_string(), url: "https://example.com".to_string(), headers: BTreeMap::new(), - response_matcher: vec![], + response_matcher: Some(vec![]), multipart: None, response_is_html: false, }, diff --git a/src/rules.rs b/src/rules.rs index ad3c9f2..ac7e2fb 100644 --- a/src/rules.rs +++ b/src/rules.rs @@ -8,7 +8,7 @@ pub mod rule; use std::{fs::File, io::BufReader, path::Path}; use anyhow::Context; -use rule::{Confidence, RuleSyntax}; +use rule::{Confidence, RuleSyntax, Validation}; use serde::de::DeserializeOwned; /// Custom error type for more granular rules loading errors. @@ -28,6 +28,9 @@ pub enum RulesError { #[error("Invalid ResponseMatcher variant in file: {0}, at line: {1}, column: {2}")] InvalidResponseMatcherVariant(String, usize, usize), + + #[error("HTTP validation for rule `{rule_id}` in file {path} missing response_matcher")] + MissingResponseMatcher { path: String, rule_id: String }, } /// Represents a collection of rule syntaxes. @@ -58,6 +61,21 @@ impl Rules { match serde_yaml::from_reader::<_, Rules>(contents) { Ok(mut rs) => { rs.rules.retain(|rule| rule.confidence.is_at_least(&confidence)); + for rule_syntax in &rs.rules { + if let Some(Validation::Http(http_val)) = &rule_syntax.validation { + if http_val + .request + .response_matcher + .as_ref() + .map_or(true, |m| m.is_empty()) + { + bail!(RulesError::MissingResponseMatcher { + path: path.display().to_string(), + rule_id: rule_syntax.id.clone(), + }); + } + } + } rules.update(rs); } Err(e) => { diff --git a/src/rules/rule.rs b/src/rules/rule.rs index 6d3c6e9..897f400 100644 --- a/src/rules/rule.rs +++ b/src/rules/rule.rs @@ -29,6 +29,15 @@ fn default_true() -> bool { true } +fn default_status_matcher() -> Vec { + vec![ResponseMatcher::StatusMatch { + r#type: "StatusMatch".to_string(), + status: vec![200], + match_all_status: false, + negative: false, + }] +} + /// Represents various types of validation that a rule can perform. #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash, Clone)] #[serde(tag = "type", content = "content")] @@ -65,7 +74,7 @@ pub struct HttpRequest { #[serde(default)] pub headers: BTreeMap, #[serde(default)] - pub response_matcher: Vec, + pub response_matcher: Option>, #[serde(default)] pub multipart: Option, // allow HTML only when explicitly set true @@ -73,6 +82,17 @@ pub struct HttpRequest { pub response_is_html: bool, } + +impl HttpRequest { + /// Return the configured response matchers or a default StatusMatch 200. + pub fn response_matchers_or_default(&self) -> Vec { + self.response_matcher + .clone() + .unwrap_or_else(default_status_matcher) + } +} + + /// Configuration for multipart HTTP requests. #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash, Clone)] pub struct MultipartConfig { diff --git a/src/validation.rs b/src/validation.rs index 7b2066b..d76616c 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -514,8 +514,11 @@ async fn timed_validate_single_match<'a>( m.validation_response_status = status; m.validation_response_body = body.clone(); + let matchers = http_validation + .request + .response_matchers_or_default(); m.validation_success = httpvalidation::validate_response( - &http_validation.request.response_matcher, + &matchers, &body, &status, &headers,