From 16c65809036f9d9ff54985bceca263ef791a717d Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Tue, 14 Apr 2026 12:59:18 -0700 Subject: [PATCH] Automate Prowler MANUAL finding verification in review-compliance-reports Adds node-level checks (kubelet file perms/ownership, kubelet config args, etcd CA separation, RBAC cluster-admin bindings) to the weekly compliance review script, and mutes the 14 MANUAL findings in Prowler with a new node-config-automated-verification compensating control. Co-Authored-By: Claude Opus 4.6 (1M context) --- argocd/manifests/prowler/kustomization.yaml | 1 + .../prowler/mutelist/manual-node-checks.yaml | 59 +++++ compensating-controls.yaml | 16 ++ .../automate-manual-prowler-checks.infra.md | 1 + mise-tasks/review-compliance-reports | 202 +++++++++++++++++- 5 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 argocd/manifests/prowler/mutelist/manual-node-checks.yaml create mode 100644 docs/changelog.d/automate-manual-prowler-checks.infra.md diff --git a/argocd/manifests/prowler/kustomization.yaml b/argocd/manifests/prowler/kustomization.yaml index 162a2ad..b6b11fe 100644 --- a/argocd/manifests/prowler/kustomization.yaml +++ b/argocd/manifests/prowler/kustomization.yaml @@ -21,6 +21,7 @@ configMapGenerator: - mutelist/apiserver.yaml - mutelist/control-plane.yaml - mutelist/core-pod-security.yaml + - mutelist/manual-node-checks.yaml - mutelist/rbac.yaml images: diff --git a/argocd/manifests/prowler/mutelist/manual-node-checks.yaml b/argocd/manifests/prowler/mutelist/manual-node-checks.yaml new file mode 100644 index 0000000..9c8354d --- /dev/null +++ b/argocd/manifests/prowler/mutelist/manual-node-checks.yaml @@ -0,0 +1,59 @@ +# Node-level and RBAC checks that Prowler reports as MANUAL because it +# cannot evaluate them from inside a pod. Compensated by automated +# verification in `mise run review-compliance-reports`, which SSHes into +# the minikube node and checks each condition directly every week. +Mutelist: + Accounts: + "*": + Checks: + "etcd_unique_ca": + Regions: ["*"] + Resources: ["^etcd-minikube$"] + Description: "CC: node-config-automated-verification. Etcd CA fingerprint verified different from cluster CA by review-compliance-reports." + "kubelet_conf_file_ownership": + Regions: ["*"] + Resources: ["^kubelet-config$"] + Description: "CC: node-config-automated-verification. File ownership verified root:root by review-compliance-reports." + "kubelet_conf_file_permissions": + Regions: ["*"] + Resources: ["^kubelet-config$"] + Description: "CC: node-config-automated-verification. File permissions verified 600 by review-compliance-reports." + "kubelet_config_yaml_ownership": + Regions: ["*"] + Resources: ["^kubelet-config$"] + Description: "CC: node-config-automated-verification. File ownership verified root:root by review-compliance-reports." + "kubelet_config_yaml_permissions": + Regions: ["*"] + Resources: ["^kubelet-config$"] + Description: "CC: node-config-automated-verification. File permissions verified 644 by review-compliance-reports." + "kubelet_service_file_ownership_root": + Regions: ["*"] + Resources: ["^kubelet-config$"] + Description: "CC: node-config-automated-verification. File ownership verified root:root by review-compliance-reports." + "kubelet_service_file_permissions": + Regions: ["*"] + Resources: ["^kubelet-config$"] + Description: "CC: node-config-automated-verification. File permissions verified 644 by review-compliance-reports." + "kubelet_disable_read_only_port": + Regions: ["*"] + Resources: ["^kubelet-config$"] + Description: "CC: node-config-automated-verification. readOnlyPort absence (defaults to 0) verified by review-compliance-reports." + "kubelet_event_record_qps": + Regions: ["*"] + Resources: ["^kubelet-config$"] + Description: "CC: node-config-automated-verification. eventRecordQPS absence (defaults to 5) verified by review-compliance-reports." + "kubelet_manage_iptables": + Regions: ["*"] + Resources: ["^kubelet-config$"] + Description: "CC: node-config-automated-verification. makeIPTablesUtilChains absence (defaults to true) verified by review-compliance-reports." + "kubelet_strong_ciphers_only": + Regions: ["*"] + Resources: ["^kubelet-config$"] + Description: "CC: node-config-automated-verification, tailscale-network-isolation. Go default ciphers used; all traffic WireGuard-encrypted via tailnet." + "rbac_cluster_admin_usage": + Regions: ["*"] + Resources: + - "^cluster-admin$" + - "^kubeadm:cluster-admins$" + - "^minikube-rbac$" + Description: "CC: node-config-automated-verification, single-user-cluster. Only built-in/minikube cluster-admin bindings present; verified by review-compliance-reports." diff --git a/compensating-controls.yaml b/compensating-controls.yaml index ae4865b..459a991 100644 --- a/compensating-controls.yaml +++ b/compensating-controls.yaml @@ -116,6 +116,22 @@ controls: for both init and runtime containers. If fsGroup alone can handle PVC ownership, remove init-chown-data and this control. + - id: node-config-automated-verification + description: >- + Prowler reports certain node-level checks as MANUAL because it runs + inside a pod and cannot evaluate kubelet file permissions, kubelet + config arguments, etcd CA separation, or cluster-admin RBAC bindings. + The review-compliance-reports script SSHes into the minikube node + weekly and programmatically verifies each condition, failing loudly + if any check deviates from expected values. + created: 2026-04-14 + last-reviewed: 2026-04-14 + notes: >- + Verification runs as part of 'mise run review-compliance-reports'. + If minikube node is unreachable, all checks report as FAIL. If new + MANUAL findings appear in Prowler, add corresponding verification + logic to the script and update the mutelist. + - id: observability-stack-audit description: >- Alloy collects pod logs and ships them to Loki, providing an diff --git a/docs/changelog.d/automate-manual-prowler-checks.infra.md b/docs/changelog.d/automate-manual-prowler-checks.infra.md new file mode 100644 index 0000000..07f132b --- /dev/null +++ b/docs/changelog.d/automate-manual-prowler-checks.infra.md @@ -0,0 +1 @@ +Automate verification of Prowler MANUAL findings (kubelet file perms, kubelet config, etcd CA, RBAC cluster-admin) in `review-compliance-reports` and mute them with `node-config-automated-verification` compensating control. diff --git a/mise-tasks/review-compliance-reports b/mise-tasks/review-compliance-reports index 075302d..080271c 100755 --- a/mise-tasks/review-compliance-reports +++ b/mise-tasks/review-compliance-reports @@ -1,7 +1,7 @@ #!/usr/bin/env -S uv run --script # /// script # requires-python = ">=3.12" -# dependencies = ["rich>=14.0.0", "typer>=0.24.0"] +# dependencies = ["rich>=14.0.0", "typer>=0.24.0", "pyyaml>=6.0"] # /// #MISE description="Summarize the latest Prowler and Kingfisher compliance reports from sifaka" #USAGE flag "--full" help="Show all unmuted failures, not just new ones" @@ -112,6 +112,200 @@ def severity_sort(r: dict) -> int: return SEVERITY_ORDER.index(sev) if sev in SEVERITY_ORDER else 99 +def _ssh_minikube(cmd: str, timeout: int = 15) -> subprocess.CompletedProcess: + """Run a command inside the minikube node via SSH.""" + return subprocess.run( + ["ssh", "indri", f"minikube ssh -- {cmd}"], + capture_output=True, + text=True, + timeout=timeout, + ) + + +def _kubectl(args: str, timeout: int = 15) -> subprocess.CompletedProcess: + """Run a kubectl command against minikube-indri.""" + return subprocess.run( + ["kubectl", "--context=minikube-indri"] + args.split(), + capture_output=True, + text=True, + timeout=timeout, + ) + + +def run_node_verification(console: Console) -> None: + """Verify node-level conditions that Prowler reports as MANUAL. + + Compensating control: node-config-automated-verification + """ + checks: list[tuple[str, str, bool]] = [] # (name, detail, passed) + + # --- File ownership and permissions --- + file_expectations = [ + ("kubelet.conf ownership", "/etc/kubernetes/kubelet.conf", "root:root", None), + ("kubelet.conf permissions", "/etc/kubernetes/kubelet.conf", None, "600"), + ("config.yaml ownership", "/var/lib/kubelet/config.yaml", "root:root", None), + ("config.yaml permissions", "/var/lib/kubelet/config.yaml", None, "644"), + ("kubelet service ownership", "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf", "root:root", None), + ("kubelet service permissions", "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf", None, "644"), + ] + + for name, path, expected_owner, expected_perms in file_expectations: + if expected_owner: + result = _ssh_minikube(f'"sudo stat -c %U:%G {path}"') + else: + result = _ssh_minikube(f'"sudo stat -c %a {path}"') + + if result.returncode != 0: + checks.append((name, f"could not stat {path}", False)) + else: + actual = result.stdout.strip() + expected = expected_owner or expected_perms + passed = actual == expected + checks.append((name, f"{actual} (expected {expected})", passed)) + + # --- Kubelet config arguments --- + kubelet_result = _ssh_minikube('"sudo cat /var/lib/kubelet/config.yaml"') + if kubelet_result.returncode != 0: + checks.append(("kubelet config", "could not read config.yaml", False)) + else: + import yaml as _yaml + + try: + kubelet_cfg = _yaml.safe_load(kubelet_result.stdout) or {} + except Exception: + kubelet_cfg = {} + checks.append(("kubelet config parse", "failed to parse config.yaml", False)) + + # readOnlyPort: absent or 0 is safe + rop = kubelet_cfg.get("readOnlyPort") + checks.append(( + "readOnlyPort", + f"{rop!r} (absent or 0 is safe)", + rop is None or rop == 0, + )) + + # makeIPTablesUtilChains: absent (defaults true) or true + miu = kubelet_cfg.get("makeIPTablesUtilChains") + checks.append(( + "makeIPTablesUtilChains", + f"{miu!r} (absent or true is safe)", + miu is None or miu is True, + )) + + # eventRecordQPS: absent (defaults 5) or > 0 + erq = kubelet_cfg.get("eventRecordQPS") + checks.append(( + "eventRecordQPS", + f"{erq!r} (absent or > 0 is safe)", + erq is None or (isinstance(erq, (int, float)) and erq > 0), + )) + + # tlsCipherSuites: absent uses Go defaults (acceptable) + tcs = kubelet_cfg.get("tlsCipherSuites") + checks.append(( + "tlsCipherSuites", + "Go defaults" if tcs is None else f"{tcs!r}", + True, # Go defaults are acceptable; explicit suites also fine + )) + + # --- Etcd CA separation --- + etcd_fp = _ssh_minikube( + '"sudo openssl x509 -in /var/lib/minikube/certs/etcd/ca.crt -noout -fingerprint -sha256"' + ) + cluster_fp = _ssh_minikube( + '"sudo openssl x509 -in /var/lib/minikube/certs/ca.crt -noout -fingerprint -sha256"' + ) + if etcd_fp.returncode != 0 or cluster_fp.returncode != 0: + checks.append(("etcd CA separation", "could not read certificates", False)) + else: + etcd_hash = etcd_fp.stdout.strip() + cluster_hash = cluster_fp.stdout.strip() + different = etcd_hash != cluster_hash + checks.append(( + "etcd CA separation", + "different CAs" if different else "SAME CA (unexpected)", + different, + )) + + # --- RBAC cluster-admin bindings --- + expected_bindings = {"cluster-admin", "kubeadm:cluster-admins", "minikube-rbac"} + # Use a jsonpath that emits "name\troleRef" pairs to avoid N+1 queries + # Tab-separated because binding names can contain colons (e.g. kubeadm:cluster-admins) + rb_result = subprocess.run( + [ + "kubectl", "--context=minikube-indri", + "get", "clusterrolebindings", + "-o", "jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.roleRef.name}{'\\n'}{end}", + ], + capture_output=True, + text=True, + timeout=15, + ) + if rb_result.returncode != 0: + checks.append(("cluster-admin bindings", "kubectl failed", False)) + else: + admin_bindings: set[str] = set() + for line in rb_result.stdout.strip().splitlines(): + if "\t" in line: + name, role = line.split("\t", 1) + if role == "cluster-admin": + admin_bindings.add(name) + + unexpected = admin_bindings - expected_bindings + if unexpected: + checks.append(( + "cluster-admin bindings", + f"unexpected: {', '.join(sorted(unexpected))}", + False, + )) + else: + checks.append(( + "cluster-admin bindings", + f"only expected: {', '.join(sorted(admin_bindings))}", + True, + )) + + # --- Display results --- + all_passed = all(passed for _, _, passed in checks) + table = Table( + show_header=True, + header_style="bold", + title="Node Verification (CC: node-config-automated-verification)", + ) + table.add_column("Check") + table.add_column("Detail") + table.add_column("Result", justify="center") + + for name, detail, passed in checks: + status = "[green]PASS[/green]" if passed else "[bold red]FAIL[/bold red]" + table.add_row(name, detail, status) + + console.print(table) + console.print() + + if all_passed: + console.print( + Panel( + "[bold green]All node-level checks passed.[/bold green] " + "Muted MANUAL findings are verified.", + title="Node Verification Verdict", + border_style="green", + ) + ) + else: + failed = [(n, d) for n, d, p in checks if not p] + console.print( + Panel( + f"[bold red]{len(failed)} node-level check(s) FAILED.[/bold red]\n" + "Review the failures above — muted MANUAL findings may no longer " + "be valid.", + title="Node Verification Verdict", + border_style="red", + ) + ) + console.print() + + def main( full: Annotated[ bool, typer.Option(help="Show all unmuted failures, not just new ones") @@ -343,6 +537,12 @@ def main( ) ) + # --- Node-level MANUAL check verification --- + # Compensating control: node-config-automated-verification + # These checks verify conditions Prowler reports as MANUAL because it + # runs inside a pod and cannot evaluate them directly. + run_node_verification(console) + # --- Kingfisher secret scanning --- # TODO: Kingfisher currently only outputs HTML. Once JSON or CSV output # is supported upstream (contribute from our spork), add parsing here: