Automate Prowler MANUAL finding verification in review-compliance-reports

Adds node-level checks (kubelet file perms/ownership, kubelet config
args, etcd CA separation, RBAC cluster-admin bindings) to the weekly
compliance review script, and mutes the 14 MANUAL findings in Prowler
with a new node-config-automated-verification compensating control.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

argocd/manifests/prowler/kustomization.yaml

@@ -21,6 +21,7 @@ configMapGenerator:
       - mutelist/apiserver.yaml
       - mutelist/control-plane.yaml
       - mutelist/core-pod-security.yaml
+      - mutelist/manual-node-checks.yaml
       - mutelist/rbac.yaml
 
 images:

argocd/manifests/prowler/mutelist/manual-node-checks.yaml

@@ -0,0 +1,59 @@
+# Node-level and RBAC checks that Prowler reports as MANUAL because it
+# cannot evaluate them from inside a pod. Compensated by automated
+# verification in `mise run review-compliance-reports`, which SSHes into
+# the minikube node and checks each condition directly every week.
+Mutelist:
+  Accounts:
+    "*":
+      Checks:
+        "etcd_unique_ca":
+          Regions: ["*"]
+          Resources: ["^etcd-minikube$"]
+          Description: "CC: node-config-automated-verification. Etcd CA fingerprint verified different from cluster CA by review-compliance-reports."
+        "kubelet_conf_file_ownership":
+          Regions: ["*"]
+          Resources: ["^kubelet-config$"]
+          Description: "CC: node-config-automated-verification. File ownership verified root:root by review-compliance-reports."
+        "kubelet_conf_file_permissions":
+          Regions: ["*"]
+          Resources: ["^kubelet-config$"]
+          Description: "CC: node-config-automated-verification. File permissions verified 600 by review-compliance-reports."
+        "kubelet_config_yaml_ownership":
+          Regions: ["*"]
+          Resources: ["^kubelet-config$"]
+          Description: "CC: node-config-automated-verification. File ownership verified root:root by review-compliance-reports."
+        "kubelet_config_yaml_permissions":
+          Regions: ["*"]
+          Resources: ["^kubelet-config$"]
+          Description: "CC: node-config-automated-verification. File permissions verified 644 by review-compliance-reports."
+        "kubelet_service_file_ownership_root":
+          Regions: ["*"]
+          Resources: ["^kubelet-config$"]
+          Description: "CC: node-config-automated-verification. File ownership verified root:root by review-compliance-reports."
+        "kubelet_service_file_permissions":
+          Regions: ["*"]
+          Resources: ["^kubelet-config$"]
+          Description: "CC: node-config-automated-verification. File permissions verified 644 by review-compliance-reports."
+        "kubelet_disable_read_only_port":
+          Regions: ["*"]
+          Resources: ["^kubelet-config$"]
+          Description: "CC: node-config-automated-verification. readOnlyPort absence (defaults to 0) verified by review-compliance-reports."
+        "kubelet_event_record_qps":
+          Regions: ["*"]
+          Resources: ["^kubelet-config$"]
+          Description: "CC: node-config-automated-verification. eventRecordQPS absence (defaults to 5) verified by review-compliance-reports."
+        "kubelet_manage_iptables":
+          Regions: ["*"]
+          Resources: ["^kubelet-config$"]
+          Description: "CC: node-config-automated-verification. makeIPTablesUtilChains absence (defaults to true) verified by review-compliance-reports."
+        "kubelet_strong_ciphers_only":
+          Regions: ["*"]
+          Resources: ["^kubelet-config$"]
+          Description: "CC: node-config-automated-verification, tailscale-network-isolation. Go default ciphers used; all traffic WireGuard-encrypted via tailnet."
+        "rbac_cluster_admin_usage":
+          Regions: ["*"]
+          Resources:
+            - "^cluster-admin$"
+            - "^kubeadm:cluster-admins$"
+            - "^minikube-rbac$"
+          Description: "CC: node-config-automated-verification, single-user-cluster. Only built-in/minikube cluster-admin bindings present; verified by review-compliance-reports."

compensating-controls.yaml

@@ -116,6 +116,22 @@ controls:
       for both init and runtime containers. If fsGroup alone can
       handle PVC ownership, remove init-chown-data and this control.
 
+  - id: node-config-automated-verification
+    description: >-
+      Prowler reports certain node-level checks as MANUAL because it runs
+      inside a pod and cannot evaluate kubelet file permissions, kubelet
+      config arguments, etcd CA separation, or cluster-admin RBAC bindings.
+      The review-compliance-reports script SSHes into the minikube node
+      weekly and programmatically verifies each condition, failing loudly
+      if any check deviates from expected values.
+    created: 2026-04-14
+    last-reviewed: 2026-04-14
+    notes: >-
+      Verification runs as part of 'mise run review-compliance-reports'.
+      If minikube node is unreachable, all checks report as FAIL. If new
+      MANUAL findings appear in Prowler, add corresponding verification
+      logic to the script and update the mutelist.
+
   - id: observability-stack-audit
     description: >-
       Alloy collects pod logs and ships them to Loki, providing an

docs/changelog.d/automate-manual-prowler-checks.infra.md

@@ -0,0 +1 @@
+Automate verification of Prowler MANUAL findings (kubelet file perms, kubelet config, etcd CA, RBAC cluster-admin) in `review-compliance-reports` and mute them with `node-config-automated-verification` compensating control.

mise-tasks/review-compliance-reports

@@ -1,7 +1,7 @@
 #!/usr/bin/env -S uv run --script
 # /// script
 # requires-python = ">=3.12"
-# dependencies = ["rich>=14.0.0", "typer>=0.24.0"]
+# dependencies = ["rich>=14.0.0", "typer>=0.24.0", "pyyaml>=6.0"]
 # ///
 #MISE description="Summarize the latest Prowler and Kingfisher compliance reports from sifaka"
 #USAGE flag "--full" help="Show all unmuted failures, not just new ones"
@@ -112,6 +112,200 @@ def severity_sort(r: dict) -> int:
     return SEVERITY_ORDER.index(sev) if sev in SEVERITY_ORDER else 99
 
 
+def _ssh_minikube(cmd: str, timeout: int = 15) -> subprocess.CompletedProcess:
+    """Run a command inside the minikube node via SSH."""
+    return subprocess.run(
+        ["ssh", "indri", f"minikube ssh -- {cmd}"],
+        capture_output=True,
+        text=True,
+        timeout=timeout,
+    )
+
+
+def _kubectl(args: str, timeout: int = 15) -> subprocess.CompletedProcess:
+    """Run a kubectl command against minikube-indri."""
+    return subprocess.run(
+        ["kubectl", "--context=minikube-indri"] + args.split(),
+        capture_output=True,
+        text=True,
+        timeout=timeout,
+    )
+
+
+def run_node_verification(console: Console) -> None:
+    """Verify node-level conditions that Prowler reports as MANUAL.
+
+    Compensating control: node-config-automated-verification
+    """
+    checks: list[tuple[str, str, bool]] = []  # (name, detail, passed)
+
+    # --- File ownership and permissions ---
+    file_expectations = [
+        ("kubelet.conf ownership", "/etc/kubernetes/kubelet.conf", "root:root", None),
+        ("kubelet.conf permissions", "/etc/kubernetes/kubelet.conf", None, "600"),
+        ("config.yaml ownership", "/var/lib/kubelet/config.yaml", "root:root", None),
+        ("config.yaml permissions", "/var/lib/kubelet/config.yaml", None, "644"),
+        ("kubelet service ownership", "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf", "root:root", None),
+        ("kubelet service permissions", "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf", None, "644"),
+    ]
+
+    for name, path, expected_owner, expected_perms in file_expectations:
+        if expected_owner:
+            result = _ssh_minikube(f'"sudo stat -c %U:%G {path}"')
+        else:
+            result = _ssh_minikube(f'"sudo stat -c %a {path}"')
+
+        if result.returncode != 0:
+            checks.append((name, f"could not stat {path}", False))
+        else:
+            actual = result.stdout.strip()
+            expected = expected_owner or expected_perms
+            passed = actual == expected
+            checks.append((name, f"{actual} (expected {expected})", passed))
+
+    # --- Kubelet config arguments ---
+    kubelet_result = _ssh_minikube('"sudo cat /var/lib/kubelet/config.yaml"')
+    if kubelet_result.returncode != 0:
+        checks.append(("kubelet config", "could not read config.yaml", False))
+    else:
+        import yaml as _yaml
+
+        try:
+            kubelet_cfg = _yaml.safe_load(kubelet_result.stdout) or {}
+        except Exception:
+            kubelet_cfg = {}
+            checks.append(("kubelet config parse", "failed to parse config.yaml", False))
+
+        # readOnlyPort: absent or 0 is safe
+        rop = kubelet_cfg.get("readOnlyPort")
+        checks.append((
+            "readOnlyPort",
+            f"{rop!r} (absent or 0 is safe)",
+            rop is None or rop == 0,
+        ))
+
+        # makeIPTablesUtilChains: absent (defaults true) or true
+        miu = kubelet_cfg.get("makeIPTablesUtilChains")
+        checks.append((
+            "makeIPTablesUtilChains",
+            f"{miu!r} (absent or true is safe)",
+            miu is None or miu is True,
+        ))
+
+        # eventRecordQPS: absent (defaults 5) or > 0
+        erq = kubelet_cfg.get("eventRecordQPS")
+        checks.append((
+            "eventRecordQPS",
+            f"{erq!r} (absent or > 0 is safe)",
+            erq is None or (isinstance(erq, (int, float)) and erq > 0),
+        ))
+
+        # tlsCipherSuites: absent uses Go defaults (acceptable)
+        tcs = kubelet_cfg.get("tlsCipherSuites")
+        checks.append((
+            "tlsCipherSuites",
+            "Go defaults" if tcs is None else f"{tcs!r}",
+            True,  # Go defaults are acceptable; explicit suites also fine
+        ))
+
+    # --- Etcd CA separation ---
+    etcd_fp = _ssh_minikube(
+        '"sudo openssl x509 -in /var/lib/minikube/certs/etcd/ca.crt -noout -fingerprint -sha256"'
+    )
+    cluster_fp = _ssh_minikube(
+        '"sudo openssl x509 -in /var/lib/minikube/certs/ca.crt -noout -fingerprint -sha256"'
+    )
+    if etcd_fp.returncode != 0 or cluster_fp.returncode != 0:
+        checks.append(("etcd CA separation", "could not read certificates", False))
+    else:
+        etcd_hash = etcd_fp.stdout.strip()
+        cluster_hash = cluster_fp.stdout.strip()
+        different = etcd_hash != cluster_hash
+        checks.append((
+            "etcd CA separation",
+            "different CAs" if different else "SAME CA (unexpected)",
+            different,
+        ))
+
+    # --- RBAC cluster-admin bindings ---
+    expected_bindings = {"cluster-admin", "kubeadm:cluster-admins", "minikube-rbac"}
+    # Use a jsonpath that emits "name\troleRef" pairs to avoid N+1 queries
+    # Tab-separated because binding names can contain colons (e.g. kubeadm:cluster-admins)
+    rb_result = subprocess.run(
+        [
+            "kubectl", "--context=minikube-indri",
+            "get", "clusterrolebindings",
+            "-o", "jsonpath={range .items[*]}{.metadata.name}{'\\t'}{.roleRef.name}{'\\n'}{end}",
+        ],
+        capture_output=True,
+        text=True,
+        timeout=15,
+    )
+    if rb_result.returncode != 0:
+        checks.append(("cluster-admin bindings", "kubectl failed", False))
+    else:
+        admin_bindings: set[str] = set()
+        for line in rb_result.stdout.strip().splitlines():
+            if "\t" in line:
+                name, role = line.split("\t", 1)
+                if role == "cluster-admin":
+                    admin_bindings.add(name)
+
+        unexpected = admin_bindings - expected_bindings
+        if unexpected:
+            checks.append((
+                "cluster-admin bindings",
+                f"unexpected: {', '.join(sorted(unexpected))}",
+                False,
+            ))
+        else:
+            checks.append((
+                "cluster-admin bindings",
+                f"only expected: {', '.join(sorted(admin_bindings))}",
+                True,
+            ))
+
+    # --- Display results ---
+    all_passed = all(passed for _, _, passed in checks)
+    table = Table(
+        show_header=True,
+        header_style="bold",
+        title="Node Verification (CC: node-config-automated-verification)",
+    )
+    table.add_column("Check")
+    table.add_column("Detail")
+    table.add_column("Result", justify="center")
+
+    for name, detail, passed in checks:
+        status = "[green]PASS[/green]" if passed else "[bold red]FAIL[/bold red]"
+        table.add_row(name, detail, status)
+
+    console.print(table)
+    console.print()
+
+    if all_passed:
+        console.print(
+            Panel(
+                "[bold green]All node-level checks passed.[/bold green] "
+                "Muted MANUAL findings are verified.",
+                title="Node Verification Verdict",
+                border_style="green",
+            )
+        )
+    else:
+        failed = [(n, d) for n, d, p in checks if not p]
+        console.print(
+            Panel(
+                f"[bold red]{len(failed)} node-level check(s) FAILED.[/bold red]\n"
+                "Review the failures above — muted MANUAL findings may no longer "
+                "be valid.",
+                title="Node Verification Verdict",
+                border_style="red",
+            )
+        )
+    console.print()
+
+
 def main(
     full: Annotated[
         bool, typer.Option(help="Show all unmuted failures, not just new ones")
@@ -343,6 +537,12 @@ def main(
                 )
             )
 
+    # --- Node-level MANUAL check verification ---
+    # Compensating control: node-config-automated-verification
+    # These checks verify conditions Prowler reports as MANUAL because it
+    # runs inside a pod and cannot evaluate them directly.
+    run_node_verification(console)
+
     # --- Kingfisher secret scanning ---
     # TODO: Kingfisher currently only outputs HTML. Once JSON or CSV output
     # is supported upstream (contribute from our spork), add parsing here: