From 261f20601a289a12e8df680dc75324fbe4ed9d4a Mon Sep 17 00:00:00 2001
From: Erich Blume <blume.erich@gmail.com>
Date: Sun, 22 Mar 2026 10:35:36 -0700
Subject: [PATCH] C2(deploy-infra-alerting): impl configure grafana alerting
 pipeline

- Enable unified alerting in grafana.ini
- Create alerting.yaml provisioning file with:
  - ntfy-infra webhook contact point (POST to ntfy.ops.eblu.me/infra-alerts)
  - Notification policy: group_wait 1m, group_interval 12h, repeat_interval 24h
  - Message templates for title and runbook links
- Mount alerting provisioning into Grafana deployment
- Add alerting.yaml to kustomization configMapGenerator

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 argocd/manifests/grafana/alerting.yaml      | 42 +++++++++++++++++++++
 argocd/manifests/grafana/deployment.yaml    |  3 ++
 argocd/manifests/grafana/grafana.ini        |  5 +++
 argocd/manifests/grafana/kustomization.yaml |  1 +
 4 files changed, 51 insertions(+)
 create mode 100644 argocd/manifests/grafana/alerting.yaml

diff --git a/argocd/manifests/grafana/alerting.yaml b/argocd/manifests/grafana/alerting.yaml
new file mode 100644
index 0000000..3ac33b0
--- /dev/null
+++ b/argocd/manifests/grafana/alerting.yaml
@@ -0,0 +1,42 @@
+apiVersion: 1
+
+contactPoints:
+  - orgId: 1
+    name: ntfy-infra
+    receivers:
+      - uid: ntfy-infra-webhook
+        type: webhook
+        settings:
+          url: https://ntfy.ops.eblu.me/infra-alerts
+          httpMethod: POST
+          title: >-
+            {{ template "ntfy-infra.title" . }}
+          message: >-
+            {{ template "ntfy-infra.message" . }}
+          maxAlerts: "0"
+        disableResolveMessage: false
+
+policies:
+  - orgId: 1
+    receiver: ntfy-infra
+    group_by:
+      - alertname
+      - service
+    group_wait: 1m
+    group_interval: 12h
+    repeat_interval: 24h
+
+templates:
+  - orgId: 1
+    name: ntfy-infra
+    template: |
+      {{ define "ntfy-infra.title" -}}
+      [{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}
+      {{- end }}
+
+      {{ define "ntfy-infra.message" -}}
+      {{ range .Alerts -}}
+      {{ .Annotations.summary }}
+      {{ if .Annotations.runbook_url }}Runbook: {{ .Annotations.runbook_url }}{{ end }}
+      {{ end -}}
+      {{- end }}
diff --git a/argocd/manifests/grafana/deployment.yaml b/argocd/manifests/grafana/deployment.yaml
index 61a2f88..5fbb8eb 100644
--- a/argocd/manifests/grafana/deployment.yaml
+++ b/argocd/manifests/grafana/deployment.yaml
@@ -277,6 +277,9 @@ spec:
             - name: config
               mountPath: /etc/grafana/provisioning/datasources/datasources.yaml
               subPath: datasources.yaml
+            - name: config
+              mountPath: /etc/grafana/provisioning/alerting/alerting.yaml
+              subPath: alerting.yaml
             - name: storage
               mountPath: /var/lib/grafana
             - name: sc-dashboard-volume
diff --git a/argocd/manifests/grafana/grafana.ini b/argocd/manifests/grafana/grafana.ini
index 61cdd7e..a0a6db8 100644
--- a/argocd/manifests/grafana/grafana.ini
+++ b/argocd/manifests/grafana/grafana.ini
@@ -30,3 +30,8 @@ allow_embedding = false
 
 [server]
 root_url = https://grafana.ops.eblu.me
+
+[unified_alerting]
+enabled = true
+evaluation_timeout = 30s
+min_interval = 10s
diff --git a/argocd/manifests/grafana/kustomization.yaml b/argocd/manifests/grafana/kustomization.yaml
index c052bf9..3aeaa26 100644
--- a/argocd/manifests/grafana/kustomization.yaml
+++ b/argocd/manifests/grafana/kustomization.yaml
@@ -25,6 +25,7 @@ configMapGenerator:
     files:
       - grafana.ini
       - datasources.yaml
+      - alerting.yaml
     options:
       labels:
         app.kubernetes.io/name: grafana