diff --git a/argocd/manifests/grafana/alerting.yaml b/argocd/manifests/grafana/alerting.yaml new file mode 100644 index 0000000..3ac33b0 --- /dev/null +++ b/argocd/manifests/grafana/alerting.yaml @@ -0,0 +1,42 @@ +apiVersion: 1 + +contactPoints: + - orgId: 1 + name: ntfy-infra + receivers: + - uid: ntfy-infra-webhook + type: webhook + settings: + url: https://ntfy.ops.eblu.me/infra-alerts + httpMethod: POST + title: >- + {{ template "ntfy-infra.title" . }} + message: >- + {{ template "ntfy-infra.message" . }} + maxAlerts: "0" + disableResolveMessage: false + +policies: + - orgId: 1 + receiver: ntfy-infra + group_by: + - alertname + - service + group_wait: 1m + group_interval: 12h + repeat_interval: 24h + +templates: + - orgId: 1 + name: ntfy-infra + template: | + {{ define "ntfy-infra.title" -}} + [{{ .Status | toUpper }}] {{ .CommonLabels.alertname }} + {{- end }} + + {{ define "ntfy-infra.message" -}} + {{ range .Alerts -}} + {{ .Annotations.summary }} + {{ if .Annotations.runbook_url }}Runbook: {{ .Annotations.runbook_url }}{{ end }} + {{ end -}} + {{- end }} diff --git a/argocd/manifests/grafana/deployment.yaml b/argocd/manifests/grafana/deployment.yaml index 61a2f88..5fbb8eb 100644 --- a/argocd/manifests/grafana/deployment.yaml +++ b/argocd/manifests/grafana/deployment.yaml @@ -277,6 +277,9 @@ spec: - name: config mountPath: /etc/grafana/provisioning/datasources/datasources.yaml subPath: datasources.yaml + - name: config + mountPath: /etc/grafana/provisioning/alerting/alerting.yaml + subPath: alerting.yaml - name: storage mountPath: /var/lib/grafana - name: sc-dashboard-volume diff --git a/argocd/manifests/grafana/grafana.ini b/argocd/manifests/grafana/grafana.ini index 61cdd7e..a0a6db8 100644 --- a/argocd/manifests/grafana/grafana.ini +++ b/argocd/manifests/grafana/grafana.ini @@ -30,3 +30,8 @@ allow_embedding = false [server] root_url = https://grafana.ops.eblu.me + +[unified_alerting] +enabled = true +evaluation_timeout = 30s +min_interval = 10s diff --git a/argocd/manifests/grafana/kustomization.yaml b/argocd/manifests/grafana/kustomization.yaml index c052bf9..3aeaa26 100644 --- a/argocd/manifests/grafana/kustomization.yaml +++ b/argocd/manifests/grafana/kustomization.yaml @@ -25,6 +25,7 @@ configMapGenerator: files: - grafana.ini - datasources.yaml + - alerting.yaml options: labels: app.kubernetes.io/name: grafana