Erich Blume
e4a8405de7
## Summary - Remove stale `/opt/homebrew/var/loki` from borgmatic backup (Loki migrated to k8s) - Add Alloy k8s DaemonSet for automatic pod log collection with auto-discovery - Add blackbox probes for miniflux, kiwix, transmission, devpi, argocd - Add transmission-exporter sidecar for full metrics (speed, torrent counts, ratios) - Replace stale devpi dashboard with probe-based metrics (status, response time, uptime) - Add unified "K8s Services Health" dashboard for service uptime/response monitoring ## Manual cleanup already performed - Deleted stale textfile metrics on indri: `devpi.prom`, `transmission.prom` - Deleted stale data directories on indri: `/opt/homebrew/var/loki/`, `/opt/homebrew/var/prometheus/` ## Deployment and Testing - [x] Sync `apps` application to pick up new alloy-k8s app - [x] Deploy alloy-k8s on feature branch: `argocd app set alloy-k8s --revision feature/observability-cleanup && argocd app sync alloy-k8s` - [x] Deploy torrent on feature branch (for transmission exporter): `argocd app set torrent --revision feature/observability-cleanup && argocd app sync torrent` - [x] Deploy prometheus on feature branch (for new scrape config): `argocd app set prometheus --revision feature/observability-cleanup && argocd app sync prometheus` - [x] Deploy grafana-config on feature branch (for dashboards): `argocd app set grafana-config --revision feature/observability-cleanup && argocd app sync grafana-config` - [x] Verify pod logs appear in Loki/Grafana - [x] Verify transmission metrics appear in Prometheus - [x] Verify service probe metrics appear in Prometheus - [x] Run `mise run provision-indri -- --tags borgmatic` to update borgmatic config - [ ] After merge, reset apps to main and resync 🤖 Generated with [Claude Code](https://claude.com/claude-code) Reviewed-on: https://forge.tail8d86e.ts.net/eblume/blumeops/pulls/43
145 lines
7.3 KiB
YAML
145 lines
7.3 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: grafana-dashboard-services
|
|
namespace: monitoring
|
|
labels:
|
|
grafana_dashboard: "1"
|
|
data:
|
|
services.json: |
|
|
{
|
|
"annotations": { "list": [] },
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 0,
|
|
"id": null,
|
|
"links": [],
|
|
"panels": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "thresholds" },
|
|
"mappings": [
|
|
{ "options": { "0": { "color": "red", "text": "DOWN" }, "1": { "color": "green", "text": "UP" } }, "type": "value" }
|
|
],
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] },
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 },
|
|
"id": 1,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "none",
|
|
"justifyMode": "center",
|
|
"orientation": "horizontal",
|
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
|
"textMode": "value_and_name"
|
|
},
|
|
"pluginVersion": "11.0.0",
|
|
"targets": [
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_success{job=\"integrations/blackbox/miniflux\"}", "legendFormat": "Miniflux", "refId": "A" },
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_success{job=\"integrations/blackbox/kiwix\"}", "legendFormat": "Kiwix", "refId": "B" },
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_success{job=\"integrations/blackbox/transmission\"}", "legendFormat": "Transmission", "refId": "C" },
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_success{job=\"integrations/blackbox/devpi\"}", "legendFormat": "Devpi", "refId": "D" },
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_success{job=\"integrations/blackbox/argocd\"}", "legendFormat": "ArgoCD", "refId": "E" }
|
|
],
|
|
"title": "Service Status",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": { "type": "linear" },
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": { "group": "A", "mode": "none" },
|
|
"thresholdsStyle": { "mode": "off" }
|
|
},
|
|
"mappings": [],
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 },
|
|
"id": 2,
|
|
"options": {
|
|
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true },
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
},
|
|
"pluginVersion": "11.0.0",
|
|
"targets": [
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_duration_seconds{job=\"integrations/blackbox/miniflux\"}", "legendFormat": "Miniflux", "refId": "A" },
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_duration_seconds{job=\"integrations/blackbox/kiwix\"}", "legendFormat": "Kiwix", "refId": "B" },
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_duration_seconds{job=\"integrations/blackbox/transmission\"}", "legendFormat": "Transmission", "refId": "C" },
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_duration_seconds{job=\"integrations/blackbox/devpi\"}", "legendFormat": "Devpi", "refId": "D" },
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_duration_seconds{job=\"integrations/blackbox/argocd\"}", "legendFormat": "ArgoCD", "refId": "E" }
|
|
],
|
|
"title": "Response Time",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "thresholds" },
|
|
"mappings": [],
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 0.95 }, { "color": "green", "value": 0.99 }] },
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 6, "w": 24, "x": 0, "y": 14 },
|
|
"id": 3,
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "horizontal",
|
|
"reduceOptions": { "calcs": ["mean"], "fields": "", "values": false },
|
|
"textMode": "value_and_name"
|
|
},
|
|
"pluginVersion": "11.0.0",
|
|
"targets": [
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "avg_over_time(probe_success{job=\"integrations/blackbox/miniflux\"}[$__range])", "legendFormat": "Miniflux", "refId": "A" },
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "avg_over_time(probe_success{job=\"integrations/blackbox/kiwix\"}[$__range])", "legendFormat": "Kiwix", "refId": "B" },
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "avg_over_time(probe_success{job=\"integrations/blackbox/transmission\"}[$__range])", "legendFormat": "Transmission", "refId": "C" },
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "avg_over_time(probe_success{job=\"integrations/blackbox/devpi\"}[$__range])", "legendFormat": "Devpi", "refId": "D" },
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "avg_over_time(probe_success{job=\"integrations/blackbox/argocd\"}[$__range])", "legendFormat": "ArgoCD", "refId": "E" }
|
|
],
|
|
"title": "Uptime (selected period)",
|
|
"type": "stat"
|
|
}
|
|
],
|
|
"refresh": "30s",
|
|
"schemaVersion": 38,
|
|
"tags": ["services", "health"],
|
|
"templating": { "list": [] },
|
|
"time": { "from": "now-24h", "to": "now" },
|
|
"timepicker": {},
|
|
"timezone": "browser",
|
|
"title": "K8s Services Health",
|
|
"uid": "k8s-services",
|
|
"version": 1,
|
|
"weekStart": ""
|
|
}
|