From d1007d29f17a6448951ea5b053530418456d8f51 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Thu, 22 Jan 2026 12:51:54 -0800 Subject: [PATCH] Add k8s observability: Alloy for pod logs, service health probes - Remove stale /opt/homebrew/var/loki from borgmatic backup (Loki migrated to k8s) - Add Alloy k8s DaemonSet for automatic pod log collection to Loki - Add blackbox probes for miniflux, kiwix, transmission, devpi, argocd - Replace stale devpi/transmission dashboards with unified services health dashboard - The new Alloy k8s deployment auto-discovers all pods including new ones Co-Authored-By: Claude Opus 4.5 --- ansible/roles/borgmatic/defaults/main.yml | 2 - argocd/apps/alloy-k8s.yaml | 17 + argocd/manifests/alloy-k8s/configmap.yaml | 158 ++++ argocd/manifests/alloy-k8s/daemonset.yaml | 65 ++ argocd/manifests/alloy-k8s/kustomization.yaml | 7 + argocd/manifests/alloy-k8s/namespace.yaml | 4 + argocd/manifests/alloy-k8s/rbac.yaml | 35 + .../dashboards/configmap-devpi.yaml | 452 ---------- .../dashboards/configmap-services.yaml | 145 ++++ .../dashboards/configmap-transmission.yaml | 814 ------------------ .../grafana-config/kustomization.yaml | 3 +- 11 files changed, 432 insertions(+), 1270 deletions(-) create mode 100644 argocd/apps/alloy-k8s.yaml create mode 100644 argocd/manifests/alloy-k8s/configmap.yaml create mode 100644 argocd/manifests/alloy-k8s/daemonset.yaml create mode 100644 argocd/manifests/alloy-k8s/kustomization.yaml create mode 100644 argocd/manifests/alloy-k8s/namespace.yaml create mode 100644 argocd/manifests/alloy-k8s/rbac.yaml delete mode 100644 argocd/manifests/grafana-config/dashboards/configmap-devpi.yaml create mode 100644 argocd/manifests/grafana-config/dashboards/configmap-services.yaml delete mode 100644 argocd/manifests/grafana-config/dashboards/configmap-transmission.yaml diff --git a/ansible/roles/borgmatic/defaults/main.yml b/ansible/roles/borgmatic/defaults/main.yml index 49816d9..14aa046 100644 --- a/ansible/roles/borgmatic/defaults/main.yml +++ b/ansible/roles/borgmatic/defaults/main.yml @@ -11,14 +11,12 @@ borgmatic_schedule_hour: 2 borgmatic_schedule_minute: 0 # Source directories to back up -# NOTE: devpi removed - now hosted in k8s (PVC handles persistence) borgmatic_source_directories: - /Users/erichblume/code/personal/zk - /opt/homebrew/var/forgejo - /Users/erichblume/.config/borgmatic - /Users/erichblume/Documents - /Users/erichblume/Pictures - - /opt/homebrew/var/loki # Backup repository borgmatic_repositories: diff --git a/argocd/apps/alloy-k8s.yaml b/argocd/apps/alloy-k8s.yaml new file mode 100644 index 0000000..29d996c --- /dev/null +++ b/argocd/apps/alloy-k8s.yaml @@ -0,0 +1,17 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: alloy-k8s + namespace: argocd +spec: + project: default + source: + repoURL: ssh://forgejo@indri.tail8d86e.ts.net:2200/eblume/blumeops.git + targetRevision: main + path: argocd/manifests/alloy-k8s + destination: + server: https://kubernetes.default.svc + namespace: alloy + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/manifests/alloy-k8s/configmap.yaml b/argocd/manifests/alloy-k8s/configmap.yaml new file mode 100644 index 0000000..2a12cd9 --- /dev/null +++ b/argocd/manifests/alloy-k8s/configmap.yaml @@ -0,0 +1,158 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: alloy-config + namespace: alloy +data: + config.alloy: | + // Alloy k8s configuration - collects pod logs from all namespaces + + // ============== K8S POD LOG DISCOVERY ============== + + // Discover all pods in the cluster + discovery.kubernetes "pods" { + role = "pod" + } + + // Relabel to extract useful metadata + discovery.relabel "pods" { + targets = discovery.kubernetes.pods.targets + + // Keep only running pods + rule { + source_labels = ["__meta_kubernetes_pod_phase"] + regex = "Pending|Succeeded|Failed|Unknown" + action = "drop" + } + + // Set namespace label + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + // Set pod name label + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + + // Set container name label + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + + // Set app label from pod labels + rule { + source_labels = ["__meta_kubernetes_pod_label_app"] + target_label = "app" + } + + // Fallback: use app.kubernetes.io/name if no app label + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] + target_label = "app" + regex = "(.+)" + action = "replace" + } + + // Set node name + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "node" + } + + // Build the log path for the pod container + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + target_label = "__path__" + separator = "/" + replacement = "/var/log/pods/*$1/$2/*.log" + } + } + + // Tail pod logs + loki.source.kubernetes "pods" { + targets = discovery.relabel.pods.output + forward_to = [loki.process.pods.receiver] + } + + // Process logs - parse JSON if present, add labels + loki.process "pods" { + forward_to = [loki.write.loki.receiver] + + // Try to parse JSON logs + stage.json { + expressions = { + level = "level", + message = "msg", + time = "time", + } + } + + // Extract level from parsed JSON if available + stage.labels { + values = { + level = "", + } + } + } + + // Write logs to Loki + loki.write "loki" { + endpoint { + url = "http://loki.monitoring.svc.cluster.local:3100/loki/api/v1/push" + } + } + + // ============== SERVICE HEALTH PROBES ============== + + // Blackbox-style HTTP probes for k8s services + prometheus.exporter.blackbox "services" { + config = "{ modules: { http_2xx: { prober: http, timeout: 5s } } }" + + target { + name = "miniflux" + address = "http://miniflux.miniflux.svc.cluster.local:8080/healthcheck" + module = "http_2xx" + } + + target { + name = "kiwix" + address = "http://kiwix.kiwix.svc.cluster.local:80/" + module = "http_2xx" + } + + target { + name = "transmission" + address = "http://transmission.torrent.svc.cluster.local:9091/transmission/web/" + module = "http_2xx" + } + + target { + name = "devpi" + address = "http://devpi.devpi.svc.cluster.local:3141/+api" + module = "http_2xx" + } + + target { + name = "argocd" + address = "http://argocd-server.argocd.svc.cluster.local:80/healthz" + module = "http_2xx" + } + } + + // Scrape blackbox probe results + prometheus.scrape "blackbox" { + targets = prometheus.exporter.blackbox.services.targets + scrape_interval = "30s" + forward_to = [prometheus.remote_write.prometheus.receiver] + } + + // Push metrics to Prometheus + prometheus.remote_write "prometheus" { + endpoint { + url = "http://prometheus.monitoring.svc.cluster.local:9090/api/v1/write" + } + } diff --git a/argocd/manifests/alloy-k8s/daemonset.yaml b/argocd/manifests/alloy-k8s/daemonset.yaml new file mode 100644 index 0000000..81a5cf8 --- /dev/null +++ b/argocd/manifests/alloy-k8s/daemonset.yaml @@ -0,0 +1,65 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: alloy + namespace: alloy + labels: + app: alloy +spec: + selector: + matchLabels: + app: alloy + template: + metadata: + labels: + app: alloy + spec: + serviceAccountName: alloy + containers: + - name: alloy + image: grafana/alloy:v1.5.1 + args: + - run + - --server.http.listen-addr=0.0.0.0:12345 + - --storage.path=/var/lib/alloy/data + - /etc/alloy/config.alloy + ports: + - containerPort: 12345 + name: http + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + volumeMounts: + - name: config + mountPath: /etc/alloy + - name: varlog + mountPath: /var/log + readOnly: true + - name: data + mountPath: /var/lib/alloy/data + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + tolerations: + - operator: Exists + volumes: + - name: config + configMap: + name: alloy-config + - name: varlog + hostPath: + path: /var/log + - name: data + emptyDir: {} diff --git a/argocd/manifests/alloy-k8s/kustomization.yaml b/argocd/manifests/alloy-k8s/kustomization.yaml new file mode 100644 index 0000000..17cd3c4 --- /dev/null +++ b/argocd/manifests/alloy-k8s/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - rbac.yaml + - configmap.yaml + - daemonset.yaml diff --git a/argocd/manifests/alloy-k8s/namespace.yaml b/argocd/manifests/alloy-k8s/namespace.yaml new file mode 100644 index 0000000..94f62be --- /dev/null +++ b/argocd/manifests/alloy-k8s/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: alloy diff --git a/argocd/manifests/alloy-k8s/rbac.yaml b/argocd/manifests/alloy-k8s/rbac.yaml new file mode 100644 index 0000000..f205d24 --- /dev/null +++ b/argocd/manifests/alloy-k8s/rbac.yaml @@ -0,0 +1,35 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: alloy + namespace: alloy +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: alloy +rules: + - apiGroups: [""] + resources: ["nodes", "nodes/proxy", "nodes/metrics", "services", "endpoints", "pods", "namespaces"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] + - nonResourceURLs: ["/metrics", "/metrics/cadvisor"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: alloy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: alloy +subjects: + - kind: ServiceAccount + name: alloy + namespace: alloy diff --git a/argocd/manifests/grafana-config/dashboards/configmap-devpi.yaml b/argocd/manifests/grafana-config/dashboards/configmap-devpi.yaml deleted file mode 100644 index 68dd32d..0000000 --- a/argocd/manifests/grafana-config/dashboards/configmap-devpi.yaml +++ /dev/null @@ -1,452 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: grafana-dashboard-devpi - namespace: monitoring - labels: - grafana_dashboard: "1" -data: - devpi.json: | - { - "annotations": { - "list": [] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "green", "value": 1 } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 }, - "id": 1, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "devpi_up", - "refId": "A" - } - ], - "title": "Devpi Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 100 }, - { "color": "red", "value": 1000 } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "devpi_web_whoosh_index_queue_size", - "refId": "A" - } - ], - "title": "Search Index Queue", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ "color": "green", "value": null }] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "devpi_serial", - "refId": "A" - } - ], - "title": "Changelog Serial", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { "type": "linear" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ "color": "green", "value": null }] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { "mode": "single", "sort": "none" } - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(devpi_server_storage_cache_hits[5m])", - "legendFormat": "Storage Cache Hits", - "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(devpi_server_storage_cache_misses[5m])", - "legendFormat": "Storage Cache Misses", - "refId": "B" - } - ], - "title": "Storage Cache Hit/Miss Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { "type": "linear" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ "color": "green", "value": null }] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { "mode": "single", "sort": "none" } - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(devpi_server_changelog_cache_hits[5m])", - "legendFormat": "Changelog Cache Hits", - "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(devpi_server_changelog_cache_misses[5m])", - "legendFormat": "Changelog Cache Misses", - "refId": "B" - } - ], - "title": "Changelog Cache Hit/Miss Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { "type": "linear" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ "color": "green", "value": null }] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { "mode": "single", "sort": "none" } - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "devpi_web_whoosh_index_queue_size", - "legendFormat": "Index Queue Size", - "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "devpi_web_whoosh_index_error_queue_size", - "legendFormat": "Index Error Queue Size", - "refId": "B" - } - ], - "title": "Search Index Queue Over Time", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ "color": "green", "value": null }] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "devpi_server_storage_cache_size", - "legendFormat": "Storage Cache Size", - "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "devpi_server_changelog_cache_size", - "legendFormat": "Changelog Cache Size", - "refId": "B" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "devpi_server_changelog_cache_items", - "legendFormat": "Changelog Cache Items", - "refId": "C" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "devpi_server_relpath_cache_size", - "legendFormat": "Relpath Cache Size", - "refId": "D" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "devpi_server_relpath_cache_items", - "legendFormat": "Relpath Cache Items", - "refId": "E" - } - ], - "title": "Cache Sizes", - "type": "stat" - } - ], - "refresh": "30s", - "schemaVersion": 38, - "tags": ["devpi", "pypi"], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Devpi PyPI Proxy", - "uid": "devpi", - "version": 1, - "weekStart": "" - } diff --git a/argocd/manifests/grafana-config/dashboards/configmap-services.yaml b/argocd/manifests/grafana-config/dashboards/configmap-services.yaml new file mode 100644 index 0000000..0b07133 --- /dev/null +++ b/argocd/manifests/grafana-config/dashboards/configmap-services.yaml @@ -0,0 +1,145 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-services + namespace: monitoring + labels: + grafana_dashboard: "1" +data: + services.json: | + { + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [ + { "options": { "0": { "color": "red", "text": "DOWN" }, "1": { "color": "green", "text": "UP" } }, "type": "value" } + ], + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 24, "x": 0, "y": 0 }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "value_and_name" + }, + "pluginVersion": "11.0.0", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_success{target=\"miniflux\"}", "legendFormat": "Miniflux", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_success{target=\"kiwix\"}", "legendFormat": "Kiwix", "refId": "B" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_success{target=\"transmission\"}", "legendFormat": "Transmission", "refId": "C" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_success{target=\"devpi\"}", "legendFormat": "Devpi", "refId": "D" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_success{target=\"argocd\"}", "legendFormat": "ArgoCD", "refId": "E" } + ], + "title": "Service Status", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 }, + "id": 2, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "pluginVersion": "11.0.0", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_duration_seconds{target=\"miniflux\"}", "legendFormat": "Miniflux", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_duration_seconds{target=\"kiwix\"}", "legendFormat": "Kiwix", "refId": "B" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_duration_seconds{target=\"transmission\"}", "legendFormat": "Transmission", "refId": "C" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_duration_seconds{target=\"devpi\"}", "legendFormat": "Devpi", "refId": "D" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "probe_duration_seconds{target=\"argocd\"}", "legendFormat": "ArgoCD", "refId": "E" } + ], + "title": "Response Time", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 0.95 }, { "color": "green", "value": 0.99 }] }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 24, "x": 0, "y": 14 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { "calcs": ["mean"], "fields": "", "values": false }, + "textMode": "value_and_name" + }, + "pluginVersion": "11.0.0", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "avg_over_time(probe_success{target=\"miniflux\"}[$__range])", "legendFormat": "Miniflux", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "avg_over_time(probe_success{target=\"kiwix\"}[$__range])", "legendFormat": "Kiwix", "refId": "B" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "avg_over_time(probe_success{target=\"transmission\"}[$__range])", "legendFormat": "Transmission", "refId": "C" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "avg_over_time(probe_success{target=\"devpi\"}[$__range])", "legendFormat": "Devpi", "refId": "D" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "avg_over_time(probe_success{target=\"argocd\"}[$__range])", "legendFormat": "ArgoCD", "refId": "E" } + ], + "title": "Uptime (selected period)", + "type": "stat" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "tags": ["services", "health"], + "templating": { "list": [] }, + "time": { "from": "now-24h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "K8s Services Health", + "uid": "k8s-services", + "version": 1, + "weekStart": "" + } diff --git a/argocd/manifests/grafana-config/dashboards/configmap-transmission.yaml b/argocd/manifests/grafana-config/dashboards/configmap-transmission.yaml deleted file mode 100644 index 3563ff5..0000000 --- a/argocd/manifests/grafana-config/dashboards/configmap-transmission.yaml +++ /dev/null @@ -1,814 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: grafana-dashboard-transmission - namespace: monitoring - labels: - grafana_dashboard: "1" -data: - transmission.json: | - { - "annotations": { - "list": [] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "green", "value": 1 } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 }, - "id": 1, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transmission_up", - "refId": "A" - } - ], - "title": "Transmission Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ "color": "green", "value": null }] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transmission_torrents_total", - "refId": "A" - } - ], - "title": "Total Torrents", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ "color": "blue", "value": null }] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 0 }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transmission_torrents_active", - "refId": "A" - } - ], - "title": "Active Torrents", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ "color": "orange", "value": null }] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { "h": 4, "w": 4, "x": 12, "y": 0 }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transmission_torrents_paused", - "refId": "A" - } - ], - "title": "Paused Torrents", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ "color": "green", "value": null }] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { "h": 4, "w": 4, "x": 16, "y": 0 }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transmission_downloaded_bytes_total", - "refId": "A" - } - ], - "title": "Total Downloaded", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ "color": "purple", "value": null }] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { "h": 4, "w": 4, "x": 20, "y": 0 }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transmission_uploaded_bytes_total", - "refId": "A" - } - ], - "title": "Total Uploaded", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 0.5 }, - { "color": "green", "value": 1 } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { "h": 4, "w": 8, "x": 0, "y": 4 }, - "id": 11, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "increase(transmission_uploaded_bytes_total[$__range]) / increase(transmission_downloaded_bytes_total[$__range])", - "refId": "A" - } - ], - "title": "Upload/Download Ratio (Period)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ "color": "green", "value": null }] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { "h": 4, "w": 8, "x": 8, "y": 4 }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "increase(transmission_downloaded_bytes_total[$__range])", - "refId": "A" - } - ], - "title": "Downloaded (Period)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "blue", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ "color": "blue", "value": null }] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { "h": 4, "w": 8, "x": 16, "y": 4 }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "increase(transmission_uploaded_bytes_total[$__range])", - "refId": "A" - } - ], - "title": "Uploaded (Period)", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ "color": "green", "value": null }] - }, - "unit": "Bps" - }, - "overrides": [ - { - "matcher": { "id": "byName", "options": "Download" }, - "properties": [ - { - "id": "color", - "value": { "fixedColor": "green", "mode": "fixed" } - } - ] - }, - { - "matcher": { "id": "byName", "options": "Upload" }, - "properties": [ - { - "id": "color", - "value": { "fixedColor": "blue", "mode": "fixed" } - } - ] - } - ] - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, - "id": 7, - "options": { - "legend": { - "calcs": ["mean", "max"], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transmission_download_speed_bytes", - "legendFormat": "Download", - "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transmission_upload_speed_bytes", - "legendFormat": "Upload", - "refId": "B" - } - ], - "title": "Transfer Speed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ "color": "green", "value": null }] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, - "id": 8, - "options": { - "legend": { - "calcs": ["mean", "max"], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transmission_torrents_total", - "legendFormat": "Total", - "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transmission_torrents_active", - "legendFormat": "Active", - "refId": "B" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transmission_torrents_paused", - "legendFormat": "Paused", - "refId": "C" - } - ], - "title": "Torrent Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ "color": "green", "value": null }] - }, - "unit": "Bps" - }, - "overrides": [ - { - "matcher": { "id": "byName", "options": "Download Rate" }, - "properties": [ - { - "id": "color", - "value": { "fixedColor": "green", "mode": "fixed" } - } - ] - }, - { - "matcher": { "id": "byName", "options": "Upload Rate" }, - "properties": [ - { - "id": "color", - "value": { "fixedColor": "blue", "mode": "fixed" } - } - ] - } - ] - }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 }, - "id": 9, - "options": { - "legend": { - "calcs": ["mean", "max", "lastNotNull"], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(transmission_downloaded_bytes_total[5m])", - "legendFormat": "Download Rate", - "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "rate(transmission_uploaded_bytes_total[5m])", - "legendFormat": "Upload Rate", - "refId": "B" - } - ], - "title": "Cumulative Transfer Rate (5m avg)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "semi-dark-purple" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [{ "color": "green", "value": null }] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 24 }, - "id": 10, - "options": { - "legend": { - "calcs": ["lastNotNull", "min", "max"], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.0.0", - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "transmission_torrents_size_bytes", - "legendFormat": "Total Torrent Size", - "refId": "A" - } - ], - "title": "Total Torrent Size", - "type": "timeseries" - } - ], - "refresh": "30s", - "schemaVersion": 38, - "tags": ["transmission", "bittorrent"], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Transmission", - "uid": "transmission", - "version": 1, - "weekStart": "" - } diff --git a/argocd/manifests/grafana-config/kustomization.yaml b/argocd/manifests/grafana-config/kustomization.yaml index d897b1d..781f139 100644 --- a/argocd/manifests/grafana-config/kustomization.yaml +++ b/argocd/manifests/grafana-config/kustomization.yaml @@ -7,11 +7,10 @@ resources: - ingress-tailscale.yaml # Dashboard ConfigMaps - discovered by Grafana sidecar via label grafana_dashboard=1 - dashboards/configmap-borgmatic.yaml - - dashboards/configmap-devpi.yaml - dashboards/configmap-loki.yaml - dashboards/configmap-macos.yaml - dashboards/configmap-minikube.yaml - dashboards/configmap-plex.yaml - dashboards/configmap-postgresql.yaml - - dashboards/configmap-transmission.yaml + - dashboards/configmap-services.yaml - dashboards/configmap-zot.yaml