Erich Blume
17023085cb
Note: the name of this branch was chosen before the scope widened to encompass the entire observability stack. Summary - Fix Grafana data source URLs (docker driver uses host.minikube.internal, not host.containers.internal) - Migrate Prometheus and Loki from indri to Kubernetes with Tailscale Ingresses - Expose CNPG PostgreSQL metrics via Tailscale and update dashboard to use cnpg_* metrics - Update Alloy to push metrics/logs to k8s endpoints (prometheus.tail8d86e.ts.net, loki.tail8d86e.ts.net) - Add ACL rule for port 9187 (CNPG metrics) - Delete obsolete ansible roles for prometheus and loki Changes - argocd/manifests/prometheus/ - New Prometheus StatefulSet with 20Gi PVC and Tailscale Ingress - argocd/manifests/loki/ - New Loki StatefulSet with 20Gi PVC and Tailscale Ingress - argocd/apps/prometheus.yaml, argocd/apps/loki.yaml - ArgoCD Applications - argocd/manifests/grafana/values.yaml - Data sources now use k8s internal DNS - argocd/manifests/databases/service-metrics-tailscale.yaml - CNPG metrics endpoint - argocd/manifests/grafana-config/dashboards/configmap-postgresql.yaml - Updated to cnpg_* metrics - ansible/roles/alloy/defaults/main.yml - Push to k8s Tailscale endpoints - pulumi/policy.hujson - ACL for port 9187 - Deleted ansible/roles/prometheus/ and ansible/roles/loki/ Deployment and Testing - Stop prometheus and loki on indri - Sync ArgoCD apps (apps, prometheus, loki, grafana) - Run mise run provision-indri -- --tags alloy - Verify Grafana dashboards show data 🤖 Generated with https://claude.ai/claude-code Reviewed-on: https://forge.tail8d86e.ts.net/eblume/blumeops/pulls/42
98 lines
3.1 KiB
YAML
98 lines
3.1 KiB
YAML
---
|
|
# Grafana Alloy configuration
|
|
#
|
|
# BUILDING FROM SOURCE (required for CGO DNS resolution on macOS):
|
|
#
|
|
# Alloy must be built with CGO_ENABLED=1 to use macOS native DNS resolver,
|
|
# which is required for Tailscale MagicDNS hostname resolution.
|
|
# The Homebrew bottle is built with CGO_ENABLED=0.
|
|
#
|
|
# Build on dev machine (gilbert), then copy to indri:
|
|
#
|
|
# 1. Clone from forge mirror:
|
|
# git clone ssh://forgejo@forge.tail8d86e.ts.net/eblume/alloy.git ~/code/3rd/alloy
|
|
#
|
|
# 2. Set up build tools via mise:
|
|
# cd ~/code/3rd/alloy && mise use go@1.25 node yarn
|
|
#
|
|
# 3. Build with CGO enabled (default in Makefile):
|
|
# cd ~/code/3rd/alloy && mise x -- make alloy
|
|
#
|
|
# 4. Copy binary to indri:
|
|
# scp ~/code/3rd/alloy/build/alloy indri:~/.local/bin/alloy
|
|
#
|
|
# 5. Run ansible to deploy config and LaunchAgent
|
|
|
|
# Binary and paths
|
|
alloy_binary: /Users/erichblume/.local/bin/alloy
|
|
alloy_config_dir: /Users/erichblume/.config/grafana-alloy
|
|
alloy_data_dir: /Users/erichblume/.local/share/grafana-alloy
|
|
alloy_log_dir: /Users/erichblume/Library/Logs
|
|
|
|
# Textfile collector directory (same as node_exporter for compatibility)
|
|
alloy_textfile_dir: /opt/homebrew/var/node_exporter/textfile
|
|
|
|
# Prometheus remote write endpoint (k8s via Tailscale)
|
|
alloy_prometheus_url: "https://prometheus.tail8d86e.ts.net/api/v1/write"
|
|
|
|
# Loki endpoint (k8s via Tailscale)
|
|
alloy_loki_url: "https://loki.tail8d86e.ts.net/loki/api/v1/push"
|
|
|
|
# Instance label for metrics
|
|
alloy_instance_label: indri
|
|
|
|
# Scrape interval
|
|
alloy_scrape_interval: "15s"
|
|
|
|
# Log paths to collect
|
|
alloy_brew_logs:
|
|
- path: /opt/homebrew/var/log/forgejo.log
|
|
service: forgejo
|
|
stream: stdout
|
|
- path: /opt/homebrew/var/log/tailscaled.log
|
|
service: tailscale
|
|
stream: stdout
|
|
|
|
alloy_mcquack_logs:
|
|
- path: /Users/erichblume/Library/Logs/mcquack.alloy.out.log
|
|
service: alloy
|
|
stream: stdout
|
|
- path: /Users/erichblume/Library/Logs/mcquack.alloy.err.log
|
|
service: alloy
|
|
stream: stderr
|
|
- path: /Users/erichblume/Library/Logs/mcquack.borgmatic.out.log
|
|
service: borgmatic
|
|
stream: stdout
|
|
- path: /Users/erichblume/Library/Logs/mcquack.borgmatic.err.log
|
|
service: borgmatic
|
|
stream: stderr
|
|
- path: /Users/erichblume/Library/Logs/mcquack.zot.out.log
|
|
service: zot
|
|
stream: stdout
|
|
- path: /Users/erichblume/Library/Logs/mcquack.zot.err.log
|
|
service: zot
|
|
stream: stderr
|
|
|
|
alloy_plex_logs:
|
|
- path: /Users/erichblume/Library/Logs/Plex Media Server/Plex Media Server.log
|
|
service: plex
|
|
stream: stdout
|
|
|
|
# Enable log collection (requires Loki to be running)
|
|
alloy_collect_logs: true
|
|
|
|
# Zot registry metrics collection
|
|
alloy_collect_zot: true
|
|
alloy_zot_metrics_url: "http://localhost:5050/metrics"
|
|
|
|
# PostgreSQL metrics collection (disabled, CNPG metrics scraped directly by k8s Prometheus)
|
|
alloy_collect_postgres: false
|
|
alloy_postgres_host: localhost
|
|
alloy_postgres_port: 5432
|
|
alloy_postgres_user: alloy
|
|
alloy_postgres_database: postgres
|
|
|
|
# 1Password settings for PostgreSQL metrics (unused when alloy_collect_postgres is false)
|
|
alloy_op_vault: vg6xf6vvfmoh5hqjjhlhbeoaie
|
|
alloy_op_postgres_item: guxu3j7ajhjyey6xxl2ovsl2ui
|
|
alloy_op_postgres_field: alloy-user-pw
|