blumeops/ansible/roles/alloy/templates/config.alloy.j2
Erich Blume 57bf8512dc Log filtering cleanup and observability improvements (#45)
## Summary
- Suppress noisy storage-provisioner Endpoints deprecation warning (upstream minikube issue)
- Disable thermal collector on indri Alloy (not supported on macOS M1)
- Add macOS power/thermal metrics collection via powermetrics LaunchDaemon
- Add Power & Thermal section to macOS Grafana dashboard
- Add logfmt parser for k8s log level extraction (Loki, Prometheus, etc.)
- Extract more fields from JSON logs (zot compatibility - uses "message" not "msg")
- Silence logfmt parse errors for non-logfmt logs
- Fix JSON escaping in devpi dashboard

## Deployment and Testing
- [x] Deployed Alloy config changes to indri via ansible
- [x] Synced alloy-k8s and grafana-config via ArgoCD
- [x] Verified power metrics appearing in Prometheus
- [x] Verified thermal collector errors stopped
- [x] Verified logfmt parse errors silenced
- [x] Verified devpi dashboard loads correctly

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Reviewed-on: https://forge.tail8d86e.ts.net/eblume/blumeops/pulls/45
2026-01-22 17:30:08 -08:00

136 lines
3.7 KiB
Django/Jinja

// {{ ansible_managed }}
// Grafana Alloy configuration for {{ alloy_instance_label }}
// Collects system metrics (replacing node_exporter) and logs
// ============== METRICS COLLECTION ==============
// System metrics exporter (replaces node_exporter)
prometheus.exporter.unix "system" {
// Disable collectors that don't work on macOS
disable_collectors = ["thermal"]
textfile {
directory = "{{ alloy_textfile_dir }}"
}
}
// Scrape system metrics
prometheus.scrape "system" {
targets = prometheus.exporter.unix.system.targets
forward_to = [prometheus.relabel.instance.receiver]
scrape_interval = "{{ alloy_scrape_interval }}"
}
// Add instance label to match existing setup
prometheus.relabel "instance" {
forward_to = [prometheus.remote_write.prometheus.receiver]
rule {
target_label = "instance"
replacement = "{{ alloy_instance_label }}"
}
}
// Push metrics to Prometheus via remote_write
prometheus.remote_write "prometheus" {
endpoint {
url = "{{ alloy_prometheus_url }}"
}
}
{% if alloy_collect_postgres | default(false) %}
// ============== POSTGRESQL METRICS ==============
// PostgreSQL exporter (read-only metrics via pg_monitor role)
prometheus.exporter.postgres "postgresql" {
data_source_names = ["postgresql://{{ alloy_postgres_user }}:{{ alloy_postgres_password | urlencode }}@{{ alloy_postgres_host }}:{{ alloy_postgres_port }}/{{ alloy_postgres_database }}?sslmode=disable"]
// Custom queries for vacuum and XID monitoring
custom_queries_config_path = "{{ alloy_config_dir }}/postgres_queries.yaml"
}
// Scrape PostgreSQL metrics
prometheus.scrape "postgresql" {
targets = prometheus.exporter.postgres.postgresql.targets
forward_to = [prometheus.relabel.instance.receiver]
scrape_interval = "{{ alloy_scrape_interval }}"
}
{% endif %}
{% if alloy_collect_zot | default(false) %}
// ============== ZOT REGISTRY METRICS ==============
// Scrape Zot's native metrics endpoint
prometheus.scrape "zot" {
targets = [{"__address__" = "localhost:5050"}]
metrics_path = "/metrics"
forward_to = [prometheus.relabel.instance.receiver]
scrape_interval = "{{ alloy_scrape_interval }}"
}
{% endif %}
{% if alloy_collect_logs %}
// ============== LOG COLLECTION ==============
// Discover log files - brew services
local.file_match "brew_logs" {
path_targets = [
{% for log in alloy_brew_logs %}
{__path__ = "{{ log.path }}", service = "{{ log.service }}", stream = "{{ log.stream }}"},
{% endfor %}
]
}
// Discover log files - mcquack LaunchAgents
local.file_match "mcquack_logs" {
path_targets = [
{% for log in alloy_mcquack_logs %}
{__path__ = "{{ log.path }}", service = "{{ log.service }}", stream = "{{ log.stream }}"},
{% endfor %}
]
}
// Discover log files - Plex Media Server
local.file_match "plex_logs" {
path_targets = [
{% for log in alloy_plex_logs %}
{__path__ = "{{ log.path }}", service = "{{ log.service }}", stream = "{{ log.stream }}"},
{% endfor %}
]
}
// Read and forward brew service logs
loki.source.file "brew_logs" {
targets = local.file_match.brew_logs.targets
forward_to = [loki.relabel.add_host.receiver]
}
// Read and forward mcquack service logs
loki.source.file "mcquack_logs" {
targets = local.file_match.mcquack_logs.targets
forward_to = [loki.relabel.add_host.receiver]
}
// Read and forward Plex logs
loki.source.file "plex_logs" {
targets = local.file_match.plex_logs.targets
forward_to = [loki.relabel.add_host.receiver]
}
// Add host label to all logs
loki.relabel "add_host" {
forward_to = [loki.write.loki.receiver]
rule {
target_label = "host"
replacement = "{{ alloy_instance_label }}"
}
}
// Write logs to Loki
loki.write "loki" {
endpoint {
url = "{{ alloy_loki_url }}"
}
}
{% endif %}