Add Grafana Alloy and Loki for unified observability (#11)

## Summary
- Add Grafana Alloy to replace node_exporter for metrics collection
- Add Loki for log aggregation and storage
- Configure Alloy to collect logs from all services (grafana, forgejo, prometheus, tailscale, transmission, devpi, kiwix, borgmatic)
- Update Prometheus to accept metrics via remote_write
- Add Loki datasource to Grafana

## Test plan
- [ ] Run \`mise run provision-indri -- --check --diff\` to verify changes
- [ ] Apply with \`mise run provision-indri\`
- [ ] Verify services: \`mise run indri-services-check\`
- [ ] Check Grafana Explore with Loki datasource
- [ ] Query logs: \`{service="grafana"}\`
- [ ] Verify metrics still flowing to Prometheus dashboards

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Reviewed-on: https://forge.tail8d86e.ts.net/eblume/blumeops/pulls/11
This commit is contained in:
Erich Blume 2026-01-15 12:24:13 -08:00
commit 242c1880de
17 changed files with 799 additions and 10 deletions

View file

@ -0,0 +1,12 @@
---
# Loki configuration
# Server settings
loki_http_port: 3100
# Storage paths
loki_data_dir: /opt/homebrew/var/loki
loki_config_file: /opt/homebrew/etc/loki-local-config.yaml
# Retention settings
loki_retention_period: 744h # 31 days

View file

@ -0,0 +1,3 @@
---
- name: restart loki
ansible.builtin.command: brew services restart loki

View file

@ -0,0 +1,2 @@
---
dependencies: []

View file

@ -0,0 +1,38 @@
---
# Loki installation and configuration
- name: Install loki via homebrew
community.general.homebrew:
name: loki
state: present
- name: Ensure loki data directory exists
ansible.builtin.file:
path: "{{ loki_data_dir }}"
state: directory
mode: '0755'
- name: Ensure loki chunks directory exists
ansible.builtin.file:
path: "{{ loki_data_dir }}/chunks"
state: directory
mode: '0755'
- name: Ensure loki rules directory exists
ansible.builtin.file:
path: "{{ loki_data_dir }}/rules"
state: directory
mode: '0755'
- name: Deploy loki configuration
ansible.builtin.template:
src: loki-config.yaml.j2
dest: "{{ loki_config_file }}"
mode: '0644'
notify: restart loki
- name: Ensure loki service is started
ansible.builtin.command: brew services start loki
register: brew_start
changed_when: "'Successfully started' in brew_start.stdout"
failed_when: false

View file

@ -0,0 +1,53 @@
# {{ ansible_managed }}
# Loki configuration for single-node deployment
auth_enabled: false
server:
http_listen_port: {{ loki_http_port }}
grpc_listen_port: 9096
common:
instance_addr: 127.0.0.1
path_prefix: {{ loki_data_dir }}
storage:
filesystem:
chunks_directory: {{ loki_data_dir }}/chunks
rules_directory: {{ loki_data_dir }}/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
query_range:
results_cache:
cache:
embedded_cache:
enabled: true
max_size_mb: 100
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
storage_config:
tsdb_shipper:
active_index_directory: {{ loki_data_dir }}/tsdb-index
cache_location: {{ loki_data_dir }}/tsdb-cache
limits_config:
retention_period: {{ loki_retention_period }}
compactor:
working_directory: {{ loki_data_dir }}/compactor
compaction_interval: 10m
retention_enabled: true
retention_delete_delay: 2h
retention_delete_worker_count: 150
delete_request_store: filesystem