Switch Alloy from Homebrew to source-built binary with LaunchAgent

CGO-enabled build required for macOS native DNS resolver (Tailscale MagicDNS).
Homebrew bottle is built with CGO_ENABLED=0 which uses Go's pure DNS resolver
that doesn't respect /etc/resolver/* on macOS.

- Remove Homebrew installation, use ~/.local/bin/alloy
- Add LaunchAgent plist (mcquack.eblume.alloy)
- Update config paths to ~/.config/grafana-alloy
- Add build instructions in defaults/main.yml
- Add alloy's own logs to mcquack_logs collection

ansible/roles/alloy/defaults/main.yml

@@ -1,5 +1,33 @@
 ---
 # Grafana Alloy configuration
+#
+# BUILDING FROM SOURCE (required for CGO DNS resolution on macOS):
+#
+# Alloy must be built with CGO_ENABLED=1 to use macOS native DNS resolver,
+# which is required for Tailscale MagicDNS hostname resolution.
+# The Homebrew bottle is built with CGO_ENABLED=0.
+#
+# Build on dev machine (gilbert), then copy to indri:
+#
+# 1. Clone from forge mirror:
+#    git clone ssh://forgejo@forge.tail8d86e.ts.net/eblume/alloy.git ~/code/3rd/alloy
+#
+# 2. Set up build tools via mise:
+#    cd ~/code/3rd/alloy && mise use go@1.25 node yarn
+#
+# 3. Build with CGO enabled (default in Makefile):
+#    cd ~/code/3rd/alloy && mise x -- make alloy
+#
+# 4. Copy binary to indri:
+#    scp ~/code/3rd/alloy/build/alloy indri:~/.local/bin/alloy
+#
+# 5. Run ansible to deploy config and LaunchAgent
+
+# Binary and paths
+alloy_binary: /Users/erichblume/.local/bin/alloy
+alloy_config_dir: /Users/erichblume/.config/grafana-alloy
+alloy_data_dir: /Users/erichblume/.local/share/grafana-alloy
+alloy_log_dir: /Users/erichblume/Library/Logs
 
 # Textfile collector directory (same as node_exporter for compatibility)
 alloy_textfile_dir: /opt/homebrew/var/node_exporter/textfile
@@ -16,10 +44,6 @@ alloy_instance_label: indri
 # Scrape interval
 alloy_scrape_interval: "15s"
 
-# Config paths
-alloy_config_dir: /opt/homebrew/etc/grafana-alloy
-alloy_data_dir: /opt/homebrew/var/lib/grafana-alloy/data
-
 # Log paths to collect
 alloy_brew_logs:
   - path: /opt/homebrew/var/log/forgejo.log
@@ -30,6 +54,12 @@ alloy_brew_logs:
     stream: stdout
 
 alloy_mcquack_logs:
+  - path: /Users/erichblume/Library/Logs/mcquack.alloy.out.log
+    service: alloy
+    stream: stdout
+  - path: /Users/erichblume/Library/Logs/mcquack.alloy.err.log
+    service: alloy
+    stream: stderr
   - path: /Users/erichblume/Library/Logs/mcquack.borgmatic.out.log
     service: borgmatic
     stream: stdout

ansible/roles/alloy/handlers/main.yml

@@ -1,6 +1,6 @@
 ---
 - name: Restart alloy
-  ansible.builtin.command: brew services restart grafana-alloy
-  async: 120
-  poll: 0
+  ansible.builtin.shell: |
+    launchctl unload ~/Library/LaunchAgents/mcquack.eblume.alloy.plist 2>/dev/null || true
+    launchctl load ~/Library/LaunchAgents/mcquack.eblume.alloy.plist
   changed_when: true

ansible/roles/alloy/tasks/main.yml

@@ -1,11 +1,18 @@
 ---
 # Grafana Alloy installation and configuration
-# Replaces node_exporter for metrics, adds log collection
+# See defaults/main.yml for build instructions
 
-- name: Install grafana-alloy via homebrew
-  community.general.homebrew:
-    name: grafana-alloy
-    state: present
+- name: Verify alloy binary exists
+  ansible.builtin.stat:
+    path: "{{ alloy_binary }}"
+  register: alloy_binary_stat
+
+- name: Fail if alloy binary not found
+  ansible.builtin.fail:
+    msg: |
+      Alloy binary not found at {{ alloy_binary }}.
+      Please build from source first (see ansible/roles/alloy/defaults/main.yml)
+  when: not alloy_binary_stat.stat.exists
 
 - name: Ensure alloy config directory exists
   ansible.builtin.file:
@@ -68,8 +75,21 @@
   notify: Restart alloy
   no_log: true
 
-- name: Ensure alloy service is started
-  ansible.builtin.command: brew services start grafana-alloy
-  register: alloy_brew_start
-  changed_when: "'Successfully started' in alloy_brew_start.stdout"
+- name: Deploy alloy LaunchAgent plist
+  ansible.builtin.template:
+    src: alloy.plist.j2
+    dest: ~/Library/LaunchAgents/mcquack.eblume.alloy.plist
+    mode: '0644'
+  notify: Restart alloy
+
+- name: Check if alloy LaunchAgent is loaded
+  ansible.builtin.command: launchctl list mcquack.eblume.alloy
+  register: alloy_launchctl_check
+  changed_when: false
+  failed_when: false
+
+- name: Load alloy LaunchAgent if not loaded
+  ansible.builtin.command: launchctl load ~/Library/LaunchAgents/mcquack.eblume.alloy.plist
+  when: alloy_launchctl_check.rc != 0
+  changed_when: true
   failed_when: false

ansible/roles/alloy/templates/alloy.plist.j2

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- {{ ansible_managed }} -->
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>Label</key>
+	<string>mcquack.eblume.alloy</string>
+	<key>ProgramArguments</key>
+	<array>
+		<string>{{ alloy_binary }}</string>
+		<string>run</string>
+		<string>{{ alloy_config_dir }}/config.alloy</string>
+		<string>--storage.path={{ alloy_data_dir }}</string>
+	</array>
+	<key>RunAtLoad</key>
+	<true/>
+	<key>KeepAlive</key>
+	<true/>
+	<key>StandardOutPath</key>
+	<string>{{ alloy_log_dir }}/mcquack.alloy.out.log</string>
+	<key>StandardErrorPath</key>
+	<string>{{ alloy_log_dir }}/mcquack.alloy.err.log</string>
+</dict>
+</plist>

ansible/roles/alloy/templates/config.alloy.j2

@@ -43,7 +43,7 @@ prometheus.exporter.postgres "postgresql" {
   data_source_names = ["postgresql://{{ alloy_postgres_user }}:{{ alloy_postgres_password | urlencode }}@{{ alloy_postgres_host }}:{{ alloy_postgres_port }}/{{ alloy_postgres_database }}?sslmode=disable"]
 
   // Custom queries for vacuum and XID monitoring
-  custom_queries_config_path = "/opt/homebrew/etc/grafana-alloy/postgres_queries.yaml"
+  custom_queries_config_path = "{{ alloy_config_dir }}/postgres_queries.yaml"
 }
 
 // Scrape PostgreSQL metrics