Add Fly.io proxy observability via embedded Alloy (#123)

## Summary

- Embed Grafana Alloy in the Fly.io proxy container to collect nginx JSON access logs (→ Loki) and derive request rate, latency histogram, cache status, and bandwidth metrics (→ Prometheus)
- Add nginx `stub_status` endpoint for connection-level metrics (active/reading/writing/waiting)
- Create two Grafana dashboards: **Docs APM** (per-service view filtered by `host="docs.eblu.me"`) and **Fly.io Proxy Health** (aggregate proxy health across all upstream services)

## Changed Files

| File | Change |
|------|--------|
| `fly/nginx.conf` | Add JSON `log_format` + `access_log`, add `stub_status` endpoint |
| `fly/Dockerfile` | COPY Alloy binary from `grafana/alloy:v1.5.1`, COPY `alloy.river` config |
| `fly/alloy.river` | **New** — Alloy config: log tailing, metric extraction, remote_write |
| `fly/start.sh` | Start Alloy after Tailscale, before nginx |
| `argocd/manifests/grafana-config/dashboards/configmap-docs-apm.yaml` | **New** — Docs APM dashboard |
| `argocd/manifests/grafana-config/dashboards/configmap-flyio.yaml` | **New** — Fly.io Proxy Health dashboard |
| `argocd/manifests/grafana-config/kustomization.yaml` | Register new dashboard configmaps |
| `docs/reference/services/flyio-proxy.md` | Document observability setup |

## Deployment and Testing

- [ ] `mise run fly-deploy` — rebuild container with Alloy
- [ ] `curl https://docs.eblu.me/` — generate traffic
- [ ] `fly logs -a blumeops-proxy` — verify Alloy startup
- [ ] Query Prometheus: `flyio_nginx_http_requests_total{instance="flyio-proxy"}`
- [ ] Query Loki: `{instance="flyio-proxy", job="flyio-nginx"}`
- [ ] `argocd app sync grafana-config` — deploy dashboards
- [ ] Verify dashboards show data in Grafana
- [ ] `mise run services-check` — no regressions

Reviewed-on: https://forge.ops.eblu.me/eblume/blumeops/pulls/123

fly/Dockerfile

@@ -7,9 +7,17 @@ COPY --from=docker.io/tailscale/tailscale:stable \
     /usr/local/bin/tailscale /usr/local/bin/tailscale
 
 RUN mkdir -p /var/run/tailscale /var/lib/tailscale \
-    && apk add --no-cache iptables ip6tables
+    && apk add --no-cache iptables ip6tables \
+    && apk add --no-cache libc6-compat
+
+# Copy Alloy binary from official image (Ubuntu-based, needs libc6-compat)
+COPY --from=docker.io/grafana/alloy:v1.5.1 \
+    /bin/alloy /usr/local/bin/alloy
+
+RUN mkdir -p /var/log/nginx /etc/alloy /tmp/alloy-data
 
 COPY nginx.conf /etc/nginx/nginx.conf
+COPY alloy.river /etc/alloy/config.alloy
 COPY start.sh /start.sh
 RUN chmod +x /start.sh
 

fly/alloy.river

@@ -0,0 +1,142 @@
+// Grafana Alloy configuration for flyio-proxy
+// Collects nginx access logs → Loki, extracts metrics → Prometheus.
+// Note: stub_status connection metrics are not collected — Alloy has no
+// built-in nginx exporter. The log-derived metrics cover the key signals.
+
+// ============== LOG COLLECTION ==============
+
+// Tail the JSON access log written by nginx
+local.file_match "nginx_access" {
+  path_targets = [
+    {__path__ = "/var/log/nginx/access.json.log", job = "flyio-nginx"},
+  ]
+}
+
+loki.source.file "nginx_access" {
+  targets    = local.file_match.nginx_access.targets
+  forward_to = [loki.process.nginx.receiver]
+}
+
+// Parse JSON fields, extract labels, derive metrics
+loki.process "nginx" {
+  forward_to = [loki.relabel.instance.receiver]
+
+  // Parse the JSON log line
+  stage.json {
+    expressions = {
+      status               = "status",
+      method               = "request_method",
+      host                 = "http_host",
+      cache_status         = "upstream_cache_status",
+      request_time         = "request_time",
+      body_bytes_sent      = "body_bytes_sent",
+      upstream_response_time = "upstream_response_time",
+    }
+  }
+
+  // Promote to labels for filtering in Loki
+  stage.labels {
+    values = {
+      status       = "",
+      method       = "",
+      host         = "",
+      cache_status = "",
+    }
+  }
+
+  // --- Derived metrics (exposed on Alloy's /metrics endpoint) ---
+
+  stage.metrics {
+    metric.counter {
+      name        = "flyio_nginx_http_requests_total"
+      description = "Total HTTP requests by status, method, and host."
+      match_all   = true
+      action      = "inc"
+    }
+  }
+
+  stage.metrics {
+    metric.histogram {
+      name        = "flyio_nginx_http_request_duration_seconds"
+      description = "HTTP request latency in seconds."
+      source      = "request_time"
+      buckets     = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]
+    }
+  }
+
+  stage.metrics {
+    metric.counter {
+      name        = "flyio_nginx_http_response_bytes_total"
+      description = "Total bytes sent in HTTP responses."
+      source      = "body_bytes_sent"
+      action      = "add"
+    }
+  }
+
+  stage.metrics {
+    metric.counter {
+      name        = "flyio_nginx_cache_requests_total"
+      description = "Total cache lookups by cache status."
+      source      = "cache_status"
+      match_all   = true
+      action      = "inc"
+    }
+  }
+}
+
+// Add instance label to logs
+loki.relabel "instance" {
+  forward_to = [loki.write.loki.receiver]
+
+  rule {
+    target_label = "instance"
+    replacement  = "flyio-proxy"
+  }
+}
+
+// Write logs to Loki via Caddy (valid TLS, no skip_verify needed)
+loki.write "loki" {
+  endpoint {
+    url = "https://loki.ops.eblu.me/loki/api/v1/push"
+  }
+}
+
+// ============== METRICS PIPELINE ==============
+
+// Self-scrape to collect the log-derived metrics from /metrics
+prometheus.scrape "self" {
+  targets         = [{"__address__" = "127.0.0.1:12345"}]
+  forward_to      = [prometheus.relabel.instance.receiver]
+  scrape_interval = "15s"
+}
+
+// Strip the "loki_process_custom_" prefix that Alloy adds to stage.metrics,
+// then add instance label. This keeps dashboard queries clean.
+prometheus.relabel "instance" {
+  forward_to = [prometheus.remote_write.prometheus.receiver]
+
+  rule {
+    source_labels = ["__name__"]
+    regex         = "loki_process_custom_(.*)"
+    target_label  = "__name__"
+    replacement   = "$1"
+  }
+
+  // Drop internal labels added by the loki pipeline
+  rule {
+    regex  = "component_id|component_path|filename"
+    action = "labeldrop"
+  }
+
+  rule {
+    target_label = "instance"
+    replacement  = "flyio-proxy"
+  }
+}
+
+// Push metrics to Prometheus via Caddy (valid TLS, no skip_verify needed)
+prometheus.remote_write "prometheus" {
+  endpoint {
+    url = "https://prometheus.ops.eblu.me/api/v1/write"
+  }
+}

fly/nginx.conf

@@ -8,6 +8,23 @@ http {
     include /etc/nginx/mime.types;
     default_type application/octet-stream;
 
+    # JSON access log for Alloy to tail → Loki + metric extraction
+    log_format json_log escape=json
+        '{'
+            '"time":"$time_iso8601",'
+            '"remote_addr":"$remote_addr",'
+            '"request_method":"$request_method",'
+            '"request_uri":"$request_uri",'
+            '"status":$status,'
+            '"body_bytes_sent":$body_bytes_sent,'
+            '"request_time":$request_time,'
+            '"upstream_response_time":"$upstream_response_time",'
+            '"upstream_cache_status":"$upstream_cache_status",'
+            '"http_host":"$http_host",'
+            '"http_user_agent":"$http_user_agent"'
+        '}';
+    access_log /var/log/nginx/access.json.log json_log;
+
     # Rate limiting zones — define per-service zones as needed
     limit_req_zone $binary_remote_addr zone=general:10m rate=10r/s;
 
@@ -54,6 +71,12 @@ http {
             return 200 "ok\n";
         }
 
+        location /stub_status {
+            stub_status;
+            allow 127.0.0.1;
+            deny all;
+        }
+
         location / {
             return 444;
         }

fly/start.sh

@@ -13,5 +13,11 @@ tailscale up --authkey="${TS_AUTHKEY}" --hostname=flyio-proxy
 until tailscale status > /dev/null 2>&1; do sleep 1; done
 echo "Tailscale connected"
 
+# Start Alloy for observability (logs → Loki, metrics → Prometheus)
+alloy run /etc/alloy/config.alloy \
+    --server.http.listen-addr=127.0.0.1:12345 \
+    --storage.path=/tmp/alloy-data &
+echo "Alloy started"
+
 # Start nginx — MagicDNS resolves *.tail8d86e.ts.net hostnames
 nginx -g "daemon off;"