Add observability to Fly.io proxy via embedded Alloy

Instrument the flyio-proxy container with Grafana Alloy to collect
nginx JSON access logs (→ Loki) and derive request/latency/cache
metrics (→ Prometheus). Adds stub_status for connection-level metrics.
Includes two Grafana dashboards: Docs APM (per-service) and Fly.io
Proxy Health (aggregate).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

argocd/manifests/grafana-config/dashboards/configmap-docs-apm.yaml

@@ -0,0 +1,262 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-dashboard-docs-apm
+  namespace: monitoring
+  labels:
+    grafana_dashboard: "1"
+data:
+  docs-apm.json: |
+    {
+      "annotations": { "list": [] },
+      "editable": true,
+      "fiscalYearStartMonth": 0,
+      "graphTooltip": 1,
+      "id": null,
+      "links": [],
+      "panels": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "req/s",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": { "legend": false, "tooltip": false, "viz": false },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": { "type": "linear" },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": { "group": "A", "mode": "normal" },
+                "thresholdsStyle": { "mode": "off" }
+              },
+              "mappings": [],
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
+              "unit": "reqps"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 8, "w": 16, "x": 0, "y": 0 },
+          "id": 1,
+          "options": {
+            "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true },
+            "tooltip": { "mode": "multi", "sort": "desc" }
+          },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (status) (rate(flyio_nginx_http_requests_total{host=\"docs.eblu.me\"}[5m]))", "legendFormat": "{{status}}", "refId": "A" }
+          ],
+          "title": "Request Rate by Status",
+          "type": "timeseries"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "thresholds" },
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.05 }] },
+              "unit": "percentunit"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 4, "w": 8, "x": 16, "y": 0 },
+          "id": 2,
+          "options": {
+            "colorMode": "background",
+            "graphMode": "area",
+            "justifyMode": "center",
+            "orientation": "auto",
+            "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+            "textMode": "auto"
+          },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_http_requests_total{host=\"docs.eblu.me\",status=~\"5..\"}[5m])) / sum(rate(flyio_nginx_http_requests_total{host=\"docs.eblu.me\"}[5m]))", "refId": "A" }
+          ],
+          "title": "Error Rate (5xx)",
+          "type": "stat"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "thresholds" },
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 0.8 }] },
+              "unit": "percentunit"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 4, "w": 4, "x": 16, "y": 4 },
+          "id": 3,
+          "options": {
+            "colorMode": "background",
+            "graphMode": "none",
+            "justifyMode": "center",
+            "orientation": "auto",
+            "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+            "textMode": "auto"
+          },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_cache_requests_total{host=\"docs.eblu.me\",cache_status=\"HIT\"}[5m])) / sum(rate(flyio_nginx_cache_requests_total{host=\"docs.eblu.me\"}[5m]))", "refId": "A" }
+          ],
+          "title": "Cache Hit Ratio",
+          "type": "stat"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "thresholds" },
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
+              "unit": "reqps"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 4, "w": 4, "x": 20, "y": 4 },
+          "id": 4,
+          "options": {
+            "colorMode": "value",
+            "graphMode": "area",
+            "justifyMode": "center",
+            "orientation": "auto",
+            "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+            "textMode": "auto"
+          },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_http_requests_total{host=\"docs.eblu.me\"}[5m]))", "refId": "A" }
+          ],
+          "title": "Current RPS",
+          "type": "stat"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "seconds",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": { "legend": false, "tooltip": false, "viz": false },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": { "type": "linear" },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": { "group": "A", "mode": "none" },
+                "thresholdsStyle": { "mode": "off" }
+              },
+              "mappings": [],
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+          "id": 5,
+          "options": {
+            "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true },
+            "tooltip": { "mode": "multi", "sort": "desc" }
+          },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(flyio_nginx_http_request_duration_seconds_bucket{host=\"docs.eblu.me\"}[5m])))", "legendFormat": "p50", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.90, sum by (le) (rate(flyio_nginx_http_request_duration_seconds_bucket{host=\"docs.eblu.me\"}[5m])))", "legendFormat": "p90", "refId": "B" },
+            { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(flyio_nginx_http_request_duration_seconds_bucket{host=\"docs.eblu.me\"}[5m])))", "legendFormat": "p99", "refId": "C" }
+          ],
+          "title": "Latency Percentiles",
+          "type": "timeseries"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": { "legend": false, "tooltip": false, "viz": false },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": { "type": "linear" },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": { "group": "A", "mode": "none" },
+                "thresholdsStyle": { "mode": "off" }
+              },
+              "mappings": [],
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
+              "unit": "Bps"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+          "id": 6,
+          "options": {
+            "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true },
+            "tooltip": { "mode": "single", "sort": "none" }
+          },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_http_response_bytes_total{host=\"docs.eblu.me\"}[5m]))", "legendFormat": "Bandwidth", "refId": "A" }
+          ],
+          "title": "Bandwidth",
+          "type": "timeseries"
+        },
+        {
+          "datasource": { "type": "loki", "uid": "loki" },
+          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 },
+          "id": 7,
+          "options": {
+            "dedupStrategy": "none",
+            "enableLogDetails": true,
+            "prettifyLogMessage": false,
+            "showCommonLabels": false,
+            "showLabels": false,
+            "showTime": true,
+            "sortOrder": "Descending",
+            "wrapLogMessage": false
+          },
+          "targets": [
+            { "datasource": { "type": "loki", "uid": "loki" }, "expr": "{instance=\"flyio-proxy\", job=\"flyio-nginx\"} |= \"docs.eblu.me\"", "refId": "A" }
+          ],
+          "title": "Recent Access Logs",
+          "type": "logs"
+        }
+      ],
+      "refresh": "30s",
+      "schemaVersion": 38,
+      "tags": ["docs", "flyio", "apm"],
+      "templating": { "list": [] },
+      "time": { "from": "now-6h", "to": "now" },
+      "timepicker": {},
+      "timezone": "",
+      "title": "Docs APM",
+      "uid": "docs-apm",
+      "version": 1,
+      "weekStart": ""
+    }

argocd/manifests/grafana-config/dashboards/configmap-flyio.yaml

@@ -0,0 +1,262 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-dashboard-flyio
+  namespace: monitoring
+  labels:
+    grafana_dashboard: "1"
+data:
+  flyio.json: |
+    {
+      "annotations": { "list": [] },
+      "editable": true,
+      "fiscalYearStartMonth": 0,
+      "graphTooltip": 1,
+      "id": null,
+      "links": [],
+      "panels": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "thresholds" },
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 100 }] },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 6, "w": 6, "x": 0, "y": 0 },
+          "id": 1,
+          "options": {
+            "colorMode": "background",
+            "graphMode": "area",
+            "justifyMode": "center",
+            "orientation": "auto",
+            "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+            "textMode": "auto"
+          },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "nginx_connections_active{instance=\"flyio-proxy\"}", "refId": "A" }
+          ],
+          "title": "Active Connections",
+          "type": "stat"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "thresholds" },
+              "mappings": [
+                { "options": { "0": { "color": "red", "text": "DOWN" }, "1": { "color": "green", "text": "UP" } }, "type": "value" }
+              ],
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 6, "w": 6, "x": 6, "y": 0 },
+          "id": 2,
+          "options": {
+            "colorMode": "background",
+            "graphMode": "none",
+            "justifyMode": "center",
+            "orientation": "auto",
+            "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
+            "textMode": "value"
+          },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "up{instance=\"flyio-proxy\"}", "refId": "A" }
+          ],
+          "title": "Alloy Health",
+          "type": "stat"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "connections",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": { "legend": false, "tooltip": false, "viz": false },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": { "type": "linear" },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": { "group": "A", "mode": "normal" },
+                "thresholdsStyle": { "mode": "off" }
+              },
+              "mappings": [],
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 },
+          "id": 3,
+          "options": {
+            "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true },
+            "tooltip": { "mode": "multi", "sort": "desc" }
+          },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "nginx_connections_reading{instance=\"flyio-proxy\"}", "legendFormat": "Reading", "refId": "A" },
+            { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "nginx_connections_writing{instance=\"flyio-proxy\"}", "legendFormat": "Writing", "refId": "B" },
+            { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "nginx_connections_waiting{instance=\"flyio-proxy\"}", "legendFormat": "Waiting", "refId": "C" }
+          ],
+          "title": "Connection States",
+          "type": "timeseries"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "req/s",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 20,
+                "gradientMode": "none",
+                "hideFrom": { "legend": false, "tooltip": false, "viz": false },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": { "type": "linear" },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": { "group": "A", "mode": "normal" },
+                "thresholdsStyle": { "mode": "off" }
+              },
+              "mappings": [],
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
+              "unit": "reqps"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
+          "id": 4,
+          "options": {
+            "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true },
+            "tooltip": { "mode": "multi", "sort": "desc" }
+          },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (host) (rate(flyio_nginx_http_requests_total{instance=\"flyio-proxy\"}[5m]))", "legendFormat": "{{host}}", "refId": "A" }
+          ],
+          "title": "Total Request Rate by Host",
+          "type": "timeseries"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "fillOpacity": 80,
+                "gradientMode": "none",
+                "hideFrom": { "legend": false, "tooltip": false, "viz": false },
+                "lineWidth": 1,
+                "scaleDistribution": { "type": "linear" }
+              },
+              "mappings": [],
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
+              "unit": "short"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
+          "id": 5,
+          "options": {
+            "barRadius": 0.05,
+            "barWidth": 0.7,
+            "fullHighlight": false,
+            "groupWidth": 0.7,
+            "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true },
+            "orientation": "horizontal",
+            "showValue": "always",
+            "stacking": "none",
+            "tooltip": { "mode": "single", "sort": "none" },
+            "xTickLabelRotation": 0
+          },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (cache_status) (increase(flyio_nginx_cache_requests_total{instance=\"flyio-proxy\"}[$__range]))", "legendFormat": "{{cache_status}}", "refId": "A", "instant": true }
+          ],
+          "title": "Cache Performance",
+          "type": "barchart"
+        },
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "fieldConfig": {
+            "defaults": {
+              "color": { "mode": "palette-classic" },
+              "custom": {
+                "axisBorderShow": false,
+                "axisCenteredZero": false,
+                "axisColorMode": "text",
+                "axisLabel": "seconds",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 10,
+                "gradientMode": "none",
+                "hideFrom": { "legend": false, "tooltip": false, "viz": false },
+                "insertNulls": false,
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": { "type": "linear" },
+                "showPoints": "never",
+                "spanNulls": false,
+                "stacking": { "group": "A", "mode": "none" },
+                "thresholdsStyle": { "mode": "off" }
+              },
+              "mappings": [],
+              "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
+              "unit": "s"
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 },
+          "id": 6,
+          "options": {
+            "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true },
+            "tooltip": { "mode": "multi", "sort": "desc" }
+          },
+          "targets": [
+            { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(flyio_nginx_http_request_duration_seconds_bucket{instance=\"flyio-proxy\"}[5m])))", "legendFormat": "p95", "refId": "A" }
+          ],
+          "title": "Upstream Response Time p95",
+          "type": "timeseries"
+        }
+      ],
+      "refresh": "30s",
+      "schemaVersion": 38,
+      "tags": ["flyio", "nginx", "proxy"],
+      "templating": { "list": [] },
+      "time": { "from": "now-6h", "to": "now" },
+      "timepicker": {},
+      "timezone": "",
+      "title": "Fly.io Proxy Health",
+      "uid": "flyio-proxy",
+      "version": 1,
+      "weekStart": ""
+    }

argocd/manifests/grafana-config/kustomization.yaml

@@ -17,6 +17,8 @@ resources:
   - dashboards/configmap-postgresql.yaml
   - dashboards/configmap-services.yaml
   - dashboards/configmap-zot.yaml
+  - dashboards/configmap-docs-apm.yaml
+  - dashboards/configmap-flyio.yaml
   # TeslaMate dashboards
   - dashboards/configmap-teslamate-overview.yaml
   - dashboards/configmap-teslamate-charges.yaml

docs/changelog.d/feature-flyio-observability.feature.md

@@ -0,0 +1 @@
+Add observability to Fly.io proxy: Alloy collects nginx access logs (→ Loki) and derived metrics (→ Prometheus), with Grafana dashboards for Docs APM and Fly.io proxy health.

docs/reference/services/flyio-proxy.md

@@ -35,9 +35,10 @@ Internet traffic hits Fly.io's Anycast edge, terminates TLS with a Let's Encrypt
 | File | Purpose |
 |------|---------|
 | `fly/fly.toml` | App configuration |
-| `fly/Dockerfile` | nginx + Tailscale container |
-| `fly/nginx.conf` | Reverse proxy, caching, rate limiting |
-| `fly/start.sh` | Entrypoint: start Tailscale, then nginx |
+| `fly/Dockerfile` | nginx + Tailscale + Alloy container |
+| `fly/nginx.conf` | Reverse proxy, caching, rate limiting, JSON logging |
+| `fly/alloy.river` | Alloy config: log tailing, metric extraction, remote_write |
+| `fly/start.sh` | Entrypoint: start Tailscale, Alloy, then nginx |
 | `pulumi/tailscale/__main__.py` | Auth key (`tag:flyio-proxy`) |
 | `pulumi/tailscale/policy.hujson` | ACL grants for proxy |
 | `pulumi/gandi/__main__.py` | DNS CNAMEs |
@@ -48,6 +49,27 @@ Fly.io runs Firecracker microVMs which support TUN devices natively. Tailscale r
 
 The Tailscale auth key is `preauthorized=True` to avoid device approval hangs on container restarts.
 
+## Observability
+
+[[alloy|Alloy]] runs inside the container alongside nginx and Tailscale, providing:
+
+- **Logs**: nginx JSON access logs tailed and pushed to [[loki|Loki]] (`{instance="flyio-proxy", job="flyio-nginx"}`)
+- **Metrics**: Derived from access logs and `stub_status`, pushed to [[prometheus|Prometheus]] via `remote_write`
+  - `flyio_nginx_http_requests_total` — request rate by status/method/host
+  - `flyio_nginx_http_request_duration_seconds` — latency histogram
+  - `flyio_nginx_http_response_bytes_total` — response bandwidth
+  - `flyio_nginx_cache_requests_total` — cache HIT/MISS/EXPIRED counts
+  - `nginx_connections_active/reading/writing/waiting` — connection states
+
+### Dashboards
+
+| Dashboard | Purpose |
+|-----------|---------|
+| **Docs APM** | Per-service view for `docs.eblu.me`: request rate, latency percentiles, cache hit ratio, error rate, bandwidth, access logs |
+| **Fly.io Proxy Health** | Aggregate proxy health: connections, total request rate by host, cache performance, upstream latency, Alloy health |
+
+Alloy listens on `127.0.0.1:12345` for self-scraping its `/metrics` endpoint. All metrics carry `instance="flyio-proxy"`.
+
 ## Secrets
 
 | Secret | Source | Description |

fly/Dockerfile

@@ -9,7 +9,14 @@ COPY --from=docker.io/tailscale/tailscale:stable \
 RUN mkdir -p /var/run/tailscale /var/lib/tailscale \
     && apk add --no-cache iptables ip6tables
 
+# Copy Alloy binary from official image
+COPY --from=docker.io/grafana/alloy:v1.5.1 \
+    /bin/alloy /usr/local/bin/alloy
+
+RUN mkdir -p /var/log/nginx /etc/alloy /tmp/alloy-data
+
 COPY nginx.conf /etc/nginx/nginx.conf
+COPY alloy.river /etc/alloy/config.alloy
 COPY start.sh /start.sh
 RUN chmod +x /start.sh
 

fly/alloy.river

@@ -0,0 +1,147 @@
+// Grafana Alloy configuration for flyio-proxy
+// Collects nginx access logs → Loki, extracts metrics → Prometheus,
+// and scrapes nginx stub_status for connection-level metrics.
+
+// ============== NGINX STUB_STATUS METRICS ==============
+
+prometheus.exporter.nginx "stub_status" {
+  nginx_address = "http://127.0.0.1:8080/stub_status"
+}
+
+prometheus.scrape "nginx" {
+  targets         = prometheus.exporter.nginx.stub_status.targets
+  forward_to      = [prometheus.relabel.instance.receiver]
+  scrape_interval = "15s"
+}
+
+// ============== LOG COLLECTION ==============
+
+// Tail the JSON access log written by nginx
+local.file_match "nginx_access" {
+  path_targets = [
+    {__path__ = "/var/log/nginx/access.json.log", job = "flyio-nginx"},
+  ]
+}
+
+loki.source.file "nginx_access" {
+  targets    = local.file_match.nginx_access.targets
+  forward_to = [loki.process.nginx.receiver]
+}
+
+// Parse JSON fields, extract labels, derive metrics
+loki.process "nginx" {
+  forward_to = [loki.relabel.instance.receiver]
+
+  // Parse the JSON log line
+  stage.json {
+    expressions = {
+      status               = "status",
+      method               = "request_method",
+      host                 = "http_host",
+      cache_status         = "upstream_cache_status",
+      request_time         = "request_time",
+      body_bytes_sent      = "body_bytes_sent",
+      upstream_response_time = "upstream_response_time",
+    }
+  }
+
+  // Promote to labels for filtering in Loki
+  stage.labels {
+    values = {
+      status       = "",
+      method       = "",
+      host         = "",
+      cache_status = "",
+    }
+  }
+
+  // --- Derived metrics (exposed on Alloy's /metrics endpoint) ---
+
+  stage.metrics {
+    metric.counter {
+      name        = "flyio_nginx_http_requests_total"
+      description = "Total HTTP requests by status, method, and host."
+      match_all   = true
+      action      = "inc"
+    }
+  }
+
+  stage.metrics {
+    metric.histogram {
+      name        = "flyio_nginx_http_request_duration_seconds"
+      description = "HTTP request latency in seconds."
+      source      = "request_time"
+      buckets     = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]
+    }
+  }
+
+  stage.metrics {
+    metric.counter {
+      name        = "flyio_nginx_http_response_bytes_total"
+      description = "Total bytes sent in HTTP responses."
+      source      = "body_bytes_sent"
+      action      = "add"
+    }
+  }
+
+  stage.metrics {
+    metric.counter {
+      name        = "flyio_nginx_cache_requests_total"
+      description = "Total cache lookups by cache status."
+      source      = "cache_status"
+      match_all   = true
+      action      = "inc"
+    }
+  }
+}
+
+// Add instance label to logs
+loki.relabel "instance" {
+  forward_to = [loki.write.loki.receiver]
+
+  rule {
+    target_label = "instance"
+    replacement  = "flyio-proxy"
+  }
+}
+
+// Write logs to Loki
+loki.write "loki" {
+  endpoint {
+    url = "https://loki.tail8d86e.ts.net/loki/api/v1/push"
+
+    tls_config {
+      insecure_skip_verify = true
+    }
+  }
+}
+
+// ============== METRICS PIPELINE ==============
+
+// Self-scrape to collect the log-derived metrics from /metrics
+prometheus.scrape "self" {
+  targets         = [{"__address__" = "127.0.0.1:12345"}]
+  forward_to      = [prometheus.relabel.instance.receiver]
+  scrape_interval = "15s"
+}
+
+// Add instance label to all metrics
+prometheus.relabel "instance" {
+  forward_to = [prometheus.remote_write.prometheus.receiver]
+
+  rule {
+    target_label = "instance"
+    replacement  = "flyio-proxy"
+  }
+}
+
+// Push metrics to Prometheus
+prometheus.remote_write "prometheus" {
+  endpoint {
+    url = "https://prometheus.tail8d86e.ts.net/api/v1/write"
+
+    tls_config {
+      insecure_skip_verify = true
+    }
+  }
+}

fly/nginx.conf

@@ -8,6 +8,23 @@ http {
     include /etc/nginx/mime.types;
     default_type application/octet-stream;
 
+    # JSON access log for Alloy to tail → Loki + metric extraction
+    log_format json_log escape=json
+        '{'
+            '"time":"$time_iso8601",'
+            '"remote_addr":"$remote_addr",'
+            '"request_method":"$request_method",'
+            '"request_uri":"$request_uri",'
+            '"status":$status,'
+            '"body_bytes_sent":$body_bytes_sent,'
+            '"request_time":$request_time,'
+            '"upstream_response_time":"$upstream_response_time",'
+            '"upstream_cache_status":"$upstream_cache_status",'
+            '"http_host":"$http_host",'
+            '"http_user_agent":"$http_user_agent"'
+        '}';
+    access_log /var/log/nginx/access.json.log json_log;
+
     # Rate limiting zones — define per-service zones as needed
     limit_req_zone $binary_remote_addr zone=general:10m rate=10r/s;
 
@@ -54,6 +71,12 @@ http {
             return 200 "ok\n";
         }
 
+        location /stub_status {
+            stub_status;
+            allow 127.0.0.1;
+            deny all;
+        }
+
         location / {
             return 444;
         }

fly/start.sh

@@ -13,5 +13,11 @@ tailscale up --authkey="${TS_AUTHKEY}" --hostname=flyio-proxy
 until tailscale status > /dev/null 2>&1; do sleep 1; done
 echo "Tailscale connected"
 
+# Start Alloy for observability (logs → Loki, metrics → Prometheus)
+alloy run /etc/alloy/config.alloy \
+    --server.http.listen-addr=127.0.0.1:12345 \
+    --storage.path=/tmp/alloy-data &
+echo "Alloy started"
+
 # Start nginx — MagicDNS resolves *.tail8d86e.ts.net hostnames
 nginx -g "daemon off;"