From a0dc7ec511d9b971ed77b8479e6d3c6b19603566 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Sun, 8 Feb 2026 08:04:29 -0800 Subject: [PATCH 1/7] Add observability to Fly.io proxy via embedded Alloy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instrument the flyio-proxy container with Grafana Alloy to collect nginx JSON access logs (→ Loki) and derive request/latency/cache metrics (→ Prometheus). Adds stub_status for connection-level metrics. Includes two Grafana dashboards: Docs APM (per-service) and Fly.io Proxy Health (aggregate). Co-Authored-By: Claude Opus 4.6 --- .../dashboards/configmap-docs-apm.yaml | 262 ++++++++++++++++++ .../dashboards/configmap-flyio.yaml | 262 ++++++++++++++++++ .../grafana-config/kustomization.yaml | 2 + .../feature-flyio-observability.feature.md | 1 + docs/reference/services/flyio-proxy.md | 28 +- fly/Dockerfile | 7 + fly/alloy.river | 147 ++++++++++ fly/nginx.conf | 23 ++ fly/start.sh | 6 + 9 files changed, 735 insertions(+), 3 deletions(-) create mode 100644 argocd/manifests/grafana-config/dashboards/configmap-docs-apm.yaml create mode 100644 argocd/manifests/grafana-config/dashboards/configmap-flyio.yaml create mode 100644 docs/changelog.d/feature-flyio-observability.feature.md create mode 100644 fly/alloy.river diff --git a/argocd/manifests/grafana-config/dashboards/configmap-docs-apm.yaml b/argocd/manifests/grafana-config/dashboards/configmap-docs-apm.yaml new file mode 100644 index 0000000..44dd184 --- /dev/null +++ b/argocd/manifests/grafana-config/dashboards/configmap-docs-apm.yaml @@ -0,0 +1,262 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-docs-apm + namespace: monitoring + labels: + grafana_dashboard: "1" +data: + docs-apm.json: | + { + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "req/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "normal" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 16, "x": 0, "y": 0 }, + "id": 1, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (status) (rate(flyio_nginx_http_requests_total{host=\"docs.eblu.me\"}[5m]))", "legendFormat": "{{status}}", "refId": "A" } + ], + "title": "Request Rate by Status", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.05 }] }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 0 }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_http_requests_total{host=\"docs.eblu.me\",status=~\"5..\"}[5m])) / sum(rate(flyio_nginx_http_requests_total{host=\"docs.eblu.me\"}[5m]))", "refId": "A" } + ], + "title": "Error Rate (5xx)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 0.8 }] }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 4 }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_cache_requests_total{host=\"docs.eblu.me\",cache_status=\"HIT\"}[5m])) / sum(rate(flyio_nginx_cache_requests_total{host=\"docs.eblu.me\"}[5m]))", "refId": "A" } + ], + "title": "Cache Hit Ratio", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 4 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_http_requests_total{host=\"docs.eblu.me\"}[5m]))", "refId": "A" } + ], + "title": "Current RPS", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "id": 5, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(flyio_nginx_http_request_duration_seconds_bucket{host=\"docs.eblu.me\"}[5m])))", "legendFormat": "p50", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.90, sum by (le) (rate(flyio_nginx_http_request_duration_seconds_bucket{host=\"docs.eblu.me\"}[5m])))", "legendFormat": "p90", "refId": "B" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(flyio_nginx_http_request_duration_seconds_bucket{host=\"docs.eblu.me\"}[5m])))", "legendFormat": "p99", "refId": "C" } + ], + "title": "Latency Percentiles", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "id": 6, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_http_response_bytes_total{host=\"docs.eblu.me\"}[5m]))", "legendFormat": "Bandwidth", "refId": "A" } + ], + "title": "Bandwidth", + "type": "timeseries" + }, + { + "datasource": { "type": "loki", "uid": "loki" }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 }, + "id": 7, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { "datasource": { "type": "loki", "uid": "loki" }, "expr": "{instance=\"flyio-proxy\", job=\"flyio-nginx\"} |= \"docs.eblu.me\"", "refId": "A" } + ], + "title": "Recent Access Logs", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "tags": ["docs", "flyio", "apm"], + "templating": { "list": [] }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "Docs APM", + "uid": "docs-apm", + "version": 1, + "weekStart": "" + } diff --git a/argocd/manifests/grafana-config/dashboards/configmap-flyio.yaml b/argocd/manifests/grafana-config/dashboards/configmap-flyio.yaml new file mode 100644 index 0000000..ca3df75 --- /dev/null +++ b/argocd/manifests/grafana-config/dashboards/configmap-flyio.yaml @@ -0,0 +1,262 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-flyio + namespace: monitoring + labels: + grafana_dashboard: "1" +data: + flyio.json: | + { + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 100 }] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 0, "y": 0 }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "nginx_connections_active{instance=\"flyio-proxy\"}", "refId": "A" } + ], + "title": "Active Connections", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [ + { "options": { "0": { "color": "red", "text": "DOWN" }, "1": { "color": "green", "text": "UP" } }, "type": "value" } + ], + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 6, "y": 0 }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "value" + }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "up{instance=\"flyio-proxy\"}", "refId": "A" } + ], + "title": "Alloy Health", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "connections", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "normal" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 }, + "id": 3, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "nginx_connections_reading{instance=\"flyio-proxy\"}", "legendFormat": "Reading", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "nginx_connections_writing{instance=\"flyio-proxy\"}", "legendFormat": "Writing", "refId": "B" }, + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "nginx_connections_waiting{instance=\"flyio-proxy\"}", "legendFormat": "Waiting", "refId": "C" } + ], + "title": "Connection States", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "req/s", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "normal" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "id": 4, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (host) (rate(flyio_nginx_http_requests_total{instance=\"flyio-proxy\"}[5m]))", "legendFormat": "{{host}}", "refId": "A" } + ], + "title": "Total Request Rate by Host", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineWidth": 1, + "scaleDistribution": { "type": "linear" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "id": 5, + "options": { + "barRadius": 0.05, + "barWidth": 0.7, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true }, + "orientation": "horizontal", + "showValue": "always", + "stacking": "none", + "tooltip": { "mode": "single", "sort": "none" }, + "xTickLabelRotation": 0 + }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (cache_status) (increase(flyio_nginx_cache_requests_total{instance=\"flyio-proxy\"}[$__range]))", "legendFormat": "{{cache_status}}", "refId": "A", "instant": true } + ], + "title": "Cache Performance", + "type": "barchart" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 }, + "id": 6, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(flyio_nginx_http_request_duration_seconds_bucket{instance=\"flyio-proxy\"}[5m])))", "legendFormat": "p95", "refId": "A" } + ], + "title": "Upstream Response Time p95", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "tags": ["flyio", "nginx", "proxy"], + "templating": { "list": [] }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "Fly.io Proxy Health", + "uid": "flyio-proxy", + "version": 1, + "weekStart": "" + } diff --git a/argocd/manifests/grafana-config/kustomization.yaml b/argocd/manifests/grafana-config/kustomization.yaml index 33b60b6..00dc5c6 100644 --- a/argocd/manifests/grafana-config/kustomization.yaml +++ b/argocd/manifests/grafana-config/kustomization.yaml @@ -17,6 +17,8 @@ resources: - dashboards/configmap-postgresql.yaml - dashboards/configmap-services.yaml - dashboards/configmap-zot.yaml + - dashboards/configmap-docs-apm.yaml + - dashboards/configmap-flyio.yaml # TeslaMate dashboards - dashboards/configmap-teslamate-overview.yaml - dashboards/configmap-teslamate-charges.yaml diff --git a/docs/changelog.d/feature-flyio-observability.feature.md b/docs/changelog.d/feature-flyio-observability.feature.md new file mode 100644 index 0000000..dae4207 --- /dev/null +++ b/docs/changelog.d/feature-flyio-observability.feature.md @@ -0,0 +1 @@ +Add observability to Fly.io proxy: Alloy collects nginx access logs (→ Loki) and derived metrics (→ Prometheus), with Grafana dashboards for Docs APM and Fly.io proxy health. diff --git a/docs/reference/services/flyio-proxy.md b/docs/reference/services/flyio-proxy.md index 17162b2..d39cacc 100644 --- a/docs/reference/services/flyio-proxy.md +++ b/docs/reference/services/flyio-proxy.md @@ -35,9 +35,10 @@ Internet traffic hits Fly.io's Anycast edge, terminates TLS with a Let's Encrypt | File | Purpose | |------|---------| | `fly/fly.toml` | App configuration | -| `fly/Dockerfile` | nginx + Tailscale container | -| `fly/nginx.conf` | Reverse proxy, caching, rate limiting | -| `fly/start.sh` | Entrypoint: start Tailscale, then nginx | +| `fly/Dockerfile` | nginx + Tailscale + Alloy container | +| `fly/nginx.conf` | Reverse proxy, caching, rate limiting, JSON logging | +| `fly/alloy.river` | Alloy config: log tailing, metric extraction, remote_write | +| `fly/start.sh` | Entrypoint: start Tailscale, Alloy, then nginx | | `pulumi/tailscale/__main__.py` | Auth key (`tag:flyio-proxy`) | | `pulumi/tailscale/policy.hujson` | ACL grants for proxy | | `pulumi/gandi/__main__.py` | DNS CNAMEs | @@ -48,6 +49,27 @@ Fly.io runs Firecracker microVMs which support TUN devices natively. Tailscale r The Tailscale auth key is `preauthorized=True` to avoid device approval hangs on container restarts. +## Observability + +[[alloy|Alloy]] runs inside the container alongside nginx and Tailscale, providing: + +- **Logs**: nginx JSON access logs tailed and pushed to [[loki|Loki]] (`{instance="flyio-proxy", job="flyio-nginx"}`) +- **Metrics**: Derived from access logs and `stub_status`, pushed to [[prometheus|Prometheus]] via `remote_write` + - `flyio_nginx_http_requests_total` — request rate by status/method/host + - `flyio_nginx_http_request_duration_seconds` — latency histogram + - `flyio_nginx_http_response_bytes_total` — response bandwidth + - `flyio_nginx_cache_requests_total` — cache HIT/MISS/EXPIRED counts + - `nginx_connections_active/reading/writing/waiting` — connection states + +### Dashboards + +| Dashboard | Purpose | +|-----------|---------| +| **Docs APM** | Per-service view for `docs.eblu.me`: request rate, latency percentiles, cache hit ratio, error rate, bandwidth, access logs | +| **Fly.io Proxy Health** | Aggregate proxy health: connections, total request rate by host, cache performance, upstream latency, Alloy health | + +Alloy listens on `127.0.0.1:12345` for self-scraping its `/metrics` endpoint. All metrics carry `instance="flyio-proxy"`. + ## Secrets | Secret | Source | Description | diff --git a/fly/Dockerfile b/fly/Dockerfile index 11af474..ca20beb 100644 --- a/fly/Dockerfile +++ b/fly/Dockerfile @@ -9,7 +9,14 @@ COPY --from=docker.io/tailscale/tailscale:stable \ RUN mkdir -p /var/run/tailscale /var/lib/tailscale \ && apk add --no-cache iptables ip6tables +# Copy Alloy binary from official image +COPY --from=docker.io/grafana/alloy:v1.5.1 \ + /bin/alloy /usr/local/bin/alloy + +RUN mkdir -p /var/log/nginx /etc/alloy /tmp/alloy-data + COPY nginx.conf /etc/nginx/nginx.conf +COPY alloy.river /etc/alloy/config.alloy COPY start.sh /start.sh RUN chmod +x /start.sh diff --git a/fly/alloy.river b/fly/alloy.river new file mode 100644 index 0000000..fe0da1f --- /dev/null +++ b/fly/alloy.river @@ -0,0 +1,147 @@ +// Grafana Alloy configuration for flyio-proxy +// Collects nginx access logs → Loki, extracts metrics → Prometheus, +// and scrapes nginx stub_status for connection-level metrics. + +// ============== NGINX STUB_STATUS METRICS ============== + +prometheus.exporter.nginx "stub_status" { + nginx_address = "http://127.0.0.1:8080/stub_status" +} + +prometheus.scrape "nginx" { + targets = prometheus.exporter.nginx.stub_status.targets + forward_to = [prometheus.relabel.instance.receiver] + scrape_interval = "15s" +} + +// ============== LOG COLLECTION ============== + +// Tail the JSON access log written by nginx +local.file_match "nginx_access" { + path_targets = [ + {__path__ = "/var/log/nginx/access.json.log", job = "flyio-nginx"}, + ] +} + +loki.source.file "nginx_access" { + targets = local.file_match.nginx_access.targets + forward_to = [loki.process.nginx.receiver] +} + +// Parse JSON fields, extract labels, derive metrics +loki.process "nginx" { + forward_to = [loki.relabel.instance.receiver] + + // Parse the JSON log line + stage.json { + expressions = { + status = "status", + method = "request_method", + host = "http_host", + cache_status = "upstream_cache_status", + request_time = "request_time", + body_bytes_sent = "body_bytes_sent", + upstream_response_time = "upstream_response_time", + } + } + + // Promote to labels for filtering in Loki + stage.labels { + values = { + status = "", + method = "", + host = "", + cache_status = "", + } + } + + // --- Derived metrics (exposed on Alloy's /metrics endpoint) --- + + stage.metrics { + metric.counter { + name = "flyio_nginx_http_requests_total" + description = "Total HTTP requests by status, method, and host." + match_all = true + action = "inc" + } + } + + stage.metrics { + metric.histogram { + name = "flyio_nginx_http_request_duration_seconds" + description = "HTTP request latency in seconds." + source = "request_time" + buckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10] + } + } + + stage.metrics { + metric.counter { + name = "flyio_nginx_http_response_bytes_total" + description = "Total bytes sent in HTTP responses." + source = "body_bytes_sent" + action = "add" + } + } + + stage.metrics { + metric.counter { + name = "flyio_nginx_cache_requests_total" + description = "Total cache lookups by cache status." + source = "cache_status" + match_all = true + action = "inc" + } + } +} + +// Add instance label to logs +loki.relabel "instance" { + forward_to = [loki.write.loki.receiver] + + rule { + target_label = "instance" + replacement = "flyio-proxy" + } +} + +// Write logs to Loki +loki.write "loki" { + endpoint { + url = "https://loki.tail8d86e.ts.net/loki/api/v1/push" + + tls_config { + insecure_skip_verify = true + } + } +} + +// ============== METRICS PIPELINE ============== + +// Self-scrape to collect the log-derived metrics from /metrics +prometheus.scrape "self" { + targets = [{"__address__" = "127.0.0.1:12345"}] + forward_to = [prometheus.relabel.instance.receiver] + scrape_interval = "15s" +} + +// Add instance label to all metrics +prometheus.relabel "instance" { + forward_to = [prometheus.remote_write.prometheus.receiver] + + rule { + target_label = "instance" + replacement = "flyio-proxy" + } +} + +// Push metrics to Prometheus +prometheus.remote_write "prometheus" { + endpoint { + url = "https://prometheus.tail8d86e.ts.net/api/v1/write" + + tls_config { + insecure_skip_verify = true + } + } +} diff --git a/fly/nginx.conf b/fly/nginx.conf index bf89e09..c1f6169 100644 --- a/fly/nginx.conf +++ b/fly/nginx.conf @@ -8,6 +8,23 @@ http { include /etc/nginx/mime.types; default_type application/octet-stream; + # JSON access log for Alloy to tail → Loki + metric extraction + log_format json_log escape=json + '{' + '"time":"$time_iso8601",' + '"remote_addr":"$remote_addr",' + '"request_method":"$request_method",' + '"request_uri":"$request_uri",' + '"status":$status,' + '"body_bytes_sent":$body_bytes_sent,' + '"request_time":$request_time,' + '"upstream_response_time":"$upstream_response_time",' + '"upstream_cache_status":"$upstream_cache_status",' + '"http_host":"$http_host",' + '"http_user_agent":"$http_user_agent"' + '}'; + access_log /var/log/nginx/access.json.log json_log; + # Rate limiting zones — define per-service zones as needed limit_req_zone $binary_remote_addr zone=general:10m rate=10r/s; @@ -54,6 +71,12 @@ http { return 200 "ok\n"; } + location /stub_status { + stub_status; + allow 127.0.0.1; + deny all; + } + location / { return 444; } diff --git a/fly/start.sh b/fly/start.sh index 8478f09..fcd9718 100644 --- a/fly/start.sh +++ b/fly/start.sh @@ -13,5 +13,11 @@ tailscale up --authkey="${TS_AUTHKEY}" --hostname=flyio-proxy until tailscale status > /dev/null 2>&1; do sleep 1; done echo "Tailscale connected" +# Start Alloy for observability (logs → Loki, metrics → Prometheus) +alloy run /etc/alloy/config.alloy \ + --server.http.listen-addr=127.0.0.1:12345 \ + --storage.path=/tmp/alloy-data & +echo "Alloy started" + # Start nginx — MagicDNS resolves *.tail8d86e.ts.net hostnames nginx -g "daemon off;" -- 2.50.1 (Apple Git-155) From 1e1e513b4a1e74251825fab44bf400314460ae3b Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Sun, 8 Feb 2026 08:45:57 -0800 Subject: [PATCH 2/7] Route Alloy through Caddy for proper TLS, update ACLs Switch Alloy endpoints from *.tail8d86e.ts.net (with insecure_skip_verify) to *.ops.eblu.me via Caddy reverse proxy with valid TLS certificates. Add tag:homelab:443 to flyio-proxy ACL grant so the proxy can reach Caddy. Co-Authored-By: Claude Opus 4.6 --- fly/alloy.river | 16 ++++------------ pulumi/tailscale/policy.hujson | 8 ++++---- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/fly/alloy.river b/fly/alloy.river index fe0da1f..7720e19 100644 --- a/fly/alloy.river +++ b/fly/alloy.river @@ -105,14 +105,10 @@ loki.relabel "instance" { } } -// Write logs to Loki +// Write logs to Loki via Caddy (valid TLS, no skip_verify needed) loki.write "loki" { endpoint { - url = "https://loki.tail8d86e.ts.net/loki/api/v1/push" - - tls_config { - insecure_skip_verify = true - } + url = "https://loki.ops.eblu.me/loki/api/v1/push" } } @@ -135,13 +131,9 @@ prometheus.relabel "instance" { } } -// Push metrics to Prometheus +// Push metrics to Prometheus via Caddy (valid TLS, no skip_verify needed) prometheus.remote_write "prometheus" { endpoint { - url = "https://prometheus.tail8d86e.ts.net/api/v1/write" - - tls_config { - insecure_skip_verify = true - } + url = "https://prometheus.ops.eblu.me/api/v1/write" } } diff --git a/pulumi/tailscale/policy.hujson b/pulumi/tailscale/policy.hujson index 43542dd..2a23872 100644 --- a/pulumi/tailscale/policy.hujson +++ b/pulumi/tailscale/policy.hujson @@ -61,10 +61,10 @@ }, // --- Fly.io proxy --- - // Public reverse proxy can reach k8s services on HTTPS only + // Public reverse proxy can reach k8s services and Caddy on HTTPS { "src": ["tag:flyio-proxy"], - "dst": ["tag:k8s"], + "dst": ["tag:k8s", "tag:homelab"], "ip": ["tcp:443"], }, @@ -175,10 +175,10 @@ "src": "tag:ci-gateway", "accept": ["tag:registry:443"], }, - // Fly.io proxy can reach k8s services (HTTPS only), nothing else + // Fly.io proxy can reach k8s and Caddy on indri (HTTPS only), nothing else { "src": "tag:flyio-proxy", - "accept": ["tag:k8s:443"], + "accept": ["tag:k8s:443", "tag:homelab:443"], "deny": ["tag:homelab:22", "tag:nas:445", "tag:registry:443"], }, ], -- 2.50.1 (Apple Git-155) From 977f63a951ef64b4980cbee2962c5fcf223e38c0 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Sun, 8 Feb 2026 09:33:05 -0800 Subject: [PATCH 3/7] =?UTF-8?q?Document=20security=20implications=20of=20f?= =?UTF-8?q?lyio-proxy=20=E2=86=92=20homelab=20ACL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new ACL grant lets the Fly.io proxy reach all Caddy-proxied services, not just Loki/Prometheus. Document the expanded attack surface and trust boundary (requires RCE on gilbert or 1Password access) in both the flyio-proxy and caddy reference cards. Co-Authored-By: Claude Opus 4.6 --- docs/reference/services/caddy.md | 4 ++++ docs/reference/services/flyio-proxy.md | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/docs/reference/services/caddy.md b/docs/reference/services/caddy.md index 49b60e7..0ef0b31 100644 --- a/docs/reference/services/caddy.md +++ b/docs/reference/services/caddy.md @@ -79,6 +79,10 @@ mise run provision-indri -- --tags caddy The token is written to `~/.config/caddy/gandi-token` (chmod 0600) and sourced by the Caddy wrapper script. +## Security Considerations + +Caddy has no authentication layer — it is a plain reverse proxy. Access control relies entirely on Tailscale ACLs restricting which devices can reach indri on port 443. Currently `tag:homelab`, `autogroup:admin`, and `tag:flyio-proxy` can reach Caddy. The [[flyio-proxy]] grant exists so Alloy can push metrics/logs to Loki and Prometheus, but it means the Fly.io container can technically reach all Caddy-proxied services. See [[flyio-proxy#Security Considerations]] for the threat model. + ## Custom Build Caddy is built from source with the Gandi DNS plugin: diff --git a/docs/reference/services/flyio-proxy.md b/docs/reference/services/flyio-proxy.md index d39cacc..e75da0d 100644 --- a/docs/reference/services/flyio-proxy.md +++ b/docs/reference/services/flyio-proxy.md @@ -70,6 +70,14 @@ The Tailscale auth key is `preauthorized=True` to avoid device approval hangs on Alloy listens on `127.0.0.1:12345` for self-scraping its `/metrics` endpoint. All metrics carry `instance="flyio-proxy"`. +## Security Considerations + +The `tag:flyio-proxy` ACL grants access to both `tag:k8s:443` (for proxying public services) and `tag:homelab:443` (for pushing metrics/logs to [[caddy|Caddy]]-proxied Loki and Prometheus). This means a compromised nginx config could route traffic to **any** Caddy-proxied service — not just the intended backends. Some of those services (Loki, Prometheus) have no auth; others ([[forgejo]], [[navidrome]], [[immich]]) do. + +Exploitation requires either pushing a malicious image to Fly.io or modifying the nginx config — both of which require RCE on [[gilbert]] (where `fly` is authenticated) or access to [[1password]] (the deploy token). This is an acceptable boundary given that 1Password is already the trust root for the entire infrastructure. + +If this surface area becomes a concern, an alternative would be to add dedicated Tailscale Ingress tags for Loki/Prometheus write endpoints and restrict `tag:flyio-proxy` to only those. + ## Secrets | Secret | Source | Description | -- 2.50.1 (Apple Git-155) From 6e4d7f59916434ff53ee44b9df423abb5b7a22b7 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Sun, 8 Feb 2026 09:40:31 -0800 Subject: [PATCH 4/7] Fix Alloy binary on Alpine: add libc6-compat The grafana/alloy image is Ubuntu-based (glibc), but our container uses nginx:alpine (musl). The binary exists but fails with "not found" because the glibc dynamic linker is missing. libc6-compat provides the compatibility shim. Co-Authored-By: Claude Opus 4.6 --- fly/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fly/Dockerfile b/fly/Dockerfile index ca20beb..6e6146c 100644 --- a/fly/Dockerfile +++ b/fly/Dockerfile @@ -7,9 +7,10 @@ COPY --from=docker.io/tailscale/tailscale:stable \ /usr/local/bin/tailscale /usr/local/bin/tailscale RUN mkdir -p /var/run/tailscale /var/lib/tailscale \ - && apk add --no-cache iptables ip6tables + && apk add --no-cache iptables ip6tables \ + && apk add --no-cache libc6-compat -# Copy Alloy binary from official image +# Copy Alloy binary from official image (Ubuntu-based, needs libc6-compat) COPY --from=docker.io/grafana/alloy:v1.5.1 \ /bin/alloy /usr/local/bin/alloy -- 2.50.1 (Apple Git-155) From 80e7d11058a92ce263b4738c50490968b9c3861f Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Sun, 8 Feb 2026 09:45:39 -0800 Subject: [PATCH 5/7] =?UTF-8?q?Remove=20nginx=20stub=5Fstatus=20exporter?= =?UTF-8?q?=20=E2=80=94=20not=20available=20in=20Alloy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Alloy has no built-in prometheus.exporter.nginx component. Remove the stub_status scraping and connection panels from the Fly.io dashboard. Replace with error rate and cache hit ratio stats. All key signals are still covered by log-derived metrics. Co-Authored-By: Claude Opus 4.6 --- .../dashboards/configmap-flyio.yaml | 128 +++++++++--------- docs/reference/services/flyio-proxy.md | 3 +- fly/alloy.river | 17 +-- 3 files changed, 71 insertions(+), 77 deletions(-) diff --git a/argocd/manifests/grafana-config/dashboards/configmap-flyio.yaml b/argocd/manifests/grafana-config/dashboards/configmap-flyio.yaml index ca3df75..7228060 100644 --- a/argocd/manifests/grafana-config/dashboards/configmap-flyio.yaml +++ b/argocd/manifests/grafana-config/dashboards/configmap-flyio.yaml @@ -15,32 +15,6 @@ data: "id": null, "links": [], "panels": [ - { - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "thresholds" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 100 }] }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { "h": 6, "w": 6, "x": 0, "y": 0 }, - "id": 1, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "textMode": "auto" - }, - "targets": [ - { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "nginx_connections_active{instance=\"flyio-proxy\"}", "refId": "A" } - ], - "title": "Active Connections", - "type": "stat" - }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { @@ -54,8 +28,8 @@ data: }, "overrides": [] }, - "gridPos": { "h": 6, "w": 6, "x": 6, "y": 0 }, - "id": 2, + "gridPos": { "h": 6, "w": 6, "x": 0, "y": 0 }, + "id": 1, "options": { "colorMode": "background", "graphMode": "none", @@ -74,47 +48,79 @@ data: "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "connections", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { "type": "linear" }, - "showPoints": "never", - "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } - }, - "mappings": [], + "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "unit": "short" + "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 }, - "id": 3, + "gridPos": { "h": 6, "w": 6, "x": 6, "y": 0 }, + "id": 2, "options": { - "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" }, "targets": [ - { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "nginx_connections_reading{instance=\"flyio-proxy\"}", "legendFormat": "Reading", "refId": "A" }, - { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "nginx_connections_writing{instance=\"flyio-proxy\"}", "legendFormat": "Writing", "refId": "B" }, - { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "nginx_connections_waiting{instance=\"flyio-proxy\"}", "legendFormat": "Waiting", "refId": "C" } + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_http_requests_total{instance=\"flyio-proxy\"}[5m]))", "refId": "A" } ], - "title": "Connection States", - "type": "timeseries" + "title": "Total RPS", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.05 }] }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 12, "y": 0 }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_http_requests_total{instance=\"flyio-proxy\",status=~\"5..\"}[5m])) / sum(rate(flyio_nginx_http_requests_total{instance=\"flyio-proxy\"}[5m]))", "refId": "A" } + ], + "title": "Error Rate (5xx)", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 0.8 }] }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 18, "y": 0 }, + "id": 7, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_cache_requests_total{instance=\"flyio-proxy\",cache_status=\"HIT\"}[5m])) / sum(rate(flyio_nginx_cache_requests_total{instance=\"flyio-proxy\"}[5m]))", "refId": "A" } + ], + "title": "Cache Hit Ratio", + "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, diff --git a/docs/reference/services/flyio-proxy.md b/docs/reference/services/flyio-proxy.md index e75da0d..e33a65f 100644 --- a/docs/reference/services/flyio-proxy.md +++ b/docs/reference/services/flyio-proxy.md @@ -54,12 +54,11 @@ The Tailscale auth key is `preauthorized=True` to avoid device approval hangs on [[alloy|Alloy]] runs inside the container alongside nginx and Tailscale, providing: - **Logs**: nginx JSON access logs tailed and pushed to [[loki|Loki]] (`{instance="flyio-proxy", job="flyio-nginx"}`) -- **Metrics**: Derived from access logs and `stub_status`, pushed to [[prometheus|Prometheus]] via `remote_write` +- **Metrics**: Derived from access logs, pushed to [[prometheus|Prometheus]] via `remote_write` - `flyio_nginx_http_requests_total` — request rate by status/method/host - `flyio_nginx_http_request_duration_seconds` — latency histogram - `flyio_nginx_http_response_bytes_total` — response bandwidth - `flyio_nginx_cache_requests_total` — cache HIT/MISS/EXPIRED counts - - `nginx_connections_active/reading/writing/waiting` — connection states ### Dashboards diff --git a/fly/alloy.river b/fly/alloy.river index 7720e19..41bd8c9 100644 --- a/fly/alloy.river +++ b/fly/alloy.river @@ -1,18 +1,7 @@ // Grafana Alloy configuration for flyio-proxy -// Collects nginx access logs → Loki, extracts metrics → Prometheus, -// and scrapes nginx stub_status for connection-level metrics. - -// ============== NGINX STUB_STATUS METRICS ============== - -prometheus.exporter.nginx "stub_status" { - nginx_address = "http://127.0.0.1:8080/stub_status" -} - -prometheus.scrape "nginx" { - targets = prometheus.exporter.nginx.stub_status.targets - forward_to = [prometheus.relabel.instance.receiver] - scrape_interval = "15s" -} +// Collects nginx access logs → Loki, extracts metrics → Prometheus. +// Note: stub_status connection metrics are not collected — Alloy has no +// built-in nginx exporter. The log-derived metrics cover the key signals. // ============== LOG COLLECTION ============== -- 2.50.1 (Apple Git-155) From 176d38be68b37d16230ed371a10d73a08bc1a82f Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Sun, 8 Feb 2026 09:50:07 -0800 Subject: [PATCH 6/7] Fix metric names: strip loki_process_custom_ prefix, drop internal labels Alloy's stage.metrics prefixes all metric names with loki_process_custom_. Add a relabel rule to strip the prefix so dashboards can query clean names (flyio_nginx_http_requests_total etc). Also drop component_id/component_path/filename labels. Co-Authored-By: Claude Opus 4.6 --- fly/alloy.river | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/fly/alloy.river b/fly/alloy.river index 41bd8c9..d2dedb7 100644 --- a/fly/alloy.river +++ b/fly/alloy.river @@ -110,10 +110,24 @@ prometheus.scrape "self" { scrape_interval = "15s" } -// Add instance label to all metrics +// Strip the "loki_process_custom_" prefix that Alloy adds to stage.metrics, +// then add instance label. This keeps dashboard queries clean. prometheus.relabel "instance" { forward_to = [prometheus.remote_write.prometheus.receiver] + rule { + source_labels = ["__name__"] + regex = "loki_process_custom_(.*)" + target_label = "__name__" + replacement = "$1" + } + + // Drop internal labels added by the loki pipeline + rule { + regex = "component_id|component_path|filename" + action = "labeldrop" + } + rule { target_label = "instance" replacement = "flyio-proxy" -- 2.50.1 (Apple Git-155) From 34fceff62782bfcc9aceff905359cfb065769b0b Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Sun, 8 Feb 2026 10:05:13 -0800 Subject: [PATCH 7/7] Update Alloy, Prometheus, Loki, Grafana reference cards for flyio-proxy Add Fly.io proxy as a third Alloy deployment, document the new remote_write source in Prometheus, new log source in Loki, and two new dashboards in Grafana. Co-Authored-By: Claude Opus 4.6 --- docs/reference/services/alloy.md | 14 +++++++++++++- docs/reference/services/grafana.md | 2 ++ docs/reference/services/loki.md | 4 ++++ docs/reference/services/prometheus.md | 1 + 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/reference/services/alloy.md b/docs/reference/services/alloy.md index cc797d3..c7958bf 100644 --- a/docs/reference/services/alloy.md +++ b/docs/reference/services/alloy.md @@ -7,9 +7,10 @@ tags: # Grafana Alloy -Unified observability collector for metrics and logs with two deployments: +Unified observability collector for metrics and logs with three deployments: 1. **Indri (host)** - System metrics and service logs from macOS host 2. **Kubernetes (DaemonSet)** - Automatic pod log collection and service health probes +3. **Fly.io proxy (embedded)** - nginx access log metrics and log forwarding from [[flyio-proxy]] ## Quick Reference @@ -20,6 +21,8 @@ Unified observability collector for metrics and logs with two deployments: | **K8s Namespace** | `alloy` | | **K8s Image** | `grafana/alloy:v1.8.2` | | **ArgoCD App** | `alloy-k8s` | +| **Fly.io Config** | `fly/alloy.river` | +| **Fly.io Image** | `grafana/alloy:v1.5.1` (binary copied into nginx container) | ## Metrics Collected @@ -33,6 +36,13 @@ Unified observability collector for metrics and logs with two deployments: - All pod logs via `loki.source.kubernetes` - Service health probes: miniflux, kiwix, transmission, devpi, argocd +### From Fly.io Proxy +- `flyio_nginx_http_requests_total` — request rate by status/method/host +- `flyio_nginx_http_request_duration_seconds` — latency histogram +- `flyio_nginx_http_response_bytes_total` — response bandwidth +- `flyio_nginx_cache_requests_total` — cache HIT/MISS/EXPIRED counts +- Pushed to [[prometheus]] via remote_write through [[caddy]] + ## Logs Collected **Brew services:** forgejo, tailscale @@ -41,6 +51,8 @@ Unified observability collector for metrics and logs with two deployments: Logs pushed to [[loki]] at `https://loki.tail8d86e.ts.net/loki/api/v1/push`. +**Fly.io proxy:** nginx JSON access logs pushed to [[loki]] at `https://loki.ops.eblu.me/loki/api/v1/push` (via [[caddy]]). + ## Why Built from Source The Homebrew bottle uses `CGO_ENABLED=0`, which breaks Tailscale MagicDNS. Building with `CGO_ENABLED=1` uses the macOS native resolver. diff --git a/docs/reference/services/grafana.md b/docs/reference/services/grafana.md index 092997a..43f2a2c 100644 --- a/docs/reference/services/grafana.md +++ b/docs/reference/services/grafana.md @@ -41,6 +41,8 @@ Optional annotation: `grafana_folder: "FolderName"` - Minikube - Kubernetes cluster overview - Borgmatic Backups - Backup status and trends - Services Health - HTTP probe results +- Docs APM - Request rate, latency, cache for docs.eblu.me +- Fly.io Proxy Health - Aggregate proxy health across all upstream services - TeslaMate (18 dashboards) - Vehicle data ## Related diff --git a/docs/reference/services/loki.md b/docs/reference/services/loki.md index 6515c53..b015802 100644 --- a/docs/reference/services/loki.md +++ b/docs/reference/services/loki.md @@ -36,12 +36,16 @@ Log aggregation system for BlumeOps infrastructure. **From Kubernetes (via Alloy DaemonSet):** - All pods in all namespaces +**From Fly.io proxy (via embedded Alloy):** +- nginx JSON access logs (`{instance="flyio-proxy", job="flyio-nginx"}`) + ## Query Examples (LogQL) ```logql {service="forgejo"} # All forgejo logs {service="borgmatic", stream="stderr"} # Borgmatic errors {host="indri"} |= "error" # All logs containing "error" +{instance="flyio-proxy"} |= "docs.eblu.me" # Fly.io proxy access logs for docs ``` ## Related diff --git a/docs/reference/services/prometheus.md b/docs/reference/services/prometheus.md index 3d3cd44..67e491f 100644 --- a/docs/reference/services/prometheus.md +++ b/docs/reference/services/prometheus.md @@ -25,6 +25,7 @@ Metrics storage and querying for BlumeOps infrastructure. ### Remote Write (from Alloy) - Indri system metrics via [[alloy|Alloy]] remote_write - Textfile metrics: minikube, borgmatic, zot, jellyfin +- [[flyio-proxy]] nginx metrics (`flyio_nginx_*`) via Alloy embedded in Fly.io container ### Scrape Targets -- 2.50.1 (Apple Git-155)