Add Fly.io proxy observability via embedded Alloy (#123)
All checks were successful
Deploy Fly.io Proxy / deploy (push) Successful in 1m16s
All checks were successful
Deploy Fly.io Proxy / deploy (push) Successful in 1m16s
## Summary - Embed Grafana Alloy in the Fly.io proxy container to collect nginx JSON access logs (→ Loki) and derive request rate, latency histogram, cache status, and bandwidth metrics (→ Prometheus) - Add nginx `stub_status` endpoint for connection-level metrics (active/reading/writing/waiting) - Create two Grafana dashboards: **Docs APM** (per-service view filtered by `host="docs.eblu.me"`) and **Fly.io Proxy Health** (aggregate proxy health across all upstream services) ## Changed Files | File | Change | |------|--------| | `fly/nginx.conf` | Add JSON `log_format` + `access_log`, add `stub_status` endpoint | | `fly/Dockerfile` | COPY Alloy binary from `grafana/alloy:v1.5.1`, COPY `alloy.river` config | | `fly/alloy.river` | **New** — Alloy config: log tailing, metric extraction, remote_write | | `fly/start.sh` | Start Alloy after Tailscale, before nginx | | `argocd/manifests/grafana-config/dashboards/configmap-docs-apm.yaml` | **New** — Docs APM dashboard | | `argocd/manifests/grafana-config/dashboards/configmap-flyio.yaml` | **New** — Fly.io Proxy Health dashboard | | `argocd/manifests/grafana-config/kustomization.yaml` | Register new dashboard configmaps | | `docs/reference/services/flyio-proxy.md` | Document observability setup | ## Deployment and Testing - [ ] `mise run fly-deploy` — rebuild container with Alloy - [ ] `curl https://docs.eblu.me/` — generate traffic - [ ] `fly logs -a blumeops-proxy` — verify Alloy startup - [ ] Query Prometheus: `flyio_nginx_http_requests_total{instance="flyio-proxy"}` - [ ] Query Loki: `{instance="flyio-proxy", job="flyio-nginx"}` - [ ] `argocd app sync grafana-config` — deploy dashboards - [ ] Verify dashboards show data in Grafana - [ ] `mise run services-check` — no regressions Reviewed-on: https://forge.ops.eblu.me/eblume/blumeops/pulls/123
This commit is contained in:
parent
61862b44c0
commit
cc54b4f565
15 changed files with 773 additions and 9 deletions
|
|
@ -0,0 +1,262 @@
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: grafana-dashboard-docs-apm
|
||||||
|
namespace: monitoring
|
||||||
|
labels:
|
||||||
|
grafana_dashboard: "1"
|
||||||
|
data:
|
||||||
|
docs-apm.json: |
|
||||||
|
{
|
||||||
|
"annotations": { "list": [] },
|
||||||
|
"editable": true,
|
||||||
|
"fiscalYearStartMonth": 0,
|
||||||
|
"graphTooltip": 1,
|
||||||
|
"id": null,
|
||||||
|
"links": [],
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "palette-classic" },
|
||||||
|
"custom": {
|
||||||
|
"axisBorderShow": false,
|
||||||
|
"axisCenteredZero": false,
|
||||||
|
"axisColorMode": "text",
|
||||||
|
"axisLabel": "req/s",
|
||||||
|
"axisPlacement": "auto",
|
||||||
|
"barAlignment": 0,
|
||||||
|
"drawStyle": "line",
|
||||||
|
"fillOpacity": 20,
|
||||||
|
"gradientMode": "none",
|
||||||
|
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||||
|
"insertNulls": false,
|
||||||
|
"lineInterpolation": "linear",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"pointSize": 5,
|
||||||
|
"scaleDistribution": { "type": "linear" },
|
||||||
|
"showPoints": "never",
|
||||||
|
"spanNulls": false,
|
||||||
|
"stacking": { "group": "A", "mode": "normal" },
|
||||||
|
"thresholdsStyle": { "mode": "off" }
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||||
|
"unit": "reqps"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 8, "w": 16, "x": 0, "y": 0 },
|
||||||
|
"id": 1,
|
||||||
|
"options": {
|
||||||
|
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true },
|
||||||
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (status) (rate(flyio_nginx_http_requests_total{host=\"docs.eblu.me\"}[5m]))", "legendFormat": "{{status}}", "refId": "A" }
|
||||||
|
],
|
||||||
|
"title": "Request Rate by Status",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "thresholds" },
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.05 }] },
|
||||||
|
"unit": "percentunit"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 4, "w": 8, "x": 16, "y": 0 },
|
||||||
|
"id": 2,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "background",
|
||||||
|
"graphMode": "area",
|
||||||
|
"justifyMode": "center",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_http_requests_total{host=\"docs.eblu.me\",status=~\"5..\"}[5m])) / sum(rate(flyio_nginx_http_requests_total{host=\"docs.eblu.me\"}[5m]))", "refId": "A" }
|
||||||
|
],
|
||||||
|
"title": "Error Rate (5xx)",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "thresholds" },
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 0.8 }] },
|
||||||
|
"unit": "percentunit"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 4 },
|
||||||
|
"id": 3,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "background",
|
||||||
|
"graphMode": "none",
|
||||||
|
"justifyMode": "center",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_cache_requests_total{host=\"docs.eblu.me\",cache_status=\"HIT\"}[5m])) / sum(rate(flyio_nginx_cache_requests_total{host=\"docs.eblu.me\"}[5m]))", "refId": "A" }
|
||||||
|
],
|
||||||
|
"title": "Cache Hit Ratio",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "thresholds" },
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||||
|
"unit": "reqps"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 4 },
|
||||||
|
"id": 4,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "area",
|
||||||
|
"justifyMode": "center",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_http_requests_total{host=\"docs.eblu.me\"}[5m]))", "refId": "A" }
|
||||||
|
],
|
||||||
|
"title": "Current RPS",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "palette-classic" },
|
||||||
|
"custom": {
|
||||||
|
"axisBorderShow": false,
|
||||||
|
"axisCenteredZero": false,
|
||||||
|
"axisColorMode": "text",
|
||||||
|
"axisLabel": "seconds",
|
||||||
|
"axisPlacement": "auto",
|
||||||
|
"barAlignment": 0,
|
||||||
|
"drawStyle": "line",
|
||||||
|
"fillOpacity": 10,
|
||||||
|
"gradientMode": "none",
|
||||||
|
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||||
|
"insertNulls": false,
|
||||||
|
"lineInterpolation": "linear",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"pointSize": 5,
|
||||||
|
"scaleDistribution": { "type": "linear" },
|
||||||
|
"showPoints": "never",
|
||||||
|
"spanNulls": false,
|
||||||
|
"stacking": { "group": "A", "mode": "none" },
|
||||||
|
"thresholdsStyle": { "mode": "off" }
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||||
|
"unit": "s"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
||||||
|
"id": 5,
|
||||||
|
"options": {
|
||||||
|
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true },
|
||||||
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(flyio_nginx_http_request_duration_seconds_bucket{host=\"docs.eblu.me\"}[5m])))", "legendFormat": "p50", "refId": "A" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.90, sum by (le) (rate(flyio_nginx_http_request_duration_seconds_bucket{host=\"docs.eblu.me\"}[5m])))", "legendFormat": "p90", "refId": "B" },
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(flyio_nginx_http_request_duration_seconds_bucket{host=\"docs.eblu.me\"}[5m])))", "legendFormat": "p99", "refId": "C" }
|
||||||
|
],
|
||||||
|
"title": "Latency Percentiles",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "palette-classic" },
|
||||||
|
"custom": {
|
||||||
|
"axisBorderShow": false,
|
||||||
|
"axisCenteredZero": false,
|
||||||
|
"axisColorMode": "text",
|
||||||
|
"axisLabel": "",
|
||||||
|
"axisPlacement": "auto",
|
||||||
|
"barAlignment": 0,
|
||||||
|
"drawStyle": "line",
|
||||||
|
"fillOpacity": 20,
|
||||||
|
"gradientMode": "none",
|
||||||
|
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||||
|
"insertNulls": false,
|
||||||
|
"lineInterpolation": "linear",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"pointSize": 5,
|
||||||
|
"scaleDistribution": { "type": "linear" },
|
||||||
|
"showPoints": "never",
|
||||||
|
"spanNulls": false,
|
||||||
|
"stacking": { "group": "A", "mode": "none" },
|
||||||
|
"thresholdsStyle": { "mode": "off" }
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||||
|
"unit": "Bps"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
||||||
|
"id": 6,
|
||||||
|
"options": {
|
||||||
|
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true },
|
||||||
|
"tooltip": { "mode": "single", "sort": "none" }
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_http_response_bytes_total{host=\"docs.eblu.me\"}[5m]))", "legendFormat": "Bandwidth", "refId": "A" }
|
||||||
|
],
|
||||||
|
"title": "Bandwidth",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "loki" },
|
||||||
|
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 },
|
||||||
|
"id": 7,
|
||||||
|
"options": {
|
||||||
|
"dedupStrategy": "none",
|
||||||
|
"enableLogDetails": true,
|
||||||
|
"prettifyLogMessage": false,
|
||||||
|
"showCommonLabels": false,
|
||||||
|
"showLabels": false,
|
||||||
|
"showTime": true,
|
||||||
|
"sortOrder": "Descending",
|
||||||
|
"wrapLogMessage": false
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "loki", "uid": "loki" }, "expr": "{instance=\"flyio-proxy\", job=\"flyio-nginx\"} |= \"docs.eblu.me\"", "refId": "A" }
|
||||||
|
],
|
||||||
|
"title": "Recent Access Logs",
|
||||||
|
"type": "logs"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"refresh": "30s",
|
||||||
|
"schemaVersion": 38,
|
||||||
|
"tags": ["docs", "flyio", "apm"],
|
||||||
|
"templating": { "list": [] },
|
||||||
|
"time": { "from": "now-6h", "to": "now" },
|
||||||
|
"timepicker": {},
|
||||||
|
"timezone": "",
|
||||||
|
"title": "Docs APM",
|
||||||
|
"uid": "docs-apm",
|
||||||
|
"version": 1,
|
||||||
|
"weekStart": ""
|
||||||
|
}
|
||||||
268
argocd/manifests/grafana-config/dashboards/configmap-flyio.yaml
Normal file
268
argocd/manifests/grafana-config/dashboards/configmap-flyio.yaml
Normal file
|
|
@ -0,0 +1,268 @@
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: grafana-dashboard-flyio
|
||||||
|
namespace: monitoring
|
||||||
|
labels:
|
||||||
|
grafana_dashboard: "1"
|
||||||
|
data:
|
||||||
|
flyio.json: |
|
||||||
|
{
|
||||||
|
"annotations": { "list": [] },
|
||||||
|
"editable": true,
|
||||||
|
"fiscalYearStartMonth": 0,
|
||||||
|
"graphTooltip": 1,
|
||||||
|
"id": null,
|
||||||
|
"links": [],
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "thresholds" },
|
||||||
|
"mappings": [
|
||||||
|
{ "options": { "0": { "color": "red", "text": "DOWN" }, "1": { "color": "green", "text": "UP" } }, "type": "value" }
|
||||||
|
],
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] },
|
||||||
|
"unit": "short"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 6, "w": 6, "x": 0, "y": 0 },
|
||||||
|
"id": 1,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "background",
|
||||||
|
"graphMode": "none",
|
||||||
|
"justifyMode": "center",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"textMode": "value"
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "up{instance=\"flyio-proxy\"}", "refId": "A" }
|
||||||
|
],
|
||||||
|
"title": "Alloy Health",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "thresholds" },
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||||
|
"unit": "reqps"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 6, "w": 6, "x": 6, "y": 0 },
|
||||||
|
"id": 2,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "area",
|
||||||
|
"justifyMode": "center",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_http_requests_total{instance=\"flyio-proxy\"}[5m]))", "refId": "A" }
|
||||||
|
],
|
||||||
|
"title": "Total RPS",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "thresholds" },
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.05 }] },
|
||||||
|
"unit": "percentunit"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 6, "w": 6, "x": 12, "y": 0 },
|
||||||
|
"id": 3,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "background",
|
||||||
|
"graphMode": "area",
|
||||||
|
"justifyMode": "center",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_http_requests_total{instance=\"flyio-proxy\",status=~\"5..\"}[5m])) / sum(rate(flyio_nginx_http_requests_total{instance=\"flyio-proxy\"}[5m]))", "refId": "A" }
|
||||||
|
],
|
||||||
|
"title": "Error Rate (5xx)",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "thresholds" },
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 0.8 }] },
|
||||||
|
"unit": "percentunit"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 6, "w": 6, "x": 18, "y": 0 },
|
||||||
|
"id": 7,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "background",
|
||||||
|
"graphMode": "none",
|
||||||
|
"justifyMode": "center",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_cache_requests_total{instance=\"flyio-proxy\",cache_status=\"HIT\"}[5m])) / sum(rate(flyio_nginx_cache_requests_total{instance=\"flyio-proxy\"}[5m]))", "refId": "A" }
|
||||||
|
],
|
||||||
|
"title": "Cache Hit Ratio",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "palette-classic" },
|
||||||
|
"custom": {
|
||||||
|
"axisBorderShow": false,
|
||||||
|
"axisCenteredZero": false,
|
||||||
|
"axisColorMode": "text",
|
||||||
|
"axisLabel": "req/s",
|
||||||
|
"axisPlacement": "auto",
|
||||||
|
"barAlignment": 0,
|
||||||
|
"drawStyle": "line",
|
||||||
|
"fillOpacity": 20,
|
||||||
|
"gradientMode": "none",
|
||||||
|
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||||
|
"insertNulls": false,
|
||||||
|
"lineInterpolation": "linear",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"pointSize": 5,
|
||||||
|
"scaleDistribution": { "type": "linear" },
|
||||||
|
"showPoints": "never",
|
||||||
|
"spanNulls": false,
|
||||||
|
"stacking": { "group": "A", "mode": "normal" },
|
||||||
|
"thresholdsStyle": { "mode": "off" }
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||||
|
"unit": "reqps"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
|
||||||
|
"id": 4,
|
||||||
|
"options": {
|
||||||
|
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true },
|
||||||
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (host) (rate(flyio_nginx_http_requests_total{instance=\"flyio-proxy\"}[5m]))", "legendFormat": "{{host}}", "refId": "A" }
|
||||||
|
],
|
||||||
|
"title": "Total Request Rate by Host",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "palette-classic" },
|
||||||
|
"custom": {
|
||||||
|
"axisBorderShow": false,
|
||||||
|
"axisCenteredZero": false,
|
||||||
|
"axisColorMode": "text",
|
||||||
|
"axisLabel": "",
|
||||||
|
"axisPlacement": "auto",
|
||||||
|
"fillOpacity": 80,
|
||||||
|
"gradientMode": "none",
|
||||||
|
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||||
|
"lineWidth": 1,
|
||||||
|
"scaleDistribution": { "type": "linear" }
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||||
|
"unit": "short"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
|
||||||
|
"id": 5,
|
||||||
|
"options": {
|
||||||
|
"barRadius": 0.05,
|
||||||
|
"barWidth": 0.7,
|
||||||
|
"fullHighlight": false,
|
||||||
|
"groupWidth": 0.7,
|
||||||
|
"legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true },
|
||||||
|
"orientation": "horizontal",
|
||||||
|
"showValue": "always",
|
||||||
|
"stacking": "none",
|
||||||
|
"tooltip": { "mode": "single", "sort": "none" },
|
||||||
|
"xTickLabelRotation": 0
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (cache_status) (increase(flyio_nginx_cache_requests_total{instance=\"flyio-proxy\"}[$__range]))", "legendFormat": "{{cache_status}}", "refId": "A", "instant": true }
|
||||||
|
],
|
||||||
|
"title": "Cache Performance",
|
||||||
|
"type": "barchart"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "palette-classic" },
|
||||||
|
"custom": {
|
||||||
|
"axisBorderShow": false,
|
||||||
|
"axisCenteredZero": false,
|
||||||
|
"axisColorMode": "text",
|
||||||
|
"axisLabel": "seconds",
|
||||||
|
"axisPlacement": "auto",
|
||||||
|
"barAlignment": 0,
|
||||||
|
"drawStyle": "line",
|
||||||
|
"fillOpacity": 10,
|
||||||
|
"gradientMode": "none",
|
||||||
|
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||||
|
"insertNulls": false,
|
||||||
|
"lineInterpolation": "linear",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"pointSize": 5,
|
||||||
|
"scaleDistribution": { "type": "linear" },
|
||||||
|
"showPoints": "never",
|
||||||
|
"spanNulls": false,
|
||||||
|
"stacking": { "group": "A", "mode": "none" },
|
||||||
|
"thresholdsStyle": { "mode": "off" }
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||||
|
"unit": "s"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 },
|
||||||
|
"id": 6,
|
||||||
|
"options": {
|
||||||
|
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true },
|
||||||
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(flyio_nginx_http_request_duration_seconds_bucket{instance=\"flyio-proxy\"}[5m])))", "legendFormat": "p95", "refId": "A" }
|
||||||
|
],
|
||||||
|
"title": "Upstream Response Time p95",
|
||||||
|
"type": "timeseries"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"refresh": "30s",
|
||||||
|
"schemaVersion": 38,
|
||||||
|
"tags": ["flyio", "nginx", "proxy"],
|
||||||
|
"templating": { "list": [] },
|
||||||
|
"time": { "from": "now-6h", "to": "now" },
|
||||||
|
"timepicker": {},
|
||||||
|
"timezone": "",
|
||||||
|
"title": "Fly.io Proxy Health",
|
||||||
|
"uid": "flyio-proxy",
|
||||||
|
"version": 1,
|
||||||
|
"weekStart": ""
|
||||||
|
}
|
||||||
|
|
@ -17,6 +17,8 @@ resources:
|
||||||
- dashboards/configmap-postgresql.yaml
|
- dashboards/configmap-postgresql.yaml
|
||||||
- dashboards/configmap-services.yaml
|
- dashboards/configmap-services.yaml
|
||||||
- dashboards/configmap-zot.yaml
|
- dashboards/configmap-zot.yaml
|
||||||
|
- dashboards/configmap-docs-apm.yaml
|
||||||
|
- dashboards/configmap-flyio.yaml
|
||||||
# TeslaMate dashboards
|
# TeslaMate dashboards
|
||||||
- dashboards/configmap-teslamate-overview.yaml
|
- dashboards/configmap-teslamate-overview.yaml
|
||||||
- dashboards/configmap-teslamate-charges.yaml
|
- dashboards/configmap-teslamate-charges.yaml
|
||||||
|
|
|
||||||
1
docs/changelog.d/feature-flyio-observability.feature.md
Normal file
1
docs/changelog.d/feature-flyio-observability.feature.md
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
Add observability to Fly.io proxy: Alloy collects nginx access logs (→ Loki) and derived metrics (→ Prometheus), with Grafana dashboards for Docs APM and Fly.io proxy health.
|
||||||
|
|
@ -7,9 +7,10 @@ tags:
|
||||||
|
|
||||||
# Grafana Alloy
|
# Grafana Alloy
|
||||||
|
|
||||||
Unified observability collector for metrics and logs with two deployments:
|
Unified observability collector for metrics and logs with three deployments:
|
||||||
1. **Indri (host)** - System metrics and service logs from macOS host
|
1. **Indri (host)** - System metrics and service logs from macOS host
|
||||||
2. **Kubernetes (DaemonSet)** - Automatic pod log collection and service health probes
|
2. **Kubernetes (DaemonSet)** - Automatic pod log collection and service health probes
|
||||||
|
3. **Fly.io proxy (embedded)** - nginx access log metrics and log forwarding from [[flyio-proxy]]
|
||||||
|
|
||||||
## Quick Reference
|
## Quick Reference
|
||||||
|
|
||||||
|
|
@ -20,6 +21,8 @@ Unified observability collector for metrics and logs with two deployments:
|
||||||
| **K8s Namespace** | `alloy` |
|
| **K8s Namespace** | `alloy` |
|
||||||
| **K8s Image** | `grafana/alloy:v1.8.2` |
|
| **K8s Image** | `grafana/alloy:v1.8.2` |
|
||||||
| **ArgoCD App** | `alloy-k8s` |
|
| **ArgoCD App** | `alloy-k8s` |
|
||||||
|
| **Fly.io Config** | `fly/alloy.river` |
|
||||||
|
| **Fly.io Image** | `grafana/alloy:v1.5.1` (binary copied into nginx container) |
|
||||||
|
|
||||||
## Metrics Collected
|
## Metrics Collected
|
||||||
|
|
||||||
|
|
@ -33,6 +36,13 @@ Unified observability collector for metrics and logs with two deployments:
|
||||||
- All pod logs via `loki.source.kubernetes`
|
- All pod logs via `loki.source.kubernetes`
|
||||||
- Service health probes: miniflux, kiwix, transmission, devpi, argocd
|
- Service health probes: miniflux, kiwix, transmission, devpi, argocd
|
||||||
|
|
||||||
|
### From Fly.io Proxy
|
||||||
|
- `flyio_nginx_http_requests_total` — request rate by status/method/host
|
||||||
|
- `flyio_nginx_http_request_duration_seconds` — latency histogram
|
||||||
|
- `flyio_nginx_http_response_bytes_total` — response bandwidth
|
||||||
|
- `flyio_nginx_cache_requests_total` — cache HIT/MISS/EXPIRED counts
|
||||||
|
- Pushed to [[prometheus]] via remote_write through [[caddy]]
|
||||||
|
|
||||||
## Logs Collected
|
## Logs Collected
|
||||||
|
|
||||||
**Brew services:** forgejo, tailscale
|
**Brew services:** forgejo, tailscale
|
||||||
|
|
@ -41,6 +51,8 @@ Unified observability collector for metrics and logs with two deployments:
|
||||||
|
|
||||||
Logs pushed to [[loki]] at `https://loki.tail8d86e.ts.net/loki/api/v1/push`.
|
Logs pushed to [[loki]] at `https://loki.tail8d86e.ts.net/loki/api/v1/push`.
|
||||||
|
|
||||||
|
**Fly.io proxy:** nginx JSON access logs pushed to [[loki]] at `https://loki.ops.eblu.me/loki/api/v1/push` (via [[caddy]]).
|
||||||
|
|
||||||
## Why Built from Source
|
## Why Built from Source
|
||||||
|
|
||||||
The Homebrew bottle uses `CGO_ENABLED=0`, which breaks Tailscale MagicDNS. Building with `CGO_ENABLED=1` uses the macOS native resolver.
|
The Homebrew bottle uses `CGO_ENABLED=0`, which breaks Tailscale MagicDNS. Building with `CGO_ENABLED=1` uses the macOS native resolver.
|
||||||
|
|
|
||||||
|
|
@ -79,6 +79,10 @@ mise run provision-indri -- --tags caddy
|
||||||
|
|
||||||
The token is written to `~/.config/caddy/gandi-token` (chmod 0600) and sourced by the Caddy wrapper script.
|
The token is written to `~/.config/caddy/gandi-token` (chmod 0600) and sourced by the Caddy wrapper script.
|
||||||
|
|
||||||
|
## Security Considerations
|
||||||
|
|
||||||
|
Caddy has no authentication layer — it is a plain reverse proxy. Access control relies entirely on Tailscale ACLs restricting which devices can reach indri on port 443. Currently `tag:homelab`, `autogroup:admin`, and `tag:flyio-proxy` can reach Caddy. The [[flyio-proxy]] grant exists so Alloy can push metrics/logs to Loki and Prometheus, but it means the Fly.io container can technically reach all Caddy-proxied services. See [[flyio-proxy#Security Considerations]] for the threat model.
|
||||||
|
|
||||||
## Custom Build
|
## Custom Build
|
||||||
|
|
||||||
Caddy is built from source with the Gandi DNS plugin:
|
Caddy is built from source with the Gandi DNS plugin:
|
||||||
|
|
|
||||||
|
|
@ -35,9 +35,10 @@ Internet traffic hits Fly.io's Anycast edge, terminates TLS with a Let's Encrypt
|
||||||
| File | Purpose |
|
| File | Purpose |
|
||||||
|------|---------|
|
|------|---------|
|
||||||
| `fly/fly.toml` | App configuration |
|
| `fly/fly.toml` | App configuration |
|
||||||
| `fly/Dockerfile` | nginx + Tailscale container |
|
| `fly/Dockerfile` | nginx + Tailscale + Alloy container |
|
||||||
| `fly/nginx.conf` | Reverse proxy, caching, rate limiting |
|
| `fly/nginx.conf` | Reverse proxy, caching, rate limiting, JSON logging |
|
||||||
| `fly/start.sh` | Entrypoint: start Tailscale, then nginx |
|
| `fly/alloy.river` | Alloy config: log tailing, metric extraction, remote_write |
|
||||||
|
| `fly/start.sh` | Entrypoint: start Tailscale, Alloy, then nginx |
|
||||||
| `pulumi/tailscale/__main__.py` | Auth key (`tag:flyio-proxy`) |
|
| `pulumi/tailscale/__main__.py` | Auth key (`tag:flyio-proxy`) |
|
||||||
| `pulumi/tailscale/policy.hujson` | ACL grants for proxy |
|
| `pulumi/tailscale/policy.hujson` | ACL grants for proxy |
|
||||||
| `pulumi/gandi/__main__.py` | DNS CNAMEs |
|
| `pulumi/gandi/__main__.py` | DNS CNAMEs |
|
||||||
|
|
@ -48,6 +49,34 @@ Fly.io runs Firecracker microVMs which support TUN devices natively. Tailscale r
|
||||||
|
|
||||||
The Tailscale auth key is `preauthorized=True` to avoid device approval hangs on container restarts.
|
The Tailscale auth key is `preauthorized=True` to avoid device approval hangs on container restarts.
|
||||||
|
|
||||||
|
## Observability
|
||||||
|
|
||||||
|
[[alloy|Alloy]] runs inside the container alongside nginx and Tailscale, providing:
|
||||||
|
|
||||||
|
- **Logs**: nginx JSON access logs tailed and pushed to [[loki|Loki]] (`{instance="flyio-proxy", job="flyio-nginx"}`)
|
||||||
|
- **Metrics**: Derived from access logs, pushed to [[prometheus|Prometheus]] via `remote_write`
|
||||||
|
- `flyio_nginx_http_requests_total` — request rate by status/method/host
|
||||||
|
- `flyio_nginx_http_request_duration_seconds` — latency histogram
|
||||||
|
- `flyio_nginx_http_response_bytes_total` — response bandwidth
|
||||||
|
- `flyio_nginx_cache_requests_total` — cache HIT/MISS/EXPIRED counts
|
||||||
|
|
||||||
|
### Dashboards
|
||||||
|
|
||||||
|
| Dashboard | Purpose |
|
||||||
|
|-----------|---------|
|
||||||
|
| **Docs APM** | Per-service view for `docs.eblu.me`: request rate, latency percentiles, cache hit ratio, error rate, bandwidth, access logs |
|
||||||
|
| **Fly.io Proxy Health** | Aggregate proxy health: connections, total request rate by host, cache performance, upstream latency, Alloy health |
|
||||||
|
|
||||||
|
Alloy listens on `127.0.0.1:12345` for self-scraping its `/metrics` endpoint. All metrics carry `instance="flyio-proxy"`.
|
||||||
|
|
||||||
|
## Security Considerations
|
||||||
|
|
||||||
|
The `tag:flyio-proxy` ACL grants access to both `tag:k8s:443` (for proxying public services) and `tag:homelab:443` (for pushing metrics/logs to [[caddy|Caddy]]-proxied Loki and Prometheus). This means a compromised nginx config could route traffic to **any** Caddy-proxied service — not just the intended backends. Some of those services (Loki, Prometheus) have no auth; others ([[forgejo]], [[navidrome]], [[immich]]) do.
|
||||||
|
|
||||||
|
Exploitation requires either pushing a malicious image to Fly.io or modifying the nginx config — both of which require RCE on [[gilbert]] (where `fly` is authenticated) or access to [[1password]] (the deploy token). This is an acceptable boundary given that 1Password is already the trust root for the entire infrastructure.
|
||||||
|
|
||||||
|
If this surface area becomes a concern, an alternative would be to add dedicated Tailscale Ingress tags for Loki/Prometheus write endpoints and restrict `tag:flyio-proxy` to only those.
|
||||||
|
|
||||||
## Secrets
|
## Secrets
|
||||||
|
|
||||||
| Secret | Source | Description |
|
| Secret | Source | Description |
|
||||||
|
|
|
||||||
|
|
@ -41,6 +41,8 @@ Optional annotation: `grafana_folder: "FolderName"`
|
||||||
- Minikube - Kubernetes cluster overview
|
- Minikube - Kubernetes cluster overview
|
||||||
- Borgmatic Backups - Backup status and trends
|
- Borgmatic Backups - Backup status and trends
|
||||||
- Services Health - HTTP probe results
|
- Services Health - HTTP probe results
|
||||||
|
- Docs APM - Request rate, latency, cache for docs.eblu.me
|
||||||
|
- Fly.io Proxy Health - Aggregate proxy health across all upstream services
|
||||||
- TeslaMate (18 dashboards) - Vehicle data
|
- TeslaMate (18 dashboards) - Vehicle data
|
||||||
|
|
||||||
## Related
|
## Related
|
||||||
|
|
|
||||||
|
|
@ -36,12 +36,16 @@ Log aggregation system for BlumeOps infrastructure.
|
||||||
**From Kubernetes (via Alloy DaemonSet):**
|
**From Kubernetes (via Alloy DaemonSet):**
|
||||||
- All pods in all namespaces
|
- All pods in all namespaces
|
||||||
|
|
||||||
|
**From Fly.io proxy (via embedded Alloy):**
|
||||||
|
- nginx JSON access logs (`{instance="flyio-proxy", job="flyio-nginx"}`)
|
||||||
|
|
||||||
## Query Examples (LogQL)
|
## Query Examples (LogQL)
|
||||||
|
|
||||||
```logql
|
```logql
|
||||||
{service="forgejo"} # All forgejo logs
|
{service="forgejo"} # All forgejo logs
|
||||||
{service="borgmatic", stream="stderr"} # Borgmatic errors
|
{service="borgmatic", stream="stderr"} # Borgmatic errors
|
||||||
{host="indri"} |= "error" # All logs containing "error"
|
{host="indri"} |= "error" # All logs containing "error"
|
||||||
|
{instance="flyio-proxy"} |= "docs.eblu.me" # Fly.io proxy access logs for docs
|
||||||
```
|
```
|
||||||
|
|
||||||
## Related
|
## Related
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,7 @@ Metrics storage and querying for BlumeOps infrastructure.
|
||||||
### Remote Write (from Alloy)
|
### Remote Write (from Alloy)
|
||||||
- Indri system metrics via [[alloy|Alloy]] remote_write
|
- Indri system metrics via [[alloy|Alloy]] remote_write
|
||||||
- Textfile metrics: minikube, borgmatic, zot, jellyfin
|
- Textfile metrics: minikube, borgmatic, zot, jellyfin
|
||||||
|
- [[flyio-proxy]] nginx metrics (`flyio_nginx_*`) via Alloy embedded in Fly.io container
|
||||||
|
|
||||||
### Scrape Targets
|
### Scrape Targets
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,9 +7,17 @@ COPY --from=docker.io/tailscale/tailscale:stable \
|
||||||
/usr/local/bin/tailscale /usr/local/bin/tailscale
|
/usr/local/bin/tailscale /usr/local/bin/tailscale
|
||||||
|
|
||||||
RUN mkdir -p /var/run/tailscale /var/lib/tailscale \
|
RUN mkdir -p /var/run/tailscale /var/lib/tailscale \
|
||||||
&& apk add --no-cache iptables ip6tables
|
&& apk add --no-cache iptables ip6tables \
|
||||||
|
&& apk add --no-cache libc6-compat
|
||||||
|
|
||||||
|
# Copy Alloy binary from official image (Ubuntu-based, needs libc6-compat)
|
||||||
|
COPY --from=docker.io/grafana/alloy:v1.5.1 \
|
||||||
|
/bin/alloy /usr/local/bin/alloy
|
||||||
|
|
||||||
|
RUN mkdir -p /var/log/nginx /etc/alloy /tmp/alloy-data
|
||||||
|
|
||||||
COPY nginx.conf /etc/nginx/nginx.conf
|
COPY nginx.conf /etc/nginx/nginx.conf
|
||||||
|
COPY alloy.river /etc/alloy/config.alloy
|
||||||
COPY start.sh /start.sh
|
COPY start.sh /start.sh
|
||||||
RUN chmod +x /start.sh
|
RUN chmod +x /start.sh
|
||||||
|
|
||||||
|
|
|
||||||
142
fly/alloy.river
Normal file
142
fly/alloy.river
Normal file
|
|
@ -0,0 +1,142 @@
|
||||||
|
// Grafana Alloy configuration for flyio-proxy
|
||||||
|
// Collects nginx access logs → Loki, extracts metrics → Prometheus.
|
||||||
|
// Note: stub_status connection metrics are not collected — Alloy has no
|
||||||
|
// built-in nginx exporter. The log-derived metrics cover the key signals.
|
||||||
|
|
||||||
|
// ============== LOG COLLECTION ==============
|
||||||
|
|
||||||
|
// Tail the JSON access log written by nginx
|
||||||
|
local.file_match "nginx_access" {
|
||||||
|
path_targets = [
|
||||||
|
{__path__ = "/var/log/nginx/access.json.log", job = "flyio-nginx"},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
loki.source.file "nginx_access" {
|
||||||
|
targets = local.file_match.nginx_access.targets
|
||||||
|
forward_to = [loki.process.nginx.receiver]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse JSON fields, extract labels, derive metrics
|
||||||
|
loki.process "nginx" {
|
||||||
|
forward_to = [loki.relabel.instance.receiver]
|
||||||
|
|
||||||
|
// Parse the JSON log line
|
||||||
|
stage.json {
|
||||||
|
expressions = {
|
||||||
|
status = "status",
|
||||||
|
method = "request_method",
|
||||||
|
host = "http_host",
|
||||||
|
cache_status = "upstream_cache_status",
|
||||||
|
request_time = "request_time",
|
||||||
|
body_bytes_sent = "body_bytes_sent",
|
||||||
|
upstream_response_time = "upstream_response_time",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Promote to labels for filtering in Loki
|
||||||
|
stage.labels {
|
||||||
|
values = {
|
||||||
|
status = "",
|
||||||
|
method = "",
|
||||||
|
host = "",
|
||||||
|
cache_status = "",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Derived metrics (exposed on Alloy's /metrics endpoint) ---
|
||||||
|
|
||||||
|
stage.metrics {
|
||||||
|
metric.counter {
|
||||||
|
name = "flyio_nginx_http_requests_total"
|
||||||
|
description = "Total HTTP requests by status, method, and host."
|
||||||
|
match_all = true
|
||||||
|
action = "inc"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stage.metrics {
|
||||||
|
metric.histogram {
|
||||||
|
name = "flyio_nginx_http_request_duration_seconds"
|
||||||
|
description = "HTTP request latency in seconds."
|
||||||
|
source = "request_time"
|
||||||
|
buckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stage.metrics {
|
||||||
|
metric.counter {
|
||||||
|
name = "flyio_nginx_http_response_bytes_total"
|
||||||
|
description = "Total bytes sent in HTTP responses."
|
||||||
|
source = "body_bytes_sent"
|
||||||
|
action = "add"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stage.metrics {
|
||||||
|
metric.counter {
|
||||||
|
name = "flyio_nginx_cache_requests_total"
|
||||||
|
description = "Total cache lookups by cache status."
|
||||||
|
source = "cache_status"
|
||||||
|
match_all = true
|
||||||
|
action = "inc"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add instance label to logs
|
||||||
|
loki.relabel "instance" {
|
||||||
|
forward_to = [loki.write.loki.receiver]
|
||||||
|
|
||||||
|
rule {
|
||||||
|
target_label = "instance"
|
||||||
|
replacement = "flyio-proxy"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write logs to Loki via Caddy (valid TLS, no skip_verify needed)
|
||||||
|
loki.write "loki" {
|
||||||
|
endpoint {
|
||||||
|
url = "https://loki.ops.eblu.me/loki/api/v1/push"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============== METRICS PIPELINE ==============
|
||||||
|
|
||||||
|
// Self-scrape to collect the log-derived metrics from /metrics
|
||||||
|
prometheus.scrape "self" {
|
||||||
|
targets = [{"__address__" = "127.0.0.1:12345"}]
|
||||||
|
forward_to = [prometheus.relabel.instance.receiver]
|
||||||
|
scrape_interval = "15s"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Strip the "loki_process_custom_" prefix that Alloy adds to stage.metrics,
|
||||||
|
// then add instance label. This keeps dashboard queries clean.
|
||||||
|
prometheus.relabel "instance" {
|
||||||
|
forward_to = [prometheus.remote_write.prometheus.receiver]
|
||||||
|
|
||||||
|
rule {
|
||||||
|
source_labels = ["__name__"]
|
||||||
|
regex = "loki_process_custom_(.*)"
|
||||||
|
target_label = "__name__"
|
||||||
|
replacement = "$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Drop internal labels added by the loki pipeline
|
||||||
|
rule {
|
||||||
|
regex = "component_id|component_path|filename"
|
||||||
|
action = "labeldrop"
|
||||||
|
}
|
||||||
|
|
||||||
|
rule {
|
||||||
|
target_label = "instance"
|
||||||
|
replacement = "flyio-proxy"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Push metrics to Prometheus via Caddy (valid TLS, no skip_verify needed)
|
||||||
|
prometheus.remote_write "prometheus" {
|
||||||
|
endpoint {
|
||||||
|
url = "https://prometheus.ops.eblu.me/api/v1/write"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -8,6 +8,23 @@ http {
|
||||||
include /etc/nginx/mime.types;
|
include /etc/nginx/mime.types;
|
||||||
default_type application/octet-stream;
|
default_type application/octet-stream;
|
||||||
|
|
||||||
|
# JSON access log for Alloy to tail → Loki + metric extraction
|
||||||
|
log_format json_log escape=json
|
||||||
|
'{'
|
||||||
|
'"time":"$time_iso8601",'
|
||||||
|
'"remote_addr":"$remote_addr",'
|
||||||
|
'"request_method":"$request_method",'
|
||||||
|
'"request_uri":"$request_uri",'
|
||||||
|
'"status":$status,'
|
||||||
|
'"body_bytes_sent":$body_bytes_sent,'
|
||||||
|
'"request_time":$request_time,'
|
||||||
|
'"upstream_response_time":"$upstream_response_time",'
|
||||||
|
'"upstream_cache_status":"$upstream_cache_status",'
|
||||||
|
'"http_host":"$http_host",'
|
||||||
|
'"http_user_agent":"$http_user_agent"'
|
||||||
|
'}';
|
||||||
|
access_log /var/log/nginx/access.json.log json_log;
|
||||||
|
|
||||||
# Rate limiting zones — define per-service zones as needed
|
# Rate limiting zones — define per-service zones as needed
|
||||||
limit_req_zone $binary_remote_addr zone=general:10m rate=10r/s;
|
limit_req_zone $binary_remote_addr zone=general:10m rate=10r/s;
|
||||||
|
|
||||||
|
|
@ -54,6 +71,12 @@ http {
|
||||||
return 200 "ok\n";
|
return 200 "ok\n";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
location /stub_status {
|
||||||
|
stub_status;
|
||||||
|
allow 127.0.0.1;
|
||||||
|
deny all;
|
||||||
|
}
|
||||||
|
|
||||||
location / {
|
location / {
|
||||||
return 444;
|
return 444;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -13,5 +13,11 @@ tailscale up --authkey="${TS_AUTHKEY}" --hostname=flyio-proxy
|
||||||
until tailscale status > /dev/null 2>&1; do sleep 1; done
|
until tailscale status > /dev/null 2>&1; do sleep 1; done
|
||||||
echo "Tailscale connected"
|
echo "Tailscale connected"
|
||||||
|
|
||||||
|
# Start Alloy for observability (logs → Loki, metrics → Prometheus)
|
||||||
|
alloy run /etc/alloy/config.alloy \
|
||||||
|
--server.http.listen-addr=127.0.0.1:12345 \
|
||||||
|
--storage.path=/tmp/alloy-data &
|
||||||
|
echo "Alloy started"
|
||||||
|
|
||||||
# Start nginx — MagicDNS resolves *.tail8d86e.ts.net hostnames
|
# Start nginx — MagicDNS resolves *.tail8d86e.ts.net hostnames
|
||||||
nginx -g "daemon off;"
|
nginx -g "daemon off;"
|
||||||
|
|
|
||||||
|
|
@ -61,10 +61,10 @@
|
||||||
},
|
},
|
||||||
|
|
||||||
// --- Fly.io proxy ---
|
// --- Fly.io proxy ---
|
||||||
// Public reverse proxy can reach k8s services on HTTPS only
|
// Public reverse proxy can reach k8s services and Caddy on HTTPS
|
||||||
{
|
{
|
||||||
"src": ["tag:flyio-proxy"],
|
"src": ["tag:flyio-proxy"],
|
||||||
"dst": ["tag:k8s"],
|
"dst": ["tag:k8s", "tag:homelab"],
|
||||||
"ip": ["tcp:443"],
|
"ip": ["tcp:443"],
|
||||||
},
|
},
|
||||||
|
|
||||||
|
|
@ -175,10 +175,10 @@
|
||||||
"src": "tag:ci-gateway",
|
"src": "tag:ci-gateway",
|
||||||
"accept": ["tag:registry:443"],
|
"accept": ["tag:registry:443"],
|
||||||
},
|
},
|
||||||
// Fly.io proxy can reach k8s services (HTTPS only), nothing else
|
// Fly.io proxy can reach k8s and Caddy on indri (HTTPS only), nothing else
|
||||||
{
|
{
|
||||||
"src": "tag:flyio-proxy",
|
"src": "tag:flyio-proxy",
|
||||||
"accept": ["tag:k8s:443"],
|
"accept": ["tag:k8s:443", "tag:homelab:443"],
|
||||||
"deny": ["tag:homelab:22", "tag:nas:445", "tag:registry:443"],
|
"deny": ["tag:homelab:22", "tag:nas:445", "tag:registry:443"],
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue