From f13bd7fca6fe58b409f3138a4395bd5a0ab0ef13 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Mon, 9 Feb 2026 10:59:23 -0800 Subject: [PATCH 1/2] Log real client IPs via Fly-Client-IP header in proxy access logs The Fly.io edge proxy forwards the original client IP in the Fly-Client-IP header. Add this as a `client_ip` field in the nginx JSON log format and extract it in the Alloy pipeline so Loki/Grafana show real visitor IPs instead of the internal proxy address (172.16.11.178). Co-Authored-By: Claude Opus 4.6 --- docs/changelog.d/fix-real-client-ip-logging.bugfix.md | 1 + fly/alloy.river | 1 + fly/nginx.conf | 1 + 3 files changed, 3 insertions(+) create mode 100644 docs/changelog.d/fix-real-client-ip-logging.bugfix.md diff --git a/docs/changelog.d/fix-real-client-ip-logging.bugfix.md b/docs/changelog.d/fix-real-client-ip-logging.bugfix.md new file mode 100644 index 0000000..466a789 --- /dev/null +++ b/docs/changelog.d/fix-real-client-ip-logging.bugfix.md @@ -0,0 +1 @@ +Log real client IPs in Fly.io proxy access logs using Fly-Client-IP header instead of showing the internal proxy address. diff --git a/fly/alloy.river b/fly/alloy.river index 213d8c5..36417d4 100644 --- a/fly/alloy.river +++ b/fly/alloy.river @@ -24,6 +24,7 @@ loki.process "nginx" { // Parse the JSON log line stage.json { expressions = { + client_ip = "client_ip", status = "status", method = "request_method", host = "http_host", diff --git a/fly/nginx.conf b/fly/nginx.conf index 135183e..f01a1a3 100644 --- a/fly/nginx.conf +++ b/fly/nginx.conf @@ -13,6 +13,7 @@ http { '{' '"time":"$time_iso8601",' '"remote_addr":"$remote_addr",' + '"client_ip":"$http_fly_client_ip",' '"request_method":"$request_method",' '"request_uri":"$request_uri",' '"status":$status,' -- 2.50.1 (Apple Git-155) From 38500704a625ab37bcb84ae30aecf19b69162369 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Mon, 9 Feb 2026 11:00:58 -0800 Subject: [PATCH 2/2] Format access log panel to show real client IPs Update the Docs APM dashboard's log panel to parse JSON and display clean formatted lines with client_ip instead of raw JSON blobs. Co-Authored-By: Claude Opus 4.6 --- .../manifests/grafana-config/dashboards/configmap-docs-apm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argocd/manifests/grafana-config/dashboards/configmap-docs-apm.yaml b/argocd/manifests/grafana-config/dashboards/configmap-docs-apm.yaml index 44dd184..8a7c60c 100644 --- a/argocd/manifests/grafana-config/dashboards/configmap-docs-apm.yaml +++ b/argocd/manifests/grafana-config/dashboards/configmap-docs-apm.yaml @@ -242,7 +242,7 @@ data: "wrapLogMessage": false }, "targets": [ - { "datasource": { "type": "loki", "uid": "loki" }, "expr": "{instance=\"flyio-proxy\", job=\"flyio-nginx\"} |= \"docs.eblu.me\"", "refId": "A" } + { "datasource": { "type": "loki", "uid": "loki" }, "expr": "{instance=\"flyio-proxy\", job=\"flyio-nginx\"} |= \"docs.eblu.me\" | json | line_format \"{{.client_ip}} {{.request_method}} {{.request_uri}} {{.status}} cache={{.upstream_cache_status}} {{.request_time}}s\"", "refId": "A" } ], "title": "Recent Access Logs", "type": "logs" -- 2.50.1 (Apple Git-155)