From 3415cad38c6c2977fe1e1ceb81e2f1fbb1b95eeb Mon Sep 17 00:00:00 2001
From: Erich Blume <blume.erich@gmail.com>
Date: Mon, 9 Feb 2026 11:02:06 -0800
Subject: [PATCH] Log real client IPs via Fly-Client-IP header (#130)

## Summary
- Add `client_ip` field to the Fly.io nginx JSON log format, sourced from `Fly-Client-IP` header
- Extract `client_ip` in the Alloy pipeline so it's available as a parsed field in Loki
- Keeps `remote_addr` (the internal proxy IP) for debugging

Fixes: Grafana access logs for docs.eblu.me showing 172.16.11.178 for every request instead of real visitor IPs.

## Deployment and Testing
- [ ] Deploy updated fly.io proxy: `fly deploy` from `fly/` directory
- [ ] Verify in Grafana that new log lines include `client_ip` with real IPs
- [ ] Confirm `remote_addr` still shows the proxy IP (preserved for debugging)

Reviewed-on: https://forge.ops.eblu.me/eblume/blumeops/pulls/130
---
 .../manifests/grafana-config/dashboards/configmap-docs-apm.yaml | 2 +-
 docs/changelog.d/fix-real-client-ip-logging.bugfix.md           | 1 +
 fly/alloy.river                                                 | 1 +
 fly/nginx.conf                                                  | 1 +
 4 files changed, 4 insertions(+), 1 deletion(-)
 create mode 100644 docs/changelog.d/fix-real-client-ip-logging.bugfix.md

diff --git a/argocd/manifests/grafana-config/dashboards/configmap-docs-apm.yaml b/argocd/manifests/grafana-config/dashboards/configmap-docs-apm.yaml
index 44dd184..8a7c60c 100644
--- a/argocd/manifests/grafana-config/dashboards/configmap-docs-apm.yaml
+++ b/argocd/manifests/grafana-config/dashboards/configmap-docs-apm.yaml
@@ -242,7 +242,7 @@ data:
             "wrapLogMessage": false
           },
           "targets": [
-            { "datasource": { "type": "loki", "uid": "loki" }, "expr": "{instance=\"flyio-proxy\", job=\"flyio-nginx\"} |= \"docs.eblu.me\"", "refId": "A" }
+            { "datasource": { "type": "loki", "uid": "loki" }, "expr": "{instance=\"flyio-proxy\", job=\"flyio-nginx\"} |= \"docs.eblu.me\" | json | line_format \"{{.client_ip}} {{.request_method}} {{.request_uri}} {{.status}} cache={{.upstream_cache_status}} {{.request_time}}s\"", "refId": "A" }
           ],
           "title": "Recent Access Logs",
           "type": "logs"
diff --git a/docs/changelog.d/fix-real-client-ip-logging.bugfix.md b/docs/changelog.d/fix-real-client-ip-logging.bugfix.md
new file mode 100644
index 0000000..466a789
--- /dev/null
+++ b/docs/changelog.d/fix-real-client-ip-logging.bugfix.md
@@ -0,0 +1 @@
+Log real client IPs in Fly.io proxy access logs using Fly-Client-IP header instead of showing the internal proxy address.
diff --git a/fly/alloy.river b/fly/alloy.river
index 213d8c5..36417d4 100644
--- a/fly/alloy.river
+++ b/fly/alloy.river
@@ -24,6 +24,7 @@ loki.process "nginx" {
   // Parse the JSON log line
   stage.json {
     expressions = {
+      client_ip            = "client_ip",
       status               = "status",
       method               = "request_method",
       host                 = "http_host",
diff --git a/fly/nginx.conf b/fly/nginx.conf
index 135183e..f01a1a3 100644
--- a/fly/nginx.conf
+++ b/fly/nginx.conf
@@ -13,6 +13,7 @@ http {
         '{'
             '"time":"$time_iso8601",'
             '"remote_addr":"$remote_addr",'
+            '"client_ip":"$http_fly_client_ip",'
             '"request_method":"$request_method",'
             '"request_uri":"$request_uri",'
             '"status":$status,'