Expose Forgejo publicly at forge.eblu.me (#278)

## Summary

Expose Forgejo publicly at `forge.eblu.me` via the Fly.io reverse proxy — the first dynamic, authenticated public-facing service.

- **Forgejo hardening:** Domain changed to forge.eblu.me, SSH stays on forge.ops.eblu.me, reverse proxy trust headers configured, local registration locked to external-only (Authentik SSO)
- **Tailscale Ingress:** ExternalName Service + Ingress in tailscale-operator creates forge.tail8d86e.ts.net endpoint
- **Fly.io proxy:** nginx server block with rate-limited auth endpoints (3r/s), fail2ban with custom nginx-deny action, security headers, /swagger blocked, WebSocket support, 512m body limit
- **Authentik:** OAuth callback updated to forge.eblu.me
- **DNS/TLS:** CNAME record in Pulumi, cert in fly-setup
- **Rename:** ~29 files updated from forge.ops.eblu.me to forge.eblu.me (HTTPS refs only; SSH, container builds, and Caddy table kept as-is)

## Deployment Order

1. `mise run provision-indri -- --tags forgejo` (config changes)
2. Verify forge.ops.eblu.me still works
3. `argocd app set tailscale-operator --revision feature/forge-public && argocd app sync tailscale-operator`
4. Verify `curl https://forge.tail8d86e.ts.net`
5. `cd fly && fly deploy`
6. Verify pre-DNS: `curl -H "Host: forge.eblu.me" https://blumeops-proxy.fly.dev/`
7. `fly certs add forge.eblu.me -a blumeops-proxy`
8. `argocd app set authentik --revision feature/forge-public && argocd app sync authentik`
9. `mise run dns-preview && mise run dns-up`
10. Full verification (see below)
11. Rehearse `mise run fly-shutoff`
12. After merge: reset ArgoCD revisions to main, re-sync

## Verification Checklist

- [ ] forge.eblu.me loads, shows public repos
- [ ] forge.ops.eblu.me still works from tailnet
- [ ] SSH clone via forge.ops.eblu.me:2222 works
- [ ] HTTPS clone via forge.eblu.me works
- [ ] UI shows forge.eblu.me for HTTPS clone, forge.ops.eblu.me for SSH
- [ ] /swagger returns 403
- [ ] Rapid login attempts trigger 429 rate limit
- [ ] fail2ban bans after 5 failed logins in 10 minutes
- [ ] ArgoCD can still sync (SSH unaffected)
- [ ] `mise run fly-shutoff` stops all public traffic
- [ ] `mise run services-check` passes

Reviewed-on: #278

fly/Dockerfile

@@ -8,7 +8,9 @@ COPY --from=docker.io/tailscale/tailscale:stable \
 
 RUN mkdir -p /var/run/tailscale /var/lib/tailscale \
     && apk add --no-cache iptables ip6tables \
-    && apk add --no-cache libc6-compat
+    && apk add --no-cache libc6-compat \
+    && apk add --no-cache fail2ban \
+    && rm -f /etc/fail2ban/jail.d/alpine-ssh.conf
 
 # Copy Alloy binary from official image (Ubuntu-based, needs libc6-compat)
 COPY --from=docker.io/grafana/alloy:v1.13.1 \
@@ -16,6 +18,10 @@ COPY --from=docker.io/grafana/alloy:v1.13.1 \
 
 RUN mkdir -p /var/log/nginx /etc/alloy /tmp/alloy-data
 
+COPY fail2ban/filter.d/forge-login.conf /etc/fail2ban/filter.d/forge-login.conf
+COPY fail2ban/jail.d/forge.conf /etc/fail2ban/jail.d/forge.conf
+COPY fail2ban/action.d/nginx-deny.conf /etc/fail2ban/action.d/nginx-deny.conf
+
 COPY nginx.conf /etc/nginx/nginx.conf
 COPY error.html /usr/share/nginx/html/error.html
 COPY alloy.river /etc/alloy/config.alloy

fly/fail2ban/action.d/nginx-deny.conf

@@ -0,0 +1,14 @@
+# Custom fail2ban action that bans IPs via an nginx deny list.
+# Standard iptables banning won't work in Fly.io because $remote_addr
+# is Fly's internal proxy IP. Instead, we write banned IPs to a file
+# that nginx checks via a geo directive keyed on $http_fly_client_ip.
+
+[Definition]
+
+actionban = echo "<ip> 1;" >> /etc/nginx/forge-deny.conf && nginx -s reload
+
+actionunban = sed -i '/<ip> 1;/d' /etc/nginx/forge-deny.conf && nginx -s reload
+
+actionstart =
+actionstop =
+actioncheck =

fly/fail2ban/filter.d/forge-login.conf

@@ -0,0 +1,10 @@
+# Filter for Forgejo login failures via nginx JSON access log.
+# Matches 401/403 responses to authentication endpoints, keyed on
+# the client_ip field (populated from Fly-Client-IP header).
+
+[Definition]
+
+# Match JSON log lines with 401 or 403 status on login-related paths
+failregex = "client_ip":"<HOST>".*"request_uri":"\/user\/(login|sign_up|forgot_password)[^"]*".*"status":(401|403)
+
+ignoreregex =

fly/fail2ban/jail.d/forge.conf

@@ -0,0 +1,8 @@
+[forge-login]
+enabled  = true
+filter   = forge-login
+logpath  = /var/log/nginx/access.json.log
+maxretry = 5
+findtime = 600
+bantime  = 3600
+banaction = nginx-deny

fly/nginx.conf

@@ -29,6 +29,19 @@ http {
     # Rate limiting zones — define per-service zones as needed
     limit_req_zone $binary_remote_addr zone=general:10m rate=10r/s;
 
+    # Forge-specific rate limit keyed on real client IP (Fly-Client-IP header).
+    # $binary_remote_addr is Fly's internal proxy IP — all clients share one
+    # bucket. $http_fly_client_ip has the actual client IP.
+    limit_req_zone $http_fly_client_ip zone=forge_auth:10m rate=3r/s;
+
+    # fail2ban deny list — banned IPs are written here by fail2ban and
+    # checked via the $forge_banned variable. The file is touched at
+    # container start to ensure it exists.
+    geo $http_fly_client_ip $forge_banned {
+        default 0;
+        include /etc/nginx/forge-deny.conf;
+    }
+
     # Proxy cache: 200MB, evict after 24h of no access
     proxy_cache_path /tmp/cache levels=1:2 keys_zone=services:10m
                      max_size=200m inactive=24h;
@@ -114,6 +127,97 @@ http {
         }
     }
 
+    # --- forge.eblu.me (dynamic, authenticated) ---
+    server {
+        listen 8080;
+        server_name forge.eblu.me;
+
+        # Block fail2ban-banned IPs
+        if ($forge_banned) {
+            return 403 "Temporarily blocked. Try again later.\n";
+        }
+
+        # General rate limit — higher burst for git operations and CI webhooks
+        limit_req zone=general burst=50 nodelay;
+
+        # Git LFS and repo uploads can be large
+        client_max_body_size 512m;
+
+        # Security headers
+        add_header X-Frame-Options "DENY" always;
+        add_header X-Content-Type-Options "nosniff" always;
+        add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
+        add_header Referrer-Policy "strict-origin-when-cross-origin" always;
+
+        error_page 502 503 504 /error.html;
+        location = /error.html {
+            root /usr/share/nginx/html;
+            internal;
+        }
+
+        # Block swagger API docs — use forge.ops.eblu.me from tailnet
+        location /swagger {
+            return 403 "API documentation is only available at forge.ops.eblu.me (tailnet).\n";
+        }
+
+        # Rate-limit authentication endpoints
+        location ~ ^/user/(login|sign_up|forgot_password) {
+            limit_req zone=forge_auth burst=5 nodelay;
+
+            set $upstream_forge https://forge.tail8d86e.ts.net;
+            proxy_pass $upstream_forge$request_uri;
+            proxy_ssl_verify off;
+            proxy_ssl_server_name on;
+            proxy_intercept_errors on;
+
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $http_fly_client_ip;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+        }
+
+        # Selectively cache static assets only
+        location ~* \.(css|js|png|jpg|svg|woff2?)$ {
+            set $upstream_forge_static https://forge.tail8d86e.ts.net;
+            proxy_pass $upstream_forge_static$request_uri;
+            proxy_ssl_verify off;
+            proxy_ssl_server_name on;
+
+            proxy_cache services;
+            proxy_cache_valid 200 7d;
+            proxy_cache_key $host$uri;
+
+            add_header X-Cache-Status $upstream_cache_status;
+            add_header X-Clacks-Overhead "GNU Terry Pratchett" always;
+        }
+
+        location / {
+            set $upstream_forge https://forge.tail8d86e.ts.net;
+            proxy_pass $upstream_forge$request_uri;
+            proxy_ssl_verify off;
+            proxy_ssl_server_name on;
+            proxy_intercept_errors on;
+
+            # NO proxy_cache — dynamic content with sessions
+
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $http_fly_client_ip;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            # WebSocket support (Forgejo uses it for live updates)
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+
+            add_header X-Clacks-Overhead "GNU Terry Pratchett" always;
+        }
+    }
+
     # Catch-all: reject unknown hosts, but serve health check
     server {
         listen 8080 default_server;

fly/start.sh

@@ -12,11 +12,22 @@ tailscale up --authkey="${TS_AUTHKEY}" --hostname=flyio-proxy
 until tailscale status > /dev/null 2>&1; do sleep 1; done
 echo "Tailscale connected"
 
+# Ensure fail2ban deny file exists before nginx starts
+touch /etc/nginx/forge-deny.conf
+
 # Start nginx — MagicDNS is available, health check passes immediately.
 nginx -g "daemon off;" &
 NGINX_PID=$!
 echo "Nginx started"
 
+# Start fail2ban for login brute-force protection.
+# Non-fatal — nginx rate limiting is the primary defense; fail2ban is additive.
+if fail2ban-server -b; then
+    echo "fail2ban started"
+else
+    echo "WARNING: fail2ban failed to start (nginx rate limiting still active)"
+fi
+
 # Start Alloy for observability (logs → Loki, metrics → Prometheus)
 alloy run /etc/alloy/config.alloy \
     --server.http.listen-addr=127.0.0.1:12345 \