eblume/blumeops
1
1
Fork 0
Code Issues Pull requests Projects Releases 83 Packages Wiki Activity Actions

Route Fly proxy through Caddy on indri for direct WireGuard peering
All checks were successful
Deploy Fly.io Proxy / deploy (push) Successful in 1m59s
Details

Browse source 
Tailscale Ingress pods in k8s can't establish direct WireGuard
connections (stuck behind pod-network NAT → DERP relay → 20s latency).
Indri's host-level Tailscale CAN peer directly with Fly.

Change all nginx upstreams to route through Caddy on indri instead of
per-service Tailscale Ingress endpoints. Tag indri as flyio-target in
the Tailscale ACL so the Fly proxy can reach it.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Erich Blume 2026-04-18 09:40:20 -07:00
commit 12b2786ca2
3 changed files with 30 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

57
fly/nginx.conf
View file

@ -54,24 +54,17 @@ http {
websocket upgrade; websocket upgrade;
} }
# --- Upstream pools with keepalive --- # --- Upstream ---
# DNS is resolved once at config load via MagicDNS. If Tailscale Ingress # DNS resolved via Tailscale MagicDNS at config load.
# pods get new IPs (restart, reschedule), run `mise run fly-reload` to
# re-resolve. A Grafana alert fires when upstreams are unreachable.
resolver 100.100.100.100 valid=30s; resolver 100.100.100.100 valid=30s;
resolver_timeout 5s; resolver_timeout 5s;
upstream forge_backend { # All services route through Caddy on indri. Indri's host-level Tailscale
server forge.tail8d86e.ts.net:443; # can establish direct WireGuard peering, avoiding the DERP relay
keepalive 8; # bottleneck that k8s-hosted Tailscale Ingress pods cannot escape.
} upstream indri_backend {
upstream docs_backend { server indri.tail8d86e.ts.net:443;
server docs.tail8d86e.ts.net:443; keepalive 16;
keepalive 4;
}
upstream cv_backend {
server cv.tail8d86e.ts.net:443;
keepalive 4;
} }
# --- docs.eblu.me (static site) --- # --- docs.eblu.me (static site) ---
@ -90,11 +83,11 @@ http {
internal; internal;
} }
location / { location / {
proxy_pass https://docs_backend$request_uri; proxy_pass https://indri_backend$request_uri;
proxy_ssl_verify off; proxy_ssl_verify off;
proxy_ssl_server_name on; proxy_ssl_server_name on;
proxy_ssl_name docs.tail8d86e.ts.net; proxy_ssl_name docs.ops.eblu.me;
proxy_set_header Host docs.tail8d86e.ts.net; proxy_set_header Host docs.ops.eblu.me;
proxy_intercept_errors on; proxy_intercept_errors on;
proxy_http_version 1.1; proxy_http_version 1.1;
@ -134,11 +127,11 @@ http {
} }
location / { location / {
proxy_pass https://cv_backend$request_uri; proxy_pass https://indri_backend$request_uri;
proxy_ssl_verify off; proxy_ssl_verify off;
proxy_ssl_server_name on; proxy_ssl_server_name on;
proxy_ssl_name cv.tail8d86e.ts.net; proxy_ssl_name cv.ops.eblu.me;
proxy_set_header Host cv.tail8d86e.ts.net; proxy_set_header Host cv.ops.eblu.me;
proxy_intercept_errors on; proxy_intercept_errors on;
proxy_http_version 1.1; proxy_http_version 1.1;
@ -209,13 +202,13 @@ http {
location ~ ^/user/(login|sign_up|forgot_password) { location ~ ^/user/(login|sign_up|forgot_password) {
limit_req zone=forge_auth burst=5 nodelay; limit_req zone=forge_auth burst=5 nodelay;
proxy_pass https://forge_backend$request_uri; proxy_pass https://indri_backend$request_uri;
proxy_ssl_verify off; proxy_ssl_verify off;
proxy_ssl_server_name on; proxy_ssl_server_name on;
proxy_ssl_name forge.tail8d86e.ts.net; proxy_ssl_name forge.ops.eblu.me;
proxy_intercept_errors on; proxy_intercept_errors on;
proxy_set_header Host $host; proxy_set_header Host forge.ops.eblu.me;
proxy_set_header X-Real-IP $http_fly_client_ip; proxy_set_header X-Real-IP $http_fly_client_ip;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Proto $scheme;
@ -228,10 +221,10 @@ http {
# Cache release artifact downloads — immutable files keyed by tag+filename. # Cache release artifact downloads — immutable files keyed by tag+filename.
# Avoids hammering Forgejo when crawlers or users re-download the same asset. # Avoids hammering Forgejo when crawlers or users re-download the same asset.
location ~ ^/[^/]+/[^/]+/releases/download/ { location ~ ^/[^/]+/[^/]+/releases/download/ {
proxy_pass https://forge_backend$request_uri; proxy_pass https://indri_backend$request_uri;
proxy_ssl_verify off; proxy_ssl_verify off;
proxy_ssl_server_name on; proxy_ssl_server_name on;
proxy_ssl_name forge.tail8d86e.ts.net; proxy_ssl_name forge.ops.eblu.me;
proxy_http_version 1.1; proxy_http_version 1.1;
proxy_set_header Connection $connection_upgrade; proxy_set_header Connection $connection_upgrade;
@ -240,7 +233,7 @@ http {
proxy_cache_valid 200 7d; proxy_cache_valid 200 7d;
proxy_cache_key $host$uri; proxy_cache_key $host$uri;
proxy_set_header Host $host; proxy_set_header Host forge.ops.eblu.me;
proxy_set_header X-Real-IP $http_fly_client_ip; proxy_set_header X-Real-IP $http_fly_client_ip;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Proto $scheme;
@ -251,10 +244,10 @@ http {
# Selectively cache static assets only # Selectively cache static assets only
location ~* \.(css|js|png|jpg|svg|woff2?)$ { location ~* \.(css|js|png|jpg|svg|woff2?)$ {
proxy_pass https://forge_backend$request_uri; proxy_pass https://indri_backend$request_uri;
proxy_ssl_verify off; proxy_ssl_verify off;
proxy_ssl_server_name on; proxy_ssl_server_name on;
proxy_ssl_name forge.tail8d86e.ts.net; proxy_ssl_name forge.ops.eblu.me;
proxy_http_version 1.1; proxy_http_version 1.1;
proxy_set_header Connection $connection_upgrade; proxy_set_header Connection $connection_upgrade;
@ -268,15 +261,15 @@ http {
} }
location / { location / {
proxy_pass https://forge_backend$request_uri; proxy_pass https://indri_backend$request_uri;
proxy_ssl_verify off; proxy_ssl_verify off;
proxy_ssl_server_name on; proxy_ssl_server_name on;
proxy_ssl_name forge.tail8d86e.ts.net; proxy_ssl_name forge.ops.eblu.me;
proxy_intercept_errors on; proxy_intercept_errors on;
# NO proxy_cache — dynamic content with sessions # NO proxy_cache — dynamic content with sessions
proxy_set_header Host $host; proxy_set_header Host forge.ops.eblu.me;
proxy_set_header X-Real-IP $http_fly_client_ip; proxy_set_header X-Real-IP $http_fly_client_ip;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Forwarded-Proto $scheme;

1
pulumi/tailscale/__main__.py
View file

@ -50,6 +50,7 @@ indri_tags = tailscale.DeviceTags(
"tag:loki", "tag:loki",
"tag:registry", # Zot container registry "tag:registry", # Zot container registry
"tag:k8s-api", # Kubernetes API server (minikube) "tag:k8s-api", # Kubernetes API server (minikube)
"tag:flyio-target", # Fly proxy routes through Caddy on indri
], ],
) )

6
pulumi/tailscale/policy.hujson
View file

@ -193,11 +193,13 @@
"src": "tag:ci-gateway", "src": "tag:ci-gateway",
"accept": ["tag:registry:443"], "accept": ["tag:registry:443"],
}, },
// Fly.io proxy can only reach flyio-target tagged endpoints, nothing else // Fly.io proxy can only reach flyio-target tagged endpoints, nothing else.
// indri has tag:flyio-target (Caddy) so tag:homelab:443 is reachable on
// indri specifically but not other homelab devices.
{ {
"src": "tag:flyio-proxy", "src": "tag:flyio-proxy",
"accept": ["tag:flyio-target:443"], "accept": ["tag:flyio-target:443"],
"deny": ["tag:k8s:443", "tag:homelab:443", "tag:homelab:22", "tag:nas:445", "tag:registry:443"], "deny": ["tag:k8s:443", "tag:homelab:22", "tag:nas:445", "tag:registry:443"],
}, },
], ],
} }