blumeops/fly/nginx.conf

worker_processes auto;

events {
    worker_connections 1024;
}

http {
    include /etc/nginx/mime.types;
    default_type application/octet-stream;

    # JSON access log for Alloy to tail → Loki + metric extraction
    log_format json_log escape=json
        '{'
            '"time":"$time_iso8601",'
            '"remote_addr":"$remote_addr",'
            '"client_ip":"$http_fly_client_ip",'
            '"request_method":"$request_method",'
            '"request_uri":"$request_uri",'
            '"status":$status,'
            '"body_bytes_sent":$body_bytes_sent,'
            '"request_time":$request_time,'
            '"upstream_response_time":"$upstream_response_time",'
            '"upstream_cache_status":"$upstream_cache_status",'
            '"http_host":"$http_host",'
            '"http_user_agent":"$http_user_agent"'
        '}';
    access_log /var/log/nginx/access.json.log json_log;

    # Rate limiting zones — define per-service zones as needed
    limit_req_zone $http_fly_client_ip zone=general:10m rate=10r/s;

    # Forge-specific rate limit keyed on real client IP (Fly-Client-IP header).
    # $binary_remote_addr is Fly's internal proxy IP — all clients share one
    # bucket. $http_fly_client_ip has the actual client IP.
    limit_req_zone $http_fly_client_ip zone=forge_auth:10m rate=3r/s;

    # Shower-specific zone: loose enough that ~30 guests sharing a single
    # venue-wifi NAT'd public IP can all scan the QR and load the splash
    # (HTML + a handful of static asset hits each) without anyone tripping
    # the limit. 50r/s + burst=200 covers the simultaneous-load spike;
    # exploit scanners still trip it (e.g. the .env-sweeping bot we saw
    # fired ~30 req in 2s — that pattern stays caught). See the
    # shower.eblu.me server block for the matching `limit_req`.
    limit_req_zone $http_fly_client_ip zone=shower_general:10m rate=50r/s;

    # fail2ban deny list — banned IPs are written here by fail2ban and
    # checked via the $forge_banned variable. The file is touched at
    # container start to ensure it exists.
    geo $http_fly_client_ip $forge_banned {
        default 0;
        include /etc/nginx/forge-deny.conf;
    }

    # Proxy cache: 200MB, evict after 24h of no access
    proxy_cache_path /tmp/cache levels=1:2 keys_zone=services:10m
                     max_size=200m inactive=24h;

    # WebSocket-aware Connection header. Only send "upgrade" when the client
    # actually requests a protocol switch; otherwise empty string to preserve
    # upstream keepalive connections.
    map $http_upgrade $connection_upgrade {
        default "";
        websocket upgrade;
    }

    # --- Upstream ---
    # DNS resolved via Tailscale MagicDNS at config load.
    resolver 100.100.100.100 valid=30s;
    resolver_timeout 5s;

    # All services route through Caddy on indri. Indri's host-level Tailscale
    # can establish direct WireGuard peering, avoiding the DERP relay
    # bottleneck that k8s-hosted Tailscale Ingress pods cannot escape.
    upstream indri_backend {
        server indri.tail8d86e.ts.net:443;
        keepalive 16;
    }

    # --- docs.eblu.me (static site) ---
    server {
        listen 8080;
        server_name docs.eblu.me;

        limit_req zone=general burst=20 nodelay;

        # Serve a friendly error page when upstreams are unreachable
        # (indri offline, Tailscale tunnel down, emergency shutoff, etc.)
        # proxy_cache_use_stale still takes priority when cached content exists.
        error_page 502 503 504 /error.html;
        location = /error.html {
            root /usr/share/nginx/html;
            internal;
        }
        location / {
            proxy_pass https://indri_backend$request_uri;
            proxy_ssl_verify off;
            proxy_ssl_server_name on;
            proxy_ssl_name docs.ops.eblu.me;
            proxy_set_header Host docs.ops.eblu.me;
            proxy_intercept_errors on;

            proxy_http_version 1.1;
            proxy_set_header Connection $connection_upgrade;

            # Cache aggressively — static site only.
            # Do NOT use these settings for dynamic services.
            proxy_cache services;
            proxy_cache_valid 200 1d;
            proxy_cache_valid 404 1m;
            proxy_cache_use_stale error timeout updating;
            proxy_cache_lock on;

            # Prevent cache-busting: ignore query strings and
            # client cache-control headers.
            # Safe for static sites; breaks dynamic services.
            proxy_cache_key $host$uri;
            proxy_ignore_headers Cache-Control Set-Cookie;

            add_header X-Cache-Status $upstream_cache_status;
            add_header X-Clacks-Overhead "GNU Terry Pratchett" always;
        }

    }

    # --- cv.eblu.me (static site) ---
    server {
        listen 8080;
        server_name cv.eblu.me;

        limit_req zone=general burst=20 nodelay;

        error_page 502 503 504 /error.html;
        location = /error.html {
            root /usr/share/nginx/html;
            internal;
        }

        location / {
            proxy_pass https://indri_backend$request_uri;
            proxy_ssl_verify off;
            proxy_ssl_server_name on;
            proxy_ssl_name cv.ops.eblu.me;
            proxy_set_header Host cv.ops.eblu.me;
            proxy_intercept_errors on;

            proxy_http_version 1.1;
            proxy_set_header Connection $connection_upgrade;

            proxy_cache services;
            proxy_cache_valid 200 1d;
            proxy_cache_valid 404 1m;
            proxy_cache_use_stale error timeout updating;
            proxy_cache_lock on;
            proxy_cache_key $host$uri;
            proxy_ignore_headers Cache-Control Set-Cookie;

            add_header X-Cache-Status $upstream_cache_status;
            add_header X-Clacks-Overhead "GNU Terry Pratchett" always;
        }
    }

    # --- forge.eblu.me (dynamic, authenticated) ---
    server {
        listen 8080;
        server_name forge.eblu.me;

        # Block fail2ban-banned IPs
        if ($forge_banned) {
            return 403 "Temporarily blocked. Try again later.\n";
        }

        # General rate limit — higher burst for git operations and CI webhooks
        limit_req zone=general burst=50 nodelay;

        # Git LFS and repo uploads can be large
        client_max_body_size 512m;

        # Security headers
        add_header X-Frame-Options "DENY" always;
        add_header X-Content-Type-Options "nosniff" always;
        add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
        add_header Referrer-Policy "strict-origin-when-cross-origin" always;

        error_page 502 503 504 /error.html;
        location = /error.html {
            root /usr/share/nginx/html;
            internal;
        }

        # Serve robots.txt directly — block crawlers from expensive endpoints
        location = /robots.txt {
            default_type text/plain;
            return 200 "User-agent: *\nDisallow: /mirrors/\nDisallow: /user/\nDisallow: /users/\nDisallow: /*/archive/\nDisallow: /*/releases/download/\n";
        }

        # Block the package registry at the public edge. Forgejo's per-user
        # visibility model treats packages as world-readable when the owner
        # has Visibility=Public — which means anyone on the internet can
        # enumerate and download every wheel/sdist/generic artifact, even
        # for private-repo releases (the sdist contains full source). We
        # like keeping eblume's profile public, so we close the hole here
        # at the proxy instead: WAN sees 403, tailnet (forge.ops.eblu.me)
        # stays open for legitimate consumers (CI workflows, gilbert).
        # See docs/tutorials/expose-service-publicly.md for the broader
        # threat model on this proxy.
        location /api/packages/ {
            return 403 "Package downloads are tailnet-only — use forge.ops.eblu.me.\n";
        }
        location /api/v1/packages {
            return 403 "Package enumeration is tailnet-only — use forge.ops.eblu.me.\n";
        }

        # Block swagger API docs — use forge.ops.eblu.me from tailnet
        location /swagger {
            return 403 "API documentation is only available at forge.ops.eblu.me (tailnet).\n";
        }

        # Black-hole the mirror repositories on WAN. These are mirrors of
        # already-public upstreams (tailscale, prometheus, mealie, …) kept
        # for supply-chain control; CI, gilbert, and tailnet clients consume
        # them via forge.ops.eblu.me. Their web UI served no public purpose
        # but AI scrapers, which crawled the near-infinite git-history URL
        # space (src/commit, commits, blame, raw) and drove ~70% of Fly
        # egress (1.24 TB/30d → a surprise bill) plus enough upstream load to
        # time out Forgejo. robots.txt already Disallows /mirrors/, but
        # meta-externalagent and GPTBot ignore it — so enforce at the edge.
        # `^~` makes this win over the regex locations below (e.g. *.css), so
        # static assets under /mirrors/ can't leak through. We also name and
        # shame: blocked requests get a "roll of dishonour" page (403 status
        # preserved) and an X-Naughty-Scrapers header. See
        # docs/explanation/ai-scraper-mitigation.md.
        location ^~ /mirrors/ {
            error_page 403 /naughty.html;
            return 403;
        }

        # Roll of dishonour — served on the /mirrors/ 403, status kept at 403.
        location = /naughty.html {
            internal;
            root /usr/share/nginx/html;
            add_header X-Naughty-Scrapers "OpenAI/GPTBot, Meta/meta-externalagent, Amazonbot, ByteDance/Bytespider — robots.txt ignorers" always;
            add_header X-Clacks-Overhead "GNU Terry Pratchett" always;
        }

        # Redirect archive endpoints to tailnet — archive requests generate full
        # git bundles on demand. Unauthenticated crawlers hitting unique commit
        # SHAs cause unbounded CPU and disk usage (DoS vector). Legitimate users
        # can download via forge.ops.eblu.me on the tailnet.
        location ~ ^/[^/]+/[^/]+/archive/ {
            default_type text/html;
            return 302 https://forge.ops.eblu.me$request_uri;
        }

        # Rate-limit authentication endpoints
        location ~ ^/user/(login|sign_up|forgot_password) {
            limit_req zone=forge_auth burst=5 nodelay;

            proxy_pass https://indri_backend$request_uri;
            proxy_ssl_verify off;
            proxy_ssl_server_name on;
            proxy_ssl_name forge.ops.eblu.me;
            proxy_intercept_errors on;

            proxy_set_header Host forge.ops.eblu.me;
            proxy_set_header X-Real-IP $http_fly_client_ip;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;

            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection $connection_upgrade;
        }

        # Cache release artifact downloads — immutable files keyed by tag+filename.
        # Avoids hammering Forgejo when crawlers or users re-download the same asset.
        location ~ ^/[^/]+/[^/]+/releases/download/ {
            proxy_pass https://indri_backend$request_uri;
            proxy_ssl_verify off;
            proxy_ssl_server_name on;
            proxy_ssl_name forge.ops.eblu.me;

            proxy_http_version 1.1;
            proxy_set_header Connection $connection_upgrade;

            proxy_cache services;
            proxy_cache_valid 200 7d;
            proxy_cache_key $host$uri;

            proxy_set_header Host forge.ops.eblu.me;
            proxy_set_header X-Real-IP $http_fly_client_ip;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;

            add_header X-Cache-Status $upstream_cache_status;
            add_header X-Clacks-Overhead "GNU Terry Pratchett" always;
        }

        # Selectively cache static assets only
        location ~* \.(css|js|png|jpg|svg|woff2?)$ {
            proxy_pass https://indri_backend$request_uri;
            proxy_ssl_verify off;
            proxy_ssl_server_name on;
            proxy_ssl_name forge.ops.eblu.me;

            proxy_http_version 1.1;
            proxy_set_header Connection $connection_upgrade;

            proxy_set_header Host forge.ops.eblu.me;
            proxy_set_header X-Real-IP $http_fly_client_ip;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;

            proxy_cache services;
            proxy_cache_valid 200 7d;
            proxy_cache_key $host$uri;

            add_header X-Cache-Status $upstream_cache_status;
            add_header X-Clacks-Overhead "GNU Terry Pratchett" always;
        }

        location / {
            proxy_pass https://indri_backend$request_uri;
            proxy_ssl_verify off;
            proxy_ssl_server_name on;
            proxy_ssl_name forge.ops.eblu.me;
            proxy_intercept_errors on;

            # NO proxy_cache — dynamic content with sessions

            proxy_set_header Host forge.ops.eblu.me;
            proxy_set_header X-Real-IP $http_fly_client_ip;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;

            # WebSocket support (Forgejo uses it for live updates)
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection $connection_upgrade;

            add_header X-Clacks-Overhead "GNU Terry Pratchett" always;
        }
    }

    # --- shower.eblu.me (Adelaide baby shower — guest-only public surface) ---
    # Only the guest paths (`/`, `/prizes/<token>/`, /static/, /media/) are
    # exposed on WAN. /host/, /admin/, and Django's login views are blocked
    # at the edge with a 403 pointing at the tailnet hostname — staff sign
    # in on shower.ops.eblu.me, which is reachable from any device with
    # Tailscale installed. Defense layers reduce to: general per-IP rate
    # limit + django-axes (5 fails / 1h) on the tailnet-side login. No
    # fail2ban needed here because the public surface no longer takes
    # credentials of any kind.
    server {
        listen 8080;
        server_name shower.eblu.me;

        # Per-IP rate limit. shower_general (50r/s, burst=200) instead of
        # the global `general` zone because at the party, guests on the
        # venue's wifi all NAT through a single Fly-Client-IP — 30 guests
        # scanning the QR at once would each fetch HTML + a few static
        # assets, easily clearing 20 burst on `general`. Exploit scanners
        # still trip it (sustained ≫ 50r/s patterns).
        limit_req zone=shower_general burst=200 nodelay;

        # Image uploads from /host/'s prize cropper are ~150-300 KiB JPEGs.
        # The host page itself isn't reachable here, but /media/ reads can
        # be larger than 1 MiB so set the cap to 5 MiB to match Django.
        client_max_body_size 5m;

        # Security headers — HSTS matches Django's SECURE_HSTS_SECONDS.
        add_header X-Frame-Options "DENY" always;
        add_header X-Content-Type-Options "nosniff" always;
        add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
        add_header Referrer-Policy "same-origin" always;
        # GNU Terry Pratchett — keep the name moving.
        add_header X-Clacks-Overhead "GNU Terry Pratchett" always;

        error_page 502 503 504 /error.html;
        location = /error.html {
            root /usr/share/nginx/html;
            internal;
        }

        # Reject indexers — there's nothing here we want crawled.
        location = /robots.txt {
            default_type text/plain;
            return 200 "User-agent: *\nDisallow: /\n";
        }

        # Admin surface: tailnet-only. Anything under /admin/ — login,
        # logout, CRUD UI, password reset — returns 403 with a pointer to
        # the tailnet host. Django's `staff_member_required` will redirect
        # /host/ to /admin/login/, which lands on this 403 if a guest
        # device wanders into it. Staff hit the tailnet host directly.
        location /admin/ {
            return 403 "Authentication is tailnet-only — visit shower.ops.eblu.me.\n";
        }

        # Operator console: tailnet-only. Same rationale as /admin/.
        location /host/ {
            return 403 "The host console is tailnet-only — visit shower.ops.eblu.me.\n";
        }

        # Static assets — WhiteNoise + CompressedManifestStaticFilesStorage
        # gives content-hashed filenames, so cache aggressively. Hashed
        # names make cache invalidation automatic on app upgrades.
        location /static/ {
            proxy_pass https://indri_backend$request_uri;
            proxy_ssl_verify off;
            proxy_ssl_server_name on;
            proxy_ssl_name shower.ops.eblu.me;

            proxy_http_version 1.1;
            proxy_set_header Connection $connection_upgrade;
            proxy_set_header Host shower.ops.eblu.me;
            proxy_set_header X-Real-IP $http_fly_client_ip;
            proxy_set_header X-Forwarded-For $http_fly_client_ip;
            proxy_set_header X-Forwarded-Proto $scheme;

            proxy_cache services;
            proxy_cache_valid 200 1y;
            proxy_cache_valid 404 1m;
            proxy_cache_use_stale error timeout updating;
            proxy_cache_lock on;
            proxy_cache_key $host$uri;
            proxy_ignore_headers Cache-Control Set-Cookie;

            add_header X-Cache-Status $upstream_cache_status;
        }

        # Prize photo uploads. Shorter TTL than /static/ because filenames
        # aren't content-hashed — operators can re-upload a prize photo
        # and we want guests to see the new image within a day.
        location /media/ {
            proxy_pass https://indri_backend$request_uri;
            proxy_ssl_verify off;
            proxy_ssl_server_name on;
            proxy_ssl_name shower.ops.eblu.me;

            proxy_http_version 1.1;
            proxy_set_header Connection $connection_upgrade;
            proxy_set_header Host shower.ops.eblu.me;
            proxy_set_header X-Real-IP $http_fly_client_ip;
            proxy_set_header X-Forwarded-For $http_fly_client_ip;
            proxy_set_header X-Forwarded-Proto $scheme;

            proxy_cache services;
            proxy_cache_valid 200 1d;
            proxy_cache_valid 404 1m;
            proxy_cache_use_stale error timeout updating;
            proxy_cache_lock on;
            proxy_cache_key $host$uri;
            proxy_ignore_headers Cache-Control Set-Cookie;

            add_header X-Cache-Status $upstream_cache_status;
        }

        location / {
            proxy_pass https://indri_backend$request_uri;
            proxy_ssl_verify off;
            proxy_ssl_server_name on;
            proxy_ssl_name shower.ops.eblu.me;
            proxy_intercept_errors on;

            # No proxy_cache — dynamic content with sessions and CSRF.

            proxy_set_header Host shower.ops.eblu.me;
            proxy_set_header X-Real-IP $http_fly_client_ip;
            proxy_set_header X-Forwarded-For $http_fly_client_ip;
            proxy_set_header X-Forwarded-Proto $scheme;

            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection $connection_upgrade;
        }
    }

    # Catch-all: reject unknown hosts, but serve health check
    server {
        listen 8080 default_server;

        location /healthz {
            return 200 "ok\n";
        }

        location /stub_status {
            stub_status;
            allow 127.0.0.1;
            deny all;
        }

        location / {
            return 444;
        }
    }
}