From 1631e111374438a843c59718a83edb106c9bdec4 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 17 Apr 2026 14:34:24 -0700 Subject: [PATCH] Add /user/ to forge robots.txt exclusion Crawlers follow auth redirects to /user/login which is pointless for them. Saves round-trips for both sides. Co-Authored-By: Claude Opus 4.6 (1M context) --- fly/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fly/nginx.conf b/fly/nginx.conf index de77ee6..9af2509 100644 --- a/fly/nginx.conf +++ b/fly/nginx.conf @@ -158,7 +158,7 @@ http { # Serve robots.txt directly — block crawlers from expensive endpoints location = /robots.txt { default_type text/plain; - return 200 "User-agent: *\nDisallow: /mirrors/\nDisallow: /users/\nDisallow: /*/archive/\nDisallow: /*/releases/download/\n"; + return 200 "User-agent: *\nDisallow: /mirrors/\nDisallow: /user/\nDisallow: /users/\nDisallow: /*/archive/\nDisallow: /*/releases/download/\n"; } # Block swagger API docs — use forge.ops.eblu.me from tailnet