From ede9a5139408d3c9567d0d5833552aa44a6d647c Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Sun, 8 Mar 2026 19:57:45 -0700 Subject: [PATCH] Fix robots.txt: block /explore/ and /tags/ (was /explorer/) The previous robots.txt had a typo blocking /explorer/ instead of /explore/, allowing Facebook's crawler to hit the spider trap. Also block /tags/ which has the same infinite relative-link issue. Co-Authored-By: Claude Opus 4.6 --- containers/quartz/default.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/containers/quartz/default.conf b/containers/quartz/default.conf index 70b8fcc..396db72 100644 --- a/containers/quartz/default.conf +++ b/containers/quartz/default.conf @@ -14,12 +14,12 @@ server { add_header Cache-Control "public, immutable"; } - # Serve robots.txt inline to prevent crawlers from entering /explorer/, + # Serve robots.txt inline to prevent crawlers from entering /explore/ and /tags/, # which is an SPA feature that generates infinite relative-link trees # when crawled (the March 2026 spider-trap incident). location = /robots.txt { default_type text/plain; - return 200 "User-agent: *\nDisallow: /explorer/\n"; + return 200 "User-agent: *\nDisallow: /explore/\nDisallow: /tags/\n"; } # SPA fallback - serve index.html for client-side routing