From d4082985a6d845d3a5fe1d4b50d817c5c5f484d5 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Tue, 3 Mar 2026 06:56:33 -0800 Subject: [PATCH 01/14] Remove deprecated forge egress proxy from tailscale-operator The egress proxy (tailscale-forge device) has been unused since Caddy took over forge routing. No k8s resources reference it as a backend. Co-Authored-By: Claude Opus 4.6 --- argocd/manifests/tailscale-operator/README.md | 3 --- .../tailscale-operator/egress-forge.yaml | 23 ------------------- .../tailscale-operator/kustomization.yaml | 1 - 3 files changed, 27 deletions(-) delete mode 100644 argocd/manifests/tailscale-operator/egress-forge.yaml diff --git a/argocd/manifests/tailscale-operator/README.md b/argocd/manifests/tailscale-operator/README.md index ee8ce1e..dc4b009 100644 --- a/argocd/manifests/tailscale-operator/README.md +++ b/argocd/manifests/tailscale-operator/README.md @@ -73,7 +73,6 @@ kubectl logs -n tailscale -l app.kubernetes.io/name=operator | `operator.yaml` | Operator deployment, CRDs, RBAC (secret removed) | | `proxyclass.yaml` | ProxyClass with fully-qualified images | | `dnsconfig.yaml` | DNSConfig for cluster-to-tailnet name resolution | -| `egress-forge.yaml` | Egress proxy for accessing forge on indri | | `secret.yaml.tpl` | 1Password template for OAuth credentials (manual) | | `README.md` | This file | @@ -86,5 +85,3 @@ kubectl logs -n tailscale -l app.kubernetes.io/name=operator annotations: tailscale.com/proxy-class: "default" ``` -- The egress proxy for forge is **deprecated**. Forge is now accessible via Caddy at - `forge.ops.eblu.me` (HTTPS) and `forge.ops.eblu.me:2222` (SSH), which pods can reach directly. diff --git a/argocd/manifests/tailscale-operator/egress-forge.yaml b/argocd/manifests/tailscale-operator/egress-forge.yaml deleted file mode 100644 index 4dc982b..0000000 --- a/argocd/manifests/tailscale-operator/egress-forge.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# DEPRECATED: This egress proxy is no longer needed. -# Forge is now accessible via Caddy at forge.ops.eblu.me (HTTPS) and -# forge.ops.eblu.me:2222 (SSH), which pods can reach directly. -# -# Keeping this file for reference during migration. Remove once verified. -# -# Original purpose: Egress proxy to expose Forgejo (forge) to the cluster -# See: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress ---- -apiVersion: v1 -kind: Service -metadata: - name: forge - namespace: tailscale - annotations: - tailscale.com/tailnet-fqdn: indri.tail8d86e.ts.net - tailscale.com/proxy-class: "default" -spec: - type: ExternalName - externalName: placeholder - ports: - - port: 3001 - targetPort: 3001 diff --git a/argocd/manifests/tailscale-operator/kustomization.yaml b/argocd/manifests/tailscale-operator/kustomization.yaml index 09fa1b8..a14ca81 100644 --- a/argocd/manifests/tailscale-operator/kustomization.yaml +++ b/argocd/manifests/tailscale-operator/kustomization.yaml @@ -7,5 +7,4 @@ namespace: tailscale resources: - ../tailscale-operator-base - proxygroup-ingress.yaml - - egress-forge.yaml - external-secret.yaml -- 2.50.1 (Apple Git-155) From d6584a2bd69e1a3fa7ad3855b8fc46e20cdfd082 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Tue, 3 Mar 2026 07:49:49 -0800 Subject: [PATCH 02/14] Docs: document forge.eblu.me public access, update routing and security guidance Update forgejo.md with public access details and security controls. Add forge.eblu.me to public services table in routing.md. Update fail2ban guidance in expose-service-publicly.md to reflect Fly.io container approach. Add changelog fragment. Co-Authored-By: Claude Opus 4.6 --- .../feature-forge-public.feature.md | 1 + .../configuration/expose-service-publicly.md | 46 +++++++++---------- docs/reference/infrastructure/routing.md | 3 +- docs/reference/services/forgejo.md | 36 +++++++++------ 4 files changed, 47 insertions(+), 39 deletions(-) create mode 100644 docs/changelog.d/feature-forge-public.feature.md diff --git a/docs/changelog.d/feature-forge-public.feature.md b/docs/changelog.d/feature-forge-public.feature.md new file mode 100644 index 0000000..44be391 --- /dev/null +++ b/docs/changelog.d/feature-forge-public.feature.md @@ -0,0 +1 @@ +Expose Forgejo publicly at forge.eblu.me via Fly.io reverse proxy with rate limiting, fail2ban, and security hardening. diff --git a/docs/how-to/configuration/expose-service-publicly.md b/docs/how-to/configuration/expose-service-publicly.md index b342481..bb6b258 100644 --- a/docs/how-to/configuration/expose-service-publicly.md +++ b/docs/how-to/configuration/expose-service-publicly.md @@ -1,7 +1,7 @@ --- title: Expose a Service Publicly -modified: 2026-02-16 -last-reviewed: 2026-02-16 +modified: 2026-03-03 +last-reviewed: 2026-03-03 tags: - how-to - fly-io @@ -259,7 +259,7 @@ server { } ``` -**Dynamic service template** (e.g., Forgejo — hypothetical, not currently deployed): +**Dynamic service template** (e.g., Forgejo — see `fly/nginx.conf` for the live configuration): ```nginx # --- forge.eblu.me (dynamic, authenticated) --- @@ -440,32 +440,30 @@ see plan history in git). ### fail2ban fail2ban monitors log files for repeated failed authentication attempts -(SSH brute force, bad login passwords, API abuse) and bans IPs via -firewall rules. +and bans offending IPs. **Static sites**: fail2ban does not apply. There is no login surface, no sessions, no credentials to brute force. -**Dynamic services with authentication** (e.g., Forgejo): fail2ban is -relevant and should be configured on **indri**, not on Fly.io. The -nginx proxy is transparent — it forwards requests but does not see -authentication outcomes. fail2ban watches the service's own logs on -indri for patterns like repeated failed logins. +**Dynamic services with authentication** (e.g., Forgejo): fail2ban +runs in the **Fly.io container**, not on indri. Standard iptables +banning won't work in Fly.io because `$remote_addr` is Fly's internal +proxy IP, not the client. Instead, fail2ban uses a custom nginx-based +ban action: -Setup considerations for Forgejo specifically: +1. fail2ban watches the nginx JSON access log for repeated 401/403 + responses to login endpoints, keyed on the `client_ip` field + (populated from the `Fly-Client-IP` header) +2. On ban, it appends the IP to `/etc/nginx/forge-deny.conf` and + reloads nginx +3. nginx uses a `geo` directive keyed on `$http_fly_client_ip` to + check the deny list and return 403 for banned IPs -- Forgejo logs failed auth attempts to its log file -- fail2ban needs a filter matching Forgejo's log format -- Banned IPs are blocked at indri's firewall (the Fly.io proxy IP is - the Tailscale address of the `flyio-proxy` node, not the end user's - IP) -- **Important**: for fail2ban to see real client IPs, the nginx proxy - must pass `X-Real-IP` / `X-Forwarded-For` headers (included in the - dynamic service nginx config above), and Forgejo must be configured - to trust the proxy and log the forwarded IP rather than the proxy's - Tailscale IP -- Disable open user registration before exposing Forgejo publicly — - require explicit invites +Ban lists are **ephemeral across deploys** — nginx rate limiting +provides the persistent baseline; fail2ban adds escalating bans for +active attacks. + +See `fly/fail2ban/` for the filter, jail, and action configuration. ### Break-glass shutoff @@ -504,7 +502,7 @@ dynamic, authenticated service like [[forgejo]]. - [ ] Disable open user registration (require invites or admin approval) - [ ] Audit access controls and permissions - [ ] Configure the service to log the forwarded client IP (not the proxy IP) -- [ ] Set up fail2ban on indri with a filter for the service's log format +- [ ] Set up fail2ban in the Fly.io container with a filter for the service's login endpoints - [ ] Tag the service's Tailscale Ingress with `tag:flyio-target` - [ ] Test the nginx config locally or in staging before deploying - [ ] Rehearse the break-glass shutoff (`mise run fly-shutoff`) diff --git a/docs/reference/infrastructure/routing.md b/docs/reference/infrastructure/routing.md index 092119f..91457e9 100644 --- a/docs/reference/infrastructure/routing.md +++ b/docs/reference/infrastructure/routing.md @@ -1,6 +1,6 @@ --- title: Routing -modified: 2026-02-09 +modified: 2026-03-03 tags: - infrastructure - networking @@ -49,6 +49,7 @@ DNS CNAMEs point to `blumeops-proxy.fly.dev`. TLS via Fly.io-managed Let's Encry | Service | URL | Description | |---------|-----|-------------| | [[docs]] | https://docs.eblu.me | Documentation site | +| [[forgejo]] | https://forge.eblu.me | Git hosting (public) | ## Tailscale-Only Services diff --git a/docs/reference/services/forgejo.md b/docs/reference/services/forgejo.md index 68fd1f4..504107e 100644 --- a/docs/reference/services/forgejo.md +++ b/docs/reference/services/forgejo.md @@ -1,6 +1,6 @@ --- title: Forgejo -modified: 2026-02-20 +modified: 2026-03-03 tags: - service - git @@ -15,7 +15,8 @@ Git forge and CI/CD platform. **Primary source of truth for blumeops** (mirrored | Property | Value | |----------|-------| -| **URL** | https://forge.ops.eblu.me | +| **URL (public)** | https://forge.eblu.me | +| **URL (internal)** | https://forge.ops.eblu.me | | **SSH** | `ssh://forgejo@forge.ops.eblu.me:2222` | | **Local Ports** | 3001 (HTTP), 2200 (SSH) | | **Config** | `ansible/roles/forgejo/templates/app.ini.j2` | @@ -94,23 +95,30 @@ This is a bootstrapping requirement - the PAT enables IaC for all other secrets. **Break-glass:** Local password login always works (with local MFA). Authentik SSO is additive — if Authentik is down, log in with local credentials. -## Future: Public Access +## Public Access -Forgejo can be exposed publicly at `forge.eblu.me` via [[flyio-proxy]]. Since Forgejo runs natively on [[indri]] (not in k8s), the pattern is: +Forgejo is publicly accessible at `https://forge.eblu.me` via [[flyio-proxy]]. This is the first dynamic, authenticated service exposed publicly. -1. Create a k8s ExternalName Service pointing to indri's Tailscale IP -2. Create a Tailscale Ingress with `tailscale.com/tags: "tag:k8s,tag:flyio-target"` -3. Add the nginx server block and DNS CNAME +| Access Method | URL | Reachable From | +|---------------|-----|----------------| +| **HTTPS (public)** | https://forge.eblu.me | Public internet | +| **HTTPS (internal)** | https://forge.ops.eblu.me | Tailnet only | +| **SSH** | `ssh://forgejo@forge.ops.eblu.me:2222` | Tailnet only | -Exposing a dynamic, authenticated service like Forgejo requires a full security review before going live: +The UI shows `forge.eblu.me` for HTTPS clone URLs and `forge.ops.eblu.me` for SSH clone URLs. -- Disable all local registration — only allow login via [[authentik]] (`DISABLE_REGISTRATION = true`, `ALLOW_ONLY_EXTERNAL_REGISTRATION = true`) -- Configure fail2ban on indri with a filter for Forgejo's log format -- Ensure Forgejo logs the forwarded client IP (`X-Real-IP`) rather than the proxy's Tailscale IP -- Audit repository visibility defaults and permissions -- Rehearse the break-glass shutoff (`mise run fly-shutoff`) +### Security Controls -See [[expose-service-publicly]] for the full howto and dynamic service checklist. +- **Registration:** Local registration disabled; only [[authentik]] SSO login allowed (`ALLOW_ONLY_EXTERNAL_REGISTRATION = true`) +- **Reverse proxy trust:** `REVERSE_PROXY_LIMIT = 2`, `REVERSE_PROXY_TRUSTED_PROXIES = *` — Forgejo logs the real client IP from `X-Real-IP` header, not the proxy's Tailscale IP +- **Rate limiting:** nginx rate limits login/signup/forgot-password endpoints (3r/s per client IP via `Fly-Client-IP` header) +- **fail2ban:** Runs in the Fly.io container; bans IPs after 5 failed logins in 10 minutes via nginx deny list (ephemeral across deploys) +- **Swagger:** Blocked at the proxy (`/swagger` returns 403); use forge.ops.eblu.me for API access +- **OAuth dead-end:** "Sign in with Authentik" redirects to the (tailnet-only) Authentik URL — SSO only works from the tailnet + +### Break-glass + +`mise run fly-shutoff` stops all public traffic immediately. forge.ops.eblu.me continues to work from the tailnet. See [[expose-service-publicly#Break-glass shutoff]]. ## Related -- 2.50.1 (Apple Git-155) From d2da346ac0bd56e15c8acb621735f1f5b04aa703 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Tue, 3 Mar 2026 07:50:25 -0800 Subject: [PATCH 03/14] Harden Forgejo for public access: domain, proxy trust, registration lockdown - Set forgejo_domain to forge.eblu.me (public URL in clone URLs) - Set forgejo_ssh_domain to forge.ops.eblu.me (SSH stays tailnet-only) - Add REVERSE_PROXY_LIMIT=2, REVERSE_PROXY_TRUSTED_PROXIES=* for correct client IP logging through Fly.io + Tailscale proxy chain - Enable ALLOW_ONLY_EXTERNAL_REGISTRATION to block local signups Co-Authored-By: Claude Opus 4.6 --- ansible/roles/forgejo/defaults/main.yml | 4 ++-- ansible/roles/forgejo/templates/app.ini.j2 | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ansible/roles/forgejo/defaults/main.yml b/ansible/roles/forgejo/defaults/main.yml index 95ffda8..be967aa 100644 --- a/ansible/roles/forgejo/defaults/main.yml +++ b/ansible/roles/forgejo/defaults/main.yml @@ -18,8 +18,8 @@ forgejo_log_path: "{{ forgejo_work_path }}/log" # Server settings forgejo_http_addr: 0.0.0.0 forgejo_http_port: 3001 -forgejo_domain: forge.ops.eblu.me -forgejo_ssh_domain: "{{ forgejo_domain }}" +forgejo_domain: forge.eblu.me +forgejo_ssh_domain: forge.ops.eblu.me forgejo_root_url: "https://{{ forgejo_domain }}/" forgejo_offline_mode: true diff --git a/ansible/roles/forgejo/templates/app.ini.j2 b/ansible/roles/forgejo/templates/app.ini.j2 index 3668827..de931be 100644 --- a/ansible/roles/forgejo/templates/app.ini.j2 +++ b/ansible/roles/forgejo/templates/app.ini.j2 @@ -20,6 +20,8 @@ SSH_LISTEN_PORT = {{ forgejo_ssh_listen_port }} LFS_START_SERVER = {{ forgejo_lfs_start_server | lower }} LFS_JWT_SECRET = {{ forgejo_lfs_jwt_secret }} OFFLINE_MODE = {{ forgejo_offline_mode | lower }} +REVERSE_PROXY_LIMIT = 2 +REVERSE_PROXY_TRUSTED_PROXIES = * [database] DB_TYPE = {{ forgejo_db_type }} @@ -40,7 +42,7 @@ ENABLED = false REGISTER_EMAIL_CONFIRM = false ENABLE_NOTIFY_MAIL = false DISABLE_REGISTRATION = {{ forgejo_disable_registration | lower }} -ALLOW_ONLY_EXTERNAL_REGISTRATION = false +ALLOW_ONLY_EXTERNAL_REGISTRATION = true ENABLE_CAPTCHA = false REQUIRE_SIGNIN_VIEW = {{ forgejo_require_signin_view | lower }} DEFAULT_KEEP_EMAIL_PRIVATE = false -- 2.50.1 (Apple Git-155) From b49ff9f8219985ba111fe3868441fdd58efe8a80 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Tue, 3 Mar 2026 07:51:28 -0800 Subject: [PATCH 04/14] Add Tailscale Ingress for Forge via ExternalName Service Create forge.tail8d86e.ts.net endpoint that proxies to Forgejo on indri:3001. Uses ExternalName Service since Forgejo runs natively on indri (not in k8s). Tagged with flyio-target for Fly.io proxy access via existing ACLs. Co-Authored-By: Claude Opus 4.6 --- .../tailscale-operator/ingress-forge.yaml | 20 +++++++++++++++++++ .../tailscale-operator/kustomization.yaml | 2 ++ .../svc-forge-external.yaml | 13 ++++++++++++ 3 files changed, 35 insertions(+) create mode 100644 argocd/manifests/tailscale-operator/ingress-forge.yaml create mode 100644 argocd/manifests/tailscale-operator/svc-forge-external.yaml diff --git a/argocd/manifests/tailscale-operator/ingress-forge.yaml b/argocd/manifests/tailscale-operator/ingress-forge.yaml new file mode 100644 index 0000000..047b59d --- /dev/null +++ b/argocd/manifests/tailscale-operator/ingress-forge.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: forge-tailscale + namespace: tailscale + annotations: + tailscale.com/proxy-class: "default" + tailscale.com/proxy-group: "ingress" + tailscale.com/tags: "tag:k8s,tag:flyio-target" +spec: + ingressClassName: tailscale + defaultBackend: + service: + name: forge-external + port: + number: 3001 + tls: + - hosts: + - forge diff --git a/argocd/manifests/tailscale-operator/kustomization.yaml b/argocd/manifests/tailscale-operator/kustomization.yaml index a14ca81..b38ee05 100644 --- a/argocd/manifests/tailscale-operator/kustomization.yaml +++ b/argocd/manifests/tailscale-operator/kustomization.yaml @@ -8,3 +8,5 @@ resources: - ../tailscale-operator-base - proxygroup-ingress.yaml - external-secret.yaml + - svc-forge-external.yaml + - ingress-forge.yaml diff --git a/argocd/manifests/tailscale-operator/svc-forge-external.yaml b/argocd/manifests/tailscale-operator/svc-forge-external.yaml new file mode 100644 index 0000000..2812acf --- /dev/null +++ b/argocd/manifests/tailscale-operator/svc-forge-external.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: forge-external + namespace: tailscale +spec: + type: ExternalName + externalName: indri.tail8d86e.ts.net + ports: + - name: http + port: 3001 + protocol: TCP -- 2.50.1 (Apple Git-155) From 80c33ccaf11f1c66fbd06fac9ae3279ada396142 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Tue, 3 Mar 2026 07:52:58 -0800 Subject: [PATCH 05/14] Add forge.eblu.me to Fly.io proxy with rate limiting and fail2ban MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit nginx configuration: - forge.eblu.me server block with WebSocket support, 512m body limit - Rate limit login/signup/forgot-password at 3r/s per real client IP (keyed on Fly-Client-IP header, not Fly's internal remote_addr) - Static asset caching (7d), no blanket caching for dynamic content - Security headers (HSTS, X-Frame-Options, X-Content-Type-Options) - Block /swagger (API docs only available via tailnet) - X-Real-IP set to real client IP for Forgejo audit logs - geo-based deny list for fail2ban integration fail2ban configuration: - Custom filter matching 401/403 on login paths in nginx JSON log - Ban after 5 failures in 10 minutes, ban duration 1 hour - Custom nginx-deny action: writes IPs to deny file and reloads nginx (iptables won't work in Fly.io — remote_addr is Fly's proxy IP) - Ban lists ephemeral across deploys (nginx rate limiting is persistent) Co-Authored-By: Claude Opus 4.6 --- fly/Dockerfile | 7 +- fly/fail2ban/action.d/nginx-deny.conf | 14 ++++ fly/fail2ban/filter.d/forge-login.conf | 10 +++ fly/fail2ban/jail.d/forge.conf | 8 ++ fly/nginx.conf | 104 +++++++++++++++++++++++++ fly/start.sh | 7 ++ 6 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 fly/fail2ban/action.d/nginx-deny.conf create mode 100644 fly/fail2ban/filter.d/forge-login.conf create mode 100644 fly/fail2ban/jail.d/forge.conf diff --git a/fly/Dockerfile b/fly/Dockerfile index 68a98d8..09099c4 100644 --- a/fly/Dockerfile +++ b/fly/Dockerfile @@ -8,7 +8,8 @@ COPY --from=docker.io/tailscale/tailscale:stable \ RUN mkdir -p /var/run/tailscale /var/lib/tailscale \ && apk add --no-cache iptables ip6tables \ - && apk add --no-cache libc6-compat + && apk add --no-cache libc6-compat \ + && apk add --no-cache fail2ban # Copy Alloy binary from official image (Ubuntu-based, needs libc6-compat) COPY --from=docker.io/grafana/alloy:v1.13.1 \ @@ -16,6 +17,10 @@ COPY --from=docker.io/grafana/alloy:v1.13.1 \ RUN mkdir -p /var/log/nginx /etc/alloy /tmp/alloy-data +COPY fail2ban/filter.d/forge-login.conf /etc/fail2ban/filter.d/forge-login.conf +COPY fail2ban/jail.d/forge.conf /etc/fail2ban/jail.d/forge.conf +COPY fail2ban/action.d/nginx-deny.conf /etc/fail2ban/action.d/nginx-deny.conf + COPY nginx.conf /etc/nginx/nginx.conf COPY error.html /usr/share/nginx/html/error.html COPY alloy.river /etc/alloy/config.alloy diff --git a/fly/fail2ban/action.d/nginx-deny.conf b/fly/fail2ban/action.d/nginx-deny.conf new file mode 100644 index 0000000..1d3737b --- /dev/null +++ b/fly/fail2ban/action.d/nginx-deny.conf @@ -0,0 +1,14 @@ +# Custom fail2ban action that bans IPs via an nginx deny list. +# Standard iptables banning won't work in Fly.io because $remote_addr +# is Fly's internal proxy IP. Instead, we write banned IPs to a file +# that nginx checks via a geo directive keyed on $http_fly_client_ip. + +[Definition] + +actionban = echo " 1;" >> /etc/nginx/forge-deny.conf && nginx -s reload + +actionunban = sed -i '/ 1;/d' /etc/nginx/forge-deny.conf && nginx -s reload + +actionstart = +actionstop = +actioncheck = diff --git a/fly/fail2ban/filter.d/forge-login.conf b/fly/fail2ban/filter.d/forge-login.conf new file mode 100644 index 0000000..6961b9a --- /dev/null +++ b/fly/fail2ban/filter.d/forge-login.conf @@ -0,0 +1,10 @@ +# Filter for Forgejo login failures via nginx JSON access log. +# Matches 401/403 responses to authentication endpoints, keyed on +# the client_ip field (populated from Fly-Client-IP header). + +[Definition] + +# Match JSON log lines with 401 or 403 status on login-related paths +failregex = "client_ip":"".*"request_uri":"\/user\/(login|sign_up|forgot_password)[^"]*".*"status":(401|403) + +ignoreregex = diff --git a/fly/fail2ban/jail.d/forge.conf b/fly/fail2ban/jail.d/forge.conf new file mode 100644 index 0000000..7b0843f --- /dev/null +++ b/fly/fail2ban/jail.d/forge.conf @@ -0,0 +1,8 @@ +[forge-login] +enabled = true +filter = forge-login +logpath = /var/log/nginx/access.json.log +maxretry = 5 +findtime = 600 +bantime = 3600 +banaction = nginx-deny diff --git a/fly/nginx.conf b/fly/nginx.conf index e50545a..992a5df 100644 --- a/fly/nginx.conf +++ b/fly/nginx.conf @@ -29,6 +29,19 @@ http { # Rate limiting zones — define per-service zones as needed limit_req_zone $binary_remote_addr zone=general:10m rate=10r/s; + # Forge-specific rate limit keyed on real client IP (Fly-Client-IP header). + # $binary_remote_addr is Fly's internal proxy IP — all clients share one + # bucket. $http_fly_client_ip has the actual client IP. + limit_req_zone $http_fly_client_ip zone=forge_auth:10m rate=3r/s; + + # fail2ban deny list — banned IPs are written here by fail2ban and + # checked via the $forge_banned variable. The file is touched at + # container start to ensure it exists. + geo $http_fly_client_ip $forge_banned { + default 0; + include /etc/nginx/forge-deny.conf; + } + # Proxy cache: 200MB, evict after 24h of no access proxy_cache_path /tmp/cache levels=1:2 keys_zone=services:10m max_size=200m inactive=24h; @@ -114,6 +127,97 @@ http { } } + # --- forge.eblu.me (dynamic, authenticated) --- + server { + listen 8080; + server_name forge.eblu.me; + + # Block fail2ban-banned IPs + if ($forge_banned) { + return 403 "Temporarily blocked. Try again later.\n"; + } + + # General rate limit — higher burst for git operations and CI webhooks + limit_req zone=general burst=50 nodelay; + + # Git LFS and repo uploads can be large + client_max_body_size 512m; + + # Security headers + add_header X-Frame-Options "DENY" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + + error_page 502 503 504 /error.html; + location = /error.html { + root /usr/share/nginx/html; + internal; + } + + # Block swagger API docs — use forge.ops.eblu.me from tailnet + location /swagger { + return 403 "API documentation is only available at forge.ops.eblu.me (tailnet).\n"; + } + + # Rate-limit authentication endpoints + location ~ ^/user/(login|sign_up|forgot_password) { + limit_req zone=forge_auth burst=5 nodelay; + + set $upstream_forge https://forge.tail8d86e.ts.net; + proxy_pass $upstream_forge$request_uri; + proxy_ssl_verify off; + proxy_ssl_server_name on; + proxy_intercept_errors on; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $http_fly_client_ip; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + # Selectively cache static assets only + location ~* \.(css|js|png|jpg|svg|woff2?)$ { + set $upstream_forge_static https://forge.tail8d86e.ts.net; + proxy_pass $upstream_forge_static$request_uri; + proxy_ssl_verify off; + proxy_ssl_server_name on; + + proxy_cache services; + proxy_cache_valid 200 7d; + proxy_cache_key $host$uri; + + add_header X-Cache-Status $upstream_cache_status; + add_header X-Clacks-Overhead "GNU Terry Pratchett" always; + } + + location / { + set $upstream_forge https://forge.tail8d86e.ts.net; + proxy_pass $upstream_forge$request_uri; + proxy_ssl_verify off; + proxy_ssl_server_name on; + proxy_intercept_errors on; + + # NO proxy_cache — dynamic content with sessions + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $http_fly_client_ip; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # WebSocket support (Forgejo uses it for live updates) + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + add_header X-Clacks-Overhead "GNU Terry Pratchett" always; + } + } + # Catch-all: reject unknown hosts, but serve health check server { listen 8080 default_server; diff --git a/fly/start.sh b/fly/start.sh index 96ccbf0..2ec7c48 100644 --- a/fly/start.sh +++ b/fly/start.sh @@ -12,11 +12,18 @@ tailscale up --authkey="${TS_AUTHKEY}" --hostname=flyio-proxy until tailscale status > /dev/null 2>&1; do sleep 1; done echo "Tailscale connected" +# Ensure fail2ban deny file exists before nginx starts +touch /etc/nginx/forge-deny.conf + # Start nginx — MagicDNS is available, health check passes immediately. nginx -g "daemon off;" & NGINX_PID=$! echo "Nginx started" +# Start fail2ban for login brute-force protection +fail2ban-server -b +echo "fail2ban started" + # Start Alloy for observability (logs → Loki, metrics → Prometheus) alloy run /etc/alloy/config.alloy \ --server.http.listen-addr=127.0.0.1:12345 \ -- 2.50.1 (Apple Git-155) From 8f47145b404269a6f0433d80a618ad5d587fca93 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Tue, 3 Mar 2026 07:53:28 -0800 Subject: [PATCH 06/14] Update Authentik Forgejo OAuth callback to forge.eblu.me MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update redirect_uris and meta_launch_url to use the new public domain. OAuth flow will dead-end naturally since Authentik is not publicly accessible — SSO only works from the tailnet. Co-Authored-By: Claude Opus 4.6 --- argocd/manifests/authentik/configmap-blueprint.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/argocd/manifests/authentik/configmap-blueprint.yaml b/argocd/manifests/authentik/configmap-blueprint.yaml index f5b4784..e867c3a 100644 --- a/argocd/manifests/authentik/configmap-blueprint.yaml +++ b/argocd/manifests/authentik/configmap-blueprint.yaml @@ -120,7 +120,7 @@ data: client_secret: !Env AUTHENTIK_FORGEJO_CLIENT_SECRET redirect_uris: - matching_mode: strict - url: https://forge.ops.eblu.me/user/oauth2/authentik/callback + url: https://forge.eblu.me/user/oauth2/authentik/callback signing_key: !Find [authentik_crypto.certificatekeypair, [name, authentik Self-signed Certificate]] property_mappings: - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]] @@ -138,7 +138,7 @@ data: name: Forgejo slug: forgejo provider: !KeyOf forgejo-provider - meta_launch_url: https://forge.ops.eblu.me + meta_launch_url: https://forge.eblu.me policy_engine_mode: any # Policy binding — restrict Forgejo to admins group -- 2.50.1 (Apple Git-155) From 32bf20525f2db86927b62bc90d80f7807def4ded Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Tue, 3 Mar 2026 07:54:20 -0800 Subject: [PATCH 07/14] Add forge.eblu.me DNS CNAME and Fly.io TLS certificate Add CNAME record pointing forge.eblu.me to blumeops-proxy.fly.dev in Pulumi Gandi config. Add forge.eblu.me to fly-setup cert list. Co-Authored-By: Claude Opus 4.6 --- mise-tasks/fly-setup | 1 + pulumi/gandi/__main__.py | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/mise-tasks/fly-setup b/mise-tasks/fly-setup index 63304db..0c5cb56 100755 --- a/mise-tasks/fly-setup +++ b/mise-tasks/fly-setup @@ -22,6 +22,7 @@ echo "IPs allocated" # Add certs for all public domains (idempotent — fly ignores duplicates) fly certs add docs.eblu.me -a "$APP" 2>/dev/null || true fly certs add cv.eblu.me -a "$APP" 2>/dev/null || true +fly certs add forge.eblu.me -a "$APP" 2>/dev/null || true echo "Certificates configured" echo "Done. Run 'mise run fly-deploy' to deploy." diff --git a/pulumi/gandi/__main__.py b/pulumi/gandi/__main__.py index df6ad34..e448ed2 100644 --- a/pulumi/gandi/__main__.py +++ b/pulumi/gandi/__main__.py @@ -76,6 +76,15 @@ cv_public = gandi.livedns.Record( values=["blumeops-proxy.fly.dev."], ) +forge_public = gandi.livedns.Record( + "forge-public", + zone=domain, + name="forge", + type="CNAME", + ttl=300, + values=["blumeops-proxy.fly.dev."], +) + # ============== Exports ============== pulumi.export("domain", domain) pulumi.export("wildcard_fqdn", f"*.{subdomain}.{domain}") @@ -83,3 +92,4 @@ pulumi.export("base_fqdn", f"{subdomain}.{domain}") pulumi.export("target_ip", tailscale_ip) pulumi.export("docs_public_fqdn", f"docs.{domain}") pulumi.export("cv_public_fqdn", f"cv.{domain}") +pulumi.export("forge_public_fqdn", f"forge.{domain}") -- 2.50.1 (Apple Git-155) From 538a8cf6c1b27640972b276c51246e57f4dd1451 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Tue, 3 Mar 2026 07:57:48 -0800 Subject: [PATCH 08/14] =?UTF-8?q?Rename=20HTTPS=20forge.ops.eblu.me=20?= =?UTF-8?q?=E2=86=92=20forge.eblu.me=20across=20codebase?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update all HTTPS references to use the new public domain. This touches workflows, ArgoCD manifests, Ansible, mise-tasks, NixOS config, and documentation (~29 files). Deliberately kept as forge.ops.eblu.me: - SSH repoURLs in argocd/apps/ (SSH stays tailnet-only) - containers/*/Dockerfile and *.nix (internal CI efficiency) - Caddy services table in routing.md - Internal URL references in forgejo.md Co-Authored-By: Claude Opus 4.6 --- .forgejo/workflows/build-blumeops.yaml | 20 +++++++++---------- .forgejo/workflows/cv-deploy.yaml | 6 +++--- ansible/playbooks/ringtail.yml | 2 +- .../forgejo_actions_secrets/defaults/main.yml | 2 +- argocd/apps/forgejo-runner.yaml | 2 +- argocd/manifests/cv/deployment.yaml | 2 +- argocd/manifests/docs/deployment.yaml | 2 +- .../manifests/forgejo-runner/deployment.yaml | 2 +- argocd/manifests/homepage/services.yaml | 4 ++-- .../authentik/build-authentik-from-source.md | 4 ++-- .../configuration/update-documentation.md | 2 +- .../deployment/build-container-image.md | 2 +- .../create-release-artifact-workflow.md | 6 +++--- .../how-to/plans/completed/adopt-dagger-ci.md | 14 ++++++------- .../how-to/plans/migrate-forgejo-from-brew.md | 8 ++++---- docs/how-to/plans/upstream-fork-strategy.md | 2 +- docs/how-to/zot/register-zot-oidc-client.md | 2 +- docs/index.md | 2 +- docs/quartz.layout.ts | 2 +- docs/reference/services/forgejo.md | 2 +- docs/tutorials/contributing.md | 2 +- mise-tasks/branch-cleanup | 2 +- mise-tasks/container-build-and-release | 2 +- mise-tasks/docs-mikado | 2 +- mise-tasks/mirror-create | 4 ++-- mise-tasks/pr-comments | 2 +- mise-tasks/runner-logs | 2 +- mise-tasks/services-check | 2 +- nixos/ringtail/configuration.nix | 2 +- 29 files changed, 54 insertions(+), 54 deletions(-) diff --git a/.forgejo/workflows/build-blumeops.yaml b/.forgejo/workflows/build-blumeops.yaml index 616d2cf..e6fe92d 100644 --- a/.forgejo/workflows/build-blumeops.yaml +++ b/.forgejo/workflows/build-blumeops.yaml @@ -11,7 +11,7 @@ # 3. The workflow creates a release with attached artifacts # # Documentation asset URL: -# https://forge.ops.eblu.me/eblume/blumeops/releases/download//docs-.tar.gz +# https://forge.eblu.me/eblume/blumeops/releases/download//docs-.tar.gz name: Build BlumeOps @@ -46,7 +46,7 @@ jobs: # Fetch latest release echo "Fetching latest release..." - LATEST=$(curl -s "https://forge.ops.eblu.me/api/v1/repos/eblume/blumeops/releases/latest" | jq -r '.tag_name // empty' || true) + LATEST=$(curl -s "https://forge.eblu.me/api/v1/repos/eblume/blumeops/releases/latest" | jq -r '.tag_name // empty' || true) if [ -z "$LATEST" ]; then LATEST="v0.0.0" @@ -94,9 +94,9 @@ jobs: esac # Check if this version already exists - if curl -sf "https://forge.ops.eblu.me/api/v1/repos/eblume/blumeops/releases/tags/$VERSION" > /dev/null 2>&1; then + if curl -sf "https://forge.eblu.me/api/v1/repos/eblume/blumeops/releases/tags/$VERSION" > /dev/null 2>&1; then echo "Error: Release $VERSION already exists" - echo "See: https://forge.ops.eblu.me/eblume/blumeops/releases/tag/$VERSION" + echo "See: https://forge.eblu.me/eblume/blumeops/releases/tag/$VERSION" exit 1 fi @@ -181,7 +181,7 @@ jobs: echo "Download \`$TARBALL\` and configure the quartz container with:" echo "" echo "\`\`\`" - echo "DOCS_RELEASE_URL=https://forge.ops.eblu.me/eblume/blumeops/releases/download/$VERSION/$TARBALL" + echo "DOCS_RELEASE_URL=https://forge.eblu.me/eblume/blumeops/releases/download/$VERSION/$TARBALL" echo "\`\`\`" } > /tmp/release_body.txt @@ -197,7 +197,7 @@ jobs: -H "Content-Type: application/json" \ -H "Authorization: token $GITHUB_TOKEN" \ -d "$RELEASE_DATA" \ - "https://forge.ops.eblu.me/api/v1/repos/eblume/blumeops/releases") + "https://forge.eblu.me/api/v1/repos/eblume/blumeops/releases") echo "API Response: $RELEASE_RESPONSE" @@ -217,7 +217,7 @@ jobs: -H "Content-Type: application/gzip" \ -H "Authorization: token $GITHUB_TOKEN" \ --data-binary "@$TARBALL" \ - "https://forge.ops.eblu.me/api/v1/repos/eblume/blumeops/releases/$RELEASE_ID/assets?name=$TARBALL") + "https://forge.eblu.me/api/v1/repos/eblume/blumeops/releases/$RELEASE_ID/assets?name=$TARBALL") echo "Upload Response: $UPLOAD_RESPONSE" echo "" @@ -228,7 +228,7 @@ jobs: VERSION="${{ steps.version.outputs.version }}" TARBALL="docs-${VERSION}.tar.gz" DEPLOYMENT_FILE="argocd/manifests/docs/deployment.yaml" - RELEASE_URL="https://forge.ops.eblu.me/eblume/blumeops/releases/download/${VERSION}/${TARBALL}" + RELEASE_URL="https://forge.eblu.me/eblume/blumeops/releases/download/${VERSION}/${TARBALL}" echo "Updating $DEPLOYMENT_FILE with new release URL..." yq -i "(.spec.template.spec.containers[0].env[] | select(.name == \"DOCS_RELEASE_URL\")).value = \"${RELEASE_URL}\"" "$DEPLOYMENT_FILE" @@ -307,7 +307,7 @@ jobs: echo "================================================" echo "" echo "Release URL:" - echo " https://forge.ops.eblu.me/eblume/blumeops/releases/tag/$VERSION" + echo " https://forge.eblu.me/eblume/blumeops/releases/tag/$VERSION" echo "" echo "Asset URL (for DOCS_RELEASE_URL ConfigMap):" - echo " https://forge.ops.eblu.me/eblume/blumeops/releases/download/$VERSION/$TARBALL" + echo " https://forge.eblu.me/eblume/blumeops/releases/download/$VERSION/$TARBALL" diff --git a/.forgejo/workflows/cv-deploy.yaml b/.forgejo/workflows/cv-deploy.yaml index 983154b..b03b925 100644 --- a/.forgejo/workflows/cv-deploy.yaml +++ b/.forgejo/workflows/cv-deploy.yaml @@ -30,7 +30,7 @@ jobs: if [ "$INPUT_VERSION" = "latest" ]; then echo "Resolving latest CV package version..." - VERSION=$(curl -s "https://forge.ops.eblu.me/api/v1/packages/eblume?type=generic&q=cv" \ + VERSION=$(curl -s "https://forge.eblu.me/api/v1/packages/eblume?type=generic&q=cv" \ | jq -r '[.[] | select(.name == "cv")] | sort_by(.version) | last | .version // empty') if [ -z "$VERSION" ]; then @@ -48,7 +48,7 @@ jobs: # Verify the package exists TARBALL="cv-${VERSION}.tar.gz" - PACKAGE_URL="https://forge.ops.eblu.me/api/packages/eblume/generic/cv/${VERSION}/${TARBALL}" + PACKAGE_URL="https://forge.eblu.me/api/packages/eblume/generic/cv/${VERSION}/${TARBALL}" if ! curl -fsSL --head "$PACKAGE_URL" > /dev/null 2>&1; then echo "Error: Package not found at $PACKAGE_URL" echo "Run the 'Release CV' workflow in the cv repo first." @@ -65,7 +65,7 @@ jobs: VERSION="${{ steps.version.outputs.version }}" TARBALL="cv-${VERSION}.tar.gz" DEPLOYMENT_FILE="argocd/manifests/cv/deployment.yaml" - RELEASE_URL="https://forge.ops.eblu.me/api/packages/eblume/generic/cv/${VERSION}/${TARBALL}" + RELEASE_URL="https://forge.eblu.me/api/packages/eblume/generic/cv/${VERSION}/${TARBALL}" echo "Updating $DEPLOYMENT_FILE with CV_RELEASE_URL..." yq -i "(.spec.template.spec.containers[0].env[] | select(.name == \"CV_RELEASE_URL\")).value = \"${RELEASE_URL}\"" "$DEPLOYMENT_FILE" diff --git a/ansible/playbooks/ringtail.yml b/ansible/playbooks/ringtail.yml index b05d67a..ee5604b 100644 --- a/ansible/playbooks/ringtail.yml +++ b/ansible/playbooks/ringtail.yml @@ -57,7 +57,7 @@ tasks: - name: Ensure blumeops repo is present ansible.builtin.git: - repo: "https://forge.ops.eblu.me/eblume/blumeops.git" + repo: "https://forge.eblu.me/eblume/blumeops.git" dest: /etc/blumeops version: "{{ ringtail_commit | default('main') }}" force: true diff --git a/ansible/roles/forgejo_actions_secrets/defaults/main.yml b/ansible/roles/forgejo_actions_secrets/defaults/main.yml index 943b8af..7742ecf 100644 --- a/ansible/roles/forgejo_actions_secrets/defaults/main.yml +++ b/ansible/roles/forgejo_actions_secrets/defaults/main.yml @@ -4,7 +4,7 @@ # This role syncs repository-level Actions secrets from 1Password to Forgejo # via the Forgejo API. -forgejo_actions_secrets_api_url: "https://forge.ops.eblu.me/api/v1" +forgejo_actions_secrets_api_url: "https://forge.eblu.me/api/v1" forgejo_actions_secrets_owner: eblume # Secrets to sync per repo. diff --git a/argocd/apps/forgejo-runner.yaml b/argocd/apps/forgejo-runner.yaml index 5bca762..7addb40 100644 --- a/argocd/apps/forgejo-runner.yaml +++ b/argocd/apps/forgejo-runner.yaml @@ -6,7 +6,7 @@ metadata: spec: project: default source: - repoURL: https://forge.ops.eblu.me/eblume/blumeops.git + repoURL: https://forge.eblu.me/eblume/blumeops.git targetRevision: main path: argocd/manifests/forgejo-runner destination: diff --git a/argocd/manifests/cv/deployment.yaml b/argocd/manifests/cv/deployment.yaml index ba969fc..2e929b4 100644 --- a/argocd/manifests/cv/deployment.yaml +++ b/argocd/manifests/cv/deployment.yaml @@ -27,7 +27,7 @@ spec: name: http env: - name: CV_RELEASE_URL - value: "https://forge.ops.eblu.me/api/packages/eblume/generic/cv/v1.0.3/cv-v1.0.3.tar.gz" + value: "https://forge.eblu.me/api/packages/eblume/generic/cv/v1.0.3/cv-v1.0.3.tar.gz" resources: requests: memory: "64Mi" diff --git a/argocd/manifests/docs/deployment.yaml b/argocd/manifests/docs/deployment.yaml index 66506c4..af8b655 100644 --- a/argocd/manifests/docs/deployment.yaml +++ b/argocd/manifests/docs/deployment.yaml @@ -27,7 +27,7 @@ spec: name: http env: - name: DOCS_RELEASE_URL - value: "https://forge.ops.eblu.me/eblume/blumeops/releases/download/v1.12.1/docs-v1.12.1.tar.gz" + value: "https://forge.eblu.me/eblume/blumeops/releases/download/v1.12.1/docs-v1.12.1.tar.gz" resources: requests: memory: "64Mi" diff --git a/argocd/manifests/forgejo-runner/deployment.yaml b/argocd/manifests/forgejo-runner/deployment.yaml index 4f67672..6e57137 100644 --- a/argocd/manifests/forgejo-runner/deployment.yaml +++ b/argocd/manifests/forgejo-runner/deployment.yaml @@ -25,7 +25,7 @@ spec: - name: DOCKER_HOST value: tcp://localhost:2375 - name: FORGEJO_URL - value: "https://forge.ops.eblu.me" + value: "https://forge.eblu.me" - name: RUNNER_NAME value: "k8s-runner" - name: RUNNER_LABELS diff --git a/argocd/manifests/homepage/services.yaml b/argocd/manifests/homepage/services.yaml index 0f35e9c..fcde74f 100644 --- a/argocd/manifests/homepage/services.yaml +++ b/argocd/manifests/homepage/services.yaml @@ -1,11 +1,11 @@ - Host Services: - Forgejo: - href: https://forge.ops.eblu.me + href: https://forge.eblu.me icon: forgejo description: Git forge widget: type: gitea - url: https://forge.ops.eblu.me + url: https://forge.eblu.me key: "{{HOMEPAGE_VAR_FORGEJO_API_KEY}}" - Registry: href: https://registry.ops.eblu.me diff --git a/docs/how-to/authentik/build-authentik-from-source.md b/docs/how-to/authentik/build-authentik-from-source.md index 169a0ed..d4411e3 100644 --- a/docs/how-to/authentik/build-authentik-from-source.md +++ b/docs/how-to/authentik/build-authentik-from-source.md @@ -36,8 +36,8 @@ The `ak` wrapper script in `default.nix` sets PATH/VIRTUAL_ENV and delegates to ## Source All derivations fetch from forge mirrors for supply chain control: -- https://forge.ops.eblu.me/mirrors/authentik (upstream: `goauthentik/authentik`) -- https://forge.ops.eblu.me/mirrors/authentik-client-go (upstream: `goauthentik/client-go`) +- https://forge.eblu.me/mirrors/authentik (upstream: `goauthentik/authentik`) +- https://forge.eblu.me/mirrors/authentik-client-go (upstream: `goauthentik/client-go`) Version and hashes are centralized in `containers/authentik/sources.nix`. diff --git a/docs/how-to/configuration/update-documentation.md b/docs/how-to/configuration/update-documentation.md index fef8ccf..7c98e24 100644 --- a/docs/how-to/configuration/update-documentation.md +++ b/docs/how-to/configuration/update-documentation.md @@ -20,7 +20,7 @@ After merging documentation changes to main: 2. Select version bump type (patch/minor/major) or enter a specific version 3. The workflow builds, releases, and deploys automatically -Direct link: https://forge.ops.eblu.me/eblume/blumeops/actions?workflow=build-blumeops.yaml +Direct link: https://forge.eblu.me/eblume/blumeops/actions?workflow=build-blumeops.yaml ## What the Workflow Does diff --git a/docs/how-to/deployment/build-container-image.md b/docs/how-to/deployment/build-container-image.md index 9f6d7b8..b3b9cbe 100644 --- a/docs/how-to/deployment/build-container-image.md +++ b/docs/how-to/deployment/build-container-image.md @@ -93,7 +93,7 @@ Container image tags include the git commit SHA they were built from (e.g. `v3.9 **The rule:** Production manifests must reference images built from a commit on main. After merging a PR that changed `containers//`: 1. The merge to main automatically triggers a rebuild (the `build-container.yaml` / `build-container-nix.yaml` workflows fire on pushes to `main` that touch `containers/**`) -2. Wait for the workflow to complete — check at `https://forge.ops.eblu.me/eblume/blumeops/actions` +2. Wait for the workflow to complete — check at `https://forge.eblu.me/eblume/blumeops/actions` 3. Find the new main-SHA tag: ```bash mise run container-list diff --git a/docs/how-to/deployment/create-release-artifact-workflow.md b/docs/how-to/deployment/create-release-artifact-workflow.md index 80e0308..63c217e 100644 --- a/docs/how-to/deployment/create-release-artifact-workflow.md +++ b/docs/how-to/deployment/create-release-artifact-workflow.md @@ -48,16 +48,16 @@ The upload step uses `FORGE_TOKEN`: -X PUT \ -H "Authorization: token $FORGE_TOKEN" \ --upload-file "./$TARBALL" \ - "https://forge.ops.eblu.me/api/packages/eblume/generic//${VERSION}/${TARBALL}" + "https://forge.eblu.me/api/packages/eblume/generic//${VERSION}/${TARBALL}" ``` ## 3. Link the package to the repo -After the first successful upload, the package appears under your **user-level** packages at `https://forge.ops.eblu.me/eblume/-/packages` but is not yet linked to the repo. +After the first successful upload, the package appears under your **user-level** packages at `https://forge.eblu.me/eblume/-/packages` but is not yet linked to the repo. To link it: -1. Go to `https://forge.ops.eblu.me/eblume/-/packages` +1. Go to `https://forge.eblu.me/eblume/-/packages` 2. Click the package name 3. Click **Settings** 4. Under **Link this package to a repository**, select the repo diff --git a/docs/how-to/plans/completed/adopt-dagger-ci.md b/docs/how-to/plans/completed/adopt-dagger-ci.md index a3ec8ce..aabae1d 100644 --- a/docs/how-to/plans/completed/adopt-dagger-ci.md +++ b/docs/how-to/plans/completed/adopt-dagger-ci.md @@ -222,12 +222,12 @@ Migrate `build-blumeops.yaml` to use Dagger for the build logic and switch from **Current:** Docs tarball uploaded as a Forgejo release asset. ``` -https://forge.ops.eblu.me/eblume/blumeops/releases/download/v1.5.2/docs-v1.5.2.tar.gz +https://forge.eblu.me/eblume/blumeops/releases/download/v1.5.2/docs-v1.5.2.tar.gz ``` **New:** Docs tarball uploaded to Forgejo generic packages registry. ``` -https://forge.ops.eblu.me/api/packages/eblume/generic/blumeops-docs/v1.6.0/docs-v1.6.0.tar.gz +https://forge.eblu.me/api/packages/eblume/generic/blumeops-docs/v1.6.0/docs-v1.6.0.tar.gz ``` This decouples the docs artifact from git releases while keeping the versioned URL pattern. Forgejo releases can still be created for changelog/announcement purposes without carrying the tarball. @@ -290,13 +290,13 @@ async def upload_docs( async with httpx.AsyncClient() as client: with open(f"/tmp/docs-{version}.tar.gz", "rb") as f: resp = await client.put( - f"https://forge.ops.eblu.me/api/packages/eblume/generic/" + f"https://forge.eblu.me/api/packages/eblume/generic/" f"blumeops-docs/{version}/docs-{version}.tar.gz", headers={"Authorization": f"token {token}"}, content=f.read(), ) resp.raise_for_status() - return f"https://forge.ops.eblu.me/api/packages/eblume/generic/blumeops-docs/{version}/docs-{version}.tar.gz" + return f"https://forge.eblu.me/api/packages/eblume/generic/blumeops-docs/{version}/docs-{version}.tar.gz" @function async def release_docs( @@ -388,7 +388,7 @@ jobs: - name: Update manifest and commit run: | VERSION="${{ steps.version.outputs.version }}" - URL="https://forge.ops.eblu.me/api/packages/eblume/generic/blumeops-docs/${VERSION}/docs-${VERSION}.tar.gz" + URL="https://forge.eblu.me/api/packages/eblume/generic/blumeops-docs/${VERSION}/docs-${VERSION}.tar.gz" sed -i "s|value: \"https://.*\"|value: \"${URL}\"|" \ argocd/manifests/docs/deployment.yaml git config user.name "Forgejo Actions" @@ -405,11 +405,11 @@ The quartz container's `DOCS_RELEASE_URL` env var in `argocd/manifests/docs/depl ```yaml # Before (Forgejo releases): - name: DOCS_RELEASE_URL - value: "https://forge.ops.eblu.me/eblume/blumeops/releases/download/v1.5.2/docs-v1.5.2.tar.gz" + value: "https://forge.eblu.me/eblume/blumeops/releases/download/v1.5.2/docs-v1.5.2.tar.gz" # After (Forgejo generic packages): - name: DOCS_RELEASE_URL - value: "https://forge.ops.eblu.me/api/packages/eblume/generic/blumeops-docs/v1.6.0/docs-v1.6.0.tar.gz" + value: "https://forge.eblu.me/api/packages/eblume/generic/blumeops-docs/v1.6.0/docs-v1.6.0.tar.gz" ``` The quartz container's `start.sh` already downloads from `DOCS_RELEASE_URL` via curl — no container changes needed, just the URL format changes. diff --git a/docs/how-to/plans/migrate-forgejo-from-brew.md b/docs/how-to/plans/migrate-forgejo-from-brew.md index 3a77154..4daad6b 100644 --- a/docs/how-to/plans/migrate-forgejo-from-brew.md +++ b/docs/how-to/plans/migrate-forgejo-from-brew.md @@ -32,7 +32,7 @@ https://codeberg.org/forgejo/forgejo.git Add the forge mirror as a secondary remote for convenience and backup: ``` -https://forge.ops.eblu.me/mirrors/forgejo.git +https://forge.eblu.me/mirrors/forgejo.git ``` ## One-Time Migration Steps @@ -48,7 +48,7 @@ ssh indri 'git clone https://codeberg.org/forgejo/forgejo.git ~/code/3rd/forgejo ### 2. Add Forge Mirror as Secondary Remote ```fish -ssh indri 'cd ~/code/3rd/forgejo && git remote add forge https://forge.ops.eblu.me/mirrors/forgejo.git' +ssh indri 'cd ~/code/3rd/forgejo && git remote add forge https://forge.eblu.me/mirrors/forgejo.git' ``` ### 3. Check Out the Desired Version Tag @@ -155,7 +155,7 @@ Replace brew install/start with binary-check + LaunchAgent pattern (matching `an # ssh indri 'git clone https://codeberg.org/forgejo/forgejo.git ~/code/3rd/forgejo' # # 2. Add forge mirror as secondary remote: -# ssh indri 'cd ~/code/3rd/forgejo && git remote add forge https://forge.ops.eblu.me/mirrors/forgejo.git' +# ssh indri 'cd ~/code/3rd/forgejo && git remote add forge https://forge.eblu.me/mirrors/forgejo.git' # # 3. Set up Go and Node via mise: # ssh indri 'cd ~/code/3rd/forgejo && mise use go@1.24 node@20' @@ -275,7 +275,7 @@ No changes needed — paths already flow through variables in `defaults/main.yml After running the migration and Ansible: - [ ] `ssh indri 'launchctl list mcquack.eblume.forgejo'` — shows running -- [ ] `curl https://forge.ops.eblu.me/api/v1/version` — returns JSON with version +- [ ] `curl https://forge.eblu.me/api/v1/version` — returns JSON with version - [ ] Git clone over SSH: `git clone ssh://forgejo@forge.ops.eblu.me:2222/eblume/blumeops.git /tmp/test-clone` - [ ] Git push works on an existing clone - [ ] Ansible dry-run is clean: `mise run provision-indri -- --tags forgejo --check --diff` diff --git a/docs/how-to/plans/upstream-fork-strategy.md b/docs/how-to/plans/upstream-fork-strategy.md index b2ce412..e2af8c8 100644 --- a/docs/how-to/plans/upstream-fork-strategy.md +++ b/docs/how-to/plans/upstream-fork-strategy.md @@ -209,7 +209,7 @@ This fork directly supports the [[adopt-dagger-ci]] plan. Once the fork exists, # After (using the BlumeOps fork): .with_exec(["git", "clone", "--depth=1", "--branch=blumeops", - "https://forge.ops.eblu.me/mirrors/quartz.git", "/tmp/quartz"]) + "https://forge.eblu.me/mirrors/quartz.git", "/tmp/quartz"]) ``` This means the `build-blumeops.yaml` workflow automatically picks up fork customizations (like `last-reviewed` rendering) when building docs — no separate integration step needed. Local iteration via `dagger call build-docs` also uses the fork, so you can test Quartz customizations against actual BlumeOps content before pushing. diff --git a/docs/how-to/zot/register-zot-oidc-client.md b/docs/how-to/zot/register-zot-oidc-client.md index ff7d805..b3696d0 100644 --- a/docs/how-to/zot/register-zot-oidc-client.md +++ b/docs/how-to/zot/register-zot-oidc-client.md @@ -12,7 +12,7 @@ tags: Register a zot OAuth2 provider and application in Authentik via blueprint, following the same pattern as Grafana and Forgejo. -Completed in PR [#236](https://forge.ops.eblu.me/eblume/blumeops/pulls/236). +Completed in PR [#236](https://forge.eblu.me/eblume/blumeops/pulls/236). ## What Was Done diff --git a/docs/index.md b/docs/index.md index 6d7822d..306e73c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -16,7 +16,7 @@ infrastructure. BlumeOps is my personal homelab infrastructure managed entirely through code. Everything lives in a [single git repository](https://github.com/eblume/blumeops), from service configs to -deployment automation. Even the [[forgejo]] instance that [hosts this repo](https://forge.ops.eblu.me/eblume/blumeops) +deployment automation. Even the [[forgejo]] instance that [hosts this repo](https://forge.eblu.me/eblume/blumeops) is defined within it, making BlumeOps fully self-hosting. It's a digital life raft I built for myself as I went, and you can see it all from within your editor of choice. (I recommend vim.) diff --git a/docs/quartz.layout.ts b/docs/quartz.layout.ts index c3f7e6d..cba29ec 100644 --- a/docs/quartz.layout.ts +++ b/docs/quartz.layout.ts @@ -14,7 +14,7 @@ export const sharedPageComponents: SharedLayout = { footer: Component.Footer({ links: { "GitHub": "https://github.com/eblume/blumeops", - "Forge": "https://forge.ops.eblu.me/eblume/blumeops", + "Forge": "https://forge.eblu.me/eblume/blumeops", }, }), } diff --git a/docs/reference/services/forgejo.md b/docs/reference/services/forgejo.md index 504107e..1f56029 100644 --- a/docs/reference/services/forgejo.md +++ b/docs/reference/services/forgejo.md @@ -72,7 +72,7 @@ mise run provision-indri -- --tags forgejo_actions_secrets The Ansible role authenticates to the Forgejo API using a Personal Access Token (PAT). This PAT must be created manually: -1. Go to https://forge.ops.eblu.me/user/settings/applications +1. Go to https://forge.eblu.me/user/settings/applications 2. Create a new token with `write:repository` scope 3. Store it in 1Password → "Forgejo Secrets" item → `api-token` field diff --git a/docs/tutorials/contributing.md b/docs/tutorials/contributing.md index 61d32e8..cddafea 100644 --- a/docs/tutorials/contributing.md +++ b/docs/tutorials/contributing.md @@ -16,7 +16,7 @@ This tutorial walks through making your first contribution to BluemeOps - from u Before contributing, you'll need: - Access to the [[tailscale|Tailscale]] network (request from Erich) -- SSH key added to [[forgejo|Forgejo]] (https://forge.ops.eblu.me) +- SSH key added to [[forgejo|Forgejo]] (https://forge.eblu.me) - Development tools installed (see below) ## Tooling Setup diff --git a/mise-tasks/branch-cleanup b/mise-tasks/branch-cleanup index 85be4e6..88e9152 100755 --- a/mise-tasks/branch-cleanup +++ b/mise-tasks/branch-cleanup @@ -44,7 +44,7 @@ from rich.console import Console from rich.table import Table PROTECTED_BRANCHES = {"main", "master"} -FORGE_API = "https://forge.ops.eblu.me/api/v1" +FORGE_API = "https://forge.eblu.me/api/v1" REPO_OWNER = "eblume" REPO_NAME = "blumeops" OP_TOKEN_REF = "op://vg6xf6vvfmoh5hqjjhlhbeoaie/w3663ffnvkewbftncqxtcpeavy/api-token" diff --git a/mise-tasks/container-build-and-release b/mise-tasks/container-build-and-release index 2226550..dd78923 100755 --- a/mise-tasks/container-build-and-release +++ b/mise-tasks/container-build-and-release @@ -21,7 +21,7 @@ import httpx import typer REGISTRY = "registry.ops.eblu.me" -FORGE_URL = "https://forge.ops.eblu.me" +FORGE_URL = "https://forge.eblu.me" FORGE_API = f"{FORGE_URL}/api/v1" REPO = "eblume/blumeops" FORGE_ACTIONS = f"{FORGE_URL}/{REPO}/actions" diff --git a/mise-tasks/docs-mikado b/mise-tasks/docs-mikado index e28c3f1..17a363d 100755 --- a/mise-tasks/docs-mikado +++ b/mise-tasks/docs-mikado @@ -238,7 +238,7 @@ def classify_branch_position(commits: list[dict]) -> str: return "unknown" -FORGE_API = "https://forge.ops.eblu.me/api/v1" +FORGE_API = "https://forge.eblu.me/api/v1" def find_pr_for_branch(branch: str) -> dict | None: diff --git a/mise-tasks/mirror-create b/mise-tasks/mirror-create index 75e0d3e..b0e82a0 100755 --- a/mise-tasks/mirror-create +++ b/mise-tasks/mirror-create @@ -6,7 +6,7 @@ #USAGE flag "--dry-run" help="Show what would be done without creating" set -euo pipefail -FORGE_API="https://forge.ops.eblu.me/api/v1" +FORGE_API="https://forge.eblu.me/api/v1" ORG="mirrors" OP_TOKEN_REF="op://blumeops/w3663ffnvkewbftncqxtcpeavy/api-token" OP_GITHUB_PAT_REF="op://blumeops/w3663ffnvkewbftncqxtcpeavy/github-mirror-pat" @@ -72,7 +72,7 @@ http_code=$(curl -s -o /tmp/mirror-create-response.json -w "%{http_code}" \ -d "$payload") if [[ "$http_code" == "201" ]]; then - echo "Created mirror: https://forge.ops.eblu.me/${ORG}/${repo_name}" + echo "Created mirror: https://forge.eblu.me/${ORG}/${repo_name}" else echo "Error (HTTP $http_code):" cat /tmp/mirror-create-response.json diff --git a/mise-tasks/pr-comments b/mise-tasks/pr-comments index 0f6b840..9c33f9d 100755 --- a/mise-tasks/pr-comments +++ b/mise-tasks/pr-comments @@ -20,7 +20,7 @@ import httpx from rich.console import Console from rich.text import Text -FORGE_API_BASE = "https://forge.ops.eblu.me/api/v1" +FORGE_API_BASE = "https://forge.eblu.me/api/v1" REPO_OWNER = "eblume" REPO_NAME = "blumeops" diff --git a/mise-tasks/runner-logs b/mise-tasks/runner-logs index 536f1fe..22e4640 100755 --- a/mise-tasks/runner-logs +++ b/mise-tasks/runner-logs @@ -27,7 +27,7 @@ import typer from rich.console import Console from rich.table import Table -FORGE_API = "https://forge.ops.eblu.me/api/v1" +FORGE_API = "https://forge.eblu.me/api/v1" REPO = "eblume/blumeops" ACTIONS_LOG_DIR = "/opt/homebrew/var/forgejo/data/actions_log/eblume/blumeops" diff --git a/mise-tasks/services-check b/mise-tasks/services-check index d09b237..dce18ee 100755 --- a/mise-tasks/services-check +++ b/mise-tasks/services-check @@ -70,7 +70,7 @@ check_http "Prometheus" "https://prometheus.ops.eblu.me/-/healthy" check_http "Loki" "https://loki.ops.eblu.me/ready" check_http "Grafana" "https://grafana.ops.eblu.me/api/health" check_http "ArgoCD" "https://argocd.ops.eblu.me/healthz" -check_http "Forgejo" "https://forge.ops.eblu.me/" +check_http "Forgejo" "https://forge.eblu.me/" check_http "Zot Registry" "https://registry.ops.eblu.me/v2/_catalog" check_http "Kiwix" "https://kiwix.ops.eblu.me/" check_http "Miniflux" "https://feed.ops.eblu.me/healthcheck" diff --git a/nixos/ringtail/configuration.nix b/nixos/ringtail/configuration.nix index 5a0035b..562cfdb 100644 --- a/nixos/ringtail/configuration.nix +++ b/nixos/ringtail/configuration.nix @@ -496,7 +496,7 @@ in instances.nix_container_builder = { enable = true; name = "ringtail-nix-builder"; - url = "https://forge.ops.eblu.me"; + url = "https://forge.eblu.me"; tokenFile = "/etc/forgejo-runner/token.env"; labels = [ "nix-container-builder:host" ]; hostPackages = with pkgs; [ -- 2.50.1 (Apple Git-155) From 17dac7ea278c049a4c3cd672d383ad87ab86a069 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Tue, 3 Mar 2026 08:17:24 -0800 Subject: [PATCH 09/14] =?UTF-8?q?Fix=20forge=20ExternalName=20=E2=86=92=20?= =?UTF-8?q?ClusterIP+Endpoints=20for=20Tailscale=20Ingress?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Tailscale ingress operator requires backends with a ClusterIP. ExternalName services don't have one, causing "invalid ClusterIP" errors. Replace with a headless Service + manual Endpoints pointing to indri's Tailscale IP (100.98.163.89). Co-Authored-By: Claude Opus 4.6 --- .../svc-forge-external.yaml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/argocd/manifests/tailscale-operator/svc-forge-external.yaml b/argocd/manifests/tailscale-operator/svc-forge-external.yaml index 2812acf..9dbcade 100644 --- a/argocd/manifests/tailscale-operator/svc-forge-external.yaml +++ b/argocd/manifests/tailscale-operator/svc-forge-external.yaml @@ -1,13 +1,28 @@ --- +# Headless service + manual Endpoints to route to Forgejo on indri. +# ExternalName services don't have a ClusterIP, which the Tailscale +# ingress operator requires. This pattern creates a real ClusterIP +# that forwards to indri's Tailscale IP. apiVersion: v1 kind: Service metadata: name: forge-external namespace: tailscale spec: - type: ExternalName - externalName: indri.tail8d86e.ts.net ports: - name: http port: 3001 protocol: TCP +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: forge-external + namespace: tailscale +subsets: + - addresses: + - ip: 100.98.163.89 + ports: + - name: http + port: 3001 + protocol: TCP -- 2.50.1 (Apple Git-155) From 27596757041f5d21175cc1e16fe2df36894116cd Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Tue, 3 Mar 2026 08:19:03 -0800 Subject: [PATCH 10/14] Split Endpoints into separate file for kustomize discovery Kustomize didn't pick up the Endpoints from the multi-document YAML in svc-forge-external.yaml. Split into a separate endpoints-forge.yaml and add to kustomization.yaml resources. Co-Authored-By: Claude Opus 4.6 --- .../tailscale-operator/endpoints-forge.yaml | 15 +++++++++++++++ .../tailscale-operator/kustomization.yaml | 1 + .../svc-forge-external.yaml | 19 +++---------------- 3 files changed, 19 insertions(+), 16 deletions(-) create mode 100644 argocd/manifests/tailscale-operator/endpoints-forge.yaml diff --git a/argocd/manifests/tailscale-operator/endpoints-forge.yaml b/argocd/manifests/tailscale-operator/endpoints-forge.yaml new file mode 100644 index 0000000..0eac8ab --- /dev/null +++ b/argocd/manifests/tailscale-operator/endpoints-forge.yaml @@ -0,0 +1,15 @@ +--- +# Manual Endpoints pointing to indri's Tailscale IP for the +# forge-external Service. Must match the Service name exactly. +apiVersion: v1 +kind: Endpoints +metadata: + name: forge-external + namespace: tailscale +subsets: + - addresses: + - ip: 100.98.163.89 + ports: + - name: http + port: 3001 + protocol: TCP diff --git a/argocd/manifests/tailscale-operator/kustomization.yaml b/argocd/manifests/tailscale-operator/kustomization.yaml index b38ee05..9b3f453 100644 --- a/argocd/manifests/tailscale-operator/kustomization.yaml +++ b/argocd/manifests/tailscale-operator/kustomization.yaml @@ -9,4 +9,5 @@ resources: - proxygroup-ingress.yaml - external-secret.yaml - svc-forge-external.yaml + - endpoints-forge.yaml - ingress-forge.yaml diff --git a/argocd/manifests/tailscale-operator/svc-forge-external.yaml b/argocd/manifests/tailscale-operator/svc-forge-external.yaml index 9dbcade..79a8645 100644 --- a/argocd/manifests/tailscale-operator/svc-forge-external.yaml +++ b/argocd/manifests/tailscale-operator/svc-forge-external.yaml @@ -1,8 +1,8 @@ --- -# Headless service + manual Endpoints to route to Forgejo on indri. +# ClusterIP service for Forgejo on indri. Paired with endpoints-forge.yaml +# which provides the actual routing to indri's Tailscale IP. # ExternalName services don't have a ClusterIP, which the Tailscale -# ingress operator requires. This pattern creates a real ClusterIP -# that forwards to indri's Tailscale IP. +# ingress operator requires. apiVersion: v1 kind: Service metadata: @@ -13,16 +13,3 @@ spec: - name: http port: 3001 protocol: TCP ---- -apiVersion: v1 -kind: Endpoints -metadata: - name: forge-external - namespace: tailscale -subsets: - - addresses: - - ip: 100.98.163.89 - ports: - - name: http - port: 3001 - protocol: TCP -- 2.50.1 (Apple Git-155) From dbfe7365d8544fbb087808a9eac80f1322d3255b Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Tue, 3 Mar 2026 08:24:24 -0800 Subject: [PATCH 11/14] Document that ArgoCD excludes Endpoints resources ArgoCD's resource.exclusions in argocd-cm skips all Endpoints objects (they're normally auto-managed by the control plane). The manual forge-external Endpoints must be applied directly with kubectl. Removed endpoints-forge.yaml from kustomization resources and added comments in both files explaining the situation and the apply command. Co-Authored-By: Claude Opus 4.6 --- argocd/manifests/tailscale-operator/endpoints-forge.yaml | 9 +++++++++ argocd/manifests/tailscale-operator/kustomization.yaml | 5 ++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/argocd/manifests/tailscale-operator/endpoints-forge.yaml b/argocd/manifests/tailscale-operator/endpoints-forge.yaml index 0eac8ab..e68aff9 100644 --- a/argocd/manifests/tailscale-operator/endpoints-forge.yaml +++ b/argocd/manifests/tailscale-operator/endpoints-forge.yaml @@ -1,6 +1,15 @@ --- # Manual Endpoints pointing to indri's Tailscale IP for the # forge-external Service. Must match the Service name exactly. +# +# NOTE: ArgoCD excludes all Endpoints resources (resource.exclusions in +# argocd-cm) because they are normally auto-managed by the control plane. +# This manual Endpoints is the exception — it must be applied directly +# with kubectl, not via ArgoCD. It is listed in kustomization.yaml for +# documentation purposes only; ArgoCD will silently skip it. +# +# kubectl --context=minikube-indri apply -f endpoints-forge.yaml +# apiVersion: v1 kind: Endpoints metadata: diff --git a/argocd/manifests/tailscale-operator/kustomization.yaml b/argocd/manifests/tailscale-operator/kustomization.yaml index 9b3f453..f1d6f89 100644 --- a/argocd/manifests/tailscale-operator/kustomization.yaml +++ b/argocd/manifests/tailscale-operator/kustomization.yaml @@ -9,5 +9,8 @@ resources: - proxygroup-ingress.yaml - external-secret.yaml - svc-forge-external.yaml - - endpoints-forge.yaml + # endpoints-forge.yaml is NOT managed by ArgoCD — Endpoints are globally + # excluded in argocd-cm resource.exclusions (too noisy for auto-managed + # Endpoints). Apply manually: + # kubectl --context=minikube-indri apply -f endpoints-forge.yaml - ingress-forge.yaml -- 2.50.1 (Apple Git-155) From 52f0308f1a24ff2ab207ecbb0870291c0eb1c5b4 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Tue, 3 Mar 2026 08:29:33 -0800 Subject: [PATCH 12/14] Fix fail2ban crash: disable default sshd jail, make non-fatal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Alpine's fail2ban ships with sshd jail enabled by default. Since there's no SSH server in the Fly.io container, fail2ban exits with an error looking for sshd logs — crashing the container via set -e. Disable the sshd jail explicitly and make fail2ban startup non-fatal since nginx rate limiting is the primary defense. Co-Authored-By: Claude Opus 4.6 --- fly/fail2ban/jail.d/forge.conf | 4 ++++ fly/start.sh | 10 +++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/fly/fail2ban/jail.d/forge.conf b/fly/fail2ban/jail.d/forge.conf index 7b0843f..50908b3 100644 --- a/fly/fail2ban/jail.d/forge.conf +++ b/fly/fail2ban/jail.d/forge.conf @@ -1,3 +1,7 @@ +# Disable the default sshd jail — no SSH server in this container +[sshd] +enabled = false + [forge-login] enabled = true filter = forge-login diff --git a/fly/start.sh b/fly/start.sh index 2ec7c48..5ec45db 100644 --- a/fly/start.sh +++ b/fly/start.sh @@ -20,9 +20,13 @@ nginx -g "daemon off;" & NGINX_PID=$! echo "Nginx started" -# Start fail2ban for login brute-force protection -fail2ban-server -b -echo "fail2ban started" +# Start fail2ban for login brute-force protection. +# Non-fatal — nginx rate limiting is the primary defense; fail2ban is additive. +if fail2ban-server -b; then + echo "fail2ban started" +else + echo "WARNING: fail2ban failed to start (nginx rate limiting still active)" +fi # Start Alloy for observability (logs → Loki, metrics → Prometheus) alloy run /etc/alloy/config.alloy \ -- 2.50.1 (Apple Git-155) From 771507e489ecf7ed04443d3997db460fbd5adf98 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Tue, 3 Mar 2026 08:31:39 -0800 Subject: [PATCH 13/14] Disable all default fail2ban jails, not just sshd Use [DEFAULT] enabled = false to disable all inherited jails globally. The previous fix only disabled sshd, but sshd-ddos (and potentially others) also fail looking for missing log files. Co-Authored-By: Claude Opus 4.6 --- fly/fail2ban/jail.d/forge.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fly/fail2ban/jail.d/forge.conf b/fly/fail2ban/jail.d/forge.conf index 50908b3..50be379 100644 --- a/fly/fail2ban/jail.d/forge.conf +++ b/fly/fail2ban/jail.d/forge.conf @@ -1,5 +1,5 @@ -# Disable the default sshd jail — no SSH server in this container -[sshd] +# Disable all default jails — this container has no SSH, mail, etc. +[DEFAULT] enabled = false [forge-login] -- 2.50.1 (Apple Git-155) From fedfdb1228a8224fe97fb5bd55e6893246818a34 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Tue, 3 Mar 2026 08:34:21 -0800 Subject: [PATCH 14/14] Remove Alpine's default SSH jails for fail2ban Alpine ships alpine-ssh.conf with sshd and sshd-ddos jails enabled. These fail on startup because there's no SSH server or /var/log/messages in the container. Remove the file after install instead of trying to override via [DEFAULT] (per-jail enabled=true beats DEFAULT). Co-Authored-By: Claude Opus 4.6 --- fly/Dockerfile | 3 ++- fly/fail2ban/jail.d/forge.conf | 4 ---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/fly/Dockerfile b/fly/Dockerfile index 09099c4..65135c1 100644 --- a/fly/Dockerfile +++ b/fly/Dockerfile @@ -9,7 +9,8 @@ COPY --from=docker.io/tailscale/tailscale:stable \ RUN mkdir -p /var/run/tailscale /var/lib/tailscale \ && apk add --no-cache iptables ip6tables \ && apk add --no-cache libc6-compat \ - && apk add --no-cache fail2ban + && apk add --no-cache fail2ban \ + && rm -f /etc/fail2ban/jail.d/alpine-ssh.conf # Copy Alloy binary from official image (Ubuntu-based, needs libc6-compat) COPY --from=docker.io/grafana/alloy:v1.13.1 \ diff --git a/fly/fail2ban/jail.d/forge.conf b/fly/fail2ban/jail.d/forge.conf index 50be379..7b0843f 100644 --- a/fly/fail2ban/jail.d/forge.conf +++ b/fly/fail2ban/jail.d/forge.conf @@ -1,7 +1,3 @@ -# Disable all default jails — this container has no SSH, mail, etc. -[DEFAULT] -enabled = false - [forge-login] enabled = true filter = forge-login -- 2.50.1 (Apple Git-155)