From a87c997ee13295d162177535d8f86c1d5f1be91e Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Tue, 3 Mar 2026 08:40:41 -0800 Subject: [PATCH] Expose Forgejo publicly at forge.eblu.me (#278) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Expose Forgejo publicly at `forge.eblu.me` via the Fly.io reverse proxy — the first dynamic, authenticated public-facing service. - **Forgejo hardening:** Domain changed to forge.eblu.me, SSH stays on forge.ops.eblu.me, reverse proxy trust headers configured, local registration locked to external-only (Authentik SSO) - **Tailscale Ingress:** ExternalName Service + Ingress in tailscale-operator creates forge.tail8d86e.ts.net endpoint - **Fly.io proxy:** nginx server block with rate-limited auth endpoints (3r/s), fail2ban with custom nginx-deny action, security headers, /swagger blocked, WebSocket support, 512m body limit - **Authentik:** OAuth callback updated to forge.eblu.me - **DNS/TLS:** CNAME record in Pulumi, cert in fly-setup - **Rename:** ~29 files updated from forge.ops.eblu.me to forge.eblu.me (HTTPS refs only; SSH, container builds, and Caddy table kept as-is) ## Deployment Order 1. `mise run provision-indri -- --tags forgejo` (config changes) 2. Verify forge.ops.eblu.me still works 3. `argocd app set tailscale-operator --revision feature/forge-public && argocd app sync tailscale-operator` 4. Verify `curl https://forge.tail8d86e.ts.net` 5. `cd fly && fly deploy` 6. Verify pre-DNS: `curl -H "Host: forge.eblu.me" https://blumeops-proxy.fly.dev/` 7. `fly certs add forge.eblu.me -a blumeops-proxy` 8. `argocd app set authentik --revision feature/forge-public && argocd app sync authentik` 9. `mise run dns-preview && mise run dns-up` 10. Full verification (see below) 11. Rehearse `mise run fly-shutoff` 12. After merge: reset ArgoCD revisions to main, re-sync ## Verification Checklist - [ ] forge.eblu.me loads, shows public repos - [ ] forge.ops.eblu.me still works from tailnet - [ ] SSH clone via forge.ops.eblu.me:2222 works - [ ] HTTPS clone via forge.eblu.me works - [ ] UI shows forge.eblu.me for HTTPS clone, forge.ops.eblu.me for SSH - [ ] /swagger returns 403 - [ ] Rapid login attempts trigger 429 rate limit - [ ] fail2ban bans after 5 failed logins in 10 minutes - [ ] ArgoCD can still sync (SSH unaffected) - [ ] `mise run fly-shutoff` stops all public traffic - [ ] `mise run services-check` passes Reviewed-on: https://forge.eblu.me/eblume/blumeops/pulls/278 --- .forgejo/workflows/build-blumeops.yaml | 20 ++-- .forgejo/workflows/cv-deploy.yaml | 6 +- ansible/playbooks/ringtail.yml | 2 +- ansible/roles/forgejo/defaults/main.yml | 4 +- ansible/roles/forgejo/templates/app.ini.j2 | 4 +- .../forgejo_actions_secrets/defaults/main.yml | 2 +- argocd/apps/forgejo-runner.yaml | 2 +- .../authentik/configmap-blueprint.yaml | 4 +- argocd/manifests/cv/deployment.yaml | 2 +- argocd/manifests/docs/deployment.yaml | 2 +- .../manifests/forgejo-runner/deployment.yaml | 2 +- argocd/manifests/homepage/services.yaml | 4 +- argocd/manifests/tailscale-operator/README.md | 3 - .../tailscale-operator/egress-forge.yaml | 23 ---- .../tailscale-operator/endpoints-forge.yaml | 24 ++++ .../tailscale-operator/ingress-forge.yaml | 20 ++++ .../tailscale-operator/kustomization.yaml | 7 +- .../svc-forge-external.yaml | 15 +++ .../feature-forge-public.feature.md | 1 + .../authentik/build-authentik-from-source.md | 4 +- .../configuration/expose-service-publicly.md | 46 ++++---- .../configuration/update-documentation.md | 2 +- .../deployment/build-container-image.md | 2 +- .../create-release-artifact-workflow.md | 6 +- .../how-to/plans/completed/adopt-dagger-ci.md | 14 +-- .../how-to/plans/migrate-forgejo-from-brew.md | 8 +- docs/how-to/plans/upstream-fork-strategy.md | 2 +- docs/how-to/zot/register-zot-oidc-client.md | 2 +- docs/index.md | 2 +- docs/quartz.layout.ts | 2 +- docs/reference/infrastructure/routing.md | 3 +- docs/reference/services/forgejo.md | 38 ++++--- docs/tutorials/contributing.md | 2 +- fly/Dockerfile | 8 +- fly/fail2ban/action.d/nginx-deny.conf | 14 +++ fly/fail2ban/filter.d/forge-login.conf | 10 ++ fly/fail2ban/jail.d/forge.conf | 8 ++ fly/nginx.conf | 104 ++++++++++++++++++ fly/start.sh | 11 ++ mise-tasks/branch-cleanup | 2 +- mise-tasks/container-build-and-release | 2 +- mise-tasks/docs-mikado | 2 +- mise-tasks/fly-setup | 1 + mise-tasks/mirror-create | 4 +- mise-tasks/pr-comments | 2 +- mise-tasks/runner-logs | 2 +- mise-tasks/services-check | 2 +- nixos/ringtail/configuration.nix | 2 +- pulumi/gandi/__main__.py | 10 ++ 49 files changed, 338 insertions(+), 126 deletions(-) delete mode 100644 argocd/manifests/tailscale-operator/egress-forge.yaml create mode 100644 argocd/manifests/tailscale-operator/endpoints-forge.yaml create mode 100644 argocd/manifests/tailscale-operator/ingress-forge.yaml create mode 100644 argocd/manifests/tailscale-operator/svc-forge-external.yaml create mode 100644 docs/changelog.d/feature-forge-public.feature.md create mode 100644 fly/fail2ban/action.d/nginx-deny.conf create mode 100644 fly/fail2ban/filter.d/forge-login.conf create mode 100644 fly/fail2ban/jail.d/forge.conf diff --git a/.forgejo/workflows/build-blumeops.yaml b/.forgejo/workflows/build-blumeops.yaml index 616d2cf..e6fe92d 100644 --- a/.forgejo/workflows/build-blumeops.yaml +++ b/.forgejo/workflows/build-blumeops.yaml @@ -11,7 +11,7 @@ # 3. The workflow creates a release with attached artifacts # # Documentation asset URL: -# https://forge.ops.eblu.me/eblume/blumeops/releases/download//docs-.tar.gz +# https://forge.eblu.me/eblume/blumeops/releases/download//docs-.tar.gz name: Build BlumeOps @@ -46,7 +46,7 @@ jobs: # Fetch latest release echo "Fetching latest release..." - LATEST=$(curl -s "https://forge.ops.eblu.me/api/v1/repos/eblume/blumeops/releases/latest" | jq -r '.tag_name // empty' || true) + LATEST=$(curl -s "https://forge.eblu.me/api/v1/repos/eblume/blumeops/releases/latest" | jq -r '.tag_name // empty' || true) if [ -z "$LATEST" ]; then LATEST="v0.0.0" @@ -94,9 +94,9 @@ jobs: esac # Check if this version already exists - if curl -sf "https://forge.ops.eblu.me/api/v1/repos/eblume/blumeops/releases/tags/$VERSION" > /dev/null 2>&1; then + if curl -sf "https://forge.eblu.me/api/v1/repos/eblume/blumeops/releases/tags/$VERSION" > /dev/null 2>&1; then echo "Error: Release $VERSION already exists" - echo "See: https://forge.ops.eblu.me/eblume/blumeops/releases/tag/$VERSION" + echo "See: https://forge.eblu.me/eblume/blumeops/releases/tag/$VERSION" exit 1 fi @@ -181,7 +181,7 @@ jobs: echo "Download \`$TARBALL\` and configure the quartz container with:" echo "" echo "\`\`\`" - echo "DOCS_RELEASE_URL=https://forge.ops.eblu.me/eblume/blumeops/releases/download/$VERSION/$TARBALL" + echo "DOCS_RELEASE_URL=https://forge.eblu.me/eblume/blumeops/releases/download/$VERSION/$TARBALL" echo "\`\`\`" } > /tmp/release_body.txt @@ -197,7 +197,7 @@ jobs: -H "Content-Type: application/json" \ -H "Authorization: token $GITHUB_TOKEN" \ -d "$RELEASE_DATA" \ - "https://forge.ops.eblu.me/api/v1/repos/eblume/blumeops/releases") + "https://forge.eblu.me/api/v1/repos/eblume/blumeops/releases") echo "API Response: $RELEASE_RESPONSE" @@ -217,7 +217,7 @@ jobs: -H "Content-Type: application/gzip" \ -H "Authorization: token $GITHUB_TOKEN" \ --data-binary "@$TARBALL" \ - "https://forge.ops.eblu.me/api/v1/repos/eblume/blumeops/releases/$RELEASE_ID/assets?name=$TARBALL") + "https://forge.eblu.me/api/v1/repos/eblume/blumeops/releases/$RELEASE_ID/assets?name=$TARBALL") echo "Upload Response: $UPLOAD_RESPONSE" echo "" @@ -228,7 +228,7 @@ jobs: VERSION="${{ steps.version.outputs.version }}" TARBALL="docs-${VERSION}.tar.gz" DEPLOYMENT_FILE="argocd/manifests/docs/deployment.yaml" - RELEASE_URL="https://forge.ops.eblu.me/eblume/blumeops/releases/download/${VERSION}/${TARBALL}" + RELEASE_URL="https://forge.eblu.me/eblume/blumeops/releases/download/${VERSION}/${TARBALL}" echo "Updating $DEPLOYMENT_FILE with new release URL..." yq -i "(.spec.template.spec.containers[0].env[] | select(.name == \"DOCS_RELEASE_URL\")).value = \"${RELEASE_URL}\"" "$DEPLOYMENT_FILE" @@ -307,7 +307,7 @@ jobs: echo "================================================" echo "" echo "Release URL:" - echo " https://forge.ops.eblu.me/eblume/blumeops/releases/tag/$VERSION" + echo " https://forge.eblu.me/eblume/blumeops/releases/tag/$VERSION" echo "" echo "Asset URL (for DOCS_RELEASE_URL ConfigMap):" - echo " https://forge.ops.eblu.me/eblume/blumeops/releases/download/$VERSION/$TARBALL" + echo " https://forge.eblu.me/eblume/blumeops/releases/download/$VERSION/$TARBALL" diff --git a/.forgejo/workflows/cv-deploy.yaml b/.forgejo/workflows/cv-deploy.yaml index 983154b..b03b925 100644 --- a/.forgejo/workflows/cv-deploy.yaml +++ b/.forgejo/workflows/cv-deploy.yaml @@ -30,7 +30,7 @@ jobs: if [ "$INPUT_VERSION" = "latest" ]; then echo "Resolving latest CV package version..." - VERSION=$(curl -s "https://forge.ops.eblu.me/api/v1/packages/eblume?type=generic&q=cv" \ + VERSION=$(curl -s "https://forge.eblu.me/api/v1/packages/eblume?type=generic&q=cv" \ | jq -r '[.[] | select(.name == "cv")] | sort_by(.version) | last | .version // empty') if [ -z "$VERSION" ]; then @@ -48,7 +48,7 @@ jobs: # Verify the package exists TARBALL="cv-${VERSION}.tar.gz" - PACKAGE_URL="https://forge.ops.eblu.me/api/packages/eblume/generic/cv/${VERSION}/${TARBALL}" + PACKAGE_URL="https://forge.eblu.me/api/packages/eblume/generic/cv/${VERSION}/${TARBALL}" if ! curl -fsSL --head "$PACKAGE_URL" > /dev/null 2>&1; then echo "Error: Package not found at $PACKAGE_URL" echo "Run the 'Release CV' workflow in the cv repo first." @@ -65,7 +65,7 @@ jobs: VERSION="${{ steps.version.outputs.version }}" TARBALL="cv-${VERSION}.tar.gz" DEPLOYMENT_FILE="argocd/manifests/cv/deployment.yaml" - RELEASE_URL="https://forge.ops.eblu.me/api/packages/eblume/generic/cv/${VERSION}/${TARBALL}" + RELEASE_URL="https://forge.eblu.me/api/packages/eblume/generic/cv/${VERSION}/${TARBALL}" echo "Updating $DEPLOYMENT_FILE with CV_RELEASE_URL..." yq -i "(.spec.template.spec.containers[0].env[] | select(.name == \"CV_RELEASE_URL\")).value = \"${RELEASE_URL}\"" "$DEPLOYMENT_FILE" diff --git a/ansible/playbooks/ringtail.yml b/ansible/playbooks/ringtail.yml index b05d67a..ee5604b 100644 --- a/ansible/playbooks/ringtail.yml +++ b/ansible/playbooks/ringtail.yml @@ -57,7 +57,7 @@ tasks: - name: Ensure blumeops repo is present ansible.builtin.git: - repo: "https://forge.ops.eblu.me/eblume/blumeops.git" + repo: "https://forge.eblu.me/eblume/blumeops.git" dest: /etc/blumeops version: "{{ ringtail_commit | default('main') }}" force: true diff --git a/ansible/roles/forgejo/defaults/main.yml b/ansible/roles/forgejo/defaults/main.yml index 95ffda8..be967aa 100644 --- a/ansible/roles/forgejo/defaults/main.yml +++ b/ansible/roles/forgejo/defaults/main.yml @@ -18,8 +18,8 @@ forgejo_log_path: "{{ forgejo_work_path }}/log" # Server settings forgejo_http_addr: 0.0.0.0 forgejo_http_port: 3001 -forgejo_domain: forge.ops.eblu.me -forgejo_ssh_domain: "{{ forgejo_domain }}" +forgejo_domain: forge.eblu.me +forgejo_ssh_domain: forge.ops.eblu.me forgejo_root_url: "https://{{ forgejo_domain }}/" forgejo_offline_mode: true diff --git a/ansible/roles/forgejo/templates/app.ini.j2 b/ansible/roles/forgejo/templates/app.ini.j2 index 3668827..de931be 100644 --- a/ansible/roles/forgejo/templates/app.ini.j2 +++ b/ansible/roles/forgejo/templates/app.ini.j2 @@ -20,6 +20,8 @@ SSH_LISTEN_PORT = {{ forgejo_ssh_listen_port }} LFS_START_SERVER = {{ forgejo_lfs_start_server | lower }} LFS_JWT_SECRET = {{ forgejo_lfs_jwt_secret }} OFFLINE_MODE = {{ forgejo_offline_mode | lower }} +REVERSE_PROXY_LIMIT = 2 +REVERSE_PROXY_TRUSTED_PROXIES = * [database] DB_TYPE = {{ forgejo_db_type }} @@ -40,7 +42,7 @@ ENABLED = false REGISTER_EMAIL_CONFIRM = false ENABLE_NOTIFY_MAIL = false DISABLE_REGISTRATION = {{ forgejo_disable_registration | lower }} -ALLOW_ONLY_EXTERNAL_REGISTRATION = false +ALLOW_ONLY_EXTERNAL_REGISTRATION = true ENABLE_CAPTCHA = false REQUIRE_SIGNIN_VIEW = {{ forgejo_require_signin_view | lower }} DEFAULT_KEEP_EMAIL_PRIVATE = false diff --git a/ansible/roles/forgejo_actions_secrets/defaults/main.yml b/ansible/roles/forgejo_actions_secrets/defaults/main.yml index 943b8af..7742ecf 100644 --- a/ansible/roles/forgejo_actions_secrets/defaults/main.yml +++ b/ansible/roles/forgejo_actions_secrets/defaults/main.yml @@ -4,7 +4,7 @@ # This role syncs repository-level Actions secrets from 1Password to Forgejo # via the Forgejo API. -forgejo_actions_secrets_api_url: "https://forge.ops.eblu.me/api/v1" +forgejo_actions_secrets_api_url: "https://forge.eblu.me/api/v1" forgejo_actions_secrets_owner: eblume # Secrets to sync per repo. diff --git a/argocd/apps/forgejo-runner.yaml b/argocd/apps/forgejo-runner.yaml index 5bca762..7addb40 100644 --- a/argocd/apps/forgejo-runner.yaml +++ b/argocd/apps/forgejo-runner.yaml @@ -6,7 +6,7 @@ metadata: spec: project: default source: - repoURL: https://forge.ops.eblu.me/eblume/blumeops.git + repoURL: https://forge.eblu.me/eblume/blumeops.git targetRevision: main path: argocd/manifests/forgejo-runner destination: diff --git a/argocd/manifests/authentik/configmap-blueprint.yaml b/argocd/manifests/authentik/configmap-blueprint.yaml index f5b4784..e867c3a 100644 --- a/argocd/manifests/authentik/configmap-blueprint.yaml +++ b/argocd/manifests/authentik/configmap-blueprint.yaml @@ -120,7 +120,7 @@ data: client_secret: !Env AUTHENTIK_FORGEJO_CLIENT_SECRET redirect_uris: - matching_mode: strict - url: https://forge.ops.eblu.me/user/oauth2/authentik/callback + url: https://forge.eblu.me/user/oauth2/authentik/callback signing_key: !Find [authentik_crypto.certificatekeypair, [name, authentik Self-signed Certificate]] property_mappings: - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]] @@ -138,7 +138,7 @@ data: name: Forgejo slug: forgejo provider: !KeyOf forgejo-provider - meta_launch_url: https://forge.ops.eblu.me + meta_launch_url: https://forge.eblu.me policy_engine_mode: any # Policy binding — restrict Forgejo to admins group diff --git a/argocd/manifests/cv/deployment.yaml b/argocd/manifests/cv/deployment.yaml index ba969fc..2e929b4 100644 --- a/argocd/manifests/cv/deployment.yaml +++ b/argocd/manifests/cv/deployment.yaml @@ -27,7 +27,7 @@ spec: name: http env: - name: CV_RELEASE_URL - value: "https://forge.ops.eblu.me/api/packages/eblume/generic/cv/v1.0.3/cv-v1.0.3.tar.gz" + value: "https://forge.eblu.me/api/packages/eblume/generic/cv/v1.0.3/cv-v1.0.3.tar.gz" resources: requests: memory: "64Mi" diff --git a/argocd/manifests/docs/deployment.yaml b/argocd/manifests/docs/deployment.yaml index 66506c4..af8b655 100644 --- a/argocd/manifests/docs/deployment.yaml +++ b/argocd/manifests/docs/deployment.yaml @@ -27,7 +27,7 @@ spec: name: http env: - name: DOCS_RELEASE_URL - value: "https://forge.ops.eblu.me/eblume/blumeops/releases/download/v1.12.1/docs-v1.12.1.tar.gz" + value: "https://forge.eblu.me/eblume/blumeops/releases/download/v1.12.1/docs-v1.12.1.tar.gz" resources: requests: memory: "64Mi" diff --git a/argocd/manifests/forgejo-runner/deployment.yaml b/argocd/manifests/forgejo-runner/deployment.yaml index 4f67672..6e57137 100644 --- a/argocd/manifests/forgejo-runner/deployment.yaml +++ b/argocd/manifests/forgejo-runner/deployment.yaml @@ -25,7 +25,7 @@ spec: - name: DOCKER_HOST value: tcp://localhost:2375 - name: FORGEJO_URL - value: "https://forge.ops.eblu.me" + value: "https://forge.eblu.me" - name: RUNNER_NAME value: "k8s-runner" - name: RUNNER_LABELS diff --git a/argocd/manifests/homepage/services.yaml b/argocd/manifests/homepage/services.yaml index 0f35e9c..fcde74f 100644 --- a/argocd/manifests/homepage/services.yaml +++ b/argocd/manifests/homepage/services.yaml @@ -1,11 +1,11 @@ - Host Services: - Forgejo: - href: https://forge.ops.eblu.me + href: https://forge.eblu.me icon: forgejo description: Git forge widget: type: gitea - url: https://forge.ops.eblu.me + url: https://forge.eblu.me key: "{{HOMEPAGE_VAR_FORGEJO_API_KEY}}" - Registry: href: https://registry.ops.eblu.me diff --git a/argocd/manifests/tailscale-operator/README.md b/argocd/manifests/tailscale-operator/README.md index ee8ce1e..dc4b009 100644 --- a/argocd/manifests/tailscale-operator/README.md +++ b/argocd/manifests/tailscale-operator/README.md @@ -73,7 +73,6 @@ kubectl logs -n tailscale -l app.kubernetes.io/name=operator | `operator.yaml` | Operator deployment, CRDs, RBAC (secret removed) | | `proxyclass.yaml` | ProxyClass with fully-qualified images | | `dnsconfig.yaml` | DNSConfig for cluster-to-tailnet name resolution | -| `egress-forge.yaml` | Egress proxy for accessing forge on indri | | `secret.yaml.tpl` | 1Password template for OAuth credentials (manual) | | `README.md` | This file | @@ -86,5 +85,3 @@ kubectl logs -n tailscale -l app.kubernetes.io/name=operator annotations: tailscale.com/proxy-class: "default" ``` -- The egress proxy for forge is **deprecated**. Forge is now accessible via Caddy at - `forge.ops.eblu.me` (HTTPS) and `forge.ops.eblu.me:2222` (SSH), which pods can reach directly. diff --git a/argocd/manifests/tailscale-operator/egress-forge.yaml b/argocd/manifests/tailscale-operator/egress-forge.yaml deleted file mode 100644 index 4dc982b..0000000 --- a/argocd/manifests/tailscale-operator/egress-forge.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# DEPRECATED: This egress proxy is no longer needed. -# Forge is now accessible via Caddy at forge.ops.eblu.me (HTTPS) and -# forge.ops.eblu.me:2222 (SSH), which pods can reach directly. -# -# Keeping this file for reference during migration. Remove once verified. -# -# Original purpose: Egress proxy to expose Forgejo (forge) to the cluster -# See: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress ---- -apiVersion: v1 -kind: Service -metadata: - name: forge - namespace: tailscale - annotations: - tailscale.com/tailnet-fqdn: indri.tail8d86e.ts.net - tailscale.com/proxy-class: "default" -spec: - type: ExternalName - externalName: placeholder - ports: - - port: 3001 - targetPort: 3001 diff --git a/argocd/manifests/tailscale-operator/endpoints-forge.yaml b/argocd/manifests/tailscale-operator/endpoints-forge.yaml new file mode 100644 index 0000000..e68aff9 --- /dev/null +++ b/argocd/manifests/tailscale-operator/endpoints-forge.yaml @@ -0,0 +1,24 @@ +--- +# Manual Endpoints pointing to indri's Tailscale IP for the +# forge-external Service. Must match the Service name exactly. +# +# NOTE: ArgoCD excludes all Endpoints resources (resource.exclusions in +# argocd-cm) because they are normally auto-managed by the control plane. +# This manual Endpoints is the exception — it must be applied directly +# with kubectl, not via ArgoCD. It is listed in kustomization.yaml for +# documentation purposes only; ArgoCD will silently skip it. +# +# kubectl --context=minikube-indri apply -f endpoints-forge.yaml +# +apiVersion: v1 +kind: Endpoints +metadata: + name: forge-external + namespace: tailscale +subsets: + - addresses: + - ip: 100.98.163.89 + ports: + - name: http + port: 3001 + protocol: TCP diff --git a/argocd/manifests/tailscale-operator/ingress-forge.yaml b/argocd/manifests/tailscale-operator/ingress-forge.yaml new file mode 100644 index 0000000..047b59d --- /dev/null +++ b/argocd/manifests/tailscale-operator/ingress-forge.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: forge-tailscale + namespace: tailscale + annotations: + tailscale.com/proxy-class: "default" + tailscale.com/proxy-group: "ingress" + tailscale.com/tags: "tag:k8s,tag:flyio-target" +spec: + ingressClassName: tailscale + defaultBackend: + service: + name: forge-external + port: + number: 3001 + tls: + - hosts: + - forge diff --git a/argocd/manifests/tailscale-operator/kustomization.yaml b/argocd/manifests/tailscale-operator/kustomization.yaml index 09fa1b8..f1d6f89 100644 --- a/argocd/manifests/tailscale-operator/kustomization.yaml +++ b/argocd/manifests/tailscale-operator/kustomization.yaml @@ -7,5 +7,10 @@ namespace: tailscale resources: - ../tailscale-operator-base - proxygroup-ingress.yaml - - egress-forge.yaml - external-secret.yaml + - svc-forge-external.yaml + # endpoints-forge.yaml is NOT managed by ArgoCD — Endpoints are globally + # excluded in argocd-cm resource.exclusions (too noisy for auto-managed + # Endpoints). Apply manually: + # kubectl --context=minikube-indri apply -f endpoints-forge.yaml + - ingress-forge.yaml diff --git a/argocd/manifests/tailscale-operator/svc-forge-external.yaml b/argocd/manifests/tailscale-operator/svc-forge-external.yaml new file mode 100644 index 0000000..79a8645 --- /dev/null +++ b/argocd/manifests/tailscale-operator/svc-forge-external.yaml @@ -0,0 +1,15 @@ +--- +# ClusterIP service for Forgejo on indri. Paired with endpoints-forge.yaml +# which provides the actual routing to indri's Tailscale IP. +# ExternalName services don't have a ClusterIP, which the Tailscale +# ingress operator requires. +apiVersion: v1 +kind: Service +metadata: + name: forge-external + namespace: tailscale +spec: + ports: + - name: http + port: 3001 + protocol: TCP diff --git a/docs/changelog.d/feature-forge-public.feature.md b/docs/changelog.d/feature-forge-public.feature.md new file mode 100644 index 0000000..44be391 --- /dev/null +++ b/docs/changelog.d/feature-forge-public.feature.md @@ -0,0 +1 @@ +Expose Forgejo publicly at forge.eblu.me via Fly.io reverse proxy with rate limiting, fail2ban, and security hardening. diff --git a/docs/how-to/authentik/build-authentik-from-source.md b/docs/how-to/authentik/build-authentik-from-source.md index 169a0ed..d4411e3 100644 --- a/docs/how-to/authentik/build-authentik-from-source.md +++ b/docs/how-to/authentik/build-authentik-from-source.md @@ -36,8 +36,8 @@ The `ak` wrapper script in `default.nix` sets PATH/VIRTUAL_ENV and delegates to ## Source All derivations fetch from forge mirrors for supply chain control: -- https://forge.ops.eblu.me/mirrors/authentik (upstream: `goauthentik/authentik`) -- https://forge.ops.eblu.me/mirrors/authentik-client-go (upstream: `goauthentik/client-go`) +- https://forge.eblu.me/mirrors/authentik (upstream: `goauthentik/authentik`) +- https://forge.eblu.me/mirrors/authentik-client-go (upstream: `goauthentik/client-go`) Version and hashes are centralized in `containers/authentik/sources.nix`. diff --git a/docs/how-to/configuration/expose-service-publicly.md b/docs/how-to/configuration/expose-service-publicly.md index b342481..bb6b258 100644 --- a/docs/how-to/configuration/expose-service-publicly.md +++ b/docs/how-to/configuration/expose-service-publicly.md @@ -1,7 +1,7 @@ --- title: Expose a Service Publicly -modified: 2026-02-16 -last-reviewed: 2026-02-16 +modified: 2026-03-03 +last-reviewed: 2026-03-03 tags: - how-to - fly-io @@ -259,7 +259,7 @@ server { } ``` -**Dynamic service template** (e.g., Forgejo — hypothetical, not currently deployed): +**Dynamic service template** (e.g., Forgejo — see `fly/nginx.conf` for the live configuration): ```nginx # --- forge.eblu.me (dynamic, authenticated) --- @@ -440,32 +440,30 @@ see plan history in git). ### fail2ban fail2ban monitors log files for repeated failed authentication attempts -(SSH brute force, bad login passwords, API abuse) and bans IPs via -firewall rules. +and bans offending IPs. **Static sites**: fail2ban does not apply. There is no login surface, no sessions, no credentials to brute force. -**Dynamic services with authentication** (e.g., Forgejo): fail2ban is -relevant and should be configured on **indri**, not on Fly.io. The -nginx proxy is transparent — it forwards requests but does not see -authentication outcomes. fail2ban watches the service's own logs on -indri for patterns like repeated failed logins. +**Dynamic services with authentication** (e.g., Forgejo): fail2ban +runs in the **Fly.io container**, not on indri. Standard iptables +banning won't work in Fly.io because `$remote_addr` is Fly's internal +proxy IP, not the client. Instead, fail2ban uses a custom nginx-based +ban action: -Setup considerations for Forgejo specifically: +1. fail2ban watches the nginx JSON access log for repeated 401/403 + responses to login endpoints, keyed on the `client_ip` field + (populated from the `Fly-Client-IP` header) +2. On ban, it appends the IP to `/etc/nginx/forge-deny.conf` and + reloads nginx +3. nginx uses a `geo` directive keyed on `$http_fly_client_ip` to + check the deny list and return 403 for banned IPs -- Forgejo logs failed auth attempts to its log file -- fail2ban needs a filter matching Forgejo's log format -- Banned IPs are blocked at indri's firewall (the Fly.io proxy IP is - the Tailscale address of the `flyio-proxy` node, not the end user's - IP) -- **Important**: for fail2ban to see real client IPs, the nginx proxy - must pass `X-Real-IP` / `X-Forwarded-For` headers (included in the - dynamic service nginx config above), and Forgejo must be configured - to trust the proxy and log the forwarded IP rather than the proxy's - Tailscale IP -- Disable open user registration before exposing Forgejo publicly — - require explicit invites +Ban lists are **ephemeral across deploys** — nginx rate limiting +provides the persistent baseline; fail2ban adds escalating bans for +active attacks. + +See `fly/fail2ban/` for the filter, jail, and action configuration. ### Break-glass shutoff @@ -504,7 +502,7 @@ dynamic, authenticated service like [[forgejo]]. - [ ] Disable open user registration (require invites or admin approval) - [ ] Audit access controls and permissions - [ ] Configure the service to log the forwarded client IP (not the proxy IP) -- [ ] Set up fail2ban on indri with a filter for the service's log format +- [ ] Set up fail2ban in the Fly.io container with a filter for the service's login endpoints - [ ] Tag the service's Tailscale Ingress with `tag:flyio-target` - [ ] Test the nginx config locally or in staging before deploying - [ ] Rehearse the break-glass shutoff (`mise run fly-shutoff`) diff --git a/docs/how-to/configuration/update-documentation.md b/docs/how-to/configuration/update-documentation.md index fef8ccf..7c98e24 100644 --- a/docs/how-to/configuration/update-documentation.md +++ b/docs/how-to/configuration/update-documentation.md @@ -20,7 +20,7 @@ After merging documentation changes to main: 2. Select version bump type (patch/minor/major) or enter a specific version 3. The workflow builds, releases, and deploys automatically -Direct link: https://forge.ops.eblu.me/eblume/blumeops/actions?workflow=build-blumeops.yaml +Direct link: https://forge.eblu.me/eblume/blumeops/actions?workflow=build-blumeops.yaml ## What the Workflow Does diff --git a/docs/how-to/deployment/build-container-image.md b/docs/how-to/deployment/build-container-image.md index 9f6d7b8..b3b9cbe 100644 --- a/docs/how-to/deployment/build-container-image.md +++ b/docs/how-to/deployment/build-container-image.md @@ -93,7 +93,7 @@ Container image tags include the git commit SHA they were built from (e.g. `v3.9 **The rule:** Production manifests must reference images built from a commit on main. After merging a PR that changed `containers//`: 1. The merge to main automatically triggers a rebuild (the `build-container.yaml` / `build-container-nix.yaml` workflows fire on pushes to `main` that touch `containers/**`) -2. Wait for the workflow to complete — check at `https://forge.ops.eblu.me/eblume/blumeops/actions` +2. Wait for the workflow to complete — check at `https://forge.eblu.me/eblume/blumeops/actions` 3. Find the new main-SHA tag: ```bash mise run container-list diff --git a/docs/how-to/deployment/create-release-artifact-workflow.md b/docs/how-to/deployment/create-release-artifact-workflow.md index 80e0308..63c217e 100644 --- a/docs/how-to/deployment/create-release-artifact-workflow.md +++ b/docs/how-to/deployment/create-release-artifact-workflow.md @@ -48,16 +48,16 @@ The upload step uses `FORGE_TOKEN`: -X PUT \ -H "Authorization: token $FORGE_TOKEN" \ --upload-file "./$TARBALL" \ - "https://forge.ops.eblu.me/api/packages/eblume/generic//${VERSION}/${TARBALL}" + "https://forge.eblu.me/api/packages/eblume/generic//${VERSION}/${TARBALL}" ``` ## 3. Link the package to the repo -After the first successful upload, the package appears under your **user-level** packages at `https://forge.ops.eblu.me/eblume/-/packages` but is not yet linked to the repo. +After the first successful upload, the package appears under your **user-level** packages at `https://forge.eblu.me/eblume/-/packages` but is not yet linked to the repo. To link it: -1. Go to `https://forge.ops.eblu.me/eblume/-/packages` +1. Go to `https://forge.eblu.me/eblume/-/packages` 2. Click the package name 3. Click **Settings** 4. Under **Link this package to a repository**, select the repo diff --git a/docs/how-to/plans/completed/adopt-dagger-ci.md b/docs/how-to/plans/completed/adopt-dagger-ci.md index a3ec8ce..aabae1d 100644 --- a/docs/how-to/plans/completed/adopt-dagger-ci.md +++ b/docs/how-to/plans/completed/adopt-dagger-ci.md @@ -222,12 +222,12 @@ Migrate `build-blumeops.yaml` to use Dagger for the build logic and switch from **Current:** Docs tarball uploaded as a Forgejo release asset. ``` -https://forge.ops.eblu.me/eblume/blumeops/releases/download/v1.5.2/docs-v1.5.2.tar.gz +https://forge.eblu.me/eblume/blumeops/releases/download/v1.5.2/docs-v1.5.2.tar.gz ``` **New:** Docs tarball uploaded to Forgejo generic packages registry. ``` -https://forge.ops.eblu.me/api/packages/eblume/generic/blumeops-docs/v1.6.0/docs-v1.6.0.tar.gz +https://forge.eblu.me/api/packages/eblume/generic/blumeops-docs/v1.6.0/docs-v1.6.0.tar.gz ``` This decouples the docs artifact from git releases while keeping the versioned URL pattern. Forgejo releases can still be created for changelog/announcement purposes without carrying the tarball. @@ -290,13 +290,13 @@ async def upload_docs( async with httpx.AsyncClient() as client: with open(f"/tmp/docs-{version}.tar.gz", "rb") as f: resp = await client.put( - f"https://forge.ops.eblu.me/api/packages/eblume/generic/" + f"https://forge.eblu.me/api/packages/eblume/generic/" f"blumeops-docs/{version}/docs-{version}.tar.gz", headers={"Authorization": f"token {token}"}, content=f.read(), ) resp.raise_for_status() - return f"https://forge.ops.eblu.me/api/packages/eblume/generic/blumeops-docs/{version}/docs-{version}.tar.gz" + return f"https://forge.eblu.me/api/packages/eblume/generic/blumeops-docs/{version}/docs-{version}.tar.gz" @function async def release_docs( @@ -388,7 +388,7 @@ jobs: - name: Update manifest and commit run: | VERSION="${{ steps.version.outputs.version }}" - URL="https://forge.ops.eblu.me/api/packages/eblume/generic/blumeops-docs/${VERSION}/docs-${VERSION}.tar.gz" + URL="https://forge.eblu.me/api/packages/eblume/generic/blumeops-docs/${VERSION}/docs-${VERSION}.tar.gz" sed -i "s|value: \"https://.*\"|value: \"${URL}\"|" \ argocd/manifests/docs/deployment.yaml git config user.name "Forgejo Actions" @@ -405,11 +405,11 @@ The quartz container's `DOCS_RELEASE_URL` env var in `argocd/manifests/docs/depl ```yaml # Before (Forgejo releases): - name: DOCS_RELEASE_URL - value: "https://forge.ops.eblu.me/eblume/blumeops/releases/download/v1.5.2/docs-v1.5.2.tar.gz" + value: "https://forge.eblu.me/eblume/blumeops/releases/download/v1.5.2/docs-v1.5.2.tar.gz" # After (Forgejo generic packages): - name: DOCS_RELEASE_URL - value: "https://forge.ops.eblu.me/api/packages/eblume/generic/blumeops-docs/v1.6.0/docs-v1.6.0.tar.gz" + value: "https://forge.eblu.me/api/packages/eblume/generic/blumeops-docs/v1.6.0/docs-v1.6.0.tar.gz" ``` The quartz container's `start.sh` already downloads from `DOCS_RELEASE_URL` via curl — no container changes needed, just the URL format changes. diff --git a/docs/how-to/plans/migrate-forgejo-from-brew.md b/docs/how-to/plans/migrate-forgejo-from-brew.md index 3a77154..4daad6b 100644 --- a/docs/how-to/plans/migrate-forgejo-from-brew.md +++ b/docs/how-to/plans/migrate-forgejo-from-brew.md @@ -32,7 +32,7 @@ https://codeberg.org/forgejo/forgejo.git Add the forge mirror as a secondary remote for convenience and backup: ``` -https://forge.ops.eblu.me/mirrors/forgejo.git +https://forge.eblu.me/mirrors/forgejo.git ``` ## One-Time Migration Steps @@ -48,7 +48,7 @@ ssh indri 'git clone https://codeberg.org/forgejo/forgejo.git ~/code/3rd/forgejo ### 2. Add Forge Mirror as Secondary Remote ```fish -ssh indri 'cd ~/code/3rd/forgejo && git remote add forge https://forge.ops.eblu.me/mirrors/forgejo.git' +ssh indri 'cd ~/code/3rd/forgejo && git remote add forge https://forge.eblu.me/mirrors/forgejo.git' ``` ### 3. Check Out the Desired Version Tag @@ -155,7 +155,7 @@ Replace brew install/start with binary-check + LaunchAgent pattern (matching `an # ssh indri 'git clone https://codeberg.org/forgejo/forgejo.git ~/code/3rd/forgejo' # # 2. Add forge mirror as secondary remote: -# ssh indri 'cd ~/code/3rd/forgejo && git remote add forge https://forge.ops.eblu.me/mirrors/forgejo.git' +# ssh indri 'cd ~/code/3rd/forgejo && git remote add forge https://forge.eblu.me/mirrors/forgejo.git' # # 3. Set up Go and Node via mise: # ssh indri 'cd ~/code/3rd/forgejo && mise use go@1.24 node@20' @@ -275,7 +275,7 @@ No changes needed — paths already flow through variables in `defaults/main.yml After running the migration and Ansible: - [ ] `ssh indri 'launchctl list mcquack.eblume.forgejo'` — shows running -- [ ] `curl https://forge.ops.eblu.me/api/v1/version` — returns JSON with version +- [ ] `curl https://forge.eblu.me/api/v1/version` — returns JSON with version - [ ] Git clone over SSH: `git clone ssh://forgejo@forge.ops.eblu.me:2222/eblume/blumeops.git /tmp/test-clone` - [ ] Git push works on an existing clone - [ ] Ansible dry-run is clean: `mise run provision-indri -- --tags forgejo --check --diff` diff --git a/docs/how-to/plans/upstream-fork-strategy.md b/docs/how-to/plans/upstream-fork-strategy.md index b2ce412..e2af8c8 100644 --- a/docs/how-to/plans/upstream-fork-strategy.md +++ b/docs/how-to/plans/upstream-fork-strategy.md @@ -209,7 +209,7 @@ This fork directly supports the [[adopt-dagger-ci]] plan. Once the fork exists, # After (using the BlumeOps fork): .with_exec(["git", "clone", "--depth=1", "--branch=blumeops", - "https://forge.ops.eblu.me/mirrors/quartz.git", "/tmp/quartz"]) + "https://forge.eblu.me/mirrors/quartz.git", "/tmp/quartz"]) ``` This means the `build-blumeops.yaml` workflow automatically picks up fork customizations (like `last-reviewed` rendering) when building docs — no separate integration step needed. Local iteration via `dagger call build-docs` also uses the fork, so you can test Quartz customizations against actual BlumeOps content before pushing. diff --git a/docs/how-to/zot/register-zot-oidc-client.md b/docs/how-to/zot/register-zot-oidc-client.md index ff7d805..b3696d0 100644 --- a/docs/how-to/zot/register-zot-oidc-client.md +++ b/docs/how-to/zot/register-zot-oidc-client.md @@ -12,7 +12,7 @@ tags: Register a zot OAuth2 provider and application in Authentik via blueprint, following the same pattern as Grafana and Forgejo. -Completed in PR [#236](https://forge.ops.eblu.me/eblume/blumeops/pulls/236). +Completed in PR [#236](https://forge.eblu.me/eblume/blumeops/pulls/236). ## What Was Done diff --git a/docs/index.md b/docs/index.md index 6d7822d..306e73c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -16,7 +16,7 @@ infrastructure. BlumeOps is my personal homelab infrastructure managed entirely through code. Everything lives in a [single git repository](https://github.com/eblume/blumeops), from service configs to -deployment automation. Even the [[forgejo]] instance that [hosts this repo](https://forge.ops.eblu.me/eblume/blumeops) +deployment automation. Even the [[forgejo]] instance that [hosts this repo](https://forge.eblu.me/eblume/blumeops) is defined within it, making BlumeOps fully self-hosting. It's a digital life raft I built for myself as I went, and you can see it all from within your editor of choice. (I recommend vim.) diff --git a/docs/quartz.layout.ts b/docs/quartz.layout.ts index c3f7e6d..cba29ec 100644 --- a/docs/quartz.layout.ts +++ b/docs/quartz.layout.ts @@ -14,7 +14,7 @@ export const sharedPageComponents: SharedLayout = { footer: Component.Footer({ links: { "GitHub": "https://github.com/eblume/blumeops", - "Forge": "https://forge.ops.eblu.me/eblume/blumeops", + "Forge": "https://forge.eblu.me/eblume/blumeops", }, }), } diff --git a/docs/reference/infrastructure/routing.md b/docs/reference/infrastructure/routing.md index 092119f..91457e9 100644 --- a/docs/reference/infrastructure/routing.md +++ b/docs/reference/infrastructure/routing.md @@ -1,6 +1,6 @@ --- title: Routing -modified: 2026-02-09 +modified: 2026-03-03 tags: - infrastructure - networking @@ -49,6 +49,7 @@ DNS CNAMEs point to `blumeops-proxy.fly.dev`. TLS via Fly.io-managed Let's Encry | Service | URL | Description | |---------|-----|-------------| | [[docs]] | https://docs.eblu.me | Documentation site | +| [[forgejo]] | https://forge.eblu.me | Git hosting (public) | ## Tailscale-Only Services diff --git a/docs/reference/services/forgejo.md b/docs/reference/services/forgejo.md index 68fd1f4..1f56029 100644 --- a/docs/reference/services/forgejo.md +++ b/docs/reference/services/forgejo.md @@ -1,6 +1,6 @@ --- title: Forgejo -modified: 2026-02-20 +modified: 2026-03-03 tags: - service - git @@ -15,7 +15,8 @@ Git forge and CI/CD platform. **Primary source of truth for blumeops** (mirrored | Property | Value | |----------|-------| -| **URL** | https://forge.ops.eblu.me | +| **URL (public)** | https://forge.eblu.me | +| **URL (internal)** | https://forge.ops.eblu.me | | **SSH** | `ssh://forgejo@forge.ops.eblu.me:2222` | | **Local Ports** | 3001 (HTTP), 2200 (SSH) | | **Config** | `ansible/roles/forgejo/templates/app.ini.j2` | @@ -71,7 +72,7 @@ mise run provision-indri -- --tags forgejo_actions_secrets The Ansible role authenticates to the Forgejo API using a Personal Access Token (PAT). This PAT must be created manually: -1. Go to https://forge.ops.eblu.me/user/settings/applications +1. Go to https://forge.eblu.me/user/settings/applications 2. Create a new token with `write:repository` scope 3. Store it in 1Password → "Forgejo Secrets" item → `api-token` field @@ -94,23 +95,30 @@ This is a bootstrapping requirement - the PAT enables IaC for all other secrets. **Break-glass:** Local password login always works (with local MFA). Authentik SSO is additive — if Authentik is down, log in with local credentials. -## Future: Public Access +## Public Access -Forgejo can be exposed publicly at `forge.eblu.me` via [[flyio-proxy]]. Since Forgejo runs natively on [[indri]] (not in k8s), the pattern is: +Forgejo is publicly accessible at `https://forge.eblu.me` via [[flyio-proxy]]. This is the first dynamic, authenticated service exposed publicly. -1. Create a k8s ExternalName Service pointing to indri's Tailscale IP -2. Create a Tailscale Ingress with `tailscale.com/tags: "tag:k8s,tag:flyio-target"` -3. Add the nginx server block and DNS CNAME +| Access Method | URL | Reachable From | +|---------------|-----|----------------| +| **HTTPS (public)** | https://forge.eblu.me | Public internet | +| **HTTPS (internal)** | https://forge.ops.eblu.me | Tailnet only | +| **SSH** | `ssh://forgejo@forge.ops.eblu.me:2222` | Tailnet only | -Exposing a dynamic, authenticated service like Forgejo requires a full security review before going live: +The UI shows `forge.eblu.me` for HTTPS clone URLs and `forge.ops.eblu.me` for SSH clone URLs. -- Disable all local registration — only allow login via [[authentik]] (`DISABLE_REGISTRATION = true`, `ALLOW_ONLY_EXTERNAL_REGISTRATION = true`) -- Configure fail2ban on indri with a filter for Forgejo's log format -- Ensure Forgejo logs the forwarded client IP (`X-Real-IP`) rather than the proxy's Tailscale IP -- Audit repository visibility defaults and permissions -- Rehearse the break-glass shutoff (`mise run fly-shutoff`) +### Security Controls -See [[expose-service-publicly]] for the full howto and dynamic service checklist. +- **Registration:** Local registration disabled; only [[authentik]] SSO login allowed (`ALLOW_ONLY_EXTERNAL_REGISTRATION = true`) +- **Reverse proxy trust:** `REVERSE_PROXY_LIMIT = 2`, `REVERSE_PROXY_TRUSTED_PROXIES = *` — Forgejo logs the real client IP from `X-Real-IP` header, not the proxy's Tailscale IP +- **Rate limiting:** nginx rate limits login/signup/forgot-password endpoints (3r/s per client IP via `Fly-Client-IP` header) +- **fail2ban:** Runs in the Fly.io container; bans IPs after 5 failed logins in 10 minutes via nginx deny list (ephemeral across deploys) +- **Swagger:** Blocked at the proxy (`/swagger` returns 403); use forge.ops.eblu.me for API access +- **OAuth dead-end:** "Sign in with Authentik" redirects to the (tailnet-only) Authentik URL — SSO only works from the tailnet + +### Break-glass + +`mise run fly-shutoff` stops all public traffic immediately. forge.ops.eblu.me continues to work from the tailnet. See [[expose-service-publicly#Break-glass shutoff]]. ## Related diff --git a/docs/tutorials/contributing.md b/docs/tutorials/contributing.md index 61d32e8..cddafea 100644 --- a/docs/tutorials/contributing.md +++ b/docs/tutorials/contributing.md @@ -16,7 +16,7 @@ This tutorial walks through making your first contribution to BluemeOps - from u Before contributing, you'll need: - Access to the [[tailscale|Tailscale]] network (request from Erich) -- SSH key added to [[forgejo|Forgejo]] (https://forge.ops.eblu.me) +- SSH key added to [[forgejo|Forgejo]] (https://forge.eblu.me) - Development tools installed (see below) ## Tooling Setup diff --git a/fly/Dockerfile b/fly/Dockerfile index 68a98d8..65135c1 100644 --- a/fly/Dockerfile +++ b/fly/Dockerfile @@ -8,7 +8,9 @@ COPY --from=docker.io/tailscale/tailscale:stable \ RUN mkdir -p /var/run/tailscale /var/lib/tailscale \ && apk add --no-cache iptables ip6tables \ - && apk add --no-cache libc6-compat + && apk add --no-cache libc6-compat \ + && apk add --no-cache fail2ban \ + && rm -f /etc/fail2ban/jail.d/alpine-ssh.conf # Copy Alloy binary from official image (Ubuntu-based, needs libc6-compat) COPY --from=docker.io/grafana/alloy:v1.13.1 \ @@ -16,6 +18,10 @@ COPY --from=docker.io/grafana/alloy:v1.13.1 \ RUN mkdir -p /var/log/nginx /etc/alloy /tmp/alloy-data +COPY fail2ban/filter.d/forge-login.conf /etc/fail2ban/filter.d/forge-login.conf +COPY fail2ban/jail.d/forge.conf /etc/fail2ban/jail.d/forge.conf +COPY fail2ban/action.d/nginx-deny.conf /etc/fail2ban/action.d/nginx-deny.conf + COPY nginx.conf /etc/nginx/nginx.conf COPY error.html /usr/share/nginx/html/error.html COPY alloy.river /etc/alloy/config.alloy diff --git a/fly/fail2ban/action.d/nginx-deny.conf b/fly/fail2ban/action.d/nginx-deny.conf new file mode 100644 index 0000000..1d3737b --- /dev/null +++ b/fly/fail2ban/action.d/nginx-deny.conf @@ -0,0 +1,14 @@ +# Custom fail2ban action that bans IPs via an nginx deny list. +# Standard iptables banning won't work in Fly.io because $remote_addr +# is Fly's internal proxy IP. Instead, we write banned IPs to a file +# that nginx checks via a geo directive keyed on $http_fly_client_ip. + +[Definition] + +actionban = echo " 1;" >> /etc/nginx/forge-deny.conf && nginx -s reload + +actionunban = sed -i '/ 1;/d' /etc/nginx/forge-deny.conf && nginx -s reload + +actionstart = +actionstop = +actioncheck = diff --git a/fly/fail2ban/filter.d/forge-login.conf b/fly/fail2ban/filter.d/forge-login.conf new file mode 100644 index 0000000..6961b9a --- /dev/null +++ b/fly/fail2ban/filter.d/forge-login.conf @@ -0,0 +1,10 @@ +# Filter for Forgejo login failures via nginx JSON access log. +# Matches 401/403 responses to authentication endpoints, keyed on +# the client_ip field (populated from Fly-Client-IP header). + +[Definition] + +# Match JSON log lines with 401 or 403 status on login-related paths +failregex = "client_ip":"".*"request_uri":"\/user\/(login|sign_up|forgot_password)[^"]*".*"status":(401|403) + +ignoreregex = diff --git a/fly/fail2ban/jail.d/forge.conf b/fly/fail2ban/jail.d/forge.conf new file mode 100644 index 0000000..7b0843f --- /dev/null +++ b/fly/fail2ban/jail.d/forge.conf @@ -0,0 +1,8 @@ +[forge-login] +enabled = true +filter = forge-login +logpath = /var/log/nginx/access.json.log +maxretry = 5 +findtime = 600 +bantime = 3600 +banaction = nginx-deny diff --git a/fly/nginx.conf b/fly/nginx.conf index e50545a..992a5df 100644 --- a/fly/nginx.conf +++ b/fly/nginx.conf @@ -29,6 +29,19 @@ http { # Rate limiting zones — define per-service zones as needed limit_req_zone $binary_remote_addr zone=general:10m rate=10r/s; + # Forge-specific rate limit keyed on real client IP (Fly-Client-IP header). + # $binary_remote_addr is Fly's internal proxy IP — all clients share one + # bucket. $http_fly_client_ip has the actual client IP. + limit_req_zone $http_fly_client_ip zone=forge_auth:10m rate=3r/s; + + # fail2ban deny list — banned IPs are written here by fail2ban and + # checked via the $forge_banned variable. The file is touched at + # container start to ensure it exists. + geo $http_fly_client_ip $forge_banned { + default 0; + include /etc/nginx/forge-deny.conf; + } + # Proxy cache: 200MB, evict after 24h of no access proxy_cache_path /tmp/cache levels=1:2 keys_zone=services:10m max_size=200m inactive=24h; @@ -114,6 +127,97 @@ http { } } + # --- forge.eblu.me (dynamic, authenticated) --- + server { + listen 8080; + server_name forge.eblu.me; + + # Block fail2ban-banned IPs + if ($forge_banned) { + return 403 "Temporarily blocked. Try again later.\n"; + } + + # General rate limit — higher burst for git operations and CI webhooks + limit_req zone=general burst=50 nodelay; + + # Git LFS and repo uploads can be large + client_max_body_size 512m; + + # Security headers + add_header X-Frame-Options "DENY" always; + add_header X-Content-Type-Options "nosniff" always; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + + error_page 502 503 504 /error.html; + location = /error.html { + root /usr/share/nginx/html; + internal; + } + + # Block swagger API docs — use forge.ops.eblu.me from tailnet + location /swagger { + return 403 "API documentation is only available at forge.ops.eblu.me (tailnet).\n"; + } + + # Rate-limit authentication endpoints + location ~ ^/user/(login|sign_up|forgot_password) { + limit_req zone=forge_auth burst=5 nodelay; + + set $upstream_forge https://forge.tail8d86e.ts.net; + proxy_pass $upstream_forge$request_uri; + proxy_ssl_verify off; + proxy_ssl_server_name on; + proxy_intercept_errors on; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $http_fly_client_ip; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + + # Selectively cache static assets only + location ~* \.(css|js|png|jpg|svg|woff2?)$ { + set $upstream_forge_static https://forge.tail8d86e.ts.net; + proxy_pass $upstream_forge_static$request_uri; + proxy_ssl_verify off; + proxy_ssl_server_name on; + + proxy_cache services; + proxy_cache_valid 200 7d; + proxy_cache_key $host$uri; + + add_header X-Cache-Status $upstream_cache_status; + add_header X-Clacks-Overhead "GNU Terry Pratchett" always; + } + + location / { + set $upstream_forge https://forge.tail8d86e.ts.net; + proxy_pass $upstream_forge$request_uri; + proxy_ssl_verify off; + proxy_ssl_server_name on; + proxy_intercept_errors on; + + # NO proxy_cache — dynamic content with sessions + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $http_fly_client_ip; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # WebSocket support (Forgejo uses it for live updates) + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + add_header X-Clacks-Overhead "GNU Terry Pratchett" always; + } + } + # Catch-all: reject unknown hosts, but serve health check server { listen 8080 default_server; diff --git a/fly/start.sh b/fly/start.sh index 96ccbf0..5ec45db 100644 --- a/fly/start.sh +++ b/fly/start.sh @@ -12,11 +12,22 @@ tailscale up --authkey="${TS_AUTHKEY}" --hostname=flyio-proxy until tailscale status > /dev/null 2>&1; do sleep 1; done echo "Tailscale connected" +# Ensure fail2ban deny file exists before nginx starts +touch /etc/nginx/forge-deny.conf + # Start nginx — MagicDNS is available, health check passes immediately. nginx -g "daemon off;" & NGINX_PID=$! echo "Nginx started" +# Start fail2ban for login brute-force protection. +# Non-fatal — nginx rate limiting is the primary defense; fail2ban is additive. +if fail2ban-server -b; then + echo "fail2ban started" +else + echo "WARNING: fail2ban failed to start (nginx rate limiting still active)" +fi + # Start Alloy for observability (logs → Loki, metrics → Prometheus) alloy run /etc/alloy/config.alloy \ --server.http.listen-addr=127.0.0.1:12345 \ diff --git a/mise-tasks/branch-cleanup b/mise-tasks/branch-cleanup index 85be4e6..88e9152 100755 --- a/mise-tasks/branch-cleanup +++ b/mise-tasks/branch-cleanup @@ -44,7 +44,7 @@ from rich.console import Console from rich.table import Table PROTECTED_BRANCHES = {"main", "master"} -FORGE_API = "https://forge.ops.eblu.me/api/v1" +FORGE_API = "https://forge.eblu.me/api/v1" REPO_OWNER = "eblume" REPO_NAME = "blumeops" OP_TOKEN_REF = "op://vg6xf6vvfmoh5hqjjhlhbeoaie/w3663ffnvkewbftncqxtcpeavy/api-token" diff --git a/mise-tasks/container-build-and-release b/mise-tasks/container-build-and-release index 2226550..dd78923 100755 --- a/mise-tasks/container-build-and-release +++ b/mise-tasks/container-build-and-release @@ -21,7 +21,7 @@ import httpx import typer REGISTRY = "registry.ops.eblu.me" -FORGE_URL = "https://forge.ops.eblu.me" +FORGE_URL = "https://forge.eblu.me" FORGE_API = f"{FORGE_URL}/api/v1" REPO = "eblume/blumeops" FORGE_ACTIONS = f"{FORGE_URL}/{REPO}/actions" diff --git a/mise-tasks/docs-mikado b/mise-tasks/docs-mikado index e28c3f1..17a363d 100755 --- a/mise-tasks/docs-mikado +++ b/mise-tasks/docs-mikado @@ -238,7 +238,7 @@ def classify_branch_position(commits: list[dict]) -> str: return "unknown" -FORGE_API = "https://forge.ops.eblu.me/api/v1" +FORGE_API = "https://forge.eblu.me/api/v1" def find_pr_for_branch(branch: str) -> dict | None: diff --git a/mise-tasks/fly-setup b/mise-tasks/fly-setup index 63304db..0c5cb56 100755 --- a/mise-tasks/fly-setup +++ b/mise-tasks/fly-setup @@ -22,6 +22,7 @@ echo "IPs allocated" # Add certs for all public domains (idempotent — fly ignores duplicates) fly certs add docs.eblu.me -a "$APP" 2>/dev/null || true fly certs add cv.eblu.me -a "$APP" 2>/dev/null || true +fly certs add forge.eblu.me -a "$APP" 2>/dev/null || true echo "Certificates configured" echo "Done. Run 'mise run fly-deploy' to deploy." diff --git a/mise-tasks/mirror-create b/mise-tasks/mirror-create index 75e0d3e..b0e82a0 100755 --- a/mise-tasks/mirror-create +++ b/mise-tasks/mirror-create @@ -6,7 +6,7 @@ #USAGE flag "--dry-run" help="Show what would be done without creating" set -euo pipefail -FORGE_API="https://forge.ops.eblu.me/api/v1" +FORGE_API="https://forge.eblu.me/api/v1" ORG="mirrors" OP_TOKEN_REF="op://blumeops/w3663ffnvkewbftncqxtcpeavy/api-token" OP_GITHUB_PAT_REF="op://blumeops/w3663ffnvkewbftncqxtcpeavy/github-mirror-pat" @@ -72,7 +72,7 @@ http_code=$(curl -s -o /tmp/mirror-create-response.json -w "%{http_code}" \ -d "$payload") if [[ "$http_code" == "201" ]]; then - echo "Created mirror: https://forge.ops.eblu.me/${ORG}/${repo_name}" + echo "Created mirror: https://forge.eblu.me/${ORG}/${repo_name}" else echo "Error (HTTP $http_code):" cat /tmp/mirror-create-response.json diff --git a/mise-tasks/pr-comments b/mise-tasks/pr-comments index 0f6b840..9c33f9d 100755 --- a/mise-tasks/pr-comments +++ b/mise-tasks/pr-comments @@ -20,7 +20,7 @@ import httpx from rich.console import Console from rich.text import Text -FORGE_API_BASE = "https://forge.ops.eblu.me/api/v1" +FORGE_API_BASE = "https://forge.eblu.me/api/v1" REPO_OWNER = "eblume" REPO_NAME = "blumeops" diff --git a/mise-tasks/runner-logs b/mise-tasks/runner-logs index 536f1fe..22e4640 100755 --- a/mise-tasks/runner-logs +++ b/mise-tasks/runner-logs @@ -27,7 +27,7 @@ import typer from rich.console import Console from rich.table import Table -FORGE_API = "https://forge.ops.eblu.me/api/v1" +FORGE_API = "https://forge.eblu.me/api/v1" REPO = "eblume/blumeops" ACTIONS_LOG_DIR = "/opt/homebrew/var/forgejo/data/actions_log/eblume/blumeops" diff --git a/mise-tasks/services-check b/mise-tasks/services-check index d09b237..dce18ee 100755 --- a/mise-tasks/services-check +++ b/mise-tasks/services-check @@ -70,7 +70,7 @@ check_http "Prometheus" "https://prometheus.ops.eblu.me/-/healthy" check_http "Loki" "https://loki.ops.eblu.me/ready" check_http "Grafana" "https://grafana.ops.eblu.me/api/health" check_http "ArgoCD" "https://argocd.ops.eblu.me/healthz" -check_http "Forgejo" "https://forge.ops.eblu.me/" +check_http "Forgejo" "https://forge.eblu.me/" check_http "Zot Registry" "https://registry.ops.eblu.me/v2/_catalog" check_http "Kiwix" "https://kiwix.ops.eblu.me/" check_http "Miniflux" "https://feed.ops.eblu.me/healthcheck" diff --git a/nixos/ringtail/configuration.nix b/nixos/ringtail/configuration.nix index 5a0035b..562cfdb 100644 --- a/nixos/ringtail/configuration.nix +++ b/nixos/ringtail/configuration.nix @@ -496,7 +496,7 @@ in instances.nix_container_builder = { enable = true; name = "ringtail-nix-builder"; - url = "https://forge.ops.eblu.me"; + url = "https://forge.eblu.me"; tokenFile = "/etc/forgejo-runner/token.env"; labels = [ "nix-container-builder:host" ]; hostPackages = with pkgs; [ diff --git a/pulumi/gandi/__main__.py b/pulumi/gandi/__main__.py index df6ad34..e448ed2 100644 --- a/pulumi/gandi/__main__.py +++ b/pulumi/gandi/__main__.py @@ -76,6 +76,15 @@ cv_public = gandi.livedns.Record( values=["blumeops-proxy.fly.dev."], ) +forge_public = gandi.livedns.Record( + "forge-public", + zone=domain, + name="forge", + type="CNAME", + ttl=300, + values=["blumeops-proxy.fly.dev."], +) + # ============== Exports ============== pulumi.export("domain", domain) pulumi.export("wildcard_fqdn", f"*.{subdomain}.{domain}") @@ -83,3 +92,4 @@ pulumi.export("base_fqdn", f"{subdomain}.{domain}") pulumi.export("target_ip", tailscale_ip) pulumi.export("docs_public_fqdn", f"docs.{domain}") pulumi.export("cv_public_fqdn", f"cv.{domain}") +pulumi.export("forge_public_fqdn", f"forge.{domain}")