From 66badfafd1b4d296951125af711cafe359d0347b Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Sun, 25 Jan 2026 12:56:31 -0800 Subject: [PATCH] Migrate k8s services to Caddy (*.ops.eblu.me) (#59) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Add Caddy reverse proxy routes for all k8s services (grafana, argocd, prometheus, loki, miniflux, devpi, kiwix, torrent, teslamate) - Add PostgreSQL via Caddy L4 TCP proxy on port 5432 - Caddy proxies to existing Tailscale endpoints - traffic stays local on indri - Both `*.ops.eblu.me` and `*.tail8d86e.ts.net` URLs continue to work ## Updated References - Alloy: prometheus/loki push endpoints → `*.ops.eblu.me` - Borgmatic: PostgreSQL backup host → `pg.ops.eblu.me` - Devpi: DEVPI_OUTSIDE_URL → `pypi.ops.eblu.me` - indri-services-check: health check URLs - CLAUDE.md: argocd login command ## Deployment and Testing - [ ] Run `mise run provision-indri -- --tags caddy` to deploy new Caddy config - [ ] Test HTTP services: `curl https://grafana.ops.eblu.me/api/health` - [ ] Test PostgreSQL: `pg_isready -h pg.ops.eblu.me -p 5432` - [ ] Run `mise run provision-indri -- --tags alloy` to update Alloy endpoints - [ ] Run `mise run provision-indri -- --tags borgmatic` to update borgmatic - [ ] Sync devpi in ArgoCD: `argocd app sync devpi` - [ ] Re-login to ArgoCD: `argocd login argocd.ops.eblu.me ...` - [ ] Run `mise run indri-services-check` to verify all services 🤖 Generated with [Claude Code](https://claude.com/claude-code) Reviewed-on: https://forge.ops.eblu.me/eblume/blumeops/pulls/59 --- CLAUDE.md | 2 +- ansible/roles/alloy/defaults/main.yml | 8 ++--- ansible/roles/borgmatic/defaults/main.yml | 6 ++-- ansible/roles/borgmatic/tasks/main.yml | 2 +- ansible/roles/caddy/defaults/main.yml | 40 ++++++++++++++++++---- ansible/roles/caddy/templates/Caddyfile.j2 | 10 +++--- argocd/manifests/devpi/statefulset.yaml | 2 +- mise-tasks/indri-services-check | 22 ++++++------ 8 files changed, 59 insertions(+), 33 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8af96e6..793ad55 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -103,7 +103,7 @@ Note: The user has fish abbreviations `ki` for `kubectl --context=minikube-indri **ArgoCD login (when token expires):** ```fish -argocd login argocd.tail8d86e.ts.net --username admin --password "$(op --vault vg6xf6vvfmoh5hqjjhlhbeoaie item get srogeebssulhtb6tnqd7ls6qey --fields password --reveal)" +argocd login argocd.ops.eblu.me --username admin --password "$(op --vault vg6xf6vvfmoh5hqjjhlhbeoaie item get srogeebssulhtb6tnqd7ls6qey --fields password --reveal)" ``` ### Indri Services (via Ansible) diff --git a/ansible/roles/alloy/defaults/main.yml b/ansible/roles/alloy/defaults/main.yml index 4f0b777..747a926 100644 --- a/ansible/roles/alloy/defaults/main.yml +++ b/ansible/roles/alloy/defaults/main.yml @@ -32,11 +32,11 @@ alloy_log_dir: /Users/erichblume/Library/Logs # Textfile collector directory (same as node_exporter for compatibility) alloy_textfile_dir: /opt/homebrew/var/node_exporter/textfile -# Prometheus remote write endpoint (k8s via Tailscale) -alloy_prometheus_url: "https://prometheus.tail8d86e.ts.net/api/v1/write" +# Prometheus remote write endpoint (k8s via Caddy) +alloy_prometheus_url: "https://prometheus.ops.eblu.me/api/v1/write" -# Loki endpoint (k8s via Tailscale) -alloy_loki_url: "https://loki.tail8d86e.ts.net/loki/api/v1/push" +# Loki endpoint (k8s via Caddy) +alloy_loki_url: "https://loki.ops.eblu.me/loki/api/v1/push" # Instance label for metrics alloy_instance_label: indri diff --git a/ansible/roles/borgmatic/defaults/main.yml b/ansible/roles/borgmatic/defaults/main.yml index 245f6ed..db09f93 100644 --- a/ansible/roles/borgmatic/defaults/main.yml +++ b/ansible/roles/borgmatic/defaults/main.yml @@ -41,12 +41,12 @@ borgmatic_keep_yearly: 1000 # pg_dump_command must be full path since LaunchAgent doesn't have homebrew in PATH borgmatic_pg_dump_command: /opt/homebrew/opt/postgresql@18/bin/pg_dump borgmatic_postgresql_databases: - # k8s PostgreSQL (CloudNativePG) + # k8s PostgreSQL (CloudNativePG) via Caddy L4 proxy - name: miniflux - hostname: pg.tail8d86e.ts.net + hostname: pg.ops.eblu.me port: 5432 username: borgmatic - name: teslamate - hostname: pg.tail8d86e.ts.net + hostname: pg.ops.eblu.me port: 5432 username: borgmatic diff --git a/ansible/roles/borgmatic/tasks/main.yml b/ansible/roles/borgmatic/tasks/main.yml index e5cc1f0..99eda45 100644 --- a/ansible/roles/borgmatic/tasks/main.yml +++ b/ansible/roles/borgmatic/tasks/main.yml @@ -14,7 +14,7 @@ ansible.builtin.copy: content: | # Managed by ansible (borgmatic role) - k8s PostgreSQL backup credentials - pg.tail8d86e.ts.net:5432:*:borgmatic:{{ borgmatic_db_password }} + pg.ops.eblu.me:5432:*:borgmatic:{{ borgmatic_db_password }} dest: ~/.pgpass mode: '0600' no_log: true diff --git a/ansible/roles/caddy/defaults/main.yml b/ansible/roles/caddy/defaults/main.yml index 8490b54..08eb341 100644 --- a/ansible/roles/caddy/defaults/main.yml +++ b/ansible/roles/caddy/defaults/main.yml @@ -29,14 +29,40 @@ caddy_services: host: "registry.{{ caddy_domain }}" backend: "http://localhost:5050" - # K8s services (via minikube NodePort or ClusterIP) - # These will be configured once we determine the correct backend URLs - # - name: grafana - # host: "grafana.{{ caddy_domain }}" - # backend: "http://minikube-ip:nodeport" + # K8s services (via Tailscale Ingress) + # Caddy proxies to existing Tailscale endpoints - traffic stays local + - name: grafana + host: "grafana.{{ caddy_domain }}" + backend: "https://grafana.tail8d86e.ts.net" + - name: argocd + host: "argocd.{{ caddy_domain }}" + backend: "https://argocd.tail8d86e.ts.net" + - name: prometheus + host: "prometheus.{{ caddy_domain }}" + backend: "https://prometheus.tail8d86e.ts.net" + - name: loki + host: "loki.{{ caddy_domain }}" + backend: "https://loki.tail8d86e.ts.net" + - name: miniflux + host: "feed.{{ caddy_domain }}" + backend: "https://feed.tail8d86e.ts.net" + - name: devpi + host: "pypi.{{ caddy_domain }}" + backend: "https://pypi.tail8d86e.ts.net" + - name: kiwix + host: "kiwix.{{ caddy_domain }}" + backend: "https://kiwix.tail8d86e.ts.net" + - name: torrent + host: "torrent.{{ caddy_domain }}" + backend: "https://torrent.tail8d86e.ts.net" + - name: teslamate + host: "tesla.{{ caddy_domain }}" + backend: "https://tesla.tail8d86e.ts.net" -# SSH services (Layer 4 TCP proxy) +# Layer 4 (TCP) services # Format: { port: external_port, backend: "host:port" } -caddy_ssh_services: +caddy_tcp_services: - port: 2222 backend: "localhost:2200" # Forgejo SSH + - port: 5432 + backend: "pg.tail8d86e.ts.net:5432" # PostgreSQL diff --git a/ansible/roles/caddy/templates/Caddyfile.j2 b/ansible/roles/caddy/templates/Caddyfile.j2 index 36c1c8d..2bc4c87 100644 --- a/ansible/roles/caddy/templates/Caddyfile.j2 +++ b/ansible/roles/caddy/templates/Caddyfile.j2 @@ -8,13 +8,13 @@ # Global options admin off -{% if caddy_ssh_services %} - # Layer 4 (TCP) routing for SSH services +{% if caddy_tcp_services %} + # Layer 4 (TCP) routing layer4 { -{% for ssh_svc in caddy_ssh_services %} - :{{ ssh_svc.port }} { +{% for tcp_svc in caddy_tcp_services %} + :{{ tcp_svc.port }} { route { - proxy {{ ssh_svc.backend }} + proxy {{ tcp_svc.backend }} } } {% endfor %} diff --git a/argocd/manifests/devpi/statefulset.yaml b/argocd/manifests/devpi/statefulset.yaml index 1aad6dd..77a4c56 100644 --- a/argocd/manifests/devpi/statefulset.yaml +++ b/argocd/manifests/devpi/statefulset.yaml @@ -27,7 +27,7 @@ spec: name: devpi-root key: password - name: DEVPI_OUTSIDE_URL - value: "https://pypi.tail8d86e.ts.net" + value: "https://pypi.ops.eblu.me" ports: - containerPort: 3141 name: http diff --git a/mise-tasks/indri-services-check b/mise-tasks/indri-services-check index 19d0928..fa05b7f 100755 --- a/mise-tasks/indri-services-check +++ b/mise-tasks/indri-services-check @@ -65,22 +65,22 @@ check_service "k8s-apiserver (indri)" "ssh indri 'kubectl get --raw /healthz'" check_service "k8s-apiserver (remote)" "kubectl --kubeconfig=$HOME/.kube/minikube-indri/config.yml --context=minikube-indri get --raw /healthz" echo "" -echo "HTTP endpoints (via Tailscale):" -check_http "Prometheus" "https://prometheus.tail8d86e.ts.net/-/healthy" -check_http "Loki" "https://loki.tail8d86e.ts.net/ready" -check_http "Grafana" "https://grafana.tail8d86e.ts.net/api/health" -check_http "ArgoCD" "https://argocd.tail8d86e.ts.net/healthz" +echo "HTTP endpoints (via Caddy):" +check_http "Prometheus" "https://prometheus.ops.eblu.me/-/healthy" +check_http "Loki" "https://loki.ops.eblu.me/ready" +check_http "Grafana" "https://grafana.ops.eblu.me/api/health" +check_http "ArgoCD" "https://argocd.ops.eblu.me/healthz" check_http "Forgejo" "https://forge.ops.eblu.me/" check_http "Zot Registry" "https://registry.ops.eblu.me/v2/_catalog" -check_http "Kiwix" "https://kiwix.tail8d86e.ts.net/" -check_http "Miniflux" "https://feed.tail8d86e.ts.net/healthcheck" -check_http "TeslaMate" "https://tesla.tail8d86e.ts.net/" -check_http "Devpi" "https://pypi.tail8d86e.ts.net/+api" -check_http "Transmission" "https://torrent.tail8d86e.ts.net/" +check_http "Kiwix" "https://kiwix.ops.eblu.me/" +check_http "Miniflux" "https://feed.ops.eblu.me/healthcheck" +check_http "TeslaMate" "https://tesla.ops.eblu.me/" +check_http "Devpi" "https://pypi.ops.eblu.me/+api" +check_http "Transmission" "https://torrent.ops.eblu.me/" echo "" echo "Database:" -check_service "PostgreSQL (k8s)" "pg_isready -h pg.tail8d86e.ts.net -p 5432" +check_service "PostgreSQL (k8s)" "pg_isready -h pg.ops.eblu.me -p 5432" echo "" echo "Kubernetes pods:"