From f80aae693f694e0abd24115503c9f6b114547b32 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Wed, 3 Jun 2026 11:02:03 -0700 Subject: [PATCH 1/5] Wire ringtail blumeops-pg into backups + Grafana MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wave-1 cutover moved paperless + teslamate (postgres) and mealie (SQLite) to ringtail, but borgmatic and the Grafana TeslaMate datasource still pointed at the minikube-hosted copies — so the migrated live data was unbacked and the dashboards would break when the minikube DBs are dropped. - Add a Tailscale Service (blumeops-pg-ringtail) + Caddy L4 route pg.ops.eblu.me:5434 for the ringtail blumeops-pg cluster. - Repoint borgmatic teslamate + paperless postgres dumps to :5434 and the mealie SQLite dump to the ringtail kubectl target (ssh:eblume@ringtail). - Repoint the Grafana TeslaMate datasource to pg.ops.eblu.me:5434. Closes the post-cutover backup gap and unblocks the wave-1 decommission. Co-Authored-By: Claude Opus 4.8 (1M context) --- ansible/roles/borgmatic/defaults/main.yml | 16 +++++++------ ansible/roles/caddy/defaults/main.yml | 2 ++ .../databases-ringtail/kustomization.yaml | 1 + .../service-blumeops-pg-tailscale.yaml | 24 +++++++++++++++++++ argocd/manifests/grafana/datasources.yaml | 4 +++- ...ckup-grafana-ringtail-blumeops-pg.infra.md | 8 +++++++ 6 files changed, 47 insertions(+), 8 deletions(-) create mode 100644 argocd/manifests/databases-ringtail/service-blumeops-pg-tailscale.yaml create mode 100644 docs/changelog.d/backup-grafana-ringtail-blumeops-pg.infra.md diff --git a/ansible/roles/borgmatic/defaults/main.yml b/ansible/roles/borgmatic/defaults/main.yml index 3a89a09..a743161 100644 --- a/ansible/roles/borgmatic/defaults/main.yml +++ b/ansible/roles/borgmatic/defaults/main.yml @@ -56,8 +56,9 @@ borgmatic_k8s_sqlite_dumps: namespace: mealie label_selector: app=mealie db_path: /app/data/mealie.db - # local kubectl, --context=minikube (indri's only configured ctx) - target: local:minikube + # migrated to ringtail (wave-1); ssh to ringtail and run k3s kubectl + # there, same as shower below. + target: ssh:eblume@ringtail - name: shower namespace: shower label_selector: app=shower @@ -102,17 +103,18 @@ borgmatic_postgresql_databases: hostname: pg.ops.eblu.me port: 5432 username: borgmatic - - name: teslamate - hostname: pg.ops.eblu.me - port: 5432 - username: borgmatic - name: authentik hostname: pg.ops.eblu.me port: 5432 username: borgmatic + # migrated to ringtail blumeops-pg (wave-1); port 5434 = Caddy L4 route + - name: teslamate + hostname: pg.ops.eblu.me + port: 5434 + username: borgmatic - name: paperless hostname: pg.ops.eblu.me - port: 5432 + port: 5434 username: borgmatic # immich-pg cluster (VectorChord) via Caddy L4 on port 5433 - name: immich diff --git a/ansible/roles/caddy/defaults/main.yml b/ansible/roles/caddy/defaults/main.yml index da6f3f9..363d09e 100644 --- a/ansible/roles/caddy/defaults/main.yml +++ b/ansible/roles/caddy/defaults/main.yml @@ -117,6 +117,8 @@ caddy_tcp_services: backend: "pg.tail8d86e.ts.net:5432" # PostgreSQL (blumeops-pg) - port: 5433 backend: "immich-pg.tail8d86e.ts.net:5432" # PostgreSQL (immich-pg) + - port: 5434 + backend: "blumeops-pg-ringtail.tail8d86e.ts.net:5432" # PostgreSQL (blumeops-pg on ringtail) - port: "{{ sifaka_node_exporter_port }}" backend: "sifaka:{{ sifaka_node_exporter_port }}" # Sifaka node_exporter - port: "{{ sifaka_smartctl_exporter_port }}" diff --git a/argocd/manifests/databases-ringtail/kustomization.yaml b/argocd/manifests/databases-ringtail/kustomization.yaml index 2bc2af3..143345c 100644 --- a/argocd/manifests/databases-ringtail/kustomization.yaml +++ b/argocd/manifests/databases-ringtail/kustomization.yaml @@ -9,6 +9,7 @@ resources: - service-immich-pg-tailscale.yaml # wave-1 indri-k8s decommission: blumeops-pg (paperless + teslamate) - blumeops-pg.yaml + - service-blumeops-pg-tailscale.yaml - external-secret-eblume.yaml - external-secret-borgmatic.yaml - external-secret-paperless.yaml diff --git a/argocd/manifests/databases-ringtail/service-blumeops-pg-tailscale.yaml b/argocd/manifests/databases-ringtail/service-blumeops-pg-tailscale.yaml new file mode 100644 index 0000000..f7ca5ef --- /dev/null +++ b/argocd/manifests/databases-ringtail/service-blumeops-pg-tailscale.yaml @@ -0,0 +1,24 @@ +# Tailscale LoadBalancer for the ringtail blumeops-pg cluster. +# Canonical hostname: blumeops-pg-ringtail.tail8d86e.ts.net (distinct from +# the minikube blumeops-pg, which still owns pg.tail8d86e.ts.net until the +# wave-1 decommission). Borgmatic on indri and the Grafana TeslaMate +# datasource reach it via the Caddy L4 route pg.ops.eblu.me:5434. +apiVersion: v1 +kind: Service +metadata: + name: blumeops-pg-tailscale + namespace: databases + annotations: + tailscale.com/hostname: "blumeops-pg-ringtail" + tailscale.com/proxy-class: "default" +spec: + type: LoadBalancer + loadBalancerClass: tailscale + selector: + cnpg.io/cluster: blumeops-pg + role: primary + ports: + - name: postgresql + port: 5432 + targetPort: 5432 + protocol: TCP diff --git a/argocd/manifests/grafana/datasources.yaml b/argocd/manifests/grafana/datasources.yaml index 5a3d0f3..64ed2bf 100644 --- a/argocd/manifests/grafana/datasources.yaml +++ b/argocd/manifests/grafana/datasources.yaml @@ -63,5 +63,7 @@ datasources: password: $TESLAMATE_DB_PASSWORD type: postgres uid: TeslaMate - url: blumeops-pg-rw.databases.svc.cluster.local:5432 + # teslamate DB migrated to ringtail blumeops-pg (wave-1); reached via the + # Caddy L4 route on indri (pg.ops.eblu.me:5434 -> blumeops-pg-ringtail). + url: pg.ops.eblu.me:5434 user: teslamate diff --git a/docs/changelog.d/backup-grafana-ringtail-blumeops-pg.infra.md b/docs/changelog.d/backup-grafana-ringtail-blumeops-pg.infra.md new file mode 100644 index 0000000..33b041f --- /dev/null +++ b/docs/changelog.d/backup-grafana-ringtail-blumeops-pg.infra.md @@ -0,0 +1,8 @@ +Wire the ringtail `blumeops-pg` cluster (which holds the wave-1-migrated +paperless + teslamate databases) into backups and Grafana. Adds a Tailscale +LoadBalancer Service (`blumeops-pg-ringtail.tail8d86e.ts.net`) and a Caddy L4 +route (`pg.ops.eblu.me:5434`), then repoints borgmatic's `teslamate` + +`paperless` postgres dumps and the `mealie` SQLite dump at ringtail, and the +Grafana TeslaMate datasource at the ringtail DB. Closes the backup gap that +opened at cutover (the migrated live data was still being backed up from the +now-frozen minikube copies) and unblocks the wave-1 decommission. -- 2.50.1 (Apple Git-155) From 83b4306edb85a2f631d5965f699159fa05c89a4a Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Wed, 3 Jun 2026 11:15:18 -0700 Subject: [PATCH 2/5] borgmatic: add .pgpass entry for ringtail blumeops-pg (port 5434) The .pgpass is a hardcoded port list, not derived from the database list, so repointing teslamate/paperless to :5434 left them without credentials (pg_dump would fail 'no password supplied'). Add the 5434 entry. Co-Authored-By: Claude Opus 4.8 (1M context) --- ansible/roles/borgmatic/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/roles/borgmatic/tasks/main.yml b/ansible/roles/borgmatic/tasks/main.yml index 4ac242c..36d3bb6 100644 --- a/ansible/roles/borgmatic/tasks/main.yml +++ b/ansible/roles/borgmatic/tasks/main.yml @@ -19,8 +19,10 @@ ansible.builtin.copy: content: | # Managed by ansible (borgmatic role) - k8s PostgreSQL backup credentials + # 5432 = minikube blumeops-pg, 5433 = immich-pg, 5434 = ringtail blumeops-pg pg.ops.eblu.me:5432:*:borgmatic:{{ borgmatic_db_password }} pg.ops.eblu.me:5433:*:borgmatic:{{ borgmatic_db_password }} + pg.ops.eblu.me:5434:*:borgmatic:{{ borgmatic_db_password }} dest: ~/.pgpass mode: '0600' no_log: true -- 2.50.1 (Apple Git-155) From 22cfd864e2a72f6c26ee7155cf7c6046f8368f59 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Wed, 3 Jun 2026 11:26:19 -0700 Subject: [PATCH 3/5] mealie: add python3 to image for the borgmatic SQLite dump helper The borgmatic k8s-sqlite-dump helper runs python3 (stdlib sqlite3 .backup) inside the pod, but the minimal Nix mealie image had no python3 on PATH, so the mealie SQLite backup produced a 0-byte file. Add pkgs.python3 (same nixpkgs build mealie uses, so negligible closure growth), matching shower. Co-Authored-By: Claude Opus 4.8 (1M context) --- containers/mealie/default.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/containers/mealie/default.nix b/containers/mealie/default.nix index fdb1430..e55efe3 100644 --- a/containers/mealie/default.nix +++ b/containers/mealie/default.nix @@ -48,6 +48,10 @@ pkgs.dockerTools.buildLayeredImage { pkgs.coreutils pkgs.cacert pkgs.tzdata + # python3 (stdlib sqlite3) for the borgmatic k8s-sqlite-dump helper, + # which runs `python3 -c "...sqlite3...backup..."` inside the pod. + # Same nixpkgs python mealie is built against, so ~no added closure. + pkgs.python3 ]; config = { -- 2.50.1 (Apple Git-155) From 0d6584f3c36bdae1f34513343363e1b32808ee8f Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Wed, 3 Jun 2026 11:27:37 -0700 Subject: [PATCH 4/5] mealie-ringtail: use python3-enabled image v3.16.0-22cfd86-nix Co-Authored-By: Claude Opus 4.8 (1M context) --- argocd/manifests/mealie-ringtail/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/argocd/manifests/mealie-ringtail/kustomization.yaml b/argocd/manifests/mealie-ringtail/kustomization.yaml index 2b6a7ef..7679032 100644 --- a/argocd/manifests/mealie-ringtail/kustomization.yaml +++ b/argocd/manifests/mealie-ringtail/kustomization.yaml @@ -12,4 +12,4 @@ resources: images: - name: registry.ops.eblu.me/blumeops/mealie - newTag: v3.16.0-fcac8e5-nix + newTag: v3.16.0-22cfd86-nix -- 2.50.1 (Apple Git-155) From 71d5759eb5bd61dc4f030fe01532c835b3bf5d69 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Wed, 3 Jun 2026 11:32:17 -0700 Subject: [PATCH 5/5] borgmatic: stage SQLite dump beside source DB (minimal images lack /tmp) The helper wrote its staging copy to /tmp, but minimal nix images (mealie) have no /tmp, so the dump failed 'unable to open database file'. Write it next to the source DB instead (always on a writable volume); works for any image, shower included. Co-Authored-By: Claude Opus 4.8 (1M context) --- ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 b/ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 index 323e717..9cc24da 100644 --- a/ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 +++ b/ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 @@ -28,7 +28,9 @@ db_path=${4:?missing db path} name=${5:?missing name} dump_target=${6:?missing dump target} -pod_tmp="/tmp/${name}-backup.db" +# Stage the backup next to the source DB (a guaranteed-writable volume); +# minimal nix images (e.g. mealie) have no /tmp. +pod_tmp="$(dirname "$db_path")/.borgmatic-backup-${name}.db" python_backup='import sqlite3; sqlite3.connect("'"$db_path"'").backup(sqlite3.connect("'"$pod_tmp"'"))' -- 2.50.1 (Apple Git-155)