diff --git a/ansible/roles/borgmatic/defaults/main.yml b/ansible/roles/borgmatic/defaults/main.yml index 3a89a09..a743161 100644 --- a/ansible/roles/borgmatic/defaults/main.yml +++ b/ansible/roles/borgmatic/defaults/main.yml @@ -56,8 +56,9 @@ borgmatic_k8s_sqlite_dumps: namespace: mealie label_selector: app=mealie db_path: /app/data/mealie.db - # local kubectl, --context=minikube (indri's only configured ctx) - target: local:minikube + # migrated to ringtail (wave-1); ssh to ringtail and run k3s kubectl + # there, same as shower below. + target: ssh:eblume@ringtail - name: shower namespace: shower label_selector: app=shower @@ -102,17 +103,18 @@ borgmatic_postgresql_databases: hostname: pg.ops.eblu.me port: 5432 username: borgmatic - - name: teslamate - hostname: pg.ops.eblu.me - port: 5432 - username: borgmatic - name: authentik hostname: pg.ops.eblu.me port: 5432 username: borgmatic + # migrated to ringtail blumeops-pg (wave-1); port 5434 = Caddy L4 route + - name: teslamate + hostname: pg.ops.eblu.me + port: 5434 + username: borgmatic - name: paperless hostname: pg.ops.eblu.me - port: 5432 + port: 5434 username: borgmatic # immich-pg cluster (VectorChord) via Caddy L4 on port 5433 - name: immich diff --git a/ansible/roles/borgmatic/tasks/main.yml b/ansible/roles/borgmatic/tasks/main.yml index 4ac242c..36d3bb6 100644 --- a/ansible/roles/borgmatic/tasks/main.yml +++ b/ansible/roles/borgmatic/tasks/main.yml @@ -19,8 +19,10 @@ ansible.builtin.copy: content: | # Managed by ansible (borgmatic role) - k8s PostgreSQL backup credentials + # 5432 = minikube blumeops-pg, 5433 = immich-pg, 5434 = ringtail blumeops-pg pg.ops.eblu.me:5432:*:borgmatic:{{ borgmatic_db_password }} pg.ops.eblu.me:5433:*:borgmatic:{{ borgmatic_db_password }} + pg.ops.eblu.me:5434:*:borgmatic:{{ borgmatic_db_password }} dest: ~/.pgpass mode: '0600' no_log: true diff --git a/ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 b/ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 index 323e717..9cc24da 100644 --- a/ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 +++ b/ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 @@ -28,7 +28,9 @@ db_path=${4:?missing db path} name=${5:?missing name} dump_target=${6:?missing dump target} -pod_tmp="/tmp/${name}-backup.db" +# Stage the backup next to the source DB (a guaranteed-writable volume); +# minimal nix images (e.g. mealie) have no /tmp. +pod_tmp="$(dirname "$db_path")/.borgmatic-backup-${name}.db" python_backup='import sqlite3; sqlite3.connect("'"$db_path"'").backup(sqlite3.connect("'"$pod_tmp"'"))' diff --git a/ansible/roles/caddy/defaults/main.yml b/ansible/roles/caddy/defaults/main.yml index da6f3f9..363d09e 100644 --- a/ansible/roles/caddy/defaults/main.yml +++ b/ansible/roles/caddy/defaults/main.yml @@ -117,6 +117,8 @@ caddy_tcp_services: backend: "pg.tail8d86e.ts.net:5432" # PostgreSQL (blumeops-pg) - port: 5433 backend: "immich-pg.tail8d86e.ts.net:5432" # PostgreSQL (immich-pg) + - port: 5434 + backend: "blumeops-pg-ringtail.tail8d86e.ts.net:5432" # PostgreSQL (blumeops-pg on ringtail) - port: "{{ sifaka_node_exporter_port }}" backend: "sifaka:{{ sifaka_node_exporter_port }}" # Sifaka node_exporter - port: "{{ sifaka_smartctl_exporter_port }}" diff --git a/argocd/manifests/databases-ringtail/kustomization.yaml b/argocd/manifests/databases-ringtail/kustomization.yaml index 2bc2af3..143345c 100644 --- a/argocd/manifests/databases-ringtail/kustomization.yaml +++ b/argocd/manifests/databases-ringtail/kustomization.yaml @@ -9,6 +9,7 @@ resources: - service-immich-pg-tailscale.yaml # wave-1 indri-k8s decommission: blumeops-pg (paperless + teslamate) - blumeops-pg.yaml + - service-blumeops-pg-tailscale.yaml - external-secret-eblume.yaml - external-secret-borgmatic.yaml - external-secret-paperless.yaml diff --git a/argocd/manifests/databases-ringtail/service-blumeops-pg-tailscale.yaml b/argocd/manifests/databases-ringtail/service-blumeops-pg-tailscale.yaml new file mode 100644 index 0000000..f7ca5ef --- /dev/null +++ b/argocd/manifests/databases-ringtail/service-blumeops-pg-tailscale.yaml @@ -0,0 +1,24 @@ +# Tailscale LoadBalancer for the ringtail blumeops-pg cluster. +# Canonical hostname: blumeops-pg-ringtail.tail8d86e.ts.net (distinct from +# the minikube blumeops-pg, which still owns pg.tail8d86e.ts.net until the +# wave-1 decommission). Borgmatic on indri and the Grafana TeslaMate +# datasource reach it via the Caddy L4 route pg.ops.eblu.me:5434. +apiVersion: v1 +kind: Service +metadata: + name: blumeops-pg-tailscale + namespace: databases + annotations: + tailscale.com/hostname: "blumeops-pg-ringtail" + tailscale.com/proxy-class: "default" +spec: + type: LoadBalancer + loadBalancerClass: tailscale + selector: + cnpg.io/cluster: blumeops-pg + role: primary + ports: + - name: postgresql + port: 5432 + targetPort: 5432 + protocol: TCP diff --git a/argocd/manifests/grafana/datasources.yaml b/argocd/manifests/grafana/datasources.yaml index 5a3d0f3..64ed2bf 100644 --- a/argocd/manifests/grafana/datasources.yaml +++ b/argocd/manifests/grafana/datasources.yaml @@ -63,5 +63,7 @@ datasources: password: $TESLAMATE_DB_PASSWORD type: postgres uid: TeslaMate - url: blumeops-pg-rw.databases.svc.cluster.local:5432 + # teslamate DB migrated to ringtail blumeops-pg (wave-1); reached via the + # Caddy L4 route on indri (pg.ops.eblu.me:5434 -> blumeops-pg-ringtail). + url: pg.ops.eblu.me:5434 user: teslamate diff --git a/argocd/manifests/mealie-ringtail/kustomization.yaml b/argocd/manifests/mealie-ringtail/kustomization.yaml index 2b6a7ef..7679032 100644 --- a/argocd/manifests/mealie-ringtail/kustomization.yaml +++ b/argocd/manifests/mealie-ringtail/kustomization.yaml @@ -12,4 +12,4 @@ resources: images: - name: registry.ops.eblu.me/blumeops/mealie - newTag: v3.16.0-fcac8e5-nix + newTag: v3.16.0-22cfd86-nix diff --git a/containers/mealie/default.nix b/containers/mealie/default.nix index fdb1430..e55efe3 100644 --- a/containers/mealie/default.nix +++ b/containers/mealie/default.nix @@ -48,6 +48,10 @@ pkgs.dockerTools.buildLayeredImage { pkgs.coreutils pkgs.cacert pkgs.tzdata + # python3 (stdlib sqlite3) for the borgmatic k8s-sqlite-dump helper, + # which runs `python3 -c "...sqlite3...backup..."` inside the pod. + # Same nixpkgs python mealie is built against, so ~no added closure. + pkgs.python3 ]; config = { diff --git a/docs/changelog.d/backup-grafana-ringtail-blumeops-pg.infra.md b/docs/changelog.d/backup-grafana-ringtail-blumeops-pg.infra.md new file mode 100644 index 0000000..33b041f --- /dev/null +++ b/docs/changelog.d/backup-grafana-ringtail-blumeops-pg.infra.md @@ -0,0 +1,8 @@ +Wire the ringtail `blumeops-pg` cluster (which holds the wave-1-migrated +paperless + teslamate databases) into backups and Grafana. Adds a Tailscale +LoadBalancer Service (`blumeops-pg-ringtail.tail8d86e.ts.net`) and a Caddy L4 +route (`pg.ops.eblu.me:5434`), then repoints borgmatic's `teslamate` + +`paperless` postgres dumps and the `mealie` SQLite dump at ringtail, and the +Grafana TeslaMate datasource at the ringtail DB. Closes the backup gap that +opened at cutover (the migrated live data was still being backed up from the +now-frozen minikube copies) and unblocks the wave-1 decommission.