From ca0c9354ee918124819fd4decefd1d3bb20a26c9 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 27 Mar 2026 16:59:58 -0700 Subject: [PATCH] Add borgmatic backups for authentik and immich databases (#314) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Add `authentik` database (blumeops-pg cluster) to borgmatic pg_dump backups - Add `immich` database (immich-pg cluster) to borgmatic pg_dump backups - For immich-pg: new borgmatic managed role with `pg_read_all_data`, ExternalSecret, Tailscale LoadBalancer service, and Caddy L4 TCP proxy on port 5433 - Update backup docs to reflect all four CNPG databases + mealie SQLite ## Deploy plan Deploy order matters — k8s resources must exist before ansible can route to them: 1. **ArgoCD (databases app):** sync to pick up immich-pg borgmatic role, ExternalSecret, and Tailscale service ``` argocd app set blumeops-pg --revision feature/borgmatic-all-pg-backups argocd app sync blumeops-pg ``` 2. **Wait** for `immich-pg-tailscale` service to get a Tailscale IP and `immich-pg.tail8d86e.ts.net` to resolve 3. **Ansible (caddy):** deploy Caddy L4 route for port 5433 ``` mise run provision-indri -- --tags caddy ``` 4. **Ansible (borgmatic):** deploy updated config and .pgpass ``` mise run provision-indri -- --tags borgmatic ``` 5. **Verify:** trigger a manual borgmatic run and check all four pg_dump streams succeed ``` borgmatic --verbosity 1 2>&1 | grep -E '(Dumping|ERROR)' ``` ## Test plan - [x] `kubectl kustomize` builds cleanly - [x] `ansible --check --diff` for borgmatic and caddy show expected changes - [ ] ArgoCD sync succeeds for databases app - [ ] `immich-pg.tail8d86e.ts.net` resolves - [ ] `pg.ops.eblu.me:5433` accepts connections - [ ] `borgmatic --verbosity 1` dumps all four databases without errors Reviewed-on: https://forge.eblu.me/eblume/blumeops/pulls/314 --- ansible/roles/borgmatic/defaults/main.yml | 9 ++++++ ansible/roles/borgmatic/tasks/main.yml | 1 + ansible/roles/caddy/defaults/main.yml | 4 ++- .../external-secret-immich-borgmatic.yaml | 29 +++++++++++++++++++ argocd/manifests/databases/immich-pg.yaml | 15 ++++++++++ argocd/manifests/databases/kustomization.yaml | 2 ++ .../service-immich-pg-tailscale.yaml | 22 ++++++++++++++ .../feature-borgmatic-all-pg-backups.infra.md | 1 + docs/reference/storage/backups.md | 13 +++++---- 9 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 argocd/manifests/databases/external-secret-immich-borgmatic.yaml create mode 100644 argocd/manifests/databases/service-immich-pg-tailscale.yaml create mode 100644 docs/changelog.d/feature-borgmatic-all-pg-backups.infra.md diff --git a/ansible/roles/borgmatic/defaults/main.yml b/ansible/roles/borgmatic/defaults/main.yml index 428f21e..93b02ba 100644 --- a/ansible/roles/borgmatic/defaults/main.yml +++ b/ansible/roles/borgmatic/defaults/main.yml @@ -70,3 +70,12 @@ borgmatic_postgresql_databases: hostname: pg.ops.eblu.me port: 5432 username: borgmatic + - name: authentik + hostname: pg.ops.eblu.me + port: 5432 + username: borgmatic + # immich-pg cluster (VectorChord) via Caddy L4 on port 5433 + - name: immich + hostname: pg.ops.eblu.me + port: 5433 + username: borgmatic diff --git a/ansible/roles/borgmatic/tasks/main.yml b/ansible/roles/borgmatic/tasks/main.yml index a4b1d7b..e7380dd 100644 --- a/ansible/roles/borgmatic/tasks/main.yml +++ b/ansible/roles/borgmatic/tasks/main.yml @@ -15,6 +15,7 @@ content: | # Managed by ansible (borgmatic role) - k8s PostgreSQL backup credentials pg.ops.eblu.me:5432:*:borgmatic:{{ borgmatic_db_password }} + pg.ops.eblu.me:5433:*:borgmatic:{{ borgmatic_db_password }} dest: ~/.pgpass mode: '0600' no_log: true diff --git a/ansible/roles/caddy/defaults/main.yml b/ansible/roles/caddy/defaults/main.yml index 6b1ec61..784738f 100644 --- a/ansible/roles/caddy/defaults/main.yml +++ b/ansible/roles/caddy/defaults/main.yml @@ -101,7 +101,9 @@ caddy_tcp_services: - port: 2222 backend: "localhost:2200" # Forgejo SSH - port: 5432 - backend: "pg.tail8d86e.ts.net:5432" # PostgreSQL + backend: "pg.tail8d86e.ts.net:5432" # PostgreSQL (blumeops-pg) + - port: 5433 + backend: "immich-pg.tail8d86e.ts.net:5432" # PostgreSQL (immich-pg) - port: "{{ sifaka_node_exporter_port }}" backend: "sifaka:{{ sifaka_node_exporter_port }}" # Sifaka node_exporter - port: "{{ sifaka_smartctl_exporter_port }}" diff --git a/argocd/manifests/databases/external-secret-immich-borgmatic.yaml b/argocd/manifests/databases/external-secret-immich-borgmatic.yaml new file mode 100644 index 0000000..8801c1a --- /dev/null +++ b/argocd/manifests/databases/external-secret-immich-borgmatic.yaml @@ -0,0 +1,29 @@ +# ExternalSecret for borgmatic backup user password on immich-pg cluster +# +# Reuses the same 1Password item as blumeops-pg-borgmatic. +# 1Password item: "borgmatic" in blumeops vault +# Field: "db-password" +# +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: immich-pg-borgmatic + namespace: databases +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: onepassword-blumeops + target: + name: immich-pg-borgmatic + creationPolicy: Owner + template: + type: kubernetes.io/basic-auth + data: + username: borgmatic + password: "{{ .password }}" + data: + - secretKey: password + remoteRef: + key: borgmatic + property: db-password diff --git a/argocd/manifests/databases/immich-pg.yaml b/argocd/manifests/databases/immich-pg.yaml index ec60387..74c6f4e 100644 --- a/argocd/manifests/databases/immich-pg.yaml +++ b/argocd/manifests/databases/immich-pg.yaml @@ -30,6 +30,21 @@ spec: - CREATE EXTENSION IF NOT EXISTS cube CASCADE; - CREATE EXTENSION IF NOT EXISTS earthdistance CASCADE; + # Managed roles + # Note: connectionLimit, ensure, inherit are CNPG defaults added to prevent ArgoCD drift + managed: + roles: + # borgmatic read-only user for backups + - name: borgmatic + login: true + connectionLimit: -1 + ensure: present + inherit: true + inRoles: + - pg_read_all_data + passwordSecret: + name: immich-pg-borgmatic + # Resource limits for minikube environment resources: requests: diff --git a/argocd/manifests/databases/kustomization.yaml b/argocd/manifests/databases/kustomization.yaml index 8c4f506..68d28b2 100644 --- a/argocd/manifests/databases/kustomization.yaml +++ b/argocd/manifests/databases/kustomization.yaml @@ -7,8 +7,10 @@ resources: - blumeops-pg.yaml - immich-pg.yaml - service-tailscale.yaml + - service-immich-pg-tailscale.yaml - service-metrics-tailscale.yaml - external-secret-eblume.yaml - external-secret-borgmatic.yaml + - external-secret-immich-borgmatic.yaml - external-secret-teslamate.yaml - external-secret-authentik.yaml diff --git a/argocd/manifests/databases/service-immich-pg-tailscale.yaml b/argocd/manifests/databases/service-immich-pg-tailscale.yaml new file mode 100644 index 0000000..78891dd --- /dev/null +++ b/argocd/manifests/databases/service-immich-pg-tailscale.yaml @@ -0,0 +1,22 @@ +# Tailscale LoadBalancer for immich-pg PostgreSQL access +# Canonical hostname: immich-pg.tail8d86e.ts.net +# Caddy L4 proxies pg.ops.eblu.me:5433 → this service for borgmatic backups +apiVersion: v1 +kind: Service +metadata: + name: immich-pg-tailscale + namespace: databases + annotations: + tailscale.com/hostname: "immich-pg" + tailscale.com/proxy-class: "default" +spec: + type: LoadBalancer + loadBalancerClass: tailscale + selector: + cnpg.io/cluster: immich-pg + role: primary + ports: + - name: postgresql + port: 5432 + targetPort: 5432 + protocol: TCP diff --git a/docs/changelog.d/feature-borgmatic-all-pg-backups.infra.md b/docs/changelog.d/feature-borgmatic-all-pg-backups.infra.md new file mode 100644 index 0000000..892ee65 --- /dev/null +++ b/docs/changelog.d/feature-borgmatic-all-pg-backups.infra.md @@ -0,0 +1 @@ +Add borgmatic pg_dump backups for authentik and immich databases. Authentik uses the existing blumeops-pg cluster on port 5432. Immich requires a new borgmatic role on the immich-pg cluster, a Tailscale service, and Caddy L4 proxy on port 5433. diff --git a/docs/reference/storage/backups.md b/docs/reference/storage/backups.md index 7061d2c..e830822 100644 --- a/docs/reference/storage/backups.md +++ b/docs/reference/storage/backups.md @@ -1,6 +1,6 @@ --- title: Backups -modified: 2026-03-15 +modified: 2026-03-27 tags: - storage - backup @@ -29,10 +29,13 @@ Daily automated backups from [[indri]] to [[sifaka|Sifaka]] NAS. ### Databases -| Database | Host | Method | -|----------|------|--------| -| miniflux | [[postgresql|pg.ops.eblu.me]] | pg_dump stream | -| teslamate | [[postgresql|pg.ops.eblu.me]] | pg_dump stream | +| Database | Cluster | Host | Method | +|----------|---------|------|--------| +| miniflux | blumeops-pg | [[postgresql|pg.ops.eblu.me:5432]] | pg_dump stream | +| teslamate | blumeops-pg | [[postgresql|pg.ops.eblu.me:5432]] | pg_dump stream | +| authentik | blumeops-pg | [[postgresql|pg.ops.eblu.me:5432]] | pg_dump stream | +| immich | immich-pg | [[postgresql|pg.ops.eblu.me:5433]] | pg_dump stream | +| mealie | — (SQLite) | k8s pod | kubectl exec sqlite3 .backup | ## Sifaka-Native Data