Erich Blume
dc69b8c68b
## Summary Nightly borgmatic backups have been failing for 2 days. Root cause: the shower SQLite dump `before_backup` hook (added in PR #349) referenced `kubectl --context=k3s-ringtail`, but indri's kubeconfig deliberately doesn't carry the ringtail credentials. The hook's failure aborted the entire run, taking out *both* the local sifaka repo and the BorgBase offsite. Verified the last good archive was `indri-2026-05-11T02:00`. ## Approach ssh into ringtail and run `k3s kubectl` there — no indri-side kubeconfig needed. `/etc/rancher/k3s/k3s.yaml` is mode 644 so no sudo required, and the existing ssh access from indri to ringtail works. Inline-shell quoting got hairy fast (fish on ringtail rejected `POD=...` bash syntax; the nix shower image lacks `tar` so `kubectl cp` fails). Pulled the dump logic into `~/bin/borgmatic-k8s-sqlite-dump`, deployed by the ansible role. Each dump entry now declares a `target`: - `local:<context>` — local kubectl with explicit context (mealie) - `ssh:<user@host>` — ssh + `k3s kubectl` on the cluster host (shower) Bytes come back via `kubectl exec ... -- cat` instead of `kubectl cp` since `cp` needs `tar` in the pod (nix-built containers don't bundle it). ## Test plan - [x] `mise run provision-indri -- --tags borgmatic --check --diff` shows expected diff - [x] Apply, helper script deployed at `~/bin/borgmatic-k8s-sqlite-dump` - [x] Helper invoked directly with `ssh:eblume@ringtail` produces a valid 288 KB SQLite file - [x] Full `borgmatic create` completes without errors — both mealie.db (1.7 MB) and shower.db (288 KB) appear in `~/.local/share/borgmatic/k8s-dumps/`, archive `indri-2026-05-13T17:31:02` written to sifaka borg repo 🤖 Generated with [Claude Code](https://claude.com/claude-code) Reviewed-on: #357
121 lines
4.3 KiB
YAML
121 lines
4.3 KiB
YAML
---
|
|
borgmatic_config: /Users/erichblume/.config/borgmatic/config.yaml
|
|
borgmatic_config_dir: /Users/erichblume/.config/borgmatic
|
|
borgmatic_log_dir: /Users/erichblume/Library/Logs
|
|
|
|
# Full path to borg binary since LaunchAgent doesn't have homebrew in PATH
|
|
borgmatic_local_path: /opt/homebrew/bin/borg
|
|
|
|
# Borgmatic version — keep in sync with mise.toml in the repo root.
|
|
# Ansible installs this via `mise install` so indri doesn't need the repo cloned.
|
|
borgmatic_version: "2.1.4"
|
|
|
|
# Full path to borgmatic binary — called directly by LaunchAgents to avoid
|
|
# routing through mise, which triggers macOS TCC permission dialogs for
|
|
# protected folders (e.g. ~/Documents) that hang headless LaunchAgent sessions.
|
|
# Uses mise's "latest" symlink so version bumps don't break the LaunchAgent path.
|
|
borgmatic_bin: /Users/erichblume/.local/share/mise/installs/pipx-borgmatic/latest/bin/borgmatic
|
|
|
|
# Schedule: runs daily at 2:00 AM
|
|
borgmatic_schedule_hour: 2
|
|
borgmatic_schedule_minute: 0
|
|
|
|
# Source directories to back up
|
|
borgmatic_source_directories:
|
|
- /Users/erichblume/code/personal/zk
|
|
- /opt/homebrew/var/forgejo
|
|
- /Users/erichblume/.config/borgmatic
|
|
- /Users/erichblume/Documents
|
|
- /Users/erichblume/.local/share/borgmatic/k8s-dumps
|
|
# Shower app prize-photo uploads (sifaka SMB mount). Mounted manually
|
|
# on indri via Finder — see docs/how-to/operations/shower-app.md.
|
|
- /Volumes/shower
|
|
|
|
# Backup repositories
|
|
borgmatic_repositories:
|
|
- path: /Volumes/backups/borg/
|
|
label: sifaka-borg-backups
|
|
encryption: repokey
|
|
append_only: true
|
|
- path: ssh://u3ugi1x1@u3ugi1x1.repo.borgbase.com/./repo
|
|
label: borgbase-offsite
|
|
encryption: repokey
|
|
append_only: true
|
|
|
|
# BorgBase SSH key (fetched from 1Password in playbook pre_tasks)
|
|
borgmatic_borgbase_ssh_key_path: /Users/erichblume/.ssh/borgbase_ed25519
|
|
|
|
# Directory for pre-backup database dumps from k8s pods
|
|
borgmatic_k8s_dump_dir: /Users/erichblume/.local/share/borgmatic/k8s-dumps
|
|
|
|
# K8s SQLite databases to dump before backup via kubectl exec
|
|
# Each entry runs: kubectl exec <pod-selector> -- sqlite3 <path> ".backup /tmp/backup.db"
|
|
# then copies the dump to borgmatic_k8s_dump_dir/<name>.db
|
|
borgmatic_k8s_sqlite_dumps:
|
|
- name: mealie
|
|
namespace: mealie
|
|
label_selector: app=mealie
|
|
db_path: /app/data/mealie.db
|
|
# local kubectl, --context=minikube (indri's only configured ctx)
|
|
target: local:minikube
|
|
- name: shower
|
|
namespace: shower
|
|
label_selector: app=shower
|
|
db_path: /app/data/db.sqlite3
|
|
# ssh to ringtail and run k3s kubectl there — avoids needing a
|
|
# ringtail kubeconfig on indri. k3s.yaml on ringtail is
|
|
# world-readable (mode 644), so no sudo required.
|
|
target: ssh:eblume@ringtail
|
|
|
|
# Exclude patterns
|
|
borgmatic_exclude_patterns: []
|
|
|
|
# Encryption passcommand (reads borg passphrase)
|
|
borgmatic_encryption_passcommand: cat /Users/erichblume/.borg/config.yaml
|
|
|
|
# Retention policy
|
|
borgmatic_keep_daily: 7
|
|
borgmatic_keep_monthly: 12
|
|
borgmatic_keep_yearly: 1000
|
|
|
|
# PostgreSQL databases to backup (streamed via pg_dump)
|
|
# Password is read from ~/.pgpass (managed by this role)
|
|
# pg_dump_command must be full path since LaunchAgent doesn't have homebrew in PATH
|
|
# --- Immich photo library backup (BorgBase offsite only) ---
|
|
borgmatic_photos_config: /Users/erichblume/.config/borgmatic/photos.yaml
|
|
borgmatic_photos_source_directories:
|
|
- /Volumes/photos/library
|
|
- /Volumes/photos/upload
|
|
borgmatic_photos_borgbase_repo: ssh://xcrtl5tg@xcrtl5tg.repo.borgbase.com/./repo
|
|
# Schedule: runs daily at 4:00 AM (offset from main backup at 2:00 AM)
|
|
borgmatic_photos_schedule_hour: 4
|
|
borgmatic_photos_schedule_minute: 0
|
|
# Retention: photos are precious, keep more history
|
|
borgmatic_photos_keep_daily: 7
|
|
borgmatic_photos_keep_monthly: 12
|
|
borgmatic_photos_keep_yearly: 1000
|
|
|
|
borgmatic_pg_dump_command: /opt/homebrew/opt/postgresql@18/bin/pg_dump
|
|
borgmatic_postgresql_databases:
|
|
# k8s PostgreSQL (CloudNativePG) via Caddy L4 proxy
|
|
- name: miniflux
|
|
hostname: pg.ops.eblu.me
|
|
port: 5432
|
|
username: borgmatic
|
|
- name: teslamate
|
|
hostname: pg.ops.eblu.me
|
|
port: 5432
|
|
username: borgmatic
|
|
- name: authentik
|
|
hostname: pg.ops.eblu.me
|
|
port: 5432
|
|
username: borgmatic
|
|
- name: paperless
|
|
hostname: pg.ops.eblu.me
|
|
port: 5432
|
|
username: borgmatic
|
|
# immich-pg cluster (VectorChord) via Caddy L4 on port 5433
|
|
- name: immich
|
|
hostname: pg.ops.eblu.me
|
|
port: 5433
|
|
username: borgmatic
|