diff --git a/ansible/roles/borgmatic/defaults/main.yml b/ansible/roles/borgmatic/defaults/main.yml index 123cb0f..3e017e3 100644 --- a/ansible/roles/borgmatic/defaults/main.yml +++ b/ansible/roles/borgmatic/defaults/main.yml @@ -61,7 +61,10 @@ borgmatic_k8s_sqlite_dumps: namespace: shower label_selector: app=shower db_path: /app/data/db.sqlite3 - context: k3s-ringtail + # ssh to ringtail and run k3s kubectl there — avoids needing a + # ringtail kubeconfig on indri. k3s.yaml on ringtail is + # world-readable (mode 644), so no sudo required. + ssh_host: eblume@ringtail # Exclude patterns borgmatic_exclude_patterns: [] diff --git a/ansible/roles/borgmatic/templates/config.yaml.j2 b/ansible/roles/borgmatic/templates/config.yaml.j2 index 85804b7..e176ca5 100644 --- a/ansible/roles/borgmatic/templates/config.yaml.j2 +++ b/ansible/roles/borgmatic/templates/config.yaml.j2 @@ -34,10 +34,24 @@ encryption_passcommand: {{ borgmatic_encryption_passcommand }} {% if borgmatic_k8s_sqlite_dumps %} # Pre-backup: dump SQLite databases from k8s pods # Uses sqlite3 .backup for a safe, consistent copy (no corruption from concurrent writes) +# +# Two execution modes per entry: +# - local kubectl with `context: `: works when indri's kubeconfig +# has that context (e.g. minikube, configured by minikube start). +# - ssh remote with `ssh_host: `: runs `k3s kubectl` on the +# remote host (no indri-side kubeconfig needed). Used for ringtail +# since indri's kubeconfig deliberately doesn't carry the ringtail +# credentials. before_backup: - mkdir -p {{ borgmatic_k8s_dump_dir }} {% for db in borgmatic_k8s_sqlite_dumps %} +{% if db.ssh_host is defined %} + - ssh {{ db.ssh_host }} 'set -e; export KUBECONFIG=/etc/rancher/k3s/k3s.yaml; POD=$(k3s kubectl -n {{ db.namespace }} get pod -l {{ db.label_selector }} -o jsonpath="{.items[0].metadata.name}"); k3s kubectl -n {{ db.namespace }} exec "$POD" -- python3 -c "import sqlite3; sqlite3.connect(\"{{ db.db_path }}\").backup(sqlite3.connect(\"/tmp/{{ db.name }}-backup.db\"))"; k3s kubectl -n {{ db.namespace }} cp "$POD:/tmp/{{ db.name }}-backup.db" /tmp/{{ db.name }}-host.db; k3s kubectl -n {{ db.namespace }} exec "$POD" -- rm -f /tmp/{{ db.name }}-backup.db' + - scp {{ db.ssh_host }}:/tmp/{{ db.name }}-host.db {{ borgmatic_k8s_dump_dir }}/{{ db.name }}.db + - ssh {{ db.ssh_host }} 'rm -f /tmp/{{ db.name }}-host.db' +{% else %} - /opt/homebrew/bin/kubectl --context={{ db.context }} exec -n {{ db.namespace }} deploy/{{ db.name }} -- python3 -c "import sqlite3; sqlite3.connect('{{ db.db_path }}').backup(sqlite3.connect('/tmp/{{ db.name }}-backup.db'))" && /opt/homebrew/bin/kubectl --context={{ db.context }} cp {{ db.namespace }}/$(/opt/homebrew/bin/kubectl --context={{ db.context }} get pod -n {{ db.namespace }} -l {{ db.label_selector }} -o jsonpath='{.items[0].metadata.name}'):/tmp/{{ db.name }}-backup.db {{ borgmatic_k8s_dump_dir }}/{{ db.name }}.db +{% endif %} {% endfor %} {% endif %} diff --git a/docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md b/docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md new file mode 100644 index 0000000..6058332 --- /dev/null +++ b/docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md @@ -0,0 +1,10 @@ +Fix nightly borgmatic backups failing for 2 days. The shower SQLite +dump hook referenced `kubectl --context=k3s-ringtail`, but indri's +kubeconfig deliberately doesn't carry the ringtail credentials. The +`before_backup` hook's failure aborted the entire run, taking out +*both* the local sifaka repo and the BorgBase offsite. Rewrite the +shower dump to ssh into ringtail and run `k3s kubectl` there +(k3s.yaml on ringtail is mode 644, so no sudo needed); fetch the +dump file back via scp. The borgmatic role's k8s-dump template now +branches on a new `ssh_host` field — entries without it keep using +local kubectl with the existing `context` field (e.g. mealie).