From 8e9fb8691c93fcfd83bb6860b44ba6973f0cb3f2 Mon Sep 17 00:00:00 2001
From: Erich Blume <blume.erich@gmail.com>
Date: Wed, 13 May 2026 17:17:45 -0700
Subject: [PATCH] C1: borgmatic shower SQLite dump via ssh to ringtail
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The shower dump hook referenced kubectl --context=k3s-ringtail, but
indri's kubeconfig deliberately doesn't carry the ringtail
credentials. Since PR #349 (2026-05-11), nightly borgmatic runs have
failed at the before_backup hook, aborting both sifaka-borg-backups
and borgbase-offsite.

Rewrite the dump to ssh into ringtail and run k3s kubectl there.
/etc/rancher/k3s/k3s.yaml on ringtail is mode 644, so no sudo is
needed; the ssh user (eblume) reads it directly. Dump file is
created in the pod via sqlite3.backup, copied to ringtail's host
filesystem via k3s kubectl cp, then scp'd back to indri.

Template gains a `ssh_host` field on dump entries — when set, uses
the ssh path; when absent (as for mealie), uses local kubectl with
the existing `context` field.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 ansible/roles/borgmatic/defaults/main.yml          |  5 ++++-
 ansible/roles/borgmatic/templates/config.yaml.j2   | 14 ++++++++++++++
 .../fix-borgmatic-shower-via-ssh.bugfix.md         | 10 ++++++++++
 3 files changed, 28 insertions(+), 1 deletion(-)
 create mode 100644 docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md

diff --git a/ansible/roles/borgmatic/defaults/main.yml b/ansible/roles/borgmatic/defaults/main.yml
index 123cb0f..3e017e3 100644
--- a/ansible/roles/borgmatic/defaults/main.yml
+++ b/ansible/roles/borgmatic/defaults/main.yml
@@ -61,7 +61,10 @@ borgmatic_k8s_sqlite_dumps:
     namespace: shower
     label_selector: app=shower
     db_path: /app/data/db.sqlite3
-    context: k3s-ringtail
+    # ssh to ringtail and run k3s kubectl there — avoids needing a
+    # ringtail kubeconfig on indri. k3s.yaml on ringtail is
+    # world-readable (mode 644), so no sudo required.
+    ssh_host: eblume@ringtail
 
 # Exclude patterns
 borgmatic_exclude_patterns: []
diff --git a/ansible/roles/borgmatic/templates/config.yaml.j2 b/ansible/roles/borgmatic/templates/config.yaml.j2
index 85804b7..e176ca5 100644
--- a/ansible/roles/borgmatic/templates/config.yaml.j2
+++ b/ansible/roles/borgmatic/templates/config.yaml.j2
@@ -34,10 +34,24 @@ encryption_passcommand: {{ borgmatic_encryption_passcommand }}
 {% if borgmatic_k8s_sqlite_dumps %}
 # Pre-backup: dump SQLite databases from k8s pods
 # Uses sqlite3 .backup for a safe, consistent copy (no corruption from concurrent writes)
+#
+# Two execution modes per entry:
+#   - local kubectl with `context: <name>`: works when indri's kubeconfig
+#     has that context (e.g. minikube, configured by minikube start).
+#   - ssh remote with `ssh_host: <user@host>`: runs `k3s kubectl` on the
+#     remote host (no indri-side kubeconfig needed). Used for ringtail
+#     since indri's kubeconfig deliberately doesn't carry the ringtail
+#     credentials.
 before_backup:
     - mkdir -p {{ borgmatic_k8s_dump_dir }}
 {% for db in borgmatic_k8s_sqlite_dumps %}
+{% if db.ssh_host is defined %}
+    - ssh {{ db.ssh_host }} 'set -e; export KUBECONFIG=/etc/rancher/k3s/k3s.yaml; POD=$(k3s kubectl -n {{ db.namespace }} get pod -l {{ db.label_selector }} -o jsonpath="{.items[0].metadata.name}"); k3s kubectl -n {{ db.namespace }} exec "$POD" -- python3 -c "import sqlite3; sqlite3.connect(\"{{ db.db_path }}\").backup(sqlite3.connect(\"/tmp/{{ db.name }}-backup.db\"))"; k3s kubectl -n {{ db.namespace }} cp "$POD:/tmp/{{ db.name }}-backup.db" /tmp/{{ db.name }}-host.db; k3s kubectl -n {{ db.namespace }} exec "$POD" -- rm -f /tmp/{{ db.name }}-backup.db'
+    - scp {{ db.ssh_host }}:/tmp/{{ db.name }}-host.db {{ borgmatic_k8s_dump_dir }}/{{ db.name }}.db
+    - ssh {{ db.ssh_host }} 'rm -f /tmp/{{ db.name }}-host.db'
+{% else %}
     - /opt/homebrew/bin/kubectl --context={{ db.context }} exec -n {{ db.namespace }} deploy/{{ db.name }} -- python3 -c "import sqlite3; sqlite3.connect('{{ db.db_path }}').backup(sqlite3.connect('/tmp/{{ db.name }}-backup.db'))" && /opt/homebrew/bin/kubectl --context={{ db.context }} cp {{ db.namespace }}/$(/opt/homebrew/bin/kubectl --context={{ db.context }} get pod -n {{ db.namespace }} -l {{ db.label_selector }} -o jsonpath='{.items[0].metadata.name}'):/tmp/{{ db.name }}-backup.db {{ borgmatic_k8s_dump_dir }}/{{ db.name }}.db
+{% endif %}
 {% endfor %}
 {% endif %}
 
diff --git a/docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md b/docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md
new file mode 100644
index 0000000..6058332
--- /dev/null
+++ b/docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md
@@ -0,0 +1,10 @@
+Fix nightly borgmatic backups failing for 2 days. The shower SQLite
+dump hook referenced `kubectl --context=k3s-ringtail`, but indri's
+kubeconfig deliberately doesn't carry the ringtail credentials. The
+`before_backup` hook's failure aborted the entire run, taking out
+*both* the local sifaka repo and the BorgBase offsite. Rewrite the
+shower dump to ssh into ringtail and run `k3s kubectl` there
+(k3s.yaml on ringtail is mode 644, so no sudo needed); fetch the
+dump file back via scp. The borgmatic role's k8s-dump template now
+branches on a new `ssh_host` field — entries without it keep using
+local kubectl with the existing `context` field (e.g. mealie).