From e43e6defa57ef1a4bf9f9c7cd3acfc356086f758 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Wed, 13 May 2026 17:17:45 -0700 Subject: [PATCH] C1: borgmatic shower SQLite dump via ssh to ringtail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shower dump hook referenced kubectl --context=k3s-ringtail, but indri's kubeconfig deliberately doesn't carry the ringtail credentials. Since PR #349 (2026-05-11), nightly borgmatic runs have failed at the before_backup hook, aborting both sifaka-borg-backups and borgbase-offsite. Rewrite the dump to ssh into ringtail and run k3s kubectl there. /etc/rancher/k3s/k3s.yaml on ringtail is mode 644, so no sudo is needed; the ssh user (eblume) reads it directly. Dump file is created in the pod via sqlite3.backup, copied to ringtail's host filesystem via k3s kubectl cp, then scp'd back to indri. Template gains a `ssh_host` field on dump entries — when set, uses the ssh path; when absent (as for mealie), uses local kubectl with the existing `context` field. Co-Authored-By: Claude Opus 4.7 (1M context) --- ansible/roles/borgmatic/defaults/main.yml | 8 ++- ansible/roles/borgmatic/tasks/main.yml | 14 ++++ .../roles/borgmatic/templates/config.yaml.j2 | 14 +++- .../borgmatic/templates/k8s-sqlite-dump.sh.j2 | 69 +++++++++++++++++++ .../fix-borgmatic-shower-via-ssh.bugfix.md | 10 +++ 5 files changed, 110 insertions(+), 5 deletions(-) create mode 100644 ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 create mode 100644 docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md diff --git a/ansible/roles/borgmatic/defaults/main.yml b/ansible/roles/borgmatic/defaults/main.yml index 123cb0f..3a89a09 100644 --- a/ansible/roles/borgmatic/defaults/main.yml +++ b/ansible/roles/borgmatic/defaults/main.yml @@ -56,12 +56,16 @@ borgmatic_k8s_sqlite_dumps: namespace: mealie label_selector: app=mealie db_path: /app/data/mealie.db - context: minikube + # local kubectl, --context=minikube (indri's only configured ctx) + target: local:minikube - name: shower namespace: shower label_selector: app=shower db_path: /app/data/db.sqlite3 - context: k3s-ringtail + # ssh to ringtail and run k3s kubectl there — avoids needing a + # ringtail kubeconfig on indri. k3s.yaml on ringtail is + # world-readable (mode 644), so no sudo required. + target: ssh:eblume@ringtail # Exclude patterns borgmatic_exclude_patterns: [] diff --git a/ansible/roles/borgmatic/tasks/main.yml b/ansible/roles/borgmatic/tasks/main.yml index eacefa5..4ac242c 100644 --- a/ansible/roles/borgmatic/tasks/main.yml +++ b/ansible/roles/borgmatic/tasks/main.yml @@ -49,6 +49,20 @@ mode: '0700' when: borgmatic_k8s_sqlite_dumps | length > 0 +- name: Ensure ~/bin exists + ansible.builtin.file: + path: "{{ ansible_env.HOME }}/bin" + state: directory + mode: '0755' + when: borgmatic_k8s_sqlite_dumps | length > 0 + +- name: Deploy k8s SQLite dump helper script + ansible.builtin.template: + src: k8s-sqlite-dump.sh.j2 + dest: "{{ ansible_env.HOME }}/bin/borgmatic-k8s-sqlite-dump" + mode: '0755' + when: borgmatic_k8s_sqlite_dumps | length > 0 + - name: Deploy borgmatic configuration ansible.builtin.template: src: config.yaml.j2 diff --git a/ansible/roles/borgmatic/templates/config.yaml.j2 b/ansible/roles/borgmatic/templates/config.yaml.j2 index 85804b7..0893dbc 100644 --- a/ansible/roles/borgmatic/templates/config.yaml.j2 +++ b/ansible/roles/borgmatic/templates/config.yaml.j2 @@ -32,12 +32,20 @@ exclude_patterns: encryption_passcommand: {{ borgmatic_encryption_passcommand }} {% if borgmatic_k8s_sqlite_dumps %} -# Pre-backup: dump SQLite databases from k8s pods -# Uses sqlite3 .backup for a safe, consistent copy (no corruption from concurrent writes) +# Pre-backup: dump SQLite databases from k8s pods. +# Uses sqlite3.backup() for a safe, consistent copy. +# +# Quoting/escaping is delegated to ~/bin/borgmatic-k8s-sqlite-dump +# (deployed by the borgmatic ansible role). Each entry's `target` +# is either: +# - local: -> local kubectl with --context (mealie etc.) +# - ssh: -> ssh + k3s kubectl on the cluster host, +# used for ringtail since indri's kubeconfig +# deliberately doesn't carry that context. before_backup: - mkdir -p {{ borgmatic_k8s_dump_dir }} {% for db in borgmatic_k8s_sqlite_dumps %} - - /opt/homebrew/bin/kubectl --context={{ db.context }} exec -n {{ db.namespace }} deploy/{{ db.name }} -- python3 -c "import sqlite3; sqlite3.connect('{{ db.db_path }}').backup(sqlite3.connect('/tmp/{{ db.name }}-backup.db'))" && /opt/homebrew/bin/kubectl --context={{ db.context }} cp {{ db.namespace }}/$(/opt/homebrew/bin/kubectl --context={{ db.context }} get pod -n {{ db.namespace }} -l {{ db.label_selector }} -o jsonpath='{.items[0].metadata.name}'):/tmp/{{ db.name }}-backup.db {{ borgmatic_k8s_dump_dir }}/{{ db.name }}.db + - {{ ansible_env.HOME }}/bin/borgmatic-k8s-sqlite-dump {{ db.target }} {{ db.namespace }} {{ db.label_selector }} {{ db.db_path }} {{ db.name }} {{ borgmatic_k8s_dump_dir }}/{{ db.name }}.db {% endfor %} {% endif %} diff --git a/ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 b/ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 new file mode 100644 index 0000000..45a5745 --- /dev/null +++ b/ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# {{ ansible_managed }} +# +# Helper script invoked by borgmatic's before_backup hook to capture a +# k8s pod's SQLite database. Keeps the borgmatic config readable by +# pulling all the quoting out of YAML. +# +# Usage: +# borgmatic-k8s-sqlite-dump \ +# +# +# is one of: +# local: - run local kubectl with --context= +# ssh: - ssh to host and run k3s kubectl there +# (no indri-side kubeconfig needed) +# +# - k8s namespace of the pod +# - label selector to find the pod (e.g. app=shower) +# - absolute path inside the pod to the SQLite DB +# - short name used for temp filenames +# - file on this host to receive the dump +set -euo pipefail + +target=${1:?missing target} +namespace=${2:?missing namespace} +selector=${3:?missing selector} +db_path=${4:?missing db path} +name=${5:?missing name} +dump_target=${6:?missing dump target} + +pod_tmp="/tmp/${name}-backup.db" + +python_backup='import sqlite3; sqlite3.connect("'"$db_path"'").backup(sqlite3.connect("'"$pod_tmp"'"))' + +mode=${target%%:*} +ref=${target#*:} + +case "$mode" in + local) + context=$ref + kubectl="/opt/homebrew/bin/kubectl --context=$context -n $namespace" + pod=$($kubectl get pod -l "$selector" \ + -o jsonpath='{.items[0].metadata.name}') + $kubectl exec "$pod" -- python3 -c "$python_backup" + $kubectl cp "$pod:$pod_tmp" "$dump_target" + $kubectl exec "$pod" -- rm -f "$pod_tmp" + ;; + ssh) + host=$ref + host_tmp="/tmp/${name}-host.db" + # Force bash on the remote (user's login shell on ringtail is + # fish). Pipe the script via stdin to dodge nested quoting. + ssh "$host" bash <&2 + echo " expected local: or ssh:" >&2 + exit 1 + ;; +esac diff --git a/docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md b/docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md new file mode 100644 index 0000000..6058332 --- /dev/null +++ b/docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md @@ -0,0 +1,10 @@ +Fix nightly borgmatic backups failing for 2 days. The shower SQLite +dump hook referenced `kubectl --context=k3s-ringtail`, but indri's +kubeconfig deliberately doesn't carry the ringtail credentials. The +`before_backup` hook's failure aborted the entire run, taking out +*both* the local sifaka repo and the BorgBase offsite. Rewrite the +shower dump to ssh into ringtail and run `k3s kubectl` there +(k3s.yaml on ringtail is mode 644, so no sudo needed); fetch the +dump file back via scp. The borgmatic role's k8s-dump template now +branches on a new `ssh_host` field — entries without it keep using +local kubectl with the existing `context` field (e.g. mealie).