From f26105dea920f85682e1220d85e8f3c72242b047 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Wed, 13 May 2026 17:17:45 -0700 Subject: [PATCH] C1: borgmatic shower SQLite dump via ssh to ringtail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shower dump hook referenced kubectl --context=k3s-ringtail, but indri's kubeconfig deliberately doesn't carry the ringtail credentials. Since PR #349 (2026-05-11), nightly borgmatic runs have failed at the before_backup hook, aborting both sifaka-borg-backups and borgbase-offsite. Rewrite the dump to ssh into ringtail and run k3s kubectl there. /etc/rancher/k3s/k3s.yaml on ringtail is mode 644, so no sudo is needed; the ssh user (eblume) reads it directly. Dump file is created in the pod via sqlite3.backup, copied to ringtail's host filesystem via k3s kubectl cp, then scp'd back to indri. Template gains a `ssh_host` field on dump entries — when set, uses the ssh path; when absent (as for mealie), uses local kubectl with the existing `context` field. Co-Authored-By: Claude Opus 4.7 (1M context) --- ansible/roles/borgmatic/defaults/main.yml | 8 ++- ansible/roles/borgmatic/tasks/main.yml | 14 ++++ .../roles/borgmatic/templates/config.yaml.j2 | 14 +++- .../borgmatic/templates/k8s-sqlite-dump.sh.j2 | 71 +++++++++++++++++++ .../fix-borgmatic-shower-via-ssh.bugfix.md | 14 ++++ 5 files changed, 116 insertions(+), 5 deletions(-) create mode 100644 ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 create mode 100644 docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md diff --git a/ansible/roles/borgmatic/defaults/main.yml b/ansible/roles/borgmatic/defaults/main.yml index 123cb0f..3a89a09 100644 --- a/ansible/roles/borgmatic/defaults/main.yml +++ b/ansible/roles/borgmatic/defaults/main.yml @@ -56,12 +56,16 @@ borgmatic_k8s_sqlite_dumps: namespace: mealie label_selector: app=mealie db_path: /app/data/mealie.db - context: minikube + # local kubectl, --context=minikube (indri's only configured ctx) + target: local:minikube - name: shower namespace: shower label_selector: app=shower db_path: /app/data/db.sqlite3 - context: k3s-ringtail + # ssh to ringtail and run k3s kubectl there — avoids needing a + # ringtail kubeconfig on indri. k3s.yaml on ringtail is + # world-readable (mode 644), so no sudo required. + target: ssh:eblume@ringtail # Exclude patterns borgmatic_exclude_patterns: [] diff --git a/ansible/roles/borgmatic/tasks/main.yml b/ansible/roles/borgmatic/tasks/main.yml index eacefa5..4ac242c 100644 --- a/ansible/roles/borgmatic/tasks/main.yml +++ b/ansible/roles/borgmatic/tasks/main.yml @@ -49,6 +49,20 @@ mode: '0700' when: borgmatic_k8s_sqlite_dumps | length > 0 +- name: Ensure ~/bin exists + ansible.builtin.file: + path: "{{ ansible_env.HOME }}/bin" + state: directory + mode: '0755' + when: borgmatic_k8s_sqlite_dumps | length > 0 + +- name: Deploy k8s SQLite dump helper script + ansible.builtin.template: + src: k8s-sqlite-dump.sh.j2 + dest: "{{ ansible_env.HOME }}/bin/borgmatic-k8s-sqlite-dump" + mode: '0755' + when: borgmatic_k8s_sqlite_dumps | length > 0 + - name: Deploy borgmatic configuration ansible.builtin.template: src: config.yaml.j2 diff --git a/ansible/roles/borgmatic/templates/config.yaml.j2 b/ansible/roles/borgmatic/templates/config.yaml.j2 index 85804b7..0893dbc 100644 --- a/ansible/roles/borgmatic/templates/config.yaml.j2 +++ b/ansible/roles/borgmatic/templates/config.yaml.j2 @@ -32,12 +32,20 @@ exclude_patterns: encryption_passcommand: {{ borgmatic_encryption_passcommand }} {% if borgmatic_k8s_sqlite_dumps %} -# Pre-backup: dump SQLite databases from k8s pods -# Uses sqlite3 .backup for a safe, consistent copy (no corruption from concurrent writes) +# Pre-backup: dump SQLite databases from k8s pods. +# Uses sqlite3.backup() for a safe, consistent copy. +# +# Quoting/escaping is delegated to ~/bin/borgmatic-k8s-sqlite-dump +# (deployed by the borgmatic ansible role). Each entry's `target` +# is either: +# - local: -> local kubectl with --context (mealie etc.) +# - ssh: -> ssh + k3s kubectl on the cluster host, +# used for ringtail since indri's kubeconfig +# deliberately doesn't carry that context. before_backup: - mkdir -p {{ borgmatic_k8s_dump_dir }} {% for db in borgmatic_k8s_sqlite_dumps %} - - /opt/homebrew/bin/kubectl --context={{ db.context }} exec -n {{ db.namespace }} deploy/{{ db.name }} -- python3 -c "import sqlite3; sqlite3.connect('{{ db.db_path }}').backup(sqlite3.connect('/tmp/{{ db.name }}-backup.db'))" && /opt/homebrew/bin/kubectl --context={{ db.context }} cp {{ db.namespace }}/$(/opt/homebrew/bin/kubectl --context={{ db.context }} get pod -n {{ db.namespace }} -l {{ db.label_selector }} -o jsonpath='{.items[0].metadata.name}'):/tmp/{{ db.name }}-backup.db {{ borgmatic_k8s_dump_dir }}/{{ db.name }}.db + - {{ ansible_env.HOME }}/bin/borgmatic-k8s-sqlite-dump {{ db.target }} {{ db.namespace }} {{ db.label_selector }} {{ db.db_path }} {{ db.name }} {{ borgmatic_k8s_dump_dir }}/{{ db.name }}.db {% endfor %} {% endif %} diff --git a/ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 b/ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 new file mode 100644 index 0000000..323e717 --- /dev/null +++ b/ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# {{ ansible_managed }} +# +# Helper script invoked by borgmatic's before_backup hook to capture a +# k8s pod's SQLite database. Keeps the borgmatic config readable by +# pulling all the quoting out of YAML. +# +# Usage: +# borgmatic-k8s-sqlite-dump \ +# +# +# is one of: +# local: - run local kubectl with --context= +# ssh: - ssh to host and run k3s kubectl there +# (no indri-side kubeconfig needed) +# +# - k8s namespace of the pod +# - label selector to find the pod (e.g. app=shower) +# - absolute path inside the pod to the SQLite DB +# - short name used for temp filenames +# - file on this host to receive the dump +set -euo pipefail + +target=${1:?missing target} +namespace=${2:?missing namespace} +selector=${3:?missing selector} +db_path=${4:?missing db path} +name=${5:?missing name} +dump_target=${6:?missing dump target} + +pod_tmp="/tmp/${name}-backup.db" + +python_backup='import sqlite3; sqlite3.connect("'"$db_path"'").backup(sqlite3.connect("'"$pod_tmp"'"))' + +mode=${target%%:*} +ref=${target#*:} + +case "$mode" in + local) + # Pulls dump bytes out via "kubectl exec -- cat" rather than + # "kubectl cp", which would otherwise need tar inside the pod + # (nix-built images like shower don't bundle tar). + context=$ref + kubectl="/opt/homebrew/bin/kubectl --context=$context -n $namespace" + pod=$($kubectl get pod -l "$selector" \ + -o jsonpath='{.items[0].metadata.name}') + $kubectl exec "$pod" -- python3 -c "$python_backup" + $kubectl exec "$pod" -- cat "$pod_tmp" > "$dump_target" + $kubectl exec "$pod" -- rm -f "$pod_tmp" + ;; + ssh) + host=$ref + # Force bash on the remote (user's login shell on ringtail is + # fish). Pipe the script via stdin to dodge nested quoting. + # The dump bytes come back over the ssh stdout stream — no + # intermediate scp, no tar requirement in the pod. + ssh "$host" bash < "$dump_target" +set -euo pipefail +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml +pod=\$(k3s kubectl -n "$namespace" get pod -l "$selector" -o jsonpath='{.items[0].metadata.name}') +k3s kubectl -n "$namespace" exec "\$pod" -- python3 -c '$python_backup' 1>&2 +k3s kubectl -n "$namespace" exec "\$pod" -- cat "$pod_tmp" +k3s kubectl -n "$namespace" exec "\$pod" -- rm -f "$pod_tmp" 1>&2 +EOF + ;; + *) + echo "borgmatic-k8s-sqlite-dump: unknown target mode: $mode" >&2 + echo " expected local: or ssh:" >&2 + exit 1 + ;; +esac diff --git a/docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md b/docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md new file mode 100644 index 0000000..e18272c --- /dev/null +++ b/docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md @@ -0,0 +1,14 @@ +Fix nightly borgmatic backups failing for 2 days. The shower SQLite +dump hook referenced `kubectl --context=k3s-ringtail`, but indri's +kubeconfig deliberately doesn't carry the ringtail credentials. The +`before_backup` hook's failure aborted the entire run, taking out +*both* the local sifaka repo and the BorgBase offsite. Replaced +the inline-shell dump with a `~/bin/borgmatic-k8s-sqlite-dump` +helper deployed by the ansible role. Each dump entry now declares a +`target` of either `local:` (mealie — kubectl uses indri's +kubeconfig) or `ssh:` (shower — ssh into ringtail and +run `k3s kubectl` there, no indri-side kubeconfig needed; k3s.yaml +on ringtail is mode 644 so no sudo required). Bytes stream back via +`kubectl exec ... -- cat` rather than `kubectl cp`, since `kubectl +cp` requires `tar` inside the pod and nix-built images like shower +don't bundle it.