From dc69b8c68be6d158f15178a08f9f09603de50381 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Wed, 13 May 2026 18:55:50 -0700 Subject: [PATCH] C1: fix borgmatic shower SQLite dump (ssh to ringtail) (#357) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Nightly borgmatic backups have been failing for 2 days. Root cause: the shower SQLite dump `before_backup` hook (added in PR #349) referenced `kubectl --context=k3s-ringtail`, but indri's kubeconfig deliberately doesn't carry the ringtail credentials. The hook's failure aborted the entire run, taking out *both* the local sifaka repo and the BorgBase offsite. Verified the last good archive was `indri-2026-05-11T02:00`. ## Approach ssh into ringtail and run `k3s kubectl` there — no indri-side kubeconfig needed. `/etc/rancher/k3s/k3s.yaml` is mode 644 so no sudo required, and the existing ssh access from indri to ringtail works. Inline-shell quoting got hairy fast (fish on ringtail rejected `POD=...` bash syntax; the nix shower image lacks `tar` so `kubectl cp` fails). Pulled the dump logic into `~/bin/borgmatic-k8s-sqlite-dump`, deployed by the ansible role. Each dump entry now declares a `target`: - `local:` — local kubectl with explicit context (mealie) - `ssh:` — ssh + `k3s kubectl` on the cluster host (shower) Bytes come back via `kubectl exec ... -- cat` instead of `kubectl cp` since `cp` needs `tar` in the pod (nix-built containers don't bundle it). ## Test plan - [x] `mise run provision-indri -- --tags borgmatic --check --diff` shows expected diff - [x] Apply, helper script deployed at `~/bin/borgmatic-k8s-sqlite-dump` - [x] Helper invoked directly with `ssh:eblume@ringtail` produces a valid 288 KB SQLite file - [x] Full `borgmatic create` completes without errors — both mealie.db (1.7 MB) and shower.db (288 KB) appear in `~/.local/share/borgmatic/k8s-dumps/`, archive `indri-2026-05-13T17:31:02` written to sifaka borg repo 🤖 Generated with [Claude Code](https://claude.com/claude-code) Reviewed-on: https://forge.eblu.me/eblume/blumeops/pulls/357 --- ansible/roles/borgmatic/defaults/main.yml | 8 ++- ansible/roles/borgmatic/tasks/main.yml | 14 ++++ .../roles/borgmatic/templates/config.yaml.j2 | 14 +++- .../borgmatic/templates/k8s-sqlite-dump.sh.j2 | 71 +++++++++++++++++++ .../fix-borgmatic-shower-via-ssh.bugfix.md | 14 ++++ 5 files changed, 116 insertions(+), 5 deletions(-) create mode 100644 ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 create mode 100644 docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md diff --git a/ansible/roles/borgmatic/defaults/main.yml b/ansible/roles/borgmatic/defaults/main.yml index 123cb0f..3a89a09 100644 --- a/ansible/roles/borgmatic/defaults/main.yml +++ b/ansible/roles/borgmatic/defaults/main.yml @@ -56,12 +56,16 @@ borgmatic_k8s_sqlite_dumps: namespace: mealie label_selector: app=mealie db_path: /app/data/mealie.db - context: minikube + # local kubectl, --context=minikube (indri's only configured ctx) + target: local:minikube - name: shower namespace: shower label_selector: app=shower db_path: /app/data/db.sqlite3 - context: k3s-ringtail + # ssh to ringtail and run k3s kubectl there — avoids needing a + # ringtail kubeconfig on indri. k3s.yaml on ringtail is + # world-readable (mode 644), so no sudo required. + target: ssh:eblume@ringtail # Exclude patterns borgmatic_exclude_patterns: [] diff --git a/ansible/roles/borgmatic/tasks/main.yml b/ansible/roles/borgmatic/tasks/main.yml index eacefa5..4ac242c 100644 --- a/ansible/roles/borgmatic/tasks/main.yml +++ b/ansible/roles/borgmatic/tasks/main.yml @@ -49,6 +49,20 @@ mode: '0700' when: borgmatic_k8s_sqlite_dumps | length > 0 +- name: Ensure ~/bin exists + ansible.builtin.file: + path: "{{ ansible_env.HOME }}/bin" + state: directory + mode: '0755' + when: borgmatic_k8s_sqlite_dumps | length > 0 + +- name: Deploy k8s SQLite dump helper script + ansible.builtin.template: + src: k8s-sqlite-dump.sh.j2 + dest: "{{ ansible_env.HOME }}/bin/borgmatic-k8s-sqlite-dump" + mode: '0755' + when: borgmatic_k8s_sqlite_dumps | length > 0 + - name: Deploy borgmatic configuration ansible.builtin.template: src: config.yaml.j2 diff --git a/ansible/roles/borgmatic/templates/config.yaml.j2 b/ansible/roles/borgmatic/templates/config.yaml.j2 index 85804b7..0893dbc 100644 --- a/ansible/roles/borgmatic/templates/config.yaml.j2 +++ b/ansible/roles/borgmatic/templates/config.yaml.j2 @@ -32,12 +32,20 @@ exclude_patterns: encryption_passcommand: {{ borgmatic_encryption_passcommand }} {% if borgmatic_k8s_sqlite_dumps %} -# Pre-backup: dump SQLite databases from k8s pods -# Uses sqlite3 .backup for a safe, consistent copy (no corruption from concurrent writes) +# Pre-backup: dump SQLite databases from k8s pods. +# Uses sqlite3.backup() for a safe, consistent copy. +# +# Quoting/escaping is delegated to ~/bin/borgmatic-k8s-sqlite-dump +# (deployed by the borgmatic ansible role). Each entry's `target` +# is either: +# - local: -> local kubectl with --context (mealie etc.) +# - ssh: -> ssh + k3s kubectl on the cluster host, +# used for ringtail since indri's kubeconfig +# deliberately doesn't carry that context. before_backup: - mkdir -p {{ borgmatic_k8s_dump_dir }} {% for db in borgmatic_k8s_sqlite_dumps %} - - /opt/homebrew/bin/kubectl --context={{ db.context }} exec -n {{ db.namespace }} deploy/{{ db.name }} -- python3 -c "import sqlite3; sqlite3.connect('{{ db.db_path }}').backup(sqlite3.connect('/tmp/{{ db.name }}-backup.db'))" && /opt/homebrew/bin/kubectl --context={{ db.context }} cp {{ db.namespace }}/$(/opt/homebrew/bin/kubectl --context={{ db.context }} get pod -n {{ db.namespace }} -l {{ db.label_selector }} -o jsonpath='{.items[0].metadata.name}'):/tmp/{{ db.name }}-backup.db {{ borgmatic_k8s_dump_dir }}/{{ db.name }}.db + - {{ ansible_env.HOME }}/bin/borgmatic-k8s-sqlite-dump {{ db.target }} {{ db.namespace }} {{ db.label_selector }} {{ db.db_path }} {{ db.name }} {{ borgmatic_k8s_dump_dir }}/{{ db.name }}.db {% endfor %} {% endif %} diff --git a/ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 b/ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 new file mode 100644 index 0000000..323e717 --- /dev/null +++ b/ansible/roles/borgmatic/templates/k8s-sqlite-dump.sh.j2 @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# {{ ansible_managed }} +# +# Helper script invoked by borgmatic's before_backup hook to capture a +# k8s pod's SQLite database. Keeps the borgmatic config readable by +# pulling all the quoting out of YAML. +# +# Usage: +# borgmatic-k8s-sqlite-dump \ +# +# +# is one of: +# local: - run local kubectl with --context= +# ssh: - ssh to host and run k3s kubectl there +# (no indri-side kubeconfig needed) +# +# - k8s namespace of the pod +# - label selector to find the pod (e.g. app=shower) +# - absolute path inside the pod to the SQLite DB +# - short name used for temp filenames +# - file on this host to receive the dump +set -euo pipefail + +target=${1:?missing target} +namespace=${2:?missing namespace} +selector=${3:?missing selector} +db_path=${4:?missing db path} +name=${5:?missing name} +dump_target=${6:?missing dump target} + +pod_tmp="/tmp/${name}-backup.db" + +python_backup='import sqlite3; sqlite3.connect("'"$db_path"'").backup(sqlite3.connect("'"$pod_tmp"'"))' + +mode=${target%%:*} +ref=${target#*:} + +case "$mode" in + local) + # Pulls dump bytes out via "kubectl exec -- cat" rather than + # "kubectl cp", which would otherwise need tar inside the pod + # (nix-built images like shower don't bundle tar). + context=$ref + kubectl="/opt/homebrew/bin/kubectl --context=$context -n $namespace" + pod=$($kubectl get pod -l "$selector" \ + -o jsonpath='{.items[0].metadata.name}') + $kubectl exec "$pod" -- python3 -c "$python_backup" + $kubectl exec "$pod" -- cat "$pod_tmp" > "$dump_target" + $kubectl exec "$pod" -- rm -f "$pod_tmp" + ;; + ssh) + host=$ref + # Force bash on the remote (user's login shell on ringtail is + # fish). Pipe the script via stdin to dodge nested quoting. + # The dump bytes come back over the ssh stdout stream — no + # intermediate scp, no tar requirement in the pod. + ssh "$host" bash < "$dump_target" +set -euo pipefail +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml +pod=\$(k3s kubectl -n "$namespace" get pod -l "$selector" -o jsonpath='{.items[0].metadata.name}') +k3s kubectl -n "$namespace" exec "\$pod" -- python3 -c '$python_backup' 1>&2 +k3s kubectl -n "$namespace" exec "\$pod" -- cat "$pod_tmp" +k3s kubectl -n "$namespace" exec "\$pod" -- rm -f "$pod_tmp" 1>&2 +EOF + ;; + *) + echo "borgmatic-k8s-sqlite-dump: unknown target mode: $mode" >&2 + echo " expected local: or ssh:" >&2 + exit 1 + ;; +esac diff --git a/docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md b/docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md new file mode 100644 index 0000000..e18272c --- /dev/null +++ b/docs/changelog.d/fix-borgmatic-shower-via-ssh.bugfix.md @@ -0,0 +1,14 @@ +Fix nightly borgmatic backups failing for 2 days. The shower SQLite +dump hook referenced `kubectl --context=k3s-ringtail`, but indri's +kubeconfig deliberately doesn't carry the ringtail credentials. The +`before_backup` hook's failure aborted the entire run, taking out +*both* the local sifaka repo and the BorgBase offsite. Replaced +the inline-shell dump with a `~/bin/borgmatic-k8s-sqlite-dump` +helper deployed by the ansible role. Each dump entry now declares a +`target` of either `local:` (mealie — kubectl uses indri's +kubeconfig) or `ssh:` (shower — ssh into ringtail and +run `k3s kubectl` there, no indri-side kubeconfig needed; k3s.yaml +on ringtail is mode 644 so no sudo required). Bytes stream back via +`kubectl exec ... -- cat` rather than `kubectl cp`, since `kubectl +cp` requires `tar` inside the pod and nix-built images like shower +don't bundle it.