Add offsite backup for immich photo library to BorgBase (#315)

## Summary

- Adds a second borgmatic config (`photos.yaml`) that backs up `/Volumes/photos` (sifaka SMB mount, ~128 GB) to a dedicated BorgBase repo (`immich-photos`), running daily at 4 AM
- Separate launchd agent (`mcquack.eblume.borgmatic-photos`) so photo backups run independently from the main backup
- Refactors `borgmatic_metrics` script to support multiple repos with a `repo` Prometheus label
- Updates Grafana "Borg Backups" dashboard with a `repo` template variable so you can filter/compare repos
- Docs updated: `backups.md`, `borgmatic.md`

## Prerequisites (manual)

- [x] Create `immich-photos` repo on BorgBase with same SSH key
- [ ] Upgrade BorgBase plan to Small ($24/yr) if currently on free tier (128 GB exceeds 10 GB limit)
- [ ] After deploy: `borg init` the new repo (borgmatic does this automatically on first run)

## Test plan

- [ ] Dry run: `mise run provision-indri -- --check --diff --tags borgmatic,borgmatic_metrics`
- [ ] Deploy borgmatic role and verify both configs deployed
- [ ] Run `borgmatic --config ~/.config/borgmatic/photos.yaml create --verbosity 1` manually for first backup (will take hours)
- [ ] Verify metrics script collects from both repos: `~/.local/bin/borgmatic-metrics && cat /opt/homebrew/var/node_exporter/textfile/borgmatic.prom`
- [ ] Sync grafana-config in ArgoCD and verify dashboard repo selector works

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Reviewed-on: #315
This commit is contained in:
Erich Blume 2026-03-27 19:43:05 -07:00
commit c78b86c72c
11 changed files with 305 additions and 136 deletions

View file

@ -59,6 +59,18 @@ borgmatic_keep_yearly: 1000
# PostgreSQL databases to backup (streamed via pg_dump)
# Password is read from ~/.pgpass (managed by this role)
# pg_dump_command must be full path since LaunchAgent doesn't have homebrew in PATH
# --- Immich photo library backup (BorgBase offsite only) ---
borgmatic_photos_config: /Users/erichblume/.config/borgmatic/photos.yaml
borgmatic_photos_source_dir: /Volumes/photos
borgmatic_photos_borgbase_repo: ssh://xcrtl5tg@xcrtl5tg.repo.borgbase.com/./repo
# Schedule: runs daily at 4:00 AM (offset from main backup at 2:00 AM)
borgmatic_photos_schedule_hour: 4
borgmatic_photos_schedule_minute: 0
# Retention: photos are precious, keep more history
borgmatic_photos_keep_daily: 7
borgmatic_photos_keep_monthly: 12
borgmatic_photos_keep_yearly: 1000
borgmatic_pg_dump_command: /opt/homebrew/opt/postgresql@18/bin/pg_dump
borgmatic_postgresql_databases:
# k8s PostgreSQL (CloudNativePG) via Caddy L4 proxy

View file

@ -4,3 +4,9 @@
launchctl unload ~/Library/LaunchAgents/mcquack.eblume.borgmatic.plist 2>/dev/null || true
launchctl load ~/Library/LaunchAgents/mcquack.eblume.borgmatic.plist
changed_when: true
- name: Reload borgmatic-photos
ansible.builtin.shell: |
launchctl unload ~/Library/LaunchAgents/mcquack.eblume.borgmatic-photos.plist 2>/dev/null || true
launchctl load ~/Library/LaunchAgents/mcquack.eblume.borgmatic-photos.plist
changed_when: true

View file

@ -28,11 +28,14 @@
mode: '0600'
no_log: true
- name: Add BorgBase host key to known_hosts
- name: Add BorgBase host keys to known_hosts
ansible.builtin.known_hosts:
name: u3ugi1x1.repo.borgbase.com
key: "u3ugi1x1.repo.borgbase.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGU0mISTyHBw9tBs6SuhSq8tvNM8m9eifQxM+88TowPO"
name: "{{ item }}"
key: "{{ item }} ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGU0mISTyHBw9tBs6SuhSq8tvNM8m9eifQxM+88TowPO"
state: present
loop:
- u3ugi1x1.repo.borgbase.com
- xcrtl5tg.repo.borgbase.com
- name: Ensure k8s dump directory exists
ansible.builtin.file:
@ -65,3 +68,30 @@
when: borgmatic_launchctl_check.rc != 0
changed_when: true
failed_when: false
# --- Immich photo library backup (BorgBase offsite only) ---
- name: Deploy borgmatic photos configuration
ansible.builtin.template:
src: photos.yaml.j2
dest: "{{ borgmatic_photos_config }}"
mode: '0600'
- name: Deploy borgmatic-photos LaunchAgent plist
ansible.builtin.template:
src: borgmatic-photos.plist.j2
dest: ~/Library/LaunchAgents/mcquack.eblume.borgmatic-photos.plist
mode: '0644'
notify: Reload borgmatic-photos
- name: Check if borgmatic-photos LaunchAgent is loaded
ansible.builtin.command: launchctl list mcquack.eblume.borgmatic-photos
register: borgmatic_photos_launchctl_check
changed_when: false
failed_when: false
- name: Load borgmatic-photos LaunchAgent if not loaded
ansible.builtin.command: launchctl load ~/Library/LaunchAgents/mcquack.eblume.borgmatic-photos.plist
when: borgmatic_photos_launchctl_check.rc != 0
changed_when: true
failed_when: false

View file

@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- {{ ansible_managed }} -->
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>KeepAlive</key>
<false/>
<key>Label</key>
<string>mcquack.eblume.borgmatic-photos</string>
<key>EnvironmentVariables</key>
<dict>
<key>PATH</key>
<string>/opt/homebrew/bin:/usr/bin:/bin</string>
</dict>
<key>ProgramArguments</key>
<array>
<string>/opt/homebrew/opt/mise/bin/mise</string>
<string>x</string>
<string>--</string>
<string>borgmatic</string>
<string>--config</string>
<string>{{ borgmatic_photos_config }}</string>
<string>create</string>
</array>
<key>RunAtLoad</key>
<false/>
<key>StandardErrorPath</key>
<string>{{ borgmatic_log_dir }}/mcquack.borgmatic-photos.err.log</string>
<key>StandardOutPath</key>
<string>{{ borgmatic_log_dir }}/mcquack.borgmatic-photos.out.log</string>
<key>StartCalendarInterval</key>
<dict>
<key>Hour</key>
<integer>{{ borgmatic_photos_schedule_hour }}</integer>
<key>Minute</key>
<integer>{{ borgmatic_photos_schedule_minute }}</integer>
</dict>
</dict>
</plist>

View file

@ -0,0 +1,29 @@
# {{ ansible_managed }}
#
# Borgmatic config for immich photo library backup.
# Backs up /Volumes/photos (sifaka SMB mount) to BorgBase offsite ONLY.
# Separate from the main borgmatic config to keep concerns isolated:
# - main config: indri data → sifaka + borgbase
# - this config: sifaka photos → borgbase (different repo)
local_path: {{ borgmatic_local_path }}
source_directories:
- {{ borgmatic_photos_source_dir }}
source_directories_must_exist: true
repositories:
- path: {{ borgmatic_photos_borgbase_repo }}
label: borgbase-immich-photos
encryption: repokey
append_only: true
encryption_passcommand: {{ borgmatic_encryption_passcommand }}
ssh_command: ssh -o IdentitiesOnly=yes -i {{ borgmatic_borgbase_ssh_key_path }}
# Retention policy — photos are precious, keep more history
keep_daily: {{ borgmatic_photos_keep_daily }}
keep_monthly: {{ borgmatic_photos_keep_monthly }}
keep_yearly: {{ borgmatic_photos_keep_yearly }}

View file

@ -1,6 +1,14 @@
---
borgmatic_metrics_repo: /Volumes/backups/borg/
# Borg repositories to collect metrics from
# Each entry needs a path (local or ssh://) and a label for Prometheus metrics
borgmatic_metrics_repos:
- path: /Volumes/backups/borg/
label: sifaka-local
- path: ssh://xcrtl5tg@xcrtl5tg.repo.borgbase.com/./repo
label: borgbase-immich-photos
borgmatic_metrics_passcommand: cat /Users/erichblume/.borg/config.yaml
borgmatic_metrics_ssh_key: /Users/erichblume/.ssh/borgbase_ed25519
borgmatic_metrics_dir: /opt/homebrew/var/node_exporter/textfile
borgmatic_metrics_script: /Users/erichblume/.local/bin/borgmatic-metrics
borgmatic_metrics_interval: 3600 # seconds between metric collection (hourly)

View file

@ -1,11 +1,12 @@
#!/bin/bash
# {{ ansible_managed }}
# Collects borg backup metrics for node_exporter textfile collector
# Supports multiple repositories with a repo label for Prometheus
set -euo pipefail
export BORG_PASSCOMMAND="{{ borgmatic_metrics_passcommand }}"
BORG_REPO="{{ borgmatic_metrics_repo }}"
export BORG_RSH="ssh -o IdentitiesOnly=yes -i {{ borgmatic_metrics_ssh_key }}"
OUTPUT_FILE="{{ borgmatic_metrics_dir }}/borgmatic.prom"
TEMP_FILE="${OUTPUT_FILE}.tmp"
@ -13,129 +14,109 @@ TEMP_FILE="${OUTPUT_FILE}.tmp"
BORG_CMD="/opt/homebrew/bin/borg"
JQ_CMD="/opt/homebrew/bin/jq"
# Get repository info
repo_json=$($BORG_CMD info --json "$BORG_REPO" 2>/dev/null) || {
echo "Failed to get borg repo info" >&2
# Write down metric
cat > "$TEMP_FILE" << 'EOF'
# Start fresh
cat > "$TEMP_FILE" << 'EOF'
# HELP borgmatic_up Borg backup repository is accessible
# TYPE borgmatic_up gauge
borgmatic_up 0
EOF
mv "$TEMP_FILE" "$OUTPUT_FILE"
exit 0
}
# Get archive list
archives_json=$($BORG_CMD list --json "$BORG_REPO" 2>/dev/null) || {
echo "Failed to list borg archives" >&2
exit 1
}
# Extract repository stats
total_size=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.total_size')
total_csize=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.total_csize')
unique_size=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.unique_size')
unique_csize=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.unique_csize')
total_chunks=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.total_chunks')
unique_chunks=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.total_unique_chunks')
# Count archives
archive_count=$(echo "$archives_json" | $JQ_CMD -r '.archives | length')
# Get last archive info
last_archive_name=$(echo "$archives_json" | $JQ_CMD -r '.archives[-1].name // empty')
if [ -n "$last_archive_name" ]; then
# Get detailed info for the last archive
last_archive_json=$($BORG_CMD info --json "${BORG_REPO}::${last_archive_name}" 2>/dev/null) || {
echo "Failed to get last archive info" >&2
last_archive_json=""
}
if [ -n "$last_archive_json" ]; then
last_original_size=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].stats.original_size')
last_compressed_size=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].stats.compressed_size')
last_deduplicated_size=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].stats.deduplicated_size')
last_nfiles=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].stats.nfiles')
last_start=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].start')
last_end=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].end')
last_duration=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].duration')
# Convert timestamp to unix epoch
last_timestamp=$(date -j -f "%Y-%m-%dT%H:%M:%S" "${last_start%.*}" "+%s" 2>/dev/null || echo "0")
fi
fi
# Write metrics
cat > "$TEMP_FILE" << EOF
# HELP borgmatic_up Borg backup repository is accessible
# TYPE borgmatic_up gauge
borgmatic_up 1
# HELP borgmatic_repo_original_size_bytes Total original size of all archives (sum of what each backup contains)
# TYPE borgmatic_repo_original_size_bytes gauge
borgmatic_repo_original_size_bytes $total_size
# HELP borgmatic_repo_compressed_size_bytes Total compressed size of all archives
# TYPE borgmatic_repo_compressed_size_bytes gauge
borgmatic_repo_compressed_size_bytes $total_csize
# HELP borgmatic_repo_deduplicated_size_bytes Actual disk usage after deduplication (unique data)
# TYPE borgmatic_repo_deduplicated_size_bytes gauge
borgmatic_repo_deduplicated_size_bytes $unique_csize
# HELP borgmatic_repo_total_chunks Total number of chunks across all archives
# TYPE borgmatic_repo_total_chunks gauge
borgmatic_repo_total_chunks $total_chunks
# HELP borgmatic_repo_unique_chunks Number of unique chunks (after deduplication)
# TYPE borgmatic_repo_unique_chunks gauge
borgmatic_repo_unique_chunks $unique_chunks
# HELP borgmatic_archive_count Number of archives in the repository
# TYPE borgmatic_archive_count gauge
borgmatic_archive_count $archive_count
EOF
# Add last archive metrics if available
if [ -n "${last_original_size:-}" ]; then
cat >> "$TEMP_FILE" << EOF
# HELP borgmatic_last_archive_original_size_bytes Original size of the last archive (data being backed up)
# TYPE borgmatic_last_archive_original_size_bytes gauge
borgmatic_last_archive_original_size_bytes $last_original_size
# HELP borgmatic_last_archive_compressed_size_bytes Compressed size of the last archive
# TYPE borgmatic_last_archive_compressed_size_bytes gauge
borgmatic_last_archive_compressed_size_bytes $last_compressed_size
# HELP borgmatic_last_archive_deduplicated_size_bytes Deduplicated size of last archive (new data added)
# TYPE borgmatic_last_archive_deduplicated_size_bytes gauge
borgmatic_last_archive_deduplicated_size_bytes $last_deduplicated_size
# HELP borgmatic_last_archive_files Number of files in the last archive
# TYPE borgmatic_last_archive_files gauge
borgmatic_last_archive_files $last_nfiles
# HELP borgmatic_last_archive_timestamp Unix timestamp of the last backup
# TYPE borgmatic_last_archive_timestamp gauge
borgmatic_last_archive_timestamp $last_timestamp
# HELP borgmatic_last_archive_duration_seconds Duration of the last backup in seconds
# TYPE borgmatic_last_archive_duration_seconds gauge
borgmatic_last_archive_duration_seconds ${last_duration:-0}
EOF
# Collect per-source-directory sizes
cat >> "$TEMP_FILE" << 'EOF'
# HELP borgmatic_source_size_bytes Size of each backup source directory in bytes
# TYPE borgmatic_source_size_bytes gauge
EOF
# List archive contents and group by source directory
$BORG_CMD list "${BORG_REPO}::${last_archive_name}" --format "{size} {path}{NL}" 2>/dev/null | awk '
collect_repo_metrics() {
local repo_path="$1"
local repo_label="$2"
# Get repository info
repo_json=$($BORG_CMD info --json "$repo_path" 2>/dev/null) || {
echo "Failed to get borg repo info for $repo_label" >&2
echo "borgmatic_up{repo=\"$repo_label\"} 0" >> "$TEMP_FILE"
return
}
# Get archive list
archives_json=$($BORG_CMD list --json "$repo_path" 2>/dev/null) || {
echo "Failed to list borg archives for $repo_label" >&2
echo "borgmatic_up{repo=\"$repo_label\"} 0" >> "$TEMP_FILE"
return
}
# Extract repository stats
total_size=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.total_size')
total_csize=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.total_csize')
unique_size=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.unique_size')
unique_csize=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.unique_csize')
total_chunks=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.total_chunks')
unique_chunks=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.total_unique_chunks')
archive_count=$(echo "$archives_json" | $JQ_CMD -r '.archives | length')
cat >> "$TEMP_FILE" << EOF
borgmatic_up{repo="$repo_label"} 1
borgmatic_repo_original_size_bytes{repo="$repo_label"} $total_size
borgmatic_repo_compressed_size_bytes{repo="$repo_label"} $total_csize
borgmatic_repo_deduplicated_size_bytes{repo="$repo_label"} $unique_csize
borgmatic_repo_total_chunks{repo="$repo_label"} $total_chunks
borgmatic_repo_unique_chunks{repo="$repo_label"} $unique_chunks
borgmatic_archive_count{repo="$repo_label"} $archive_count
EOF
# Get last archive info
last_archive_name=$(echo "$archives_json" | $JQ_CMD -r '.archives[-1].name // empty')
if [ -z "$last_archive_name" ]; then
return
fi
# Get detailed info for the last archive
last_archive_json=$($BORG_CMD info --json "${repo_path}::${last_archive_name}" 2>/dev/null) || {
echo "Failed to get last archive info for $repo_label" >&2
return
}
last_original_size=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].stats.original_size')
last_compressed_size=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].stats.compressed_size')
last_deduplicated_size=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].stats.deduplicated_size')
last_nfiles=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].stats.nfiles')
last_start=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].start')
last_duration=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].duration')
# Convert timestamp to unix epoch
last_timestamp=$(date -j -f "%Y-%m-%dT%H:%M:%S" "${last_start%.*}" "+%s" 2>/dev/null || echo "0")
cat >> "$TEMP_FILE" << EOF
borgmatic_last_archive_original_size_bytes{repo="$repo_label"} $last_original_size
borgmatic_last_archive_compressed_size_bytes{repo="$repo_label"} $last_compressed_size
borgmatic_last_archive_deduplicated_size_bytes{repo="$repo_label"} $last_deduplicated_size
borgmatic_last_archive_files{repo="$repo_label"} $last_nfiles
borgmatic_last_archive_timestamp{repo="$repo_label"} $last_timestamp
borgmatic_last_archive_duration_seconds{repo="$repo_label"} ${last_duration:-0}
EOF
# Collect per-source-directory sizes
$BORG_CMD list "${repo_path}::${last_archive_name}" --format "{size} {path}{NL}" 2>/dev/null | awk -v repo="$repo_label" '
{
size = $1
path = $2
@ -145,8 +126,10 @@ EOF
else if (path ~ /^Users\/[^\/]+\/devpi/) { source = "devpi" }
else if (path ~ /^Users\/[^\/]+\/code\/personal\/zk/) { source = "Zettelkasten" }
else if (path ~ /^Users\/[^\/]+\/.config\/borgmatic/) { source = "borgmatic_config" }
else if (path ~ /^Users\/[^\/]+\/.local\/share\/borgmatic/) { source = "k8s_dumps" }
else if (path ~ /^opt\/homebrew\/var\/forgejo/) { source = "Forgejo" }
else if (path ~ /^opt\/homebrew\/var\/loki/) { source = "Loki" }
else if (path ~ /^Volumes\/photos/) { source = "immich_photos" }
else if (path ~ /^borgmatic\/postgresql_databases/) { source = "PostgreSQL" }
else if (path ~ /^borgmatic\//) { source = "borgmatic_metadata" }
else { source = "other" }
@ -155,10 +138,15 @@ EOF
}
END {
for (src in totals) {
printf "borgmatic_source_size_bytes{source=\"%s\"} %.0f\n", src, totals[src]
printf "borgmatic_source_size_bytes{repo=\"%s\",source=\"%s\"} %.0f\n", repo, src, totals[src]
}
}' >> "$TEMP_FILE"
fi
}
# Collect metrics for each configured repository
{% for repo in borgmatic_metrics_repos %}
collect_repo_metrics "{{ repo.path }}" "{{ repo.label }}"
{% endfor %}
# Atomic move
mv "$TEMP_FILE" "$OUTPUT_FILE"