From c1e7497e12b2d038cb1165935fc6e3acf0100be4 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 27 Mar 2026 19:20:12 -0700 Subject: [PATCH] Add offsite backup for immich photo library to BorgBase Adds a second borgmatic config (photos.yaml) that backs up /Volumes/photos (sifaka SMB mount) to a dedicated BorgBase repo, running daily at 4 AM. Refactors borgmatic metrics to support multiple repos with a `repo` label, and updates the Grafana dashboard with a repo selector variable. Co-Authored-By: Claude Opus 4.6 (1M context) --- ansible/roles/borgmatic/defaults/main.yml | 12 ++ ansible/roles/borgmatic/handlers/main.yml | 6 + ansible/roles/borgmatic/tasks/main.yml | 36 +++- .../templates/borgmatic-photos.plist.j2 | 39 ++++ .../roles/borgmatic/templates/photos.yaml.j2 | 29 +++ .../roles/borgmatic_metrics/defaults/main.yml | 10 +- .../templates/borgmatic-metrics.sh.j2 | 180 ++++++++---------- .../dashboards/configmap-borgmatic.yaml | 93 ++++++--- .../immich-photos-backup.feature.md | 1 + docs/reference/services/borgmatic.md | 12 +- docs/reference/storage/backups.md | 25 ++- 11 files changed, 306 insertions(+), 137 deletions(-) create mode 100644 ansible/roles/borgmatic/templates/borgmatic-photos.plist.j2 create mode 100644 ansible/roles/borgmatic/templates/photos.yaml.j2 create mode 100644 docs/changelog.d/immich-photos-backup.feature.md diff --git a/ansible/roles/borgmatic/defaults/main.yml b/ansible/roles/borgmatic/defaults/main.yml index 93b02ba..c7a9793 100644 --- a/ansible/roles/borgmatic/defaults/main.yml +++ b/ansible/roles/borgmatic/defaults/main.yml @@ -59,6 +59,18 @@ borgmatic_keep_yearly: 1000 # PostgreSQL databases to backup (streamed via pg_dump) # Password is read from ~/.pgpass (managed by this role) # pg_dump_command must be full path since LaunchAgent doesn't have homebrew in PATH +# --- Immich photo library backup (BorgBase offsite only) --- +borgmatic_photos_config: /Users/erichblume/.config/borgmatic/photos.yaml +borgmatic_photos_source_dir: /Volumes/photos +borgmatic_photos_borgbase_repo: ssh://xcrtl5tg@xcrtl5tg.repo.borgbase.com/./repo +# Schedule: runs daily at 4:00 AM (offset from main backup at 2:00 AM) +borgmatic_photos_schedule_hour: 4 +borgmatic_photos_schedule_minute: 0 +# Retention: photos are precious, keep more history +borgmatic_photos_keep_daily: 7 +borgmatic_photos_keep_monthly: 12 +borgmatic_photos_keep_yearly: 1000 + borgmatic_pg_dump_command: /opt/homebrew/opt/postgresql@18/bin/pg_dump borgmatic_postgresql_databases: # k8s PostgreSQL (CloudNativePG) via Caddy L4 proxy diff --git a/ansible/roles/borgmatic/handlers/main.yml b/ansible/roles/borgmatic/handlers/main.yml index 5fd6174..3463cce 100644 --- a/ansible/roles/borgmatic/handlers/main.yml +++ b/ansible/roles/borgmatic/handlers/main.yml @@ -4,3 +4,9 @@ launchctl unload ~/Library/LaunchAgents/mcquack.eblume.borgmatic.plist 2>/dev/null || true launchctl load ~/Library/LaunchAgents/mcquack.eblume.borgmatic.plist changed_when: true + +- name: Reload borgmatic-photos + ansible.builtin.shell: | + launchctl unload ~/Library/LaunchAgents/mcquack.eblume.borgmatic-photos.plist 2>/dev/null || true + launchctl load ~/Library/LaunchAgents/mcquack.eblume.borgmatic-photos.plist + changed_when: true diff --git a/ansible/roles/borgmatic/tasks/main.yml b/ansible/roles/borgmatic/tasks/main.yml index e7380dd..dd6efdd 100644 --- a/ansible/roles/borgmatic/tasks/main.yml +++ b/ansible/roles/borgmatic/tasks/main.yml @@ -28,11 +28,14 @@ mode: '0600' no_log: true -- name: Add BorgBase host key to known_hosts +- name: Add BorgBase host keys to known_hosts ansible.builtin.known_hosts: - name: u3ugi1x1.repo.borgbase.com - key: "u3ugi1x1.repo.borgbase.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGU0mISTyHBw9tBs6SuhSq8tvNM8m9eifQxM+88TowPO" + name: "{{ item }}" + key: "{{ item }} ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGU0mISTyHBw9tBs6SuhSq8tvNM8m9eifQxM+88TowPO" state: present + loop: + - u3ugi1x1.repo.borgbase.com + - xcrtl5tg.repo.borgbase.com - name: Ensure k8s dump directory exists ansible.builtin.file: @@ -65,3 +68,30 @@ when: borgmatic_launchctl_check.rc != 0 changed_when: true failed_when: false + +# --- Immich photo library backup (BorgBase offsite only) --- + +- name: Deploy borgmatic photos configuration + ansible.builtin.template: + src: photos.yaml.j2 + dest: "{{ borgmatic_photos_config }}" + mode: '0600' + +- name: Deploy borgmatic-photos LaunchAgent plist + ansible.builtin.template: + src: borgmatic-photos.plist.j2 + dest: ~/Library/LaunchAgents/mcquack.eblume.borgmatic-photos.plist + mode: '0644' + notify: Reload borgmatic-photos + +- name: Check if borgmatic-photos LaunchAgent is loaded + ansible.builtin.command: launchctl list mcquack.eblume.borgmatic-photos + register: borgmatic_photos_launchctl_check + changed_when: false + failed_when: false + +- name: Load borgmatic-photos LaunchAgent if not loaded + ansible.builtin.command: launchctl load ~/Library/LaunchAgents/mcquack.eblume.borgmatic-photos.plist + when: borgmatic_photos_launchctl_check.rc != 0 + changed_when: true + failed_when: false diff --git a/ansible/roles/borgmatic/templates/borgmatic-photos.plist.j2 b/ansible/roles/borgmatic/templates/borgmatic-photos.plist.j2 new file mode 100644 index 0000000..6e69159 --- /dev/null +++ b/ansible/roles/borgmatic/templates/borgmatic-photos.plist.j2 @@ -0,0 +1,39 @@ + + + + + + KeepAlive + + Label + mcquack.eblume.borgmatic-photos + EnvironmentVariables + + PATH + /opt/homebrew/bin:/usr/bin:/bin + + ProgramArguments + + /opt/homebrew/opt/mise/bin/mise + x + -- + borgmatic + --config + {{ borgmatic_photos_config }} + create + + RunAtLoad + + StandardErrorPath + {{ borgmatic_log_dir }}/mcquack.borgmatic-photos.err.log + StandardOutPath + {{ borgmatic_log_dir }}/mcquack.borgmatic-photos.out.log + StartCalendarInterval + + Hour + {{ borgmatic_photos_schedule_hour }} + Minute + {{ borgmatic_photos_schedule_minute }} + + + diff --git a/ansible/roles/borgmatic/templates/photos.yaml.j2 b/ansible/roles/borgmatic/templates/photos.yaml.j2 new file mode 100644 index 0000000..1c118df --- /dev/null +++ b/ansible/roles/borgmatic/templates/photos.yaml.j2 @@ -0,0 +1,29 @@ +# {{ ansible_managed }} +# +# Borgmatic config for immich photo library backup. +# Backs up /Volumes/photos (sifaka SMB mount) to BorgBase offsite ONLY. +# Separate from the main borgmatic config to keep concerns isolated: +# - main config: indri data → sifaka + borgbase +# - this config: sifaka photos → borgbase (different repo) + +local_path: {{ borgmatic_local_path }} + +source_directories: + - {{ borgmatic_photos_source_dir }} + +source_directories_must_exist: true + +repositories: + - path: {{ borgmatic_photos_borgbase_repo }} + label: borgbase-immich-photos + encryption: repokey + append_only: true + +encryption_passcommand: {{ borgmatic_encryption_passcommand }} + +ssh_command: ssh -o IdentitiesOnly=yes -i {{ borgmatic_borgbase_ssh_key_path }} + +# Retention policy — photos are precious, keep more history +keep_daily: {{ borgmatic_photos_keep_daily }} +keep_monthly: {{ borgmatic_photos_keep_monthly }} +keep_yearly: {{ borgmatic_photos_keep_yearly }} diff --git a/ansible/roles/borgmatic_metrics/defaults/main.yml b/ansible/roles/borgmatic_metrics/defaults/main.yml index c8207ba..8fcd91e 100644 --- a/ansible/roles/borgmatic_metrics/defaults/main.yml +++ b/ansible/roles/borgmatic_metrics/defaults/main.yml @@ -1,6 +1,14 @@ --- -borgmatic_metrics_repo: /Volumes/backups/borg/ +# Borg repositories to collect metrics from +# Each entry needs a path (local or ssh://) and a label for Prometheus metrics +borgmatic_metrics_repos: + - path: /Volumes/backups/borg/ + label: sifaka-local + - path: ssh://xcrtl5tg@xcrtl5tg.repo.borgbase.com/./repo + label: borgbase-immich-photos + borgmatic_metrics_passcommand: cat /Users/erichblume/.borg/config.yaml +borgmatic_metrics_ssh_key: /Users/erichblume/.ssh/borgbase_ed25519 borgmatic_metrics_dir: /opt/homebrew/var/node_exporter/textfile borgmatic_metrics_script: /Users/erichblume/.local/bin/borgmatic-metrics borgmatic_metrics_interval: 3600 # seconds between metric collection (hourly) diff --git a/ansible/roles/borgmatic_metrics/templates/borgmatic-metrics.sh.j2 b/ansible/roles/borgmatic_metrics/templates/borgmatic-metrics.sh.j2 index 856cbe9..b3ad605 100644 --- a/ansible/roles/borgmatic_metrics/templates/borgmatic-metrics.sh.j2 +++ b/ansible/roles/borgmatic_metrics/templates/borgmatic-metrics.sh.j2 @@ -1,11 +1,12 @@ #!/bin/bash # {{ ansible_managed }} # Collects borg backup metrics for node_exporter textfile collector +# Supports multiple repositories with a repo label for Prometheus set -euo pipefail export BORG_PASSCOMMAND="{{ borgmatic_metrics_passcommand }}" -BORG_REPO="{{ borgmatic_metrics_repo }}" +export BORG_RSH="ssh -o IdentitiesOnly=yes -i {{ borgmatic_metrics_ssh_key }}" OUTPUT_FILE="{{ borgmatic_metrics_dir }}/borgmatic.prom" TEMP_FILE="${OUTPUT_FILE}.tmp" @@ -13,129 +14,109 @@ TEMP_FILE="${OUTPUT_FILE}.tmp" BORG_CMD="/opt/homebrew/bin/borg" JQ_CMD="/opt/homebrew/bin/jq" -# Get repository info -repo_json=$($BORG_CMD info --json "$BORG_REPO" 2>/dev/null) || { - echo "Failed to get borg repo info" >&2 - # Write down metric - cat > "$TEMP_FILE" << 'EOF' +# Start fresh +cat > "$TEMP_FILE" << 'EOF' # HELP borgmatic_up Borg backup repository is accessible # TYPE borgmatic_up gauge -borgmatic_up 0 -EOF - mv "$TEMP_FILE" "$OUTPUT_FILE" - exit 0 -} - -# Get archive list -archives_json=$($BORG_CMD list --json "$BORG_REPO" 2>/dev/null) || { - echo "Failed to list borg archives" >&2 - exit 1 -} - -# Extract repository stats -total_size=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.total_size') -total_csize=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.total_csize') -unique_size=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.unique_size') -unique_csize=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.unique_csize') -total_chunks=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.total_chunks') -unique_chunks=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.total_unique_chunks') - -# Count archives -archive_count=$(echo "$archives_json" | $JQ_CMD -r '.archives | length') - -# Get last archive info -last_archive_name=$(echo "$archives_json" | $JQ_CMD -r '.archives[-1].name // empty') - -if [ -n "$last_archive_name" ]; then - # Get detailed info for the last archive - last_archive_json=$($BORG_CMD info --json "${BORG_REPO}::${last_archive_name}" 2>/dev/null) || { - echo "Failed to get last archive info" >&2 - last_archive_json="" - } - - if [ -n "$last_archive_json" ]; then - last_original_size=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].stats.original_size') - last_compressed_size=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].stats.compressed_size') - last_deduplicated_size=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].stats.deduplicated_size') - last_nfiles=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].stats.nfiles') - last_start=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].start') - last_end=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].end') - last_duration=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].duration') - - # Convert timestamp to unix epoch - last_timestamp=$(date -j -f "%Y-%m-%dT%H:%M:%S" "${last_start%.*}" "+%s" 2>/dev/null || echo "0") - fi -fi - -# Write metrics -cat > "$TEMP_FILE" << EOF -# HELP borgmatic_up Borg backup repository is accessible -# TYPE borgmatic_up gauge -borgmatic_up 1 - # HELP borgmatic_repo_original_size_bytes Total original size of all archives (sum of what each backup contains) # TYPE borgmatic_repo_original_size_bytes gauge -borgmatic_repo_original_size_bytes $total_size - # HELP borgmatic_repo_compressed_size_bytes Total compressed size of all archives # TYPE borgmatic_repo_compressed_size_bytes gauge -borgmatic_repo_compressed_size_bytes $total_csize - # HELP borgmatic_repo_deduplicated_size_bytes Actual disk usage after deduplication (unique data) # TYPE borgmatic_repo_deduplicated_size_bytes gauge -borgmatic_repo_deduplicated_size_bytes $unique_csize - # HELP borgmatic_repo_total_chunks Total number of chunks across all archives # TYPE borgmatic_repo_total_chunks gauge -borgmatic_repo_total_chunks $total_chunks - # HELP borgmatic_repo_unique_chunks Number of unique chunks (after deduplication) # TYPE borgmatic_repo_unique_chunks gauge -borgmatic_repo_unique_chunks $unique_chunks - # HELP borgmatic_archive_count Number of archives in the repository # TYPE borgmatic_archive_count gauge -borgmatic_archive_count $archive_count -EOF - -# Add last archive metrics if available -if [ -n "${last_original_size:-}" ]; then - cat >> "$TEMP_FILE" << EOF - # HELP borgmatic_last_archive_original_size_bytes Original size of the last archive (data being backed up) # TYPE borgmatic_last_archive_original_size_bytes gauge -borgmatic_last_archive_original_size_bytes $last_original_size - # HELP borgmatic_last_archive_compressed_size_bytes Compressed size of the last archive # TYPE borgmatic_last_archive_compressed_size_bytes gauge -borgmatic_last_archive_compressed_size_bytes $last_compressed_size - # HELP borgmatic_last_archive_deduplicated_size_bytes Deduplicated size of last archive (new data added) # TYPE borgmatic_last_archive_deduplicated_size_bytes gauge -borgmatic_last_archive_deduplicated_size_bytes $last_deduplicated_size - # HELP borgmatic_last_archive_files Number of files in the last archive # TYPE borgmatic_last_archive_files gauge -borgmatic_last_archive_files $last_nfiles - # HELP borgmatic_last_archive_timestamp Unix timestamp of the last backup # TYPE borgmatic_last_archive_timestamp gauge -borgmatic_last_archive_timestamp $last_timestamp - # HELP borgmatic_last_archive_duration_seconds Duration of the last backup in seconds # TYPE borgmatic_last_archive_duration_seconds gauge -borgmatic_last_archive_duration_seconds ${last_duration:-0} -EOF - - # Collect per-source-directory sizes - cat >> "$TEMP_FILE" << 'EOF' - # HELP borgmatic_source_size_bytes Size of each backup source directory in bytes # TYPE borgmatic_source_size_bytes gauge EOF - # List archive contents and group by source directory - $BORG_CMD list "${BORG_REPO}::${last_archive_name}" --format "{size} {path}{NL}" 2>/dev/null | awk ' +collect_repo_metrics() { + local repo_path="$1" + local repo_label="$2" + + # Get repository info + repo_json=$($BORG_CMD info --json "$repo_path" 2>/dev/null) || { + echo "Failed to get borg repo info for $repo_label" >&2 + echo "borgmatic_up{repo=\"$repo_label\"} 0" >> "$TEMP_FILE" + return + } + + # Get archive list + archives_json=$($BORG_CMD list --json "$repo_path" 2>/dev/null) || { + echo "Failed to list borg archives for $repo_label" >&2 + echo "borgmatic_up{repo=\"$repo_label\"} 0" >> "$TEMP_FILE" + return + } + + # Extract repository stats + total_size=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.total_size') + total_csize=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.total_csize') + unique_size=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.unique_size') + unique_csize=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.unique_csize') + total_chunks=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.total_chunks') + unique_chunks=$(echo "$repo_json" | $JQ_CMD -r '.cache.stats.total_unique_chunks') + archive_count=$(echo "$archives_json" | $JQ_CMD -r '.archives | length') + + cat >> "$TEMP_FILE" << EOF +borgmatic_up{repo="$repo_label"} 1 +borgmatic_repo_original_size_bytes{repo="$repo_label"} $total_size +borgmatic_repo_compressed_size_bytes{repo="$repo_label"} $total_csize +borgmatic_repo_deduplicated_size_bytes{repo="$repo_label"} $unique_csize +borgmatic_repo_total_chunks{repo="$repo_label"} $total_chunks +borgmatic_repo_unique_chunks{repo="$repo_label"} $unique_chunks +borgmatic_archive_count{repo="$repo_label"} $archive_count +EOF + + # Get last archive info + last_archive_name=$(echo "$archives_json" | $JQ_CMD -r '.archives[-1].name // empty') + + if [ -z "$last_archive_name" ]; then + return + fi + + # Get detailed info for the last archive + last_archive_json=$($BORG_CMD info --json "${repo_path}::${last_archive_name}" 2>/dev/null) || { + echo "Failed to get last archive info for $repo_label" >&2 + return + } + + last_original_size=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].stats.original_size') + last_compressed_size=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].stats.compressed_size') + last_deduplicated_size=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].stats.deduplicated_size') + last_nfiles=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].stats.nfiles') + last_start=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].start') + last_duration=$(echo "$last_archive_json" | $JQ_CMD -r '.archives[0].duration') + + # Convert timestamp to unix epoch + last_timestamp=$(date -j -f "%Y-%m-%dT%H:%M:%S" "${last_start%.*}" "+%s" 2>/dev/null || echo "0") + + cat >> "$TEMP_FILE" << EOF +borgmatic_last_archive_original_size_bytes{repo="$repo_label"} $last_original_size +borgmatic_last_archive_compressed_size_bytes{repo="$repo_label"} $last_compressed_size +borgmatic_last_archive_deduplicated_size_bytes{repo="$repo_label"} $last_deduplicated_size +borgmatic_last_archive_files{repo="$repo_label"} $last_nfiles +borgmatic_last_archive_timestamp{repo="$repo_label"} $last_timestamp +borgmatic_last_archive_duration_seconds{repo="$repo_label"} ${last_duration:-0} +EOF + + # Collect per-source-directory sizes + $BORG_CMD list "${repo_path}::${last_archive_name}" --format "{size} {path}{NL}" 2>/dev/null | awk -v repo="$repo_label" ' { size = $1 path = $2 @@ -145,8 +126,10 @@ EOF else if (path ~ /^Users\/[^\/]+\/devpi/) { source = "devpi" } else if (path ~ /^Users\/[^\/]+\/code\/personal\/zk/) { source = "Zettelkasten" } else if (path ~ /^Users\/[^\/]+\/.config\/borgmatic/) { source = "borgmatic_config" } + else if (path ~ /^Users\/[^\/]+\/.local\/share\/borgmatic/) { source = "k8s_dumps" } else if (path ~ /^opt\/homebrew\/var\/forgejo/) { source = "Forgejo" } else if (path ~ /^opt\/homebrew\/var\/loki/) { source = "Loki" } + else if (path ~ /^Volumes\/photos/) { source = "immich_photos" } else if (path ~ /^borgmatic\/postgresql_databases/) { source = "PostgreSQL" } else if (path ~ /^borgmatic\//) { source = "borgmatic_metadata" } else { source = "other" } @@ -155,10 +138,15 @@ EOF } END { for (src in totals) { - printf "borgmatic_source_size_bytes{source=\"%s\"} %.0f\n", src, totals[src] + printf "borgmatic_source_size_bytes{repo=\"%s\",source=\"%s\"} %.0f\n", repo, src, totals[src] } }' >> "$TEMP_FILE" -fi +} + +# Collect metrics for each configured repository +{% for repo in borgmatic_metrics_repos %} +collect_repo_metrics "{{ repo.path }}" "{{ repo.label }}" +{% endfor %} # Atomic move mv "$TEMP_FILE" "$OUTPUT_FILE" diff --git a/argocd/manifests/grafana-config/dashboards/configmap-borgmatic.yaml b/argocd/manifests/grafana-config/dashboards/configmap-borgmatic.yaml index 0e16982..c29f021 100644 --- a/argocd/manifests/grafana-config/dashboards/configmap-borgmatic.yaml +++ b/argocd/manifests/grafana-config/dashboards/configmap-borgmatic.yaml @@ -70,7 +70,8 @@ data: "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "borgmatic_up", + "expr": "borgmatic_up{repo=~\"$repo\"}", + "legendFormat": "{{repo}}", "refId": "A" } ], @@ -114,7 +115,8 @@ data: "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "borgmatic_repo_deduplicated_size_bytes", + "expr": "borgmatic_repo_deduplicated_size_bytes{repo=~\"$repo\"}", + "legendFormat": "{{repo}}", "refId": "A" } ], @@ -158,7 +160,8 @@ data: "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "borgmatic_last_archive_original_size_bytes", + "expr": "borgmatic_last_archive_original_size_bytes{repo=~\"$repo\"}", + "legendFormat": "{{repo}}", "refId": "A" } ], @@ -202,7 +205,8 @@ data: "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "borgmatic_archive_count", + "expr": "borgmatic_archive_count{repo=~\"$repo\"}", + "legendFormat": "{{repo}}", "refId": "A" } ], @@ -250,7 +254,8 @@ data: "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "time() - borgmatic_last_archive_timestamp", + "expr": "time() - borgmatic_last_archive_timestamp{repo=~\"$repo\"}", + "legendFormat": "{{repo}}", "refId": "A" } ], @@ -299,7 +304,8 @@ data: "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "borgmatic_repo_original_size_bytes / borgmatic_repo_deduplicated_size_bytes", + "expr": "borgmatic_repo_original_size_bytes{repo=~\"$repo\"} / borgmatic_repo_deduplicated_size_bytes{repo=~\"$repo\"}", + "legendFormat": "{{repo}}", "refId": "A" } ], @@ -343,7 +349,8 @@ data: "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "borgmatic_last_archive_deduplicated_size_bytes", + "expr": "borgmatic_last_archive_deduplicated_size_bytes{repo=~\"$repo\"}", + "legendFormat": "{{repo}}", "refId": "A" } ], @@ -387,7 +394,8 @@ data: "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "borgmatic_last_archive_files", + "expr": "borgmatic_last_archive_files{repo=~\"$repo\"}", + "legendFormat": "{{repo}}", "refId": "A" } ], @@ -435,7 +443,8 @@ data: "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "borgmatic_last_archive_duration_seconds", + "expr": "borgmatic_last_archive_duration_seconds{repo=~\"$repo\"}", + "legendFormat": "{{repo}}", "refId": "A" } ], @@ -479,7 +488,8 @@ data: "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "borgmatic_repo_unique_chunks", + "expr": "borgmatic_repo_unique_chunks{repo=~\"$repo\"}", + "legendFormat": "{{repo}}", "refId": "A" } ], @@ -524,8 +534,8 @@ data: "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "topk(10, borgmatic_source_size_bytes)", - "legendFormat": "{{source}}", + "expr": "topk(10, borgmatic_source_size_bytes{repo=~\"$repo\"})", + "legendFormat": "{{repo}} / {{source}}", "refId": "A" } ], @@ -541,8 +551,7 @@ data: "fieldConfig": { "defaults": { "color": { - "fixedColor": "green", - "mode": "fixed" + "mode": "palette-classic" }, "custom": { "axisBorderShow": false, @@ -603,8 +612,8 @@ data: "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "borgmatic_last_archive_original_size_bytes", - "legendFormat": "Backup Size (if extracted)", + "expr": "borgmatic_last_archive_original_size_bytes{repo=~\"$repo\"}", + "legendFormat": "{{repo}}", "refId": "A" } ], @@ -620,8 +629,7 @@ data: "fieldConfig": { "defaults": { "color": { - "fixedColor": "blue", - "mode": "fixed" + "mode": "palette-classic" }, "custom": { "axisBorderShow": false, @@ -682,8 +690,8 @@ data: "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "borgmatic_repo_deduplicated_size_bytes", - "legendFormat": "Repository Size on Disk", + "expr": "borgmatic_repo_deduplicated_size_bytes{repo=~\"$repo\"}", + "legendFormat": "{{repo}}", "refId": "A" } ], @@ -699,8 +707,7 @@ data: "fieldConfig": { "defaults": { "color": { - "fixedColor": "orange", - "mode": "fixed" + "mode": "palette-classic" }, "custom": { "axisBorderShow": false, @@ -761,8 +768,8 @@ data: "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "borgmatic_last_archive_deduplicated_size_bytes", - "legendFormat": "New Data Added", + "expr": "borgmatic_last_archive_deduplicated_size_bytes{repo=~\"$repo\"}", + "legendFormat": "{{repo}}", "refId": "A" } ], @@ -778,8 +785,7 @@ data: "fieldConfig": { "defaults": { "color": { - "fixedColor": "yellow", - "mode": "fixed" + "mode": "palette-classic" }, "custom": { "axisBorderShow": false, @@ -844,8 +850,8 @@ data: "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "expr": "borgmatic_last_archive_duration_seconds", - "legendFormat": "Backup Duration", + "expr": "borgmatic_last_archive_duration_seconds{repo=~\"$repo\"}", + "legendFormat": "{{repo}}", "refId": "A" } ], @@ -858,7 +864,36 @@ data: "schemaVersion": 38, "tags": ["borg", "backup"], "templating": { - "list": [] + "list": [ + { + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(borgmatic_up, repo)", + "hide": 0, + "includeAll": true, + "label": "Repository", + "multi": true, + "name": "repo", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(borgmatic_up, repo)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] }, "time": { "from": "now-30d", diff --git a/docs/changelog.d/immich-photos-backup.feature.md b/docs/changelog.d/immich-photos-backup.feature.md new file mode 100644 index 0000000..6391af5 --- /dev/null +++ b/docs/changelog.d/immich-photos-backup.feature.md @@ -0,0 +1 @@ +Add offsite backup for immich photo library to BorgBase, running daily at 4 AM from indri via sifaka SMB mount. diff --git a/docs/reference/services/borgmatic.md b/docs/reference/services/borgmatic.md index 1020327..fea4551 100644 --- a/docs/reference/services/borgmatic.md +++ b/docs/reference/services/borgmatic.md @@ -15,9 +15,12 @@ Daily backup system using Borg backup, running on indri. | Property | Value | |----------|-------| | **Install** | mise (pipx) | -| **Config** | `~/.config/borgmatic/config.yaml` | -| **Schedule** | Daily at 2:00 AM | -| **Repository** | `/Volumes/backups/borg/` on [[sifaka|Sifaka]] | +| **Main config** | `~/.config/borgmatic/config.yaml` | +| **Photos config** | `~/.config/borgmatic/photos.yaml` | +| **Main schedule** | Daily at 2:00 AM | +| **Photos schedule** | Daily at 4:00 AM | +| **Main targets** | [[sifaka]] local + BorgBase offsite | +| **Photos target** | BorgBase offsite only | ## What Gets Backed Up @@ -35,6 +38,9 @@ Daily backup system using Borg backup, running on indri. **K8s SQLite databases (pre-backup dump via kubectl exec):** - [[mealie]] - Recipe manager (`/app/data/mealie.db`) +**Immich photo library** (separate config, BorgBase offsite only): +- `/Volumes/photos` (sifaka SMB mount, ~128 GB) + **Not backed up (by design):** - ZIM archives (re-downloadable) - Prometheus metrics (ephemeral) diff --git a/docs/reference/storage/backups.md b/docs/reference/storage/backups.md index e830822..9ca3bcb 100644 --- a/docs/reference/storage/backups.md +++ b/docs/reference/storage/backups.md @@ -37,9 +37,23 @@ Daily automated backups from [[indri]] to [[sifaka|Sifaka]] NAS. | immich | immich-pg | [[postgresql|pg.ops.eblu.me:5433]] | pg_dump stream | | mealie | — (SQLite) | k8s pod | kubectl exec sqlite3 .backup | +## Immich Photo Library (Offsite Only) + +The [[immich]] photo library lives on [[sifaka]] at `/volume1/photos` (SMB-mounted on [[indri]] as `/Volumes/photos`). Since sifaka is already the local backup target, photos are backed up to BorgBase offsite only — not back to sifaka. + +| Property | Value | +|----------|-------| +| **Config** | `~/.config/borgmatic/photos.yaml` | +| **Schedule** | Daily at 4:00 AM (offset from main backup) | +| **Source** | `/Volumes/photos` (sifaka SMB mount) | +| **Target** | BorgBase `borgbase-immich-photos` repo | +| **Size** | ~128 GB | + +Uses the same encryption passphrase and SSH key as the main borgmatic config. + ## Sifaka-Native Data -Some data lives directly on [[sifaka]] rather than being backed up to it (photos via [[immich]], music via [[navidrome]], video via [[jellyfin]]). See [[sifaka]] for data protection details. +Other data lives directly on [[sifaka]] (music via [[navidrome]], video via [[jellyfin]]). See [[sifaka]] for data protection details. ## What Is NOT Backed Up @@ -60,10 +74,11 @@ Some data lives directly on [[sifaka]] rather than being backed up to it (photos ## Backup Targets -| Repository | Location | Label | -|------------|----------|-------| -| `/Volumes/backups/borg/` | [[sifaka]] (local NAS) | — | -| `ssh://u3ugi1x1@u3ugi1x1.repo.borgbase.com/./repo` | BorgBase (offsite) | `borgbase-offsite` | +| Repository | Location | Label | Backs up | +|------------|----------|-------|----------| +| `/Volumes/backups/borg/` | [[sifaka]] (local NAS) | `sifaka-borg-backups` | indri data | +| `ssh://u3ugi1x1@...repo.borgbase.com/./repo` | BorgBase (offsite) | `borgbase-offsite` | indri data | +| `ssh://xcrtl5tg@...repo.borgbase.com/./repo` | BorgBase (offsite) | `borgbase-immich-photos` | immich photos | ## Monitoring