From 3962e5a7de2f241f2cbf62a7f002fe5761b9b31d Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Sat, 17 Jan 2026 09:22:01 -0800 Subject: [PATCH] Fix borgmatic PostgreSQL backup and update backup sources (#21) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Fix PostgreSQL backup failure by adding explicit `pg_dump_command` path (was failing with "pg_dump: command not found" in LaunchAgent) - Remove `~/code/3rd/kiwix-tools` from backups (was just symlinks to ZIM archives in transmission) - Enable Loki log backup by removing from exclude_patterns ## Deployment and Testing - [x] Dry run with `--check --diff` shows expected changes - [ ] Deploy with `mise run provision-indri -- --tags borgmatic` - [ ] Verify config deployed: `ssh indri 'cat ~/.config/borgmatic/config.yaml'` - [ ] Run manual backup to test: `ssh indri 'mise x -- borgmatic create --verbosity 1'` - [ ] Verify PostgreSQL dump succeeds (no "pg_dump: command not found" error) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Reviewed-on: https://forge.tail8d86e.ts.net/eblume/blumeops/pulls/21 --- CLAUDE.md | 6 +- ansible/playbooks/indri.yml | 2 + ansible/roles/borgmatic/defaults/main.yml | 4 +- .../roles/borgmatic/templates/config.yaml.j2 | 3 + .../roles/borgmatic_metrics/defaults/main.yml | 7 + .../roles/borgmatic_metrics/handlers/main.yml | 6 + .../roles/borgmatic_metrics/tasks/main.yml | 43 + .../templates/borgmatic-metrics.plist.j2 | 21 + .../templates/borgmatic-metrics.sh.j2 | 163 ++++ .../grafana/files/dashboards/borgmatic.json | 781 ++++++++++++++++++ 10 files changed, 1032 insertions(+), 4 deletions(-) create mode 100644 ansible/roles/borgmatic_metrics/defaults/main.yml create mode 100644 ansible/roles/borgmatic_metrics/handlers/main.yml create mode 100644 ansible/roles/borgmatic_metrics/tasks/main.yml create mode 100644 ansible/roles/borgmatic_metrics/templates/borgmatic-metrics.plist.j2 create mode 100644 ansible/roles/borgmatic_metrics/templates/borgmatic-metrics.sh.j2 create mode 100644 ansible/roles/grafana/files/dashboards/borgmatic.json diff --git a/CLAUDE.md b/CLAUDE.md index 292061b..023c0a1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -35,9 +35,11 @@ The user will review your work as you go, and will merge the pr as the last step 5. Services are typically hosted on hostname "indri" and are launched from LaunchAgents of the user `erichblume`. If a service is available from `brew services` that is typically used, otherwise there is a utility called `mcquack` (`mcquack --help`) hosted at `https://forge.tail8d86e.ts.net/eblume/mcquack` - but you can just edit the mcquack launchagents directly via ansible. -6. Try to always test changes before applying them. Use syntax checkers, do dry runs, run commands manually via `ssh indri 'some command'`, etc. +6. Try to always test changes before applying them. Use syntax checkers, do dry runs (`--check --diff`), run commands manually via `ssh indri 'some command'`, etc. -7. After making changes, try to verify the result. Use `mise run indri-services-check` to do a general service health check. +7. **Wait for user review before deploying.** After creating a PR, do not run `mise run provision-indri` or other deployment commands until the user has had a chance to review the changes. The user will indicate when they're ready to deploy. + +8. After deploying changes, try to verify the result. Use `mise run indri-services-check` to do a general service health check. ## Project structure Some important places you can look: diff --git a/ansible/playbooks/indri.yml b/ansible/playbooks/indri.yml index 0759ab6..4645cb4 100644 --- a/ansible/playbooks/indri.yml +++ b/ansible/playbooks/indri.yml @@ -91,6 +91,8 @@ tags: kiwix - role: borgmatic tags: borgmatic + - role: borgmatic_metrics + tags: borgmatic_metrics - role: forgejo tags: forgejo - role: devpi diff --git a/ansible/roles/borgmatic/defaults/main.yml b/ansible/roles/borgmatic/defaults/main.yml index 243ba4d..a62e5de 100644 --- a/ansible/roles/borgmatic/defaults/main.yml +++ b/ansible/roles/borgmatic/defaults/main.yml @@ -11,7 +11,6 @@ borgmatic_schedule_minute: 0 borgmatic_source_directories: - /Users/erichblume/code/personal/zk - /opt/homebrew/var/forgejo - - /Users/erichblume/code/3rd/kiwix-tools - /Users/erichblume/.config/borgmatic - /Users/erichblume/Documents - /Users/erichblume/Pictures @@ -29,7 +28,6 @@ borgmatic_repositories: borgmatic_exclude_patterns: # Exclude mirrored PyPI cache (only backup private packages) - /Users/erichblume/devpi/+files/root/pypi - - /opt/homebrew/var/loki # Encryption passcommand (reads borg passphrase) borgmatic_encryption_passcommand: cat /Users/erichblume/.borg/config.yaml @@ -41,6 +39,8 @@ borgmatic_keep_yearly: 1000 # PostgreSQL databases to backup (streamed via pg_dump) # Password is read from ~/.pgpass (managed by postgresql role) +# pg_dump_command must be full path since LaunchAgent doesn't have homebrew in PATH +borgmatic_pg_dump_command: /opt/homebrew/opt/postgresql@18/bin/pg_dump borgmatic_postgresql_databases: - name: miniflux hostname: localhost diff --git a/ansible/roles/borgmatic/templates/config.yaml.j2 b/ansible/roles/borgmatic/templates/config.yaml.j2 index 4a782bf..6bfb835 100644 --- a/ansible/roles/borgmatic/templates/config.yaml.j2 +++ b/ansible/roles/borgmatic/templates/config.yaml.j2 @@ -41,5 +41,8 @@ postgresql_databases: hostname: {{ db.hostname | default('localhost') }} port: {{ db.port | default(5432) }} username: {{ db.username }} +{% if borgmatic_pg_dump_command is defined %} + pg_dump_command: {{ borgmatic_pg_dump_command }} +{% endif %} {% endfor %} {% endif %} diff --git a/ansible/roles/borgmatic_metrics/defaults/main.yml b/ansible/roles/borgmatic_metrics/defaults/main.yml new file mode 100644 index 0000000..368730a --- /dev/null +++ b/ansible/roles/borgmatic_metrics/defaults/main.yml @@ -0,0 +1,7 @@ +--- +borgmatic_metrics_repo: /Volumes/backups/borg/ +borgmatic_metrics_passcommand: cat /Users/erichblume/.borg/config.yaml +borgmatic_metrics_dir: /opt/homebrew/var/node_exporter/textfile +borgmatic_metrics_script: /Users/erichblume/bin/borgmatic-metrics +borgmatic_metrics_interval: 3600 # seconds between metric collection (hourly) +borgmatic_metrics_log_dir: /opt/homebrew/var/log diff --git a/ansible/roles/borgmatic_metrics/handlers/main.yml b/ansible/roles/borgmatic_metrics/handlers/main.yml new file mode 100644 index 0000000..0b91591 --- /dev/null +++ b/ansible/roles/borgmatic_metrics/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: Reload borgmatic-metrics + ansible.builtin.shell: | + launchctl unload ~/Library/LaunchAgents/mcquack.eblume.borgmatic-metrics.plist 2>/dev/null || true + launchctl load ~/Library/LaunchAgents/mcquack.eblume.borgmatic-metrics.plist + changed_when: true diff --git a/ansible/roles/borgmatic_metrics/tasks/main.yml b/ansible/roles/borgmatic_metrics/tasks/main.yml new file mode 100644 index 0000000..fb73cfa --- /dev/null +++ b/ansible/roles/borgmatic_metrics/tasks/main.yml @@ -0,0 +1,43 @@ +--- +- name: Ensure metrics directory exists + ansible.builtin.file: + path: "{{ borgmatic_metrics_dir }}" + state: directory + mode: '0755' + +- name: Ensure log directory exists + ansible.builtin.file: + path: "{{ borgmatic_metrics_log_dir }}" + state: directory + mode: '0755' + +- name: Ensure bin directory exists + ansible.builtin.file: + path: "{{ borgmatic_metrics_script | dirname }}" + state: directory + mode: '0755' + +- name: Deploy borgmatic-metrics script + ansible.builtin.template: + src: borgmatic-metrics.sh.j2 + dest: "{{ borgmatic_metrics_script }}" + mode: '0755' + +- name: Deploy borgmatic-metrics LaunchAgent plist + ansible.builtin.template: + src: borgmatic-metrics.plist.j2 + dest: ~/Library/LaunchAgents/mcquack.eblume.borgmatic-metrics.plist + mode: '0644' + notify: Reload borgmatic-metrics + +- name: Check if borgmatic-metrics LaunchAgent is loaded + ansible.builtin.command: launchctl list mcquack.eblume.borgmatic-metrics + register: borgmatic_metrics_launchctl_check + changed_when: false + failed_when: false + +- name: Load borgmatic-metrics LaunchAgent if not loaded + ansible.builtin.command: launchctl load ~/Library/LaunchAgents/mcquack.eblume.borgmatic-metrics.plist + when: borgmatic_metrics_launchctl_check.rc != 0 + changed_when: true + failed_when: false diff --git a/ansible/roles/borgmatic_metrics/templates/borgmatic-metrics.plist.j2 b/ansible/roles/borgmatic_metrics/templates/borgmatic-metrics.plist.j2 new file mode 100644 index 0000000..dea0f65 --- /dev/null +++ b/ansible/roles/borgmatic_metrics/templates/borgmatic-metrics.plist.j2 @@ -0,0 +1,21 @@ + + + + + + Label + mcquack.eblume.borgmatic-metrics + ProgramArguments + + {{ borgmatic_metrics_script }} + + StartInterval + {{ borgmatic_metrics_interval }} + RunAtLoad + + StandardErrorPath + {{ borgmatic_metrics_log_dir }}/mcquack.borgmatic-metrics.err.log + StandardOutPath + {{ borgmatic_metrics_log_dir }}/mcquack.borgmatic-metrics.out.log + + diff --git a/ansible/roles/borgmatic_metrics/templates/borgmatic-metrics.sh.j2 b/ansible/roles/borgmatic_metrics/templates/borgmatic-metrics.sh.j2 new file mode 100644 index 0000000..99ce1ca --- /dev/null +++ b/ansible/roles/borgmatic_metrics/templates/borgmatic-metrics.sh.j2 @@ -0,0 +1,163 @@ +#!/bin/bash +# {{ ansible_managed }} +# Collects borg backup metrics for node_exporter textfile collector + +set -euo pipefail + +export BORG_PASSCOMMAND="{{ borgmatic_metrics_passcommand }}" +BORG_REPO="{{ borgmatic_metrics_repo }}" +OUTPUT_FILE="{{ borgmatic_metrics_dir }}/borgmatic.prom" +TEMP_FILE="${OUTPUT_FILE}.tmp" + +# Check if borg is available via mise +BORG_CMD="mise x -- borg" + +# Get repository info +repo_json=$($BORG_CMD info --json "$BORG_REPO" 2>/dev/null) || { + echo "Failed to get borg repo info" >&2 + # Write down metric + cat > "$TEMP_FILE" << 'EOF' +# HELP borgmatic_up Borg backup repository is accessible +# TYPE borgmatic_up gauge +borgmatic_up 0 +EOF + mv "$TEMP_FILE" "$OUTPUT_FILE" + exit 0 +} + +# Get archive list +archives_json=$($BORG_CMD list --json "$BORG_REPO" 2>/dev/null) || { + echo "Failed to list borg archives" >&2 + exit 1 +} + +# Extract repository stats +total_size=$(echo "$repo_json" | jq -r '.cache.stats.total_size') +total_csize=$(echo "$repo_json" | jq -r '.cache.stats.total_csize') +unique_size=$(echo "$repo_json" | jq -r '.cache.stats.unique_size') +unique_csize=$(echo "$repo_json" | jq -r '.cache.stats.unique_csize') +total_chunks=$(echo "$repo_json" | jq -r '.cache.stats.total_chunks') +unique_chunks=$(echo "$repo_json" | jq -r '.cache.stats.total_unique_chunks') + +# Count archives +archive_count=$(echo "$archives_json" | jq -r '.archives | length') + +# Get last archive info +last_archive_name=$(echo "$archives_json" | jq -r '.archives[-1].name // empty') + +if [ -n "$last_archive_name" ]; then + # Get detailed info for the last archive + last_archive_json=$($BORG_CMD info --json "${BORG_REPO}::${last_archive_name}" 2>/dev/null) || { + echo "Failed to get last archive info" >&2 + last_archive_json="" + } + + if [ -n "$last_archive_json" ]; then + last_original_size=$(echo "$last_archive_json" | jq -r '.archives[0].stats.original_size') + last_compressed_size=$(echo "$last_archive_json" | jq -r '.archives[0].stats.compressed_size') + last_deduplicated_size=$(echo "$last_archive_json" | jq -r '.archives[0].stats.deduplicated_size') + last_nfiles=$(echo "$last_archive_json" | jq -r '.archives[0].stats.nfiles') + last_start=$(echo "$last_archive_json" | jq -r '.archives[0].start') + last_end=$(echo "$last_archive_json" | jq -r '.archives[0].end') + last_duration=$(echo "$last_archive_json" | jq -r '.archives[0].duration') + + # Convert timestamp to unix epoch + last_timestamp=$(date -j -f "%Y-%m-%dT%H:%M:%S" "${last_start%.*}" "+%s" 2>/dev/null || echo "0") + fi +fi + +# Write metrics +cat > "$TEMP_FILE" << EOF +# HELP borgmatic_up Borg backup repository is accessible +# TYPE borgmatic_up gauge +borgmatic_up 1 + +# HELP borgmatic_repo_original_size_bytes Total original size of all archives (sum of what each backup contains) +# TYPE borgmatic_repo_original_size_bytes gauge +borgmatic_repo_original_size_bytes $total_size + +# HELP borgmatic_repo_compressed_size_bytes Total compressed size of all archives +# TYPE borgmatic_repo_compressed_size_bytes gauge +borgmatic_repo_compressed_size_bytes $total_csize + +# HELP borgmatic_repo_deduplicated_size_bytes Actual disk usage after deduplication (unique data) +# TYPE borgmatic_repo_deduplicated_size_bytes gauge +borgmatic_repo_deduplicated_size_bytes $unique_csize + +# HELP borgmatic_repo_total_chunks Total number of chunks across all archives +# TYPE borgmatic_repo_total_chunks gauge +borgmatic_repo_total_chunks $total_chunks + +# HELP borgmatic_repo_unique_chunks Number of unique chunks (after deduplication) +# TYPE borgmatic_repo_unique_chunks gauge +borgmatic_repo_unique_chunks $unique_chunks + +# HELP borgmatic_archive_count Number of archives in the repository +# TYPE borgmatic_archive_count gauge +borgmatic_archive_count $archive_count +EOF + +# Add last archive metrics if available +if [ -n "${last_original_size:-}" ]; then + cat >> "$TEMP_FILE" << EOF + +# HELP borgmatic_last_archive_original_size_bytes Original size of the last archive (data being backed up) +# TYPE borgmatic_last_archive_original_size_bytes gauge +borgmatic_last_archive_original_size_bytes $last_original_size + +# HELP borgmatic_last_archive_compressed_size_bytes Compressed size of the last archive +# TYPE borgmatic_last_archive_compressed_size_bytes gauge +borgmatic_last_archive_compressed_size_bytes $last_compressed_size + +# HELP borgmatic_last_archive_deduplicated_size_bytes Deduplicated size of last archive (new data added) +# TYPE borgmatic_last_archive_deduplicated_size_bytes gauge +borgmatic_last_archive_deduplicated_size_bytes $last_deduplicated_size + +# HELP borgmatic_last_archive_files Number of files in the last archive +# TYPE borgmatic_last_archive_files gauge +borgmatic_last_archive_files $last_nfiles + +# HELP borgmatic_last_archive_timestamp Unix timestamp of the last backup +# TYPE borgmatic_last_archive_timestamp gauge +borgmatic_last_archive_timestamp $last_timestamp + +# HELP borgmatic_last_archive_duration_seconds Duration of the last backup in seconds +# TYPE borgmatic_last_archive_duration_seconds gauge +borgmatic_last_archive_duration_seconds ${last_duration:-0} +EOF + + # Collect per-source-directory sizes + cat >> "$TEMP_FILE" << 'EOF' + +# HELP borgmatic_source_size_bytes Size of each backup source directory in bytes +# TYPE borgmatic_source_size_bytes gauge +EOF + + # List archive contents and group by source directory + $BORG_CMD list "${BORG_REPO}::${last_archive_name}" --format "{size} {path}{NL}" 2>/dev/null | awk ' + { + size = $1 + path = $2 + # Map paths to friendly source names + if (path ~ /^Users\/[^\/]+\/Pictures/) { source = "Pictures" } + else if (path ~ /^Users\/[^\/]+\/Documents/) { source = "Documents" } + else if (path ~ /^Users\/[^\/]+\/devpi/) { source = "devpi" } + else if (path ~ /^Users\/[^\/]+\/code\/personal\/zk/) { source = "Zettelkasten" } + else if (path ~ /^Users\/[^\/]+\/.config\/borgmatic/) { source = "borgmatic_config" } + else if (path ~ /^opt\/homebrew\/var\/forgejo/) { source = "Forgejo" } + else if (path ~ /^opt\/homebrew\/var\/loki/) { source = "Loki" } + else if (path ~ /^borgmatic\/postgresql_databases/) { source = "PostgreSQL" } + else if (path ~ /^borgmatic\//) { source = "borgmatic_metadata" } + else { source = "other" } + + totals[source] += size + } + END { + for (src in totals) { + printf "borgmatic_source_size_bytes{source=\"%s\"} %.0f\n", src, totals[src] + } + }' >> "$TEMP_FILE" +fi + +# Atomic move +mv "$TEMP_FILE" "$OUTPUT_FILE" diff --git a/ansible/roles/grafana/files/dashboards/borgmatic.json b/ansible/roles/grafana/files/dashboards/borgmatic.json new file mode 100644 index 0000000..d2233ed --- /dev/null +++ b/ansible/roles/grafana/files/dashboards/borgmatic.json @@ -0,0 +1,781 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { "color": "red", "index": 0, "text": "DOWN" } + }, + "type": "value" + }, + { + "options": { + "1": { "color": "green", "index": 1, "text": "UP" } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "borgmatic_up", + "refId": "A" + } + ], + "title": "Repository Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "blue", "value": null }] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 5, "x": 4, "y": 0 }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "borgmatic_repo_deduplicated_size_bytes", + "refId": "A" + } + ], + "title": "Disk Usage (Deduplicated)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 5, "x": 9, "y": 0 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "borgmatic_last_archive_original_size_bytes", + "refId": "A" + } + ], + "title": "Data Being Backed Up", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "purple", "value": null }] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 14, "y": 0 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "borgmatic_archive_count", + "refId": "A" + } + ], + "title": "Archives", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 86400 }, + { "color": "red", "value": 172800 } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 17, "y": 0 }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "time() - borgmatic_last_archive_timestamp", + "refId": "A" + } + ], + "title": "Time Since Last Backup", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 2 }, + { "color": "green", "value": 5 } + ] + }, + "unit": "x" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 3, "x": 21, "y": 0 }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "borgmatic_repo_original_size_bytes / borgmatic_repo_deduplicated_size_bytes", + "refId": "A" + } + ], + "title": "Dedup Ratio", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "orange", "value": null }] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 4 }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "borgmatic_last_archive_deduplicated_size_bytes", + "refId": "A" + } + ], + "title": "Last Backup New Data", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "cyan", "value": null }] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 4 }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "borgmatic_last_archive_files", + "refId": "A" + } + ], + "title": "Files in Backup", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 60 }, + { "color": "red", "value": 300 } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 4 }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "borgmatic_last_archive_duration_seconds", + "refId": "A" + } + ], + "title": "Last Backup Duration", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "semi-dark-blue", "value": null }] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 4 }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "borgmatic_repo_unique_chunks", + "refId": "A" + } + ], + "title": "Unique Chunks", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 4 }, + "id": 14, + "options": { + "displayMode": "basic", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true, + "valueMode": "color" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "topk(10, borgmatic_source_size_bytes)", + "legendFormat": "{{source}}", + "refId": "A" + } + ], + "title": "Backup Size by Source", + "description": "Breakdown of backup size by source directory", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "id": 11, + "options": { + "legend": { + "calcs": ["lastNotNull", "min", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "borgmatic_last_archive_original_size_bytes", + "legendFormat": "Backup Size (if extracted)", + "refId": "A" + } + ], + "title": "Backup Size Over Time", + "description": "The original (uncompressed, non-deduplicated) size of each backup - this is how much data would be restored", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "blue", "value": null }] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "id": 12, + "options": { + "legend": { + "calcs": ["lastNotNull", "min", "max"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "borgmatic_repo_deduplicated_size_bytes", + "legendFormat": "Repository Size on Disk", + "refId": "A" + } + ], + "title": "Repository Disk Usage Over Time", + "description": "Actual disk space used by the backup repository (after compression and deduplication)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "orange", "value": null }] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 }, + "id": 13, + "options": { + "legend": { + "calcs": ["mean", "max", "lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "expr": "borgmatic_last_archive_deduplicated_size_bytes", + "legendFormat": "New Data Added", + "refId": "A" + } + ], + "title": "New Data Per Backup", + "description": "How much new (deduplicated) data each backup added to the repository", + "type": "timeseries" + } + ], + "refresh": "5m", + "schemaVersion": 38, + "tags": ["borgmatic", "backup", "borg"], + "templating": { + "list": [] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Borgmatic Backups", + "uid": "borgmatic", + "version": 1, + "weekStart": "" +}