CLAUDE.md

@@ -35,9 +35,11 @@ The user will review your work as you go, and will merge the pr as the last step
 
 5. Services are typically hosted on hostname "indri" and are launched from LaunchAgents of the user `erichblume`. If a service is available from `brew services` that is typically used, otherwise there is a utility called `mcquack` (`mcquack --help`) hosted at `https://forge.tail8d86e.ts.net/eblume/mcquack` - but you can just edit the mcquack launchagents directly via ansible.
 
-6. Try to always test changes before applying them. Use syntax checkers, do dry runs, run commands manually via `ssh indri 'some command'`, etc.
+6. Try to always test changes before applying them. Use syntax checkers, do dry runs (`--check --diff`), run commands manually via `ssh indri 'some command'`, etc.
 
-7. After making changes, try to verify the result. Use `mise run indri-services-check` to do a general service health check.
+7. **Wait for user review before deploying.** After creating a PR, do not run `mise run provision-indri` or other deployment commands until the user has had a chance to review the changes. The user will indicate when they're ready to deploy.
+
+8. After deploying changes, try to verify the result. Use `mise run indri-services-check` to do a general service health check.
 
 ## Project structure
 Some important places you can look:

ansible/playbooks/indri.yml

@@ -91,6 +91,8 @@
       tags: kiwix
     - role: borgmatic
       tags: borgmatic
+    - role: borgmatic_metrics
+      tags: borgmatic_metrics
     - role: forgejo
       tags: forgejo
     - role: devpi

ansible/roles/borgmatic/defaults/main.yml

@@ -11,7 +11,6 @@ borgmatic_schedule_minute: 0
 borgmatic_source_directories:
   - /Users/erichblume/code/personal/zk
   - /opt/homebrew/var/forgejo
-  - /Users/erichblume/code/3rd/kiwix-tools
   - /Users/erichblume/.config/borgmatic
   - /Users/erichblume/Documents
   - /Users/erichblume/Pictures
@@ -29,7 +28,6 @@ borgmatic_repositories:
 borgmatic_exclude_patterns:
   # Exclude mirrored PyPI cache (only backup private packages)
   - /Users/erichblume/devpi/+files/root/pypi
-  - /opt/homebrew/var/loki
 
 # Encryption passcommand (reads borg passphrase)
 borgmatic_encryption_passcommand: cat /Users/erichblume/.borg/config.yaml
@@ -41,6 +39,8 @@ borgmatic_keep_yearly: 1000
 
 # PostgreSQL databases to backup (streamed via pg_dump)
 # Password is read from ~/.pgpass (managed by postgresql role)
+# pg_dump_command must be full path since LaunchAgent doesn't have homebrew in PATH
+borgmatic_pg_dump_command: /opt/homebrew/opt/postgresql@18/bin/pg_dump
 borgmatic_postgresql_databases:
   - name: miniflux
     hostname: localhost

ansible/roles/borgmatic/templates/config.yaml.j2

@@ -41,5 +41,8 @@ postgresql_databases:
       hostname: {{ db.hostname | default('localhost') }}
       port: {{ db.port | default(5432) }}
       username: {{ db.username }}
+{% if borgmatic_pg_dump_command is defined %}
+      pg_dump_command: {{ borgmatic_pg_dump_command }}
+{% endif %}
 {% endfor %}
 {% endif %}

ansible/roles/borgmatic_metrics/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+borgmatic_metrics_repo: /Volumes/backups/borg/
+borgmatic_metrics_passcommand: cat /Users/erichblume/.borg/config.yaml
+borgmatic_metrics_dir: /opt/homebrew/var/node_exporter/textfile
+borgmatic_metrics_script: /Users/erichblume/bin/borgmatic-metrics
+borgmatic_metrics_interval: 3600  # seconds between metric collection (hourly)
+borgmatic_metrics_log_dir: /opt/homebrew/var/log

ansible/roles/borgmatic_metrics/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Reload borgmatic-metrics
+  ansible.builtin.shell: |
+    launchctl unload ~/Library/LaunchAgents/mcquack.eblume.borgmatic-metrics.plist 2>/dev/null || true
+    launchctl load ~/Library/LaunchAgents/mcquack.eblume.borgmatic-metrics.plist
+  changed_when: true

ansible/roles/borgmatic_metrics/tasks/main.yml

@@ -0,0 +1,43 @@
+---
+- name: Ensure metrics directory exists
+  ansible.builtin.file:
+    path: "{{ borgmatic_metrics_dir }}"
+    state: directory
+    mode: '0755'
+
+- name: Ensure log directory exists
+  ansible.builtin.file:
+    path: "{{ borgmatic_metrics_log_dir }}"
+    state: directory
+    mode: '0755'
+
+- name: Ensure bin directory exists
+  ansible.builtin.file:
+    path: "{{ borgmatic_metrics_script | dirname }}"
+    state: directory
+    mode: '0755'
+
+- name: Deploy borgmatic-metrics script
+  ansible.builtin.template:
+    src: borgmatic-metrics.sh.j2
+    dest: "{{ borgmatic_metrics_script }}"
+    mode: '0755'
+
+- name: Deploy borgmatic-metrics LaunchAgent plist
+  ansible.builtin.template:
+    src: borgmatic-metrics.plist.j2
+    dest: ~/Library/LaunchAgents/mcquack.eblume.borgmatic-metrics.plist
+    mode: '0644'
+  notify: Reload borgmatic-metrics
+
+- name: Check if borgmatic-metrics LaunchAgent is loaded
+  ansible.builtin.command: launchctl list mcquack.eblume.borgmatic-metrics
+  register: borgmatic_metrics_launchctl_check
+  changed_when: false
+  failed_when: false
+
+- name: Load borgmatic-metrics LaunchAgent if not loaded
+  ansible.builtin.command: launchctl load ~/Library/LaunchAgents/mcquack.eblume.borgmatic-metrics.plist
+  when: borgmatic_metrics_launchctl_check.rc != 0
+  changed_when: true
+  failed_when: false

ansible/roles/borgmatic_metrics/templates/borgmatic-metrics.plist.j2

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- {{ ansible_managed }} -->
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>Label</key>
+	<string>mcquack.eblume.borgmatic-metrics</string>
+	<key>ProgramArguments</key>
+	<array>
+		<string>{{ borgmatic_metrics_script }}</string>
+	</array>
+	<key>StartInterval</key>
+	<integer>{{ borgmatic_metrics_interval }}</integer>
+	<key>RunAtLoad</key>
+	<true/>
+	<key>StandardErrorPath</key>
+	<string>{{ borgmatic_metrics_log_dir }}/mcquack.borgmatic-metrics.err.log</string>
+	<key>StandardOutPath</key>
+	<string>{{ borgmatic_metrics_log_dir }}/mcquack.borgmatic-metrics.out.log</string>
+</dict>
+</plist>

ansible/roles/borgmatic_metrics/templates/borgmatic-metrics.sh.j2

@@ -0,0 +1,163 @@
+#!/bin/bash
+# {{ ansible_managed }}
+# Collects borg backup metrics for node_exporter textfile collector
+
+set -euo pipefail
+
+export BORG_PASSCOMMAND="{{ borgmatic_metrics_passcommand }}"
+BORG_REPO="{{ borgmatic_metrics_repo }}"
+OUTPUT_FILE="{{ borgmatic_metrics_dir }}/borgmatic.prom"
+TEMP_FILE="${OUTPUT_FILE}.tmp"
+
+# Check if borg is available via mise
+BORG_CMD="mise x -- borg"
+
+# Get repository info
+repo_json=$($BORG_CMD info --json "$BORG_REPO" 2>/dev/null) || {
+    echo "Failed to get borg repo info" >&2
+    # Write down metric
+    cat > "$TEMP_FILE" << 'EOF'
+# HELP borgmatic_up Borg backup repository is accessible
+# TYPE borgmatic_up gauge
+borgmatic_up 0
+EOF
+    mv "$TEMP_FILE" "$OUTPUT_FILE"
+    exit 0
+}
+
+# Get archive list
+archives_json=$($BORG_CMD list --json "$BORG_REPO" 2>/dev/null) || {
+    echo "Failed to list borg archives" >&2
+    exit 1
+}
+
+# Extract repository stats
+total_size=$(echo "$repo_json" | jq -r '.cache.stats.total_size')
+total_csize=$(echo "$repo_json" | jq -r '.cache.stats.total_csize')
+unique_size=$(echo "$repo_json" | jq -r '.cache.stats.unique_size')
+unique_csize=$(echo "$repo_json" | jq -r '.cache.stats.unique_csize')
+total_chunks=$(echo "$repo_json" | jq -r '.cache.stats.total_chunks')
+unique_chunks=$(echo "$repo_json" | jq -r '.cache.stats.total_unique_chunks')
+
+# Count archives
+archive_count=$(echo "$archives_json" | jq -r '.archives | length')
+
+# Get last archive info
+last_archive_name=$(echo "$archives_json" | jq -r '.archives[-1].name // empty')
+
+if [ -n "$last_archive_name" ]; then
+    # Get detailed info for the last archive
+    last_archive_json=$($BORG_CMD info --json "${BORG_REPO}::${last_archive_name}" 2>/dev/null) || {
+        echo "Failed to get last archive info" >&2
+        last_archive_json=""
+    }
+
+    if [ -n "$last_archive_json" ]; then
+        last_original_size=$(echo "$last_archive_json" | jq -r '.archives[0].stats.original_size')
+        last_compressed_size=$(echo "$last_archive_json" | jq -r '.archives[0].stats.compressed_size')
+        last_deduplicated_size=$(echo "$last_archive_json" | jq -r '.archives[0].stats.deduplicated_size')
+        last_nfiles=$(echo "$last_archive_json" | jq -r '.archives[0].stats.nfiles')
+        last_start=$(echo "$last_archive_json" | jq -r '.archives[0].start')
+        last_end=$(echo "$last_archive_json" | jq -r '.archives[0].end')
+        last_duration=$(echo "$last_archive_json" | jq -r '.archives[0].duration')
+
+        # Convert timestamp to unix epoch
+        last_timestamp=$(date -j -f "%Y-%m-%dT%H:%M:%S" "${last_start%.*}" "+%s" 2>/dev/null || echo "0")
+    fi
+fi
+
+# Write metrics
+cat > "$TEMP_FILE" << EOF
+# HELP borgmatic_up Borg backup repository is accessible
+# TYPE borgmatic_up gauge
+borgmatic_up 1
+
+# HELP borgmatic_repo_original_size_bytes Total original size of all archives (sum of what each backup contains)
+# TYPE borgmatic_repo_original_size_bytes gauge
+borgmatic_repo_original_size_bytes $total_size
+
+# HELP borgmatic_repo_compressed_size_bytes Total compressed size of all archives
+# TYPE borgmatic_repo_compressed_size_bytes gauge
+borgmatic_repo_compressed_size_bytes $total_csize
+
+# HELP borgmatic_repo_deduplicated_size_bytes Actual disk usage after deduplication (unique data)
+# TYPE borgmatic_repo_deduplicated_size_bytes gauge
+borgmatic_repo_deduplicated_size_bytes $unique_csize
+
+# HELP borgmatic_repo_total_chunks Total number of chunks across all archives
+# TYPE borgmatic_repo_total_chunks gauge
+borgmatic_repo_total_chunks $total_chunks
+
+# HELP borgmatic_repo_unique_chunks Number of unique chunks (after deduplication)
+# TYPE borgmatic_repo_unique_chunks gauge
+borgmatic_repo_unique_chunks $unique_chunks
+
+# HELP borgmatic_archive_count Number of archives in the repository
+# TYPE borgmatic_archive_count gauge
+borgmatic_archive_count $archive_count
+EOF
+
+# Add last archive metrics if available
+if [ -n "${last_original_size:-}" ]; then
+    cat >> "$TEMP_FILE" << EOF
+
+# HELP borgmatic_last_archive_original_size_bytes Original size of the last archive (data being backed up)
+# TYPE borgmatic_last_archive_original_size_bytes gauge
+borgmatic_last_archive_original_size_bytes $last_original_size
+
+# HELP borgmatic_last_archive_compressed_size_bytes Compressed size of the last archive
+# TYPE borgmatic_last_archive_compressed_size_bytes gauge
+borgmatic_last_archive_compressed_size_bytes $last_compressed_size
+
+# HELP borgmatic_last_archive_deduplicated_size_bytes Deduplicated size of last archive (new data added)
+# TYPE borgmatic_last_archive_deduplicated_size_bytes gauge
+borgmatic_last_archive_deduplicated_size_bytes $last_deduplicated_size
+
+# HELP borgmatic_last_archive_files Number of files in the last archive
+# TYPE borgmatic_last_archive_files gauge
+borgmatic_last_archive_files $last_nfiles
+
+# HELP borgmatic_last_archive_timestamp Unix timestamp of the last backup
+# TYPE borgmatic_last_archive_timestamp gauge
+borgmatic_last_archive_timestamp $last_timestamp
+
+# HELP borgmatic_last_archive_duration_seconds Duration of the last backup in seconds
+# TYPE borgmatic_last_archive_duration_seconds gauge
+borgmatic_last_archive_duration_seconds ${last_duration:-0}
+EOF
+
+    # Collect per-source-directory sizes
+    cat >> "$TEMP_FILE" << 'EOF'
+
+# HELP borgmatic_source_size_bytes Size of each backup source directory in bytes
+# TYPE borgmatic_source_size_bytes gauge
+EOF
+
+    # List archive contents and group by source directory
+    $BORG_CMD list "${BORG_REPO}::${last_archive_name}" --format "{size} {path}{NL}" 2>/dev/null | awk '
+    {
+        size = $1
+        path = $2
+        # Map paths to friendly source names
+        if (path ~ /^Users\/[^\/]+\/Pictures/) { source = "Pictures" }
+        else if (path ~ /^Users\/[^\/]+\/Documents/) { source = "Documents" }
+        else if (path ~ /^Users\/[^\/]+\/devpi/) { source = "devpi" }
+        else if (path ~ /^Users\/[^\/]+\/code\/personal\/zk/) { source = "Zettelkasten" }
+        else if (path ~ /^Users\/[^\/]+\/.config\/borgmatic/) { source = "borgmatic_config" }
+        else if (path ~ /^opt\/homebrew\/var\/forgejo/) { source = "Forgejo" }
+        else if (path ~ /^opt\/homebrew\/var\/loki/) { source = "Loki" }
+        else if (path ~ /^borgmatic\/postgresql_databases/) { source = "PostgreSQL" }
+        else if (path ~ /^borgmatic\//) { source = "borgmatic_metadata" }
+        else { source = "other" }
+
+        totals[source] += size
+    }
+    END {
+        for (src in totals) {
+            printf "borgmatic_source_size_bytes{source=\"%s\"} %.0f\n", src, totals[src]
+        }
+    }' >> "$TEMP_FILE"
+fi
+
+# Atomic move
+mv "$TEMP_FILE" "$OUTPUT_FILE"

ansible/roles/grafana/files/dashboards/borgmatic.json

@@ -0,0 +1,781 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": { "color": "red", "index": 0, "text": "DOWN" }
+              },
+              "type": "value"
+            },
+            {
+              "options": {
+                "1": { "color": "green", "index": 1, "text": "UP" }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red", "value": null },
+              { "color": "green", "value": 1 }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "borgmatic_up",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Status",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "blue", "value": null }]
+          },
+          "unit": "decbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 5, "x": 4, "y": 0 },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "borgmatic_repo_deduplicated_size_bytes",
+          "refId": "A"
+        }
+      ],
+      "title": "Disk Usage (Deduplicated)",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "unit": "decbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 5, "x": 9, "y": 0 },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "borgmatic_last_archive_original_size_bytes",
+          "refId": "A"
+        }
+      ],
+      "title": "Data Being Backed Up",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "purple", "value": null }]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 3, "x": 14, "y": 0 },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "borgmatic_archive_count",
+          "refId": "A"
+        }
+      ],
+      "title": "Archives",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 86400 },
+              { "color": "red", "value": 172800 }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 4, "x": 17, "y": 0 },
+      "id": 5,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "time() - borgmatic_last_archive_timestamp",
+          "refId": "A"
+        }
+      ],
+      "title": "Time Since Last Backup",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "decimals": 1,
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "red", "value": null },
+              { "color": "yellow", "value": 2 },
+              { "color": "green", "value": 5 }
+            ]
+          },
+          "unit": "x"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 3, "x": 21, "y": 0 },
+      "id": 6,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "borgmatic_repo_original_size_bytes / borgmatic_repo_deduplicated_size_bytes",
+          "refId": "A"
+        }
+      ],
+      "title": "Dedup Ratio",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "orange", "value": null }]
+          },
+          "unit": "decbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 4, "x": 0, "y": 4 },
+      "id": 7,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "borgmatic_last_archive_deduplicated_size_bytes",
+          "refId": "A"
+        }
+      ],
+      "title": "Last Backup New Data",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "cyan", "value": null }]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 4, "x": 4, "y": 4 },
+      "id": 8,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "borgmatic_last_archive_files",
+          "refId": "A"
+        }
+      ],
+      "title": "Files in Backup",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "yellow", "value": 60 },
+              { "color": "red", "value": 300 }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 4, "x": 8, "y": 4 },
+      "id": 9,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "borgmatic_last_archive_duration_seconds",
+          "refId": "A"
+        }
+      ],
+      "title": "Last Backup Duration",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "semi-dark-blue", "value": null }]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 4, "x": 12, "y": 4 },
+      "id": 10,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "borgmatic_repo_unique_chunks",
+          "refId": "A"
+        }
+      ],
+      "title": "Unique Chunks",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "unit": "decbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 4 },
+      "id": 14,
+      "options": {
+        "displayMode": "basic",
+        "minVizHeight": 10,
+        "minVizWidth": 0,
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": ["lastNotNull"],
+          "fields": "",
+          "values": false
+        },
+        "showUnfilled": true,
+        "valueMode": "color"
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "topk(10, borgmatic_source_size_bytes)",
+          "legendFormat": "{{source}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Backup Size by Source",
+      "description": "Breakdown of backup size by source directory",
+      "type": "bargauge"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "green",
+            "mode": "fixed"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "opacity",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "green", "value": null }]
+          },
+          "unit": "decbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+      "id": 11,
+      "options": {
+        "legend": {
+          "calcs": ["lastNotNull", "min", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "borgmatic_last_archive_original_size_bytes",
+          "legendFormat": "Backup Size (if extracted)",
+          "refId": "A"
+        }
+      ],
+      "title": "Backup Size Over Time",
+      "description": "The original (uncompressed, non-deduplicated) size of each backup - this is how much data would be restored",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "blue",
+            "mode": "fixed"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "opacity",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "blue", "value": null }]
+          },
+          "unit": "decbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+      "id": 12,
+      "options": {
+        "legend": {
+          "calcs": ["lastNotNull", "min", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "borgmatic_repo_deduplicated_size_bytes",
+          "legendFormat": "Repository Size on Disk",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Disk Usage Over Time",
+      "description": "Actual disk space used by the backup repository (after compression and deduplication)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "orange",
+            "mode": "fixed"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "bars",
+            "fillOpacity": 80,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [{ "color": "orange", "value": null }]
+          },
+          "unit": "decbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 },
+      "id": 13,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max", "lastNotNull"],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.0.0",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "expr": "borgmatic_last_archive_deduplicated_size_bytes",
+          "legendFormat": "New Data Added",
+          "refId": "A"
+        }
+      ],
+      "title": "New Data Per Backup",
+      "description": "How much new (deduplicated) data each backup added to the repository",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5m",
+  "schemaVersion": 38,
+  "tags": ["borgmatic", "backup", "borg"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-30d",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Borgmatic Backups",
+  "uid": "borgmatic",
+  "version": 1,
+  "weekStart": ""
+}