diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 607114e..9d912a1 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -4,7 +4,9 @@ "Bash(mcquack --help:*)", "Bash(tea help:*)", "Bash(git add:*)", - "Bash(git commit:*)" + "Bash(git commit:*)", + "WebFetch(domain:devpi.net)", + "WebFetch(domain:docs.devpi.net)" ] } } diff --git a/ansible/playbooks/indri.yml b/ansible/playbooks/indri.yml index d6cf954..12efb5d 100644 --- a/ansible/playbooks/indri.yml +++ b/ansible/playbooks/indri.yml @@ -18,3 +18,7 @@ tags: borgmatic - role: forgejo tags: forgejo + - role: devpi + tags: devpi + - role: devpi_metrics + tags: devpi_metrics diff --git a/ansible/roles/devpi/defaults/main.yml b/ansible/roles/devpi/defaults/main.yml new file mode 100644 index 0000000..0fc0569 --- /dev/null +++ b/ansible/roles/devpi/defaults/main.yml @@ -0,0 +1,7 @@ +--- +devpi_port: 3141 +devpi_serverdir: /Users/erichblume/devpi +devpi_log_dir: /Users/erichblume/Library/Logs +devpi_host: 0.0.0.0 # Listen on all interfaces for Tailscale +devpi_outside_url: https://pypi.tail8d86e.ts.net # URL for Tailscale proxy +devpi_secretfile: /Users/erichblume/devpi/.secret # Persistent auth secret diff --git a/ansible/roles/devpi/handlers/main.yml b/ansible/roles/devpi/handlers/main.yml new file mode 100644 index 0000000..788895e --- /dev/null +++ b/ansible/roles/devpi/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: reload devpi + ansible.builtin.shell: | + launchctl unload ~/Library/LaunchAgents/mcquack.eblume.devpi.plist 2>/dev/null || true + launchctl load ~/Library/LaunchAgents/mcquack.eblume.devpi.plist diff --git a/ansible/roles/devpi/tasks/main.yml b/ansible/roles/devpi/tasks/main.yml new file mode 100644 index 0000000..b35989c --- /dev/null +++ b/ansible/roles/devpi/tasks/main.yml @@ -0,0 +1,54 @@ +--- +# Note: devpi is installed via mise (pipx/uvx), not managed here. +# +# ONE-TIME SETUP (before running ansible): +# +# 1. Add to ~/.config/mise/config.toml on indri: +# +# [tools] +# "pipx:devpi-server" = { version = "latest", uvx = "true", uvx_args = "--with devpi-web" } +# "pipx:devpi-client" = { version = "latest", uvx = "true" } +# +# 2. Install: mise install +# +# 3. Initialize with root password (generate password in 1password): +# mise x -- devpi-init --serverdir {{ devpi_serverdir }} --root-passwd YOUR_PASSWORD +# +# 4. Run ansible to deploy LaunchAgent +# +# 5. Set up Tailscale service (see management log) + +- name: Ensure devpi data directory exists + ansible.builtin.file: + path: "{{ devpi_serverdir }}" + state: directory + mode: '0755' + +- name: Generate devpi secret file if not exists + ansible.builtin.shell: | + openssl rand -hex 32 > "{{ devpi_secretfile }}" + args: + creates: "{{ devpi_secretfile }}" + +- name: Ensure devpi secret file has secure permissions + ansible.builtin.file: + path: "{{ devpi_secretfile }}" + mode: '0600' + +- name: Deploy devpi LaunchAgent plist + ansible.builtin.template: + src: devpi.plist.j2 + dest: ~/Library/LaunchAgents/mcquack.eblume.devpi.plist + mode: '0644' + notify: reload devpi + +- name: Check if devpi LaunchAgent is loaded + ansible.builtin.command: launchctl list mcquack.eblume.devpi + register: launchctl_check + changed_when: false + failed_when: false + +- name: Load devpi LaunchAgent if not loaded + ansible.builtin.command: launchctl load ~/Library/LaunchAgents/mcquack.eblume.devpi.plist + when: launchctl_check.rc != 0 + failed_when: false diff --git a/ansible/roles/devpi/templates/devpi.plist.j2 b/ansible/roles/devpi/templates/devpi.plist.j2 new file mode 100644 index 0000000..b2ed6aa --- /dev/null +++ b/ansible/roles/devpi/templates/devpi.plist.j2 @@ -0,0 +1,39 @@ + + + + + + KeepAlive + + Label + mcquack.eblume.devpi + EnvironmentVariables + + PATH + /opt/homebrew/bin:/usr/bin:/bin + + ProgramArguments + + /opt/homebrew/opt/mise/bin/mise + x + -- + devpi-server + --serverdir + {{ devpi_serverdir }} + --host + {{ devpi_host }} + --port + {{ devpi_port }} + --outside-url + {{ devpi_outside_url }} + --secretfile + {{ devpi_secretfile }} + + RunAtLoad + + StandardErrorPath + {{ devpi_log_dir }}/mcquack.devpi.err.log + StandardOutPath + {{ devpi_log_dir }}/mcquack.devpi.out.log + + diff --git a/ansible/roles/devpi_metrics/defaults/main.yml b/ansible/roles/devpi_metrics/defaults/main.yml new file mode 100644 index 0000000..d1aa18b --- /dev/null +++ b/ansible/roles/devpi_metrics/defaults/main.yml @@ -0,0 +1,6 @@ +--- +devpi_metrics_url: http://localhost:3141/+status +devpi_metrics_dir: /opt/homebrew/var/node_exporter/textfile +devpi_metrics_script: /Users/erichblume/bin/devpi-metrics +devpi_metrics_interval: 60 # seconds between metric collection +devpi_metrics_log_dir: /opt/homebrew/var/log diff --git a/ansible/roles/devpi_metrics/handlers/main.yml b/ansible/roles/devpi_metrics/handlers/main.yml new file mode 100644 index 0000000..c37757b --- /dev/null +++ b/ansible/roles/devpi_metrics/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: reload devpi-metrics + ansible.builtin.shell: | + launchctl unload ~/Library/LaunchAgents/mcquack.eblume.devpi-metrics.plist 2>/dev/null || true + launchctl load ~/Library/LaunchAgents/mcquack.eblume.devpi-metrics.plist diff --git a/ansible/roles/devpi_metrics/meta/main.yml b/ansible/roles/devpi_metrics/meta/main.yml new file mode 100644 index 0000000..355e454 --- /dev/null +++ b/ansible/roles/devpi_metrics/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - role: node_exporter + - role: devpi diff --git a/ansible/roles/devpi_metrics/tasks/main.yml b/ansible/roles/devpi_metrics/tasks/main.yml new file mode 100644 index 0000000..e680641 --- /dev/null +++ b/ansible/roles/devpi_metrics/tasks/main.yml @@ -0,0 +1,36 @@ +--- +- name: Ensure metrics directory exists + ansible.builtin.file: + path: "{{ devpi_metrics_dir }}" + state: directory + mode: '0755' + +- name: Ensure log directory exists + ansible.builtin.file: + path: "{{ devpi_metrics_log_dir }}" + state: directory + mode: '0755' + +- name: Deploy devpi-metrics script + ansible.builtin.template: + src: devpi-metrics.sh.j2 + dest: "{{ devpi_metrics_script }}" + mode: '0755' + +- name: Deploy devpi-metrics LaunchAgent plist + ansible.builtin.template: + src: devpi-metrics.plist.j2 + dest: ~/Library/LaunchAgents/mcquack.eblume.devpi-metrics.plist + mode: '0644' + notify: reload devpi-metrics + +- name: Check if devpi-metrics LaunchAgent is loaded + ansible.builtin.command: launchctl list mcquack.eblume.devpi-metrics + register: launchctl_check + changed_when: false + failed_when: false + +- name: Load devpi-metrics LaunchAgent if not loaded + ansible.builtin.command: launchctl load ~/Library/LaunchAgents/mcquack.eblume.devpi-metrics.plist + when: launchctl_check.rc != 0 + failed_when: false diff --git a/ansible/roles/devpi_metrics/templates/devpi-metrics.plist.j2 b/ansible/roles/devpi_metrics/templates/devpi-metrics.plist.j2 new file mode 100644 index 0000000..a8141df --- /dev/null +++ b/ansible/roles/devpi_metrics/templates/devpi-metrics.plist.j2 @@ -0,0 +1,21 @@ + + + + + + Label + mcquack.eblume.devpi-metrics + ProgramArguments + + {{ devpi_metrics_script }} + + StartInterval + {{ devpi_metrics_interval }} + RunAtLoad + + StandardErrorPath + {{ devpi_metrics_log_dir }}/mcquack.devpi-metrics.err.log + StandardOutPath + {{ devpi_metrics_log_dir }}/mcquack.devpi-metrics.out.log + + diff --git a/ansible/roles/devpi_metrics/templates/devpi-metrics.sh.j2 b/ansible/roles/devpi_metrics/templates/devpi-metrics.sh.j2 new file mode 100644 index 0000000..2a1141d --- /dev/null +++ b/ansible/roles/devpi_metrics/templates/devpi-metrics.sh.j2 @@ -0,0 +1,54 @@ +#!/bin/bash +# {{ ansible_managed }} +# Collects devpi-server metrics for node_exporter textfile collector + +set -euo pipefail + +STATUS_URL="{{ devpi_metrics_url }}" +OUTPUT_FILE="{{ devpi_metrics_dir }}/devpi.prom" +TEMP_FILE="${OUTPUT_FILE}.tmp" + +# Fetch status JSON +status_json=$(curl -s -H "Accept: application/json" "$STATUS_URL" 2>/dev/null) + +if [ -z "$status_json" ] || ! echo "$status_json" | jq -e '.result' >/dev/null 2>&1; then + echo "Failed to fetch devpi status" >&2 + exit 1 +fi + +# Start output file +cat > "$TEMP_FILE" << 'HEADER' +# HELP devpi_up devpi-server is up and responding +# TYPE devpi_up gauge +devpi_up 1 + +HEADER + +# Extract serial number using jq +serial=$(echo "$status_json" | jq -r '.result.serial // empty') +if [ -n "$serial" ]; then + cat >> "$TEMP_FILE" << EOF +# HELP devpi_serial Current changelog serial number +# TYPE devpi_serial gauge +devpi_serial $serial + +EOF +fi + +# Parse metrics array using jq - format is ["name", "type", value] +echo "$status_json" | jq -r '.result.metrics[]? | @json' | while read -r metric_json; do + name=$(echo "$metric_json" | jq -r '.[0]') + type=$(echo "$metric_json" | jq -r '.[1]') + value=$(echo "$metric_json" | jq -r '.[2]') + + # Write metric in Prometheus format + cat >> "$TEMP_FILE" << EOF +# HELP $name devpi metric +# TYPE $name $type +$name $value + +EOF +done + +# Atomic move +mv "$TEMP_FILE" "$OUTPUT_FILE" diff --git a/ansible/roles/grafana/files/dashboards/devpi.json b/ansible/roles/grafana/files/dashboards/devpi.json new file mode 100644 index 0000000..a317380 --- /dev/null +++ b/ansible/roles/grafana/files/dashboards/devpi.json @@ -0,0 +1,438 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 0, "y": 0}, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_up", + "refId": "A" + } + ], + "title": "Devpi Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 100}, + {"color": "red", "value": 1000} + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 4, "y": 0}, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_web_whoosh_index_queue_size", + "refId": "A" + } + ], + "title": "Search Index Queue", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null} + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 8, "y": 0}, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_serial", + "refId": "A" + } + ], + "title": "Changelog Serial", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": {"legend": false, "tooltip": false, "viz": false}, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": {"type": "linear"}, + "showPoints": "never", + "spanNulls": false, + "stacking": {"group": "A", "mode": "none"}, + "thresholdsStyle": {"mode": "off"} + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null} + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 4}, + "id": 4, + "options": { + "legend": {"calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true}, + "tooltip": {"mode": "single", "sort": "none"} + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "rate(devpi_server_storage_cache_hits[5m])", + "legendFormat": "Storage Cache Hits", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "rate(devpi_server_storage_cache_misses[5m])", + "legendFormat": "Storage Cache Misses", + "refId": "B" + } + ], + "title": "Storage Cache Hit/Miss Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": {"legend": false, "tooltip": false, "viz": false}, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": {"type": "linear"}, + "showPoints": "never", + "spanNulls": false, + "stacking": {"group": "A", "mode": "none"}, + "thresholdsStyle": {"mode": "off"} + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null} + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 4}, + "id": 5, + "options": { + "legend": {"calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true}, + "tooltip": {"mode": "single", "sort": "none"} + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "rate(devpi_server_changelog_cache_hits[5m])", + "legendFormat": "Changelog Cache Hits", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "rate(devpi_server_changelog_cache_misses[5m])", + "legendFormat": "Changelog Cache Misses", + "refId": "B" + } + ], + "title": "Changelog Cache Hit/Miss Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": {"legend": false, "tooltip": false, "viz": false}, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": {"type": "linear"}, + "showPoints": "never", + "spanNulls": false, + "stacking": {"group": "A", "mode": "none"}, + "thresholdsStyle": {"mode": "off"} + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null} + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 12}, + "id": 6, + "options": { + "legend": {"calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true}, + "tooltip": {"mode": "single", "sort": "none"} + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_web_whoosh_index_queue_size", + "legendFormat": "Index Queue Size", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_web_whoosh_index_error_queue_size", + "legendFormat": "Index Error Queue Size", + "refId": "B" + } + ], + "title": "Search Index Queue Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null} + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 12}, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_server_storage_cache_size", + "legendFormat": "Storage Cache Size", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_server_changelog_cache_size", + "legendFormat": "Changelog Cache Size", + "refId": "B" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_server_changelog_cache_items", + "legendFormat": "Changelog Cache Items", + "refId": "C" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_server_relpath_cache_size", + "legendFormat": "Relpath Cache Size", + "refId": "D" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_server_relpath_cache_items", + "legendFormat": "Relpath Cache Items", + "refId": "E" + } + ], + "title": "Cache Sizes", + "type": "stat" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "tags": ["devpi", "pypi"], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Devpi PyPI Proxy", + "uid": "devpi", + "version": 1, + "weekStart": "" +} diff --git a/ansible/roles/transmission_metrics/defaults/main.yml b/ansible/roles/transmission_metrics/defaults/main.yml index 189c24d..7423637 100644 --- a/ansible/roles/transmission_metrics/defaults/main.yml +++ b/ansible/roles/transmission_metrics/defaults/main.yml @@ -2,6 +2,6 @@ transmission_rpc_host: 127.0.0.1 transmission_rpc_port: 9091 transmission_metrics_dir: /opt/homebrew/var/node_exporter/textfile -transmission_metrics_script: /opt/homebrew/bin/transmission-metrics +transmission_metrics_script: /Users/erichblume/bin/transmission-metrics transmission_metrics_interval: 60 # seconds between metric collection transmission_log_dir: /opt/homebrew/var/log diff --git a/mise-tasks/indri-services-check b/mise-tasks/indri-services-check index beeb6de..ea794fd 100755 --- a/mise-tasks/indri-services-check +++ b/mise-tasks/indri-services-check @@ -50,6 +50,7 @@ check_service "transmission" "ssh indri 'brew services list | grep transmission check_service "transmission-metrics" "ssh indri 'launchctl list | grep transmission-metrics | grep -v \"^-\"'" check_service "kiwix-serve" "ssh indri 'launchctl list | grep kiwix | grep -v \"^-\"'" check_service "forgejo" "ssh indri 'brew services list | grep forgejo | grep started'" +check_service "devpi" "ssh indri 'launchctl list | grep devpi | grep -v \"^-\"'" echo "" echo "HTTP endpoints (via Tailscale):" @@ -57,6 +58,7 @@ check_http "Prometheus" "http://indri:9090/-/healthy" check_http "Grafana" "http://indri:3000/api/health" check_http "Kiwix" "http://indri:5501/" check_http "Forgejo" "http://indri:3001/" +check_http "Devpi" "http://indri:3141/+api" # Transmission RPC is localhost-only by design, check via SSH check_service "Transmission RPC" "ssh indri 'curl -sf http://127.0.0.1:9091/transmission/rpc'" # Check that transmission metrics are being collected