diff --git a/ansible/playbooks/indri.yml b/ansible/playbooks/indri.yml index fbad4c7..12efb5d 100644 --- a/ansible/playbooks/indri.yml +++ b/ansible/playbooks/indri.yml @@ -20,3 +20,5 @@ tags: forgejo - role: devpi tags: devpi + - role: devpi_metrics + tags: devpi_metrics diff --git a/ansible/roles/devpi_metrics/defaults/main.yml b/ansible/roles/devpi_metrics/defaults/main.yml new file mode 100644 index 0000000..81ce7bf --- /dev/null +++ b/ansible/roles/devpi_metrics/defaults/main.yml @@ -0,0 +1,6 @@ +--- +devpi_metrics_url: http://localhost:3141/+status +devpi_metrics_dir: /opt/homebrew/var/node_exporter/textfile +devpi_metrics_script: /opt/homebrew/bin/devpi-metrics +devpi_metrics_interval: 60 # seconds between metric collection +devpi_metrics_log_dir: /opt/homebrew/var/log diff --git a/ansible/roles/devpi_metrics/handlers/main.yml b/ansible/roles/devpi_metrics/handlers/main.yml new file mode 100644 index 0000000..c37757b --- /dev/null +++ b/ansible/roles/devpi_metrics/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: reload devpi-metrics + ansible.builtin.shell: | + launchctl unload ~/Library/LaunchAgents/mcquack.eblume.devpi-metrics.plist 2>/dev/null || true + launchctl load ~/Library/LaunchAgents/mcquack.eblume.devpi-metrics.plist diff --git a/ansible/roles/devpi_metrics/meta/main.yml b/ansible/roles/devpi_metrics/meta/main.yml new file mode 100644 index 0000000..355e454 --- /dev/null +++ b/ansible/roles/devpi_metrics/meta/main.yml @@ -0,0 +1,4 @@ +--- +dependencies: + - role: node_exporter + - role: devpi diff --git a/ansible/roles/devpi_metrics/tasks/main.yml b/ansible/roles/devpi_metrics/tasks/main.yml new file mode 100644 index 0000000..e680641 --- /dev/null +++ b/ansible/roles/devpi_metrics/tasks/main.yml @@ -0,0 +1,36 @@ +--- +- name: Ensure metrics directory exists + ansible.builtin.file: + path: "{{ devpi_metrics_dir }}" + state: directory + mode: '0755' + +- name: Ensure log directory exists + ansible.builtin.file: + path: "{{ devpi_metrics_log_dir }}" + state: directory + mode: '0755' + +- name: Deploy devpi-metrics script + ansible.builtin.template: + src: devpi-metrics.sh.j2 + dest: "{{ devpi_metrics_script }}" + mode: '0755' + +- name: Deploy devpi-metrics LaunchAgent plist + ansible.builtin.template: + src: devpi-metrics.plist.j2 + dest: ~/Library/LaunchAgents/mcquack.eblume.devpi-metrics.plist + mode: '0644' + notify: reload devpi-metrics + +- name: Check if devpi-metrics LaunchAgent is loaded + ansible.builtin.command: launchctl list mcquack.eblume.devpi-metrics + register: launchctl_check + changed_when: false + failed_when: false + +- name: Load devpi-metrics LaunchAgent if not loaded + ansible.builtin.command: launchctl load ~/Library/LaunchAgents/mcquack.eblume.devpi-metrics.plist + when: launchctl_check.rc != 0 + failed_when: false diff --git a/ansible/roles/devpi_metrics/templates/devpi-metrics.plist.j2 b/ansible/roles/devpi_metrics/templates/devpi-metrics.plist.j2 new file mode 100644 index 0000000..a8141df --- /dev/null +++ b/ansible/roles/devpi_metrics/templates/devpi-metrics.plist.j2 @@ -0,0 +1,21 @@ + + + + + + Label + mcquack.eblume.devpi-metrics + ProgramArguments + + {{ devpi_metrics_script }} + + StartInterval + {{ devpi_metrics_interval }} + RunAtLoad + + StandardErrorPath + {{ devpi_metrics_log_dir }}/mcquack.devpi-metrics.err.log + StandardOutPath + {{ devpi_metrics_log_dir }}/mcquack.devpi-metrics.out.log + + diff --git a/ansible/roles/devpi_metrics/templates/devpi-metrics.sh.j2 b/ansible/roles/devpi_metrics/templates/devpi-metrics.sh.j2 new file mode 100644 index 0000000..2a1141d --- /dev/null +++ b/ansible/roles/devpi_metrics/templates/devpi-metrics.sh.j2 @@ -0,0 +1,54 @@ +#!/bin/bash +# {{ ansible_managed }} +# Collects devpi-server metrics for node_exporter textfile collector + +set -euo pipefail + +STATUS_URL="{{ devpi_metrics_url }}" +OUTPUT_FILE="{{ devpi_metrics_dir }}/devpi.prom" +TEMP_FILE="${OUTPUT_FILE}.tmp" + +# Fetch status JSON +status_json=$(curl -s -H "Accept: application/json" "$STATUS_URL" 2>/dev/null) + +if [ -z "$status_json" ] || ! echo "$status_json" | jq -e '.result' >/dev/null 2>&1; then + echo "Failed to fetch devpi status" >&2 + exit 1 +fi + +# Start output file +cat > "$TEMP_FILE" << 'HEADER' +# HELP devpi_up devpi-server is up and responding +# TYPE devpi_up gauge +devpi_up 1 + +HEADER + +# Extract serial number using jq +serial=$(echo "$status_json" | jq -r '.result.serial // empty') +if [ -n "$serial" ]; then + cat >> "$TEMP_FILE" << EOF +# HELP devpi_serial Current changelog serial number +# TYPE devpi_serial gauge +devpi_serial $serial + +EOF +fi + +# Parse metrics array using jq - format is ["name", "type", value] +echo "$status_json" | jq -r '.result.metrics[]? | @json' | while read -r metric_json; do + name=$(echo "$metric_json" | jq -r '.[0]') + type=$(echo "$metric_json" | jq -r '.[1]') + value=$(echo "$metric_json" | jq -r '.[2]') + + # Write metric in Prometheus format + cat >> "$TEMP_FILE" << EOF +# HELP $name devpi metric +# TYPE $name $type +$name $value + +EOF +done + +# Atomic move +mv "$TEMP_FILE" "$OUTPUT_FILE" diff --git a/ansible/roles/grafana/files/dashboards/devpi.json b/ansible/roles/grafana/files/dashboards/devpi.json new file mode 100644 index 0000000..a317380 --- /dev/null +++ b/ansible/roles/grafana/files/dashboards/devpi.json @@ -0,0 +1,438 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 0, "y": 0}, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_up", + "refId": "A" + } + ], + "title": "Devpi Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 100}, + {"color": "red", "value": 1000} + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 4, "y": 0}, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_web_whoosh_index_queue_size", + "refId": "A" + } + ], + "title": "Search Index Queue", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null} + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": {"h": 4, "w": 4, "x": 8, "y": 0}, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_serial", + "refId": "A" + } + ], + "title": "Changelog Serial", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": {"legend": false, "tooltip": false, "viz": false}, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": {"type": "linear"}, + "showPoints": "never", + "spanNulls": false, + "stacking": {"group": "A", "mode": "none"}, + "thresholdsStyle": {"mode": "off"} + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null} + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 4}, + "id": 4, + "options": { + "legend": {"calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true}, + "tooltip": {"mode": "single", "sort": "none"} + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "rate(devpi_server_storage_cache_hits[5m])", + "legendFormat": "Storage Cache Hits", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "rate(devpi_server_storage_cache_misses[5m])", + "legendFormat": "Storage Cache Misses", + "refId": "B" + } + ], + "title": "Storage Cache Hit/Miss Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": {"legend": false, "tooltip": false, "viz": false}, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": {"type": "linear"}, + "showPoints": "never", + "spanNulls": false, + "stacking": {"group": "A", "mode": "none"}, + "thresholdsStyle": {"mode": "off"} + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null} + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 4}, + "id": 5, + "options": { + "legend": {"calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true}, + "tooltip": {"mode": "single", "sort": "none"} + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "rate(devpi_server_changelog_cache_hits[5m])", + "legendFormat": "Changelog Cache Hits", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "rate(devpi_server_changelog_cache_misses[5m])", + "legendFormat": "Changelog Cache Misses", + "refId": "B" + } + ], + "title": "Changelog Cache Hit/Miss Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": {"legend": false, "tooltip": false, "viz": false}, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": {"type": "linear"}, + "showPoints": "never", + "spanNulls": false, + "stacking": {"group": "A", "mode": "none"}, + "thresholdsStyle": {"mode": "off"} + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null} + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 12}, + "id": 6, + "options": { + "legend": {"calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true}, + "tooltip": {"mode": "single", "sort": "none"} + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_web_whoosh_index_queue_size", + "legendFormat": "Index Queue Size", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_web_whoosh_index_error_queue_size", + "legendFormat": "Index Error Queue Size", + "refId": "B" + } + ], + "title": "Search Index Queue Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null} + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 12}, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.0.0", + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_server_storage_cache_size", + "legendFormat": "Storage Cache Size", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_server_changelog_cache_size", + "legendFormat": "Changelog Cache Size", + "refId": "B" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_server_changelog_cache_items", + "legendFormat": "Changelog Cache Items", + "refId": "C" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_server_relpath_cache_size", + "legendFormat": "Relpath Cache Size", + "refId": "D" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "devpi_server_relpath_cache_items", + "legendFormat": "Relpath Cache Items", + "refId": "E" + } + ], + "title": "Cache Sizes", + "type": "stat" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "tags": ["devpi", "pypi"], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Devpi PyPI Proxy", + "uid": "devpi", + "version": 1, + "weekStart": "" +}