Add devpi PyPI caching proxy role for indri (#9)

## Summary
- Add ansible role for devpi-server as a transparent PyPI caching proxy
- LaunchAgent with KeepAlive runs via `mise x -- devpi-server`
- Listens on port 3141, data stored in `~/devpi`
- Health checks added to `indri-services-check` script

## Manual Setup Required (on indri, before provisioning)
1. Add to `~/.config/mise/config.toml`:
   ```toml
   [tools]
   "pipx:devpi-server" = "latest"
   "pipx:devpi-web" = "latest"
   "pipx:devpi-client" = "latest"
   ```
2. Run `mise install`
3. Initialize: `mise x -- devpi-init --serverdir ~/devpi`

## Post-Provisioning
- Set up Tailscale service `pypi` on port 443 → 3141
- Configure client pip.conf with index-url

## Test plan
- [x] Ansible syntax check passes
- [x] Dry-run: `mise run provision-indri -- --check --diff`
- [x] Apply: `mise run provision-indri`
- [x] Health check: `mise run indri-services-check`

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Reviewed-on: https://forge.tail8d86e.ts.net/eblume/blumeops/pulls/9
This commit is contained in:
Erich Blume 2026-01-15 08:31:09 -08:00
commit d8a0ef6482
15 changed files with 679 additions and 2 deletions

View file

@ -0,0 +1,7 @@
---
devpi_port: 3141
devpi_serverdir: /Users/erichblume/devpi
devpi_log_dir: /Users/erichblume/Library/Logs
devpi_host: 0.0.0.0 # Listen on all interfaces for Tailscale
devpi_outside_url: https://pypi.tail8d86e.ts.net # URL for Tailscale proxy
devpi_secretfile: /Users/erichblume/devpi/.secret # Persistent auth secret

View file

@ -0,0 +1,5 @@
---
- name: reload devpi
ansible.builtin.shell: |
launchctl unload ~/Library/LaunchAgents/mcquack.eblume.devpi.plist 2>/dev/null || true
launchctl load ~/Library/LaunchAgents/mcquack.eblume.devpi.plist

View file

@ -0,0 +1,54 @@
---
# Note: devpi is installed via mise (pipx/uvx), not managed here.
#
# ONE-TIME SETUP (before running ansible):
#
# 1. Add to ~/.config/mise/config.toml on indri:
#
# [tools]
# "pipx:devpi-server" = { version = "latest", uvx = "true", uvx_args = "--with devpi-web" }
# "pipx:devpi-client" = { version = "latest", uvx = "true" }
#
# 2. Install: mise install
#
# 3. Initialize with root password (generate password in 1password):
# mise x -- devpi-init --serverdir {{ devpi_serverdir }} --root-passwd YOUR_PASSWORD
#
# 4. Run ansible to deploy LaunchAgent
#
# 5. Set up Tailscale service (see management log)
- name: Ensure devpi data directory exists
ansible.builtin.file:
path: "{{ devpi_serverdir }}"
state: directory
mode: '0755'
- name: Generate devpi secret file if not exists
ansible.builtin.shell: |
openssl rand -hex 32 > "{{ devpi_secretfile }}"
args:
creates: "{{ devpi_secretfile }}"
- name: Ensure devpi secret file has secure permissions
ansible.builtin.file:
path: "{{ devpi_secretfile }}"
mode: '0600'
- name: Deploy devpi LaunchAgent plist
ansible.builtin.template:
src: devpi.plist.j2
dest: ~/Library/LaunchAgents/mcquack.eblume.devpi.plist
mode: '0644'
notify: reload devpi
- name: Check if devpi LaunchAgent is loaded
ansible.builtin.command: launchctl list mcquack.eblume.devpi
register: launchctl_check
changed_when: false
failed_when: false
- name: Load devpi LaunchAgent if not loaded
ansible.builtin.command: launchctl load ~/Library/LaunchAgents/mcquack.eblume.devpi.plist
when: launchctl_check.rc != 0
failed_when: false

View file

@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- {{ ansible_managed }} -->
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>KeepAlive</key>
<true/>
<key>Label</key>
<string>mcquack.eblume.devpi</string>
<key>EnvironmentVariables</key>
<dict>
<key>PATH</key>
<string>/opt/homebrew/bin:/usr/bin:/bin</string>
</dict>
<key>ProgramArguments</key>
<array>
<string>/opt/homebrew/opt/mise/bin/mise</string>
<string>x</string>
<string>--</string>
<string>devpi-server</string>
<string>--serverdir</string>
<string>{{ devpi_serverdir }}</string>
<string>--host</string>
<string>{{ devpi_host }}</string>
<string>--port</string>
<string>{{ devpi_port }}</string>
<string>--outside-url</string>
<string>{{ devpi_outside_url }}</string>
<string>--secretfile</string>
<string>{{ devpi_secretfile }}</string>
</array>
<key>RunAtLoad</key>
<true/>
<key>StandardErrorPath</key>
<string>{{ devpi_log_dir }}/mcquack.devpi.err.log</string>
<key>StandardOutPath</key>
<string>{{ devpi_log_dir }}/mcquack.devpi.out.log</string>
</dict>
</plist>

View file

@ -0,0 +1,6 @@
---
devpi_metrics_url: http://localhost:3141/+status
devpi_metrics_dir: /opt/homebrew/var/node_exporter/textfile
devpi_metrics_script: /Users/erichblume/bin/devpi-metrics
devpi_metrics_interval: 60 # seconds between metric collection
devpi_metrics_log_dir: /opt/homebrew/var/log

View file

@ -0,0 +1,5 @@
---
- name: reload devpi-metrics
ansible.builtin.shell: |
launchctl unload ~/Library/LaunchAgents/mcquack.eblume.devpi-metrics.plist 2>/dev/null || true
launchctl load ~/Library/LaunchAgents/mcquack.eblume.devpi-metrics.plist

View file

@ -0,0 +1,4 @@
---
dependencies:
- role: node_exporter
- role: devpi

View file

@ -0,0 +1,36 @@
---
- name: Ensure metrics directory exists
ansible.builtin.file:
path: "{{ devpi_metrics_dir }}"
state: directory
mode: '0755'
- name: Ensure log directory exists
ansible.builtin.file:
path: "{{ devpi_metrics_log_dir }}"
state: directory
mode: '0755'
- name: Deploy devpi-metrics script
ansible.builtin.template:
src: devpi-metrics.sh.j2
dest: "{{ devpi_metrics_script }}"
mode: '0755'
- name: Deploy devpi-metrics LaunchAgent plist
ansible.builtin.template:
src: devpi-metrics.plist.j2
dest: ~/Library/LaunchAgents/mcquack.eblume.devpi-metrics.plist
mode: '0644'
notify: reload devpi-metrics
- name: Check if devpi-metrics LaunchAgent is loaded
ansible.builtin.command: launchctl list mcquack.eblume.devpi-metrics
register: launchctl_check
changed_when: false
failed_when: false
- name: Load devpi-metrics LaunchAgent if not loaded
ansible.builtin.command: launchctl load ~/Library/LaunchAgents/mcquack.eblume.devpi-metrics.plist
when: launchctl_check.rc != 0
failed_when: false

View file

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- {{ ansible_managed }} -->
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>mcquack.eblume.devpi-metrics</string>
<key>ProgramArguments</key>
<array>
<string>{{ devpi_metrics_script }}</string>
</array>
<key>StartInterval</key>
<integer>{{ devpi_metrics_interval }}</integer>
<key>RunAtLoad</key>
<true/>
<key>StandardErrorPath</key>
<string>{{ devpi_metrics_log_dir }}/mcquack.devpi-metrics.err.log</string>
<key>StandardOutPath</key>
<string>{{ devpi_metrics_log_dir }}/mcquack.devpi-metrics.out.log</string>
</dict>
</plist>

View file

@ -0,0 +1,54 @@
#!/bin/bash
# {{ ansible_managed }}
# Collects devpi-server metrics for node_exporter textfile collector
set -euo pipefail
STATUS_URL="{{ devpi_metrics_url }}"
OUTPUT_FILE="{{ devpi_metrics_dir }}/devpi.prom"
TEMP_FILE="${OUTPUT_FILE}.tmp"
# Fetch status JSON
status_json=$(curl -s -H "Accept: application/json" "$STATUS_URL" 2>/dev/null)
if [ -z "$status_json" ] || ! echo "$status_json" | jq -e '.result' >/dev/null 2>&1; then
echo "Failed to fetch devpi status" >&2
exit 1
fi
# Start output file
cat > "$TEMP_FILE" << 'HEADER'
# HELP devpi_up devpi-server is up and responding
# TYPE devpi_up gauge
devpi_up 1
HEADER
# Extract serial number using jq
serial=$(echo "$status_json" | jq -r '.result.serial // empty')
if [ -n "$serial" ]; then
cat >> "$TEMP_FILE" << EOF
# HELP devpi_serial Current changelog serial number
# TYPE devpi_serial gauge
devpi_serial $serial
EOF
fi
# Parse metrics array using jq - format is ["name", "type", value]
echo "$status_json" | jq -r '.result.metrics[]? | @json' | while read -r metric_json; do
name=$(echo "$metric_json" | jq -r '.[0]')
type=$(echo "$metric_json" | jq -r '.[1]')
value=$(echo "$metric_json" | jq -r '.[2]')
# Write metric in Prometheus format
cat >> "$TEMP_FILE" << EOF
# HELP $name devpi metric
# TYPE $name $type
$name $value
EOF
done
# Atomic move
mv "$TEMP_FILE" "$OUTPUT_FILE"

View file

@ -0,0 +1,438 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{"color": "red", "value": null},
{"color": "green", "value": 1}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 0},
"id": 1,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "10.0.0",
"targets": [
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"expr": "devpi_up",
"refId": "A"
}
],
"title": "Devpi Status",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{"color": "green", "value": null},
{"color": "yellow", "value": 100},
{"color": "red", "value": 1000}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 0},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "10.0.0",
"targets": [
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"expr": "devpi_web_whoosh_index_queue_size",
"refId": "A"
}
],
"title": "Search Index Queue",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{"color": "green", "value": null}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {"h": 4, "w": 4, "x": 8, "y": 0},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "10.0.0",
"targets": [
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"expr": "devpi_serial",
"refId": "A"
}
],
"title": "Changelog Serial",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {"legend": false, "tooltip": false, "viz": false},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {"type": "linear"},
"showPoints": "never",
"spanNulls": false,
"stacking": {"group": "A", "mode": "none"},
"thresholdsStyle": {"mode": "off"}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{"color": "green", "value": null}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 4},
"id": 4,
"options": {
"legend": {"calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true},
"tooltip": {"mode": "single", "sort": "none"}
},
"pluginVersion": "10.0.0",
"targets": [
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"expr": "rate(devpi_server_storage_cache_hits[5m])",
"legendFormat": "Storage Cache Hits",
"refId": "A"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"expr": "rate(devpi_server_storage_cache_misses[5m])",
"legendFormat": "Storage Cache Misses",
"refId": "B"
}
],
"title": "Storage Cache Hit/Miss Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {"legend": false, "tooltip": false, "viz": false},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {"type": "linear"},
"showPoints": "never",
"spanNulls": false,
"stacking": {"group": "A", "mode": "none"},
"thresholdsStyle": {"mode": "off"}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{"color": "green", "value": null}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 4},
"id": 5,
"options": {
"legend": {"calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true},
"tooltip": {"mode": "single", "sort": "none"}
},
"pluginVersion": "10.0.0",
"targets": [
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"expr": "rate(devpi_server_changelog_cache_hits[5m])",
"legendFormat": "Changelog Cache Hits",
"refId": "A"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"expr": "rate(devpi_server_changelog_cache_misses[5m])",
"legendFormat": "Changelog Cache Misses",
"refId": "B"
}
],
"title": "Changelog Cache Hit/Miss Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {"legend": false, "tooltip": false, "viz": false},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {"type": "linear"},
"showPoints": "never",
"spanNulls": false,
"stacking": {"group": "A", "mode": "none"},
"thresholdsStyle": {"mode": "off"}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{"color": "green", "value": null}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 12},
"id": 6,
"options": {
"legend": {"calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true},
"tooltip": {"mode": "single", "sort": "none"}
},
"pluginVersion": "10.0.0",
"targets": [
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"expr": "devpi_web_whoosh_index_queue_size",
"legendFormat": "Index Queue Size",
"refId": "A"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"expr": "devpi_web_whoosh_index_error_queue_size",
"legendFormat": "Index Error Queue Size",
"refId": "B"
}
],
"title": "Search Index Queue Over Time",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{"color": "green", "value": null}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 12},
"id": 7,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "10.0.0",
"targets": [
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"expr": "devpi_server_storage_cache_size",
"legendFormat": "Storage Cache Size",
"refId": "A"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"expr": "devpi_server_changelog_cache_size",
"legendFormat": "Changelog Cache Size",
"refId": "B"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"expr": "devpi_server_changelog_cache_items",
"legendFormat": "Changelog Cache Items",
"refId": "C"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"expr": "devpi_server_relpath_cache_size",
"legendFormat": "Relpath Cache Size",
"refId": "D"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"expr": "devpi_server_relpath_cache_items",
"legendFormat": "Relpath Cache Items",
"refId": "E"
}
],
"title": "Cache Sizes",
"type": "stat"
}
],
"refresh": "30s",
"schemaVersion": 38,
"tags": ["devpi", "pypi"],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Devpi PyPI Proxy",
"uid": "devpi",
"version": 1,
"weekStart": ""
}

View file

@ -2,6 +2,6 @@
transmission_rpc_host: 127.0.0.1
transmission_rpc_port: 9091
transmission_metrics_dir: /opt/homebrew/var/node_exporter/textfile
transmission_metrics_script: /opt/homebrew/bin/transmission-metrics
transmission_metrics_script: /Users/erichblume/bin/transmission-metrics
transmission_metrics_interval: 60 # seconds between metric collection
transmission_log_dir: /opt/homebrew/var/log