From 14ca0160ba5f76ab8ad348f64f68c75fb6ec3659 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Wed, 29 Apr 2026 13:38:36 -0700 Subject: [PATCH] Migrate devpi from minikube to indri (launchd) (#341) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Devpi was crash-looping under memory pressure on the minikube StatefulSet, breaking the Python toolchain across the repo (`mise run docs-mikado`, `prek`, every `uv pip install`). It moves to indri as a native LaunchAgent. ## What changed - **New ansible role** `ansible/roles/devpi/`: installs `devpi-server` + `devpi-web` into a uv-managed venv, initializes the server-dir on first run via 1Password root password, runs as a LaunchAgent (`mcquack.eblume.devpi`) bound to `127.0.0.1:3141`. Bootstraps from upstream PyPI (so devpi can install itself on a fresh box). - **Caddy**: `pypi.ops.eblu.me` now proxies to `http://localhost:3141`. - **Playbook**: `indri.yml` gains pre_tasks for the root password and the new role. - **service-versions.yaml**: devpi flipped from `type: argocd` to `type: ansible`. - **ArgoCD**: removed `apps/devpi.yaml` and `manifests/devpi/`. The in-cluster Application, namespace, and PVC have been deleted. - **Docs**: new how-to `docs/how-to/operations/devpi-on-indri.md`; `restart-indri.md` lists devpi in the LaunchAgent stop list. ## Already deployed (live on indri) - Service running: `launchctl list mcquack.eblume.devpi` → PID 53888 - `curl https://pypi.ops.eblu.me/+api` returns 200 ✅ - `mise run docs-mikado` works again ✅ - 1.0G of cached PyPI data was migrated from the PVC to `~erichblume/devpi/server-dir/` - Minikube namespace and PVC fully reclaimed ## Test plan - [ ] `mise run services-check` (after merge) - [ ] CI workflows that use devpi succeed - [ ] No regressions in tools that depend on `pypi.ops.eblu.me` (prek, uv-script tasks, dagger pipelines) ## Context This is the C1 prelude to a planned C2 chain (`mikado/retire-minikube-indri`) to retire minikube on indri entirely. Doing devpi as a standalone C1 was the right call because (a) it was urgent — it was breaking the toolchain — and (b) it shakes out the migration recipe before we commit to a multi-leaf chain. Reviewed-on: https://forge.eblu.me/eblume/blumeops/pulls/341 --- ansible/playbooks/indri.yml | 19 +++++ ansible/roles/caddy/defaults/main.yml | 2 +- ansible/roles/devpi/defaults/main.yml | 21 ++++++ ansible/roles/devpi/handlers/main.yml | 6 ++ ansible/roles/devpi/tasks/main.yml | 71 ++++++++++++++++++ ansible/roles/devpi/templates/devpi.plist.j2 | 34 +++++++++ argocd/apps/devpi.yaml | 29 -------- argocd/manifests/alloy-k8s/config.alloy | 4 +- argocd/manifests/devpi/README.md | 72 ------------------ argocd/manifests/devpi/external-secret.yaml | 25 ------- argocd/manifests/devpi/ingress-tailscale.yaml | 25 ------- argocd/manifests/devpi/kustomization.yaml | 14 ---- argocd/manifests/devpi/service.yaml | 13 ---- argocd/manifests/devpi/statefulset.yaml | 64 ---------------- .../migrate-devpi-to-indri.infra.md | 1 + docs/how-to/operations/devpi-on-indri.md | 74 +++++++++++++++++++ .../operations/rebuild-minikube-cluster.md | 20 +---- docs/how-to/operations/restart-indri.md | 1 + docs/reference/infrastructure/tailscale.md | 2 +- docs/reference/services/devpi.md | 34 +++++---- docs/reference/storage/backups.md | 2 +- pulumi/tailscale/__main__.py | 2 +- pulumi/tailscale/policy.hujson | 9 +-- service-versions.yaml | 5 +- 24 files changed, 260 insertions(+), 289 deletions(-) create mode 100644 ansible/roles/devpi/defaults/main.yml create mode 100644 ansible/roles/devpi/handlers/main.yml create mode 100644 ansible/roles/devpi/tasks/main.yml create mode 100644 ansible/roles/devpi/templates/devpi.plist.j2 delete mode 100644 argocd/apps/devpi.yaml delete mode 100644 argocd/manifests/devpi/README.md delete mode 100644 argocd/manifests/devpi/external-secret.yaml delete mode 100644 argocd/manifests/devpi/ingress-tailscale.yaml delete mode 100644 argocd/manifests/devpi/kustomization.yaml delete mode 100644 argocd/manifests/devpi/service.yaml delete mode 100644 argocd/manifests/devpi/statefulset.yaml create mode 100644 docs/changelog.d/migrate-devpi-to-indri.infra.md create mode 100644 docs/how-to/operations/devpi-on-indri.md diff --git a/ansible/playbooks/indri.yml b/ansible/playbooks/indri.yml index ce6a930..fa87b36 100644 --- a/ansible/playbooks/indri.yml +++ b/ansible/playbooks/indri.yml @@ -212,6 +212,23 @@ no_log: true tags: [forgejo_metrics] + # Devpi root password (PyPI mirror admin) + - name: Fetch devpi root password + ansible.builtin.command: + cmd: op read "op://vg6xf6vvfmoh5hqjjhlhbeoaie/kyhzfifryqnuk7jeyibmmjvxxm/add more/root password" + delegate_to: localhost + register: _devpi_root_password + changed_when: false + no_log: true + check_mode: false + tags: [devpi] + + - name: Set devpi root password fact + ansible.builtin.set_fact: + devpi_root_password: "{{ _devpi_root_password.stdout }}" + no_log: true + tags: [devpi] + roles: - role: alloy tags: alloy @@ -227,6 +244,8 @@ tags: zot - role: zot_metrics tags: zot_metrics + - role: devpi + tags: devpi - role: minikube tags: minikube - role: minikube_metrics diff --git a/ansible/roles/caddy/defaults/main.yml b/ansible/roles/caddy/defaults/main.yml index ebb210b..80993ee 100644 --- a/ansible/roles/caddy/defaults/main.yml +++ b/ansible/roles/caddy/defaults/main.yml @@ -51,7 +51,7 @@ caddy_services: backend: "https://feed.tail8d86e.ts.net" - name: devpi host: "pypi.{{ caddy_domain }}" - backend: "https://pypi.tail8d86e.ts.net" + backend: "http://localhost:3141" - name: kiwix host: "kiwix.{{ caddy_domain }}" backend: "https://kiwix.tail8d86e.ts.net" diff --git a/ansible/roles/devpi/defaults/main.yml b/ansible/roles/devpi/defaults/main.yml new file mode 100644 index 0000000..6d52b9b --- /dev/null +++ b/ansible/roles/devpi/defaults/main.yml @@ -0,0 +1,21 @@ +--- +# devpi PyPI caching mirror (native launchd, replaces minikube StatefulSet) + +devpi_home: /Users/erichblume/devpi +devpi_venv: "{{ devpi_home }}/venv" +devpi_server_dir: "{{ devpi_home }}/server-dir" +devpi_binary: "{{ devpi_venv }}/bin/devpi-server" +devpi_init_binary: "{{ devpi_venv }}/bin/devpi-init" + +devpi_python_version: "3.12" +devpi_server_version: "6.19.3" +devpi_web_version: "5.0.2" + +devpi_host: 127.0.0.1 +devpi_port: 3141 +devpi_outside_url: "https://pypi.ops.eblu.me" + +devpi_log_dir: /Users/erichblume/Library/Logs + +# uv binary on indri — mise shim so version bumps via `mise upgrade uv` flow through transparently +devpi_uv_binary: /Users/erichblume/.local/share/mise/shims/uv diff --git a/ansible/roles/devpi/handlers/main.yml b/ansible/roles/devpi/handlers/main.yml new file mode 100644 index 0000000..2765850 --- /dev/null +++ b/ansible/roles/devpi/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: Restart devpi + ansible.builtin.shell: | + launchctl unload ~/Library/LaunchAgents/mcquack.eblume.devpi.plist 2>/dev/null || true + launchctl load ~/Library/LaunchAgents/mcquack.eblume.devpi.plist + changed_when: true diff --git a/ansible/roles/devpi/tasks/main.yml b/ansible/roles/devpi/tasks/main.yml new file mode 100644 index 0000000..985ca46 --- /dev/null +++ b/ansible/roles/devpi/tasks/main.yml @@ -0,0 +1,71 @@ +--- +# devpi role — devpi-server in a uv-managed venv, run via LaunchAgent. +# Replaces the prior minikube StatefulSet; see [[devpi-on-indri]]. +# +# The root password is fetched in the indri.yml playbook pre_tasks and +# exposed as `devpi_root_password`. + +- name: Ensure devpi home exists + ansible.builtin.file: + path: "{{ devpi_home }}" + state: directory + mode: '0755' + +- name: Ensure devpi server-dir exists + ansible.builtin.file: + path: "{{ devpi_server_dir }}" + state: directory + mode: '0700' + +- name: Create devpi venv if missing + ansible.builtin.command: + cmd: "{{ devpi_uv_binary }} venv --python {{ devpi_python_version }} {{ devpi_venv }}" + creates: "{{ devpi_venv }}/bin/python" + +- name: Install devpi-server and devpi-web into venv + # Always bootstrap from upstream PyPI — devpi is the index it would otherwise resolve through, + # and that's a circular dependency (devpi cannot install itself from itself). + ansible.builtin.command: + cmd: >- + {{ devpi_uv_binary }} pip install + --python {{ devpi_venv }}/bin/python + --index-url https://pypi.org/simple/ + devpi-server=={{ devpi_server_version }} + devpi-web=={{ devpi_web_version }} + register: devpi_pip_install + changed_when: "'Installed' in devpi_pip_install.stdout or 'Uninstalled' in devpi_pip_install.stdout" + notify: Restart devpi + +- name: Check if devpi server-dir is initialized + ansible.builtin.stat: + path: "{{ devpi_server_dir }}/.serverversion" + register: devpi_serverversion + +- name: Initialize devpi server-dir + ansible.builtin.command: + cmd: >- + {{ devpi_init_binary }} + --serverdir {{ devpi_server_dir }} + --root-passwd {{ devpi_root_password }} + when: not devpi_serverversion.stat.exists + changed_when: true + no_log: true + +- name: Deploy devpi LaunchAgent plist + ansible.builtin.template: + src: devpi.plist.j2 + dest: ~/Library/LaunchAgents/mcquack.eblume.devpi.plist + mode: '0644' + notify: Restart devpi + +- name: Check if devpi LaunchAgent is loaded + ansible.builtin.command: launchctl list mcquack.eblume.devpi + register: devpi_launchctl_check + changed_when: false + failed_when: false + +- name: Load devpi LaunchAgent if not loaded + ansible.builtin.command: launchctl load ~/Library/LaunchAgents/mcquack.eblume.devpi.plist + when: devpi_launchctl_check.rc != 0 + changed_when: true + failed_when: false diff --git a/ansible/roles/devpi/templates/devpi.plist.j2 b/ansible/roles/devpi/templates/devpi.plist.j2 new file mode 100644 index 0000000..b9485e6 --- /dev/null +++ b/ansible/roles/devpi/templates/devpi.plist.j2 @@ -0,0 +1,34 @@ + + + + + + Label + mcquack.eblume.devpi + ProgramArguments + + {{ devpi_binary }} + --serverdir + {{ devpi_server_dir }} + --host + {{ devpi_host }} + --port + {{ devpi_port }} + --outside-url + {{ devpi_outside_url }} + + RunAtLoad + + KeepAlive + + EnvironmentVariables + + PATH + {{ devpi_venv }}/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin + + StandardOutPath + {{ devpi_log_dir }}/mcquack.devpi.out.log + StandardErrorPath + {{ devpi_log_dir }}/mcquack.devpi.err.log + + diff --git a/argocd/apps/devpi.yaml b/argocd/apps/devpi.yaml deleted file mode 100644 index 4a15672..0000000 --- a/argocd/apps/devpi.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# devpi PyPI Caching Proxy -# Provides PyPI cache and private package hosting -# -# After first deployment, initialize devpi: -# kubectl -n devpi exec -it devpi-0 -- devpi-init --serverdir /devpi --root-passwd -# kubectl -n devpi rollout restart statefulset devpi -# -# Then create user/index: -# uvx devpi use https://pypi.tail8d86e.ts.net -# uvx devpi login root -# uvx devpi user -c eblume email=blume.erich@gmail.com -# uvx devpi index -c eblume/dev bases=root/pypi -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: devpi - namespace: argocd -spec: - project: default - source: - repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/blumeops.git - targetRevision: main - path: argocd/manifests/devpi - destination: - server: https://kubernetes.default.svc - namespace: devpi - syncPolicy: - syncOptions: - - CreateNamespace=true diff --git a/argocd/manifests/alloy-k8s/config.alloy b/argocd/manifests/alloy-k8s/config.alloy index a716ddc..56a2e13 100644 --- a/argocd/manifests/alloy-k8s/config.alloy +++ b/argocd/manifests/alloy-k8s/config.alloy @@ -159,8 +159,10 @@ prometheus.exporter.blackbox "services" { } target { + // devpi runs natively on indri (LaunchAgent), not in-cluster. + // We probe through Caddy (https://pypi.ops.eblu.me) which the cluster can reach via Tailscale. name = "devpi" - address = "http://devpi.devpi.svc.cluster.local:3141/+api" + address = "https://pypi.ops.eblu.me/+api" module = "http_2xx" } diff --git a/argocd/manifests/devpi/README.md b/argocd/manifests/devpi/README.md deleted file mode 100644 index 11fd697..0000000 --- a/argocd/manifests/devpi/README.md +++ /dev/null @@ -1,72 +0,0 @@ -# devpi PyPI Caching Proxy - -devpi-server running in Kubernetes, providing: -- PyPI caching proxy at `root/pypi` -- Private package hosting at `eblume/dev` - -## Setup - -### 1. Create the root password secret - -```fish -kubectl create namespace devpi -op inject -i argocd/manifests/devpi/secret-root.yaml.tpl | kubectl apply -f - -``` - -### 2. Deploy via ArgoCD - -```fish -argocd app sync apps -argocd app sync devpi -``` - -The container will auto-initialize on first startup using the root password from the secret. - -### 3. Create user and index (first time only) - -After the pod is running: - -```fish -# Login to devpi as root -uvx --from devpi-client devpi use https://pypi.tail8d86e.ts.net -uvx --from devpi-client devpi login root -# Enter root password when prompted - -# Create eblume user (prompts for password - use the one from 1Password) -uvx --from devpi-client devpi user -c eblume email=blume.erich@gmail.com - -# Create private index inheriting from PyPI -uvx --from devpi-client devpi index -c eblume/dev bases=root/pypi -``` - -## Usage - -### As pip index (caching proxy) - -Configure `~/.config/pip/pip.conf`: - -```ini -[global] -index-url = https://pypi.tail8d86e.ts.net/root/pypi/+simple/ -trusted-host = pypi.tail8d86e.ts.net -``` - -### Upload private packages - -```fish -cd ~/code/personal/your-package -uv build -uv publish --publish-url https://pypi.tail8d86e.ts.net/eblume/dev/ -``` - -## URLs - -- Web UI: https://pypi.tail8d86e.ts.net -- PyPI cache: https://pypi.tail8d86e.ts.net/root/pypi/+simple/ -- Private index: https://pypi.tail8d86e.ts.net/eblume/dev/+simple/ - -## Credentials - -Stored in 1Password vault `blumeops`, item `kyhzfifryqnuk7jeyibmmjvxxm`: -- `root password` - devpi root user -- `password` - eblume user password diff --git a/argocd/manifests/devpi/external-secret.yaml b/argocd/manifests/devpi/external-secret.yaml deleted file mode 100644 index 290ea67..0000000 --- a/argocd/manifests/devpi/external-secret.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# ExternalSecret for devpi root password -# -# Replaces the manual op inject workflow from secret-root.yaml.tpl -# -# 1Password item: "devpi" in blumeops vault -# Field: "root password" -# -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: devpi-root - namespace: devpi -spec: - refreshInterval: 1h - secretStoreRef: - kind: ClusterSecretStore - name: onepassword-blumeops - target: - name: devpi-root - creationPolicy: Owner - data: - - secretKey: password - remoteRef: - key: devpi - property: root password diff --git a/argocd/manifests/devpi/ingress-tailscale.yaml b/argocd/manifests/devpi/ingress-tailscale.yaml deleted file mode 100644 index 474bf72..0000000 --- a/argocd/manifests/devpi/ingress-tailscale.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: devpi-tailscale - namespace: devpi - annotations: - tailscale.com/proxy-class: "default" - tailscale.com/proxy-group: "ingress" - gethomepage.dev/enabled: "true" - gethomepage.dev/name: "PyPI" - gethomepage.dev/group: "Infrastructure" - gethomepage.dev/icon: "pypi.png" - gethomepage.dev/description: "PyPI cache" - gethomepage.dev/href: "https://pypi.ops.eblu.me" - gethomepage.dev/pod-selector: "app=devpi" -spec: - ingressClassName: tailscale - defaultBackend: - service: - name: devpi - port: - number: 3141 - tls: - - hosts: - - pypi diff --git a/argocd/manifests/devpi/kustomization.yaml b/argocd/manifests/devpi/kustomization.yaml deleted file mode 100644 index 2083aaa..0000000 --- a/argocd/manifests/devpi/kustomization.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namespace: devpi - -resources: - - statefulset.yaml - - service.yaml - - ingress-tailscale.yaml - - external-secret.yaml - -images: - - name: registry.ops.eblu.me/blumeops/devpi - newTag: v6.19.3-37b8a21 diff --git a/argocd/manifests/devpi/service.yaml b/argocd/manifests/devpi/service.yaml deleted file mode 100644 index 42e1543..0000000 --- a/argocd/manifests/devpi/service.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: devpi - namespace: devpi -spec: - selector: - app: devpi - ports: - - name: http - port: 3141 - targetPort: 3141 - protocol: TCP diff --git a/argocd/manifests/devpi/statefulset.yaml b/argocd/manifests/devpi/statefulset.yaml deleted file mode 100644 index 91875df..0000000 --- a/argocd/manifests/devpi/statefulset.yaml +++ /dev/null @@ -1,64 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: devpi - namespace: devpi -spec: - serviceName: devpi - replicas: 1 - selector: - matchLabels: - app: devpi - template: - metadata: - labels: - app: devpi - spec: - securityContext: - fsGroup: 1000 - seccompProfile: - type: RuntimeDefault - containers: - - name: devpi - image: registry.ops.eblu.me/blumeops/devpi:kustomized - env: - - name: DEVPI_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: devpi-root - key: password - - name: DEVPI_OUTSIDE_URL - value: "https://pypi.ops.eblu.me" - ports: - - containerPort: 3141 - name: http - volumeMounts: - - name: data - mountPath: /devpi - resources: - requests: - memory: "256Mi" - cpu: "100m" - limits: - memory: "2Gi" # High limit for initial PyPI index build, reclaimed after - cpu: "500m" - livenessProbe: - httpGet: - path: /+api - port: 3141 - initialDelaySeconds: 30 - periodSeconds: 30 - readinessProbe: - httpGet: - path: /+api - port: 3141 - initialDelaySeconds: 10 - periodSeconds: 10 - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: 50Gi diff --git a/docs/changelog.d/migrate-devpi-to-indri.infra.md b/docs/changelog.d/migrate-devpi-to-indri.infra.md new file mode 100644 index 0000000..418db70 --- /dev/null +++ b/docs/changelog.d/migrate-devpi-to-indri.infra.md @@ -0,0 +1 @@ +Migrated devpi (PyPI mirror at `pypi.ops.eblu.me`) from a minikube StatefulSet to a launchd-managed service on indri. devpi-server now runs in a uv-managed venv with pinned `devpi-server` and `devpi-web` versions, listens on `127.0.0.1:3141`, and is fronted by Caddy. The minikube StatefulSet was crash-looping under memory pressure (and breaking the Python toolchain everywhere); the new layout removes a layer of dependency on cluster health for critical-path tooling. See [[devpi-on-indri]]. diff --git a/docs/how-to/operations/devpi-on-indri.md b/docs/how-to/operations/devpi-on-indri.md new file mode 100644 index 0000000..0334d37 --- /dev/null +++ b/docs/how-to/operations/devpi-on-indri.md @@ -0,0 +1,74 @@ +--- +title: Devpi on Indri +modified: 2026-04-29 +last-reviewed: 2026-04-29 +tags: + - how-to + - operations +--- + +# Devpi on Indri + +How devpi (the PyPI caching mirror at `pypi.ops.eblu.me`) is deployed on indri as a launchd-managed native service. Replaces the prior minikube StatefulSet. + +## Why native, not Kubernetes + +Devpi has no runtime dependencies beyond a Python interpreter, a writable directory, and outbound HTTPS to upstream PyPI. Running it on indri natively removes a layer of operational complexity, frees minikube resources, and decouples this critical-path tooling (used by every Python build, including `mise run docs-mikado` itself) from cluster health. + +## Layout + +| Concern | Path / detail | +|---|---| +| Service binary | `/Users/erichblume/devpi/venv/bin/devpi-server` | +| Server-dir (data) | `/Users/erichblume/devpi/server-dir/` | +| Logs | `/Users/erichblume/Library/Logs/mcquack.devpi.{out,err}.log` | +| LaunchAgent label | `mcquack.eblume.devpi` | +| LaunchAgent plist | `~/Library/LaunchAgents/mcquack.eblume.devpi.plist` | +| Listen address | `127.0.0.1:3141` (loopback only) | +| Public URL | `https://pypi.ops.eblu.me` (via Caddy reverse proxy) | +| Root password secret | 1Password item `devpi`, field `root password` | + +The venv is built fresh by ansible from a pinned `devpi-server` and `devpi-web` version; bumping versions is a config change in `ansible/roles/devpi/defaults/main.yml`. + +## Deploy + +```fish +mise run provision-indri -- --tags devpi +``` + +Ansible will: + +1. Fetch the root password from 1Password (in playbook `pre_tasks`) +2. Create the venv at `~/devpi/venv` if absent and install/upgrade `devpi-server` + `devpi-web` to the pinned versions +3. Initialize the server-dir (only on first run, when `.serverversion` is missing) +4. Render and load the LaunchAgent plist +5. Restart the service if the plist or config changed + +Caddy already proxies `pypi.ops.eblu.me` → `127.0.0.1:3141`; nothing else routes traffic. + +## Verify + +```fish +ssh indri 'launchctl list mcquack.eblume.devpi' +curl -fsS https://pypi.ops.eblu.me/+api | jq +uv pip install --index-url https://pypi.ops.eblu.me/root/pypi/+simple/ requests +``` + +## Logs + +```fish +ssh indri 'tail -f ~/Library/Logs/mcquack.devpi.err.log' +``` + +## Bumping devpi versions + +Edit `devpi_server_version` / `devpi_web_version` in `ansible/roles/devpi/defaults/main.yml`, then re-run the playbook with `--tags devpi`. The role rebuilds the venv in-place; the server-dir survives. + +## Backup + +The server-dir is **not** in `borgmatic_source_directories` and is not backed up. The PyPI cache (`+files/`) is re-fetchable from upstream on first request; the local `eblume/dev` index can be republished from source. If retention becomes important, add `/Users/erichblume/devpi/server-dir/` to the borgmatic source list. + +## Related + +- [[restart-indri]] — devpi is one of the LaunchAgents to stop on graceful shutdown +- [[connect-to-postgres]] — pattern for indri-native services (different stack, similar shape) diff --git a/docs/how-to/operations/rebuild-minikube-cluster.md b/docs/how-to/operations/rebuild-minikube-cluster.md index e23d027..0d924e9 100644 --- a/docs/how-to/operations/rebuild-minikube-cluster.md +++ b/docs/how-to/operations/rebuild-minikube-cluster.md @@ -235,25 +235,7 @@ mise run services-check ## Post-Rebuild: Cold Cache Failures -### Devpi (PyPI Cache) - -After a rebuild, devpi's package cache is empty. The first Dagger-based container build will trigger a flood of concurrent package downloads. Devpi uses lazy caching — it serves package metadata (simple index) immediately from upstream PyPI but fetches wheel files on demand. Under heavy concurrent load with a cold cache, the upstream fetch can race with the client request, causing devpi to return `no such file` (HTTP 404) for packages it knows about but hasn't finished downloading yet. - -**Why devpi, not PyPI?** The repo's `uv.lock` was generated with devpi as the index, so every package source URL points at `pypi.ops.eblu.me`. Dagger's Python SDK runtime does a locked install (`uv sync`), not fresh resolution — it fetches from whatever URLs are in the lockfile. This is intentional (supply chain control), but means all builds — local and CI — depend on devpi being available and warm. - -**Symptoms:** Forgejo Actions Dagger builds fail during module initialization with errors like: -``` -Failed to download `googleapis-common-protos==1.74.0` -HTTP status client error (404 Not Found) for url (https://pypi.ops.eblu.me/root/pypi/+f/...) -``` - -**Fix:** Re-run the failed build. The first attempt warms the cache; subsequent builds succeed. Alternatively, warm the cache manually before triggering CI builds: - -```bash -# From any machine that can reach pypi.ops.eblu.me, install the Dagger SDK -# to pre-populate the most common packages: -pip install --dry-run --index-url https://pypi.ops.eblu.me/root/pypi/+simple/ dagger-io -``` +Devpi runs natively on indri (see [[devpi-on-indri]]) and is unaffected by minikube rebuilds, so the historical "devpi cold cache after rebuild" failure mode no longer applies. If devpi itself goes cold (fresh server-dir), the same lazy-cache race can still cause `404` on the first Dagger build under concurrent load — re-run the build to warm the cache, or pre-warm with `uv pip install --dry-run --index-url https://pypi.ops.eblu.me/root/pypi/+simple/ dagger-io`. ## Related diff --git a/docs/how-to/operations/restart-indri.md b/docs/how-to/operations/restart-indri.md index a956644..e92581e 100644 --- a/docs/how-to/operations/restart-indri.md +++ b/docs/how-to/operations/restart-indri.md @@ -41,6 +41,7 @@ Native services managed by launchd will stop automatically during macOS shutdown ssh indri 'launchctl unload ~/Library/LaunchAgents/mcquack.eblume.forgejo.plist' ssh indri 'launchctl unload ~/Library/LaunchAgents/mcquack.eblume.caddy.plist' ssh indri 'launchctl unload ~/Library/LaunchAgents/mcquack.eblume.zot.plist' +ssh indri 'launchctl unload ~/Library/LaunchAgents/mcquack.eblume.devpi.plist' # see [[devpi-on-indri]] ssh indri 'launchctl unload ~/Library/LaunchAgents/mcquack.jellyfin.plist' ssh indri 'launchctl unload ~/Library/LaunchAgents/mcquack.eblume.alloy.plist' ssh indri 'launchctl unload ~/Library/LaunchAgents/mcquack.eblume.borgmatic.plist' diff --git a/docs/reference/infrastructure/tailscale.md b/docs/reference/infrastructure/tailscale.md index 2794111..9c15d83 100644 --- a/docs/reference/infrastructure/tailscale.md +++ b/docs/reference/infrastructure/tailscale.md @@ -33,7 +33,7 @@ ACLs managed via Pulumi in `pulumi/tailscale/policy.hujson`. | `tag:loki` | indri | Loki log aggregation | | `tag:k8s-api` | indri | Kubernetes API server (minikube) | | `tag:k8s-operator` | (operator pod) | Tailscale operator for k8s — see [[tailscale-operator]] | -| `tag:k8s` | (Ingress proxy pods) | Kubernetes Tailscale Ingress nodes; each also carries a per-service tag (`tag:grafana`, `tag:kiwix`, `tag:devpi`, `tag:feed`, `tag:pg`) | +| `tag:k8s` | (Ingress proxy pods) | Kubernetes Tailscale Ingress nodes; each also carries a per-service tag (`tag:grafana`, `tag:kiwix`, `tag:feed`, `tag:pg`) | | `tag:ci-gateway` | (ephemeral CI containers) | CI containers pushing images to registry | | `tag:flyio-proxy` | (Fly.io proxy container) | Public reverse proxy | | `tag:flyio-target` | indri, designated Ingress endpoints | Endpoints reachable by the Fly.io proxy (indri for Caddy routing, Ingress pods for Alloy metrics/logs) | diff --git a/docs/reference/services/devpi.md b/docs/reference/services/devpi.md index c6493fe..589a802 100644 --- a/docs/reference/services/devpi.md +++ b/docs/reference/services/devpi.md @@ -1,7 +1,7 @@ --- title: Devpi -modified: 2026-03-23 -last-reviewed: 2026-03-23 +modified: 2026-04-29 +last-reviewed: 2026-04-29 tags: - service - python @@ -9,31 +9,37 @@ tags: # devpi (PyPI Proxy) -PyPI caching proxy and private package index. +PyPI caching proxy and private package index. Runs natively on [[indri]] as a LaunchAgent (not in-cluster). See [[devpi-on-indri]] for deploy and operations. ## Quick Reference | Property | Value | |----------|-------| -| **URL** | https://pypi.ops.eblu.me | -| **Namespace** | `devpi` | -| **ArgoCD App** | `devpi` | -| **Storage** | 50Gi PVC | -| **Image** | `registry.ops.eblu.me/blumeops/devpi` (see `argocd/manifests/devpi/kustomization.yaml` for current tag) | +| **URL** | `https://pypi.ops.eblu.me` | +| **Listen** | `127.0.0.1:3141` (loopback only; reached via Caddy) | +| **Service** | LaunchAgent `mcquack.eblume.devpi` on indri | +| **Server-dir** | `/Users/erichblume/devpi/server-dir/` | +| **Runtime** | uv-managed venv at `/Users/erichblume/devpi/venv/` | +| **Ansible role** | `ansible/roles/devpi/` | +| **Versions** | Pinned in `ansible/roles/devpi/defaults/main.yml` (`devpi_server_version`, `devpi_web_version`) | ## Indices | Index | Purpose | |-------|---------| -| `root/pypi` | PyPI mirror/cache (auto-created) | -| `eblume/dev` | Private packages (inherits from root/pypi) | +| `root/pypi` | PyPI mirror/cache (auto-created by `devpi-init`) | +| `eblume/dev` | Private packages (inherits from `root/pypi`) | ## Credentials -Root password stored in 1Password (blumeops vault), injected via ExternalSecret. +Root password stored in 1Password (`blumeops` vault, item `devpi`, field `root password`). Fetched via `op read` in the `ansible/playbooks/indri.yml` `pre_tasks` and passed to the role on first init. + +## Backup + +The server-dir is **not** backed up. The PyPI cache (`+files/`) is re-fetchable from upstream on first request. The local `eblume/dev` index metadata is small but also not critical to retain — packages can be republished from source. If retention becomes important, add `/Users/erichblume/devpi/server-dir/` to `borgmatic_source_directories`. ## Related -- [[use-pypi-proxy]] - Client configuration and package uploads -- [[argocd]] - Deployment -- [[1password]] - Secrets management +- [[devpi-on-indri]] — Deploy, verify, and version-bump procedures +- [[use-pypi-proxy]] — Client configuration and package uploads +- [[1password]] — Secrets management diff --git a/docs/reference/storage/backups.md b/docs/reference/storage/backups.md index 9ca3bcb..14dbcea 100644 --- a/docs/reference/storage/backups.md +++ b/docs/reference/storage/backups.md @@ -62,7 +62,7 @@ Other data lives directly on [[sifaka]] (music via [[navidrome]], video via [[je | ZIM archives (`~/transmission/`) | Re-downloadable via torrent | | Prometheus metrics | Ephemeral, in k8s PVC | | Loki logs | Ephemeral, in k8s PVC | -| devpi cache | Re-fetchable from PyPI | +| devpi cache (`~/devpi/server-dir/` on indri) | Re-fetchable from PyPI on first request | ## Retention Policy diff --git a/pulumi/tailscale/__main__.py b/pulumi/tailscale/__main__.py index 2f5262b..3acbb62 100644 --- a/pulumi/tailscale/__main__.py +++ b/pulumi/tailscale/__main__.py @@ -37,7 +37,7 @@ acl = tailscale.Acl( # indri - Mac Mini M1, primary homelab server # Hosts forge, loki, zot registry, and the k8s control plane. -# Other services (grafana, kiwix, devpi, etc.) run in k8s with their own Tailscale devices. +# Other services (grafana, kiwix, etc.) run in k8s with their own Tailscale devices. indri = tailscale.get_device(name="indri.tail8d86e.ts.net") indri_tags = tailscale.DeviceTags( "indri-tags", diff --git a/pulumi/tailscale/policy.hujson b/pulumi/tailscale/policy.hujson index 84f1f17..88408ef 100644 --- a/pulumi/tailscale/policy.hujson +++ b/pulumi/tailscale/policy.hujson @@ -20,7 +20,8 @@ }, // --- Members: user-facing services only --- - // Kiwix, Forge, devpi, Miniflux, PostgreSQL + // Kiwix, Forge, Miniflux, PostgreSQL + // (devpi moved off-cluster to indri; reachable via Caddy on tag:flyio-target) { "src": ["autogroup:member"], "dst": ["tag:kiwix"], @@ -31,11 +32,6 @@ "dst": ["tag:forge"], "ip": ["tcp:443", "tcp:22"], }, - { - "src": ["autogroup:member"], - "dst": ["tag:devpi"], - "ip": ["tcp:443"], - }, { "src": ["autogroup:member"], "dst": ["tag:feed"], @@ -152,7 +148,6 @@ "tag:grafana": ["autogroup:admin", "tag:blumeops"], "tag:kiwix": ["autogroup:admin", "tag:blumeops"], "tag:forge": ["autogroup:admin", "tag:blumeops"], - "tag:devpi": ["autogroup:admin", "tag:blumeops"], "tag:loki": ["autogroup:admin", "tag:blumeops"], "tag:pg": ["autogroup:admin", "tag:blumeops"], "tag:feed": ["autogroup:admin", "tag:blumeops"], diff --git a/service-versions.yaml b/service-versions.yaml index 0a4fe93..e819c6c 100644 --- a/service-versions.yaml +++ b/service-versions.yaml @@ -214,10 +214,11 @@ services: upstream-source: https://github.com/kiwix/kiwix-tools/releases - name: devpi - type: argocd - last-reviewed: 2026-04-18 + type: ansible + last-reviewed: 2026-04-29 current-version: "6.19.3" upstream-source: https://github.com/devpi/devpi/releases + notes: Installed via uv into a venv on indri; version pinned in ansible/roles/devpi/defaults/main.yml - name: cv type: argocd