P6: Migrate Kiwix and Transmission to Kubernetes (#39)

## Summary
- Add Transmission BitTorrent daemon to k8s (torrent namespace)
- Add Kiwix ZIM archive server to k8s (kiwix namespace)
- NFS storage from sifaka for shared torrent/ZIM data
- Torrent-sync sidecar in kiwix deployment to manage declarative ZIM list
- ZIM-watcher CronJob to auto-restart kiwix when new archives appear
- Remove transmission, transmission_metrics, and kiwix ansible roles from indri
- Remove svc:kiwix from tailscale_serve defaults

## Key Decisions
- Direct NFS mount for kiwix (no PVC) since it shares storage with transmission
- Shell wrapper for kiwix-serve command (glob expansion)
- Accept HTTP 409 as "ready" in torrent sync (transmission session ID mechanism)
- Completed downloads stored in `/downloads/complete/` on sifaka

## Deployment and Testing
- [x] Deployed transmission to k8s
- [x] Verified transmission web UI at torrent.tail8d86e.ts.net
- [x] Moved existing ZIM files to complete folder
- [x] Deployed kiwix to k8s
- [x] Verified kiwix web UI at kiwix.tail8d86e.ts.net
- [x] Stopped old services on indri
- [x] Cleared svc:kiwix from Tailscale serve on indri
- [x] Updated zk documentation

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Reviewed-on: https://forge.tail8d86e.ts.net/eblume/blumeops/pulls/39

ansible/playbooks/indri.yml

@@ -29,20 +29,12 @@
       tags: alloy
     - role: prometheus
       tags: prometheus
-    # NOTE: grafana role removed - now hosted in k8s (see argocd/apps/grafana.yaml)
-    - role: transmission
-      tags: transmission
-    - role: transmission_metrics
-      tags: transmission_metrics
-    - role: kiwix
-      tags: kiwix
     - role: borgmatic
       tags: borgmatic
     - role: borgmatic_metrics
       tags: borgmatic_metrics
     - role: forgejo
       tags: forgejo
-    # NOTE: devpi and devpi_metrics roles removed - now hosted in k8s (see argocd/apps/devpi.yaml)
     - role: zot
       tags: zot
     - role: zot_metrics
@@ -53,6 +45,5 @@
       tags: minikube_metrics
     - role: plex_metrics
       tags: plex_metrics
-    # NOTE: postgresql and miniflux roles removed - now hosted in k8s
     - role: tailscale_serve
       tags: tailscale-serve

ansible/roles/tailscale_serve/defaults/main.yml

@@ -3,9 +3,6 @@
 # Each service maps a Tailscale service name to local endpoints
 
 tailscale_serve_services:
-  # NOTE: svc:grafana, svc:pg, svc:feed, svc:pypi removed - now hosted in k8s
-  # NOTE: svc:k8s is configured by the minikube role (port is dynamic with docker driver)
-
   - name: svc:forge
     https:
       port: 443
@@ -14,11 +11,6 @@ tailscale_serve_services:
       port: 22
       upstream: tcp://localhost:2200
 
-  - name: svc:kiwix
-    https:
-      port: 443
-      upstream: http://localhost:5501
-
   - name: svc:registry
     https:
       port: 443

argocd/apps/kiwix.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: kiwix
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: ssh://forgejo@indri.tail8d86e.ts.net:2200/eblume/blumeops.git
+    targetRevision: main
+    path: argocd/manifests/kiwix
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: kiwix
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=true

argocd/apps/torrent.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: torrent
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: ssh://forgejo@indri.tail8d86e.ts.net:2200/eblume/blumeops.git
+    targetRevision: main
+    path: argocd/manifests/torrent
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: torrent
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=true

argocd/manifests/kiwix/configmap-sync-script.yaml

@@ -0,0 +1,68 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: zim-torrent-sync-script
+  namespace: kiwix
+data:
+  sync-zim-torrents.sh: |
+    #!/bin/bash
+    # Sync ZIM torrents from kiwix ConfigMap to Transmission
+    # Runs as a sidecar in the kiwix deployment
+    set -euo pipefail
+
+    TORRENT_LIST="${TORRENT_LIST:-/config/torrents.txt}"
+    TRANSMISSION_HOST="${TRANSMISSION_HOST:-transmission.torrent.svc.cluster.local}"
+    TRANSMISSION_PORT="${TRANSMISSION_PORT:-9091}"
+
+    echo "Syncing ZIM torrents to transmission at ${TRANSMISSION_HOST}:${TRANSMISSION_PORT}"
+
+    # Wait for transmission to be ready
+    # Transmission RPC returns 409 on first request (to provide session ID), which is fine
+    echo "Waiting for Transmission RPC..."
+    max_attempts=30
+    attempt=0
+    until curl -s -o /dev/null -w "%{http_code}" "http://${TRANSMISSION_HOST}:${TRANSMISSION_PORT}/transmission/rpc" | grep -qE "^(200|409)$"; do
+        attempt=$((attempt + 1))
+        if [[ $attempt -ge $max_attempts ]]; then
+            echo "Transmission not ready after ${max_attempts} attempts, will retry next cycle"
+            exit 0  # Don't fail, just skip this sync
+        fi
+        sleep 10
+    done
+    echo "Transmission is ready"
+
+    # Get current torrents from transmission
+    # transmission-remote returns header + data + footer, extract just torrent names
+    current=$(transmission-remote "${TRANSMISSION_HOST}:${TRANSMISSION_PORT}" -l 2>/dev/null | \
+              tail -n +2 | head -n -1 | awk '{print $NF}' || true)
+
+    added=0
+    skipped=0
+
+    while IFS= read -r url || [[ -n "$url" ]]; do
+        # Skip empty lines and comments
+        [[ -z "$url" || "$url" =~ ^[[:space:]]*# ]] && continue
+        # Trim whitespace
+        url=$(echo "$url" | xargs)
+        [[ -z "$url" ]] && continue
+
+        # Extract base name from URL (remove .torrent extension)
+        basename=$(basename "$url" .torrent)
+        # Also try without .zim in case transmission reports it differently
+        basename_no_zim="${basename%.zim}"
+
+        # Check if already in transmission
+        if echo "$current" | grep -qF "$basename_no_zim"; then
+            ((skipped++)) || true
+        else
+            if transmission-remote "${TRANSMISSION_HOST}:${TRANSMISSION_PORT}" -a "$url" 2>/dev/null; then
+                echo "Added: $basename"
+                ((added++)) || true
+            else
+                echo "Warning: Failed to add $url" >&2
+            fi
+        fi
+    done < "$TORRENT_LIST"
+
+    echo "Sync complete: $added added, $skipped already present"

argocd/manifests/kiwix/configmap-zim-torrents.yaml

@@ -0,0 +1,69 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kiwix-zim-torrents
+  namespace: kiwix
+data:
+  torrents.txt: |
+    # Declarative ZIM archive torrent URLs
+    # These are synced to transmission automatically by the kiwix sidecar
+    # Format: one URL per line, comments start with #
+    #
+    # Users can also add ZIM torrents manually via torrent.tail8d86e.ts.net
+    # and kiwix will pick them up automatically.
+
+    # Wikipedia - Top 1M English articles (43G)
+    https://download.kiwix.org/zim/wikipedia/wikipedia_en_top1m_maxi_2025-09.zim.torrent
+
+    # Project Gutenberg - Public domain books (72G)
+    https://download.kiwix.org/zim/gutenberg/gutenberg_en_all_2023-08.zim.torrent
+
+    # iFixit - Repair guides (3.3G)
+    https://download.kiwix.org/zim/ifixit/ifixit_en_all_2025-12.zim.torrent
+
+    # Stack Exchange
+    https://download.kiwix.org/zim/stack_exchange/superuser.com_en_all_2025-12.zim.torrent
+    https://download.kiwix.org/zim/stack_exchange/math.stackexchange.com_en_all_2025-12.zim.torrent
+
+    # LibreTexts - Open educational resources
+    https://download.kiwix.org/zim/libretexts/libretexts.org_en_bio_2025-01.zim.torrent
+    https://download.kiwix.org/zim/libretexts/libretexts.org_en_chem_2025-01.zim.torrent
+    https://download.kiwix.org/zim/libretexts/libretexts.org_en_eng_2025-01.zim.torrent
+    https://download.kiwix.org/zim/libretexts/libretexts.org_en_math_2025-01.zim.torrent
+    https://download.kiwix.org/zim/libretexts/libretexts.org_en_phys_2025-01.zim.torrent
+    https://download.kiwix.org/zim/libretexts/libretexts.org_en_human_2025-01.zim.torrent
+
+    # DevDocs - Programming documentation
+    https://download.kiwix.org/zim/devdocs/devdocs_en_bash_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_c_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_click_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_cmake_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_cpp_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_css_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_django-rest-framework_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_django_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_docker_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_duckdb_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_fish_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_gcc_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_git_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_go_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_godot_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_hammerspoon_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_homebrew_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_javascript_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_kubectl_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_kubernetes_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_latex_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_lua_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_markdown_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_nginx_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_nix_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_postgresql_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_python_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_redis_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_sqlite_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_typescript_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_werkzeug_2026-01.zim.torrent
+    https://download.kiwix.org/zim/devdocs/devdocs_en_zig_2026-01.zim.torrent

argocd/manifests/kiwix/cronjob-zim-watcher.yaml

@@ -0,0 +1,85 @@
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: zim-watcher
+  namespace: kiwix
+spec:
+  schedule: "0 * * * *"  # Every hour
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          serviceAccountName: zim-watcher
+          containers:
+            - name: watcher
+              image: bitnami/kubectl:latest
+              command: ["/bin/bash", "-c"]
+              args:
+                - |
+                  set -euo pipefail
+
+                  # Get current ZIM files (among all downloads)
+                  # This picks up ZIMs from both declarative list AND manually added torrents
+                  current_zims=$(ls -1 /data/complete/*.zim 2>/dev/null | sort | md5sum | cut -d' ' -f1 || echo "empty")
+
+                  # Get stored hash from deployment annotation
+                  JSONPATH='{.metadata.annotations.kiwix\.blumeops/zim-hash}'
+                  stored_hash=$(kubectl get deployment kiwix -n kiwix -o jsonpath="$JSONPATH" 2>/dev/null || echo "")
+
+                  echo "Current ZIMs hash: $current_zims"
+                  echo "Stored hash: $stored_hash"
+
+                  # Also list what ZIMs we found
+                  echo "ZIM files found:"
+                  ls -la /data/complete/*.zim 2>/dev/null || echo "  (none)"
+
+                  if [[ "$current_zims" != "$stored_hash" && "$current_zims" != "empty" ]]; then
+                    echo "ZIM files changed, restarting kiwix deployment..."
+                    kubectl annotate deployment kiwix -n kiwix "kiwix.blumeops/zim-hash=$current_zims" --overwrite
+                    kubectl rollout restart deployment/kiwix -n kiwix
+                    echo "Restart triggered"
+                  else
+                    echo "No changes detected"
+                  fi
+              volumeMounts:
+                - name: torrents
+                  mountPath: /data
+                  readOnly: true
+          restartPolicy: OnFailure
+          volumes:
+            - name: torrents
+              nfs:
+                server: sifaka
+                path: /volume1/torrents
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: zim-watcher
+  namespace: kiwix
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: zim-watcher
+  namespace: kiwix
+rules:
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["get", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: zim-watcher
+  namespace: kiwix
+subjects:
+  - kind: ServiceAccount
+    name: zim-watcher
+    namespace: kiwix
+roleRef:
+  kind: Role
+  name: zim-watcher
+  apiGroup: rbac.authorization.k8s.io

argocd/manifests/kiwix/deployment.yaml

@@ -0,0 +1,98 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kiwix
+  namespace: kiwix
+  annotations:
+    # Track ZIM file changes for restart detection
+    kiwix.blumeops/zim-hash: ""
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kiwix
+  template:
+    metadata:
+      labels:
+        app: kiwix
+    spec:
+      containers:
+        # Main kiwix-serve container
+        - name: kiwix-serve
+          image: ghcr.io/kiwix/kiwix-serve:3.8.1
+          command: ["/bin/sh", "-c"]
+          args:
+            - "kiwix-serve --port=80 /data/complete/*.zim"
+          ports:
+            - containerPort: 80
+              name: http
+          volumeMounts:
+            - name: torrents
+              mountPath: /data
+              readOnly: true
+          resources:
+            requests:
+              memory: "256Mi"
+              cpu: "100m"
+            limits:
+              memory: "1Gi"
+          livenessProbe:
+            httpGet:
+              path: /
+              port: 80
+            initialDelaySeconds: 10
+            periodSeconds: 30
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 80
+            initialDelaySeconds: 5
+            periodSeconds: 10
+
+        # Sidecar: Syncs declarative ZIM torrents to transmission
+        - name: torrent-sync
+          image: lscr.io/linuxserver/transmission:latest  # Has transmission-remote CLI
+          command: ["/bin/bash", "-c"]
+          args:
+            - |
+              echo "Starting ZIM torrent sync sidecar"
+              # Initial sync
+              /scripts/sync-zim-torrents.sh || echo "Initial sync failed, will retry"
+              # Periodic sync every 30 minutes
+              while true; do
+                sleep 1800
+                /scripts/sync-zim-torrents.sh || echo "Sync failed, will retry"
+              done
+          env:
+            - name: TRANSMISSION_HOST
+              value: "transmission.torrent.svc.cluster.local"
+            - name: TRANSMISSION_PORT
+              value: "9091"
+            - name: TORRENT_LIST
+              value: "/config/torrents.txt"
+          volumeMounts:
+            - name: zim-torrents-config
+              mountPath: /config/torrents.txt
+              subPath: torrents.txt
+            - name: sync-script
+              mountPath: /scripts
+          resources:
+            requests:
+              memory: "32Mi"
+              cpu: "10m"
+            limits:
+              memory: "64Mi"
+
+      volumes:
+        - name: torrents
+          nfs:
+            server: sifaka
+            path: /volume1/torrents
+        - name: zim-torrents-config
+          configMap:
+            name: kiwix-zim-torrents
+        - name: sync-script
+          configMap:
+            name: zim-torrent-sync-script
+            defaultMode: 493  # 0755 in decimal

argocd/manifests/kiwix/ingress-tailscale.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: kiwix-tailscale
+  namespace: kiwix
+  annotations:
+    tailscale.com/proxy-class: "default"
+spec:
+  ingressClassName: tailscale
+  defaultBackend:
+    service:
+      name: kiwix
+      port:
+        number: 80
+  tls:
+    - hosts:
+        - kiwix

argocd/manifests/kiwix/kustomization.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: kiwix
+resources:
+  - configmap-zim-torrents.yaml
+  - configmap-sync-script.yaml
+  - deployment.yaml
+  - service.yaml
+  - ingress-tailscale.yaml
+  - cronjob-zim-watcher.yaml

argocd/manifests/kiwix/service.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kiwix
+  namespace: kiwix
+spec:
+  selector:
+    app: kiwix
+  ports:
+    - name: http
+      port: 80
+      targetPort: 80

argocd/manifests/torrent/deployment.yaml

@@ -0,0 +1,63 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: transmission
+  namespace: torrent
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: transmission
+  template:
+    metadata:
+      labels:
+        app: transmission
+    spec:
+      containers:
+        - name: transmission
+          image: lscr.io/linuxserver/transmission:latest
+          env:
+            - name: PUID
+              value: "1000"
+            - name: PGID
+              value: "1000"
+            - name: TZ
+              value: "America/Los_Angeles"
+          ports:
+            - containerPort: 9091
+              name: web
+            - containerPort: 51413
+              name: peer-tcp
+            - containerPort: 51413
+              protocol: UDP
+              name: peer-udp
+          volumeMounts:
+            - name: downloads
+              mountPath: /downloads
+            - name: config
+              mountPath: /config
+          resources:
+            requests:
+              memory: "256Mi"
+              cpu: "100m"
+            limits:
+              memory: "512Mi"
+          livenessProbe:
+            httpGet:
+              path: /transmission/web/
+              port: 9091
+            initialDelaySeconds: 30
+            periodSeconds: 30
+          readinessProbe:
+            httpGet:
+              path: /transmission/web/
+              port: 9091
+            initialDelaySeconds: 10
+            periodSeconds: 10
+      volumes:
+        - name: downloads
+          persistentVolumeClaim:
+            claimName: torrents-storage
+        - name: config
+          emptyDir: {}  # Config is ephemeral; torrents persist in NFS

argocd/manifests/torrent/ingress-tailscale.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: transmission-tailscale
+  namespace: torrent
+  annotations:
+    tailscale.com/proxy-class: "default"
+spec:
+  ingressClassName: tailscale
+  defaultBackend:
+    service:
+      name: transmission
+      port:
+        number: 9091
+  tls:
+    - hosts:
+        - torrent

argocd/manifests/torrent/kustomization.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: torrent
+resources:
+  - pv-nfs.yaml
+  - pvc.yaml
+  - deployment.yaml
+  - service.yaml
+  - ingress-tailscale.yaml

argocd/manifests/torrent/pv-nfs.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: torrents-nfs-pv
+spec:
+  capacity:
+    storage: 1Ti
+  accessModes:
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: ""
+  nfs:
+    server: sifaka
+    path: /volume1/torrents

argocd/manifests/torrent/pvc.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: torrents-storage
+  namespace: torrent
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: ""
+  volumeName: torrents-nfs-pv
+  resources:
+    requests:
+      storage: 1Ti

argocd/manifests/torrent/service.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: transmission
+  namespace: torrent
+spec:
+  selector:
+    app: transmission
+  ports:
+    - name: web
+      port: 9091
+      targetPort: 9091