Merge pull request 'Migrate Immich from Helm to kustomize (v2.5.6 → v2.6.3)' (#324) from immich-kustomize-v2.6.3 into main

ansible/roles/minikube/tasks/main.yml

@@ -77,6 +77,37 @@
     msg: "WARNING: minikube may not have started properly. Run 'minikube start' manually on indri if needed. Status: {{ minikube_final_status.stdout | default('unknown') }}"
   when: minikube_final_status.rc != 0
 
+# The storage-provisioner is a bare Pod (no controller). If the node restarts
+# via Docker Desktop rather than `minikube start`, kubelet brings back static
+# pods (apiserver, etcd) but bare pods like storage-provisioner are lost.
+# `minikube start` on a running cluster is safe and re-applies all addons.
+- name: Check storage-provisioner pod is running
+  ansible.builtin.command:
+    cmd: kubectl -n kube-system get pod storage-provisioner -o jsonpath='{.status.phase}'
+  register: minikube_storage_provisioner
+  changed_when: false
+  failed_when: false
+  when: minikube_final_status.rc == 0
+
+- name: Re-run minikube start to restore addons
+  ansible.builtin.command:
+    cmd: >
+      minikube start
+      --driver={{ minikube_driver }}
+      --container-runtime={{ minikube_container_runtime }}
+      --cpus={{ minikube_cpus }}
+      --memory={{ minikube_memory }}
+      --disk-size={{ minikube_disk_size }}
+      {% for name in minikube_apiserver_names %}
+      --apiserver-names={{ name }}
+      {% endfor %}
+      --apiserver-port={{ minikube_apiserver_port }}
+      --listen-address={{ minikube_listen_address }}
+  when:
+    - minikube_final_status.rc == 0
+    - minikube_storage_provisioner.stdout | default('') != 'Running'
+  changed_when: true
+
 # Configure containerd to use zot registry as pull-through cache
 # With docker driver, use host.minikube.internal to reach the host
 # Zot runs on indri:5050 and caches images from docker.io, ghcr.io, quay.io

argocd/apps/immich-storage.yaml

@@ -1,25 +0,0 @@
-# Immich Storage - PersistentVolume and PVC for photo library
-# Must be synced BEFORE the main immich app
-#
-# Prerequisites:
-# 1. NFS share on sifaka at /volume1/photos with permissions for indri
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: immich-storage
-  namespace: argocd
-spec:
-  project: default
-  source:
-    repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/blumeops.git
-    targetRevision: main
-    path: argocd/manifests/immich
-    # Only deploy storage resources (PV/PVC/Ingress), not Helm values.yaml
-    directory:
-      include: "{pv-nfs.yaml,pvc.yaml,ingress-tailscale.yaml}"
-  destination:
-    server: https://kubernetes.default.svc
-    namespace: immich
-  syncPolicy:
-    syncOptions:
-      - CreateNamespace=true

argocd/apps/immich.yaml

@@ -1,15 +1,16 @@
 # Immich - Self-hosted photo and video management
 # High-performance Google Photos/iCloud alternative with AI features
 #
-# Chart mirrored from https://github.com/immich-app/immich-charts to forge
+# Kustomize manifests in argocd/manifests/immich/
+# Components: server, machine-learning, valkey (Redis)
 #
 # Prerequisites:
-# 1. Mirror immich-charts to forge: https://github.com/immich-app/immich-charts
+# 1. Create immich namespace and secrets:
-# 2. Create immich namespace and secrets:
 #    kubectl create namespace immich
-#    op inject -i argocd/manifests/immich/secret-db.yaml.tpl | kubectl apply -f -
+#    kubectl --context=minikube-indri create secret generic immich-db -n immich \
-# 3. Create immich-pg database and user (see immich-pg app)
+#      --from-literal=password="$(kubectl --context=minikube-indri -n databases get secret immich-pg-app -o jsonpath='{.data.password}' | base64 -d)"
-# 4. Mount photos directory from indri to minikube
+# 2. Create immich-pg database and user (see immich-pg app)
+# 3. NFS share on sifaka at /volume1/photos with read/write for indri
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
@@ -17,18 +18,10 @@ metadata:
   namespace: argocd
 spec:
   project: default
-  sources:
+  source:
-    # Helm chart from forge mirror (SSH via egress)
+    repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/blumeops.git
-    - repoURL: ssh://forgejo@forge.ops.eblu.me:2222/mirrors/immich-charts.git
-      targetRevision: immich-0.10.3
-      path: charts/immich
-      helm:
-        releaseName: immich
-        valueFiles:
-          - $values/argocd/manifests/immich/values.yaml
-    - repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/blumeops.git
     targetRevision: main
-      ref: values
+    path: argocd/manifests/immich
   destination:
     server: https://kubernetes.default.svc
     namespace: immich

argocd/manifests/immich/README.md

@@ -11,11 +11,18 @@ Self-hosted photo and video management solution with AI-powered search and face
 ## Deployment Order
 
 1. Sync `blumeops-pg` (to get CloudNativePG operator if not already running)
-2. Sync `immich-storage` (creates PV, PVC, and Tailscale Ingress)
+2. Wait for `immich-pg` cluster to be healthy
-3. Wait for `immich-pg` cluster to be healthy
+3. Create secrets (see below)
-4. Create secrets (see below)
+4. Sync `immich` (deploys all resources: storage, services, deployments)
-5. Sync `immich` (deploys the Helm chart)
+5. Run `mise run provision-indri -- --tags caddy` to update Caddy config
-6. Run `mise run provision-indri -- --tags caddy` to update Caddy config
+
+## Components
+
+| Component | Deployment | Service | Port |
+|-----------|------------|---------|------|
+| Server (web/API) | `immich-server` | `immich-server` | 2283 |
+| Machine Learning | `immich-machine-learning` | `immich-machine-learning` | 3003 |
+| Valkey (Redis) | `immich-valkey` | `immich-valkey` | 6379 |
 
 ## Secret Setup
 
@@ -72,30 +79,37 @@ To import existing photos from iCloud sync on indri:
 └─────────────────┘
 ```
 
-## Helm Values
+## Version Management
 
-The Helm chart is configured via `values.yaml`. Key settings:
+Image versions are controlled via `kustomization.yaml`:
 
-- `image.tag`: Immich version (update manually)
+```yaml
-- `immich.persistence.library.existingClaim`: Points to `immich-library` PVC
+images:
-- `machine-learning.enabled`: AI features for face/object recognition
+  - name: ghcr.io/immich-app/immich-server
-- `valkey.enabled`: Redis cache included in chart
+    newTag: v2.6.3
+  - name: ghcr.io/immich-app/immich-machine-learning
+    newTag: v2.6.3
+  - name: docker.io/valkey/valkey
+    newTag: "8.1-alpine"
+```
+
+To upgrade, update `newTag` values and sync via ArgoCD.
 
 ## Troubleshooting
 
 ```bash
 # Check pods
-kubectl -n immich get pods
+kubectl --context=minikube-indri -n immich get pods
 
 # Check immich-pg cluster
-kubectl -n databases get cluster immich-pg
+kubectl --context=minikube-indri -n databases get cluster immich-pg
 
 # View server logs
-kubectl -n immich logs -l app.kubernetes.io/name=immich-server
+kubectl --context=minikube-indri -n immich logs -l app=immich,component=server
 
 # View ML logs
-kubectl -n immich logs -l app.kubernetes.io/name=immich-machine-learning
+kubectl --context=minikube-indri -n immich logs -l app=immich,component=machine-learning
 
 # Check PVC binding
-kubectl -n immich get pvc
+kubectl --context=minikube-indri -n immich get pvc
 ```

argocd/manifests/immich/deployment-ml.yaml

@@ -0,0 +1,60 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: immich-machine-learning
+  namespace: immich
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: immich
+      component: machine-learning
+  template:
+    metadata:
+      labels:
+        app: immich
+        component: machine-learning
+    spec:
+      containers:
+        - name: machine-learning
+          image: ghcr.io/immich-app/immich-machine-learning:kustomized
+          ports:
+            - name: http
+              containerPort: 3003
+          env:
+            - name: TZ
+              value: "America/Los_Angeles"
+            - name: TRANSFORMERS_CACHE
+              value: /cache
+            - name: HF_XET_CACHE
+              value: /cache/huggingface-xet
+            - name: MPLCONFIGDIR
+              value: /cache/matplotlib-config
+          volumeMounts:
+            - name: cache
+              mountPath: /cache
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: 3003
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /ping
+              port: 3003
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            timeoutSeconds: 5
+          resources:
+            requests:
+              memory: "512Mi"
+              cpu: "100m"
+            limits:
+              memory: "4Gi"
+      volumes:
+        - name: cache
+          persistentVolumeClaim:
+            claimName: immich-ml-cache

argocd/manifests/immich/deployment-server.yaml

@@ -0,0 +1,71 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: immich-server
+  namespace: immich
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: immich
+      component: server
+  template:
+    metadata:
+      labels:
+        app: immich
+        component: server
+    spec:
+      containers:
+        - name: server
+          image: ghcr.io/immich-app/immich-server:kustomized
+          ports:
+            - name: http
+              containerPort: 2283
+          env:
+            - name: TZ
+              value: "America/Los_Angeles"
+            - name: DB_HOSTNAME
+              value: "immich-pg-rw.databases.svc.cluster.local"
+            - name: DB_PORT
+              value: "5432"
+            - name: DB_DATABASE_NAME
+              value: "immich"
+            - name: DB_USERNAME
+              value: "immich"
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: immich-db
+                  key: password
+            - name: REDIS_HOSTNAME
+              value: immich-valkey
+            - name: IMMICH_MACHINE_LEARNING_URL
+              value: "http://immich-machine-learning:3003"
+          volumeMounts:
+            - name: library
+              mountPath: /usr/src/app/upload
+          livenessProbe:
+            httpGet:
+              path: /api/server/ping
+              port: 2283
+            initialDelaySeconds: 30
+            periodSeconds: 30
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /api/server/ping
+              port: 2283
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            timeoutSeconds: 5
+          resources:
+            requests:
+              memory: "256Mi"
+              cpu: "100m"
+            limits:
+              memory: "2Gi"
+      volumes:
+        - name: library
+          persistentVolumeClaim:
+            claimName: immich-library

argocd/manifests/immich/deployment-valkey.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: immich-valkey
+  namespace: immich
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: immich
+      component: valkey
+  template:
+    metadata:
+      labels:
+        app: immich
+        component: valkey
+    spec:
+      containers:
+        - name: valkey
+          image: docker.io/valkey/valkey:kustomized
+          ports:
+            - name: redis
+              containerPort: 6379
+          volumeMounts:
+            - name: data
+              mountPath: /data
+          resources:
+            requests:
+              memory: "64Mi"
+              cpu: "25m"
+            limits:
+              memory: "256Mi"
+      volumes:
+        - name: data
+          emptyDir:
+            sizeLimit: 1Gi

argocd/manifests/immich/kustomization.yaml

@@ -1,11 +1,22 @@
-# Immich non-Helm resources (storage)
+---
-# These must be deployed before the Helm chart
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-
 namespace: immich
-
 resources:
+  - deployment-server.yaml
+  - deployment-ml.yaml
+  - deployment-valkey.yaml
+  - service.yaml
+  - service-ml.yaml
+  - service-valkey.yaml
+  - pvc-ml-cache.yaml
   - pv-nfs.yaml
   - pvc.yaml
   - ingress-tailscale.yaml
+images:
+  - name: ghcr.io/immich-app/immich-server
+    newTag: v2.6.3
+  - name: ghcr.io/immich-app/immich-machine-learning
+    newTag: v2.6.3
+  - name: docker.io/valkey/valkey
+    newTag: "8.1-alpine"

argocd/manifests/immich/pvc-ml-cache.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: immich-ml-cache
+  namespace: immich
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi

argocd/manifests/immich/service-ml.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: immich-machine-learning
+  namespace: immich
+spec:
+  selector:
+    app: immich
+    component: machine-learning
+  ports:
+    - name: http
+      port: 3003
+      targetPort: 3003

argocd/manifests/immich/service-valkey.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: immich-valkey
+  namespace: immich
+spec:
+  selector:
+    app: immich
+    component: valkey
+  ports:
+    - name: redis
+      port: 6379
+      targetPort: 6379

argocd/manifests/immich/service.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: immich-server
+  namespace: immich
+spec:
+  selector:
+    app: immich
+    component: server
+  ports:
+    - name: http
+      port: 2283
+      targetPort: 2283

argocd/manifests/immich/values.yaml

@@ -1,91 +0,0 @@
-# Immich Helm values for blumeops
-# Chart: https://github.com/immich-app/immich-charts (v0.10.3)
-#
-# Immich requires:
-# - PostgreSQL with VectorChord extension (separate immich-pg cluster)
-# - Redis/Valkey (included in chart)
-# - Library storage PVC (photos directory from sifaka NFS)
-
-# Shared environment variables
-env:
-  TZ: "America/Los_Angeles"
-
-# Shared controller settings - image tag and DB connection
-controllers:
-  main:
-    containers:
-      main:
-        image:
-          tag: v2.5.6
-        env:
-          DB_HOSTNAME: "immich-pg-rw.databases.svc.cluster.local"
-          DB_PORT: "5432"
-          DB_DATABASE_NAME: "immich"
-          DB_USERNAME: "immich"
-          DB_PASSWORD:
-            valueFrom:
-              secretKeyRef:
-                name: immich-db
-                key: password
-
-# Immich server configuration
-immich:
-  persistence:
-    library:
-      existingClaim: immich-library
-
-# Machine Learning service
-machine-learning:
-  enabled: true
-  controllers:
-    main:
-      containers:
-        main:
-          resources:
-            requests:
-              memory: "512Mi"
-              cpu: "100m"
-            limits:
-              memory: "4Gi"
-          probes:
-            liveness:
-              spec:
-                timeoutSeconds: 5
-            readiness:
-              spec:
-                timeoutSeconds: 5
-  persistence:
-    cache:
-      enabled: true
-      type: persistentVolumeClaim
-      accessMode: ReadWriteOnce
-      size: 10Gi
-
-# Valkey (Redis fork) - included in chart
-valkey:
-  enabled: true
-  persistence:
-    data:
-      enabled: true
-      type: emptyDir
-      size: 1Gi
-
-# Server configuration
-server:
-  controllers:
-    main:
-      containers:
-        main:
-          resources:
-            requests:
-              memory: "256Mi"
-              cpu: "100m"
-            limits:
-              memory: "2Gi"
-          probes:
-            liveness:
-              spec:
-                timeoutSeconds: 5
-            readiness:
-              spec:
-                timeoutSeconds: 5

docs/changelog.d/immich-kustomize-v2.6.3.infra.md

@@ -0,0 +1 @@
+Migrate Immich from Helm chart to kustomize manifests and upgrade from v2.5.6 to v2.6.3

docs/explanation/no-helm-policy.md

@@ -0,0 +1,48 @@
+---
+title: No Helm Policy
+modified: 2026-04-04
+tags:
+  - explanation
+  - kubernetes
+---
+
+# No Helm Policy
+
+> **Note:** This article was drafted by AI and reviewed by Erich. I plan to rewrite all explanatory content in my own words - these serve as placeholders to establish the documentation structure.
+
+BlumeOps avoids Helm charts as a deployment mechanism. Plain kustomize manifests are the standard for all services.
+
+## Rationale
+
+Helm templates add a layer of abstraction that works against the simplicity of Kubernetes YAML manifests. Go templates embedded in YAML are hard to read, hard to diff, and hard to reason about. A manifest should be a manifest — not a program that generates one.
+
+Kustomize overlays preserve the readability of plain YAML while providing the composition and patching features needed for environment-specific configuration. Version bumps are a one-line `newTag` edit in `kustomization.yaml`, and `kubectl diff` shows exactly what will change.
+
+## Current State
+
+All services in blumeops use kustomize manifests except:
+
+- **1Password Connect** — still deployed via Helm chart (`connect-helm-charts v2.3.0`). Migration is a future goal.
+
+## Migration History
+
+Services previously deployed via Helm that have been migrated to kustomize:
+
+| Service | Migrated | Notes |
+|---------|----------|-------|
+| Grafana | 2026-02 | Converted during v12.x upgrade |
+| CloudNative-PG | 2026-02 | Switched to upstream release manifest via forge mirror |
+| External Secrets | 2026-03 | Static manifests rendered from chart |
+| Homepage | 2026-02 | Replaced chart with plain manifests |
+| Immich | 2026-04 | Converted during v2.6.3 upgrade |
+
+## Guidelines
+
+- **Do not introduce new Helm chart dependencies.** When deploying a new service, write kustomize manifests directly — even if the upstream project provides a Helm chart. The chart's `helm template` output is a fine starting point for writing those manifests.
+- **When upgrading a Helm-based service**, consider whether it's a good time to migrate off Helm as part of the upgrade.
+- **Upstream manifests** can be referenced directly in `kustomization.yaml` resources (like ArgoCD and Tailscale operator do) or applied via ArgoCD's `directory.include` (like CloudNative-PG). Both avoid Helm.
+
+## Related
+
+- [[review-services]] — Service review process
+- [[architecture]] — Overall infrastructure design

docs/how-to/knowledgebase/review-services.md

@@ -118,8 +118,13 @@ After reviewing, edit `service-versions.yaml` (repo root) and update the service
 
 Commit this change alongside any upgrades you make during the review.
 
+## Deployment Policy
+
+BlumeOps uses kustomize manifests for all services. Helm charts should not be introduced for new services. See [[no-helm-policy]] for rationale and migration history.
+
 ## Related
 
+- [[no-helm-policy]] - Why blumeops avoids Helm charts
 - [[review-documentation]] - Periodically review documentation cards
 - [[deploy-k8s-service]] - Deploy changes to Kubernetes services
 - [[build-container-image]] - Build and release custom container images

docs/reference/kubernetes/apps.md

@@ -24,9 +24,9 @@ Registry of all applications deployed via [[argocd]].
 | `blumeops-pg` | databases | `argocd/manifests/databases/` | [[postgresql]] |
 | `prometheus` | monitoring | `argocd/manifests/prometheus/` | [[prometheus]] |
 | `loki` | monitoring | `argocd/manifests/loki/` | [[loki]] |
-| `grafana` | monitoring | Helm chart (forge mirror) | [[grafana]] |
+| `grafana` | monitoring | `argocd/manifests/grafana/` | [[grafana]] |
 | `grafana-config` | monitoring | `argocd/manifests/grafana-config/` | [[grafana]] |
-| `immich` | immich | Helm chart | [[immich]] |
+| `immich` | immich | `argocd/manifests/immich/` | [[immich]] |
 | `tempo` | monitoring | `argocd/manifests/tempo/` | [[tempo]] |
 | `alloy-k8s` | alloy | `argocd/manifests/alloy-k8s/` | [[alloy|Alloy]] |
 | `alloy-tracing-ringtail` | alloy | `argocd/manifests/alloy-tracing-ringtail/` | [[alloy|Alloy]] (eBPF tracing) |

docs/reference/services/immich.md

@@ -1,6 +1,6 @@
 ---
 title: Immich
-modified: 2026-02-07
+modified: 2026-04-04
 last-reviewed: 2026-03-23
 tags:
   - service
@@ -17,7 +17,7 @@ Self-hosted photo and video management.
 |----------|-------|
 | **URL** | https://photos.ops.eblu.me |
 | **Namespace** | `immich` |
-| **Deployment** | Helm chart (k8s) |
+| **Deployment** | Kustomize (k8s) |
 | **Database** | [[postgresql]] (CNPG) |
 | **Storage** | [[sifaka|Sifaka]] photos volume |
 

mise-tasks/service-review

@@ -169,7 +169,7 @@ def main(
     if svc_type == "argocd":
         checklist_parts += [
             "\n[bold]ArgoCD Deployment:[/bold]\n",
-            "• Update image tag or Helm chart version in argocd/manifests/\n",
+            "• Update image tag in argocd/manifests/<service>/kustomization.yaml\n",
             f"• Verify sync status: argocd app get {top_svc['name']}\n",
         ]
     elif svc_type == "ansible":

service-versions.yaml

@@ -120,10 +120,10 @@ services:
 
   - name: immich
     type: argocd
-    last-reviewed: 2026-02-25
+    last-reviewed: 2026-04-04
-    current-version: "v2.5.6"
+    current-version: "v2.6.3"
     upstream-source: https://github.com/immich-app/immich/releases
-    notes: Deployed via Helm chart
+    notes: Kustomize manifests with upstream images
 
   - name: external-secrets
     type: argocd