From 1897eb1c5bf4ef1f6d3dfe3601f875b49b8ba2a4 Mon Sep 17 00:00:00 2001
From: Erich Blume <blume.erich@gmail.com>
Date: Sun, 17 May 2026 08:46:22 -0700
Subject: [PATCH] C0: move immich blackbox probe to ringtail alloy

Immich migrated to ringtail's k3s cluster but the probe still targeted
the in-cluster service DNS on indri's minikube, firing ServiceProbeFailure
indefinitely. Moved the target into alloy-ringtail's config so the probe
runs in the cluster where immich actually lives.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 argocd/manifests/alloy-k8s/config.alloy       |  6 ------
 argocd/manifests/alloy-ringtail/config.alloy  | 20 +++++++++++++++++++
 .../+immich-probe-ringtail.infra.md           |  1 +
 3 files changed, 21 insertions(+), 6 deletions(-)
 create mode 100644 docs/changelog.d/+immich-probe-ringtail.infra.md

diff --git a/argocd/manifests/alloy-k8s/config.alloy b/argocd/manifests/alloy-k8s/config.alloy
index 56a2e13..5a0a8f9 100644
--- a/argocd/manifests/alloy-k8s/config.alloy
+++ b/argocd/manifests/alloy-k8s/config.alloy
@@ -196,12 +196,6 @@ prometheus.exporter.blackbox "services" {
     module  = "http_2xx"
   }
 
-  target {
-    name    = "immich"
-    address = "http://immich-server.immich.svc.cluster.local:2283/api/server/ping"
-    module  = "http_2xx"
-  }
-
   target {
     name    = "navidrome"
     address = "http://navidrome.navidrome.svc.cluster.local:4533/"
diff --git a/argocd/manifests/alloy-ringtail/config.alloy b/argocd/manifests/alloy-ringtail/config.alloy
index e92ab0f..e5cc045 100644
--- a/argocd/manifests/alloy-ringtail/config.alloy
+++ b/argocd/manifests/alloy-ringtail/config.alloy
@@ -45,6 +45,26 @@ prometheus.scrape "kube_state_metrics" {
   forward_to      = [prometheus.remote_write.prometheus.receiver]
 }
 
+// ============== SERVICE HEALTH PROBES ==============
+
+// Blackbox-style HTTP probes for in-cluster services on ringtail
+prometheus.exporter.blackbox "services" {
+  config = "{ modules: { http_2xx: { prober: http, timeout: 5s } } }"
+
+  target {
+    name    = "immich"
+    address = "http://immich-server.immich.svc.cluster.local:2283/api/server/ping"
+    module  = "http_2xx"
+  }
+}
+
+// Scrape blackbox probe results
+prometheus.scrape "blackbox" {
+  targets         = prometheus.exporter.blackbox.services.targets
+  scrape_interval = "30s"
+  forward_to      = [prometheus.remote_write.prometheus.receiver]
+}
+
 // Push metrics to indri Prometheus
 prometheus.remote_write "prometheus" {
   external_labels = { cluster = "ringtail" }
diff --git a/docs/changelog.d/+immich-probe-ringtail.infra.md b/docs/changelog.d/+immich-probe-ringtail.infra.md
new file mode 100644
index 0000000..f2d3dee
--- /dev/null
+++ b/docs/changelog.d/+immich-probe-ringtail.infra.md
@@ -0,0 +1 @@
+Moved the Immich blackbox health probe from indri's alloy to ringtail's alloy. After the immich migration to ringtail, the probe still targeted `immich-server.immich.svc.cluster.local` on indri's cluster where the service no longer exists, causing a persistent `ServiceProbeFailure` alert.