eblume/blumeops
1
1
Fork 0
Code Issues Pull requests Projects Releases 83 Packages Wiki Activity Actions
blumeops/mise-tasks/services-check
Erich Blume 918df9e642 Add k3s, 1Password Connect, and systemd nix-container-builder to ringtail (#209) 
## Summary

  Extends ringtail from a desktop/gaming NixOS box into an infrastructure node with a k3s cluster, secrets management, and a Forgejo Actions
  runner for building containers with Nix.

  ### K3s cluster
  - Single-node k3s with Traefik/ServiceLB/metrics-server disabled (minimal footprint)
  - TLS SAN set to `ringtail.tail8d86e.ts.net` so ArgoCD on indri can manage it via Tailscale
  - Containerd registry mirrors pull through Zot on indri (`k3s-registries.yaml`)
  - Tailscale interface added to `trustedInterfaces` for cross-node ArgoCD access
  - `kubectl` added to system packages

  ### 1Password Connect + External Secrets Operator
  - Four new ArgoCD apps targeting `k3s-ringtail`: `1password-connect-ringtail`, `external-secrets-crds-ringtail`, `external-secrets-ringtail`,
  `external-secrets-config-ringtail`
  - Reuses the same Helm charts/values as indri, just pointed at ringtail's k3s API server
  - Bootstrap secrets (`op-credentials`, `onepassword-token`) provisioned by Ansible pre_tasks via `op read`, then applied to the `1password`
  namespace in post_tasks

  ### Systemd Forgejo Actions runner
  - Native `services.gitea-actions-runner` with `forgejo-runner` package — no DinD, no k8s pod, runs directly on the NixOS host
  - Label `nix-container-builder:host` — jobs execute on the host with `nix`, `skopeo`, `nodejs`, etc. in PATH
  - Registration token fetched from 1Password (`Forgejo Secrets/runner_reg`) by Ansible and written to `/etc/forgejo-runner/token.env`
  - Runner's dynamic user (`gitea-runner`) added to `nix.settings.trusted-users` for nix daemon access

  ### Nix container build workflow
  - New `.forgejo/workflows/build-container-nix.yaml` triggers on `*-nix-v[0-9]*` tags (e.g. `nettest-nix-v1.0.0`)
  - Builds with `nix build -f containers/<name>/default.nix`, pushes to Zot via `skopeo copy`
  - Existing Dockerfile workflow guarded with `if: !contains(github.ref_name, '-nix-v')` to avoid double-triggering

  ### Mise task updates
  - `container-tag-and-release` auto-detects `default.nix` vs `Dockerfile` and uses the appropriate tag format (`-nix-v` vs `-v`)
  - `container-list` shows build type indicator (`[nix]` / `[dockerfile]`)

  ## Post-merge

  1. `mise run provision-ringtail` — deploys k3s token, runner token, NixOS rebuild
  2. Register k3s cluster in ArgoCD (first time only):
     ```fish
     ssh ringtail 'sudo cat /etc/rancher/k3s/k3s.yaml' | \
       sed 's|127.0.0.1|ringtail.tail8d86e.ts.net|' > /tmp/k3s-ringtail.yaml
     set -x KUBECONFIG /tmp/k3s-ringtail.yaml
     argocd cluster add default --name k3s-ringtail
  3. Sync ArgoCD apps in order: 1password-connect-ringtail -> external-secrets-crds-ringtail -> external-secrets-ringtail ->
  external-secrets-config-ringtail
  4. Verify runner: ssh ringtail 'systemctl status gitea-runner-nix-container-builder'
  5. Check Forgejo admin panel for ringtail-nix-builder runner online
  6. Test: create containers/<name>/default.nix, tag with <name>-nix-v0.1.0

Reviewed-on: https://forge.ops.eblu.me/eblume/blumeops/pulls/209
2026-02-18 21:15:30 -08:00

138 lines
6.5 KiB
Bash
Executable file
Raw Blame History

#!/usr/bin/env bash
#MISE description="Check that all services are online and responding"
set -euo pipefail
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
NC='\033[0m' # No Color
FAILED=0
check_service() {
local name="$1"
local check_cmd="$2"
printf "%-24s " "$name..."
if eval "$check_cmd" > /dev/null 2>&1; then
echo -e "${GREEN}OK${NC}"
else
echo -e "${RED}FAILED${NC}"
FAILED=1
fi
}
check_http() {
local name="$1"
local url="$2"
printf "%-24s " "$name..."
if curl -sf --max-time 5 "$url" > /dev/null 2>&1; then
echo -e "${GREEN}OK${NC}"
else
echo -e "${RED}FAILED${NC}"
FAILED=1
fi
}
echo "Checking services..."
echo "===================="
echo ""
# Local services on indri
echo "Local services on indri:"
check_service "forgejo (brew)" "ssh indri 'brew services list | grep forgejo | grep started'"
check_service "alloy" "ssh indri 'launchctl list mcquack.eblume.alloy | grep -v \"^-\"'"
check_service "borgmatic" "ssh indri 'launchctl list mcquack.eblume.borgmatic | grep -v \"^-\"'"
check_service "borgmatic-metrics" "ssh indri 'launchctl list mcquack.borgmatic-metrics | grep -v \"^-\"'"
check_service "zot" "ssh indri 'launchctl list mcquack.eblume.zot | grep -v \"^-\"'"
check_service "zot-metrics" "ssh indri 'launchctl list mcquack.zot-metrics | grep -v \"^-\"'"
check_service "minikube-metrics" "ssh indri 'launchctl list mcquack.minikube-metrics | grep -v \"^-\"'"
check_service "jellyfin-metrics" "ssh indri 'launchctl list mcquack.eblume.jellyfin-metrics | grep -v \"^-\"'"
echo ""
echo "Metrics textfiles:"
check_service "borgmatic.prom" "ssh indri 'test -f /opt/homebrew/var/node_exporter/textfile/borgmatic.prom'"
check_service "zot.prom" "ssh indri 'test -f /opt/homebrew/var/node_exporter/textfile/zot.prom'"
check_service "minikube.prom" "ssh indri 'test -f /opt/homebrew/var/node_exporter/textfile/minikube.prom'"
check_service "jellyfin.prom" "ssh indri 'test -f /opt/homebrew/var/node_exporter/textfile/jellyfin.prom'"
echo ""
echo "Kubernetes cluster:"
check_service "minikube" "ssh indri 'minikube status --format={{.Host}} | grep -q Running'"
check_service "k8s-apiserver (indri)" "ssh indri 'kubectl get --raw /healthz'"
check_service "k8s-apiserver (remote)" "kubectl --kubeconfig=$HOME/.kube/minikube-indri/config.yml --context=minikube-indri get --raw /healthz"
echo ""
echo "HTTP endpoints (via Caddy):"
check_http "Prometheus" "https://prometheus.ops.eblu.me/-/healthy"
check_http "Loki" "https://loki.ops.eblu.me/ready"
check_http "Grafana" "https://grafana.ops.eblu.me/api/health"
check_http "ArgoCD" "https://argocd.ops.eblu.me/healthz"
check_http "Forgejo" "https://forge.ops.eblu.me/"
check_http "Zot Registry" "https://registry.ops.eblu.me/v2/_catalog"
check_http "Kiwix" "https://kiwix.ops.eblu.me/"
check_http "Miniflux" "https://feed.ops.eblu.me/healthcheck"
check_http "TeslaMate" "https://tesla.ops.eblu.me/"
check_http "Devpi" "https://pypi.ops.eblu.me/+api"
check_http "Transmission" "https://torrent.ops.eblu.me/"
check_http "Immich" "https://photos.ops.eblu.me/"
check_http "Navidrome" "https://dj.ops.eblu.me/"
check_http "CV" "https://cv.ops.eblu.me/"
check_http "Ntfy" "https://ntfy.ops.eblu.me/v1/health"
check_http "Frigate" "https://nvr.ops.eblu.me/api/version"
echo ""
echo "Ringtail (NixOS):"
check_service "ssh" "ssh -o ConnectTimeout=5 ringtail true"
check_service "tailscale" "ssh ringtail 'tailscale status --self --json' | jq -e '.Self.Online' > /dev/null"
check_service "k3s" "ssh ringtail 'k3s kubectl get nodes --no-headers | grep -q Ready'"
check_service "k3s-apiserver (remote)" "kubectl --context=k3s-ringtail get --raw /healthz"
check_service "forgejo-runner" "ssh ringtail 'systemctl is-active gitea-runner-nix_container_builder.service'"
echo ""
echo "Public services (via Fly.io):"
check_http "Docs (public)" "https://docs.eblu.me/"
check_http "CV (public)" "https://cv.eblu.me/"
check_http "Fly.io healthz" "https://blumeops-proxy.fly.dev/healthz"
echo ""
echo "Database:"
check_service "PostgreSQL (k8s)" "pg_isready -h pg.ops.eblu.me -p 5432"
echo ""
echo "Kubernetes pods:"
check_service "prometheus-0" "kubectl --context=minikube-indri -n monitoring get pod prometheus-0 -o jsonpath='{.status.phase}' | grep -q Running"
check_service "loki-0" "kubectl --context=minikube-indri -n monitoring get pod loki-0 -o jsonpath='{.status.phase}' | grep -q Running"
check_service "grafana" "kubectl --context=minikube-indri -n monitoring get pods -l app.kubernetes.io/name=grafana -o jsonpath='{.items[0].status.phase}' | grep -q Running"
check_service "miniflux" "kubectl --context=minikube-indri -n miniflux get pods -l app=miniflux -o jsonpath='{.items[0].status.phase}' | grep -q Running"
check_service "teslamate" "kubectl --context=minikube-indri -n teslamate get pods -l app=teslamate -o jsonpath='{.items[0].status.phase}' | grep -q Running"
check_service "blumeops-pg" "kubectl --context=minikube-indri -n databases get pods -l cnpg.io/cluster=blumeops-pg -o jsonpath='{.items[0].status.phase}' | grep -q Running"
check_service "mosquitto" "kubectl --context=minikube-indri -n mqtt get pods -l app=mosquitto -o jsonpath='{.items[0].status.phase}' | grep -q Running"
check_service "ntfy" "kubectl --context=minikube-indri -n ntfy get pods -l app=ntfy -o jsonpath='{.items[0].status.phase}' | grep -q Running"
check_service "frigate" "kubectl --context=minikube-indri -n frigate get pods -l app=frigate -o jsonpath='{.items[0].status.phase}' | grep -q Running"
check_service "frigate-notify" "kubectl --context=minikube-indri -n frigate get pods -l app=frigate-notify -o jsonpath='{.items[0].status.phase}' | grep -q Running"
echo ""
echo "ArgoCD app sync status:"
printf "%-20s %-12s %-12s %s\n" "NAME" "SYNC" "HEALTH" "TARGET"
while read -r name sync health target; do
if [[ "$sync" == "Synced" ]]; then
printf "%-20s ${GREEN}%-12s${NC} %-12s %s\n" "$name" "$sync" "$health" "$target"
elif [[ "$sync" == "OutOfSync" ]]; then
printf "%-20s ${RED}%-12s${NC} %-12s %s\n" "$name" "$sync" "$health" "$target"
FAILED=1
else
printf "%-20s %-12s %-12s %s\n" "$name" "$sync" "$health" "$target"
fi
done < <(kubectl --context=minikube-indri get applications -n argocd --no-headers -o custom-columns='NAME:.metadata.name,SYNC:.status.sync.status,HEALTH:.status.health.status,TARGET:.spec.source.targetRevision' 2>/dev/null)
echo ""
if [ $FAILED -eq 0 ]; then
echo -e "${GREEN}All services healthy!${NC}"
exit 0
else
echo -e "${RED}Some services failed health check${NC}"
exit 1
fi
Reference in a new issue View git blame Copy permalink