Add k3s, 1Password Connect, and systemd nix-container-builder to ringtail (#209)

## Summary

  Extends ringtail from a desktop/gaming NixOS box into an infrastructure node with a k3s cluster, secrets management, and a Forgejo Actions
  runner for building containers with Nix.

  ### K3s cluster
  - Single-node k3s with Traefik/ServiceLB/metrics-server disabled (minimal footprint)
  - TLS SAN set to `ringtail.tail8d86e.ts.net` so ArgoCD on indri can manage it via Tailscale
  - Containerd registry mirrors pull through Zot on indri (`k3s-registries.yaml`)
  - Tailscale interface added to `trustedInterfaces` for cross-node ArgoCD access
  - `kubectl` added to system packages

  ### 1Password Connect + External Secrets Operator
  - Four new ArgoCD apps targeting `k3s-ringtail`: `1password-connect-ringtail`, `external-secrets-crds-ringtail`, `external-secrets-ringtail`,
  `external-secrets-config-ringtail`
  - Reuses the same Helm charts/values as indri, just pointed at ringtail's k3s API server
  - Bootstrap secrets (`op-credentials`, `onepassword-token`) provisioned by Ansible pre_tasks via `op read`, then applied to the `1password`
  namespace in post_tasks

  ### Systemd Forgejo Actions runner
  - Native `services.gitea-actions-runner` with `forgejo-runner` package — no DinD, no k8s pod, runs directly on the NixOS host
  - Label `nix-container-builder:host` — jobs execute on the host with `nix`, `skopeo`, `nodejs`, etc. in PATH
  - Registration token fetched from 1Password (`Forgejo Secrets/runner_reg`) by Ansible and written to `/etc/forgejo-runner/token.env`
  - Runner's dynamic user (`gitea-runner`) added to `nix.settings.trusted-users` for nix daemon access

  ### Nix container build workflow
  - New `.forgejo/workflows/build-container-nix.yaml` triggers on `*-nix-v[0-9]*` tags (e.g. `nettest-nix-v1.0.0`)
  - Builds with `nix build -f containers/<name>/default.nix`, pushes to Zot via `skopeo copy`
  - Existing Dockerfile workflow guarded with `if: !contains(github.ref_name, '-nix-v')` to avoid double-triggering

  ### Mise task updates
  - `container-tag-and-release` auto-detects `default.nix` vs `Dockerfile` and uses the appropriate tag format (`-nix-v` vs `-v`)
  - `container-list` shows build type indicator (`[nix]` / `[dockerfile]`)

  ## Post-merge

  1. `mise run provision-ringtail` — deploys k3s token, runner token, NixOS rebuild
  2. Register k3s cluster in ArgoCD (first time only):
     ```fish
     ssh ringtail 'sudo cat /etc/rancher/k3s/k3s.yaml' | \
       sed 's|127.0.0.1|ringtail.tail8d86e.ts.net|' > /tmp/k3s-ringtail.yaml
     set -x KUBECONFIG /tmp/k3s-ringtail.yaml
     argocd cluster add default --name k3s-ringtail
  3. Sync ArgoCD apps in order: 1password-connect-ringtail -> external-secrets-crds-ringtail -> external-secrets-ringtail ->
  external-secrets-config-ringtail
  4. Verify runner: ssh ringtail 'systemctl status gitea-runner-nix-container-builder'
  5. Check Forgejo admin panel for ringtail-nix-builder runner online
  6. Test: create containers/<name>/default.nix, tag with <name>-nix-v0.1.0

Reviewed-on: https://forge.ops.eblu.me/eblume/blumeops/pulls/209

mise-tasks/container-list

@@ -10,16 +10,24 @@ echo "Container Images"
 echo "================"
 echo ""
 
-# Find all container directories with Dockerfiles
+# Find all container directories with Dockerfiles or default.nix
 for dir in "$CONTAINER_DIR"/*/; do
     [[ -d "$dir" ]] || continue
-    [[ -f "$dir/Dockerfile" ]] || continue
+
+    # Determine build type
+    if [[ -f "$dir/default.nix" ]]; then
+        build_type="nix"
+    elif [[ -f "$dir/Dockerfile" ]]; then
+        build_type="dockerfile"
+    else
+        continue
+    fi
 
     # Extract container name from directory
     container=$(basename "$dir")
     image="blumeops/$container"
 
-    echo "📦 $container"
+    echo "[$build_type] $container"
     echo "   Image: $REGISTRY/$image"
     echo "   Path: $dir"
 

mise-tasks/container-tag-and-release

@@ -19,28 +19,39 @@ if [[ ! "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
     exit 1
 fi
 
-TAG="${CONTAINER}-${VERSION}"
+# Determine build type: Nix or Dockerfile
+CONTAINER_DIR="containers/${CONTAINER}"
+if [[ -f "$CONTAINER_DIR/default.nix" ]]; then
+    BUILD_TYPE="nix"
+    TAG="${CONTAINER}-nix-${VERSION}"
+elif [[ -f "$CONTAINER_DIR/Dockerfile" ]]; then
+    BUILD_TYPE="dockerfile"
+    TAG="${CONTAINER}-${VERSION}"
+else
+    echo "Error: No Dockerfile or default.nix found in '$CONTAINER_DIR'"
+    echo ""
+    echo "Available containers:"
+    for dir in containers/*/; do
+        [[ -d "$dir" ]] || continue
+        name=$(basename "$dir")
+        if [[ -f "$dir/default.nix" ]]; then
+            echo "  - $name (nix)"
+        elif [[ -f "$dir/Dockerfile" ]]; then
+            echo "  - $name (dockerfile)"
+        fi
+    done
+    exit 1
+fi
 
 echo "Creating release tag: $TAG"
+echo "Build type: $BUILD_TYPE"
 echo ""
 
 # Check if tag already exists
 if git rev-parse "$TAG" >/dev/null 2>&1; then
     echo "Error: Tag '$TAG' already exists"
     echo "Existing tags for $CONTAINER:"
-    git tag -l "${CONTAINER}-v*" | sort -V | tail -5
-    exit 1
-fi
-
-# Check if container directory exists
-CONTAINER_DIR="containers/${CONTAINER}"
-if [[ ! -f "$CONTAINER_DIR/Dockerfile" ]]; then
-    echo "Error: No Dockerfile found at '$CONTAINER_DIR/Dockerfile'"
-    echo ""
-    echo "Available containers:"
-    for dir in containers/*/; do
-        [[ -d "$dir" ]] && echo "  - $(basename "$dir")"
-    done
+    git tag -l "${CONTAINER}-*v*" | sort -V | tail -5
     exit 1
 fi
 

mise-tasks/ensure-k3s-ringtail-kubectl-config

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+#MISE description="Ensure kubectl config for k3s-ringtail is set up on this workstation"
+
+set -euo pipefail
+
+CONFIG_DIR="$HOME/.kube/k3s-ringtail"
+CONFIG_FILE="$CONFIG_DIR/config.yml"
+
+echo "Ensuring k3s-ringtail kubectl config..."
+
+# Create directory if needed
+mkdir -p "$CONFIG_DIR"
+
+# Fetch kubeconfig from ringtail and extract the CA cert
+echo "Fetching kubeconfig from ringtail..."
+RAW_CONFIG=$(ssh ringtail 'sudo cat /etc/rancher/k3s/k3s.yaml')
+
+# Extract and decode the CA certificate
+echo "$RAW_CONFIG" | grep certificate-authority-data | awk '{print $2}' | base64 -d > "$CONFIG_DIR/ca.crt"
+
+# Extract and decode the client certificate
+echo "$RAW_CONFIG" | grep client-certificate-data | awk '{print $2}' | base64 -d > "$CONFIG_DIR/client.crt"
+
+# Extract and decode the client key
+echo "$RAW_CONFIG" | grep client-key-data | awk '{print $2}' | base64 -d > "$CONFIG_DIR/client.key"
+chmod 600 "$CONFIG_DIR/client.key"
+
+# Write kubeconfig with file-based certs and tailscale hostname
+cat > "$CONFIG_FILE" << EOF
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    certificate-authority: $CONFIG_DIR/ca.crt
+    server: https://ringtail.tail8d86e.ts.net:6443
+  name: k3s-ringtail
+contexts:
+- context:
+    cluster: k3s-ringtail
+    user: k3s-ringtail
+  name: k3s-ringtail
+current-context: k3s-ringtail
+users:
+- name: k3s-ringtail
+  user:
+    client-certificate: $CONFIG_DIR/client.crt
+    client-key: $CONFIG_DIR/client.key
+EOF
+
+echo "Config written to $CONFIG_FILE"
+
+# Warn if KUBECONFIG doesn't include this file
+if [[ -z "${KUBECONFIG:-}" ]] || [[ ":$KUBECONFIG:" != *":$CONFIG_FILE:"* ]]; then
+    echo ""
+    echo "WARNING: KUBECONFIG does not include $CONFIG_FILE"
+    echo "Add this to your shell config:"
+    echo "  export KUBECONFIG=\"\$KUBECONFIG:$CONFIG_FILE\""
+fi
+
+echo ""
+echo "Test with: kubectl --context=k3s-ringtail get nodes"

mise-tasks/services-check

@@ -87,6 +87,9 @@ echo ""
 echo "Ringtail (NixOS):"
 check_service "ssh" "ssh -o ConnectTimeout=5 ringtail true"
 check_service "tailscale" "ssh ringtail 'tailscale status --self --json' | jq -e '.Self.Online' > /dev/null"
+check_service "k3s" "ssh ringtail 'k3s kubectl get nodes --no-headers | grep -q Ready'"
+check_service "k3s-apiserver (remote)" "kubectl --context=k3s-ringtail get --raw /healthz"
+check_service "forgejo-runner" "ssh ringtail 'systemctl is-active gitea-runner-nix_container_builder.service'"
 
 echo ""
 echo "Public services (via Fly.io):"