.forgejo/workflows/build-container-nix.yaml

@@ -0,0 +1,89 @@
+# Nix container build workflow
+# Triggers on tags matching: <container>-nix-v<version>
+# Builds from containers/<container>/default.nix using nix build
+# Pushes to Zot registry via skopeo
+#
+# Examples:
+#   nettest-nix-v1.0.0  -> builds containers/nettest/default.nix
+#   myapp-nix-v2.1.0    -> builds containers/myapp/default.nix
+name: Build Container (Nix)
+
+on:
+  push:
+    tags:
+      - '*-nix-v[0-9]*'
+
+jobs:
+  build:
+    runs-on: nix-container-builder
+    steps:
+      - name: Parse tag
+        id: parse
+        run: |
+          TAG="${GITHUB_REF_NAME}"
+          echo "Tag: $TAG"
+
+          # Extract container name (everything before -nix-v)
+          # e.g., "nettest-nix-v1.0.0" -> "nettest"
+          CONTAINER="${TAG%-nix-v[0-9]*}"
+          VERSION="${TAG#"${CONTAINER}"-nix-}"
+
+          echo "container=$CONTAINER" >> "$GITHUB_OUTPUT"
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "Container: $CONTAINER"
+          echo "Version: $VERSION"
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Check if nix container exists
+        id: check
+        run: |
+          CONTAINER="${{ steps.parse.outputs.container }}"
+          CONTEXT="containers/$CONTAINER"
+
+          if [ -f "$CONTEXT/default.nix" ]; then
+            echo "Found $CONTEXT/default.nix"
+            echo "exists=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "No default.nix found at $CONTEXT/default.nix"
+            echo "exists=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Skip if container not found
+        if: steps.check.outputs.exists != 'true'
+        run: |
+          echo "========================================"
+          echo "Nix container not found: ${{ steps.parse.outputs.container }}"
+          echo "========================================"
+          echo ""
+          echo "Tag '${{ github.ref_name }}' does not match any nix container in containers/"
+          echo ""
+          echo "Available nix containers:"
+          for nix in containers/*/default.nix; do
+            [ -f "$nix" ] && echo "  - $(basename "$(dirname "$nix")")"
+          done
+          echo ""
+          echo "Skipping build."
+
+      - name: Build with nix
+        if: steps.check.outputs.exists == 'true'
+        id: build
+        run: |
+          CONTAINER="${{ steps.parse.outputs.container }}"
+          echo "Building containers/$CONTAINER/default.nix"
+          nix build -f "containers/$CONTAINER/default.nix" -o result
+          echo "Build complete: $(readlink result)"
+
+      - name: Push to registry
+        if: steps.check.outputs.exists == 'true'
+        run: |
+          CONTAINER="${{ steps.parse.outputs.container }}"
+          VERSION="${{ steps.parse.outputs.version }}"
+          IMAGE="registry.ops.eblu.me/blumeops/$CONTAINER:$VERSION"
+
+          echo "Pushing to $IMAGE"
+          skopeo copy \
+            "docker-archive:result" \
+            "docker://$IMAGE"
+          echo "Push complete: $IMAGE"

.forgejo/workflows/build-container.yaml

@@ -17,6 +17,7 @@ on:
 
 jobs:
   build:
+    if: "!contains(github.ref_name, '-nix-v')"
     runs-on: k8s
     steps:
       - name: Parse tag

.github/actionlint.yaml

@@ -1,3 +1,4 @@
 self-hosted-runner:
   labels:
     - k8s
+    - nix-container-builder

ansible/playbooks/ringtail.yml

@@ -3,6 +3,57 @@
   hosts: ringtail
   become: true
 
+  pre_tasks:
+    - name: Fetch 1Password Connect credentials from 1Password
+      ansible.builtin.command:
+        cmd: op read "op://vg6xf6vvfmoh5hqjjhlhbeoaie/1Password Connect/credentials-file"
+      register: _op_credentials
+      changed_when: false
+      delegate_to: localhost
+      become: false
+
+    - name: Fetch 1Password Connect token from 1Password
+      ansible.builtin.command:
+        cmd: op read "op://vg6xf6vvfmoh5hqjjhlhbeoaie/1Password Connect/token"
+      register: _op_token
+      changed_when: false
+      delegate_to: localhost
+      become: false
+
+    - name: Fetch Forgejo runner registration token from 1Password
+      ansible.builtin.command:
+        cmd: op read "op://vg6xf6vvfmoh5hqjjhlhbeoaie/Forgejo Secrets/runner_reg"
+      register: _runner_reg
+      changed_when: false
+      delegate_to: localhost
+      become: false
+
+    - name: Ensure /etc/forgejo-runner directory exists
+      ansible.builtin.file:
+        path: /etc/forgejo-runner
+        state: directory
+        mode: "0700"
+
+    - name: Write Forgejo runner token file
+      ansible.builtin.copy:
+        content: "TOKEN={{ _runner_reg.stdout }}"
+        dest: /etc/forgejo-runner/token.env
+        mode: "0600"
+      no_log: true
+
+    - name: Ensure /etc/k3s directory exists
+      ansible.builtin.file:
+        path: /etc/k3s
+        state: directory
+        mode: "0700"
+
+    - name: Generate k3s token if not present
+      ansible.builtin.copy:
+        content: "{{ lookup('ansible.builtin.password', '/dev/null', chars=['hexdigits'], length=32) }}"
+        dest: /etc/k3s/token
+        mode: "0600"
+        force: false
+
   tasks:
     - name: Ensure blumeops repo is present
       ansible.builtin.git:
@@ -24,3 +75,42 @@
       register: _ts_status
       changed_when: false
       failed_when: "'Running' not in _ts_status.stdout"
+
+  post_tasks:
+    - name: Wait for k3s to be ready
+      ansible.builtin.command: k3s kubectl get nodes
+      register: _k3s_ready
+      changed_when: false
+      retries: 30
+      delay: 5
+      until: _k3s_ready.rc == 0
+
+    - name: Create 1password namespace
+      ansible.builtin.command: k3s kubectl create namespace 1password
+      register: _ns
+      changed_when: _ns.rc == 0
+      failed_when: _ns.rc != 0 and 'AlreadyExists' not in _ns.stderr
+
+    - name: Create or update op-credentials secret
+      ansible.builtin.shell:
+        cmd: |
+          set -o pipefail
+          k3s kubectl create secret generic op-credentials \
+            --namespace=1password \
+            --from-literal=1password-credentials.json='{{ _op_credentials.stdout }}' \
+            --dry-run=client -o yaml | k3s kubectl apply -f -
+        executable: /run/current-system/sw/bin/bash
+      changed_when: true
+      no_log: true
+
+    - name: Create or update onepassword-token secret
+      ansible.builtin.shell:
+        cmd: |
+          set -o pipefail
+          k3s kubectl create secret generic onepassword-token \
+            --namespace=1password \
+            --from-literal=token={{ _op_token.stdout }} \
+            --dry-run=client -o yaml | k3s kubectl apply -f -
+        executable: /run/current-system/sw/bin/bash
+      changed_when: true
+      no_log: true

argocd/apps/1password-connect-ringtail.yaml

@@ -0,0 +1,32 @@
+# 1Password Connect for ringtail k3s cluster
+# Same chart/values as indri, different destination
+#
+# Prerequisites:
+# 1. Bootstrap secrets via ansible (provision-ringtail creates 1password namespace,
+#    op-credentials and onepassword-token secrets)
+# 2. Sync BEFORE external-secrets-ringtail
+#
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: 1password-connect-ringtail
+  namespace: argocd
+spec:
+  project: default
+  sources:
+    - repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/connect-helm-charts.git
+      targetRevision: connect-2.3.0
+      path: charts/connect
+      helm:
+        releaseName: onepassword-connect
+        valueFiles:
+          - $values/argocd/manifests/1password-connect/values.yaml
+    - repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/blumeops.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://ringtail.tail8d86e.ts.net:6443
+    namespace: 1password
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=true

argocd/apps/external-secrets-config-ringtail.yaml

@@ -0,0 +1,24 @@
+# External Secrets Configuration for ringtail k3s cluster
+# Same ClusterSecretStore manifests as indri, different destination
+#
+# Prerequisites:
+# - 1password-connect-ringtail is deployed and healthy
+# - external-secrets-ringtail operator is deployed and CRDs are installed
+#
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: external-secrets-config-ringtail
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/blumeops.git
+    targetRevision: main
+    path: argocd/manifests/external-secrets
+  destination:
+    server: https://ringtail.tail8d86e.ts.net:6443
+    namespace: external-secrets
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=true

argocd/apps/external-secrets-crds-ringtail.yaml

@@ -0,0 +1,24 @@
+# External Secrets Operator CRDs for ringtail k3s cluster
+# Same CRDs source as indri, different destination
+#
+# Must be synced BEFORE external-secrets-ringtail operator app.
+#
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: external-secrets-crds-ringtail
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/external-secrets.git
+    targetRevision: helm-chart-2.0.0
+    path: config/crds/bases
+    directory:
+      exclude: 'kustomization.yaml'
+  destination:
+    server: https://ringtail.tail8d86e.ts.net:6443
+  syncPolicy:
+    syncOptions:
+      - ServerSideApply=true
+      - CreateNamespace=false

argocd/apps/external-secrets-ringtail.yaml

@@ -0,0 +1,32 @@
+# External Secrets Operator for ringtail k3s cluster
+# Same chart/values as indri, different destination
+#
+# Prerequisites:
+# - 1password-connect-ringtail must be deployed and healthy
+# - external-secrets-crds-ringtail must be synced first
+#
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: external-secrets-ringtail
+  namespace: argocd
+spec:
+  project: default
+  sources:
+    - repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/external-secrets.git
+      targetRevision: helm-chart-2.0.0
+      path: deploy/charts/external-secrets
+      helm:
+        releaseName: external-secrets
+        valueFiles:
+          - $values/argocd/manifests/external-secrets/values.yaml
+    - repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/blumeops.git
+      targetRevision: main
+      ref: values
+  destination:
+    server: https://ringtail.tail8d86e.ts.net:6443
+    namespace: external-secrets
+  syncPolicy:
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true

docs/changelog.d/feature-k3s-ringtail-runner.feature.md

@@ -0,0 +1 @@
+Systemd Forgejo Actions runner on ringtail (`nix-container-builder` label) for building containers with `nix build` and pushing via `skopeo`. K3s cluster retained for future workloads. 1Password Connect + External Secrets Operator available for k8s secret management.

docs/reference/infrastructure/ringtail.md

@@ -45,6 +45,51 @@ mise run provision-ringtail
 
 This updates `flake.lock` via Dagger, verifies the current commit is pushed to forge, then deploys the exact commit via ansible. If the lockfile changed, it stages the file and exits so you can commit and re-run.
 
+## K3s Cluster
+
+Ringtail runs a single-node k3s cluster for native amd64 workloads, registered in [[argocd|ArgoCD]] on indri as `k3s-ringtail`.
+
+- **Disabled components:** Traefik, ServiceLB, metrics-server (minimal footprint)
+- **TLS SAN:** `ringtail.tail8d86e.ts.net` (ArgoCD connects via Tailscale)
+- **Registry mirrors:** Containerd pulls through Zot on indri (`registry.ops.eblu.me`)
+- **Token:** `/etc/k3s/token` (generated on first provision)
+- **Kubeconfig:** `/etc/rancher/k3s/k3s.yaml` (world-readable via `--write-kubeconfig-mode=644`)
+
+### Secrets Management
+
+1Password Connect + External Secrets Operator syncs secrets from 1Password to k8s, matching the [[1password|indri pattern]]. Bootstrap credentials (`op-credentials`, `onepassword-token`) are provisioned by Ansible; ArgoCD manages the operator stack.
+
+Sync order: `1password-connect-ringtail` -> `external-secrets-crds-ringtail` -> `external-secrets-ringtail` -> `external-secrets-config-ringtail`
+
+### Workloads
+
+No k8s workloads currently deployed. K3s is available for future workloads (e.g. Frigate, running nix-built containers).
+
+### Manual Cluster Registration
+
+After first provision, register the cluster in ArgoCD:
+
+```fish
+ssh ringtail 'sudo cat /etc/rancher/k3s/k3s.yaml' | \
+  sed 's|127.0.0.1|ringtail.tail8d86e.ts.net|' > /tmp/k3s-ringtail.yaml
+set -x KUBECONFIG /tmp/k3s-ringtail.yaml
+kubectl get nodes  # verify access
+argocd cluster add default --name k3s-ringtail
+```
+
+## Systemd Services
+
+### Forgejo Actions Runner
+
+A native Forgejo Actions runner (`ringtail-nix-builder`) runs as a systemd service via the NixOS `services.gitea-actions-runner` module. It builds containers using `nix build` and pushes them to Zot via `skopeo`.
+
+| Property | Value |
+|----------|-------|
+| **Label** | `nix-container-builder` |
+| **Execution** | Host (no containers) |
+| **Token** | `/etc/forgejo-runner/token.env` (provisioned by Ansible) |
+| **Service unit** | `gitea-runner-nix_container_builder.service` |
+
 ## Maintenance Notes
 
 **1Password:** Desktop app must be running for `op` CLI. Use `$mod+Shift+minus` to send to scratchpad.

mise-tasks/container-list

@@ -10,16 +10,24 @@ echo "Container Images"
 echo "================"
 echo ""
 
-# Find all container directories with Dockerfiles
+# Find all container directories with Dockerfiles or default.nix
 for dir in "$CONTAINER_DIR"/*/; do
     [[ -d "$dir" ]] || continue
-    [[ -f "$dir/Dockerfile" ]] || continue
+
+    # Determine build type
+    if [[ -f "$dir/default.nix" ]]; then
+        build_type="nix"
+    elif [[ -f "$dir/Dockerfile" ]]; then
+        build_type="dockerfile"
+    else
+        continue
+    fi
 
     # Extract container name from directory
     container=$(basename "$dir")
     image="blumeops/$container"
 
-    echo "📦 $container"
+    echo "[$build_type] $container"
     echo "   Image: $REGISTRY/$image"
     echo "   Path: $dir"
 

mise-tasks/container-tag-and-release

@@ -19,28 +19,39 @@ if [[ ! "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
     exit 1
 fi
 
+# Determine build type: Nix or Dockerfile
+CONTAINER_DIR="containers/${CONTAINER}"
+if [[ -f "$CONTAINER_DIR/default.nix" ]]; then
+    BUILD_TYPE="nix"
+    TAG="${CONTAINER}-nix-${VERSION}"
+elif [[ -f "$CONTAINER_DIR/Dockerfile" ]]; then
+    BUILD_TYPE="dockerfile"
     TAG="${CONTAINER}-${VERSION}"
+else
+    echo "Error: No Dockerfile or default.nix found in '$CONTAINER_DIR'"
+    echo ""
+    echo "Available containers:"
+    for dir in containers/*/; do
+        [[ -d "$dir" ]] || continue
+        name=$(basename "$dir")
+        if [[ -f "$dir/default.nix" ]]; then
+            echo "  - $name (nix)"
+        elif [[ -f "$dir/Dockerfile" ]]; then
+            echo "  - $name (dockerfile)"
+        fi
+    done
+    exit 1
+fi
 
 echo "Creating release tag: $TAG"
+echo "Build type: $BUILD_TYPE"
 echo ""
 
 # Check if tag already exists
 if git rev-parse "$TAG" >/dev/null 2>&1; then
     echo "Error: Tag '$TAG' already exists"
     echo "Existing tags for $CONTAINER:"
-    git tag -l "${CONTAINER}-v*" | sort -V | tail -5
-    exit 1
-fi
-
-# Check if container directory exists
-CONTAINER_DIR="containers/${CONTAINER}"
-if [[ ! -f "$CONTAINER_DIR/Dockerfile" ]]; then
-    echo "Error: No Dockerfile found at '$CONTAINER_DIR/Dockerfile'"
-    echo ""
-    echo "Available containers:"
-    for dir in containers/*/; do
-        [[ -d "$dir" ]] && echo "  - $(basename "$dir")"
-    done
+    git tag -l "${CONTAINER}-*v*" | sort -V | tail -5
     exit 1
 fi
 

mise-tasks/ensure-k3s-ringtail-kubectl-config

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+#MISE description="Ensure kubectl config for k3s-ringtail is set up on this workstation"
+
+set -euo pipefail
+
+CONFIG_DIR="$HOME/.kube/k3s-ringtail"
+CONFIG_FILE="$CONFIG_DIR/config.yml"
+
+echo "Ensuring k3s-ringtail kubectl config..."
+
+# Create directory if needed
+mkdir -p "$CONFIG_DIR"
+
+# Fetch kubeconfig from ringtail and extract the CA cert
+echo "Fetching kubeconfig from ringtail..."
+RAW_CONFIG=$(ssh ringtail 'sudo cat /etc/rancher/k3s/k3s.yaml')
+
+# Extract and decode the CA certificate
+echo "$RAW_CONFIG" | grep certificate-authority-data | awk '{print $2}' | base64 -d > "$CONFIG_DIR/ca.crt"
+
+# Extract and decode the client certificate
+echo "$RAW_CONFIG" | grep client-certificate-data | awk '{print $2}' | base64 -d > "$CONFIG_DIR/client.crt"
+
+# Extract and decode the client key
+echo "$RAW_CONFIG" | grep client-key-data | awk '{print $2}' | base64 -d > "$CONFIG_DIR/client.key"
+chmod 600 "$CONFIG_DIR/client.key"
+
+# Write kubeconfig with file-based certs and tailscale hostname
+cat > "$CONFIG_FILE" << EOF
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    certificate-authority: $CONFIG_DIR/ca.crt
+    server: https://ringtail.tail8d86e.ts.net:6443
+  name: k3s-ringtail
+contexts:
+- context:
+    cluster: k3s-ringtail
+    user: k3s-ringtail
+  name: k3s-ringtail
+current-context: k3s-ringtail
+users:
+- name: k3s-ringtail
+  user:
+    client-certificate: $CONFIG_DIR/client.crt
+    client-key: $CONFIG_DIR/client.key
+EOF
+
+echo "Config written to $CONFIG_FILE"
+
+# Warn if KUBECONFIG doesn't include this file
+if [[ -z "${KUBECONFIG:-}" ]] || [[ ":$KUBECONFIG:" != *":$CONFIG_FILE:"* ]]; then
+    echo ""
+    echo "WARNING: KUBECONFIG does not include $CONFIG_FILE"
+    echo "Add this to your shell config:"
+    echo "  export KUBECONFIG=\"\$KUBECONFIG:$CONFIG_FILE\""
+fi
+
+echo ""
+echo "Test with: kubectl --context=k3s-ringtail get nodes"

mise-tasks/services-check

@@ -87,6 +87,9 @@ echo ""
 echo "Ringtail (NixOS):"
 check_service "ssh" "ssh -o ConnectTimeout=5 ringtail true"
 check_service "tailscale" "ssh ringtail 'tailscale status --self --json' | jq -e '.Self.Online' > /dev/null"
+check_service "k3s" "ssh ringtail 'k3s kubectl get nodes --no-headers | grep -q Ready'"
+check_service "k3s-apiserver (remote)" "kubectl --context=k3s-ringtail get --raw /healthz"
+check_service "forgejo-runner" "ssh ringtail 'systemctl is-active gitea-runner-nix_container_builder.service'"
 
 echo ""
 echo "Public services (via Fly.io):"

nixos/ringtail/configuration.nix

@@ -96,12 +96,32 @@ in
     dedicatedServer.openFirewall = true;
   };
 
+  # K3s single-node cluster
+  services.k3s = {
+    enable = true;
+    role = "server";
+    tokenFile = "/etc/k3s/token";
+    extraFlags = toString [
+      "--disable=traefik"
+      "--disable=servicelb"
+      "--disable=metrics-server"
+      "--write-kubeconfig-mode=644"
+      "--tls-san=ringtail.tail8d86e.ts.net"
+    ];
+  };
+
+  # K3s containerd registry mirrors (pull through Zot on indri)
+  environment.etc."rancher/k3s/registries.yaml".source = ./k3s-registries.yaml;
+
   # Tailscale
   services.tailscale = {
     enable = true;
     extraUpFlags = [ "--accept-routes" "--ssh" ];
   };
 
+  # Trust Tailscale interface (ArgoCD on indri connects via tailnet)
+  networking.firewall.trustedInterfaces = [ "tailscale0" ];
+
   # SSH
   services.openssh = {
     enable = true;
@@ -124,6 +144,7 @@ in
   # System packages
   environment.systemPackages = with pkgs; [
     git
+    kubectl
     python3 # required for Ansible
     vim
     htop
@@ -369,9 +390,35 @@ in
     "d /mnt/storage2 0755 eblume users -"
   ];
 
+  # Forgejo Actions runner (nix container builder)
+  services.gitea-actions-runner = {
+    package = pkgs.forgejo-runner;
+    instances.nix_container_builder = {
+      enable = true;
+      name = "ringtail-nix-builder";
+      url = "https://forge.ops.eblu.me";
+      tokenFile = "/etc/forgejo-runner/token.env";
+      labels = [ "nix-container-builder:host" ];
+      hostPackages = with pkgs; [
+        bash coreutils curl gawk gitMinimal gnused nodejs wget
+        nix skopeo
+      ];
+      settings = {
+        log.level = "info";
+        runner = {
+          capacity = 1;
+          timeout = "3h";
+        };
+      };
+    };
+  };
+
   # Enable nix flakes
   nix.settings.experimental-features = [ "nix-command" "flakes" ];
 
+  # Allow the runner's dynamic user to access the nix daemon
+  nix.settings.trusted-users = [ "gitea-runner" ];
+
   # NixOS release
   system.stateVersion = "25.11";
 }

nixos/ringtail/k3s-registries.yaml

@@ -0,0 +1,13 @@
+mirrors:
+  docker.io:
+    endpoint:
+      - "https://registry.ops.eblu.me"
+  ghcr.io:
+    endpoint:
+      - "https://registry.ops.eblu.me"
+  quay.io:
+    endpoint:
+      - "https://registry.ops.eblu.me"
+  registry.ops.eblu.me:
+    endpoint:
+      - "https://registry.ops.eblu.me"