Add k3s, 1Password Connect, and systemd nix-container-builder to ringtail (#209)

## Summary

  Extends ringtail from a desktop/gaming NixOS box into an infrastructure node with a k3s cluster, secrets management, and a Forgejo Actions
  runner for building containers with Nix.

  ### K3s cluster
  - Single-node k3s with Traefik/ServiceLB/metrics-server disabled (minimal footprint)
  - TLS SAN set to `ringtail.tail8d86e.ts.net` so ArgoCD on indri can manage it via Tailscale
  - Containerd registry mirrors pull through Zot on indri (`k3s-registries.yaml`)
  - Tailscale interface added to `trustedInterfaces` for cross-node ArgoCD access
  - `kubectl` added to system packages

  ### 1Password Connect + External Secrets Operator
  - Four new ArgoCD apps targeting `k3s-ringtail`: `1password-connect-ringtail`, `external-secrets-crds-ringtail`, `external-secrets-ringtail`,
  `external-secrets-config-ringtail`
  - Reuses the same Helm charts/values as indri, just pointed at ringtail's k3s API server
  - Bootstrap secrets (`op-credentials`, `onepassword-token`) provisioned by Ansible pre_tasks via `op read`, then applied to the `1password`
  namespace in post_tasks

  ### Systemd Forgejo Actions runner
  - Native `services.gitea-actions-runner` with `forgejo-runner` package — no DinD, no k8s pod, runs directly on the NixOS host
  - Label `nix-container-builder:host` — jobs execute on the host with `nix`, `skopeo`, `nodejs`, etc. in PATH
  - Registration token fetched from 1Password (`Forgejo Secrets/runner_reg`) by Ansible and written to `/etc/forgejo-runner/token.env`
  - Runner's dynamic user (`gitea-runner`) added to `nix.settings.trusted-users` for nix daemon access

  ### Nix container build workflow
  - New `.forgejo/workflows/build-container-nix.yaml` triggers on `*-nix-v[0-9]*` tags (e.g. `nettest-nix-v1.0.0`)
  - Builds with `nix build -f containers/<name>/default.nix`, pushes to Zot via `skopeo copy`
  - Existing Dockerfile workflow guarded with `if: !contains(github.ref_name, '-nix-v')` to avoid double-triggering

  ### Mise task updates
  - `container-tag-and-release` auto-detects `default.nix` vs `Dockerfile` and uses the appropriate tag format (`-nix-v` vs `-v`)
  - `container-list` shows build type indicator (`[nix]` / `[dockerfile]`)

  ## Post-merge

  1. `mise run provision-ringtail` — deploys k3s token, runner token, NixOS rebuild
  2. Register k3s cluster in ArgoCD (first time only):
     ```fish
     ssh ringtail 'sudo cat /etc/rancher/k3s/k3s.yaml' | \
       sed 's|127.0.0.1|ringtail.tail8d86e.ts.net|' > /tmp/k3s-ringtail.yaml
     set -x KUBECONFIG /tmp/k3s-ringtail.yaml
     argocd cluster add default --name k3s-ringtail
  3. Sync ArgoCD apps in order: 1password-connect-ringtail -> external-secrets-crds-ringtail -> external-secrets-ringtail ->
  external-secrets-config-ringtail
  4. Verify runner: ssh ringtail 'systemctl status gitea-runner-nix-container-builder'
  5. Check Forgejo admin panel for ringtail-nix-builder runner online
  6. Test: create containers/<name>/default.nix, tag with <name>-nix-v0.1.0

Reviewed-on: https://forge.ops.eblu.me/eblume/blumeops/pulls/209
This commit is contained in:
Erich Blume 2026-02-18 21:15:30 -08:00
commit 918df9e642
16 changed files with 499 additions and 17 deletions

View file

@ -0,0 +1,89 @@
# Nix container build workflow
# Triggers on tags matching: <container>-nix-v<version>
# Builds from containers/<container>/default.nix using nix build
# Pushes to Zot registry via skopeo
#
# Examples:
# nettest-nix-v1.0.0 -> builds containers/nettest/default.nix
# myapp-nix-v2.1.0 -> builds containers/myapp/default.nix
name: Build Container (Nix)
on:
push:
tags:
- '*-nix-v[0-9]*'
jobs:
build:
runs-on: nix-container-builder
steps:
- name: Parse tag
id: parse
run: |
TAG="${GITHUB_REF_NAME}"
echo "Tag: $TAG"
# Extract container name (everything before -nix-v)
# e.g., "nettest-nix-v1.0.0" -> "nettest"
CONTAINER="${TAG%-nix-v[0-9]*}"
VERSION="${TAG#"${CONTAINER}"-nix-}"
echo "container=$CONTAINER" >> "$GITHUB_OUTPUT"
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
echo "Container: $CONTAINER"
echo "Version: $VERSION"
- name: Checkout
uses: actions/checkout@v4
- name: Check if nix container exists
id: check
run: |
CONTAINER="${{ steps.parse.outputs.container }}"
CONTEXT="containers/$CONTAINER"
if [ -f "$CONTEXT/default.nix" ]; then
echo "Found $CONTEXT/default.nix"
echo "exists=true" >> "$GITHUB_OUTPUT"
else
echo "No default.nix found at $CONTEXT/default.nix"
echo "exists=false" >> "$GITHUB_OUTPUT"
fi
- name: Skip if container not found
if: steps.check.outputs.exists != 'true'
run: |
echo "========================================"
echo "Nix container not found: ${{ steps.parse.outputs.container }}"
echo "========================================"
echo ""
echo "Tag '${{ github.ref_name }}' does not match any nix container in containers/"
echo ""
echo "Available nix containers:"
for nix in containers/*/default.nix; do
[ -f "$nix" ] && echo " - $(basename "$(dirname "$nix")")"
done
echo ""
echo "Skipping build."
- name: Build with nix
if: steps.check.outputs.exists == 'true'
id: build
run: |
CONTAINER="${{ steps.parse.outputs.container }}"
echo "Building containers/$CONTAINER/default.nix"
nix build -f "containers/$CONTAINER/default.nix" -o result
echo "Build complete: $(readlink result)"
- name: Push to registry
if: steps.check.outputs.exists == 'true'
run: |
CONTAINER="${{ steps.parse.outputs.container }}"
VERSION="${{ steps.parse.outputs.version }}"
IMAGE="registry.ops.eblu.me/blumeops/$CONTAINER:$VERSION"
echo "Pushing to $IMAGE"
skopeo copy \
"docker-archive:result" \
"docker://$IMAGE"
echo "Push complete: $IMAGE"

View file

@ -17,6 +17,7 @@ on:
jobs:
build:
if: "!contains(github.ref_name, '-nix-v')"
runs-on: k8s
steps:
- name: Parse tag

View file

@ -1,3 +1,4 @@
self-hosted-runner:
labels:
- k8s
- nix-container-builder

View file

@ -3,6 +3,57 @@
hosts: ringtail
become: true
pre_tasks:
- name: Fetch 1Password Connect credentials from 1Password
ansible.builtin.command:
cmd: op read "op://vg6xf6vvfmoh5hqjjhlhbeoaie/1Password Connect/credentials-file"
register: _op_credentials
changed_when: false
delegate_to: localhost
become: false
- name: Fetch 1Password Connect token from 1Password
ansible.builtin.command:
cmd: op read "op://vg6xf6vvfmoh5hqjjhlhbeoaie/1Password Connect/token"
register: _op_token
changed_when: false
delegate_to: localhost
become: false
- name: Fetch Forgejo runner registration token from 1Password
ansible.builtin.command:
cmd: op read "op://vg6xf6vvfmoh5hqjjhlhbeoaie/Forgejo Secrets/runner_reg"
register: _runner_reg
changed_when: false
delegate_to: localhost
become: false
- name: Ensure /etc/forgejo-runner directory exists
ansible.builtin.file:
path: /etc/forgejo-runner
state: directory
mode: "0700"
- name: Write Forgejo runner token file
ansible.builtin.copy:
content: "TOKEN={{ _runner_reg.stdout }}"
dest: /etc/forgejo-runner/token.env
mode: "0600"
no_log: true
- name: Ensure /etc/k3s directory exists
ansible.builtin.file:
path: /etc/k3s
state: directory
mode: "0700"
- name: Generate k3s token if not present
ansible.builtin.copy:
content: "{{ lookup('ansible.builtin.password', '/dev/null', chars=['hexdigits'], length=32) }}"
dest: /etc/k3s/token
mode: "0600"
force: false
tasks:
- name: Ensure blumeops repo is present
ansible.builtin.git:
@ -24,3 +75,42 @@
register: _ts_status
changed_when: false
failed_when: "'Running' not in _ts_status.stdout"
post_tasks:
- name: Wait for k3s to be ready
ansible.builtin.command: k3s kubectl get nodes
register: _k3s_ready
changed_when: false
retries: 30
delay: 5
until: _k3s_ready.rc == 0
- name: Create 1password namespace
ansible.builtin.command: k3s kubectl create namespace 1password
register: _ns
changed_when: _ns.rc == 0
failed_when: _ns.rc != 0 and 'AlreadyExists' not in _ns.stderr
- name: Create or update op-credentials secret
ansible.builtin.shell:
cmd: |
set -o pipefail
k3s kubectl create secret generic op-credentials \
--namespace=1password \
--from-literal=1password-credentials.json='{{ _op_credentials.stdout }}' \
--dry-run=client -o yaml | k3s kubectl apply -f -
executable: /run/current-system/sw/bin/bash
changed_when: true
no_log: true
- name: Create or update onepassword-token secret
ansible.builtin.shell:
cmd: |
set -o pipefail
k3s kubectl create secret generic onepassword-token \
--namespace=1password \
--from-literal=token={{ _op_token.stdout }} \
--dry-run=client -o yaml | k3s kubectl apply -f -
executable: /run/current-system/sw/bin/bash
changed_when: true
no_log: true

View file

@ -0,0 +1,32 @@
# 1Password Connect for ringtail k3s cluster
# Same chart/values as indri, different destination
#
# Prerequisites:
# 1. Bootstrap secrets via ansible (provision-ringtail creates 1password namespace,
# op-credentials and onepassword-token secrets)
# 2. Sync BEFORE external-secrets-ringtail
#
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: 1password-connect-ringtail
namespace: argocd
spec:
project: default
sources:
- repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/connect-helm-charts.git
targetRevision: connect-2.3.0
path: charts/connect
helm:
releaseName: onepassword-connect
valueFiles:
- $values/argocd/manifests/1password-connect/values.yaml
- repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/blumeops.git
targetRevision: main
ref: values
destination:
server: https://ringtail.tail8d86e.ts.net:6443
namespace: 1password
syncPolicy:
syncOptions:
- CreateNamespace=true

View file

@ -0,0 +1,24 @@
# External Secrets Configuration for ringtail k3s cluster
# Same ClusterSecretStore manifests as indri, different destination
#
# Prerequisites:
# - 1password-connect-ringtail is deployed and healthy
# - external-secrets-ringtail operator is deployed and CRDs are installed
#
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: external-secrets-config-ringtail
namespace: argocd
spec:
project: default
source:
repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/blumeops.git
targetRevision: main
path: argocd/manifests/external-secrets
destination:
server: https://ringtail.tail8d86e.ts.net:6443
namespace: external-secrets
syncPolicy:
syncOptions:
- CreateNamespace=true

View file

@ -0,0 +1,24 @@
# External Secrets Operator CRDs for ringtail k3s cluster
# Same CRDs source as indri, different destination
#
# Must be synced BEFORE external-secrets-ringtail operator app.
#
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: external-secrets-crds-ringtail
namespace: argocd
spec:
project: default
source:
repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/external-secrets.git
targetRevision: helm-chart-2.0.0
path: config/crds/bases
directory:
exclude: 'kustomization.yaml'
destination:
server: https://ringtail.tail8d86e.ts.net:6443
syncPolicy:
syncOptions:
- ServerSideApply=true
- CreateNamespace=false

View file

@ -0,0 +1,32 @@
# External Secrets Operator for ringtail k3s cluster
# Same chart/values as indri, different destination
#
# Prerequisites:
# - 1password-connect-ringtail must be deployed and healthy
# - external-secrets-crds-ringtail must be synced first
#
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: external-secrets-ringtail
namespace: argocd
spec:
project: default
sources:
- repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/external-secrets.git
targetRevision: helm-chart-2.0.0
path: deploy/charts/external-secrets
helm:
releaseName: external-secrets
valueFiles:
- $values/argocd/manifests/external-secrets/values.yaml
- repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/blumeops.git
targetRevision: main
ref: values
destination:
server: https://ringtail.tail8d86e.ts.net:6443
namespace: external-secrets
syncPolicy:
syncOptions:
- CreateNamespace=true
- ServerSideApply=true

View file

@ -0,0 +1 @@
Systemd Forgejo Actions runner on ringtail (`nix-container-builder` label) for building containers with `nix build` and pushing via `skopeo`. K3s cluster retained for future workloads. 1Password Connect + External Secrets Operator available for k8s secret management.

View file

@ -45,6 +45,51 @@ mise run provision-ringtail
This updates `flake.lock` via Dagger, verifies the current commit is pushed to forge, then deploys the exact commit via ansible. If the lockfile changed, it stages the file and exits so you can commit and re-run.
## K3s Cluster
Ringtail runs a single-node k3s cluster for native amd64 workloads, registered in [[argocd|ArgoCD]] on indri as `k3s-ringtail`.
- **Disabled components:** Traefik, ServiceLB, metrics-server (minimal footprint)
- **TLS SAN:** `ringtail.tail8d86e.ts.net` (ArgoCD connects via Tailscale)
- **Registry mirrors:** Containerd pulls through Zot on indri (`registry.ops.eblu.me`)
- **Token:** `/etc/k3s/token` (generated on first provision)
- **Kubeconfig:** `/etc/rancher/k3s/k3s.yaml` (world-readable via `--write-kubeconfig-mode=644`)
### Secrets Management
1Password Connect + External Secrets Operator syncs secrets from 1Password to k8s, matching the [[1password|indri pattern]]. Bootstrap credentials (`op-credentials`, `onepassword-token`) are provisioned by Ansible; ArgoCD manages the operator stack.
Sync order: `1password-connect-ringtail` -> `external-secrets-crds-ringtail` -> `external-secrets-ringtail` -> `external-secrets-config-ringtail`
### Workloads
No k8s workloads currently deployed. K3s is available for future workloads (e.g. Frigate, running nix-built containers).
### Manual Cluster Registration
After first provision, register the cluster in ArgoCD:
```fish
ssh ringtail 'sudo cat /etc/rancher/k3s/k3s.yaml' | \
sed 's|127.0.0.1|ringtail.tail8d86e.ts.net|' > /tmp/k3s-ringtail.yaml
set -x KUBECONFIG /tmp/k3s-ringtail.yaml
kubectl get nodes # verify access
argocd cluster add default --name k3s-ringtail
```
## Systemd Services
### Forgejo Actions Runner
A native Forgejo Actions runner (`ringtail-nix-builder`) runs as a systemd service via the NixOS `services.gitea-actions-runner` module. It builds containers using `nix build` and pushes them to Zot via `skopeo`.
| Property | Value |
|----------|-------|
| **Label** | `nix-container-builder` |
| **Execution** | Host (no containers) |
| **Token** | `/etc/forgejo-runner/token.env` (provisioned by Ansible) |
| **Service unit** | `gitea-runner-nix_container_builder.service` |
## Maintenance Notes
**1Password:** Desktop app must be running for `op` CLI. Use `$mod+Shift+minus` to send to scratchpad.

View file

@ -10,16 +10,24 @@ echo "Container Images"
echo "================"
echo ""
# Find all container directories with Dockerfiles
# Find all container directories with Dockerfiles or default.nix
for dir in "$CONTAINER_DIR"/*/; do
[[ -d "$dir" ]] || continue
[[ -f "$dir/Dockerfile" ]] || continue
# Determine build type
if [[ -f "$dir/default.nix" ]]; then
build_type="nix"
elif [[ -f "$dir/Dockerfile" ]]; then
build_type="dockerfile"
else
continue
fi
# Extract container name from directory
container=$(basename "$dir")
image="blumeops/$container"
echo "📦 $container"
echo "[$build_type] $container"
echo " Image: $REGISTRY/$image"
echo " Path: $dir"

View file

@ -19,28 +19,39 @@ if [[ ! "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
exit 1
fi
TAG="${CONTAINER}-${VERSION}"
# Determine build type: Nix or Dockerfile
CONTAINER_DIR="containers/${CONTAINER}"
if [[ -f "$CONTAINER_DIR/default.nix" ]]; then
BUILD_TYPE="nix"
TAG="${CONTAINER}-nix-${VERSION}"
elif [[ -f "$CONTAINER_DIR/Dockerfile" ]]; then
BUILD_TYPE="dockerfile"
TAG="${CONTAINER}-${VERSION}"
else
echo "Error: No Dockerfile or default.nix found in '$CONTAINER_DIR'"
echo ""
echo "Available containers:"
for dir in containers/*/; do
[[ -d "$dir" ]] || continue
name=$(basename "$dir")
if [[ -f "$dir/default.nix" ]]; then
echo " - $name (nix)"
elif [[ -f "$dir/Dockerfile" ]]; then
echo " - $name (dockerfile)"
fi
done
exit 1
fi
echo "Creating release tag: $TAG"
echo "Build type: $BUILD_TYPE"
echo ""
# Check if tag already exists
if git rev-parse "$TAG" >/dev/null 2>&1; then
echo "Error: Tag '$TAG' already exists"
echo "Existing tags for $CONTAINER:"
git tag -l "${CONTAINER}-v*" | sort -V | tail -5
exit 1
fi
# Check if container directory exists
CONTAINER_DIR="containers/${CONTAINER}"
if [[ ! -f "$CONTAINER_DIR/Dockerfile" ]]; then
echo "Error: No Dockerfile found at '$CONTAINER_DIR/Dockerfile'"
echo ""
echo "Available containers:"
for dir in containers/*/; do
[[ -d "$dir" ]] && echo " - $(basename "$dir")"
done
git tag -l "${CONTAINER}-*v*" | sort -V | tail -5
exit 1
fi

View file

@ -0,0 +1,61 @@
#!/usr/bin/env bash
#MISE description="Ensure kubectl config for k3s-ringtail is set up on this workstation"
set -euo pipefail
CONFIG_DIR="$HOME/.kube/k3s-ringtail"
CONFIG_FILE="$CONFIG_DIR/config.yml"
echo "Ensuring k3s-ringtail kubectl config..."
# Create directory if needed
mkdir -p "$CONFIG_DIR"
# Fetch kubeconfig from ringtail and extract the CA cert
echo "Fetching kubeconfig from ringtail..."
RAW_CONFIG=$(ssh ringtail 'sudo cat /etc/rancher/k3s/k3s.yaml')
# Extract and decode the CA certificate
echo "$RAW_CONFIG" | grep certificate-authority-data | awk '{print $2}' | base64 -d > "$CONFIG_DIR/ca.crt"
# Extract and decode the client certificate
echo "$RAW_CONFIG" | grep client-certificate-data | awk '{print $2}' | base64 -d > "$CONFIG_DIR/client.crt"
# Extract and decode the client key
echo "$RAW_CONFIG" | grep client-key-data | awk '{print $2}' | base64 -d > "$CONFIG_DIR/client.key"
chmod 600 "$CONFIG_DIR/client.key"
# Write kubeconfig with file-based certs and tailscale hostname
cat > "$CONFIG_FILE" << EOF
apiVersion: v1
kind: Config
clusters:
- cluster:
certificate-authority: $CONFIG_DIR/ca.crt
server: https://ringtail.tail8d86e.ts.net:6443
name: k3s-ringtail
contexts:
- context:
cluster: k3s-ringtail
user: k3s-ringtail
name: k3s-ringtail
current-context: k3s-ringtail
users:
- name: k3s-ringtail
user:
client-certificate: $CONFIG_DIR/client.crt
client-key: $CONFIG_DIR/client.key
EOF
echo "Config written to $CONFIG_FILE"
# Warn if KUBECONFIG doesn't include this file
if [[ -z "${KUBECONFIG:-}" ]] || [[ ":$KUBECONFIG:" != *":$CONFIG_FILE:"* ]]; then
echo ""
echo "WARNING: KUBECONFIG does not include $CONFIG_FILE"
echo "Add this to your shell config:"
echo " export KUBECONFIG=\"\$KUBECONFIG:$CONFIG_FILE\""
fi
echo ""
echo "Test with: kubectl --context=k3s-ringtail get nodes"

View file

@ -87,6 +87,9 @@ echo ""
echo "Ringtail (NixOS):"
check_service "ssh" "ssh -o ConnectTimeout=5 ringtail true"
check_service "tailscale" "ssh ringtail 'tailscale status --self --json' | jq -e '.Self.Online' > /dev/null"
check_service "k3s" "ssh ringtail 'k3s kubectl get nodes --no-headers | grep -q Ready'"
check_service "k3s-apiserver (remote)" "kubectl --context=k3s-ringtail get --raw /healthz"
check_service "forgejo-runner" "ssh ringtail 'systemctl is-active gitea-runner-nix_container_builder.service'"
echo ""
echo "Public services (via Fly.io):"

View file

@ -96,12 +96,32 @@ in
dedicatedServer.openFirewall = true;
};
# K3s single-node cluster
services.k3s = {
enable = true;
role = "server";
tokenFile = "/etc/k3s/token";
extraFlags = toString [
"--disable=traefik"
"--disable=servicelb"
"--disable=metrics-server"
"--write-kubeconfig-mode=644"
"--tls-san=ringtail.tail8d86e.ts.net"
];
};
# K3s containerd registry mirrors (pull through Zot on indri)
environment.etc."rancher/k3s/registries.yaml".source = ./k3s-registries.yaml;
# Tailscale
services.tailscale = {
enable = true;
extraUpFlags = [ "--accept-routes" "--ssh" ];
};
# Trust Tailscale interface (ArgoCD on indri connects via tailnet)
networking.firewall.trustedInterfaces = [ "tailscale0" ];
# SSH
services.openssh = {
enable = true;
@ -124,6 +144,7 @@ in
# System packages
environment.systemPackages = with pkgs; [
git
kubectl
python3 # required for Ansible
vim
htop
@ -369,9 +390,35 @@ in
"d /mnt/storage2 0755 eblume users -"
];
# Forgejo Actions runner (nix container builder)
services.gitea-actions-runner = {
package = pkgs.forgejo-runner;
instances.nix_container_builder = {
enable = true;
name = "ringtail-nix-builder";
url = "https://forge.ops.eblu.me";
tokenFile = "/etc/forgejo-runner/token.env";
labels = [ "nix-container-builder:host" ];
hostPackages = with pkgs; [
bash coreutils curl gawk gitMinimal gnused nodejs wget
nix skopeo
];
settings = {
log.level = "info";
runner = {
capacity = 1;
timeout = "3h";
};
};
};
};
# Enable nix flakes
nix.settings.experimental-features = [ "nix-command" "flakes" ];
# Allow the runner's dynamic user to access the nix daemon
nix.settings.trusted-users = [ "gitea-runner" ];
# NixOS release
system.stateVersion = "25.11";
}

View file

@ -0,0 +1,13 @@
mirrors:
docker.io:
endpoint:
- "https://registry.ops.eblu.me"
ghcr.io:
endpoint:
- "https://registry.ops.eblu.me"
quay.io:
endpoint:
- "https://registry.ops.eblu.me"
registry.ops.eblu.me:
endpoint:
- "https://registry.ops.eblu.me"