Replace k8s Forgejo runner with systemd nix-container-builder

Remove the DinD-based k8s runner and add a native systemd Forgejo Actions runner on ringtail for building containers with nix build and pushing via skopeo. The runner uses the NixOS services.gitea-actions-runner module with host execution (no containers), and Ansible provisions the registration token from 1Password. Adds a new build-container-nix workflow for -nix- tags and updates mise tasks to support both Dockerfile and Nix builds. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-18 20:21:39 -08:00 · 2026-02-18 20:21:39 -08:00 · c098199f8b
commit c098199f8b
parent 0d3269e8d6
15 changed files with 190 additions and 190 deletions
--- a/.forgejo/workflows/build-container-nix.yaml
+++ b/.forgejo/workflows/build-container-nix.yaml
@ -0,0 +1,90 @@
+# Nix container build workflow
+# Triggers on tags matching: <container>-nix-v<version>
+# Builds from containers/<container>/default.nix using nix build
+# Pushes to Zot registry via skopeo
+#
+# Examples:
+#   nettest-nix-v1.0.0  -> builds containers/nettest/default.nix
+#   myapp-nix-v2.1.0    -> builds containers/myapp/default.nix
+name: Build Container (Nix)
+
+on:
+  push:
+    tags:
+      - '*-nix-v[0-9]*'
+
+jobs:
+  build:
+    runs-on: nix-container-builder
+    steps:
+      - name: Parse tag
+        id: parse
+        run: |
+          TAG="${GITHUB_REF_NAME}"
+          echo "Tag: $TAG"
+
+          # Extract container name (everything before -nix-v)
+          # e.g., "nettest-nix-v1.0.0" -> "nettest"
+          CONTAINER="${TAG%-nix-v[0-9]*}"
+          VERSION="${TAG#"${CONTAINER}"-nix-}"
+
+          echo "container=$CONTAINER" >> "$GITHUB_OUTPUT"
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "Container: $CONTAINER"
+          echo "Version: $VERSION"
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Check if nix container exists
+        id: check
+        run: |
+          CONTAINER="${{ steps.parse.outputs.container }}"
+          CONTEXT="containers/$CONTAINER"
+
+          if [ -f "$CONTEXT/default.nix" ]; then
+            echo "Found $CONTEXT/default.nix"
+            echo "exists=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "No default.nix found at $CONTEXT/default.nix"
+            echo "exists=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Skip if container not found
+        if: steps.check.outputs.exists != 'true'
+        run: |
+          echo "========================================"
+          echo "Nix container not found: ${{ steps.parse.outputs.container }}"
+          echo "========================================"
+          echo ""
+          echo "Tag '${{ github.ref_name }}' does not match any nix container in containers/"
+          echo ""
+          echo "Available nix containers:"
+          for nix in containers/*/default.nix; do
+            [ -f "$nix" ] && echo "  - $(basename "$(dirname "$nix")")"
+          done
+          echo ""
+          echo "Skipping build."
+
+      - name: Build with nix
+        if: steps.check.outputs.exists == 'true'
+        id: build
+        run: |
+          CONTAINER="${{ steps.parse.outputs.container }}"
+          echo "Building containers/$CONTAINER/default.nix"
+          nix build -f "containers/$CONTAINER/default.nix" -o result
+          echo "Build complete: $(readlink result)"
+
+      - name: Push to registry
+        if: steps.check.outputs.exists == 'true'
+        run: |
+          CONTAINER="${{ steps.parse.outputs.container }}"
+          VERSION="${{ steps.parse.outputs.version }}"
+          IMAGE="registry.ops.eblu.me/blumeops/$CONTAINER:$VERSION"
+
+          echo "Pushing to $IMAGE"
+          skopeo copy \
+            --dest-tls-verify=false \
+            "docker-archive:result" \
+            "docker://$IMAGE"
+          echo "Push complete: $IMAGE"
--- a/.forgejo/workflows/build-container.yaml
+++ b/.forgejo/workflows/build-container.yaml
@ -17,6 +17,7 @@ on:

 jobs:
  build:
+    if: "!contains(github.ref_name, '-nix-v')"
    runs-on: k8s
    steps:
      - name: Parse tag
--- a/.github/actionlint.yaml
+++ b/.github/actionlint.yaml
@ -1,3 +1,4 @@
 self-hosted-runner:
  labels:
    - k8s
+    - nix-container-builder
--- a/ansible/playbooks/ringtail.yml
+++ b/ansible/playbooks/ringtail.yml
@ -20,6 +20,27 @@
      delegate_to: localhost
      become: false

+    - name: Fetch Forgejo runner registration token from 1Password
+      ansible.builtin.command:
+        cmd: op read "op://vg6xf6vvfmoh5hqjjhlhbeoaie/Forgejo Secrets/runner_reg"
+      register: _runner_reg
+      changed_when: false
+      delegate_to: localhost
+      become: false
+
+    - name: Ensure /etc/forgejo-runner directory exists
+      ansible.builtin.file:
+        path: /etc/forgejo-runner
+        state: directory
+        mode: "0700"
+
+    - name: Write Forgejo runner token file
+      ansible.builtin.copy:
+        content: "TOKEN={{ _runner_reg.stdout }}"
+        dest: /etc/forgejo-runner/token.env
+        mode: "0600"
+      no_log: true
+
    - name: Ensure /etc/k3s directory exists
      ansible.builtin.file:
        path: /etc/k3s
--- a/argocd/apps/forgejo-runner-amd64.yaml
+++ b/argocd/apps/forgejo-runner-amd64.yaml
@ -1,17 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Application
-metadata:
-  name: forgejo-runner-amd64
-  namespace: argocd
-spec:
-  project: default
-  source:
-    repoURL: https://forge.ops.eblu.me/eblume/blumeops.git
-    targetRevision: main
-    path: argocd/manifests/forgejo-runner-amd64
-  destination:
-    server: https://ringtail.tail8d86e.ts.net:6443
-    namespace: forgejo-runner
-  syncPolicy:
-    syncOptions:
-      - CreateNamespace=true
--- a/argocd/manifests/forgejo-runner-amd64/configmap.yaml
+++ b/argocd/manifests/forgejo-runner-amd64/configmap.yaml
@ -1,25 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: forgejo-runner-config
-  namespace: forgejo-runner
-data:
-  config.yaml: |
-    log:
-      level: info
-
-    runner:
-      file: /data/.runner
-      capacity: 2
-      timeout: 3h
-      envs:
-        DOCKER_HOST: tcp://127.0.0.1:2375
-        TZ: America/Los_Angeles
-
-    container:
-      network: "host"
-      docker_host: tcp://127.0.0.1:2375
-  daemon.json: |
-    {
-      "registry-mirrors": ["https://registry.ops.eblu.me"]
-    }
--- a/argocd/manifests/forgejo-runner-amd64/deployment.yaml
+++ b/argocd/manifests/forgejo-runner-amd64/deployment.yaml
@ -1,89 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: forgejo-runner-amd64
-  namespace: forgejo-runner
-  labels:
-    app: forgejo-runner-amd64
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: forgejo-runner-amd64
-  template:
-    metadata:
-      labels:
-        app: forgejo-runner-amd64
-    spec:
-      containers:
-        # Forgejo runner daemon
-        - name: runner
-          image: code.forgejo.org/forgejo/runner:6.3.1
-          env:
-            - name: TZ
-              value: America/Los_Angeles
-            - name: DOCKER_HOST
-              value: tcp://localhost:2375
-            - name: FORGEJO_URL
-              value: "https://forge.ops.eblu.me"
-            - name: RUNNER_NAME
-              value: "k8s-amd64-runner"
-            - name: RUNNER_LABELS
-              value: "k8s-amd64:docker://registry.ops.eblu.me/blumeops/forgejo-runner:v3.2.0-amd64"
-          command:
-            - /bin/sh
-            - -c
-            - |
-              # Wait for DinD to be ready
-              echo "Waiting for Docker daemon..."
-              while ! wget -q -O /dev/null http://localhost:2375/_ping 2>/dev/null; do
-                sleep 1
-              done
-              echo "Docker daemon ready"
-
-              # Register if not already registered
-              if [ ! -f /data/.runner ]; then
-                echo "Registering runner..."
-                forgejo-runner register \
-                  --instance "$FORGEJO_URL" \
-                  --token "$RUNNER_TOKEN" \
-                  --name "$RUNNER_NAME" \
-                  --labels "$RUNNER_LABELS" \
-                  --no-interactive
-              fi
-
-              # Start daemon
-              exec forgejo-runner daemon --config /config/config.yaml
-          envFrom:
-            - secretRef:
-                name: forgejo-runner-env
-          volumeMounts:
-            - name: data
-              mountPath: /data
-            - name: config
-              mountPath: /config
-
-        # Docker-in-Docker sidecar
-        - name: dind
-          image: docker:27-dind
-          securityContext:
-            privileged: true
-          env:
-            - name: DOCKER_TLS_CERTDIR
-              value: ""
-          volumeMounts:
-            - name: dind-storage
-              mountPath: /var/lib/docker
-            - name: config
-              mountPath: /etc/docker/daemon.json
-              subPath: daemon.json
-              readOnly: true
-
-      volumes:
-        - name: data
-          emptyDir: {}
-        - name: dind-storage
-          emptyDir: {}
-        - name: config
-          configMap:
-            name: forgejo-runner-config
--- a/argocd/manifests/forgejo-runner-amd64/external-secret.yaml
+++ b/argocd/manifests/forgejo-runner-amd64/external-secret.yaml
@ -1,27 +0,0 @@
-# ExternalSecret for Forgejo Runner token (amd64)
-#
-# 1Password item: "Forgejo Secrets" in blumeops vault
-# Field: runner_reg (runner registration token)
-#
-# Non-secret env vars (FORGEJO_URL, RUNNER_NAME, RUNNER_LABELS) live in the
-# deployment spec so that changes (e.g. image version bumps) trigger a rollout
-# automatically.
-#
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: forgejo-runner-env
-  namespace: forgejo-runner
-spec:
-  refreshInterval: 1h
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: onepassword-blumeops
-  target:
-    name: forgejo-runner-env
-    creationPolicy: Owner
-  data:
-  - secretKey: RUNNER_TOKEN
-    remoteRef:
-      key: Forgejo Secrets
-      property: runner_reg
--- a/argocd/manifests/forgejo-runner-amd64/kustomization.yaml
+++ b/argocd/manifests/forgejo-runner-amd64/kustomization.yaml
@ -1,7 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - namespace.yaml
-  - configmap.yaml
-  - deployment.yaml
-  - external-secret.yaml
--- a/argocd/manifests/forgejo-runner-amd64/namespace.yaml
+++ b/argocd/manifests/forgejo-runner-amd64/namespace.yaml
@ -1,4 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: forgejo-runner
--- a/docs/changelog.d/feature-k3s-ringtail-runner.feature.md
+++ b/docs/changelog.d/feature-k3s-ringtail-runner.feature.md
@ -1 +1 @@
-K3s cluster on ringtail with Forgejo Actions runner (`k8s-amd64` label) for native amd64 container builds, managed via ArgoCD multi-cluster. Includes 1Password Connect + External Secrets Operator for automated secret management, matching the indri pattern.
+Systemd Forgejo Actions runner on ringtail (`nix-container-builder` label) for building containers with `nix build` and pushing via `skopeo`. K3s cluster retained for future workloads. 1Password Connect + External Secrets Operator available for k8s secret management.
--- a/docs/reference/infrastructure/ringtail.md
+++ b/docs/reference/infrastructure/ringtail.md
@ -63,9 +63,7 @@ Sync order: `1password-connect-ringtail` -> `external-secrets-crds-ringtail` ->

 ### Workloads

-| Workload | Namespace | Label |
-|----------|-----------|-------|
-| Forgejo Runner (amd64) | `forgejo-runner` | `k8s-amd64` |
+No k8s workloads currently deployed. K3s is available for future workloads (e.g. Frigate, running nix-built containers).

 ### Manual Cluster Registration

@ -79,6 +77,19 @@ kubectl get nodes  # verify access
 argocd cluster add default --name k3s-ringtail
 ```

+## Systemd Services
+
+### Forgejo Actions Runner
+
+A native Forgejo Actions runner (`ringtail-nix-builder`) runs as a systemd service via the NixOS `services.gitea-actions-runner` module. It builds containers using `nix build` and pushes them to Zot via `skopeo`.
+
+| Property | Value |
+|----------|-------|
+| **Label** | `nix-container-builder` |
+| **Execution** | Host (no containers) |
+| **Token** | `/etc/forgejo-runner/token.env` (provisioned by Ansible) |
+| **Service unit** | `gitea-runner-nix-container-builder.service` |
+
 ## Maintenance Notes

 **1Password:** Desktop app must be running for `op` CLI. Use `$mod+Shift+minus` to send to scratchpad.
--- a/mise-tasks/container-list
+++ b/mise-tasks/container-list
@ -10,16 +10,24 @@ echo "Container Images"
 echo "================"
 echo ""

-# Find all container directories with Dockerfiles
+# Find all container directories with Dockerfiles or default.nix
 for dir in "$CONTAINER_DIR"/*/; do
    [[ -d "$dir" ]] || continue
-    [[ -f "$dir/Dockerfile" ]] || continue
+
+    # Determine build type
+    if [[ -f "$dir/default.nix" ]]; then
+        build_type="nix"
+    elif [[ -f "$dir/Dockerfile" ]]; then
+        build_type="dockerfile"
+    else
+        continue
+    fi

    # Extract container name from directory
    container=$(basename "$dir")
    image="blumeops/$container"

-    echo "📦 $container"
+    echo "[$build_type] $container"
    echo "   Image: $REGISTRY/$image"
    echo "   Path: $dir"

--- a/mise-tasks/container-tag-and-release
+++ b/mise-tasks/container-tag-and-release
@ -19,28 +19,39 @@ if [[ ! "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
    exit 1
 fi

-TAG="${CONTAINER}-${VERSION}"
+# Determine build type: Nix or Dockerfile
+CONTAINER_DIR="containers/${CONTAINER}"
+if [[ -f "$CONTAINER_DIR/default.nix" ]]; then
+    BUILD_TYPE="nix"
+    TAG="${CONTAINER}-nix-${VERSION}"
+elif [[ -f "$CONTAINER_DIR/Dockerfile" ]]; then
+    BUILD_TYPE="dockerfile"
+    TAG="${CONTAINER}-${VERSION}"
+else
+    echo "Error: No Dockerfile or default.nix found in '$CONTAINER_DIR'"
+    echo ""
+    echo "Available containers:"
+    for dir in containers/*/; do
+        [[ -d "$dir" ]] || continue
+        name=$(basename "$dir")
+        if [[ -f "$dir/default.nix" ]]; then
+            echo "  - $name (nix)"
+        elif [[ -f "$dir/Dockerfile" ]]; then
+            echo "  - $name (dockerfile)"
+        fi
+    done
+    exit 1
+fi

 echo "Creating release tag: $TAG"
+echo "Build type: $BUILD_TYPE"
 echo ""

 # Check if tag already exists
 if git rev-parse "$TAG" >/dev/null 2>&1; then
    echo "Error: Tag '$TAG' already exists"
    echo "Existing tags for $CONTAINER:"
-    git tag -l "${CONTAINER}-v*" | sort -V | tail -5
-    exit 1
-fi
-
-# Check if container directory exists
-CONTAINER_DIR="containers/${CONTAINER}"
-if [[ ! -f "$CONTAINER_DIR/Dockerfile" ]]; then
-    echo "Error: No Dockerfile found at '$CONTAINER_DIR/Dockerfile'"
-    echo ""
-    echo "Available containers:"
-    for dir in containers/*/; do
-        [[ -d "$dir" ]] && echo "  - $(basename "$dir")"
-    done
+    git tag -l "${CONTAINER}-*v*" | sort -V | tail -5
    exit 1
 fi

--- a/nixos/ringtail/configuration.nix
+++ b/nixos/ringtail/configuration.nix
@ -390,9 +390,35 @@ in
    "d /mnt/storage2 0755 eblume users -"
  ];

+  # Forgejo Actions runner (nix container builder)
+  services.gitea-actions-runner = {
+    package = pkgs.forgejo-runner;
+    instances.nix-container-builder = {
+      enable = true;
+      name = "ringtail-nix-builder";
+      url = "https://forge.ops.eblu.me";
+      tokenFile = "/etc/forgejo-runner/token.env";
+      labels = [ "nix-container-builder:host" ];
+      hostPackages = with pkgs; [
+        bash coreutils curl gawk gitMinimal gnused nodejs wget
+        nix skopeo
+      ];
+      settings = {
+        log.level = "info";
+        runner = {
+          capacity = 1;
+          timeout = "3h";
+        };
+      };
+    };
+  };
+
  # Enable nix flakes
  nix.settings.experimental-features = [ "nix-command" "flakes" ];

+  # Allow the runner's dynamic user to access the nix daemon
+  nix.settings.trusted-users = [ "gitea-runner" ];
+
  # NixOS release
  system.stateVersion = "25.11";
 }