Switch container builds to indri docker-builder runner

- Use Docker instead of buildah in composite action
- Build workflows now run on docker-builder label
- Add actionlint config for custom runner labels
- Avoids nested containerization complexity in k8s

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

.forgejo/actions/build-push-image/action.yaml

@@ -0,0 +1,54 @@
+name: 'Build and Push Image'
+description: 'Build a container image with Docker and push to zot registry'
+
+# TODO: Investigate zot tag immutability to prevent overwriting released versions
+# See: https://zotregistry.dev/v2.1.1/articles/immutable-tags/
+
+inputs:
+  context:
+    description: 'Build context path'
+    required: true
+  dockerfile:
+    description: 'Dockerfile path (relative to context)'
+    required: false
+    default: 'Dockerfile'
+  image_name:
+    description: 'Image name (without registry, e.g. blumeops/devpi)'
+    required: true
+  version:
+    description: 'Version tag (e.g. v1.0.0)'
+    required: true
+  registry:
+    description: 'Registry URL'
+    required: false
+    default: 'registry.tail8d86e.ts.net'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Build image with Docker
+      shell: bash
+      run: |
+        echo "Building ${{ inputs.image_name }}:${{ inputs.version }}"
+        docker build \
+          --tag ${{ inputs.registry }}/${{ inputs.image_name }}:${{ inputs.version }} \
+          --tag ${{ inputs.registry }}/${{ inputs.image_name }}:${{ github.sha }} \
+          --file ${{ inputs.context }}/${{ inputs.dockerfile }} \
+          ${{ inputs.context }}
+
+    - name: Push to registry
+      shell: bash
+      run: |
+        echo "Pushing ${{ inputs.image_name }}:${{ inputs.version }}"
+        docker push ${{ inputs.registry }}/${{ inputs.image_name }}:${{ inputs.version }}
+        docker push ${{ inputs.registry }}/${{ inputs.image_name }}:${{ github.sha }}
+
+    - name: Summary
+      shell: bash
+      run: |
+        echo "Built and pushed:"
+        echo "   ${{ inputs.registry }}/${{ inputs.image_name }}:${{ inputs.version }}"
+        echo "   ${{ inputs.registry }}/${{ inputs.image_name }}:${{ github.sha }}"
+        echo ""
+        echo "Registry tags:"
+        curl -sf "https://${{ inputs.registry }}/v2/${{ inputs.image_name }}/tags/list" | jq -r '.tags[]' | sort -V | tail -10 || true

.forgejo/workflows/build-devpi.yaml

@@ -0,0 +1,37 @@
+name: Build devpi
+
+on:
+  push:
+    tags:
+      - 'devpi-v*'
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version (e.g. v1.0.0)'
+        required: true
+
+jobs:
+  build:
+    runs-on: docker-builder
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Extract version from tag
+        id: version
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            VERSION="${{ github.event.inputs.version }}"
+          else
+            # Extract version from tag: devpi-v1.0.0 -> v1.0.0
+            VERSION="${GITHUB_REF_NAME#devpi-}"
+          fi
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "Building version: $VERSION"
+
+      - name: Build and push
+        uses: ./.forgejo/actions/build-push-image
+        with:
+          context: argocd/manifests/devpi
+          image_name: blumeops/devpi
+          version: ${{ steps.version.outputs.version }}

.forgejo/workflows/build-runner.yaml

@@ -0,0 +1,37 @@
+name: Build forgejo-runner
+
+on:
+  push:
+    tags:
+      - 'runner-v*'
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version (e.g. v1.0.0)'
+        required: true
+
+jobs:
+  build:
+    runs-on: docker-builder
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Extract version from tag
+        id: version
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            VERSION="${{ github.event.inputs.version }}"
+          else
+            # Extract version from tag: runner-v1.0.0 -> v1.0.0
+            VERSION="${GITHUB_REF_NAME#runner-}"
+          fi
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "Building version: $VERSION"
+
+      - name: Build and push
+        uses: ./.forgejo/actions/build-push-image
+        with:
+          context: argocd/manifests/forgejo-runner
+          image_name: blumeops/forgejo-runner
+          version: ${{ steps.version.outputs.version }}

.forgejo/workflows/test.yaml

@@ -23,14 +23,15 @@ jobs:
           git --version
           echo ""
           echo "=== Build tools ==="
-          make --version | head -1
-          gcc --version | head -1
+          make --version 2>&1 | head -1 || true
+          gcc --version 2>&1 | head -1 || true
           echo ""
-          echo "=== Docker ==="
-          docker --version
+          echo "=== Container tools (Buildah) ==="
+          buildah --version
+          podman --version
           echo ""
           echo "=== Other tools ==="
-          curl --version | head -1
+          curl --version 2>&1 | head -1 || true
           jq --version
 
       - name: Show repo info

.github/actionlint.yaml

@@ -0,0 +1,5 @@
+self-hosted-runner:
+  labels:
+    - docker-builder
+    - ubuntu-latest
+    - ubuntu-22.04

.pre-commit-config.yaml

@@ -86,4 +86,5 @@ repos:
     rev: v1.7.10
     hooks:
       - id: actionlint-system
+        args: ['-config-file', '.github/actionlint.yaml']
         files: ^\.forgejo/workflows/

CLAUDE.md

@@ -134,6 +134,13 @@ When migrating a service from indri to k8s, the Tailscale hostname must be freed
 
 Use `ssh indri 'tailscale serve status --json'` to check current serve entries (the non-JSON output may be empty even when entries exist).
 
+## Container Image Releases
+
+```fish
+mise run container-list                       # Show containers and recent tags
+mise run container-release runner v1.0.0      # Tag and trigger build workflow
+```
+
 ## Third-Party Projects
 
 When a task requires cloning or using a third-party git repository (e.g., for building from source), **ask the user to mirror it on forge first**, then clone from the mirror:

ansible/playbooks/indri.yml

@@ -61,6 +61,23 @@
       no_log: true
       tags: [forgejo]
 
+    # Forgejo runner token (for indri-based runner)
+    - name: Fetch forgejo runner token
+      ansible.builtin.command:
+        cmd: op --vault vg6xf6vvfmoh5hqjjhlhbeoaie item get w3663ffnvkewbftncqxtcpeavy --fields runner_reg --reveal
+      delegate_to: localhost
+      register: _forgejo_runner_token
+      changed_when: false
+      no_log: true
+      check_mode: false
+      tags: [forgejo_runner]
+
+    - name: Set forgejo runner token fact
+      ansible.builtin.set_fact:
+        forgejo_runner_token: "{{ _forgejo_runner_token.stdout }}"
+      no_log: true
+      tags: [forgejo_runner]
+
   roles:
     - role: alloy
       tags: alloy
@@ -82,3 +99,5 @@
       tags: plex_metrics
     - role: tailscale_serve
       tags: tailscale-serve
+    - role: forgejo_runner
+      tags: forgejo_runner

ansible/roles/forgejo_runner/defaults/main.yml

@@ -0,0 +1,19 @@
+---
+forgejo_runner_repo_dir: /Users/erichblume/code/3rd/forgejo-runner
+forgejo_runner_binary: "{{ forgejo_runner_repo_dir }}/forgejo-runner"
+forgejo_runner_data_dir: /Users/erichblume/.forgejo-runner
+forgejo_runner_config_dir: /Users/erichblume/.config/forgejo-runner
+forgejo_runner_log_dir: /Users/erichblume/Library/Logs
+
+# Runner registration
+forgejo_runner_instance_url: "http://localhost:3001"
+forgejo_runner_name: "indri-docker-runner"
+forgejo_runner_labels: "docker-builder:docker"
+
+# Runner config
+forgejo_runner_capacity: 2
+forgejo_runner_timeout: 3h
+
+# Docker container settings for jobs
+forgejo_runner_docker_network: bridge
+forgejo_runner_privileged: true  # Needed for container builds

ansible/roles/forgejo_runner/handlers/main.yml

@@ -0,0 +1,7 @@
+---
+- name: Restart forgejo-runner
+  listen: Restart forgejo-runner
+  ansible.builtin.shell: |
+    launchctl unload ~/Library/LaunchAgents/mcquack.eblume.forgejo-runner.plist 2>/dev/null || true
+    launchctl load ~/Library/LaunchAgents/mcquack.eblume.forgejo-runner.plist
+  changed_when: true

ansible/roles/forgejo_runner/tasks/main.yml

@@ -0,0 +1,83 @@
+---
+# Forgejo Runner on indri
+#
+# Uses Docker container mode for job isolation.
+# Can build containers using Docker (via socket).
+#
+# ONE-TIME SETUP (before running ansible):
+#
+# 1. Clone forgejo-runner from forge mirror:
+#    ssh indri 'git clone https://forge.tail8d86e.ts.net/eblume/forgejo-runner.git ~/code/3rd/forgejo-runner'
+#
+# 2. Set up Go via mise:
+#    ssh indri 'cd ~/code/3rd/forgejo-runner && mise use go@1.24'
+#
+# 3. Build:
+#    ssh indri 'cd ~/code/3rd/forgejo-runner && mise x -- make build'
+#
+# 4. Run ansible to deploy config and LaunchAgent
+
+- name: Verify forgejo-runner binary exists
+  ansible.builtin.stat:
+    path: "{{ forgejo_runner_binary }}"
+  register: forgejo_runner_binary_stat
+
+- name: Fail if forgejo-runner binary not found
+  ansible.builtin.fail:
+    msg: |
+      Forgejo-runner binary not found at {{ forgejo_runner_binary }}.
+      Please build from source first:
+        ssh indri 'cd ~/code/3rd/forgejo-runner && mise x -- make build'
+  when: not forgejo_runner_binary_stat.stat.exists
+
+- name: Ensure forgejo-runner directories exist
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: '0755'
+  loop:
+    - "{{ forgejo_runner_data_dir }}"
+    - "{{ forgejo_runner_config_dir }}"
+
+- name: Deploy forgejo-runner config
+  ansible.builtin.template:
+    src: config.yaml.j2
+    dest: "{{ forgejo_runner_config_dir }}/config.yaml"
+    mode: '0644'
+  notify: Restart forgejo-runner
+
+- name: Check if runner is registered
+  ansible.builtin.stat:
+    path: "{{ forgejo_runner_data_dir }}/.runner"
+  register: forgejo_runner_registered
+
+- name: Register runner with Forgejo
+  ansible.builtin.command:
+    cmd: >
+      {{ forgejo_runner_binary }} register
+      --instance "{{ forgejo_runner_instance_url }}"
+      --token "{{ forgejo_runner_token }}"
+      --name "{{ forgejo_runner_name }}"
+      --labels "{{ forgejo_runner_labels }}"
+      --no-interactive
+    chdir: "{{ forgejo_runner_data_dir }}"
+  when: not forgejo_runner_registered.stat.exists
+  changed_when: true
+
+- name: Deploy forgejo-runner LaunchAgent plist
+  ansible.builtin.template:
+    src: forgejo-runner.plist.j2
+    dest: ~/Library/LaunchAgents/mcquack.eblume.forgejo-runner.plist
+    mode: '0644'
+  notify: Restart forgejo-runner
+
+- name: Check if forgejo-runner LaunchAgent is loaded
+  ansible.builtin.command: launchctl list mcquack.eblume.forgejo-runner
+  register: forgejo_runner_launchctl_check
+  changed_when: false
+  failed_when: false
+
+- name: Load forgejo-runner LaunchAgent if not loaded
+  ansible.builtin.command: launchctl load ~/Library/LaunchAgents/mcquack.eblume.forgejo-runner.plist
+  when: forgejo_runner_launchctl_check.rc != 0
+  changed_when: true

ansible/roles/forgejo_runner/templates/config.yaml.j2

@@ -0,0 +1,15 @@
+# {{ ansible_managed }}
+log:
+  level: info
+
+runner:
+  file: {{ forgejo_runner_data_dir }}/.runner
+  capacity: {{ forgejo_runner_capacity }}
+  timeout: {{ forgejo_runner_timeout }}
+
+container:
+  network: "{{ forgejo_runner_docker_network }}"
+  privileged: {{ forgejo_runner_privileged | lower }}
+  # Mount Docker socket so jobs can build containers
+  valid_volumes:
+    - /var/run/docker.sock

ansible/roles/forgejo_runner/templates/forgejo-runner.plist.j2

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- {{ ansible_managed }} -->
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>Label</key>
+	<string>mcquack.eblume.forgejo-runner</string>
+	<key>ProgramArguments</key>
+	<array>
+		<string>{{ forgejo_runner_binary }}</string>
+		<string>daemon</string>
+		<string>--config</string>
+		<string>{{ forgejo_runner_config_dir }}/config.yaml</string>
+	</array>
+	<key>WorkingDirectory</key>
+	<string>{{ forgejo_runner_data_dir }}</string>
+	<key>RunAtLoad</key>
+	<true/>
+	<key>KeepAlive</key>
+	<true/>
+	<key>StandardOutPath</key>
+	<string>{{ forgejo_runner_log_dir }}/mcquack.forgejo-runner.out.log</string>
+	<key>StandardErrorPath</key>
+	<string>{{ forgejo_runner_log_dir }}/mcquack.forgejo-runner.err.log</string>
+</dict>
+</plist>

argocd/manifests/forgejo-runner/Dockerfile

@@ -1,29 +1,67 @@
-FROM code.forgejo.org/forgejo/runner:3.5.1
+# Build forgejo-runner from source
+# Source: https://forge.tail8d86e.ts.net/eblume/forgejo-runner (mirror of code.forgejo.org/forgejo/runner)
 
-# Switch to root to install packages
-USER root
+FROM golang:1.24-alpine AS builder
 
-# The base image is Alpine Linux
-# Install tools needed for GitHub Actions and builds
+ARG FORGEJO_RUNNER_VERSION=v3.5.1
+
+RUN apk add --no-cache git make build-base
+
+WORKDIR /src
+RUN git clone --depth 1 --branch ${FORGEJO_RUNNER_VERSION} \
+    https://forge.tail8d86e.ts.net/eblume/forgejo-runner.git .
+
+RUN make clean && make build
+
+# Runtime image
+FROM alpine:3.21
+
+# Create runner user with proper passwd entry (required by buildah)
+# Also configure subuid/subgid for rootless container builds
+RUN addgroup -g 1000 runner && \
+    adduser -D -u 1000 -G runner -h /data runner && \
+    echo "runner:100000:65536" >> /etc/subuid && \
+    echo "runner:100000:65536" >> /etc/subgid
+
+# Install runtime dependencies
 RUN apk add --no-cache \
     # Required for actions/checkout and other Node-based actions
     nodejs \
     npm \
-    # Build essentials
+    # Core tools
     git \
+    bash \
     curl \
     wget \
     jq \
+    # Build essentials
     make \
     gcc \
     g++ \
     musl-dev \
-    # For container builds
-    ca-certificates \
-    docker-cli
+    # For container builds (daemonless, no Docker socket needed)
+    buildah \
+    podman \
+    fuse-overlayfs \
+    ca-certificates
+
+# Copy runner binary from builder
+COPY --from=builder /src/forgejo-runner /bin/forgejo-runner
+
+# Configure buildah for rootless operation
+RUN mkdir -p /etc/containers && \
+    printf '[storage]\ndriver = "overlay"\nrunroot = "/tmp/containers-run"\ngraphroot = "/tmp/containers-storage"\n[storage.options.overlay]\nmount_program = "/usr/bin/fuse-overlayfs"\n' \
+    > /etc/containers/storage.conf
+
+# Configure registries (allow insecure for local registry)
+RUN printf 'unqualified-search-registries = ["docker.io"]\n[[registry]]\nlocation = "registry.tail8d86e.ts.net"\ninsecure = true\n' \
+    > /etc/containers/registries.conf
 
 # Verify tools are available
-RUN node --version && npm --version && docker --version
+RUN node --version && npm --version && buildah --version && /bin/forgejo-runner --version
 
-# Switch back to non-root user
-USER 1000
+ENV HOME=/data
+WORKDIR /data
+USER runner
+
+CMD ["/bin/forgejo-runner"]

argocd/manifests/forgejo-runner/deployment.yaml

@@ -16,7 +16,7 @@ spec:
       serviceAccountName: forgejo-runner
       containers:
         - name: runner
-          image: registry.tail8d86e.ts.net/blumeops/forgejo-runner:latest
+          image: registry.tail8d86e.ts.net/blumeops/forgejo-runner:v1.0.3
           env:
             # Use internal k8s service via Tailscale operator egress
             - name: FORGEJO_INSTANCE_URL

mise-tasks/container-list

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+#MISE description="List available containers and their recent tags"
+
+set -euo pipefail
+
+REGISTRY="registry.tail8d86e.ts.net"
+WORKFLOW_DIR=".forgejo/workflows"
+
+echo "Container Images"
+echo "================"
+echo ""
+
+# Find all build-*.yaml workflows
+for workflow in "$WORKFLOW_DIR"/build-*.yaml; do
+    [[ -f "$workflow" ]] || continue
+
+    # Extract container name from filename: build-runner.yaml -> runner
+    filename=$(basename "$workflow")
+    container="${filename#build-}"
+    container="${container%.yaml}"
+
+    # Skip if not a container build workflow (check for image_name)
+    if ! grep -q "image_name:" "$workflow" 2>/dev/null; then
+        continue
+    fi
+
+    # Extract image name from workflow
+    image=$(grep -E "^\s+image_name:" "$workflow" | head -1 | awk '{print $2}')
+
+    echo "📦 $container"
+    echo "   Image: $REGISTRY/$image"
+    echo "   Workflow: $workflow"
+
+    # Query zot for recent tags
+    tags=$(curl -sf "https://$REGISTRY/v2/$image/tags/list" 2>/dev/null | jq -r '.tags // [] | .[]' | grep -E '^v[0-9]' | sort -V | tail -4 || true)
+
+    if [[ -n "$tags" ]]; then
+        echo "   Recent tags:"
+        echo "$tags" | while read -r tag; do
+            echo "     - $tag"
+        done
+    else
+        echo "   Recent tags: (none)"
+    fi
+    echo ""
+done
+
+echo "---"
+echo "To release a new version:"
+echo "  mise run container-release <container> <version>"
+echo ""
+echo "Example:"
+echo "  mise run container-release runner v1.0.0"

mise-tasks/container-release

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+#MISE description="Release a container image by creating a git tag"
+
+set -euo pipefail
+
+CONTAINER="${1:-}"
+VERSION="${2:-}"
+
+if [[ -z "$CONTAINER" || -z "$VERSION" ]]; then
+    echo "Usage: mise run container-release <container> <version>"
+    echo ""
+    echo "Run 'mise run container-list' to see available containers and recent tags."
+    exit 1
+fi
+
+# Validate version format
+if [[ ! "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    echo "Error: Version must be in format vX.Y.Z (e.g. v1.0.0)"
+    exit 1
+fi
+
+TAG="${CONTAINER}-${VERSION}"
+
+echo "Creating release tag: $TAG"
+echo ""
+
+# Check if tag already exists
+if git rev-parse "$TAG" >/dev/null 2>&1; then
+    echo "Error: Tag '$TAG' already exists"
+    echo "Existing tags for $CONTAINER:"
+    git tag -l "${CONTAINER}-v*" | sort -V | tail -5
+    exit 1
+fi
+
+# Find the workflow file to determine image name
+WORKFLOW_FILE=".forgejo/workflows/build-${CONTAINER}.yaml"
+if [[ ! -f "$WORKFLOW_FILE" ]]; then
+    echo "Error: No workflow found for container '$CONTAINER'"
+    echo ""
+    echo "Run 'mise run container-list' to see available containers."
+    exit 1
+fi
+
+# Extract image name from workflow
+IMAGE=$(grep -E "^\s+image_name:" "$WORKFLOW_FILE" | head -1 | awk '{print $2}')
+if [[ -z "$IMAGE" ]]; then
+    echo "Error: Could not determine image name from $WORKFLOW_FILE"
+    exit 1
+fi
+
+echo "Container: $CONTAINER"
+echo "Workflow: $WORKFLOW_FILE"
+echo "Image: registry.tail8d86e.ts.net/$IMAGE:$VERSION"
+echo ""
+
+# Confirm
+read -p "Create tag and push? [y/N] " -n 1 -r
+echo
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    echo "Aborted."
+    exit 0
+fi
+
+# Create and push tag
+git tag "$TAG"
+git push origin "$TAG"
+
+echo ""
+echo "✅ Tag '$TAG' created and pushed"
+echo ""
+echo "The workflow will now build and push:"
+echo "  registry.tail8d86e.ts.net/$IMAGE:$VERSION"
+echo ""
+echo "Monitor the build at:"
+echo "  https://forge.tail8d86e.ts.net/eblume/blumeops/actions"