diff --git a/ansible/playbooks/ringtail.yml b/ansible/playbooks/ringtail.yml index f7e085a..3cc1a0b 100644 --- a/ansible/playbooks/ringtail.yml +++ b/ansible/playbooks/ringtail.yml @@ -3,6 +3,28 @@ hosts: ringtail become: true + pre_tasks: + - name: Fetch Forgejo runner registration token from 1Password + ansible.builtin.command: + cmd: op read "op://vg6xf6vvfmoh5hqjjhlhbeoaie/Forgejo Secrets/runner_reg" + register: _runner_token + changed_when: false + delegate_to: localhost + become: false + + - name: Ensure /etc/k3s directory exists + ansible.builtin.file: + path: /etc/k3s + state: directory + mode: "0700" + + - name: Generate k3s token if not present + ansible.builtin.copy: + content: "{{ lookup('ansible.builtin.password', '/dev/null', chars=['hexdigits'], length=32) }}" + dest: /etc/k3s/token + mode: "0600" + force: false + tasks: - name: Ensure blumeops repo is present ansible.builtin.git: @@ -24,3 +46,46 @@ register: _ts_status changed_when: false failed_when: "'Running' not in _ts_status.stdout" + + post_tasks: + - name: Wait for k3s to be ready + ansible.builtin.command: k3s kubectl get nodes + register: _k3s_ready + changed_when: false + retries: 30 + delay: 5 + until: _k3s_ready.rc == 0 + + - name: Create forgejo-runner namespace + ansible.builtin.command: k3s kubectl create namespace forgejo-runner + register: _ns + changed_when: _ns.rc == 0 + failed_when: _ns.rc != 0 and 'AlreadyExists' not in _ns.stderr + + - name: Check if forgejo-runner-env secret exists + ansible.builtin.command: k3s kubectl get secret forgejo-runner-env -n forgejo-runner + register: _secret_exists + changed_when: false + failed_when: false + + - name: Create forgejo-runner-env secret + ansible.builtin.command: > + k3s kubectl create secret generic forgejo-runner-env + --namespace=forgejo-runner + --from-literal=RUNNER_TOKEN={{ _runner_token.stdout }} + changed_when: true + when: _secret_exists.rc != 0 + no_log: true + + - name: Update forgejo-runner-env secret + ansible.builtin.shell: + cmd: | + set -o pipefail + k3s kubectl create secret generic forgejo-runner-env \ + --namespace=forgejo-runner \ + --from-literal=RUNNER_TOKEN={{ _runner_token.stdout }} \ + --dry-run=client -o yaml | k3s kubectl apply -f - + executable: /bin/bash + when: _secret_exists.rc == 0 + changed_when: true + no_log: true diff --git a/argocd/apps/forgejo-runner-amd64.yaml b/argocd/apps/forgejo-runner-amd64.yaml new file mode 100644 index 0000000..1e82c79 --- /dev/null +++ b/argocd/apps/forgejo-runner-amd64.yaml @@ -0,0 +1,17 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: forgejo-runner-amd64 + namespace: argocd +spec: + project: default + source: + repoURL: https://forge.ops.eblu.me/eblume/blumeops.git + targetRevision: main + path: argocd/manifests/forgejo-runner-amd64 + destination: + server: https://ringtail.tail8d86e.ts.net:6443 + namespace: forgejo-runner + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/manifests/forgejo-runner-amd64/configmap.yaml b/argocd/manifests/forgejo-runner-amd64/configmap.yaml new file mode 100644 index 0000000..bc14942 --- /dev/null +++ b/argocd/manifests/forgejo-runner-amd64/configmap.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: forgejo-runner-config + namespace: forgejo-runner +data: + config.yaml: | + log: + level: info + + runner: + file: /data/.runner + capacity: 2 + timeout: 3h + envs: + DOCKER_HOST: tcp://127.0.0.1:2375 + TZ: America/Los_Angeles + + container: + network: "host" + docker_host: tcp://127.0.0.1:2375 + daemon.json: | + { + "registry-mirrors": ["https://registry.ops.eblu.me"] + } diff --git a/argocd/manifests/forgejo-runner-amd64/deployment.yaml b/argocd/manifests/forgejo-runner-amd64/deployment.yaml new file mode 100644 index 0000000..ae426aa --- /dev/null +++ b/argocd/manifests/forgejo-runner-amd64/deployment.yaml @@ -0,0 +1,89 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: forgejo-runner-amd64 + namespace: forgejo-runner + labels: + app: forgejo-runner-amd64 +spec: + replicas: 1 + selector: + matchLabels: + app: forgejo-runner-amd64 + template: + metadata: + labels: + app: forgejo-runner-amd64 + spec: + containers: + # Forgejo runner daemon + - name: runner + image: code.forgejo.org/forgejo/runner:6.3.1 + env: + - name: TZ + value: America/Los_Angeles + - name: DOCKER_HOST + value: tcp://localhost:2375 + - name: FORGEJO_URL + value: "https://forge.ops.eblu.me" + - name: RUNNER_NAME + value: "k8s-amd64-runner" + - name: RUNNER_LABELS + value: "k8s-amd64:docker://registry.ops.eblu.me/blumeops/forgejo-runner:v3.2.0-amd64" + command: + - /bin/sh + - -c + - | + # Wait for DinD to be ready + echo "Waiting for Docker daemon..." + while ! wget -q -O /dev/null http://localhost:2375/_ping 2>/dev/null; do + sleep 1 + done + echo "Docker daemon ready" + + # Register if not already registered + if [ ! -f /data/.runner ]; then + echo "Registering runner..." + forgejo-runner register \ + --instance "$FORGEJO_URL" \ + --token "$RUNNER_TOKEN" \ + --name "$RUNNER_NAME" \ + --labels "$RUNNER_LABELS" \ + --no-interactive + fi + + # Start daemon + exec forgejo-runner daemon --config /config/config.yaml + envFrom: + - secretRef: + name: forgejo-runner-env + volumeMounts: + - name: data + mountPath: /data + - name: config + mountPath: /config + + # Docker-in-Docker sidecar + - name: dind + image: docker:27-dind + securityContext: + privileged: true + env: + - name: DOCKER_TLS_CERTDIR + value: "" + volumeMounts: + - name: dind-storage + mountPath: /var/lib/docker + - name: config + mountPath: /etc/docker/daemon.json + subPath: daemon.json + readOnly: true + + volumes: + - name: data + emptyDir: {} + - name: dind-storage + emptyDir: {} + - name: config + configMap: + name: forgejo-runner-config diff --git a/argocd/manifests/forgejo-runner-amd64/kustomization.yaml b/argocd/manifests/forgejo-runner-amd64/kustomization.yaml new file mode 100644 index 0000000..6f6f6e5 --- /dev/null +++ b/argocd/manifests/forgejo-runner-amd64/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - configmap.yaml + - deployment.yaml diff --git a/argocd/manifests/forgejo-runner-amd64/namespace.yaml b/argocd/manifests/forgejo-runner-amd64/namespace.yaml new file mode 100644 index 0000000..19441b1 --- /dev/null +++ b/argocd/manifests/forgejo-runner-amd64/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: forgejo-runner diff --git a/docs/changelog.d/feature-k3s-ringtail-runner.feature.md b/docs/changelog.d/feature-k3s-ringtail-runner.feature.md new file mode 100644 index 0000000..5d69fda --- /dev/null +++ b/docs/changelog.d/feature-k3s-ringtail-runner.feature.md @@ -0,0 +1 @@ +K3s cluster on ringtail with Forgejo Actions runner (`k8s-amd64` label) for native amd64 container builds, managed via ArgoCD multi-cluster. diff --git a/docs/reference/infrastructure/ringtail.md b/docs/reference/infrastructure/ringtail.md index 7906979..686c0eb 100644 --- a/docs/reference/infrastructure/ringtail.md +++ b/docs/reference/infrastructure/ringtail.md @@ -45,6 +45,34 @@ mise run provision-ringtail This updates `flake.lock` via Dagger, verifies the current commit is pushed to forge, then deploys the exact commit via ansible. If the lockfile changed, it stages the file and exits so you can commit and re-run. +## K3s Cluster + +Ringtail runs a single-node k3s cluster for native amd64 workloads, registered in [[argocd|ArgoCD]] on indri as `k3s-ringtail`. + +- **Disabled components:** Traefik, ServiceLB, metrics-server (minimal footprint) +- **TLS SAN:** `ringtail.tail8d86e.ts.net` (ArgoCD connects via Tailscale) +- **Registry mirrors:** Containerd pulls through Zot on indri (`registry.ops.eblu.me`) +- **Token:** `/etc/k3s/token` (generated on first provision) +- **Kubeconfig:** `/etc/rancher/k3s/k3s.yaml` (world-readable via `--write-kubeconfig-mode=644`) + +### Workloads + +| Workload | Namespace | Label | +|----------|-----------|-------| +| Forgejo Runner (amd64) | `forgejo-runner` | `k8s-amd64` | + +### Manual Cluster Registration + +After first provision, register the cluster in ArgoCD: + +```fish +ssh ringtail 'sudo cat /etc/rancher/k3s/k3s.yaml' | \ + sed 's|127.0.0.1|ringtail.tail8d86e.ts.net|' > /tmp/k3s-ringtail.yaml +set -x KUBECONFIG /tmp/k3s-ringtail.yaml +kubectl get nodes # verify access +argocd cluster add default --name k3s-ringtail +``` + ## Maintenance Notes **1Password:** Desktop app must be running for `op` CLI. Use `$mod+Shift+minus` to send to scratchpad. diff --git a/nixos/ringtail/configuration.nix b/nixos/ringtail/configuration.nix index 9e5ecec..79bcd79 100644 --- a/nixos/ringtail/configuration.nix +++ b/nixos/ringtail/configuration.nix @@ -96,12 +96,32 @@ in dedicatedServer.openFirewall = true; }; + # K3s single-node cluster + services.k3s = { + enable = true; + role = "server"; + tokenFile = "/etc/k3s/token"; + extraFlags = toString [ + "--disable=traefik" + "--disable=servicelb" + "--disable=metrics-server" + "--write-kubeconfig-mode=644" + "--tls-san=ringtail.tail8d86e.ts.net" + ]; + }; + + # K3s containerd registry mirrors (pull through Zot on indri) + environment.etc."rancher/k3s/registries.yaml".source = ./k3s-registries.yaml; + # Tailscale services.tailscale = { enable = true; extraUpFlags = [ "--accept-routes" "--ssh" ]; }; + # Trust Tailscale interface (ArgoCD on indri connects via tailnet) + networking.firewall.trustedInterfaces = [ "tailscale0" ]; + # SSH services.openssh = { enable = true; @@ -124,6 +144,7 @@ in # System packages environment.systemPackages = with pkgs; [ git + kubectl python3 # required for Ansible vim htop diff --git a/nixos/ringtail/k3s-registries.yaml b/nixos/ringtail/k3s-registries.yaml new file mode 100644 index 0000000..312362c --- /dev/null +++ b/nixos/ringtail/k3s-registries.yaml @@ -0,0 +1,13 @@ +mirrors: + docker.io: + endpoint: + - "https://registry.ops.eblu.me" + ghcr.io: + endpoint: + - "https://registry.ops.eblu.me" + quay.io: + endpoint: + - "https://registry.ops.eblu.me" + registry.ops.eblu.me: + endpoint: + - "https://registry.ops.eblu.me"