diff --git a/.forgejo/workflows/build-container-nix.yaml b/.forgejo/workflows/build-container-nix.yaml new file mode 100644 index 0000000..f66691f --- /dev/null +++ b/.forgejo/workflows/build-container-nix.yaml @@ -0,0 +1,89 @@ +# Nix container build workflow +# Triggers on tags matching: -nix-v +# Builds from containers//default.nix using nix build +# Pushes to Zot registry via skopeo +# +# Examples: +# nettest-nix-v1.0.0 -> builds containers/nettest/default.nix +# myapp-nix-v2.1.0 -> builds containers/myapp/default.nix +name: Build Container (Nix) + +on: + push: + tags: + - '*-nix-v[0-9]*' + +jobs: + build: + runs-on: nix-container-builder + steps: + - name: Parse tag + id: parse + run: | + TAG="${GITHUB_REF_NAME}" + echo "Tag: $TAG" + + # Extract container name (everything before -nix-v) + # e.g., "nettest-nix-v1.0.0" -> "nettest" + CONTAINER="${TAG%-nix-v[0-9]*}" + VERSION="${TAG#"${CONTAINER}"-nix-}" + + echo "container=$CONTAINER" >> "$GITHUB_OUTPUT" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "Container: $CONTAINER" + echo "Version: $VERSION" + + - name: Checkout + uses: actions/checkout@v4 + + - name: Check if nix container exists + id: check + run: | + CONTAINER="${{ steps.parse.outputs.container }}" + CONTEXT="containers/$CONTAINER" + + if [ -f "$CONTEXT/default.nix" ]; then + echo "Found $CONTEXT/default.nix" + echo "exists=true" >> "$GITHUB_OUTPUT" + else + echo "No default.nix found at $CONTEXT/default.nix" + echo "exists=false" >> "$GITHUB_OUTPUT" + fi + + - name: Skip if container not found + if: steps.check.outputs.exists != 'true' + run: | + echo "========================================" + echo "Nix container not found: ${{ steps.parse.outputs.container }}" + echo "========================================" + echo "" + echo "Tag '${{ github.ref_name }}' does not match any nix container in containers/" + echo "" + echo "Available nix containers:" + for nix in containers/*/default.nix; do + [ -f "$nix" ] && echo " - $(basename "$(dirname "$nix")")" + done + echo "" + echo "Skipping build." + + - name: Build with nix + if: steps.check.outputs.exists == 'true' + id: build + run: | + CONTAINER="${{ steps.parse.outputs.container }}" + echo "Building containers/$CONTAINER/default.nix" + nix build -f "containers/$CONTAINER/default.nix" -o result + echo "Build complete: $(readlink result)" + + - name: Push to registry + if: steps.check.outputs.exists == 'true' + run: | + CONTAINER="${{ steps.parse.outputs.container }}" + VERSION="${{ steps.parse.outputs.version }}" + IMAGE="registry.ops.eblu.me/blumeops/$CONTAINER:$VERSION" + + echo "Pushing to $IMAGE" + skopeo copy \ + "docker-archive:result" \ + "docker://$IMAGE" + echo "Push complete: $IMAGE" diff --git a/.forgejo/workflows/build-container.yaml b/.forgejo/workflows/build-container.yaml index 98231cf..b76978f 100644 --- a/.forgejo/workflows/build-container.yaml +++ b/.forgejo/workflows/build-container.yaml @@ -17,6 +17,7 @@ on: jobs: build: + if: "!contains(github.ref_name, '-nix-v')" runs-on: k8s steps: - name: Parse tag diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 20281ec..ffc2bdf 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,3 +1,4 @@ self-hosted-runner: labels: - k8s + - nix-container-builder diff --git a/ansible/playbooks/ringtail.yml b/ansible/playbooks/ringtail.yml index f7e085a..74c4f54 100644 --- a/ansible/playbooks/ringtail.yml +++ b/ansible/playbooks/ringtail.yml @@ -3,6 +3,57 @@ hosts: ringtail become: true + pre_tasks: + - name: Fetch 1Password Connect credentials from 1Password + ansible.builtin.command: + cmd: op read "op://vg6xf6vvfmoh5hqjjhlhbeoaie/1Password Connect/credentials-file" + register: _op_credentials + changed_when: false + delegate_to: localhost + become: false + + - name: Fetch 1Password Connect token from 1Password + ansible.builtin.command: + cmd: op read "op://vg6xf6vvfmoh5hqjjhlhbeoaie/1Password Connect/token" + register: _op_token + changed_when: false + delegate_to: localhost + become: false + + - name: Fetch Forgejo runner registration token from 1Password + ansible.builtin.command: + cmd: op read "op://vg6xf6vvfmoh5hqjjhlhbeoaie/Forgejo Secrets/runner_reg" + register: _runner_reg + changed_when: false + delegate_to: localhost + become: false + + - name: Ensure /etc/forgejo-runner directory exists + ansible.builtin.file: + path: /etc/forgejo-runner + state: directory + mode: "0700" + + - name: Write Forgejo runner token file + ansible.builtin.copy: + content: "TOKEN={{ _runner_reg.stdout }}" + dest: /etc/forgejo-runner/token.env + mode: "0600" + no_log: true + + - name: Ensure /etc/k3s directory exists + ansible.builtin.file: + path: /etc/k3s + state: directory + mode: "0700" + + - name: Generate k3s token if not present + ansible.builtin.copy: + content: "{{ lookup('ansible.builtin.password', '/dev/null', chars=['hexdigits'], length=32) }}" + dest: /etc/k3s/token + mode: "0600" + force: false + tasks: - name: Ensure blumeops repo is present ansible.builtin.git: @@ -24,3 +75,42 @@ register: _ts_status changed_when: false failed_when: "'Running' not in _ts_status.stdout" + + post_tasks: + - name: Wait for k3s to be ready + ansible.builtin.command: k3s kubectl get nodes + register: _k3s_ready + changed_when: false + retries: 30 + delay: 5 + until: _k3s_ready.rc == 0 + + - name: Create 1password namespace + ansible.builtin.command: k3s kubectl create namespace 1password + register: _ns + changed_when: _ns.rc == 0 + failed_when: _ns.rc != 0 and 'AlreadyExists' not in _ns.stderr + + - name: Create or update op-credentials secret + ansible.builtin.shell: + cmd: | + set -o pipefail + k3s kubectl create secret generic op-credentials \ + --namespace=1password \ + --from-literal=1password-credentials.json='{{ _op_credentials.stdout }}' \ + --dry-run=client -o yaml | k3s kubectl apply -f - + executable: /run/current-system/sw/bin/bash + changed_when: true + no_log: true + + - name: Create or update onepassword-token secret + ansible.builtin.shell: + cmd: | + set -o pipefail + k3s kubectl create secret generic onepassword-token \ + --namespace=1password \ + --from-literal=token={{ _op_token.stdout }} \ + --dry-run=client -o yaml | k3s kubectl apply -f - + executable: /run/current-system/sw/bin/bash + changed_when: true + no_log: true diff --git a/argocd/apps/1password-connect-ringtail.yaml b/argocd/apps/1password-connect-ringtail.yaml new file mode 100644 index 0000000..408eb23 --- /dev/null +++ b/argocd/apps/1password-connect-ringtail.yaml @@ -0,0 +1,32 @@ +# 1Password Connect for ringtail k3s cluster +# Same chart/values as indri, different destination +# +# Prerequisites: +# 1. Bootstrap secrets via ansible (provision-ringtail creates 1password namespace, +# op-credentials and onepassword-token secrets) +# 2. Sync BEFORE external-secrets-ringtail +# +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: 1password-connect-ringtail + namespace: argocd +spec: + project: default + sources: + - repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/connect-helm-charts.git + targetRevision: connect-2.3.0 + path: charts/connect + helm: + releaseName: onepassword-connect + valueFiles: + - $values/argocd/manifests/1password-connect/values.yaml + - repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/blumeops.git + targetRevision: main + ref: values + destination: + server: https://ringtail.tail8d86e.ts.net:6443 + namespace: 1password + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/apps/external-secrets-config-ringtail.yaml b/argocd/apps/external-secrets-config-ringtail.yaml new file mode 100644 index 0000000..d3f9e58 --- /dev/null +++ b/argocd/apps/external-secrets-config-ringtail.yaml @@ -0,0 +1,24 @@ +# External Secrets Configuration for ringtail k3s cluster +# Same ClusterSecretStore manifests as indri, different destination +# +# Prerequisites: +# - 1password-connect-ringtail is deployed and healthy +# - external-secrets-ringtail operator is deployed and CRDs are installed +# +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: external-secrets-config-ringtail + namespace: argocd +spec: + project: default + source: + repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/blumeops.git + targetRevision: main + path: argocd/manifests/external-secrets + destination: + server: https://ringtail.tail8d86e.ts.net:6443 + namespace: external-secrets + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/apps/external-secrets-crds-ringtail.yaml b/argocd/apps/external-secrets-crds-ringtail.yaml new file mode 100644 index 0000000..a23eae3 --- /dev/null +++ b/argocd/apps/external-secrets-crds-ringtail.yaml @@ -0,0 +1,24 @@ +# External Secrets Operator CRDs for ringtail k3s cluster +# Same CRDs source as indri, different destination +# +# Must be synced BEFORE external-secrets-ringtail operator app. +# +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: external-secrets-crds-ringtail + namespace: argocd +spec: + project: default + source: + repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/external-secrets.git + targetRevision: helm-chart-2.0.0 + path: config/crds/bases + directory: + exclude: 'kustomization.yaml' + destination: + server: https://ringtail.tail8d86e.ts.net:6443 + syncPolicy: + syncOptions: + - ServerSideApply=true + - CreateNamespace=false diff --git a/argocd/apps/external-secrets-ringtail.yaml b/argocd/apps/external-secrets-ringtail.yaml new file mode 100644 index 0000000..c54c51b --- /dev/null +++ b/argocd/apps/external-secrets-ringtail.yaml @@ -0,0 +1,32 @@ +# External Secrets Operator for ringtail k3s cluster +# Same chart/values as indri, different destination +# +# Prerequisites: +# - 1password-connect-ringtail must be deployed and healthy +# - external-secrets-crds-ringtail must be synced first +# +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: external-secrets-ringtail + namespace: argocd +spec: + project: default + sources: + - repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/external-secrets.git + targetRevision: helm-chart-2.0.0 + path: deploy/charts/external-secrets + helm: + releaseName: external-secrets + valueFiles: + - $values/argocd/manifests/external-secrets/values.yaml + - repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/blumeops.git + targetRevision: main + ref: values + destination: + server: https://ringtail.tail8d86e.ts.net:6443 + namespace: external-secrets + syncPolicy: + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/docs/changelog.d/feature-k3s-ringtail-runner.feature.md b/docs/changelog.d/feature-k3s-ringtail-runner.feature.md new file mode 100644 index 0000000..e51251d --- /dev/null +++ b/docs/changelog.d/feature-k3s-ringtail-runner.feature.md @@ -0,0 +1 @@ +Systemd Forgejo Actions runner on ringtail (`nix-container-builder` label) for building containers with `nix build` and pushing via `skopeo`. K3s cluster retained for future workloads. 1Password Connect + External Secrets Operator available for k8s secret management. diff --git a/docs/reference/infrastructure/ringtail.md b/docs/reference/infrastructure/ringtail.md index 7906979..45bc757 100644 --- a/docs/reference/infrastructure/ringtail.md +++ b/docs/reference/infrastructure/ringtail.md @@ -45,6 +45,51 @@ mise run provision-ringtail This updates `flake.lock` via Dagger, verifies the current commit is pushed to forge, then deploys the exact commit via ansible. If the lockfile changed, it stages the file and exits so you can commit and re-run. +## K3s Cluster + +Ringtail runs a single-node k3s cluster for native amd64 workloads, registered in [[argocd|ArgoCD]] on indri as `k3s-ringtail`. + +- **Disabled components:** Traefik, ServiceLB, metrics-server (minimal footprint) +- **TLS SAN:** `ringtail.tail8d86e.ts.net` (ArgoCD connects via Tailscale) +- **Registry mirrors:** Containerd pulls through Zot on indri (`registry.ops.eblu.me`) +- **Token:** `/etc/k3s/token` (generated on first provision) +- **Kubeconfig:** `/etc/rancher/k3s/k3s.yaml` (world-readable via `--write-kubeconfig-mode=644`) + +### Secrets Management + +1Password Connect + External Secrets Operator syncs secrets from 1Password to k8s, matching the [[1password|indri pattern]]. Bootstrap credentials (`op-credentials`, `onepassword-token`) are provisioned by Ansible; ArgoCD manages the operator stack. + +Sync order: `1password-connect-ringtail` -> `external-secrets-crds-ringtail` -> `external-secrets-ringtail` -> `external-secrets-config-ringtail` + +### Workloads + +No k8s workloads currently deployed. K3s is available for future workloads (e.g. Frigate, running nix-built containers). + +### Manual Cluster Registration + +After first provision, register the cluster in ArgoCD: + +```fish +ssh ringtail 'sudo cat /etc/rancher/k3s/k3s.yaml' | \ + sed 's|127.0.0.1|ringtail.tail8d86e.ts.net|' > /tmp/k3s-ringtail.yaml +set -x KUBECONFIG /tmp/k3s-ringtail.yaml +kubectl get nodes # verify access +argocd cluster add default --name k3s-ringtail +``` + +## Systemd Services + +### Forgejo Actions Runner + +A native Forgejo Actions runner (`ringtail-nix-builder`) runs as a systemd service via the NixOS `services.gitea-actions-runner` module. It builds containers using `nix build` and pushes them to Zot via `skopeo`. + +| Property | Value | +|----------|-------| +| **Label** | `nix-container-builder` | +| **Execution** | Host (no containers) | +| **Token** | `/etc/forgejo-runner/token.env` (provisioned by Ansible) | +| **Service unit** | `gitea-runner-nix_container_builder.service` | + ## Maintenance Notes **1Password:** Desktop app must be running for `op` CLI. Use `$mod+Shift+minus` to send to scratchpad. diff --git a/mise-tasks/container-list b/mise-tasks/container-list index 4a168ea..0122e77 100755 --- a/mise-tasks/container-list +++ b/mise-tasks/container-list @@ -10,16 +10,24 @@ echo "Container Images" echo "================" echo "" -# Find all container directories with Dockerfiles +# Find all container directories with Dockerfiles or default.nix for dir in "$CONTAINER_DIR"/*/; do [[ -d "$dir" ]] || continue - [[ -f "$dir/Dockerfile" ]] || continue + + # Determine build type + if [[ -f "$dir/default.nix" ]]; then + build_type="nix" + elif [[ -f "$dir/Dockerfile" ]]; then + build_type="dockerfile" + else + continue + fi # Extract container name from directory container=$(basename "$dir") image="blumeops/$container" - echo "📦 $container" + echo "[$build_type] $container" echo " Image: $REGISTRY/$image" echo " Path: $dir" diff --git a/mise-tasks/container-tag-and-release b/mise-tasks/container-tag-and-release index 068b164..493f00f 100755 --- a/mise-tasks/container-tag-and-release +++ b/mise-tasks/container-tag-and-release @@ -19,28 +19,39 @@ if [[ ! "$VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then exit 1 fi -TAG="${CONTAINER}-${VERSION}" +# Determine build type: Nix or Dockerfile +CONTAINER_DIR="containers/${CONTAINER}" +if [[ -f "$CONTAINER_DIR/default.nix" ]]; then + BUILD_TYPE="nix" + TAG="${CONTAINER}-nix-${VERSION}" +elif [[ -f "$CONTAINER_DIR/Dockerfile" ]]; then + BUILD_TYPE="dockerfile" + TAG="${CONTAINER}-${VERSION}" +else + echo "Error: No Dockerfile or default.nix found in '$CONTAINER_DIR'" + echo "" + echo "Available containers:" + for dir in containers/*/; do + [[ -d "$dir" ]] || continue + name=$(basename "$dir") + if [[ -f "$dir/default.nix" ]]; then + echo " - $name (nix)" + elif [[ -f "$dir/Dockerfile" ]]; then + echo " - $name (dockerfile)" + fi + done + exit 1 +fi echo "Creating release tag: $TAG" +echo "Build type: $BUILD_TYPE" echo "" # Check if tag already exists if git rev-parse "$TAG" >/dev/null 2>&1; then echo "Error: Tag '$TAG' already exists" echo "Existing tags for $CONTAINER:" - git tag -l "${CONTAINER}-v*" | sort -V | tail -5 - exit 1 -fi - -# Check if container directory exists -CONTAINER_DIR="containers/${CONTAINER}" -if [[ ! -f "$CONTAINER_DIR/Dockerfile" ]]; then - echo "Error: No Dockerfile found at '$CONTAINER_DIR/Dockerfile'" - echo "" - echo "Available containers:" - for dir in containers/*/; do - [[ -d "$dir" ]] && echo " - $(basename "$dir")" - done + git tag -l "${CONTAINER}-*v*" | sort -V | tail -5 exit 1 fi diff --git a/mise-tasks/ensure-k3s-ringtail-kubectl-config b/mise-tasks/ensure-k3s-ringtail-kubectl-config new file mode 100755 index 0000000..d8a1b80 --- /dev/null +++ b/mise-tasks/ensure-k3s-ringtail-kubectl-config @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +#MISE description="Ensure kubectl config for k3s-ringtail is set up on this workstation" + +set -euo pipefail + +CONFIG_DIR="$HOME/.kube/k3s-ringtail" +CONFIG_FILE="$CONFIG_DIR/config.yml" + +echo "Ensuring k3s-ringtail kubectl config..." + +# Create directory if needed +mkdir -p "$CONFIG_DIR" + +# Fetch kubeconfig from ringtail and extract the CA cert +echo "Fetching kubeconfig from ringtail..." +RAW_CONFIG=$(ssh ringtail 'sudo cat /etc/rancher/k3s/k3s.yaml') + +# Extract and decode the CA certificate +echo "$RAW_CONFIG" | grep certificate-authority-data | awk '{print $2}' | base64 -d > "$CONFIG_DIR/ca.crt" + +# Extract and decode the client certificate +echo "$RAW_CONFIG" | grep client-certificate-data | awk '{print $2}' | base64 -d > "$CONFIG_DIR/client.crt" + +# Extract and decode the client key +echo "$RAW_CONFIG" | grep client-key-data | awk '{print $2}' | base64 -d > "$CONFIG_DIR/client.key" +chmod 600 "$CONFIG_DIR/client.key" + +# Write kubeconfig with file-based certs and tailscale hostname +cat > "$CONFIG_FILE" << EOF +apiVersion: v1 +kind: Config +clusters: +- cluster: + certificate-authority: $CONFIG_DIR/ca.crt + server: https://ringtail.tail8d86e.ts.net:6443 + name: k3s-ringtail +contexts: +- context: + cluster: k3s-ringtail + user: k3s-ringtail + name: k3s-ringtail +current-context: k3s-ringtail +users: +- name: k3s-ringtail + user: + client-certificate: $CONFIG_DIR/client.crt + client-key: $CONFIG_DIR/client.key +EOF + +echo "Config written to $CONFIG_FILE" + +# Warn if KUBECONFIG doesn't include this file +if [[ -z "${KUBECONFIG:-}" ]] || [[ ":$KUBECONFIG:" != *":$CONFIG_FILE:"* ]]; then + echo "" + echo "WARNING: KUBECONFIG does not include $CONFIG_FILE" + echo "Add this to your shell config:" + echo " export KUBECONFIG=\"\$KUBECONFIG:$CONFIG_FILE\"" +fi + +echo "" +echo "Test with: kubectl --context=k3s-ringtail get nodes" diff --git a/mise-tasks/services-check b/mise-tasks/services-check index 31c8cc5..a77c53d 100755 --- a/mise-tasks/services-check +++ b/mise-tasks/services-check @@ -87,6 +87,9 @@ echo "" echo "Ringtail (NixOS):" check_service "ssh" "ssh -o ConnectTimeout=5 ringtail true" check_service "tailscale" "ssh ringtail 'tailscale status --self --json' | jq -e '.Self.Online' > /dev/null" +check_service "k3s" "ssh ringtail 'k3s kubectl get nodes --no-headers | grep -q Ready'" +check_service "k3s-apiserver (remote)" "kubectl --context=k3s-ringtail get --raw /healthz" +check_service "forgejo-runner" "ssh ringtail 'systemctl is-active gitea-runner-nix_container_builder.service'" echo "" echo "Public services (via Fly.io):" diff --git a/nixos/ringtail/configuration.nix b/nixos/ringtail/configuration.nix index 9e5ecec..d765a23 100644 --- a/nixos/ringtail/configuration.nix +++ b/nixos/ringtail/configuration.nix @@ -96,12 +96,32 @@ in dedicatedServer.openFirewall = true; }; + # K3s single-node cluster + services.k3s = { + enable = true; + role = "server"; + tokenFile = "/etc/k3s/token"; + extraFlags = toString [ + "--disable=traefik" + "--disable=servicelb" + "--disable=metrics-server" + "--write-kubeconfig-mode=644" + "--tls-san=ringtail.tail8d86e.ts.net" + ]; + }; + + # K3s containerd registry mirrors (pull through Zot on indri) + environment.etc."rancher/k3s/registries.yaml".source = ./k3s-registries.yaml; + # Tailscale services.tailscale = { enable = true; extraUpFlags = [ "--accept-routes" "--ssh" ]; }; + # Trust Tailscale interface (ArgoCD on indri connects via tailnet) + networking.firewall.trustedInterfaces = [ "tailscale0" ]; + # SSH services.openssh = { enable = true; @@ -124,6 +144,7 @@ in # System packages environment.systemPackages = with pkgs; [ git + kubectl python3 # required for Ansible vim htop @@ -369,9 +390,35 @@ in "d /mnt/storage2 0755 eblume users -" ]; + # Forgejo Actions runner (nix container builder) + services.gitea-actions-runner = { + package = pkgs.forgejo-runner; + instances.nix_container_builder = { + enable = true; + name = "ringtail-nix-builder"; + url = "https://forge.ops.eblu.me"; + tokenFile = "/etc/forgejo-runner/token.env"; + labels = [ "nix-container-builder:host" ]; + hostPackages = with pkgs; [ + bash coreutils curl gawk gitMinimal gnused nodejs wget + nix skopeo + ]; + settings = { + log.level = "info"; + runner = { + capacity = 1; + timeout = "3h"; + }; + }; + }; + }; + # Enable nix flakes nix.settings.experimental-features = [ "nix-command" "flakes" ]; + # Allow the runner's dynamic user to access the nix daemon + nix.settings.trusted-users = [ "gitea-runner" ]; + # NixOS release system.stateVersion = "25.11"; } diff --git a/nixos/ringtail/k3s-registries.yaml b/nixos/ringtail/k3s-registries.yaml new file mode 100644 index 0000000..312362c --- /dev/null +++ b/nixos/ringtail/k3s-registries.yaml @@ -0,0 +1,13 @@ +mirrors: + docker.io: + endpoint: + - "https://registry.ops.eblu.me" + ghcr.io: + endpoint: + - "https://registry.ops.eblu.me" + quay.io: + endpoint: + - "https://registry.ops.eblu.me" + registry.ops.eblu.me: + endpoint: + - "https://registry.ops.eblu.me"