diff --git a/ansible/roles/minikube/defaults/main.yml b/ansible/roles/minikube/defaults/main.yml index e753f38..2558c91 100644 --- a/ansible/roles/minikube/defaults/main.yml +++ b/ansible/roles/minikube/defaults/main.yml @@ -1,15 +1,16 @@ --- # Minikube cluster configuration +# Uses docker driver - requires Docker Desktop to be installed and running +# with at least 12GB memory allocated in Docker Desktop settings minikube_cpus: 6 -minikube_memory: 12288 +minikube_memory: 11264 # Leave ~1GB headroom for Docker Desktop overhead minikube_disk_size: "200g" -minikube_driver: qemu2 -minikube_network: socket_vmnet -minikube_container_runtime: containerd +minikube_driver: docker +minikube_container_runtime: docker # Remote access configuration # These allow kubectl from other machines (e.g., gilbert) to connect -# k8s.tail8d86e.ts.net is exposed via Tailscale service (TCP passthrough) +# k8s.tail8d86e.ts.net is exposed via Tailscale service (TCP passthrough to localhost) minikube_apiserver_names: - k8s.tail8d86e.ts.net - indri diff --git a/ansible/roles/minikube/files/com.blumeops.minikube-mount.plist b/ansible/roles/minikube/files/com.blumeops.minikube-mount.plist deleted file mode 100644 index 2313c52..0000000 --- a/ansible/roles/minikube/files/com.blumeops.minikube-mount.plist +++ /dev/null @@ -1,41 +0,0 @@ - - - - - Label - com.blumeops.minikube-mount - ProgramArguments - - /bin/bash - -c - -# Wait for minikube to be running -for i in {1..60}; do - if /opt/homebrew/bin/minikube status | grep -q "Running"; then - break - fi - sleep 5 -done - -# Wait for NFS mount to be available -for i in {1..30}; do - if mount | grep -q "/Volumes/torrents-nfs"; then - break - fi - sleep 2 -done - -# Start the mount (this blocks until killed) -exec /opt/homebrew/bin/minikube mount /Volumes/torrents-nfs:/mnt/torrents - - - RunAtLoad - - KeepAlive - - StandardErrorPath - /tmp/minikube-mount.err - StandardOutPath - /tmp/minikube-mount.log - - diff --git a/ansible/roles/minikube/files/com.blumeops.nfs-torrents.plist b/ansible/roles/minikube/files/com.blumeops.nfs-torrents.plist deleted file mode 100644 index 1cec7a6..0000000 --- a/ansible/roles/minikube/files/com.blumeops.nfs-torrents.plist +++ /dev/null @@ -1,24 +0,0 @@ - - - - - Label - com.blumeops.nfs-torrents - ProgramArguments - - /sbin/mount - -t - nfs - -o - resvport,rw - sifaka:/volume1/torrents - /Volumes/torrents-nfs - - RunAtLoad - - StandardErrorPath - /tmp/nfs-torrents.err - StandardOutPath - /tmp/nfs-torrents.log - - diff --git a/ansible/roles/minikube/files/zot-mirror.conf b/ansible/roles/minikube/files/zot-mirror.conf deleted file mode 100644 index df9bc11..0000000 --- a/ansible/roles/minikube/files/zot-mirror.conf +++ /dev/null @@ -1,43 +0,0 @@ -# Zot pull-through cache on indri -# Uses host.containers.internal which is stable across restarts -# Applied by ansible minikube role - -# Direct access to Zot for private images (blumeops/*) -[[registry]] -prefix = "host.containers.internal:5050" -location = "host.containers.internal:5050" -insecure = true - -# Tailscale hostname for Zot - redirects to local access -# Allows manifests to use registry.tail8d86e.ts.net which is cleaner -[[registry]] -prefix = "registry.tail8d86e.ts.net" -location = "registry.tail8d86e.ts.net" - -[[registry.mirror]] -location = "host.containers.internal:5050" -insecure = true - -[[registry]] -prefix = "docker.io" -location = "docker.io" - -[[registry.mirror]] -location = "host.containers.internal:5050/docker.io" -insecure = true - -[[registry]] -prefix = "ghcr.io" -location = "ghcr.io" - -[[registry.mirror]] -location = "host.containers.internal:5050/ghcr.io" -insecure = true - -[[registry]] -prefix = "quay.io" -location = "quay.io" - -[[registry.mirror]] -location = "host.containers.internal:5050/quay.io" -insecure = true diff --git a/ansible/roles/minikube/handlers/main.yml b/ansible/roles/minikube/handlers/main.yml index 7d62a8b..44ad747 100644 --- a/ansible/roles/minikube/handlers/main.yml +++ b/ansible/roles/minikube/handlers/main.yml @@ -12,16 +12,3 @@ ansible.builtin.command: cmd: minikube ssh --native-ssh=false "sudo systemctl restart containerd" changed_when: true - -- name: Load NFS mount LaunchDaemon - ansible.builtin.command: - cmd: launchctl load /Library/LaunchDaemons/com.blumeops.nfs-torrents.plist - become: true - failed_when: false - changed_when: true - -- name: Load minikube mount LaunchAgent - ansible.builtin.command: - cmd: launchctl load {{ ansible_facts['env']['HOME'] }}/Library/LaunchAgents/com.blumeops.minikube-mount.plist - failed_when: false - changed_when: true diff --git a/ansible/roles/minikube/tasks/main.yml b/ansible/roles/minikube/tasks/main.yml index a471d82..9e9fdd3 100644 --- a/ansible/roles/minikube/tasks/main.yml +++ b/ansible/roles/minikube/tasks/main.yml @@ -1,41 +1,20 @@ --- # Minikube installation and cluster setup for indri -# Uses qemu2 driver for full VM with kernel mount capabilities (NFS, SMB, etc.) -# Requires socket_vmnet for proper networking (minikube service/tunnel commands) +# Uses docker driver - requires Docker Desktop to be installed manually first +# (Docker Desktop requires GUI setup, so it's not automated in this role) +# +# Prerequisites: +# 1. Install Docker Desktop: brew install --cask docker +# 2. Launch Docker Desktop and complete setup wizard +# 3. Configure Docker Desktop with at least 12GB memory # # NOTE: minikube start may have issues when run via SSH. # If cluster fails to start, manually run on indri: -# minikube start --driver=qemu2 --network=socket_vmnet --container-runtime=containerd \ -# --cpus=6 --memory=12288 --disk-size=200g \ +# minikube start --driver=docker --container-runtime=docker \ +# --cpus=6 --memory=11264 --disk-size=200g \ # --apiserver-names=k8s.tail8d86e.ts.net --apiserver-names=indri \ # --apiserver-port=6443 --listen-address=0.0.0.0 -- name: Install qemu via homebrew (required for qemu2 driver) - community.general.homebrew: - name: qemu - state: present - -- name: Install socket_vmnet via homebrew (required for qemu2 networking) - community.general.homebrew: - name: socket_vmnet - state: present - -- name: Check if socket_vmnet process is running - ansible.builtin.command: - cmd: pgrep socket_vmnet - register: minikube_socket_vmnet_status - changed_when: false - failed_when: false - -- name: Start socket_vmnet service - ansible.builtin.command: - cmd: brew services start socket_vmnet - become: true - register: minikube_socket_vmnet_start - changed_when: "'Successfully started' in minikube_socket_vmnet_start.stdout" - failed_when: false - when: minikube_socket_vmnet_status.rc != 0 - - name: Install minikube via homebrew community.general.homebrew: name: minikube @@ -46,6 +25,18 @@ name: kubectl state: present +- name: Check if Docker is running + ansible.builtin.command: + cmd: docker info + register: minikube_docker_status + changed_when: false + failed_when: false + +- name: Warn if Docker is not running + ansible.builtin.debug: + msg: "WARNING: Docker does not appear to be running. Please start Docker Desktop manually." + when: minikube_docker_status.rc != 0 + - name: Check if minikube cluster exists ansible.builtin.command: cmd: minikube status --format={% raw %}'{{.Host}}'{% endraw %} @@ -58,7 +49,6 @@ cmd: > minikube start --driver={{ minikube_driver }} - --network={{ minikube_network }} --container-runtime={{ minikube_container_runtime }} --cpus={{ minikube_cpus }} --memory={{ minikube_memory }} @@ -70,8 +60,10 @@ --listen-address={{ minikube_listen_address }} register: minikube_start changed_when: minikube_start.rc == 0 - failed_when: false # Don't fail - may need manual intervention like podman - when: minikube_status.rc != 0 or 'Running' not in minikube_status.stdout + failed_when: false # Don't fail - may need manual intervention + when: + - minikube_docker_status.rc == 0 + - minikube_status.rc != 0 or 'Running' not in minikube_status.stdout - name: Check minikube status after start attempt ansible.builtin.command: @@ -85,84 +77,146 @@ msg: "WARNING: minikube may not have started properly. Run 'minikube start' manually on indri if needed. Status: {{ minikube_final_status.stdout | default('unknown') }}" when: minikube_final_status.rc != 0 or 'Running' not in minikube_final_status.stdout -# Configure VM to access zot registry on host -# The VM can't resolve Tailscale hostnames, so we add a hosts entry -# and configure containerd to use the local zot instance -- name: Add registry hostname to VM hosts file - ansible.builtin.command: - cmd: minikube ssh --native-ssh=false "grep -q 'registry.tail8d86e.ts.net' /etc/hosts || echo '192.168.105.1 registry.tail8d86e.ts.net' | sudo tee -a /etc/hosts" - register: minikube_hosts_entry - changed_when: "'registry.tail8d86e.ts.net' in minikube_hosts_entry.stdout" - when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout +# Configure containerd to use zot registry as pull-through cache +# With docker driver, use host.minikube.internal to reach the host +# Zot runs on indri:5050 and caches images from docker.io, ghcr.io, quay.io -- name: Create containerd registry mirror directory +- name: Create containerd registry mirror directories ansible.builtin.command: - cmd: minikube ssh --native-ssh=false "sudo mkdir -p /etc/containerd/certs.d/registry.tail8d86e.ts.net" - register: minikube_registry_dir + cmd: minikube ssh --native-ssh=false "sudo mkdir -p /etc/containerd/certs.d/{{ item }}" + loop: + - registry.tail8d86e.ts.net + - docker.io + - ghcr.io + - quay.io changed_when: false when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout -- name: Check containerd registry mirror config +# Private registry (registry.tail8d86e.ts.net) - direct to zot +- name: Check registry.tail8d86e.ts.net config ansible.builtin.command: cmd: minikube ssh --native-ssh=false "cat /etc/containerd/certs.d/registry.tail8d86e.ts.net/hosts.toml 2>/dev/null || echo ''" - register: minikube_registry_config_current + register: minikube_registry_config changed_when: false when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout -- name: Configure containerd registry mirror for zot +- name: Configure registry.tail8d86e.ts.net mirror ansible.builtin.command: cmd: | minikube ssh --native-ssh=false 'echo "server = \"http://host.minikube.internal:5050\" [host.\"http://host.minikube.internal:5050\"] - capabilities = [\"pull\", \"resolve\"] + capabilities = [\"pull\", \"resolve\", \"push\"] skip_verify = true" | sudo tee /etc/containerd/certs.d/registry.tail8d86e.ts.net/hosts.toml' - register: minikube_registry_config changed_when: true when: - minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout - - "'host.minikube.internal:5050' not in minikube_registry_config_current.stdout" + - "'host.minikube.internal:5050' not in minikube_registry_config.stdout" notify: Restart containerd in minikube -# Set up persistent NFS mount from sifaka and minikube mount passthrough -# NFS mount uses LaunchDaemon (runs as root at boot) -# Minikube mount uses LaunchAgent (runs in user GUI session at login) -# -# NOTE: Tasks with become:true require passwordless sudo on indri -# (configured via /etc/sudoers.d/erichblume) +# Docker Hub (docker.io) - zot pull-through cache +- name: Check docker.io config + ansible.builtin.command: + cmd: minikube ssh --native-ssh=false "cat /etc/containerd/certs.d/docker.io/hosts.toml 2>/dev/null || echo ''" + register: minikube_dockerio_config + changed_when: false + when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout -- name: Check if NFS mount point exists - ansible.builtin.stat: - path: /Volumes/torrents-nfs - register: minikube_nfs_mount_point +- name: Configure docker.io mirror through zot + ansible.builtin.command: + cmd: | + minikube ssh --native-ssh=false 'echo "server = \"https://registry-1.docker.io\" -- name: Create NFS mount point - ansible.builtin.file: - path: /Volumes/torrents-nfs - state: directory - mode: "0755" - become: true - when: not minikube_nfs_mount_point.stat.exists + [host.\"http://host.minikube.internal:5050\"] + capabilities = [\"pull\", \"resolve\"] + skip_verify = true" | sudo tee /etc/containerd/certs.d/docker.io/hosts.toml' + changed_when: true + when: + - minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout + - "'host.minikube.internal:5050' not in minikube_dockerio_config.stdout" + notify: Restart containerd in minikube -- name: Check if NFS LaunchDaemon is installed - ansible.builtin.stat: - path: /Library/LaunchDaemons/com.blumeops.nfs-torrents.plist - register: minikube_nfs_launchdaemon +# GitHub Container Registry (ghcr.io) - zot pull-through cache +- name: Check ghcr.io config + ansible.builtin.command: + cmd: minikube ssh --native-ssh=false "cat /etc/containerd/certs.d/ghcr.io/hosts.toml 2>/dev/null || echo ''" + register: minikube_ghcr_config + changed_when: false + when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout -- name: Install NFS mount LaunchDaemon - ansible.builtin.copy: - src: com.blumeops.nfs-torrents.plist - dest: /Library/LaunchDaemons/com.blumeops.nfs-torrents.plist - owner: root - group: wheel - mode: "0644" - become: true - notify: Load NFS mount LaunchDaemon - when: not minikube_nfs_launchdaemon.stat.exists +- name: Configure ghcr.io mirror through zot + ansible.builtin.command: + cmd: | + minikube ssh --native-ssh=false 'echo "server = \"https://ghcr.io\" -- name: Install minikube mount LaunchAgent - ansible.builtin.copy: - src: com.blumeops.minikube-mount.plist - dest: "{{ ansible_facts['env']['HOME'] }}/Library/LaunchAgents/com.blumeops.minikube-mount.plist" - mode: "0644" - notify: Load minikube mount LaunchAgent + [host.\"http://host.minikube.internal:5050\"] + capabilities = [\"pull\", \"resolve\"] + skip_verify = true" | sudo tee /etc/containerd/certs.d/ghcr.io/hosts.toml' + changed_when: true + when: + - minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout + - "'host.minikube.internal:5050' not in minikube_ghcr_config.stdout" + notify: Restart containerd in minikube + +# Quay.io - zot pull-through cache +- name: Check quay.io config + ansible.builtin.command: + cmd: minikube ssh --native-ssh=false "cat /etc/containerd/certs.d/quay.io/hosts.toml 2>/dev/null || echo ''" + register: minikube_quay_config + changed_when: false + when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout + +- name: Configure quay.io mirror through zot + ansible.builtin.command: + cmd: | + minikube ssh --native-ssh=false 'echo "server = \"https://quay.io\" + + [host.\"http://host.minikube.internal:5050\"] + capabilities = [\"pull\", \"resolve\"] + skip_verify = true" | sudo tee /etc/containerd/certs.d/quay.io/hosts.toml' + changed_when: true + when: + - minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout + - "'host.minikube.internal:5050' not in minikube_quay_config.stdout" + notify: Restart containerd in minikube + +# Configure Tailscale serve for k8s API access +# With docker driver, the API server port is dynamic (not fixed at 6443) +# We query the current port and configure tailscale serve accordingly +- name: Get minikube API server URL + ansible.builtin.command: + cmd: kubectl config view --minify -o jsonpath="{.clusters[0].cluster.server}" + register: minikube_api_url + changed_when: false + when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout + +- name: Extract API server port from URL + ansible.builtin.set_fact: + minikube_api_port: "{{ minikube_api_url.stdout | regex_search(':([0-9]+)$', '\\1') | first }}" + when: + - minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout + - minikube_api_url.stdout is defined + +- name: Check current tailscale serve config for k8s + ansible.builtin.command: + cmd: tailscale serve status --json + register: minikube_tailscale_serve_status + changed_when: false + when: minikube_api_port is defined + +- name: Parse tailscale serve k8s config + ansible.builtin.set_fact: + minikube_tailscale_k8s_tcp: "{{ ((minikube_tailscale_serve_status.stdout | from_json).Services['svc:k8s'].TCP['443'].TCPForward | default('')) }}" + when: + - minikube_api_port is defined + - minikube_tailscale_serve_status.stdout is defined + - "'svc:k8s' in (minikube_tailscale_serve_status.stdout | from_json).Services | default({})" + failed_when: false + +- name: Configure tailscale serve for k8s API + ansible.builtin.command: + cmd: tailscale serve --service="svc:k8s" --tcp=443 tcp://localhost:{{ minikube_api_port }} + when: + - minikube_api_port is defined + - minikube_tailscale_k8s_tcp is not defined or minikube_tailscale_k8s_tcp != 'localhost:' + minikube_api_port + changed_when: true diff --git a/ansible/roles/tailscale_serve/defaults/main.yml b/ansible/roles/tailscale_serve/defaults/main.yml index e9c5970..3f775fd 100644 --- a/ansible/roles/tailscale_serve/defaults/main.yml +++ b/ansible/roles/tailscale_serve/defaults/main.yml @@ -4,6 +4,7 @@ tailscale_serve_services: # NOTE: svc:grafana, svc:pg, svc:feed, svc:pypi removed - now hosted in k8s + # NOTE: svc:k8s is configured by the minikube role (port is dynamic with docker driver) - name: svc:forge https: @@ -22,10 +23,3 @@ tailscale_serve_services: https: port: 443 upstream: http://localhost:5050 - - # Kubernetes API server (TCP passthrough for mTLS) - # With qemu2 driver, API server is inside VM at 192.168.105.2:6443 - - name: svc:k8s - tcp: - port: 443 - upstream: tcp://192.168.105.2:6443