eblume/blumeops
1
1
Fork 0
Code Issues Pull requests Projects Releases 83 Packages Wiki Activity Actions

Migrate minikube ansible role from qemu2 to docker driver

Browse source 
- Change driver from qemu2 to docker
- Remove socket_vmnet and qemu dependencies
- Remove NFS mount and minikube mount LaunchAgent/LaunchDaemon
- Remove old podman zot-mirror.conf
- Update containerd registry mirror config for docker driver
  - Uses host.minikube.internal:5050 to reach zot
  - Configures pull-through cache for docker.io, ghcr.io, quay.io
- Add dynamic tailscale serve configuration for k8s API
  (port is dynamic with docker driver, not fixed at 6443)
- Remove svc:k8s from tailscale_serve defaults (minikube role handles it)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Erich Blume 2026-01-21 13:52:52 -08:00
commit 9fac4439b1
7 changed files with 153 additions and 225 deletions
Download patch file Download diff file Expand all files Collapse all files

11
ansible/roles/minikube/defaults/main.yml
View file

@ -1,15 +1,16 @@
---
# Minikube cluster configuration
# Uses docker driver - requires Docker Desktop to be installed and running
# with at least 12GB memory allocated in Docker Desktop settings
minikube_cpus: 6
minikube_memory: 12288
minikube_memory: 11264 # Leave ~1GB headroom for Docker Desktop overhead
minikube_disk_size: "200g"
minikube_driver: qemu2
minikube_network: socket_vmnet
minikube_container_runtime: containerd
minikube_driver: docker
minikube_container_runtime: docker
# Remote access configuration
# These allow kubectl from other machines (e.g., gilbert) to connect
# k8s.tail8d86e.ts.net is exposed via Tailscale service (TCP passthrough)
# k8s.tail8d86e.ts.net is exposed via Tailscale service (TCP passthrough to localhost)
minikube_apiserver_names:
- k8s.tail8d86e.ts.net
- indri

41
ansible/roles/minikube/files/com.blumeops.minikube-mount.plist
View file

@ -1,41 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>com.blumeops.minikube-mount</string>
<key>ProgramArguments</key>
<array>
<string>/bin/bash</string>
<string>-c</string>
<string>
# Wait for minikube to be running
for i in {1..60}; do
if /opt/homebrew/bin/minikube status | grep -q "Running"; then
break
fi
sleep 5
done
# Wait for NFS mount to be available
for i in {1..30}; do
if mount | grep -q "/Volumes/torrents-nfs"; then
break
fi
sleep 2
done
# Start the mount (this blocks until killed)
exec /opt/homebrew/bin/minikube mount /Volumes/torrents-nfs:/mnt/torrents
</string>
</array>
<key>RunAtLoad</key>
<true/>
<key>KeepAlive</key>
<true/>
<key>StandardErrorPath</key>
<string>/tmp/minikube-mount.err</string>
<key>StandardOutPath</key>
<string>/tmp/minikube-mount.log</string>
</dict>
</plist>

24
ansible/roles/minikube/files/com.blumeops.nfs-torrents.plist
View file

@ -1,24 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>com.blumeops.nfs-torrents</string>
<key>ProgramArguments</key>
<array>
<string>/sbin/mount</string>
<string>-t</string>
<string>nfs</string>
<string>-o</string>
<string>resvport,rw</string>
<string>sifaka:/volume1/torrents</string>
<string>/Volumes/torrents-nfs</string>
</array>
<key>RunAtLoad</key>
<true/>
<key>StandardErrorPath</key>
<string>/tmp/nfs-torrents.err</string>
<key>StandardOutPath</key>
<string>/tmp/nfs-torrents.log</string>
</dict>
</plist>

43
ansible/roles/minikube/files/zot-mirror.conf
View file

@ -1,43 +0,0 @@
# Zot pull-through cache on indri
# Uses host.containers.internal which is stable across restarts
# Applied by ansible minikube role
# Direct access to Zot for private images (blumeops/*)
[[registry]]
prefix = "host.containers.internal:5050"
location = "host.containers.internal:5050"
insecure = true
# Tailscale hostname for Zot - redirects to local access
# Allows manifests to use registry.tail8d86e.ts.net which is cleaner
[[registry]]
prefix = "registry.tail8d86e.ts.net"
location = "registry.tail8d86e.ts.net"
[[registry.mirror]]
location = "host.containers.internal:5050"
insecure = true
[[registry]]
prefix = "docker.io"
location = "docker.io"
[[registry.mirror]]
location = "host.containers.internal:5050/docker.io"
insecure = true
[[registry]]
prefix = "ghcr.io"
location = "ghcr.io"
[[registry.mirror]]
location = "host.containers.internal:5050/ghcr.io"
insecure = true
[[registry]]
prefix = "quay.io"
location = "quay.io"
[[registry.mirror]]
location = "host.containers.internal:5050/quay.io"
insecure = true

13
ansible/roles/minikube/handlers/main.yml
View file

@ -12,16 +12,3 @@
ansible.builtin.command:
cmd: minikube ssh --native-ssh=false "sudo systemctl restart containerd"
changed_when: true
- name: Load NFS mount LaunchDaemon
ansible.builtin.command:
cmd: launchctl load /Library/LaunchDaemons/com.blumeops.nfs-torrents.plist
become: true
failed_when: false
changed_when: true
- name: Load minikube mount LaunchAgent
ansible.builtin.command:
cmd: launchctl load {{ ansible_facts['env']['HOME'] }}/Library/LaunchAgents/com.blumeops.minikube-mount.plist
failed_when: false
changed_when: true

230
ansible/roles/minikube/tasks/main.yml
View file

@ -1,41 +1,20 @@
---
# Minikube installation and cluster setup for indri
# Uses qemu2 driver for full VM with kernel mount capabilities (NFS, SMB, etc.)
# Requires socket_vmnet for proper networking (minikube service/tunnel commands)
# Uses docker driver - requires Docker Desktop to be installed manually first
# (Docker Desktop requires GUI setup, so it's not automated in this role)
#
# Prerequisites:
# 1. Install Docker Desktop: brew install --cask docker
# 2. Launch Docker Desktop and complete setup wizard
# 3. Configure Docker Desktop with at least 12GB memory
#
# NOTE: minikube start may have issues when run via SSH.
# If cluster fails to start, manually run on indri:
# minikube start --driver=qemu2 --network=socket_vmnet --container-runtime=containerd \
# --cpus=6 --memory=12288 --disk-size=200g \
# minikube start --driver=docker --container-runtime=docker \
# --cpus=6 --memory=11264 --disk-size=200g \
# --apiserver-names=k8s.tail8d86e.ts.net --apiserver-names=indri \
# --apiserver-port=6443 --listen-address=0.0.0.0
- name: Install qemu via homebrew (required for qemu2 driver)
community.general.homebrew:
name: qemu
state: present
- name: Install socket_vmnet via homebrew (required for qemu2 networking)
community.general.homebrew:
name: socket_vmnet
state: present
- name: Check if socket_vmnet process is running
ansible.builtin.command:
cmd: pgrep socket_vmnet
register: minikube_socket_vmnet_status
changed_when: false
failed_when: false
- name: Start socket_vmnet service
ansible.builtin.command:
cmd: brew services start socket_vmnet
become: true
register: minikube_socket_vmnet_start
changed_when: "'Successfully started' in minikube_socket_vmnet_start.stdout"
failed_when: false
when: minikube_socket_vmnet_status.rc != 0
- name: Install minikube via homebrew
community.general.homebrew:
name: minikube
@ -46,6 +25,18 @@
name: kubectl
state: present
- name: Check if Docker is running
ansible.builtin.command:
cmd: docker info
register: minikube_docker_status
changed_when: false
failed_when: false
- name: Warn if Docker is not running
ansible.builtin.debug:
msg: "WARNING: Docker does not appear to be running. Please start Docker Desktop manually."
when: minikube_docker_status.rc != 0
- name: Check if minikube cluster exists
ansible.builtin.command:
cmd: minikube status --format={% raw %}'{{.Host}}'{% endraw %}
@ -58,7 +49,6 @@
cmd: >
minikube start
--driver={{ minikube_driver }}
--network={{ minikube_network }}
--container-runtime={{ minikube_container_runtime }}
--cpus={{ minikube_cpus }}
--memory={{ minikube_memory }}
@ -70,8 +60,10 @@
--listen-address={{ minikube_listen_address }}
register: minikube_start
changed_when: minikube_start.rc == 0
failed_when: false # Don't fail - may need manual intervention like podman
when: minikube_status.rc != 0 or 'Running' not in minikube_status.stdout
failed_when: false # Don't fail - may need manual intervention
when:
- minikube_docker_status.rc == 0
- minikube_status.rc != 0 or 'Running' not in minikube_status.stdout
- name: Check minikube status after start attempt
ansible.builtin.command:
@ -85,84 +77,146 @@
msg: "WARNING: minikube may not have started properly. Run 'minikube start' manually on indri if needed. Status: {{ minikube_final_status.stdout | default('unknown') }}"
when: minikube_final_status.rc != 0 or 'Running' not in minikube_final_status.stdout
# Configure VM to access zot registry on host
# The VM can't resolve Tailscale hostnames, so we add a hosts entry
# and configure containerd to use the local zot instance
- name: Add registry hostname to VM hosts file
ansible.builtin.command:
cmd: minikube ssh --native-ssh=false "grep -q 'registry.tail8d86e.ts.net' /etc/hosts || echo '192.168.105.1 registry.tail8d86e.ts.net' | sudo tee -a /etc/hosts"
register: minikube_hosts_entry
changed_when: "'registry.tail8d86e.ts.net' in minikube_hosts_entry.stdout"
when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout
# Configure containerd to use zot registry as pull-through cache
# With docker driver, use host.minikube.internal to reach the host
# Zot runs on indri:5050 and caches images from docker.io, ghcr.io, quay.io
- name: Create containerd registry mirror directory
- name: Create containerd registry mirror directories
ansible.builtin.command:
cmd: minikube ssh --native-ssh=false "sudo mkdir -p /etc/containerd/certs.d/registry.tail8d86e.ts.net"
register: minikube_registry_dir
cmd: minikube ssh --native-ssh=false "sudo mkdir -p /etc/containerd/certs.d/{{ item }}"
loop:
- registry.tail8d86e.ts.net
- docker.io
- ghcr.io
- quay.io
changed_when: false
when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout
- name: Check containerd registry mirror config
# Private registry (registry.tail8d86e.ts.net) - direct to zot
- name: Check registry.tail8d86e.ts.net config
ansible.builtin.command:
cmd: minikube ssh --native-ssh=false "cat /etc/containerd/certs.d/registry.tail8d86e.ts.net/hosts.toml 2>/dev/null || echo ''"
register: minikube_registry_config_current
register: minikube_registry_config
changed_when: false
when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout
- name: Configure containerd registry mirror for zot
- name: Configure registry.tail8d86e.ts.net mirror
ansible.builtin.command:
cmd: |
minikube ssh --native-ssh=false 'echo "server = \"http://host.minikube.internal:5050\"
[host.\"http://host.minikube.internal:5050\"]
capabilities = [\"pull\", \"resolve\"]
capabilities = [\"pull\", \"resolve\", \"push\"]
skip_verify = true" | sudo tee /etc/containerd/certs.d/registry.tail8d86e.ts.net/hosts.toml'
register: minikube_registry_config
changed_when: true
when:
- minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout
- "'host.minikube.internal:5050' not in minikube_registry_config_current.stdout"
- "'host.minikube.internal:5050' not in minikube_registry_config.stdout"
notify: Restart containerd in minikube
# Set up persistent NFS mount from sifaka and minikube mount passthrough
# NFS mount uses LaunchDaemon (runs as root at boot)
# Minikube mount uses LaunchAgent (runs in user GUI session at login)
#
# NOTE: Tasks with become:true require passwordless sudo on indri
# (configured via /etc/sudoers.d/erichblume)
# Docker Hub (docker.io) - zot pull-through cache
- name: Check docker.io config
ansible.builtin.command:
cmd: minikube ssh --native-ssh=false "cat /etc/containerd/certs.d/docker.io/hosts.toml 2>/dev/null || echo ''"
register: minikube_dockerio_config
changed_when: false
when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout
- name: Check if NFS mount point exists
ansible.builtin.stat:
path: /Volumes/torrents-nfs
register: minikube_nfs_mount_point
- name: Configure docker.io mirror through zot
ansible.builtin.command:
cmd: |
minikube ssh --native-ssh=false 'echo "server = \"https://registry-1.docker.io\"
- name: Create NFS mount point
ansible.builtin.file:
path: /Volumes/torrents-nfs
state: directory
mode: "0755"
become: true
when: not minikube_nfs_mount_point.stat.exists
[host.\"http://host.minikube.internal:5050\"]
capabilities = [\"pull\", \"resolve\"]
skip_verify = true" | sudo tee /etc/containerd/certs.d/docker.io/hosts.toml'
changed_when: true
when:
- minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout
- "'host.minikube.internal:5050' not in minikube_dockerio_config.stdout"
notify: Restart containerd in minikube
- name: Check if NFS LaunchDaemon is installed
ansible.builtin.stat:
path: /Library/LaunchDaemons/com.blumeops.nfs-torrents.plist
register: minikube_nfs_launchdaemon
# GitHub Container Registry (ghcr.io) - zot pull-through cache
- name: Check ghcr.io config
ansible.builtin.command:
cmd: minikube ssh --native-ssh=false "cat /etc/containerd/certs.d/ghcr.io/hosts.toml 2>/dev/null || echo ''"
register: minikube_ghcr_config
changed_when: false
when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout
- name: Install NFS mount LaunchDaemon
ansible.builtin.copy:
src: com.blumeops.nfs-torrents.plist
dest: /Library/LaunchDaemons/com.blumeops.nfs-torrents.plist
owner: root
group: wheel
mode: "0644"
become: true
notify: Load NFS mount LaunchDaemon
when: not minikube_nfs_launchdaemon.stat.exists
- name: Configure ghcr.io mirror through zot
ansible.builtin.command:
cmd: |
minikube ssh --native-ssh=false 'echo "server = \"https://ghcr.io\"
- name: Install minikube mount LaunchAgent
ansible.builtin.copy:
src: com.blumeops.minikube-mount.plist
dest: "{{ ansible_facts['env']['HOME'] }}/Library/LaunchAgents/com.blumeops.minikube-mount.plist"
mode: "0644"
notify: Load minikube mount LaunchAgent
[host.\"http://host.minikube.internal:5050\"]
capabilities = [\"pull\", \"resolve\"]
skip_verify = true" | sudo tee /etc/containerd/certs.d/ghcr.io/hosts.toml'
changed_when: true
when:
- minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout
- "'host.minikube.internal:5050' not in minikube_ghcr_config.stdout"
notify: Restart containerd in minikube
# Quay.io - zot pull-through cache
- name: Check quay.io config
ansible.builtin.command:
cmd: minikube ssh --native-ssh=false "cat /etc/containerd/certs.d/quay.io/hosts.toml 2>/dev/null || echo ''"
register: minikube_quay_config
changed_when: false
when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout
- name: Configure quay.io mirror through zot
ansible.builtin.command:
cmd: |
minikube ssh --native-ssh=false 'echo "server = \"https://quay.io\"
[host.\"http://host.minikube.internal:5050\"]
capabilities = [\"pull\", \"resolve\"]
skip_verify = true" | sudo tee /etc/containerd/certs.d/quay.io/hosts.toml'
changed_when: true
when:
- minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout
- "'host.minikube.internal:5050' not in minikube_quay_config.stdout"
notify: Restart containerd in minikube
# Configure Tailscale serve for k8s API access
# With docker driver, the API server port is dynamic (not fixed at 6443)
# We query the current port and configure tailscale serve accordingly
- name: Get minikube API server URL
ansible.builtin.command:
cmd: kubectl config view --minify -o jsonpath="{.clusters[0].cluster.server}"
register: minikube_api_url
changed_when: false
when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout
- name: Extract API server port from URL
ansible.builtin.set_fact:
minikube_api_port: "{{ minikube_api_url.stdout | regex_search(':([0-9]+)$', '\\1') | first }}"
when:
- minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout
- minikube_api_url.stdout is defined
- name: Check current tailscale serve config for k8s
ansible.builtin.command:
cmd: tailscale serve status --json
register: minikube_tailscale_serve_status
changed_when: false
when: minikube_api_port is defined
- name: Parse tailscale serve k8s config
ansible.builtin.set_fact:
minikube_tailscale_k8s_tcp: "{{ ((minikube_tailscale_serve_status.stdout | from_json).Services['svc:k8s'].TCP['443'].TCPForward | default('')) }}"
when:
- minikube_api_port is defined
- minikube_tailscale_serve_status.stdout is defined
- "'svc:k8s' in (minikube_tailscale_serve_status.stdout | from_json).Services | default({})"
failed_when: false
- name: Configure tailscale serve for k8s API
ansible.builtin.command:
cmd: tailscale serve --service="svc:k8s" --tcp=443 tcp://localhost:{{ minikube_api_port }}
when:
- minikube_api_port is defined
- minikube_tailscale_k8s_tcp is not defined or minikube_tailscale_k8s_tcp != 'localhost:' + minikube_api_port
changed_when: true

8
ansible/roles/tailscale_serve/defaults/main.yml
View file

@ -4,6 +4,7 @@
tailscale_serve_services:
# NOTE: svc:grafana, svc:pg, svc:feed, svc:pypi removed - now hosted in k8s
# NOTE: svc:k8s is configured by the minikube role (port is dynamic with docker driver)
- name: svc:forge
https:
@ -22,10 +23,3 @@ tailscale_serve_services:
https:
port: 443
upstream: http://localhost:5050
# Kubernetes API server (TCP passthrough for mTLS)
# With qemu2 driver, API server is inside VM at 192.168.105.2:6443
- name: svc:k8s
tcp:
port: 443
upstream: tcp://192.168.105.2:6443