From 26ec02e1be22213c3b087b468a659a0771b75be2 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Wed, 21 Jan 2026 08:03:21 -0800 Subject: [PATCH] P5.1: Add VM config to ansible role, mark phase complete - Add hosts file entry for registry.tail8d86e.ts.net in VM - Configure containerd registry mirror to use local zot - Update P5.1 doc with implementation notes and manual steps - Mark P5.1 as complete Manual steps still required after cluster creation: 1. sudo brew services start socket_vmnet (once per reboot) 2. sudo mount -t nfs sifaka:/volume1/torrents /Volumes/torrents-nfs 3. minikube mount /Volumes/torrents-nfs:/mnt/torrents (GUI session) Co-Authored-By: Claude Opus 4.5 --- ansible/roles/minikube/tasks/main.yml | 38 +++++++++------ plans/k8s-migration/P5.1_qemu2_migration.md | 52 +++++++++++++++++---- 2 files changed, 67 insertions(+), 23 deletions(-) diff --git a/ansible/roles/minikube/tasks/main.yml b/ansible/roles/minikube/tasks/main.yml index aca060f..d704ba0 100644 --- a/ansible/roles/minikube/tasks/main.yml +++ b/ansible/roles/minikube/tasks/main.yml @@ -77,22 +77,32 @@ msg: "WARNING: minikube may not have started properly. Run 'minikube start' manually on indri if needed. Status: {{ minikube_final_status.stdout | default('unknown') }}" when: minikube_final_status.rc != 0 or 'Running' not in minikube_final_status.stdout -# Configure containerd to use zot as pull-through cache -# With qemu2 driver, host is accessible via host.minikube.internal -# Zot listens on indri:5050 (localhost:5050 from host perspective) -- name: Get host IP for registry mirror config +# Configure VM to access zot registry on host +# The VM can't resolve Tailscale hostnames, so we add a hosts entry +# and configure containerd to use the local zot instance +- name: Add registry hostname to VM hosts file ansible.builtin.command: - cmd: minikube ssh --native-ssh=false "getent hosts host.minikube.internal | awk '{print \$1}'" - register: minikube_host_ip - changed_when: false - failed_when: false + cmd: minikube ssh --native-ssh=false "grep -q 'registry.tail8d86e.ts.net' /etc/hosts || echo '192.168.105.1 registry.tail8d86e.ts.net' | sudo tee -a /etc/hosts" + register: minikube_hosts_entry + changed_when: "'registry.tail8d86e.ts.net' in minikube_hosts_entry.stdout" when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout -- name: Configure insecure registry for zot +- name: Create containerd registry mirror directory ansible.builtin.command: - cmd: "minikube addons configure registry-creds" + cmd: minikube ssh --native-ssh=false "sudo mkdir -p /etc/containerd/certs.d/registry.tail8d86e.ts.net" + register: minikube_registry_dir changed_when: false - failed_when: false - when: false # TODO: Configure containerd registry mirrors after basic migration works - # For now, images will be pulled directly from public registries - # We can add zot mirror config later via containerd config or minikube addons + when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout + +- name: Configure containerd registry mirror for zot + ansible.builtin.command: + cmd: | + minikube ssh --native-ssh=false 'echo "server = \"http://host.minikube.internal:5050\" + + [host.\"http://host.minikube.internal:5050\"] + capabilities = [\"pull\", \"resolve\"] + skip_verify = true" | sudo tee /etc/containerd/certs.d/registry.tail8d86e.ts.net/hosts.toml' + register: minikube_registry_config + changed_when: minikube_registry_config.rc == 0 + when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout + notify: Restart containerd in minikube diff --git a/plans/k8s-migration/P5.1_qemu2_migration.md b/plans/k8s-migration/P5.1_qemu2_migration.md index 032baea..64c5be7 100644 --- a/plans/k8s-migration/P5.1_qemu2_migration.md +++ b/plans/k8s-migration/P5.1_qemu2_migration.md @@ -2,7 +2,7 @@ **Goal**: Replace the podman driver with qemu2 to enable proper volume mounts (hostPath, NFS, SMB CSI) -**Status**: Planning +**Status**: Complete (2026-01-21) **Prerequisites**: [Phase 5](P5_devpi.complete.md) complete @@ -254,15 +254,49 @@ All state is in git, so cluster recreation is straightforward. - socket_vmnet provides better networking but requires sudo setup - Consider creating a LaunchAgent for `minikube mount` if using that approach -## Post-Migration Tasks +## Implementation Notes (2026-01-21) -### Zot Registry Mirror (TODO) +### What Actually Worked -The CRI-O registry mirror config (`zot-mirror.conf`) is not compatible with containerd. After the basic QEMU2 migration is verified working, we need to: +**Volume mounting solution**: NFS mount on indri (host) + `minikube mount` to pass through to VM -1. **Reconfigure zot mirrors for containerd** - containerd uses a different config format at `/etc/containerd/config.toml` -2. **Update host address** - `host.containers.internal` (podman-specific) won't work; use `host.minikube.internal` or the host IP instead -3. **Test registry caching** - verify images are being cached through zot -4. **Update ansible role** - add containerd-specific registry mirror configuration +1. Mount sifaka's torrents share on indri via NFS: `sudo mount -t nfs sifaka:/volume1/torrents /Volumes/torrents-nfs` +2. Run `minikube mount /Volumes/torrents-nfs:/mnt/torrents` from indri console (GUI session required due to macOS security) +3. Pods can access `/mnt/torrents` via hostPath -For now, images will be pulled directly from public registries (docker.io, ghcr.io, etc.) which is fine for the migration but loses the caching benefits. +**Why NFS from inside VM didn't work**: Despite allowing 192.168.105.0/24 in Synology NFS settings, the VM got "access denied". Root cause unknown - may be Synology NFS quirk. + +**Why SMB didn't work**: The minikube containerd kernel doesn't include the CIFS module. + +### Zot Registry Mirror (Implemented) + +The ansible role now configures containerd to redirect `registry.tail8d86e.ts.net` to `host.minikube.internal:5050`: +- Adds hosts file entry in VM +- Creates containerd registry mirror config at `/etc/containerd/certs.d/registry.tail8d86e.ts.net/hosts.toml` + +### Manual Steps Still Required + +These steps cannot be fully automated via ansible and must be done manually: + +1. **socket_vmnet service (once per reboot)**: + ```bash + # On indri console: + sudo brew services start socket_vmnet + ``` + +2. **NFS mount on indri (once per reboot)**: + ```bash + # On indri console: + sudo mount -t nfs sifaka:/volume1/torrents /Volumes/torrents-nfs + ``` + +3. **minikube mount (must run in GUI session)**: + ```bash + # On indri console (not SSH - requires GUI session for macOS security): + minikube mount /Volumes/torrents-nfs:/mnt/torrents + # Keep this terminal open - the mount dies if process exits + ``` + +### TODO: LaunchAgent for Persistent Mount + +Create a LaunchAgent to run `minikube mount` at login. Challenge: must run in GUI session context for macOS security model.