From 55f0335a1e4f332b33efad693078ed924032086b Mon Sep 17 00:00:00 2001
From: Erich Blume <blume.erich@gmail.com>
Date: Sat, 17 Jan 2026 22:23:45 -0800
Subject: [PATCH] Add podman role with known issue documentation

- Create ansible/roles/podman for podman machine setup on indri
- Document known reliability issue with podman machine init/start via SSH
  (race condition from containers/podman#16945)
- Role attempts init/start but doesn't fail if start hangs
- Workaround: manual init/start on indri if needed
- Update k8s-migration plan with implementation details

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
---
 ansible/playbooks/indri.yml            |  2 +
 ansible/roles/podman/handlers/main.yml |  3 ++
 ansible/roles/podman/tasks/main.yml    | 55 ++++++++++++++++++++++++++
 plans/k8s-migration.md                 | 11 ++++++
 4 files changed, 71 insertions(+)
 create mode 100644 ansible/roles/podman/handlers/main.yml
 create mode 100644 ansible/roles/podman/tasks/main.yml

diff --git a/ansible/playbooks/indri.yml b/ansible/playbooks/indri.yml
index 570b60f..e8dd762 100644
--- a/ansible/playbooks/indri.yml
+++ b/ansible/playbooks/indri.yml
@@ -103,6 +103,8 @@
       tags: zot
     - role: zot_metrics
       tags: zot_metrics
+    - role: podman
+      tags: podman
     - role: plex_metrics
       tags: plex_metrics
     - role: postgresql
diff --git a/ansible/roles/podman/handlers/main.yml b/ansible/roles/podman/handlers/main.yml
new file mode 100644
index 0000000..89a6a94
--- /dev/null
+++ b/ansible/roles/podman/handlers/main.yml
@@ -0,0 +1,3 @@
+---
+# No handlers currently - podman machine start is unreliable via Ansible
+# See known issue in tasks/main.yml
diff --git a/ansible/roles/podman/tasks/main.yml b/ansible/roles/podman/tasks/main.yml
new file mode 100644
index 0000000..86a3cda
--- /dev/null
+++ b/ansible/roles/podman/tasks/main.yml
@@ -0,0 +1,55 @@
+---
+# Podman installation and machine setup for indri
+# Used as container runtime for minikube
+#
+# KNOWN ISSUE: podman machine init/start has reliability issues when run via
+# Ansible/SSH. The machine sometimes gets stuck in "Starting" state due to a
+# race condition (see https://github.com/containers/podman/issues/16945).
+# Additionally, Apple Hypervisor may require GUI session context.
+#
+# WORKAROUND: If the machine fails to start via Ansible, manually run on indri:
+#   podman machine rm -f podman-machine-default
+#   podman machine init --cpus 4 --memory 8192 --disk-size 220
+#   podman machine start
+#
+# TODO: Investigate proper LaunchAgent or other solution for reliable automation.
+
+- name: Install podman via homebrew
+  community.general.homebrew:
+    name: podman
+    state: present
+
+- name: Check if podman machine exists
+  ansible.builtin.command:
+    cmd: podman machine list --format json
+  register: podman_machine_list
+  changed_when: false
+
+- name: Initialize podman machine (if not exists)
+  ansible.builtin.command:
+    cmd: podman machine init --cpus 4 --memory 8192 --disk-size 220
+  register: podman_init
+  changed_when: podman_init.rc == 0
+  failed_when: podman_init.rc not in [0, 125]  # 125 = already exists
+  when: podman_machine_list.stdout == '[]'
+
+- name: Check if podman machine is running
+  ansible.builtin.command:
+    cmd: podman machine list --format "{{ '{{' }}.Running{{ '}}' }}"
+  register: podman_running
+  changed_when: false
+
+- name: Start podman machine (if stopped)
+  ansible.builtin.command:
+    cmd: podman machine start
+  register: podman_start
+  changed_when: "'started successfully' in podman_start.stdout"
+  failed_when: false  # Don't fail - see known issue above
+  when: "'true' not in podman_running.stdout"
+
+- name: Warn if podman machine failed to start
+  ansible.builtin.debug:
+    msg: "WARNING: podman machine may not have started. Run 'podman machine start' manually on indri if needed."
+  when:
+    - "'true' not in podman_running.stdout"
+    - podman_start.rc != 0 or "'started successfully' not in podman_start.stdout"
diff --git a/plans/k8s-migration.md b/plans/k8s-migration.md
index 53bf6a4..12ea944 100644
--- a/plans/k8s-migration.md
+++ b/plans/k8s-migration.md
@@ -521,6 +521,17 @@ ssh indri 'podman info'
 ssh indri 'podman run --rm hello-world'
 ```
 
+**Implementation Details:**
+- **KNOWN ISSUE**: `podman machine init` and `podman machine start` have reliability issues when run via Ansible/SSH. The machine sometimes gets stuck in "Starting" state due to a race condition (see https://github.com/containers/podman/issues/16945). Apple Hypervisor may also require GUI session context.
+- **WORKAROUND**: If the machine fails to start via Ansible, manually run on indri:
+  ```bash
+  podman machine rm -f podman-machine-default
+  podman machine init --cpus 4 --memory 8192 --disk-size 220
+  podman machine start
+  ```
+- LaunchAgent approach was attempted but didn't resolve the issue reliably.
+- TODO: Investigate proper automation solution for reliable podman machine management.
+
 ---
 
 ### Step 0.9: Install and Configure Minikube