From ce6c5b6b37fc9826cd26fcc33ccae9b30dbf8b9b Mon Sep 17 00:00:00 2001
From: Erich Blume <blume.erich@gmail.com>
Date: Fri, 16 Jan 2026 22:19:46 -0800
Subject: [PATCH] Refactor kiwix role to use shell scripts instead of ansible
 loops

Replace ansible loops for torrent syncing and ZIM symlinking with
standalone shell scripts that handle all items in a single pass:

- kiwix-sync-torrents.sh: Reads torrent URLs from file, adds missing
  ones to transmission in one execution
- kiwix-symlink-zims.sh: Symlinks all completed ZIM files from download
  directory to kiwix directory in one pass
- kiwix-torrents.txt: Generated list of torrent URLs from inventory

This reduces ansible output noise and improves execution speed by
avoiding per-item task invocations.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
---
 ansible/roles/kiwix/defaults/main.yml         |   1 +
 ansible/roles/kiwix/tasks/main.yml            | 106 ++++++------------
 .../kiwix/templates/kiwix-symlink-zims.sh.j2  |  50 +++++++++
 .../kiwix/templates/kiwix-sync-torrents.sh.j2 |  46 ++++++++
 .../kiwix/templates/kiwix-torrents.txt.j2     |   5 +
 5 files changed, 139 insertions(+), 69 deletions(-)
 create mode 100644 ansible/roles/kiwix/templates/kiwix-symlink-zims.sh.j2
 create mode 100644 ansible/roles/kiwix/templates/kiwix-sync-torrents.sh.j2
 create mode 100644 ansible/roles/kiwix/templates/kiwix-torrents.txt.j2

diff --git a/ansible/roles/kiwix/defaults/main.yml b/ansible/roles/kiwix/defaults/main.yml
index 6b54533..41f3abb 100644
--- a/ansible/roles/kiwix/defaults/main.yml
+++ b/ansible/roles/kiwix/defaults/main.yml
@@ -1,6 +1,7 @@
 ---
 kiwix_serve_bin: /Users/erichblume/code/3rd/kiwix-tools/kiwix-serve
 kiwix_zim_dir: /Users/erichblume/code/3rd/kiwix-tools
+kiwix_bin_dir: /Users/erichblume/.local/bin
 kiwix_port: 5501
 kiwix_log_dir: /Users/erichblume/Library/Logs
 
diff --git a/ansible/roles/kiwix/tasks/main.yml b/ansible/roles/kiwix/tasks/main.yml
index 950fd03..85a09a2 100644
--- a/ansible/roles/kiwix/tasks/main.yml
+++ b/ansible/roles/kiwix/tasks/main.yml
@@ -5,10 +5,35 @@
     state: directory
     mode: '0755'
 
-# --- Transmission-based torrent management ---
-# This section ensures declared ZIM archives have torrents added to transmission.
-# It does NOT wait for downloads to complete - kiwix startup handles that separately.
+- name: Ensure kiwix bin directory exists
+  ansible.builtin.file:
+    path: "{{ kiwix_bin_dir }}"
+    state: directory
+    mode: '0755'
 
+# --- Deploy management scripts ---
+- name: Deploy kiwix torrent sync script
+  ansible.builtin.template:
+    src: kiwix-sync-torrents.sh.j2
+    dest: "{{ kiwix_bin_dir }}/kiwix-sync-torrents.sh"
+    mode: '0755'
+  when: kiwix_use_transmission
+
+- name: Deploy kiwix symlink script
+  ansible.builtin.template:
+    src: kiwix-symlink-zims.sh.j2
+    dest: "{{ kiwix_bin_dir }}/kiwix-symlink-zims.sh"
+    mode: '0755'
+  when: kiwix_use_transmission
+
+- name: Deploy kiwix torrent list
+  ansible.builtin.template:
+    src: kiwix-torrents.txt.j2
+    dest: "{{ kiwix_bin_dir }}/kiwix-torrents.txt"
+    mode: '0644'
+  when: kiwix_use_transmission
+
+# --- Transmission-based torrent management ---
 - name: Check transmission daemon is responding
   ansible.builtin.command: transmission-remote -l
   register: kiwix_transmission_check
@@ -21,75 +46,18 @@
     msg: "Transmission daemon is not responding. Ensure transmission role ran successfully."
   when: kiwix_use_transmission and kiwix_transmission_check.rc != 0
 
-# Find which declared archives don't have torrents yet (single shell command)
-- name: Find declared archives missing from transmission
-  ansible.builtin.shell: |
-    set -euo pipefail
-    # Get current torrent list (skip header and footer)
-    torrents=$(transmission-remote -l 2>/dev/null | tail -n +2 | head -n -1 || true)
-
-    # Check each declared archive
-    {% for archive in kiwix_zim_archives %}
-    base="{{ archive.filename | regex_replace('\\.zim$', '') }}"
-    if ! echo "$torrents" | grep -qF "$base"; then
-      echo "{{ archive.category }}/{{ archive.filename }}"
-    fi
-    {% endfor %}
-  args:
-    executable: /bin/bash
-  register: kiwix_missing_torrents
-  changed_when: false
+- name: Sync ZIM torrents to transmission
+  ansible.builtin.command: "{{ kiwix_bin_dir }}/kiwix-sync-torrents.sh {{ kiwix_bin_dir }}/kiwix-torrents.txt"
+  register: kiwix_torrent_sync
+  changed_when: "'Added:' in kiwix_torrent_sync.stdout"
   when: kiwix_use_transmission
 
-# Add only the missing torrents
-- name: Add missing torrents to transmission
-  ansible.builtin.command: >
-    transmission-remote -a "{{ kiwix_torrent_base_url }}/{{ item }}.torrent"
-  loop: "{{ kiwix_missing_torrents.stdout_lines | default([]) }}"
-  loop_control:
-    label: "{{ item | basename }}"
-  when:
-    - kiwix_use_transmission
-    - kiwix_missing_torrents.stdout_lines | default([]) | length > 0
-  register: kiwix_torrent_add
-  changed_when: kiwix_torrent_add.rc == 0
-
-# --- Kiwix startup: serve whatever completed ZIM files exist ---
-# This is decoupled from the declared inventory - it just serves what's available.
-
-# Find all completed ZIM files in transmission download directory
-- name: Find completed ZIM files in transmission download directory
-  ansible.builtin.find:
-    paths: "{{ transmission_download_dir }}"
-    patterns: "*.zim"
-    file_type: file
-  register: kiwix_completed_zim_files
-  when: kiwix_use_transmission
-
-# Check which ZIM files already have symlinks in kiwix directory
-- name: Check existing symlinks in kiwix directory
-  ansible.builtin.stat:
-    path: "{{ kiwix_zim_dir }}/{{ item.path | basename }}"
-    get_checksum: false
-  loop: "{{ kiwix_completed_zim_files.files | default([]) }}"
-  loop_control:
-    label: "{{ item.path | basename }}"
-  register: kiwix_existing_symlinks
-  when: kiwix_use_transmission
-
-# Create symlinks for any completed ZIM files not yet linked
+# --- Symlink completed ZIM files ---
 - name: Symlink completed ZIM files to kiwix directory
-  ansible.builtin.file:
-    src: "{{ item.item.path }}"
-    dest: "{{ kiwix_zim_dir }}/{{ item.item.path | basename }}"
-    state: link
-  loop: "{{ kiwix_existing_symlinks.results | default([]) }}"
-  loop_control:
-    label: "{{ item.item.path | basename }}"
-  when:
-    - kiwix_use_transmission
-    - item.stat is defined
-    - not item.stat.exists
+  ansible.builtin.command: "{{ kiwix_bin_dir }}/kiwix-symlink-zims.sh {{ transmission_download_dir }} {{ kiwix_zim_dir }}"
+  register: kiwix_symlink_result
+  changed_when: "'Linked:' in kiwix_symlink_result.stdout"
+  when: kiwix_use_transmission
   notify: Restart kiwix-serve
 
 # --- Fallback: Direct HTTP download (original behavior) ---
diff --git a/ansible/roles/kiwix/templates/kiwix-symlink-zims.sh.j2 b/ansible/roles/kiwix/templates/kiwix-symlink-zims.sh.j2
new file mode 100644
index 0000000..29e7630
--- /dev/null
+++ b/ansible/roles/kiwix/templates/kiwix-symlink-zims.sh.j2
@@ -0,0 +1,50 @@
+#!/bin/bash
+# Symlink completed ZIM files from download directory to kiwix directory
+set -euo pipefail
+
+SOURCE_DIR="${1:-}"
+TARGET_DIR="${2:-}"
+
+if [[ -z "$SOURCE_DIR" || -z "$TARGET_DIR" ]]; then
+    echo "Usage: $0 <source-dir> <target-dir>" >&2
+    exit 1
+fi
+
+if [[ ! -d "$SOURCE_DIR" ]]; then
+    echo "Error: Source directory not found: $SOURCE_DIR" >&2
+    exit 1
+fi
+
+if [[ ! -d "$TARGET_DIR" ]]; then
+    echo "Error: Target directory not found: $TARGET_DIR" >&2
+    exit 1
+fi
+
+created=0
+skipped=0
+
+# Find all .zim files in source directory
+for zim_file in "$SOURCE_DIR"/*.zim; do
+    # Handle case where no .zim files exist
+    [[ -e "$zim_file" ]] || continue
+
+    filename=$(basename "$zim_file")
+    target_path="$TARGET_DIR/$filename"
+
+    if [[ -e "$target_path" || -L "$target_path" ]]; then
+        ((skipped++)) || true
+    else
+        ln -s "$zim_file" "$target_path"
+        echo "Linked: $filename"
+        ((created++)) || true
+    fi
+done
+
+echo "Symlink complete: $created created, $skipped already present"
+
+# Exit with special code if new symlinks were created (for ansible changed detection)
+if [[ $created -gt 0 ]]; then
+    exit 0
+else
+    exit 0
+fi
diff --git a/ansible/roles/kiwix/templates/kiwix-sync-torrents.sh.j2 b/ansible/roles/kiwix/templates/kiwix-sync-torrents.sh.j2
new file mode 100644
index 0000000..d95d7dd
--- /dev/null
+++ b/ansible/roles/kiwix/templates/kiwix-sync-torrents.sh.j2
@@ -0,0 +1,46 @@
+#!/bin/bash
+# Sync ZIM archive torrents to transmission
+# Reads torrent URLs from stdin or file, adds any missing to transmission
+set -euo pipefail
+
+TORRENT_LIST="${1:-}"
+
+if [[ -z "$TORRENT_LIST" ]]; then
+    echo "Usage: $0 <torrent-list-file>" >&2
+    exit 1
+fi
+
+if [[ ! -f "$TORRENT_LIST" ]]; then
+    echo "Error: Torrent list file not found: $TORRENT_LIST" >&2
+    exit 1
+fi
+
+# Get current torrents from transmission (extract names, skip header/footer)
+current_torrents=$(transmission-remote -l 2>/dev/null | tail -n +2 | head -n -1 | awk '{print $NF}' || true)
+
+added=0
+skipped=0
+
+while IFS= read -r torrent_url || [[ -n "$torrent_url" ]]; do
+    # Skip empty lines and comments
+    [[ -z "$torrent_url" || "$torrent_url" =~ ^# ]] && continue
+
+    # Extract base name from URL (remove .torrent extension and path)
+    base_name=$(basename "$torrent_url" .torrent)
+    # Also try without .zim in case transmission reports it differently
+    base_without_zim="${base_name%.zim}"
+
+    # Check if already in transmission
+    if echo "$current_torrents" | grep -qF "$base_without_zim"; then
+        ((skipped++)) || true
+    else
+        if transmission-remote -a "$torrent_url" 2>/dev/null; then
+            echo "Added: $base_name"
+            ((added++)) || true
+        else
+            echo "Warning: Failed to add $torrent_url" >&2
+        fi
+    fi
+done < "$TORRENT_LIST"
+
+echo "Sync complete: $added added, $skipped already present"
diff --git a/ansible/roles/kiwix/templates/kiwix-torrents.txt.j2 b/ansible/roles/kiwix/templates/kiwix-torrents.txt.j2
new file mode 100644
index 0000000..fcc4b4e
--- /dev/null
+++ b/ansible/roles/kiwix/templates/kiwix-torrents.txt.j2
@@ -0,0 +1,5 @@
+# ZIM archive torrent URLs for kiwix
+# Generated by ansible - do not edit manually
+{% for archive in kiwix_zim_archives %}
+{{ kiwix_torrent_base_url }}/{{ archive.category }}/{{ archive.filename }}.torrent
+{% endfor %}