From 21e6b4f9e8ba71604d6ef052f7363a86c49030bb Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 16 Jan 2026 22:06:31 -0800 Subject: [PATCH 1/3] Remove ansible role meta dependencies to fix duplicate execution Ansible's tag accumulation behavior prevents proper role deduplication when using meta/main.yml dependencies. When a role is pulled in as a dependency, the parent role's tags are added to the dependency's tags, making them appear as different invocations to Ansible. Role ordering is now controlled entirely by indri.yml playbook. Also fixes incorrect roles path in CLAUDE.md (was playbooks/roles, should be just roles). Co-Authored-By: Claude Opus 4.5 --- CLAUDE.md | 2 +- ansible/roles/alloy/meta/main.yml | 6 +++--- ansible/roles/devpi_metrics/meta/main.yml | 6 +++--- ansible/roles/grafana/meta/main.yml | 6 +++--- ansible/roles/kiwix/meta/main.yml | 5 +++-- ansible/roles/miniflux/meta/main.yml | 5 +++-- ansible/roles/plex_metrics/meta/main.yml | 5 +++-- ansible/roles/tailscale_serve/meta/main.yml | 9 +++------ ansible/roles/transmission_metrics/meta/main.yml | 6 +++--- 9 files changed, 25 insertions(+), 25 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 298240b..292061b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -44,7 +44,7 @@ Some important places you can look: ``` ./mise-tasks/ # management and utility scripts run via `mise run` ./ansible/playbooks/indri.yml # primary blumeops provisioning script -./ansible/playbooks/roles/ # role dirs here give good overview of services; dependencies tracked via meta/main.yml +./ansible/roles/ # role dirs here give good overview of services ./pulumi/ # python (via uv) pulumi script for provisioning the tailnet and other cloud resources ~/code/personal/ # projects managed by the user ~/code/3rd/ # external projects, mirrored or downloaded diff --git a/ansible/roles/alloy/meta/main.yml b/ansible/roles/alloy/meta/main.yml index 9e57ded..b05a43b 100644 --- a/ansible/roles/alloy/meta/main.yml +++ b/ansible/roles/alloy/meta/main.yml @@ -1,4 +1,4 @@ --- -dependencies: - - role: prometheus - - role: loki +# Role ordering is controlled by indri.yml playbook - do not add dependencies here +# (Ansible's tag accumulation prevents proper deduplication when using meta dependencies) +dependencies: [] diff --git a/ansible/roles/devpi_metrics/meta/main.yml b/ansible/roles/devpi_metrics/meta/main.yml index f5c4308..b05a43b 100644 --- a/ansible/roles/devpi_metrics/meta/main.yml +++ b/ansible/roles/devpi_metrics/meta/main.yml @@ -1,4 +1,4 @@ --- -dependencies: - - role: alloy - - role: devpi +# Role ordering is controlled by indri.yml playbook - do not add dependencies here +# (Ansible's tag accumulation prevents proper deduplication when using meta dependencies) +dependencies: [] diff --git a/ansible/roles/grafana/meta/main.yml b/ansible/roles/grafana/meta/main.yml index 9e57ded..b05a43b 100644 --- a/ansible/roles/grafana/meta/main.yml +++ b/ansible/roles/grafana/meta/main.yml @@ -1,4 +1,4 @@ --- -dependencies: - - role: prometheus - - role: loki +# Role ordering is controlled by indri.yml playbook - do not add dependencies here +# (Ansible's tag accumulation prevents proper deduplication when using meta dependencies) +dependencies: [] diff --git a/ansible/roles/kiwix/meta/main.yml b/ansible/roles/kiwix/meta/main.yml index 32004b6..b05a43b 100644 --- a/ansible/roles/kiwix/meta/main.yml +++ b/ansible/roles/kiwix/meta/main.yml @@ -1,3 +1,4 @@ --- -dependencies: - - role: transmission +# Role ordering is controlled by indri.yml playbook - do not add dependencies here +# (Ansible's tag accumulation prevents proper deduplication when using meta dependencies) +dependencies: [] diff --git a/ansible/roles/miniflux/meta/main.yml b/ansible/roles/miniflux/meta/main.yml index 92e0133..b05a43b 100644 --- a/ansible/roles/miniflux/meta/main.yml +++ b/ansible/roles/miniflux/meta/main.yml @@ -1,3 +1,4 @@ --- -dependencies: - - role: postgresql +# Role ordering is controlled by indri.yml playbook - do not add dependencies here +# (Ansible's tag accumulation prevents proper deduplication when using meta dependencies) +dependencies: [] diff --git a/ansible/roles/plex_metrics/meta/main.yml b/ansible/roles/plex_metrics/meta/main.yml index 2925213..b05a43b 100644 --- a/ansible/roles/plex_metrics/meta/main.yml +++ b/ansible/roles/plex_metrics/meta/main.yml @@ -1,3 +1,4 @@ --- -dependencies: - - role: alloy +# Role ordering is controlled by indri.yml playbook - do not add dependencies here +# (Ansible's tag accumulation prevents proper deduplication when using meta dependencies) +dependencies: [] diff --git a/ansible/roles/tailscale_serve/meta/main.yml b/ansible/roles/tailscale_serve/meta/main.yml index bfc3021..b05a43b 100644 --- a/ansible/roles/tailscale_serve/meta/main.yml +++ b/ansible/roles/tailscale_serve/meta/main.yml @@ -1,7 +1,4 @@ --- -dependencies: - - role: grafana - - role: forgejo - - role: kiwix - - role: devpi - - role: miniflux +# Role ordering is controlled by indri.yml playbook - do not add dependencies here +# (Ansible's tag accumulation prevents proper deduplication when using meta dependencies) +dependencies: [] diff --git a/ansible/roles/transmission_metrics/meta/main.yml b/ansible/roles/transmission_metrics/meta/main.yml index 14f47d4..b05a43b 100644 --- a/ansible/roles/transmission_metrics/meta/main.yml +++ b/ansible/roles/transmission_metrics/meta/main.yml @@ -1,4 +1,4 @@ --- -dependencies: - - role: alloy - - role: transmission +# Role ordering is controlled by indri.yml playbook - do not add dependencies here +# (Ansible's tag accumulation prevents proper deduplication when using meta dependencies) +dependencies: [] -- 2.50.1 (Apple Git-155) From ce6c5b6b37fc9826cd26fcc33ccae9b30dbf8b9b Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 16 Jan 2026 22:19:46 -0800 Subject: [PATCH 2/3] Refactor kiwix role to use shell scripts instead of ansible loops Replace ansible loops for torrent syncing and ZIM symlinking with standalone shell scripts that handle all items in a single pass: - kiwix-sync-torrents.sh: Reads torrent URLs from file, adds missing ones to transmission in one execution - kiwix-symlink-zims.sh: Symlinks all completed ZIM files from download directory to kiwix directory in one pass - kiwix-torrents.txt: Generated list of torrent URLs from inventory This reduces ansible output noise and improves execution speed by avoiding per-item task invocations. Co-Authored-By: Claude Opus 4.5 --- ansible/roles/kiwix/defaults/main.yml | 1 + ansible/roles/kiwix/tasks/main.yml | 106 ++++++------------ .../kiwix/templates/kiwix-symlink-zims.sh.j2 | 50 +++++++++ .../kiwix/templates/kiwix-sync-torrents.sh.j2 | 46 ++++++++ .../kiwix/templates/kiwix-torrents.txt.j2 | 5 + 5 files changed, 139 insertions(+), 69 deletions(-) create mode 100644 ansible/roles/kiwix/templates/kiwix-symlink-zims.sh.j2 create mode 100644 ansible/roles/kiwix/templates/kiwix-sync-torrents.sh.j2 create mode 100644 ansible/roles/kiwix/templates/kiwix-torrents.txt.j2 diff --git a/ansible/roles/kiwix/defaults/main.yml b/ansible/roles/kiwix/defaults/main.yml index 6b54533..41f3abb 100644 --- a/ansible/roles/kiwix/defaults/main.yml +++ b/ansible/roles/kiwix/defaults/main.yml @@ -1,6 +1,7 @@ --- kiwix_serve_bin: /Users/erichblume/code/3rd/kiwix-tools/kiwix-serve kiwix_zim_dir: /Users/erichblume/code/3rd/kiwix-tools +kiwix_bin_dir: /Users/erichblume/.local/bin kiwix_port: 5501 kiwix_log_dir: /Users/erichblume/Library/Logs diff --git a/ansible/roles/kiwix/tasks/main.yml b/ansible/roles/kiwix/tasks/main.yml index 950fd03..85a09a2 100644 --- a/ansible/roles/kiwix/tasks/main.yml +++ b/ansible/roles/kiwix/tasks/main.yml @@ -5,10 +5,35 @@ state: directory mode: '0755' -# --- Transmission-based torrent management --- -# This section ensures declared ZIM archives have torrents added to transmission. -# It does NOT wait for downloads to complete - kiwix startup handles that separately. +- name: Ensure kiwix bin directory exists + ansible.builtin.file: + path: "{{ kiwix_bin_dir }}" + state: directory + mode: '0755' +# --- Deploy management scripts --- +- name: Deploy kiwix torrent sync script + ansible.builtin.template: + src: kiwix-sync-torrents.sh.j2 + dest: "{{ kiwix_bin_dir }}/kiwix-sync-torrents.sh" + mode: '0755' + when: kiwix_use_transmission + +- name: Deploy kiwix symlink script + ansible.builtin.template: + src: kiwix-symlink-zims.sh.j2 + dest: "{{ kiwix_bin_dir }}/kiwix-symlink-zims.sh" + mode: '0755' + when: kiwix_use_transmission + +- name: Deploy kiwix torrent list + ansible.builtin.template: + src: kiwix-torrents.txt.j2 + dest: "{{ kiwix_bin_dir }}/kiwix-torrents.txt" + mode: '0644' + when: kiwix_use_transmission + +# --- Transmission-based torrent management --- - name: Check transmission daemon is responding ansible.builtin.command: transmission-remote -l register: kiwix_transmission_check @@ -21,75 +46,18 @@ msg: "Transmission daemon is not responding. Ensure transmission role ran successfully." when: kiwix_use_transmission and kiwix_transmission_check.rc != 0 -# Find which declared archives don't have torrents yet (single shell command) -- name: Find declared archives missing from transmission - ansible.builtin.shell: | - set -euo pipefail - # Get current torrent list (skip header and footer) - torrents=$(transmission-remote -l 2>/dev/null | tail -n +2 | head -n -1 || true) - - # Check each declared archive - {% for archive in kiwix_zim_archives %} - base="{{ archive.filename | regex_replace('\\.zim$', '') }}" - if ! echo "$torrents" | grep -qF "$base"; then - echo "{{ archive.category }}/{{ archive.filename }}" - fi - {% endfor %} - args: - executable: /bin/bash - register: kiwix_missing_torrents - changed_when: false +- name: Sync ZIM torrents to transmission + ansible.builtin.command: "{{ kiwix_bin_dir }}/kiwix-sync-torrents.sh {{ kiwix_bin_dir }}/kiwix-torrents.txt" + register: kiwix_torrent_sync + changed_when: "'Added:' in kiwix_torrent_sync.stdout" when: kiwix_use_transmission -# Add only the missing torrents -- name: Add missing torrents to transmission - ansible.builtin.command: > - transmission-remote -a "{{ kiwix_torrent_base_url }}/{{ item }}.torrent" - loop: "{{ kiwix_missing_torrents.stdout_lines | default([]) }}" - loop_control: - label: "{{ item | basename }}" - when: - - kiwix_use_transmission - - kiwix_missing_torrents.stdout_lines | default([]) | length > 0 - register: kiwix_torrent_add - changed_when: kiwix_torrent_add.rc == 0 - -# --- Kiwix startup: serve whatever completed ZIM files exist --- -# This is decoupled from the declared inventory - it just serves what's available. - -# Find all completed ZIM files in transmission download directory -- name: Find completed ZIM files in transmission download directory - ansible.builtin.find: - paths: "{{ transmission_download_dir }}" - patterns: "*.zim" - file_type: file - register: kiwix_completed_zim_files - when: kiwix_use_transmission - -# Check which ZIM files already have symlinks in kiwix directory -- name: Check existing symlinks in kiwix directory - ansible.builtin.stat: - path: "{{ kiwix_zim_dir }}/{{ item.path | basename }}" - get_checksum: false - loop: "{{ kiwix_completed_zim_files.files | default([]) }}" - loop_control: - label: "{{ item.path | basename }}" - register: kiwix_existing_symlinks - when: kiwix_use_transmission - -# Create symlinks for any completed ZIM files not yet linked +# --- Symlink completed ZIM files --- - name: Symlink completed ZIM files to kiwix directory - ansible.builtin.file: - src: "{{ item.item.path }}" - dest: "{{ kiwix_zim_dir }}/{{ item.item.path | basename }}" - state: link - loop: "{{ kiwix_existing_symlinks.results | default([]) }}" - loop_control: - label: "{{ item.item.path | basename }}" - when: - - kiwix_use_transmission - - item.stat is defined - - not item.stat.exists + ansible.builtin.command: "{{ kiwix_bin_dir }}/kiwix-symlink-zims.sh {{ transmission_download_dir }} {{ kiwix_zim_dir }}" + register: kiwix_symlink_result + changed_when: "'Linked:' in kiwix_symlink_result.stdout" + when: kiwix_use_transmission notify: Restart kiwix-serve # --- Fallback: Direct HTTP download (original behavior) --- diff --git a/ansible/roles/kiwix/templates/kiwix-symlink-zims.sh.j2 b/ansible/roles/kiwix/templates/kiwix-symlink-zims.sh.j2 new file mode 100644 index 0000000..29e7630 --- /dev/null +++ b/ansible/roles/kiwix/templates/kiwix-symlink-zims.sh.j2 @@ -0,0 +1,50 @@ +#!/bin/bash +# Symlink completed ZIM files from download directory to kiwix directory +set -euo pipefail + +SOURCE_DIR="${1:-}" +TARGET_DIR="${2:-}" + +if [[ -z "$SOURCE_DIR" || -z "$TARGET_DIR" ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +if [[ ! -d "$SOURCE_DIR" ]]; then + echo "Error: Source directory not found: $SOURCE_DIR" >&2 + exit 1 +fi + +if [[ ! -d "$TARGET_DIR" ]]; then + echo "Error: Target directory not found: $TARGET_DIR" >&2 + exit 1 +fi + +created=0 +skipped=0 + +# Find all .zim files in source directory +for zim_file in "$SOURCE_DIR"/*.zim; do + # Handle case where no .zim files exist + [[ -e "$zim_file" ]] || continue + + filename=$(basename "$zim_file") + target_path="$TARGET_DIR/$filename" + + if [[ -e "$target_path" || -L "$target_path" ]]; then + ((skipped++)) || true + else + ln -s "$zim_file" "$target_path" + echo "Linked: $filename" + ((created++)) || true + fi +done + +echo "Symlink complete: $created created, $skipped already present" + +# Exit with special code if new symlinks were created (for ansible changed detection) +if [[ $created -gt 0 ]]; then + exit 0 +else + exit 0 +fi diff --git a/ansible/roles/kiwix/templates/kiwix-sync-torrents.sh.j2 b/ansible/roles/kiwix/templates/kiwix-sync-torrents.sh.j2 new file mode 100644 index 0000000..d95d7dd --- /dev/null +++ b/ansible/roles/kiwix/templates/kiwix-sync-torrents.sh.j2 @@ -0,0 +1,46 @@ +#!/bin/bash +# Sync ZIM archive torrents to transmission +# Reads torrent URLs from stdin or file, adds any missing to transmission +set -euo pipefail + +TORRENT_LIST="${1:-}" + +if [[ -z "$TORRENT_LIST" ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi + +if [[ ! -f "$TORRENT_LIST" ]]; then + echo "Error: Torrent list file not found: $TORRENT_LIST" >&2 + exit 1 +fi + +# Get current torrents from transmission (extract names, skip header/footer) +current_torrents=$(transmission-remote -l 2>/dev/null | tail -n +2 | head -n -1 | awk '{print $NF}' || true) + +added=0 +skipped=0 + +while IFS= read -r torrent_url || [[ -n "$torrent_url" ]]; do + # Skip empty lines and comments + [[ -z "$torrent_url" || "$torrent_url" =~ ^# ]] && continue + + # Extract base name from URL (remove .torrent extension and path) + base_name=$(basename "$torrent_url" .torrent) + # Also try without .zim in case transmission reports it differently + base_without_zim="${base_name%.zim}" + + # Check if already in transmission + if echo "$current_torrents" | grep -qF "$base_without_zim"; then + ((skipped++)) || true + else + if transmission-remote -a "$torrent_url" 2>/dev/null; then + echo "Added: $base_name" + ((added++)) || true + else + echo "Warning: Failed to add $torrent_url" >&2 + fi + fi +done < "$TORRENT_LIST" + +echo "Sync complete: $added added, $skipped already present" diff --git a/ansible/roles/kiwix/templates/kiwix-torrents.txt.j2 b/ansible/roles/kiwix/templates/kiwix-torrents.txt.j2 new file mode 100644 index 0000000..fcc4b4e --- /dev/null +++ b/ansible/roles/kiwix/templates/kiwix-torrents.txt.j2 @@ -0,0 +1,5 @@ +# ZIM archive torrent URLs for kiwix +# Generated by ansible - do not edit manually +{% for archive in kiwix_zim_archives %} +{{ kiwix_torrent_base_url }}/{{ archive.category }}/{{ archive.filename }}.torrent +{% endfor %} -- 2.50.1 (Apple Git-155) From 53a0bf58133db54d77bdfa14b4cbbc550e913e53 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 16 Jan 2026 22:36:28 -0800 Subject: [PATCH 3/3] Fix macOS compatibility in kiwix torrent sync script BSD head (macOS) doesn't support negative line counts like GNU head. Use `sed '$d'` instead of `head -n -1` to remove the last line. Co-Authored-By: Claude Opus 4.5 --- ansible/roles/kiwix/templates/kiwix-sync-torrents.sh.j2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/roles/kiwix/templates/kiwix-sync-torrents.sh.j2 b/ansible/roles/kiwix/templates/kiwix-sync-torrents.sh.j2 index d95d7dd..4a293b3 100644 --- a/ansible/roles/kiwix/templates/kiwix-sync-torrents.sh.j2 +++ b/ansible/roles/kiwix/templates/kiwix-sync-torrents.sh.j2 @@ -16,7 +16,8 @@ if [[ ! -f "$TORRENT_LIST" ]]; then fi # Get current torrents from transmission (extract names, skip header/footer) -current_torrents=$(transmission-remote -l 2>/dev/null | tail -n +2 | head -n -1 | awk '{print $NF}' || true) +# Note: Use sed '$d' instead of head -n -1 for macOS compatibility +current_torrents=$(transmission-remote -l 2>/dev/null | tail -n +2 | sed '$d' | awk '{print $NF}' || true) added=0 skipped=0 -- 2.50.1 (Apple Git-155)