From 248e1181029b5b81e9e6ffb96f4f1bd182138905 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 16 Jan 2026 07:26:59 -0800 Subject: [PATCH 01/15] Add PostgreSQL and Miniflux services to tailnet - Add postgresql ansible role (postgresql@18 via homebrew) - Creates miniflux database and user - Configures pg_hba.conf for local scram-sha-256 auth - Exposed via Tailscale at pg.tail8d86e.ts.net:5432 - Add miniflux ansible role (RSS/Atom feed reader) - Depends on postgresql role - Configures via /opt/homebrew/etc/miniflux.conf - Reads DB password from ~/.miniflux-db-password - Supports first-run admin creation via miniflux_create_admin flag - Exposed via Tailscale at feed.tail8d86e.ts.net - Update Pulumi ACL tags (tag:pg, tag:feed) - Update tailscale_serve role with new service definitions - Update Alloy log collection for both services - Update indri.yml playbook with new roles Co-Authored-By: Claude Opus 4.5 --- ansible/playbooks/indri.yml | 4 + ansible/roles/alloy/defaults/main.yml | 6 ++ ansible/roles/miniflux/defaults/main.yml | 34 +++++++ ansible/roles/miniflux/handlers/main.yml | 5 ++ ansible/roles/miniflux/meta/main.yml | 3 + ansible/roles/miniflux/tasks/main.yml | 88 ++++++++++++++++++ .../roles/miniflux/templates/miniflux.conf.j2 | 31 +++++++ ansible/roles/postgresql/defaults/main.yml | 23 +++++ ansible/roles/postgresql/handlers/main.yml | 5 ++ ansible/roles/postgresql/tasks/main.yml | 89 +++++++++++++++++++ .../roles/postgresql/templates/pg_hba.conf.j2 | 13 +++ .../roles/tailscale_serve/defaults/main.yml | 10 +++ ansible/roles/tailscale_serve/meta/main.yml | 1 + pulumi/policy.hujson | 6 ++ 14 files changed, 318 insertions(+) create mode 100644 ansible/roles/miniflux/defaults/main.yml create mode 100644 ansible/roles/miniflux/handlers/main.yml create mode 100644 ansible/roles/miniflux/meta/main.yml create mode 100644 ansible/roles/miniflux/tasks/main.yml create mode 100644 ansible/roles/miniflux/templates/miniflux.conf.j2 create mode 100644 ansible/roles/postgresql/defaults/main.yml create mode 100644 ansible/roles/postgresql/handlers/main.yml create mode 100644 ansible/roles/postgresql/tasks/main.yml create mode 100644 ansible/roles/postgresql/templates/pg_hba.conf.j2 diff --git a/ansible/playbooks/indri.yml b/ansible/playbooks/indri.yml index d038c89..8191bb3 100644 --- a/ansible/playbooks/indri.yml +++ b/ansible/playbooks/indri.yml @@ -26,5 +26,9 @@ tags: devpi_metrics - role: plex_metrics tags: plex_metrics + - role: postgresql + tags: postgresql + - role: miniflux + tags: miniflux - role: tailscale_serve tags: tailscale-serve diff --git a/ansible/roles/alloy/defaults/main.yml b/ansible/roles/alloy/defaults/main.yml index a81d0ef..f4cf069 100644 --- a/ansible/roles/alloy/defaults/main.yml +++ b/ansible/roles/alloy/defaults/main.yml @@ -40,6 +40,12 @@ alloy_brew_logs: - path: /opt/homebrew/var/transmission/transmission-daemon.log service: transmission stream: stdout + - path: /opt/homebrew/var/log/postgresql@18.log + service: postgresql + stream: stdout + - path: /opt/homebrew/var/log/miniflux.log + service: miniflux + stream: stdout alloy_mcquack_logs: - path: /Users/erichblume/Library/Logs/mcquack.devpi.out.log diff --git a/ansible/roles/miniflux/defaults/main.yml b/ansible/roles/miniflux/defaults/main.yml new file mode 100644 index 0000000..68a8d58 --- /dev/null +++ b/ansible/roles/miniflux/defaults/main.yml @@ -0,0 +1,34 @@ +--- +# Miniflux configuration + +# Network settings +miniflux_listen_addr: "127.0.0.1:8080" +miniflux_base_url: "https://feed.tail8d86e.ts.net/" + +# Database connection (password read from file on host) +miniflux_db_host: localhost +miniflux_db_port: 5432 +miniflux_db_name: miniflux +miniflux_db_user: miniflux +miniflux_db_password_file: ~/.miniflux-db-password + +# Config paths +miniflux_config_file: /opt/homebrew/etc/miniflux.conf + +# First run settings +# Set miniflux_create_admin to 1 for initial setup, then 0 after +miniflux_create_admin: 0 +miniflux_admin_username: admin +miniflux_admin_password_file: ~/.miniflux-admin-password + +# Always run migrations to keep schema updated +miniflux_run_migrations: 1 + +# Polling settings +miniflux_polling_frequency: 60 +miniflux_batch_size: 100 +miniflux_polling_scheduler: "entry_frequency" + +# Cleanup settings (archive old entries) +miniflux_cleanup_archive_unread_days: 180 +miniflux_cleanup_archive_read_days: 60 diff --git a/ansible/roles/miniflux/handlers/main.yml b/ansible/roles/miniflux/handlers/main.yml new file mode 100644 index 0000000..936d0a3 --- /dev/null +++ b/ansible/roles/miniflux/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: restart miniflux + ansible.builtin.command: brew services restart miniflux + async: 120 + poll: 0 diff --git a/ansible/roles/miniflux/meta/main.yml b/ansible/roles/miniflux/meta/main.yml new file mode 100644 index 0000000..92e0133 --- /dev/null +++ b/ansible/roles/miniflux/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: postgresql diff --git a/ansible/roles/miniflux/tasks/main.yml b/ansible/roles/miniflux/tasks/main.yml new file mode 100644 index 0000000..99c6ca0 --- /dev/null +++ b/ansible/roles/miniflux/tasks/main.yml @@ -0,0 +1,88 @@ +--- +# Miniflux installation and configuration +# +# ONE-TIME SETUP (before running ansible): +# +# 1. Create miniflux database password in 1Password (same as PostgreSQL user password) +# 2. Create the password file on indri: +# +# ssh indri 'echo "your-password" > ~/.miniflux-db-password && chmod 600 ~/.miniflux-db-password' +# +# FIRST RUN (admin creation): +# +# 1. Create admin password in 1Password +# 2. Create admin password file on indri: +# +# ssh indri 'echo "your-admin-password" > ~/.miniflux-admin-password && chmod 600 ~/.miniflux-admin-password' +# +# 3. Set miniflux_create_admin: 1 in playbook vars or via --extra-vars +# 4. Run ansible: mise run provision-indri -- --tags miniflux -e miniflux_create_admin=1 +# 5. After successful first run, remove admin password file: +# +# ssh indri 'rm ~/.miniflux-admin-password' +# + +- name: Install miniflux via homebrew + community.general.homebrew: + name: miniflux + state: present + +- name: Check if database password file exists + ansible.builtin.stat: + path: "{{ miniflux_db_password_file }}" + register: db_password_file + +- name: Fail if database password not configured + ansible.builtin.fail: + msg: | + Miniflux database password not found at {{ miniflux_db_password_file }} + Create it with: + echo "YOUR_PASSWORD" > {{ miniflux_db_password_file }} && chmod 600 {{ miniflux_db_password_file }} + Password should match what was set for the miniflux PostgreSQL user. + when: not db_password_file.stat.exists + +- name: Read database password + ansible.builtin.slurp: + src: "{{ miniflux_db_password_file }}" + register: db_password_content + when: db_password_file.stat.exists + +- name: Set database password fact + ansible.builtin.set_fact: + miniflux_db_password: "{{ db_password_content.content | b64decode | trim }}" + when: db_password_file.stat.exists + +# Handle first run admin setup +- name: Check if admin password file exists (for first run) + ansible.builtin.stat: + path: "{{ miniflux_admin_password_file }}" + register: admin_password_file + when: miniflux_create_admin | int == 1 + +- name: Read admin password for first run + ansible.builtin.slurp: + src: "{{ miniflux_admin_password_file }}" + register: admin_password_content + when: + - miniflux_create_admin | int == 1 + - admin_password_file.stat.exists | default(false) + +- name: Set admin password fact + ansible.builtin.set_fact: + miniflux_admin_password: "{{ admin_password_content.content | b64decode | trim }}" + when: + - miniflux_create_admin | int == 1 + - admin_password_file.stat.exists | default(false) + +- name: Deploy miniflux configuration + ansible.builtin.template: + src: miniflux.conf.j2 + dest: "{{ miniflux_config_file }}" + mode: '0600' + notify: restart miniflux + +- name: Ensure miniflux service is started + ansible.builtin.command: brew services start miniflux + register: brew_start + changed_when: "'Successfully started' in brew_start.stdout" + failed_when: false diff --git a/ansible/roles/miniflux/templates/miniflux.conf.j2 b/ansible/roles/miniflux/templates/miniflux.conf.j2 new file mode 100644 index 0000000..2a479f4 --- /dev/null +++ b/ansible/roles/miniflux/templates/miniflux.conf.j2 @@ -0,0 +1,31 @@ +# {{ ansible_managed }} +# Miniflux configuration - KEY=VALUE format + +# Server settings +LISTEN_ADDR={{ miniflux_listen_addr }} +BASE_URL={{ miniflux_base_url }} + +# Database connection +DATABASE_URL=postgres://{{ miniflux_db_user }}:{{ miniflux_db_password }}@{{ miniflux_db_host }}:{{ miniflux_db_port }}/{{ miniflux_db_name }}?sslmode=disable + +# Migrations (always run to keep schema updated) +RUN_MIGRATIONS={{ miniflux_run_migrations }} + +{% if miniflux_create_admin | int == 1 and miniflux_admin_password is defined %} +# First run admin creation +CREATE_ADMIN=1 +ADMIN_USERNAME={{ miniflux_admin_username }} +ADMIN_PASSWORD={{ miniflux_admin_password }} +{% endif %} + +# Polling settings +POLLING_FREQUENCY={{ miniflux_polling_frequency }} +BATCH_SIZE={{ miniflux_batch_size }} +POLLING_SCHEDULER={{ miniflux_polling_scheduler }} + +# Cleanup settings +CLEANUP_ARCHIVE_UNREAD_DAYS={{ miniflux_cleanup_archive_unread_days }} +CLEANUP_ARCHIVE_READ_DAYS={{ miniflux_cleanup_archive_read_days }} + +# Logging +LOG_LEVEL=info diff --git a/ansible/roles/postgresql/defaults/main.yml b/ansible/roles/postgresql/defaults/main.yml new file mode 100644 index 0000000..89bd25a --- /dev/null +++ b/ansible/roles/postgresql/defaults/main.yml @@ -0,0 +1,23 @@ +--- +# PostgreSQL configuration + +# Formula and version +postgresql_formula: postgresql@18 + +# Paths (keg-only formula on macOS) +postgresql_bin_dir: /opt/homebrew/opt/postgresql@18/bin +postgresql_data_dir: /opt/homebrew/var/postgresql@18 +postgresql_config_dir: /opt/homebrew/var/postgresql@18 + +# Network settings +postgresql_port: 5432 +postgresql_listen_addresses: "localhost" + +# Databases to create +postgresql_databases: + - name: miniflux + owner: miniflux + +# Users to create (passwords set manually via 1Password) +postgresql_users: + - name: miniflux diff --git a/ansible/roles/postgresql/handlers/main.yml b/ansible/roles/postgresql/handlers/main.yml new file mode 100644 index 0000000..a6ce462 --- /dev/null +++ b/ansible/roles/postgresql/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: restart postgresql + ansible.builtin.command: brew services restart {{ postgresql_formula }} + async: 120 + poll: 0 diff --git a/ansible/roles/postgresql/tasks/main.yml b/ansible/roles/postgresql/tasks/main.yml new file mode 100644 index 0000000..4471aae --- /dev/null +++ b/ansible/roles/postgresql/tasks/main.yml @@ -0,0 +1,89 @@ +--- +# PostgreSQL installation and configuration +# +# ONE-TIME SETUP (after running ansible): +# +# 1. Create the miniflux database password in 1Password +# 2. SSH to indri and set the password: +# +# /opt/homebrew/opt/postgresql@18/bin/psql -c "ALTER USER miniflux PASSWORD 'your-password-here';" +# +# 3. For borgmatic backups, create ~/.pgpass: +# echo "localhost:5432:miniflux:miniflux:YOUR_PASSWORD" > ~/.pgpass +# chmod 600 ~/.pgpass +# + +- name: Install {{ postgresql_formula }} via homebrew + community.general.homebrew: + name: "{{ postgresql_formula }}" + state: present + +- name: Check if postgresql data directory is initialized + ansible.builtin.stat: + path: "{{ postgresql_data_dir }}/PG_VERSION" + register: pg_data + +- name: Initialize postgresql database cluster + ansible.builtin.command: > + {{ postgresql_bin_dir }}/initdb + --locale=en_US.UTF-8 -E UTF-8 + {{ postgresql_data_dir }} + when: not pg_data.stat.exists + +- name: Deploy pg_hba.conf + ansible.builtin.template: + src: pg_hba.conf.j2 + dest: "{{ postgresql_config_dir }}/pg_hba.conf" + mode: '0600' + notify: restart postgresql + +- name: Ensure postgresql service is started + ansible.builtin.command: brew services start {{ postgresql_formula }} + register: brew_start + changed_when: "'Successfully started' in brew_start.stdout" + failed_when: false + +- name: Wait for postgresql to accept connections + ansible.builtin.command: > + {{ postgresql_bin_dir }}/pg_isready -h localhost -p {{ postgresql_port }} + register: pg_ready + until: pg_ready.rc == 0 + retries: 10 + delay: 2 + changed_when: false + +# Create database users (without passwords - set manually via 1Password) +- name: Check if postgresql users exist + ansible.builtin.command: > + {{ postgresql_bin_dir }}/psql -tAc + "SELECT 1 FROM pg_roles WHERE rolname = '{{ item.name }}';" + loop: "{{ postgresql_users }}" + register: user_check + changed_when: false + failed_when: false + +- name: Create postgresql users + ansible.builtin.command: > + {{ postgresql_bin_dir }}/psql -c "CREATE USER {{ item.item.name }};" + loop: "{{ user_check.results }}" + when: item.stdout != "1" + changed_when: true + +# Create databases +- name: Check if postgresql databases exist + ansible.builtin.command: > + {{ postgresql_bin_dir }}/psql -tAc + "SELECT 1 FROM pg_database WHERE datname = '{{ item.name }}';" + loop: "{{ postgresql_databases }}" + register: db_check + changed_when: false + failed_when: false + +- name: Create postgresql databases + ansible.builtin.command: > + {{ postgresql_bin_dir }}/createdb + --owner={{ item.item.owner }} + {{ item.item.name }} + loop: "{{ db_check.results }}" + when: item.stdout != "1" + changed_when: true diff --git a/ansible/roles/postgresql/templates/pg_hba.conf.j2 b/ansible/roles/postgresql/templates/pg_hba.conf.j2 new file mode 100644 index 0000000..248392b --- /dev/null +++ b/ansible/roles/postgresql/templates/pg_hba.conf.j2 @@ -0,0 +1,13 @@ +# {{ ansible_managed }} +# PostgreSQL Client Authentication Configuration File + +# TYPE DATABASE USER ADDRESS METHOD + +# Local connections (Unix socket) - trust for initial setup +local all all trust + +# IPv4 local connections (password auth for services) +host all all 127.0.0.1/32 scram-sha-256 + +# IPv6 local connections +host all all ::1/128 scram-sha-256 diff --git a/ansible/roles/tailscale_serve/defaults/main.yml b/ansible/roles/tailscale_serve/defaults/main.yml index 8b1523d..b739213 100644 --- a/ansible/roles/tailscale_serve/defaults/main.yml +++ b/ansible/roles/tailscale_serve/defaults/main.yml @@ -25,3 +25,13 @@ tailscale_services: https: port: 443 upstream: http://127.0.0.1:3141 + + - name: svc:pg + tcp: + port: 5432 + upstream: tcp://localhost:5432 + + - name: svc:feed + https: + port: 443 + upstream: http://localhost:8080 diff --git a/ansible/roles/tailscale_serve/meta/main.yml b/ansible/roles/tailscale_serve/meta/main.yml index 904ced2..bfc3021 100644 --- a/ansible/roles/tailscale_serve/meta/main.yml +++ b/ansible/roles/tailscale_serve/meta/main.yml @@ -4,3 +4,4 @@ dependencies: - role: forgejo - role: kiwix - role: devpi + - role: miniflux diff --git a/pulumi/policy.hujson b/pulumi/policy.hujson index 64bc11d..d135b1a 100644 --- a/pulumi/policy.hujson +++ b/pulumi/policy.hujson @@ -89,6 +89,12 @@ // Loki log collection "tag:loki": ["autogroup:admin"], + // PostgreSQL database server + "tag:pg": ["autogroup:admin"], + + // Miniflux RSS/Atom feed reader + "tag:feed": ["autogroup:admin"], + // This tag is applied to resources modified by blumeops-pulumi IaC "tag:blumeops": ["autogroup:admin"], }, -- 2.50.1 (Apple Git-155) From b7ccca87f35ea4f084c73a105cf49d009f593b87 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 16 Jan 2026 08:06:29 -0800 Subject: [PATCH 02/15] Secure password management via 1Password CLI in ansible - All passwords fetched from 1Password at runtime using `op` CLI - pg_hba.conf uses scram-sha-256 everywhere (no trust mode) - initdb uses --pwfile for secure superuser password bootstrap - All password-handling tasks use no_log: true - Add borgmatic user with pg_read_all_data for backup dumps - Remove pg-setup mise task (no longer needed) - Miniflux fetches password directly from 1Password Requires: `op signin` before running ansible Co-Authored-By: Claude Opus 4.5 --- ansible/roles/miniflux/defaults/main.yml | 8 +- ansible/roles/miniflux/tasks/main.yml | 84 ++++-------- .../roles/miniflux/templates/miniflux.conf.j2 | 3 +- ansible/roles/postgresql/defaults/main.yml | 15 ++- ansible/roles/postgresql/tasks/main.yml | 125 +++++++++++++++--- .../roles/postgresql/templates/pg_hba.conf.j2 | 9 +- 6 files changed, 161 insertions(+), 83 deletions(-) diff --git a/ansible/roles/miniflux/defaults/main.yml b/ansible/roles/miniflux/defaults/main.yml index 68a8d58..5419803 100644 --- a/ansible/roles/miniflux/defaults/main.yml +++ b/ansible/roles/miniflux/defaults/main.yml @@ -5,21 +5,23 @@ miniflux_listen_addr: "127.0.0.1:8080" miniflux_base_url: "https://feed.tail8d86e.ts.net/" -# Database connection (password read from file on host) +# Database connection (password fetched from 1Password) miniflux_db_host: localhost miniflux_db_port: 5432 miniflux_db_name: miniflux miniflux_db_user: miniflux -miniflux_db_password_file: ~/.miniflux-db-password # Config paths miniflux_config_file: /opt/homebrew/etc/miniflux.conf +# 1Password settings for admin password +miniflux_op_vault: vg6xf6vvfmoh5hqjjhlhbeoaie +miniflux_op_item: ns6wylqiuqgczpo7gq2akaxbti + # First run settings # Set miniflux_create_admin to 1 for initial setup, then 0 after miniflux_create_admin: 0 miniflux_admin_username: admin -miniflux_admin_password_file: ~/.miniflux-admin-password # Always run migrations to keep schema updated miniflux_run_migrations: 1 diff --git a/ansible/roles/miniflux/tasks/main.yml b/ansible/roles/miniflux/tasks/main.yml index 99c6ca0..d5cdb62 100644 --- a/ansible/roles/miniflux/tasks/main.yml +++ b/ansible/roles/miniflux/tasks/main.yml @@ -1,78 +1,49 @@ --- # Miniflux installation and configuration # -# ONE-TIME SETUP (before running ansible): -# -# 1. Create miniflux database password in 1Password (same as PostgreSQL user password) -# 2. Create the password file on indri: -# -# ssh indri 'echo "your-password" > ~/.miniflux-db-password && chmod 600 ~/.miniflux-db-password' -# -# FIRST RUN (admin creation): -# -# 1. Create admin password in 1Password -# 2. Create admin password file on indri: -# -# ssh indri 'echo "your-admin-password" > ~/.miniflux-admin-password && chmod 600 ~/.miniflux-admin-password' -# -# 3. Set miniflux_create_admin: 1 in playbook vars or via --extra-vars -# 4. Run ansible: mise run provision-indri -- --tags miniflux -e miniflux_create_admin=1 -# 5. After successful first run, remove admin password file: -# -# ssh indri 'rm ~/.miniflux-admin-password' +# Prerequisites: +# - PostgreSQL role has run (creates database, user, and ~/.miniflux-db-password) +# - 1Password CLI authenticated on control machine # +# First run: +# mise run provision-indri -- --tags miniflux -e miniflux_create_admin=1 - name: Install miniflux via homebrew community.general.homebrew: name: miniflux state: present -- name: Check if database password file exists - ansible.builtin.stat: - path: "{{ miniflux_db_password_file }}" - register: db_password_file +# === Fetch passwords from 1Password === -- name: Fail if database password not configured - ansible.builtin.fail: - msg: | - Miniflux database password not found at {{ miniflux_db_password_file }} - Create it with: - echo "YOUR_PASSWORD" > {{ miniflux_db_password_file }} && chmod 600 {{ miniflux_db_password_file }} - Password should match what was set for the miniflux PostgreSQL user. - when: not db_password_file.stat.exists - -- name: Read database password - ansible.builtin.slurp: - src: "{{ miniflux_db_password_file }}" - register: db_password_content - when: db_password_file.stat.exists +- name: Fetch miniflux database password from 1Password + ansible.builtin.command: + cmd: op --vault {{ miniflux_op_vault }} item get {{ miniflux_op_item }} --fields password --reveal + delegate_to: localhost + register: miniflux_db_password_result + changed_when: false + no_log: true - name: Set database password fact ansible.builtin.set_fact: - miniflux_db_password: "{{ db_password_content.content | b64decode | trim }}" - when: db_password_file.stat.exists + miniflux_db_password: "{{ miniflux_db_password_result.stdout }}" + no_log: true -# Handle first run admin setup -- name: Check if admin password file exists (for first run) - ansible.builtin.stat: - path: "{{ miniflux_admin_password_file }}" - register: admin_password_file +- name: Fetch miniflux admin password from 1Password (for first run) + ansible.builtin.command: + cmd: op --vault {{ miniflux_op_vault }} item get {{ miniflux_op_item }} --fields admin-password --reveal + delegate_to: localhost + register: miniflux_admin_password_result + changed_when: false + no_log: true when: miniflux_create_admin | int == 1 -- name: Read admin password for first run - ansible.builtin.slurp: - src: "{{ miniflux_admin_password_file }}" - register: admin_password_content - when: - - miniflux_create_admin | int == 1 - - admin_password_file.stat.exists | default(false) - - name: Set admin password fact ansible.builtin.set_fact: - miniflux_admin_password: "{{ admin_password_content.content | b64decode | trim }}" - when: - - miniflux_create_admin | int == 1 - - admin_password_file.stat.exists | default(false) + miniflux_admin_password: "{{ miniflux_admin_password_result.stdout }}" + no_log: true + when: miniflux_create_admin | int == 1 + +# === Deploy configuration === - name: Deploy miniflux configuration ansible.builtin.template: @@ -80,6 +51,7 @@ dest: "{{ miniflux_config_file }}" mode: '0600' notify: restart miniflux + no_log: true - name: Ensure miniflux service is started ansible.builtin.command: brew services start miniflux diff --git a/ansible/roles/miniflux/templates/miniflux.conf.j2 b/ansible/roles/miniflux/templates/miniflux.conf.j2 index 2a479f4..2e23e07 100644 --- a/ansible/roles/miniflux/templates/miniflux.conf.j2 +++ b/ansible/roles/miniflux/templates/miniflux.conf.j2 @@ -1,5 +1,6 @@ # {{ ansible_managed }} # Miniflux configuration - KEY=VALUE format +# Passwords fetched from 1Password at deploy time. # Server settings LISTEN_ADDR={{ miniflux_listen_addr }} @@ -12,7 +13,7 @@ DATABASE_URL=postgres://{{ miniflux_db_user }}:{{ miniflux_db_password }}@{{ min RUN_MIGRATIONS={{ miniflux_run_migrations }} {% if miniflux_create_admin | int == 1 and miniflux_admin_password is defined %} -# First run admin creation +# First run admin creation (remove these after initial setup) CREATE_ADMIN=1 ADMIN_USERNAME={{ miniflux_admin_username }} ADMIN_PASSWORD={{ miniflux_admin_password }} diff --git a/ansible/roles/postgresql/defaults/main.yml b/ansible/roles/postgresql/defaults/main.yml index 89bd25a..eb18646 100644 --- a/ansible/roles/postgresql/defaults/main.yml +++ b/ansible/roles/postgresql/defaults/main.yml @@ -13,11 +13,24 @@ postgresql_config_dir: /opt/homebrew/var/postgresql@18 postgresql_port: 5432 postgresql_listen_addresses: "localhost" +# 1Password vault and item IDs for credentials +postgresql_op_vault: vg6xf6vvfmoh5hqjjhlhbeoaie +postgresql_op_superuser_item: guxu3j7ajhjyey6xxl2ovsl2ui +postgresql_op_miniflux_item: ns6wylqiuqgczpo7gq2akaxbti +postgresql_op_borgmatic_item: mw2bv5we7woicjza7hc6s44yvy + # Databases to create postgresql_databases: - name: miniflux owner: miniflux -# Users to create (passwords set manually via 1Password) +# Users to create (passwords fetched from 1Password) postgresql_users: - name: miniflux + op_item: "{{ postgresql_op_miniflux_item }}" + op_field: password + - name: borgmatic + op_item: "{{ postgresql_op_borgmatic_item }}" + op_field: db-password + roles: + - pg_read_all_data diff --git a/ansible/roles/postgresql/tasks/main.yml b/ansible/roles/postgresql/tasks/main.yml index 4471aae..c4e8658 100644 --- a/ansible/roles/postgresql/tasks/main.yml +++ b/ansible/roles/postgresql/tasks/main.yml @@ -1,35 +1,75 @@ --- # PostgreSQL installation and configuration # -# ONE-TIME SETUP (after running ansible): -# -# 1. Create the miniflux database password in 1Password -# 2. SSH to indri and set the password: -# -# /opt/homebrew/opt/postgresql@18/bin/psql -c "ALTER USER miniflux PASSWORD 'your-password-here';" -# -# 3. For borgmatic backups, create ~/.pgpass: -# echo "localhost:5432:miniflux:miniflux:YOUR_PASSWORD" > ~/.pgpass -# chmod 600 ~/.pgpass -# +# Passwords are fetched from 1Password at runtime using the `op` CLI. +# Requires: `op` authenticated on the control machine (run `op signin` first). - name: Install {{ postgresql_formula }} via homebrew community.general.homebrew: name: "{{ postgresql_formula }}" state: present +# === Fetch passwords from 1Password (on control machine) === + +- name: Fetch superuser password from 1Password + ansible.builtin.command: + cmd: op --vault {{ postgresql_op_vault }} item get {{ postgresql_op_superuser_item }} --fields password --reveal + delegate_to: localhost + register: pg_superuser_password_result + changed_when: false + no_log: true + +- name: Set superuser password fact + ansible.builtin.set_fact: + pg_superuser_password: "{{ pg_superuser_password_result.stdout }}" + no_log: true + +- name: Fetch user passwords from 1Password + ansible.builtin.command: + cmd: op --vault {{ postgresql_op_vault }} item get {{ item.op_item }} --fields {{ item.op_field }} --reveal + delegate_to: localhost + loop: "{{ postgresql_users }}" + register: pg_user_passwords_result + changed_when: false + no_log: true + +- name: Build user password lookup + ansible.builtin.set_fact: + pg_user_passwords: "{{ pg_user_passwords | default({}) | combine({item.item.name: item.stdout}) }}" + loop: "{{ pg_user_passwords_result.results }}" + no_log: true + +# === Initialize PostgreSQL cluster === + - name: Check if postgresql data directory is initialized ansible.builtin.stat: path: "{{ postgresql_data_dir }}/PG_VERSION" register: pg_data -- name: Initialize postgresql database cluster +- name: Create temporary password file for initdb + ansible.builtin.copy: + content: "{{ pg_superuser_password }}" + dest: /tmp/.pg_init_pwfile + mode: '0600' + when: not pg_data.stat.exists + no_log: true + +- name: Initialize postgresql database cluster with superuser password ansible.builtin.command: > {{ postgresql_bin_dir }}/initdb --locale=en_US.UTF-8 -E UTF-8 + --pwfile=/tmp/.pg_init_pwfile {{ postgresql_data_dir }} when: not pg_data.stat.exists +- name: Remove temporary password file + ansible.builtin.file: + path: /tmp/.pg_init_pwfile + state: absent + when: not pg_data.stat.exists + +# === Configure and start PostgreSQL === + - name: Deploy pg_hba.conf ansible.builtin.template: src: pg_hba.conf.j2 @@ -52,38 +92,85 @@ delay: 2 changed_when: false -# Create database users (without passwords - set manually via 1Password) +# === Create users with passwords === + - name: Check if postgresql users exist ansible.builtin.command: > - {{ postgresql_bin_dir }}/psql -tAc + {{ postgresql_bin_dir }}/psql -h localhost -tAc "SELECT 1 FROM pg_roles WHERE rolname = '{{ item.name }}';" + environment: + PGPASSWORD: "{{ pg_superuser_password }}" loop: "{{ postgresql_users }}" register: user_check changed_when: false failed_when: false + no_log: true -- name: Create postgresql users +- name: Create postgresql users with passwords ansible.builtin.command: > - {{ postgresql_bin_dir }}/psql -c "CREATE USER {{ item.item.name }};" + {{ postgresql_bin_dir }}/psql -h localhost -c + "CREATE USER {{ item.item.name }} WITH PASSWORD '{{ pg_user_passwords[item.item.name] }}';" + environment: + PGPASSWORD: "{{ pg_superuser_password }}" loop: "{{ user_check.results }}" when: item.stdout != "1" changed_when: true + no_log: true + +- name: Update postgresql user passwords (idempotent) + ansible.builtin.command: > + {{ postgresql_bin_dir }}/psql -h localhost -c + "ALTER USER {{ item.name }} WITH PASSWORD '{{ pg_user_passwords[item.name] }}';" + environment: + PGPASSWORD: "{{ pg_superuser_password }}" + loop: "{{ postgresql_users }}" + changed_when: false + no_log: true + +# === Grant roles to users === + +- name: Grant roles to users + ansible.builtin.command: > + {{ postgresql_bin_dir }}/psql -h localhost -c "GRANT {{ item.1 }} TO {{ item.0.name }};" + environment: + PGPASSWORD: "{{ pg_superuser_password }}" + loop: "{{ postgresql_users | subelements('roles', skip_missing=True) }}" + changed_when: false + no_log: true + +# === Create databases === -# Create databases - name: Check if postgresql databases exist ansible.builtin.command: > - {{ postgresql_bin_dir }}/psql -tAc + {{ postgresql_bin_dir }}/psql -h localhost -tAc "SELECT 1 FROM pg_database WHERE datname = '{{ item.name }}';" + environment: + PGPASSWORD: "{{ pg_superuser_password }}" loop: "{{ postgresql_databases }}" register: db_check changed_when: false failed_when: false + no_log: true - name: Create postgresql databases ansible.builtin.command: > - {{ postgresql_bin_dir }}/createdb + {{ postgresql_bin_dir }}/createdb -h localhost --owner={{ item.item.owner }} {{ item.item.name }} + environment: + PGPASSWORD: "{{ pg_superuser_password }}" loop: "{{ db_check.results }}" when: item.stdout != "1" changed_when: true + no_log: true + +# === Write credential files for local access === + +- name: Write .pgpass file for local authentication + ansible.builtin.copy: + content: | + localhost:{{ postgresql_port }}:*:{{ ansible_user_id }}:{{ pg_superuser_password }} + localhost:{{ postgresql_port }}:*:borgmatic:{{ pg_user_passwords['borgmatic'] }} + dest: ~/.pgpass + mode: '0600' + no_log: true diff --git a/ansible/roles/postgresql/templates/pg_hba.conf.j2 b/ansible/roles/postgresql/templates/pg_hba.conf.j2 index 248392b..992897a 100644 --- a/ansible/roles/postgresql/templates/pg_hba.conf.j2 +++ b/ansible/roles/postgresql/templates/pg_hba.conf.j2 @@ -1,12 +1,15 @@ # {{ ansible_managed }} # PostgreSQL Client Authentication Configuration File +# +# All connections require password authentication (scram-sha-256). +# Passwords are managed via 1Password and fetched by ansible at runtime. # TYPE DATABASE USER ADDRESS METHOD -# Local connections (Unix socket) - trust for initial setup -local all all trust +# Local connections (Unix socket) +local all all scram-sha-256 -# IPv4 local connections (password auth for services) +# IPv4 local connections (services connect via TCP) host all all 127.0.0.1/32 scram-sha-256 # IPv6 local connections -- 2.50.1 (Apple Git-155) From f9757d71e3b7914c92138595cb63cda5811f2a1a Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 16 Jan 2026 08:11:10 -0800 Subject: [PATCH 03/15] Add note that zettelkasten uses Obsidian Sync, not git Co-Authored-By: Claude Opus 4.5 --- CLAUDE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 44e655e..004b197 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,6 +20,8 @@ This displays all cards tagged `blumeops`, with the main project card first and You are encouraged to explore the zk, follow links, and propose updates to it as the project evolves. **Always keep the zettelkasten documentation up to date with any changes you make.** +Note: The zettelkasten is synced via Obsidian Sync, not git. You don't need to commit or push zk changes. + ## Rules for all sessions 1. Always start by reading the zk docs with the command above. -- 2.50.1 (Apple Git-155) From 8b701bb5aaae109a3dce950986a8b4b82a0b986a Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 16 Jan 2026 08:18:54 -0800 Subject: [PATCH 04/15] Add PostgreSQL metrics collection and Grafana dashboard - Add 'alloy' PostgreSQL user with pg_monitor role (read-only) - Configure Alloy prometheus.exporter.postgres for metrics collection - Password fetched from 1Password with no_log protection - Add PostgreSQL Grafana dashboard with: - Connection stats (active, total, by state) - Database sizes - Tuple operations rate - Transaction rate (commits/rollbacks) Co-Authored-By: Claude Opus 4.5 --- ansible/roles/alloy/defaults/main.yml | 12 + ansible/roles/alloy/tasks/main.yml | 22 +- ansible/roles/alloy/templates/config.alloy.j2 | 16 + .../grafana/files/dashboards/postgresql.json | 519 ++++++++++++++++++ ansible/roles/postgresql/defaults/main.yml | 5 + 5 files changed, 573 insertions(+), 1 deletion(-) create mode 100644 ansible/roles/grafana/files/dashboards/postgresql.json diff --git a/ansible/roles/alloy/defaults/main.yml b/ansible/roles/alloy/defaults/main.yml index f4cf069..45cc52b 100644 --- a/ansible/roles/alloy/defaults/main.yml +++ b/ansible/roles/alloy/defaults/main.yml @@ -74,3 +74,15 @@ alloy_plex_logs: # Enable log collection (requires Loki to be running) alloy_collect_logs: true + +# PostgreSQL metrics collection +alloy_collect_postgres: true +alloy_postgres_host: localhost +alloy_postgres_port: 5432 +alloy_postgres_user: alloy +alloy_postgres_database: postgres + +# 1Password settings for PostgreSQL metrics +alloy_op_vault: vg6xf6vvfmoh5hqjjhlhbeoaie +alloy_op_postgres_item: guxu3j7ajhjyey6xxl2ovsl2ui +alloy_op_postgres_field: alloy-user-pw diff --git a/ansible/roles/alloy/tasks/main.yml b/ansible/roles/alloy/tasks/main.yml index b6f5dad..568dbbb 100644 --- a/ansible/roles/alloy/tasks/main.yml +++ b/ansible/roles/alloy/tasks/main.yml @@ -25,12 +25,32 @@ state: directory mode: '0755' +# === Fetch PostgreSQL password from 1Password === + +- name: Fetch PostgreSQL metrics password from 1Password + ansible.builtin.command: + cmd: op --vault {{ alloy_op_vault }} item get {{ alloy_op_postgres_item }} --fields {{ alloy_op_postgres_field }} --reveal + delegate_to: localhost + register: alloy_postgres_password_result + changed_when: false + no_log: true + when: alloy_collect_postgres | default(false) + +- name: Set PostgreSQL password fact + ansible.builtin.set_fact: + alloy_postgres_password: "{{ alloy_postgres_password_result.stdout }}" + no_log: true + when: alloy_collect_postgres | default(false) + +# === Deploy configuration === + - name: Deploy alloy configuration ansible.builtin.template: src: config.alloy.j2 dest: "{{ alloy_config_dir }}/config.alloy" - mode: '0644' + mode: '0600' notify: restart alloy + no_log: true - name: Ensure alloy service is started ansible.builtin.command: brew services start grafana-alloy diff --git a/ansible/roles/alloy/templates/config.alloy.j2 b/ansible/roles/alloy/templates/config.alloy.j2 index 069d8d4..5b73991 100644 --- a/ansible/roles/alloy/templates/config.alloy.j2 +++ b/ansible/roles/alloy/templates/config.alloy.j2 @@ -35,6 +35,22 @@ prometheus.remote_write "prometheus" { } } +{% if alloy_collect_postgres | default(false) %} +// ============== POSTGRESQL METRICS ============== + +// PostgreSQL exporter (read-only metrics via pg_monitor role) +prometheus.exporter.postgres "postgresql" { + data_source_names = ["postgresql://{{ alloy_postgres_user }}:{{ alloy_postgres_password }}@{{ alloy_postgres_host }}:{{ alloy_postgres_port }}/{{ alloy_postgres_database }}?sslmode=disable"] +} + +// Scrape PostgreSQL metrics +prometheus.scrape "postgresql" { + targets = prometheus.exporter.postgres.postgresql.targets + forward_to = [prometheus.relabel.instance.receiver] + scrape_interval = "{{ alloy_scrape_interval }}" +} +{% endif %} + {% if alloy_collect_logs %} // ============== LOG COLLECTION ============== diff --git a/ansible/roles/grafana/files/dashboards/postgresql.json b/ansible/roles/grafana/files/dashboards/postgresql.json new file mode 100644 index 0000000..990a278 --- /dev/null +++ b/ansible/roles/grafana/files/dashboards/postgresql.json @@ -0,0 +1,519 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 80 }, + { "color": "red", "value": 100 } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "pg_up", + "refId": "A" + } + ], + "title": "PostgreSQL Up", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "pg_stat_activity_count{state=\"active\"}", + "refId": "A" + } + ], + "title": "Active Connections", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(pg_stat_activity_count)", + "refId": "A" + } + ], + "title": "Total Connections", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "decbytes" + } + }, + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(pg_database_size_bytes)", + "refId": "A" + } + ], + "title": "Total Database Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "expr": "pg_stat_activity_count", + "legendFormat": "{{state}}", + "refId": "A" + } + ], + "title": "Connections by State", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "decbytes" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "expr": "pg_database_size_bytes{datname!~\"template.*\"}", + "legendFormat": "{{datname}}", + "refId": "A" + } + ], + "title": "Database Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "ops" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "expr": "rate(pg_stat_database_tup_fetched{datname!~\"template.*\"}[5m])", + "legendFormat": "{{datname}} fetched", + "refId": "A" + }, + { + "expr": "rate(pg_stat_database_tup_inserted{datname!~\"template.*\"}[5m])", + "legendFormat": "{{datname}} inserted", + "refId": "B" + }, + { + "expr": "rate(pg_stat_database_tup_updated{datname!~\"template.*\"}[5m])", + "legendFormat": "{{datname}} updated", + "refId": "C" + }, + { + "expr": "rate(pg_stat_database_tup_deleted{datname!~\"template.*\"}[5m])", + "legendFormat": "{{datname}} deleted", + "refId": "D" + } + ], + "title": "Tuple Operations Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "expr": "rate(pg_stat_database_xact_commit{datname!~\"template.*\"}[5m])", + "legendFormat": "{{datname}} commits", + "refId": "A" + }, + { + "expr": "rate(pg_stat_database_xact_rollback{datname!~\"template.*\"}[5m])", + "legendFormat": "{{datname}} rollbacks", + "refId": "B" + } + ], + "title": "Transactions Rate", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": ["postgresql", "database"], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "PostgreSQL", + "uid": "postgresql", + "version": 1 +} diff --git a/ansible/roles/postgresql/defaults/main.yml b/ansible/roles/postgresql/defaults/main.yml index eb18646..5e145bf 100644 --- a/ansible/roles/postgresql/defaults/main.yml +++ b/ansible/roles/postgresql/defaults/main.yml @@ -34,3 +34,8 @@ postgresql_users: op_field: db-password roles: - pg_read_all_data + - name: alloy + op_item: "{{ postgresql_op_superuser_item }}" + op_field: alloy-user-pw + roles: + - pg_monitor -- 2.50.1 (Apple Git-155) From f293b0e7e51ff1b1ee418b3cea2cbe81af974f29 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 16 Jan 2026 09:47:05 -0800 Subject: [PATCH 05/15] Add Pulumi DeviceTags for indri to manage device tags via IaC Instead of manually applying tags to indri in Tailscale admin, use tailscale.DeviceTags resource to manage them declaratively. This includes all service tags (grafana, forge, kiwix, devpi, loki, pg, feed) plus homelab and blumeops tags. Co-Authored-By: Claude Opus 4.5 --- pulumi/__main__.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pulumi/__main__.py b/pulumi/__main__.py index 8971cae..8c5cd39 100644 --- a/pulumi/__main__.py +++ b/pulumi/__main__.py @@ -14,5 +14,28 @@ acl = tailscale.Acl( acl=policy_content, ) +# ============== Device Tags ============== +# Manage tags for devices in the tailnet + +# indri - Mac Mini M1 running homelab services +indri = tailscale.get_device(name="indri.tail8d86e.ts.net") +indri_tags = tailscale.DeviceTags( + "indri-tags", + device_id=indri.node_id, + tags=[ + "tag:homelab", + "tag:grafana", + "tag:forge", + "tag:kiwix", + "tag:devpi", + "tag:loki", + "tag:pg", + "tag:feed", + "tag:blumeops", + ], +) + # Export useful info pulumi.export("acl_id", acl.id) +pulumi.export("indri_device_id", indri.node_id) +pulumi.export("indri_tags", indri_tags.tags) -- 2.50.1 (Apple Git-155) From 544682e92eab0646884cf8b7e8fde696a524a036 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 16 Jan 2026 09:51:30 -0800 Subject: [PATCH 06/15] Restrict .pgpass to read-only borgmatic user only Remove superuser from .pgpass since it's not needed for automated operations. Only borgmatic (with pg_read_all_data role) needs passwordless access for pg_dump backups. Co-Authored-By: Claude Opus 4.5 --- ansible/roles/postgresql/tasks/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ansible/roles/postgresql/tasks/main.yml b/ansible/roles/postgresql/tasks/main.yml index c4e8658..6a6ff30 100644 --- a/ansible/roles/postgresql/tasks/main.yml +++ b/ansible/roles/postgresql/tasks/main.yml @@ -166,10 +166,12 @@ # === Write credential files for local access === -- name: Write .pgpass file for local authentication +# .pgpass is used by borgmatic for pg_dump backups +# Only includes read-only roles (borgmatic has pg_read_all_data) +- name: Write .pgpass file for borgmatic backups ansible.builtin.copy: content: | - localhost:{{ postgresql_port }}:*:{{ ansible_user_id }}:{{ pg_superuser_password }} + # Managed by ansible - only read-only roles localhost:{{ postgresql_port }}:*:borgmatic:{{ pg_user_passwords['borgmatic'] }} dest: ~/.pgpass mode: '0600' -- 2.50.1 (Apple Git-155) From 4e9c8c11f7a2b0bbae2994b3df8310eb6b580f8f Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 16 Jan 2026 09:57:20 -0800 Subject: [PATCH 07/15] Add tag:blumeops as owner of service tags for IaC management The OAuth client acts as tag:blumeops, so it needs to own all tags it manages on devices. This enables Pulumi to set device tags automatically instead of requiring manual Tailscale admin console changes. Co-Authored-By: Claude Opus 4.5 --- pulumi/policy.hujson | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pulumi/policy.hujson b/pulumi/policy.hujson index d135b1a..45ad401 100644 --- a/pulumi/policy.hujson +++ b/pulumi/policy.hujson @@ -72,31 +72,32 @@ "tagOwners": { // Grafana service host tag - "tag:grafana": ["autogroup:admin"], + "tag:grafana": ["autogroup:admin", "tag:blumeops"], // This tag applies to instances which are meant to be accessible in my homelab. These instances can be SSH'ed in to by any member of the admin autogroup. - "tag:homelab": ["autogroup:admin"], + "tag:homelab": ["autogroup:admin", "tag:blumeops"], // Kiwix, a local wiki server. I use it to create mirrors of wikipedia. - "tag:kiwix": ["autogroup:admin"], + "tag:kiwix": ["autogroup:admin", "tag:blumeops"], // Service tag for forgejo, scm host and code forge - "tag:forge": ["autogroup:admin"], + "tag:forge": ["autogroup:admin", "tag:blumeops"], // devpi pypi index - "tag:devpi": ["autogroup:admin"], + "tag:devpi": ["autogroup:admin", "tag:blumeops"], // Loki log collection - "tag:loki": ["autogroup:admin"], + "tag:loki": ["autogroup:admin", "tag:blumeops"], // PostgreSQL database server - "tag:pg": ["autogroup:admin"], + "tag:pg": ["autogroup:admin", "tag:blumeops"], // Miniflux RSS/Atom feed reader - "tag:feed": ["autogroup:admin"], + "tag:feed": ["autogroup:admin", "tag:blumeops"], // This tag is applied to resources modified by blumeops-pulumi IaC - "tag:blumeops": ["autogroup:admin"], + // Includes itself so the OAuth client can apply it to devices + "tag:blumeops": ["autogroup:admin", "tag:blumeops"], }, // Test access rules every time they're saved. -- 2.50.1 (Apple Git-155) From 02a2c9d6fbde8e0c7a8a1f9108c28803603e3978 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 16 Jan 2026 10:39:40 -0800 Subject: [PATCH 08/15] Fix psql commands to specify -d postgres database Without specifying a database, psql defaults to connecting to a database named after the current user, which doesn't exist on a fresh install. Co-Authored-By: Claude Opus 4.5 --- ansible/roles/postgresql/tasks/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ansible/roles/postgresql/tasks/main.yml b/ansible/roles/postgresql/tasks/main.yml index 6a6ff30..4ee473e 100644 --- a/ansible/roles/postgresql/tasks/main.yml +++ b/ansible/roles/postgresql/tasks/main.yml @@ -96,7 +96,7 @@ - name: Check if postgresql users exist ansible.builtin.command: > - {{ postgresql_bin_dir }}/psql -h localhost -tAc + {{ postgresql_bin_dir }}/psql -h localhost -d postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname = '{{ item.name }}';" environment: PGPASSWORD: "{{ pg_superuser_password }}" @@ -108,7 +108,7 @@ - name: Create postgresql users with passwords ansible.builtin.command: > - {{ postgresql_bin_dir }}/psql -h localhost -c + {{ postgresql_bin_dir }}/psql -h localhost -d postgres -c "CREATE USER {{ item.item.name }} WITH PASSWORD '{{ pg_user_passwords[item.item.name] }}';" environment: PGPASSWORD: "{{ pg_superuser_password }}" @@ -119,7 +119,7 @@ - name: Update postgresql user passwords (idempotent) ansible.builtin.command: > - {{ postgresql_bin_dir }}/psql -h localhost -c + {{ postgresql_bin_dir }}/psql -h localhost -d postgres -c "ALTER USER {{ item.name }} WITH PASSWORD '{{ pg_user_passwords[item.name] }}';" environment: PGPASSWORD: "{{ pg_superuser_password }}" @@ -131,7 +131,7 @@ - name: Grant roles to users ansible.builtin.command: > - {{ postgresql_bin_dir }}/psql -h localhost -c "GRANT {{ item.1 }} TO {{ item.0.name }};" + {{ postgresql_bin_dir }}/psql -h localhost -d postgres -c "GRANT {{ item.1 }} TO {{ item.0.name }};" environment: PGPASSWORD: "{{ pg_superuser_password }}" loop: "{{ postgresql_users | subelements('roles', skip_missing=True) }}" @@ -142,7 +142,7 @@ - name: Check if postgresql databases exist ansible.builtin.command: > - {{ postgresql_bin_dir }}/psql -h localhost -tAc + {{ postgresql_bin_dir }}/psql -h localhost -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '{{ item.name }}';" environment: PGPASSWORD: "{{ pg_superuser_password }}" -- 2.50.1 (Apple Git-155) From cdb843269195d2ac232b05e5c74cd20fc3ef604b Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 16 Jan 2026 11:32:03 -0800 Subject: [PATCH 09/15] Add ansible-managed borgmatic config with PostgreSQL backup - Move borgmatic config.yaml from manual to ansible-managed template - Add postgresql_databases backup for miniflux database - Consolidate 1Password credential fetching to playbook pre_tasks to reduce auth prompts during full playbook runs - Roles now check if credentials are already defined before fetching, so they still work when running with --tags Co-Authored-By: Claude Opus 4.5 --- ansible/playbooks/indri.yml | 60 +++++++++++++++++++ ansible/roles/alloy/tasks/main.yml | 10 +++- ansible/roles/borgmatic/defaults/main.yml | 41 +++++++++++++ ansible/roles/borgmatic/tasks/main.yml | 14 ++++- .../roles/borgmatic/templates/config.yaml.j2 | 45 ++++++++++++++ ansible/roles/miniflux/tasks/main.yml | 4 ++ ansible/roles/postgresql/tasks/main.yml | 6 ++ 7 files changed, 177 insertions(+), 3 deletions(-) create mode 100644 ansible/roles/borgmatic/templates/config.yaml.j2 diff --git a/ansible/playbooks/indri.yml b/ansible/playbooks/indri.yml index 8191bb3..5b6489f 100644 --- a/ansible/playbooks/indri.yml +++ b/ansible/playbooks/indri.yml @@ -1,6 +1,66 @@ --- - name: Configure indri hosts: indri + + # Fetch all 1Password credentials upfront to minimize prompts + # Each role also fetches its own credentials (with 'when: is not defined') + # so they still work when running with --tags + pre_tasks: + - name: Fetch PostgreSQL superuser password + ansible.builtin.command: + cmd: op --vault vg6xf6vvfmoh5hqjjhlhbeoaie item get guxu3j7ajhjyey6xxl2ovsl2ui --fields password --reveal + delegate_to: localhost + register: _pg_superuser_pw + changed_when: false + no_log: true + + - name: Set PostgreSQL superuser password fact + ansible.builtin.set_fact: + pg_superuser_password: "{{ _pg_superuser_pw.stdout }}" + no_log: true + + - name: Fetch PostgreSQL alloy user password + ansible.builtin.command: + cmd: op --vault vg6xf6vvfmoh5hqjjhlhbeoaie item get guxu3j7ajhjyey6xxl2ovsl2ui --fields alloy-user-pw --reveal + delegate_to: localhost + register: _pg_alloy_pw + changed_when: false + no_log: true + + - name: Set PostgreSQL alloy password fact + ansible.builtin.set_fact: + alloy_postgres_password: "{{ _pg_alloy_pw.stdout }}" + no_log: true + + - name: Fetch miniflux database password + ansible.builtin.command: + cmd: op --vault vg6xf6vvfmoh5hqjjhlhbeoaie item get ns6wylqiuqgczpo7gq2akaxbti --fields password --reveal + delegate_to: localhost + register: _miniflux_db_pw + changed_when: false + no_log: true + + - name: Set miniflux passwords fact + ansible.builtin.set_fact: + miniflux_db_password: "{{ _miniflux_db_pw.stdout }}" + no_log: true + + - name: Fetch borgmatic database password + ansible.builtin.command: + cmd: op --vault vg6xf6vvfmoh5hqjjhlhbeoaie item get mw2bv5we7woicjza7hc6s44yvy --fields db-password --reveal + delegate_to: localhost + register: _borgmatic_db_pw + changed_when: false + no_log: true + + - name: Build PostgreSQL user password lookup + ansible.builtin.set_fact: + pg_user_passwords: + miniflux: "{{ _miniflux_db_pw.stdout }}" + borgmatic: "{{ _borgmatic_db_pw.stdout }}" + alloy: "{{ _pg_alloy_pw.stdout }}" + no_log: true + roles: - role: loki tags: loki diff --git a/ansible/roles/alloy/tasks/main.yml b/ansible/roles/alloy/tasks/main.yml index 568dbbb..23ac3c3 100644 --- a/ansible/roles/alloy/tasks/main.yml +++ b/ansible/roles/alloy/tasks/main.yml @@ -26,6 +26,8 @@ mode: '0755' # === Fetch PostgreSQL password from 1Password === +# Skipped when running full playbook (pre_tasks sets it) +# but runs when using --tags alloy - name: Fetch PostgreSQL metrics password from 1Password ansible.builtin.command: @@ -34,13 +36,17 @@ register: alloy_postgres_password_result changed_when: false no_log: true - when: alloy_collect_postgres | default(false) + when: + - alloy_collect_postgres | default(false) + - alloy_postgres_password is not defined - name: Set PostgreSQL password fact ansible.builtin.set_fact: alloy_postgres_password: "{{ alloy_postgres_password_result.stdout }}" no_log: true - when: alloy_collect_postgres | default(false) + when: + - alloy_collect_postgres | default(false) + - alloy_postgres_password is not defined # === Deploy configuration === diff --git a/ansible/roles/borgmatic/defaults/main.yml b/ansible/roles/borgmatic/defaults/main.yml index 1727c48..243ba4d 100644 --- a/ansible/roles/borgmatic/defaults/main.yml +++ b/ansible/roles/borgmatic/defaults/main.yml @@ -1,7 +1,48 @@ --- borgmatic_config: /Users/erichblume/.config/borgmatic/config.yaml +borgmatic_config_dir: /Users/erichblume/.config/borgmatic borgmatic_log_dir: /Users/erichblume/Library/Logs # Schedule: runs daily at 2:00 AM borgmatic_schedule_hour: 2 borgmatic_schedule_minute: 0 + +# Source directories to back up +borgmatic_source_directories: + - /Users/erichblume/code/personal/zk + - /opt/homebrew/var/forgejo + - /Users/erichblume/code/3rd/kiwix-tools + - /Users/erichblume/.config/borgmatic + - /Users/erichblume/Documents + - /Users/erichblume/Pictures + - /Users/erichblume/devpi + - /opt/homebrew/var/loki + +# Backup repository +borgmatic_repositories: + - path: /Volumes/backups/borg/ + label: sifaka-borg-backups + encryption: repokey + append_only: true + +# Exclude patterns +borgmatic_exclude_patterns: + # Exclude mirrored PyPI cache (only backup private packages) + - /Users/erichblume/devpi/+files/root/pypi + - /opt/homebrew/var/loki + +# Encryption passcommand (reads borg passphrase) +borgmatic_encryption_passcommand: cat /Users/erichblume/.borg/config.yaml + +# Retention policy +borgmatic_keep_daily: 7 +borgmatic_keep_monthly: 12 +borgmatic_keep_yearly: 1000 + +# PostgreSQL databases to backup (streamed via pg_dump) +# Password is read from ~/.pgpass (managed by postgresql role) +borgmatic_postgresql_databases: + - name: miniflux + hostname: localhost + port: 5432 + username: borgmatic diff --git a/ansible/roles/borgmatic/tasks/main.yml b/ansible/roles/borgmatic/tasks/main.yml index f0c749a..db0704d 100644 --- a/ansible/roles/borgmatic/tasks/main.yml +++ b/ansible/roles/borgmatic/tasks/main.yml @@ -1,6 +1,18 @@ --- # Note: borgmatic is installed via mise (pipx), not managed here. -# This role manages only the scheduled LaunchAgent. +# This role manages the config file and scheduled LaunchAgent. + +- name: Ensure borgmatic config directory exists + ansible.builtin.file: + path: "{{ borgmatic_config_dir }}" + state: directory + mode: '0700' + +- name: Deploy borgmatic configuration + ansible.builtin.template: + src: config.yaml.j2 + dest: "{{ borgmatic_config }}" + mode: '0600' - name: Deploy borgmatic LaunchAgent plist ansible.builtin.template: diff --git a/ansible/roles/borgmatic/templates/config.yaml.j2 b/ansible/roles/borgmatic/templates/config.yaml.j2 new file mode 100644 index 0000000..4a782bf --- /dev/null +++ b/ansible/roles/borgmatic/templates/config.yaml.j2 @@ -0,0 +1,45 @@ +# {{ ansible_managed }} + +source_directories: +{% for dir in borgmatic_source_directories %} + - {{ dir }} +{% endfor %} + +source_directories_must_exist: true + +repositories: +{% for repo in borgmatic_repositories %} + - path: {{ repo.path }} + label: {{ repo.label }} +{% if repo.encryption is defined %} + encryption: {{ repo.encryption }} +{% endif %} +{% if repo.append_only is defined %} + append_only: {{ repo.append_only | lower }} +{% endif %} +{% endfor %} + +{% if borgmatic_exclude_patterns %} +exclude_patterns: +{% for pattern in borgmatic_exclude_patterns %} + - {{ pattern }} +{% endfor %} +{% endif %} + +encryption_passcommand: {{ borgmatic_encryption_passcommand }} + +# Retention policy +keep_daily: {{ borgmatic_keep_daily }} +keep_monthly: {{ borgmatic_keep_monthly }} +keep_yearly: {{ borgmatic_keep_yearly }} + +{% if borgmatic_postgresql_databases %} +# PostgreSQL database backups (streamed via pg_dump) +postgresql_databases: +{% for db in borgmatic_postgresql_databases %} + - name: {{ db.name }} + hostname: {{ db.hostname | default('localhost') }} + port: {{ db.port | default(5432) }} + username: {{ db.username }} +{% endfor %} +{% endif %} diff --git a/ansible/roles/miniflux/tasks/main.yml b/ansible/roles/miniflux/tasks/main.yml index d5cdb62..8c3f691 100644 --- a/ansible/roles/miniflux/tasks/main.yml +++ b/ansible/roles/miniflux/tasks/main.yml @@ -14,6 +14,8 @@ state: present # === Fetch passwords from 1Password === +# These are skipped when running full playbook (pre_tasks sets them) +# but run when using --tags miniflux - name: Fetch miniflux database password from 1Password ansible.builtin.command: @@ -22,11 +24,13 @@ register: miniflux_db_password_result changed_when: false no_log: true + when: miniflux_db_password is not defined - name: Set database password fact ansible.builtin.set_fact: miniflux_db_password: "{{ miniflux_db_password_result.stdout }}" no_log: true + when: miniflux_db_password is not defined - name: Fetch miniflux admin password from 1Password (for first run) ansible.builtin.command: diff --git a/ansible/roles/postgresql/tasks/main.yml b/ansible/roles/postgresql/tasks/main.yml index 4ee473e..65a98f6 100644 --- a/ansible/roles/postgresql/tasks/main.yml +++ b/ansible/roles/postgresql/tasks/main.yml @@ -10,6 +10,8 @@ state: present # === Fetch passwords from 1Password (on control machine) === +# These are skipped when running full playbook (pre_tasks sets them) +# but run when using --tags postgresql - name: Fetch superuser password from 1Password ansible.builtin.command: @@ -18,11 +20,13 @@ register: pg_superuser_password_result changed_when: false no_log: true + when: pg_superuser_password is not defined - name: Set superuser password fact ansible.builtin.set_fact: pg_superuser_password: "{{ pg_superuser_password_result.stdout }}" no_log: true + when: pg_superuser_password is not defined - name: Fetch user passwords from 1Password ansible.builtin.command: @@ -32,12 +36,14 @@ register: pg_user_passwords_result changed_when: false no_log: true + when: pg_user_passwords is not defined - name: Build user password lookup ansible.builtin.set_fact: pg_user_passwords: "{{ pg_user_passwords | default({}) | combine({item.item.name: item.stdout}) }}" loop: "{{ pg_user_passwords_result.results }}" no_log: true + when: pg_user_passwords is not defined # === Initialize PostgreSQL cluster === -- 2.50.1 (Apple Git-155) From 88cf7fac7e9c9402bdde4bac0d57111e0b2d6a4f Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 16 Jan 2026 11:59:54 -0800 Subject: [PATCH 10/15] Fix idempotency issues in playbook and tailscale_serve role - Add tags to pre_tasks so they only run when relevant roles are included - Make tailscale_serve idempotent by checking serve status JSON before configuring services (skips if already configured) Co-Authored-By: Claude Opus 4.5 --- ansible/playbooks/indri.yml | 9 +++++++++ ansible/roles/tailscale_serve/tasks/main.yml | 19 +++++++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/ansible/playbooks/indri.yml b/ansible/playbooks/indri.yml index 5b6489f..40ca301 100644 --- a/ansible/playbooks/indri.yml +++ b/ansible/playbooks/indri.yml @@ -5,6 +5,7 @@ # Fetch all 1Password credentials upfront to minimize prompts # Each role also fetches its own credentials (with 'when: is not defined') # so they still work when running with --tags + # Tags ensure pre_tasks only run when relevant roles are included pre_tasks: - name: Fetch PostgreSQL superuser password ansible.builtin.command: @@ -13,11 +14,13 @@ register: _pg_superuser_pw changed_when: false no_log: true + tags: [postgresql] - name: Set PostgreSQL superuser password fact ansible.builtin.set_fact: pg_superuser_password: "{{ _pg_superuser_pw.stdout }}" no_log: true + tags: [postgresql] - name: Fetch PostgreSQL alloy user password ansible.builtin.command: @@ -26,11 +29,13 @@ register: _pg_alloy_pw changed_when: false no_log: true + tags: [alloy, postgresql] - name: Set PostgreSQL alloy password fact ansible.builtin.set_fact: alloy_postgres_password: "{{ _pg_alloy_pw.stdout }}" no_log: true + tags: [alloy, postgresql] - name: Fetch miniflux database password ansible.builtin.command: @@ -39,11 +44,13 @@ register: _miniflux_db_pw changed_when: false no_log: true + tags: [miniflux, postgresql] - name: Set miniflux passwords fact ansible.builtin.set_fact: miniflux_db_password: "{{ _miniflux_db_pw.stdout }}" no_log: true + tags: [miniflux, postgresql] - name: Fetch borgmatic database password ansible.builtin.command: @@ -52,6 +59,7 @@ register: _borgmatic_db_pw changed_when: false no_log: true + tags: [postgresql] - name: Build PostgreSQL user password lookup ansible.builtin.set_fact: @@ -60,6 +68,7 @@ borgmatic: "{{ _borgmatic_db_pw.stdout }}" alloy: "{{ _pg_alloy_pw.stdout }}" no_log: true + tags: [postgresql] roles: - role: loki diff --git a/ansible/roles/tailscale_serve/tasks/main.yml b/ansible/roles/tailscale_serve/tasks/main.yml index 6ed7442..d7e9a8f 100644 --- a/ansible/roles/tailscale_serve/tasks/main.yml +++ b/ansible/roles/tailscale_serve/tasks/main.yml @@ -4,22 +4,33 @@ register: serve_status changed_when: false +- name: Parse serve status + ansible.builtin.set_fact: + serve_config: "{{ (serve_status.stdout | from_json).Services | default({}) }}" + +# Configure HTTPS if service doesn't have Web config yet - name: Configure HTTPS services ansible.builtin.command: > tailscale serve --service="{{ item.name }}" --https={{ item.https.port }} {{ item.https.upstream }} loop: "{{ tailscale_services }}" - when: item.https is defined + when: + - item.https is defined + - serve_config[item.name] is not defined or serve_config[item.name].Web is not defined register: https_result - changed_when: "'already serving' not in https_result.stderr | default('')" failed_when: false +# Configure TCP if service doesn't have the specific port configured yet - name: Configure TCP services ansible.builtin.command: > tailscale serve --service="{{ item.name }}" --tcp={{ item.tcp.port }} {{ item.tcp.upstream }} loop: "{{ tailscale_services }}" - when: item.tcp is defined + when: + - item.tcp is defined + - serve_config[item.name] is not defined or + serve_config[item.name].TCP is not defined or + serve_config[item.name].TCP[item.tcp.port | string] is not defined or + serve_config[item.name].TCP[item.tcp.port | string].TCPForward is not defined register: tcp_result - changed_when: "'already serving' not in tcp_result.stderr | default('')" failed_when: false -- 2.50.1 (Apple Git-155) From cce4c3776e1ad996c405757534dcc00ca7e23d61 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 16 Jan 2026 12:02:52 -0800 Subject: [PATCH 11/15] Fix PostgreSQL metrics by URL-encoding password in alloy config The password contained special characters (@, !, *) that broke the connection string URL parsing. Added urlencode filter to the template. Co-Authored-By: Claude Opus 4.5 --- ansible/roles/alloy/templates/config.alloy.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/alloy/templates/config.alloy.j2 b/ansible/roles/alloy/templates/config.alloy.j2 index 5b73991..08e3cdf 100644 --- a/ansible/roles/alloy/templates/config.alloy.j2 +++ b/ansible/roles/alloy/templates/config.alloy.j2 @@ -40,7 +40,7 @@ prometheus.remote_write "prometheus" { // PostgreSQL exporter (read-only metrics via pg_monitor role) prometheus.exporter.postgres "postgresql" { - data_source_names = ["postgresql://{{ alloy_postgres_user }}:{{ alloy_postgres_password }}@{{ alloy_postgres_host }}:{{ alloy_postgres_port }}/{{ alloy_postgres_database }}?sslmode=disable"] + data_source_names = ["postgresql://{{ alloy_postgres_user }}:{{ alloy_postgres_password | urlencode }}@{{ alloy_postgres_host }}:{{ alloy_postgres_port }}/{{ alloy_postgres_database }}?sslmode=disable"] } // Scrape PostgreSQL metrics -- 2.50.1 (Apple Git-155) From 7dca29aea921fb6609f39c88851a442aff03a24b Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 16 Jan 2026 12:15:45 -0800 Subject: [PATCH 12/15] Add XID exhaustion tracking to PostgreSQL dashboard - Add custom query for pg_database XID age monitoring - Add gauge showing XID age with threshold warnings (yellow at 150M, red at 180M) - Add time series chart for XID age trends - URL-encode postgres password in alloy connection string XID (transaction ID) exhaustion can cause PostgreSQL to shut down to prevent wraparound. Default autovacuum_freeze_max_age is 200M, so warnings start well before that threshold. Co-Authored-By: Claude Opus 4.5 --- ansible/roles/alloy/tasks/main.yml | 8 ++ ansible/roles/alloy/templates/config.alloy.j2 | 3 + .../alloy/templates/postgres_queries.yaml.j2 | 20 +++ .../grafana/files/dashboards/postgresql.json | 127 +++++++++++++++++- 4 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 ansible/roles/alloy/templates/postgres_queries.yaml.j2 diff --git a/ansible/roles/alloy/tasks/main.yml b/ansible/roles/alloy/tasks/main.yml index 23ac3c3..4fb6225 100644 --- a/ansible/roles/alloy/tasks/main.yml +++ b/ansible/roles/alloy/tasks/main.yml @@ -50,6 +50,14 @@ # === Deploy configuration === +- name: Deploy PostgreSQL custom queries config + ansible.builtin.template: + src: postgres_queries.yaml.j2 + dest: "{{ alloy_config_dir }}/postgres_queries.yaml" + mode: '0600' + notify: restart alloy + when: alloy_collect_postgres | default(false) + - name: Deploy alloy configuration ansible.builtin.template: src: config.alloy.j2 diff --git a/ansible/roles/alloy/templates/config.alloy.j2 b/ansible/roles/alloy/templates/config.alloy.j2 index 08e3cdf..e0c1cad 100644 --- a/ansible/roles/alloy/templates/config.alloy.j2 +++ b/ansible/roles/alloy/templates/config.alloy.j2 @@ -41,6 +41,9 @@ prometheus.remote_write "prometheus" { // PostgreSQL exporter (read-only metrics via pg_monitor role) prometheus.exporter.postgres "postgresql" { data_source_names = ["postgresql://{{ alloy_postgres_user }}:{{ alloy_postgres_password | urlencode }}@{{ alloy_postgres_host }}:{{ alloy_postgres_port }}/{{ alloy_postgres_database }}?sslmode=disable"] + + // Custom queries for vacuum and XID monitoring + custom_queries_config_path = "/opt/homebrew/etc/grafana-alloy/postgres_queries.yaml" } // Scrape PostgreSQL metrics diff --git a/ansible/roles/alloy/templates/postgres_queries.yaml.j2 b/ansible/roles/alloy/templates/postgres_queries.yaml.j2 new file mode 100644 index 0000000..6714995 --- /dev/null +++ b/ansible/roles/alloy/templates/postgres_queries.yaml.j2 @@ -0,0 +1,20 @@ +# {{ ansible_managed }} +# Custom PostgreSQL queries for XID exhaustion monitoring + +pg_database_xid_age: + query: | + SELECT datname, + age(datfrozenxid) as xid_age, + current_setting('autovacuum_freeze_max_age')::bigint as freeze_max_age + FROM pg_database + WHERE datallowconn + metrics: + - datname: + usage: "LABEL" + description: "Database name" + - xid_age: + usage: "GAUGE" + description: "Age of oldest unfrozen transaction ID" + - freeze_max_age: + usage: "GAUGE" + description: "autovacuum_freeze_max_age setting" diff --git a/ansible/roles/grafana/files/dashboards/postgresql.json b/ansible/roles/grafana/files/dashboards/postgresql.json index 990a278..6935416 100644 --- a/ansible/roles/grafana/files/dashboards/postgresql.json +++ b/ansible/roles/grafana/files/dashboards/postgresql.json @@ -500,7 +500,132 @@ ], "title": "Transactions Rate", "type": "timeseries" - } + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 200000000, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 150000000 }, + { "color": "red", "value": 180000000 } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 20 }, + "id": 9, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "targets": [ + { + "expr": "pg_database_xid_age_xid_age", + "legendFormat": "{{datname}}", + "refId": "A" + } + ], + "title": "Transaction ID Age (XID Exhaustion Risk)", + "description": "Age of oldest unfrozen XID. Approaches 2 billion = wraparound danger. Yellow at 150M, Red at 180M (autovacuum_freeze_max_age default is 200M).", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 20 }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "expr": "pg_database_xid_age_xid_age", + "legendFormat": "{{datname}}", + "refId": "A" + } + ], + "title": "XID Age Over Time", + "type": "timeseries" + }, ], "schemaVersion": 39, "tags": ["postgresql", "database"], -- 2.50.1 (Apple Git-155) From d56231d8134d83853815706e274ff67b1329e022 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 16 Jan 2026 12:20:55 -0800 Subject: [PATCH 13/15] Bump PostgreSQL dashboard version to force reload --- ansible/roles/grafana/files/dashboards/postgresql.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/grafana/files/dashboards/postgresql.json b/ansible/roles/grafana/files/dashboards/postgresql.json index 6935416..8479dff 100644 --- a/ansible/roles/grafana/files/dashboards/postgresql.json +++ b/ansible/roles/grafana/files/dashboards/postgresql.json @@ -640,5 +640,5 @@ "timezone": "browser", "title": "PostgreSQL", "uid": "postgresql", - "version": 1 + "version": 2 } -- 2.50.1 (Apple Git-155) From a1341cdaa6b75ffae50f97aa95a13616f53483e1 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 16 Jan 2026 12:24:34 -0800 Subject: [PATCH 14/15] Fix trailing comma in PostgreSQL dashboard JSON --- ansible/roles/grafana/files/dashboards/postgresql.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/grafana/files/dashboards/postgresql.json b/ansible/roles/grafana/files/dashboards/postgresql.json index 8479dff..b797499 100644 --- a/ansible/roles/grafana/files/dashboards/postgresql.json +++ b/ansible/roles/grafana/files/dashboards/postgresql.json @@ -625,7 +625,7 @@ ], "title": "XID Age Over Time", "type": "timeseries" - }, + } ], "schemaVersion": 39, "tags": ["postgresql", "database"], @@ -640,5 +640,5 @@ "timezone": "browser", "title": "PostgreSQL", "uid": "postgresql", - "version": 2 + "version": 3 } -- 2.50.1 (Apple Git-155) From b1c08af5596cf4d6126d570ef3216fd8e0a84a9c Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Fri, 16 Jan 2026 12:28:01 -0800 Subject: [PATCH 15/15] Remove XID gauge panel, keep only time series --- .../grafana/files/dashboards/postgresql.json | 52 +------------------ 1 file changed, 1 insertion(+), 51 deletions(-) diff --git a/ansible/roles/grafana/files/dashboards/postgresql.json b/ansible/roles/grafana/files/dashboards/postgresql.json index b797499..8a561c7 100644 --- a/ansible/roles/grafana/files/dashboards/postgresql.json +++ b/ansible/roles/grafana/files/dashboards/postgresql.json @@ -501,56 +501,6 @@ "title": "Transactions Rate", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 200000000, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 150000000 }, - { "color": "red", "value": 180000000 } - ] - }, - "unit": "short" - } - }, - "gridPos": { "h": 6, "w": 12, "x": 0, "y": 20 }, - "id": 9, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "horizontal", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "targets": [ - { - "expr": "pg_database_xid_age_xid_age", - "legendFormat": "{{datname}}", - "refId": "A" - } - ], - "title": "Transaction ID Age (XID Exhaustion Risk)", - "description": "Age of oldest unfrozen XID. Approaches 2 billion = wraparound danger. Yellow at 150M, Red at 180M (autovacuum_freeze_max_age default is 200M).", - "type": "gauge" - }, { "datasource": { "type": "prometheus", @@ -640,5 +590,5 @@ "timezone": "browser", "title": "PostgreSQL", "uid": "postgresql", - "version": 3 + "version": 4 } -- 2.50.1 (Apple Git-155)