From d75fdfdad626f7a7ab0679cf995d625fb659dcba Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Mon, 19 Jan 2026 08:55:08 -0800 Subject: [PATCH] Add PostgreSQL cluster manifest for Step 7 - Create blumeops-pg Cluster with CloudNativePG - Add eblume superuser role (matches current brew pg setup) - Configure pg_hba for password auth from any IP (Tailscale handles security) - Add secret template for eblume password from 1Password - Create ArgoCD Application with manual sync policy - Update Phase 1 plan with implementation notes Co-Authored-By: Claude Opus 4.5 --- argocd/apps/blumeops-pg.yaml | 24 +++ argocd/manifests/databases/README.md | 91 +++++++++++ argocd/manifests/databases/blumeops-pg.yaml | 52 ++++++ argocd/manifests/databases/kustomization.yaml | 7 + .../databases/secret-eblume.yaml.tpl | 13 ++ plans/k8s-migration/P1_k8s_infrastructure.md | 153 ++++++++++++++++++ 6 files changed, 340 insertions(+) create mode 100644 argocd/apps/blumeops-pg.yaml create mode 100644 argocd/manifests/databases/README.md create mode 100644 argocd/manifests/databases/blumeops-pg.yaml create mode 100644 argocd/manifests/databases/kustomization.yaml create mode 100644 argocd/manifests/databases/secret-eblume.yaml.tpl diff --git a/argocd/apps/blumeops-pg.yaml b/argocd/apps/blumeops-pg.yaml new file mode 100644 index 0000000..9550ba4 --- /dev/null +++ b/argocd/apps/blumeops-pg.yaml @@ -0,0 +1,24 @@ +# PostgreSQL Cluster for blumeops services +# Requires: CloudNativePG operator (cloudnative-pg app) and manual secret setup +# +# Before syncing, create the eblume password secret: +# kubectl create namespace databases +# op inject -i argocd/manifests/databases/secret-eblume.yaml.tpl | kubectl apply -f - +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: blumeops-pg + namespace: argocd +spec: + project: default + source: + repoURL: ssh://forgejo@indri.tail8d86e.ts.net:2200/eblume/blumeops.git + targetRevision: feature/k8s-phase1-kickoff + path: argocd/manifests/databases + destination: + server: https://kubernetes.default.svc + namespace: databases + syncPolicy: + syncOptions: + - CreateNamespace=true + # Manual sync only - no automated sync on git push diff --git a/argocd/manifests/databases/README.md b/argocd/manifests/databases/README.md new file mode 100644 index 0000000..9bd005e --- /dev/null +++ b/argocd/manifests/databases/README.md @@ -0,0 +1,91 @@ +# Database Manifests + +PostgreSQL clusters managed by CloudNativePG operator. + +## blumeops-pg + +Single-instance PostgreSQL cluster for blumeops services. + +### Configuration + +- **Instances**: 1 (single-node for minikube) +- **Storage**: 10Gi on `standard` storage class +- **Initial database**: `miniflux` owned by `miniflux` user + +### Users/Roles + +| User | Role | Purpose | Password Source | +|----------|-------------|----------------------------------|------------------------------------| +| postgres | superuser | CNPG internal (avoid using) | `blumeops-pg-superuser` secret | +| miniflux | app owner | Owns miniflux database | `blumeops-pg-app` secret | +| eblume | superuser | Admin access (matches brew pg) | `blumeops-pg-eblume` secret (manual) | + +### Manual Secret Setup + +Before deploying, create the eblume password secret: + +```bash +# Create namespace first +kubectl create namespace databases + +# Apply eblume password from 1Password +op inject -i argocd/manifests/databases/secret-eblume.yaml.tpl | kubectl apply -f - +``` + +The `miniflux` user password is auto-generated by CloudNativePG and stored in `blumeops-pg-app`. + +### Connection Information + +After the cluster is healthy: + +```bash +# Connect as eblume (same style as current brew pg) +# Uses same password as pg.tail8d86e.ts.net +PGPASSWORD=$(op --vault blumeops item get guxu3j7ajhjyey6xxl2ovsl2ui --fields password --reveal) \ + psql -h -U eblume -d miniflux + +# Get miniflux app credentials (for applications) +kubectl -n databases get secret blumeops-pg-app -o jsonpath='{.data.uri}' | base64 -d + +# Get postgres superuser credentials (emergency only) +kubectl -n databases get secret blumeops-pg-superuser -o jsonpath='{.data.password}' | base64 -d +``` + +### Connecting via kubectl port-forward + +Until Tailscale exposure is configured: + +```bash +# Terminal 1: Port-forward to the primary +kubectl -n databases port-forward svc/blumeops-pg-rw 5432:5432 + +# Terminal 2: Connect as eblume +PGPASSWORD=$(op --vault blumeops item get guxu3j7ajhjyey6xxl2ovsl2ui --fields password --reveal) \ + psql -h localhost -U eblume -d miniflux +``` + +### Status + +```bash +# Check cluster health +kubectl -n databases get cluster blumeops-pg + +# Check pods +kubectl -n databases get pods -l cnpg.io/cluster=blumeops-pg + +# Check managed roles status +kubectl -n databases get cluster blumeops-pg -o jsonpath='{.status.managedRolesStatus}' | jq + +# Operator logs +kubectl -n databases logs -l cnpg.io/cluster=blumeops-pg +``` + +## Future: Tailscale Exposure + +The cluster is currently internal-only. In Phase 4, after miniflux migrates to k8s, +the `pg.tail8d86e.ts.net` Tailscale service will be pointed to this cluster. + +When exposed, you'll be able to connect with: +```bash +psql -h pg.tail8d86e.ts.net -U eblume -W -d miniflux +``` diff --git a/argocd/manifests/databases/blumeops-pg.yaml b/argocd/manifests/databases/blumeops-pg.yaml new file mode 100644 index 0000000..93d30a2 --- /dev/null +++ b/argocd/manifests/databases/blumeops-pg.yaml @@ -0,0 +1,52 @@ +# PostgreSQL Cluster for blumeops services +# Managed by CloudNativePG operator +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: blumeops-pg + namespace: databases +spec: + instances: 1 + + storage: + size: 10Gi + storageClass: standard + + # Bootstrap creates initial database and owner + bootstrap: + initdb: + database: miniflux + owner: miniflux + + # Managed roles - additional users beyond the bootstrap owner + managed: + roles: + # eblume superuser for admin access (matches current brew pg setup) + - name: eblume + login: true + superuser: true + createdb: true + createrole: true + passwordSecret: + name: blumeops-pg-eblume + + # Resource limits for minikube environment + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "1Gi" + cpu: "500m" + + # PostgreSQL configuration + postgresql: + parameters: + max_connections: "50" + shared_buffers: "128MB" + password_encryption: "scram-sha-256" + pg_hba: + # Allow all users to connect from any IP with password auth + # Network security is handled by Tailscale + - host all all 0.0.0.0/0 scram-sha-256 + - host all all ::/0 scram-sha-256 diff --git a/argocd/manifests/databases/kustomization.yaml b/argocd/manifests/databases/kustomization.yaml new file mode 100644 index 0000000..9e4ff0a --- /dev/null +++ b/argocd/manifests/databases/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: databases + +resources: + - blumeops-pg.yaml diff --git a/argocd/manifests/databases/secret-eblume.yaml.tpl b/argocd/manifests/databases/secret-eblume.yaml.tpl new file mode 100644 index 0000000..481bd96 --- /dev/null +++ b/argocd/manifests/databases/secret-eblume.yaml.tpl @@ -0,0 +1,13 @@ +# Template for eblume superuser password +# Apply with: op inject -i secret-eblume.yaml.tpl | kubectl apply -f - +# +# Uses the same 1Password item as the brew PostgreSQL setup on indri +apiVersion: v1 +kind: Secret +metadata: + name: blumeops-pg-eblume + namespace: databases +type: kubernetes.io/basic-auth +stringData: + username: eblume + password: {{ op://vg6xf6vvfmoh5hqjjhlhbeoaie/guxu3j7ajhjyey6xxl2ovsl2ui/password }} diff --git a/plans/k8s-migration/P1_k8s_infrastructure.md b/plans/k8s-migration/P1_k8s_infrastructure.md index f05a7fa..9e02286 100644 --- a/plans/k8s-migration/P1_k8s_infrastructure.md +++ b/plans/k8s-migration/P1_k8s_infrastructure.md @@ -502,3 +502,156 @@ kubectl delete namespace tailscale-system git checkout pulumi/policy.hujson mise run tailnet-up ``` + +--- + +## Implementation Notes (Deviations from Plan) + +*Added during implementation for retrospective review* + +### Git Source: Forge Instead of GitHub + +**Plan**: Use GitHub mirror (`github.com/eblume/blumeops`) +**Actual**: Use internal Forgejo (`ssh://forgejo@indri.tail8d86e.ts.net:2200/eblume/blumeops.git`) + +**Why**: User preference to use internal infrastructure, accepting circular dependency for later. + +**Required changes**: +- Deploy key added to forge for ArgoCD SSH access +- Repository secret `repo-forge` with SSH private key from 1Password +- Discovered: `op read` requires `?ssh-format=openssh` query parameter for ArgoCD-compatible key format +- Egress proxy service to reach forge from cluster (targets `indri.tail8d86e.ts.net` not `forge.tail8d86e.ts.net` due to Tailscale Serve limitation) +- DNSConfig CRD for cluster-to-tailnet MagicDNS resolution +- ACL grant: `tag:k8s` → `tag:homelab` on ports 3001 (HTTP) and 2200 (SSH) + +### ArgoCD Exposure: Ingress Instead of LoadBalancer + +**Plan**: LoadBalancer service with `tailscale.com/hostname` annotation +**Actual**: Tailscale Ingress with Let's Encrypt TLS termination + +**Why**: Ingress provides automatic TLS certificates and is the recommended approach. + +**File**: `argocd/manifests/argocd/service-tailscale.yaml` uses `kind: Ingress` with `ingressClassName: tailscale` + +### Namespace: `tailscale` Instead of `tailscale-system` + +**Plan**: `tailscale-system` namespace +**Actual**: `tailscale` namespace + +**Why**: Matches upstream Tailscale operator defaults. + +### Sync Policy: Manual Instead of Automated + +**Plan**: `syncPolicy.automated` with prune and selfHeal +**Actual**: Manual sync policy for workload apps; auto-sync only for app-of-apps + +**Why**: User preference for explicit control over deployments during initial migration phase. + +**Pattern**: +- `apps.yaml` (app-of-apps): auto-sync to pick up new Application manifests +- All workload apps: manual sync requires `argocd app sync ` + +### CloudNativePG: Helm Chart Instead of Raw Manifest + +**Plan**: Download raw CNPG manifest +**Actual**: Multi-source Application using official Helm chart from `https://cloudnative-pg.github.io/charts` + +**Why**: Helm chart is the officially supported distribution method. + +**Additional fix**: Required `ServerSideApply=true` sync option due to large CRD exceeding annotation size limit. + +### App-of-Apps: Named `apps` Instead of `root` + +**Plan**: `argocd/apps/root.yaml` +**Actual**: `argocd/apps/apps.yaml` with Application named `apps` + +**Why**: Clearer naming; `apps` manages apps, `argocd` manages itself. + +### ArgoCD Self-Management Added + +**Plan**: Not explicitly planned +**Actual**: `argocd/apps/argocd.yaml` Application for ArgoCD self-management + +**Why**: Standard GitOps pattern - ArgoCD manages its own deployment after bootstrap. + +### CRI-O Registry Mirror for Zot + +**Plan**: Not in original plan +**Actual**: Configured CRI-O to use zot as pull-through cache for docker.io, ghcr.io, quay.io + +**Why**: Reduces external bandwidth, speeds up pulls, avoids rate limits. + +**Implementation**: Ansible `minikube` role applies `/etc/containers/registries.conf.d/zot-mirror.conf` inside minikube VM using stable hostname `host.containers.internal:5050`. + +### ProxyClass for CRI-O Image Compatibility + +**Plan**: Not mentioned +**Actual**: Required `ProxyClass` with fully-qualified image paths (`docker.io/tailscale/...`) + +**Why**: CRI-O requires fully-qualified image references; default Tailscale operator uses short names. + +### Actual File Structure + +``` +argocd/ + apps/ + apps.yaml # App-of-apps (auto-sync) + argocd.yaml # ArgoCD self-management (manual sync) + tailscale-operator.yaml # Tailscale operator (manual sync) + cloudnative-pg.yaml # CNPG operator via Helm (manual sync) + manifests/ + tailscale-operator/ + kustomization.yaml + operator.yaml + proxyclass.yaml # CRI-O compatibility + dnsconfig.yaml # Cluster-to-tailnet DNS + egress-forge.yaml # Egress proxy for forge + secret.yaml.tpl # OAuth secret template (manual) + README.md + argocd/ + kustomization.yaml # Uses remote base from upstream + service-tailscale.yaml # Ingress (not LoadBalancer) + argocd-cmd-params-cm.yaml # Disable HTTPS redirect + repo-forge-secret.yaml.tpl # SSH key template (manual) + README.md + cloudnative-pg/ + values.yaml # Helm values (currently minimal) + README.md +``` + +### Bootstrap Commands (Actual) + +```bash +# 1. Create namespaces +kubectl create namespace tailscale +kubectl create namespace argocd + +# 2. Apply secrets (manual, uses 1Password) +op inject -i argocd/manifests/tailscale-operator/secret.yaml.tpl | kubectl apply -f - + +PRIV_KEY=$(op read "op://vg6xf6vvfmoh5hqjjhlhbeoaie/csjncynh6htjvnh2l2da65y32q/private key?ssh-format=openssh")$'\n' && \ +kubectl create secret generic repo-forge -n argocd \ + --from-literal=type=git \ + --from-literal=url='ssh://forgejo@indri.tail8d86e.ts.net:2200/eblume/blumeops.git' \ + --from-literal=insecure=true \ + --from-literal=sshPrivateKey="$PRIV_KEY" && \ +kubectl label secret repo-forge -n argocd argocd.argoproj.io/secret-type=repository + +# 3. Bootstrap tailscale-operator +kubectl apply -k argocd/manifests/tailscale-operator/ + +# 4. Bootstrap ArgoCD +kubectl apply -k argocd/manifests/argocd/ + +# 5. Login and change password +argocd login argocd.tail8d86e.ts.net --username admin --grpc-web +argocd account update-password + +# 6. Apply ArgoCD Applications +kubectl apply -f argocd/apps/argocd.yaml +kubectl apply -f argocd/apps/apps.yaml + +# 7. Sync workloads +argocd app sync tailscale-operator +argocd app sync cloudnative-pg +```