diff --git a/.yamllint.yaml b/.yamllint.yaml index 81338de..15b4de5 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -27,3 +27,5 @@ rules: ignore: - .venv/ - pulumi/.venv/ + # Third-party k8s manifest with non-standard formatting + - argocd/manifests/tailscale-operator/operator.yaml diff --git a/Brewfile b/Brewfile index d4186a6..2f962c5 100644 --- a/Brewfile +++ b/Brewfile @@ -1,4 +1,5 @@ # CLI tools for blumeops management +brew "argocd" # ArgoCD CLI for GitOps management brew "bat" # Syntax-highlighted file concatenation brew "tea" # Gitea/Forgejo CLI for forge.tail8d86e.ts.net brew "podman" # Container CLI (uses VM on macOS, for building/pushing images) diff --git a/ansible/roles/minikube/files/zot-mirror.conf b/ansible/roles/minikube/files/zot-mirror.conf new file mode 100644 index 0000000..434e406 --- /dev/null +++ b/ansible/roles/minikube/files/zot-mirror.conf @@ -0,0 +1,27 @@ +# Zot pull-through cache on indri +# Uses host.containers.internal which is stable across restarts +# Applied by ansible minikube role + +[[registry]] +prefix = "docker.io" +location = "docker.io" + +[[registry.mirror]] +location = "host.containers.internal:5050/docker.io" +insecure = true + +[[registry]] +prefix = "ghcr.io" +location = "ghcr.io" + +[[registry.mirror]] +location = "host.containers.internal:5050/ghcr.io" +insecure = true + +[[registry]] +prefix = "quay.io" +location = "quay.io" + +[[registry.mirror]] +location = "host.containers.internal:5050/quay.io" +insecure = true diff --git a/ansible/roles/minikube/handlers/main.yml b/ansible/roles/minikube/handlers/main.yml index 7ba10c9..b473dbb 100644 --- a/ansible/roles/minikube/handlers/main.yml +++ b/ansible/roles/minikube/handlers/main.yml @@ -7,3 +7,8 @@ minikube stop 2>/dev/null || true minikube start changed_when: true + +- name: Restart CRI-O in minikube + ansible.builtin.command: + cmd: minikube ssh "sudo systemctl restart crio" + changed_when: true diff --git a/ansible/roles/minikube/tasks/main.yml b/ansible/roles/minikube/tasks/main.yml index db8c8bf..4416ec0 100644 --- a/ansible/roles/minikube/tasks/main.yml +++ b/ansible/roles/minikube/tasks/main.yml @@ -56,3 +56,42 @@ ansible.builtin.debug: msg: "WARNING: minikube may not have started properly. Run 'minikube start' manually on indri if needed. Status: {{ minikube_final_status.stdout | default('unknown') }}" when: minikube_final_status.rc != 0 or 'Running' not in minikube_final_status.stdout + +# Configure CRI-O to use zot as pull-through cache +- name: Copy zot mirror config to temp location + ansible.builtin.copy: + src: zot-mirror.conf + dest: /tmp/zot-mirror.conf + mode: "0644" + when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout + +- name: Check if zot mirror config exists in minikube + ansible.builtin.command: + cmd: minikube ssh "cat /etc/containers/registries.conf.d/zot-mirror.conf 2>/dev/null || echo ''" + register: minikube_existing_zot_config + changed_when: false + when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout + +- name: Read local zot mirror config + ansible.builtin.slurp: + src: /tmp/zot-mirror.conf + register: minikube_local_zot_config + when: minikube_final_status.rc == 0 and 'Running' in minikube_final_status.stdout + +- name: Apply zot mirror config to minikube + ansible.builtin.shell: + cmd: | + set -o pipefail + cat /tmp/zot-mirror.conf | minikube ssh "sudo tee /etc/containers/registries.conf.d/zot-mirror.conf > /dev/null" + executable: /bin/bash + changed_when: true + when: + - minikube_final_status.rc == 0 + - "'Running' in minikube_final_status.stdout" + - minikube_existing_zot_config.stdout != (minikube_local_zot_config.content | b64decode) + notify: Restart CRI-O in minikube + +- name: Clean up temp config file + ansible.builtin.file: + path: /tmp/zot-mirror.conf + state: absent diff --git a/argocd/apps/apps.yaml b/argocd/apps/apps.yaml new file mode 100644 index 0000000..32a0e4f --- /dev/null +++ b/argocd/apps/apps.yaml @@ -0,0 +1,24 @@ +# App-of-apps root Application +# Watches argocd/apps/ and creates/manages all Application resources +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: apps + namespace: argocd +spec: + project: default + source: + repoURL: ssh://forgejo@indri.tail8d86e.ts.net:2200/eblume/blumeops.git + targetRevision: feature/k8s-phase1-kickoff + path: argocd/apps + destination: + server: https://kubernetes.default.svc + namespace: argocd + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + # Auto-sync enabled: new/changed Application manifests appear automatically + # but child apps still require manual sync (they have manual sync policy) diff --git a/argocd/apps/argocd.yaml b/argocd/apps/argocd.yaml new file mode 100644 index 0000000..b737160 --- /dev/null +++ b/argocd/apps/argocd.yaml @@ -0,0 +1,20 @@ +# ArgoCD self-management Application +# After bootstrap, ArgoCD manages its own deployment +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: argocd + namespace: argocd +spec: + project: default + source: + repoURL: ssh://forgejo@indri.tail8d86e.ts.net:2200/eblume/blumeops.git + targetRevision: feature/k8s-phase1-kickoff + path: argocd/manifests/argocd + destination: + server: https://kubernetes.default.svc + namespace: argocd + syncPolicy: + syncOptions: + - CreateNamespace=true + # Manual sync only - no automated sync on git push diff --git a/argocd/apps/blumeops-pg.yaml b/argocd/apps/blumeops-pg.yaml new file mode 100644 index 0000000..9550ba4 --- /dev/null +++ b/argocd/apps/blumeops-pg.yaml @@ -0,0 +1,24 @@ +# PostgreSQL Cluster for blumeops services +# Requires: CloudNativePG operator (cloudnative-pg app) and manual secret setup +# +# Before syncing, create the eblume password secret: +# kubectl create namespace databases +# op inject -i argocd/manifests/databases/secret-eblume.yaml.tpl | kubectl apply -f - +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: blumeops-pg + namespace: argocd +spec: + project: default + source: + repoURL: ssh://forgejo@indri.tail8d86e.ts.net:2200/eblume/blumeops.git + targetRevision: feature/k8s-phase1-kickoff + path: argocd/manifests/databases + destination: + server: https://kubernetes.default.svc + namespace: databases + syncPolicy: + syncOptions: + - CreateNamespace=true + # Manual sync only - no automated sync on git push diff --git a/argocd/apps/cloudnative-pg.yaml b/argocd/apps/cloudnative-pg.yaml new file mode 100644 index 0000000..e128f4e --- /dev/null +++ b/argocd/apps/cloudnative-pg.yaml @@ -0,0 +1,30 @@ +# CloudNativePG Operator - PostgreSQL for Kubernetes +# Deploys the operator only; PostgreSQL clusters are created separately +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: cloudnative-pg + namespace: argocd +spec: + project: default + sources: + # Helm chart from upstream + - repoURL: https://cloudnative-pg.github.io/charts + chart: cloudnative-pg + targetRevision: "0.23.0" + helm: + releaseName: cloudnative-pg + valueFiles: + - $values/argocd/manifests/cloudnative-pg/values.yaml + # Values from our git repo + - repoURL: ssh://forgejo@indri.tail8d86e.ts.net:2200/eblume/blumeops.git + targetRevision: feature/k8s-phase1-kickoff + ref: values + destination: + server: https://kubernetes.default.svc + namespace: cnpg-system + syncPolicy: + syncOptions: + - CreateNamespace=true + - ServerSideApply=true # Required for large CRDs that exceed annotation size limit + # Manual sync only - no automated sync on git push diff --git a/argocd/apps/tailscale-operator.yaml b/argocd/apps/tailscale-operator.yaml new file mode 100644 index 0000000..5d9c6c4 --- /dev/null +++ b/argocd/apps/tailscale-operator.yaml @@ -0,0 +1,26 @@ +# ArgoCD Application for Tailscale Kubernetes Operator +# Note: OAuth secret is managed separately (not in git) +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: tailscale-operator + namespace: argocd +spec: + project: default + # Tailscale operator mutates externalName from "placeholder" to actual proxy service + ignoreDifferences: + - group: "" + kind: Service + jsonPointers: + - /spec/externalName + source: + repoURL: ssh://forgejo@indri.tail8d86e.ts.net:2200/eblume/blumeops.git + targetRevision: feature/k8s-phase1-kickoff + path: argocd/manifests/tailscale-operator + destination: + server: https://kubernetes.default.svc + namespace: tailscale + syncPolicy: + syncOptions: + - CreateNamespace=true + # Manual sync only - no automated sync on git push diff --git a/argocd/manifests/argocd/README.md b/argocd/manifests/argocd/README.md new file mode 100644 index 0000000..5e27c2e --- /dev/null +++ b/argocd/manifests/argocd/README.md @@ -0,0 +1,114 @@ +# ArgoCD + +GitOps continuous delivery for Kubernetes, with self-management via ArgoCD. + +## Prerequisites + +- Tailscale operator deployed (see `argocd/manifests/tailscale-operator/README.md`) +- Deploy key added to forge for SSH access to blumeops repo + +## Manual Bootstrap + +Bootstrap is required when setting up a new cluster. After bootstrap, ArgoCD manages itself. + +```bash +# 1. Create namespace +kubectl create namespace argocd + +# 2. Apply ArgoCD manifests via kustomize +kubectl apply -k argocd/manifests/argocd/ + +# 3. Wait for ArgoCD to be ready +kubectl wait --for=condition=available deployment/argocd-server -n argocd --timeout=300s + +# 4. Get initial admin password +kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d && echo + +# 5. Login and change password +argocd login argocd.tail8d86e.ts.net --username admin --grpc-web +argocd account update-password + +# 6. Apply repo-forge secret for SSH access to forge +PRIV_KEY=$(op read "op://vg6xf6vvfmoh5hqjjhlhbeoaie/csjncynh6htjvnh2l2da65y32q/private key?ssh-format=openssh")$'\n' && \ +kubectl create secret generic repo-forge -n argocd \ + --from-literal=type=git \ + --from-literal=url='ssh://forgejo@indri.tail8d86e.ts.net:2200/eblume/blumeops.git' \ + --from-literal=insecure=true \ + --from-literal=sshPrivateKey="$PRIV_KEY" && \ +kubectl label secret repo-forge -n argocd argocd.argoproj.io/secret-type=repository + +# 7. Apply ArgoCD Applications (self-management + app-of-apps) +kubectl apply -f argocd/apps/argocd.yaml +kubectl apply -f argocd/apps/apps.yaml +``` + +After step 7, ArgoCD manages itself and all applications defined in `argocd/apps/`. + +## Access + +- URL: https://argocd.tail8d86e.ts.net +- Username: `admin` +- Password: Stored in 1Password after initial setup + +## ArgoCD CLI Commands + +```bash +# Check all applications +argocd app list + +# Sync a specific application +argocd app sync + +# Check application status +argocd app get + +# Hard refresh (clear git cache) +argocd app get --hard-refresh +``` + +## Adding New Applications + +1. Create an Application manifest in `argocd/apps/.yaml` +2. Commit and push to forge +3. ArgoCD (via app-of-apps) automatically picks it up + +Example Application: +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: my-app + namespace: argocd +spec: + project: default + source: + repoURL: ssh://forgejo@indri.tail8d86e.ts.net:2200/eblume/blumeops.git + targetRevision: main + path: argocd/manifests/my-app + destination: + server: https://kubernetes.default.svc + namespace: my-app + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true +``` + +## Files + +| File | Description | +|------|-------------| +| `kustomization.yaml` | References upstream install.yaml + local customizations | +| `service-tailscale.yaml` | Tailscale Ingress for external access with Let's Encrypt TLS | +| `argocd-cmd-params-cm.yaml` | Patch to disable HTTPS redirect (TLS terminates at Ingress) | +| `repo-forge-secret.yaml.tpl` | Template documenting the forge SSH secret (manual) | +| `README.md` | This file | + +## Notes + +- **TODO:** Secrets (`repo-forge`) are not managed by ArgoCD and must be applied manually. + Future improvement: integrate with a secrets operator (e.g., External Secrets). +- ArgoCD uses Tailscale Ingress with Let's Encrypt for TLS termination. +- The `--grpc-web` flag is required for CLI access through the Tailscale ingress. diff --git a/argocd/manifests/argocd/argocd-cmd-params-cm.yaml b/argocd/manifests/argocd/argocd-cmd-params-cm.yaml new file mode 100644 index 0000000..af4c45b --- /dev/null +++ b/argocd/manifests/argocd/argocd-cmd-params-cm.yaml @@ -0,0 +1,8 @@ +# ArgoCD server parameters +# Disables HTTPS redirect since TLS is terminated at Tailscale Ingress +apiVersion: v1 +kind: ConfigMap +metadata: + name: argocd-cmd-params-cm +data: + server.insecure: "true" diff --git a/argocd/manifests/argocd/kustomization.yaml b/argocd/manifests/argocd/kustomization.yaml new file mode 100644 index 0000000..bcca27b --- /dev/null +++ b/argocd/manifests/argocd/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: argocd + +resources: + - https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml + - service-tailscale.yaml + +patchesStrategicMerge: + - argocd-cmd-params-cm.yaml diff --git a/argocd/manifests/argocd/repo-forge-secret.yaml.tpl b/argocd/manifests/argocd/repo-forge-secret.yaml.tpl new file mode 100644 index 0000000..f4a2d53 --- /dev/null +++ b/argocd/manifests/argocd/repo-forge-secret.yaml.tpl @@ -0,0 +1,27 @@ +# ArgoCD repository secret for forge SSH access +# +# IMPORTANT: Use ?ssh-format=openssh to get OpenSSH format (required by ArgoCD) +# +# Create the secret with: +# +# PRIV_KEY=$(op read "op://vg6xf6vvfmoh5hqjjhlhbeoaie/csjncynh6htjvnh2l2da65y32q/private key?ssh-format=openssh")$'\n' && \ +# kubectl create secret generic repo-forge -n argocd \ +# --from-literal=type=git \ +# --from-literal=url='ssh://forgejo@indri.tail8d86e.ts.net:2200/eblume/blumeops.git' \ +# --from-literal=insecure=true \ +# --from-literal=sshPrivateKey="$PRIV_KEY" && \ +# kubectl label secret repo-forge -n argocd argocd.argoproj.io/secret-type=repository +# +apiVersion: v1 +kind: Secret +metadata: + name: repo-forge + namespace: argocd + labels: + argocd.argoproj.io/secret-type: repository +stringData: + type: git + url: ssh://forgejo@indri.tail8d86e.ts.net:2200/eblume/blumeops.git + insecure: "true" + sshPrivateKey: | + # Key from 1Password: op://vg6xf6vvfmoh5hqjjhlhbeoaie/csjncynh6htjvnh2l2da65y32q/private key diff --git a/argocd/manifests/argocd/service-tailscale.yaml b/argocd/manifests/argocd/service-tailscale.yaml new file mode 100644 index 0000000..2c95cd6 --- /dev/null +++ b/argocd/manifests/argocd/service-tailscale.yaml @@ -0,0 +1,23 @@ +# Tailscale Ingress for ArgoCD +# Exposes ArgoCD at https://argocd.tail8d86e.ts.net with Let's Encrypt TLS +# +# Using Ingress instead of LoadBalancer to get automatic TLS certificates. +# See: https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: argocd-server-tailscale + namespace: argocd + annotations: + tailscale.com/proxy-class: "crio-compat" +spec: + ingressClassName: tailscale + defaultBackend: + service: + name: argocd-server + port: + number: 80 + tls: + - hosts: + - argocd diff --git a/argocd/manifests/cloudnative-pg/README.md b/argocd/manifests/cloudnative-pg/README.md new file mode 100644 index 0000000..c6c1fb1 --- /dev/null +++ b/argocd/manifests/cloudnative-pg/README.md @@ -0,0 +1,52 @@ +# CloudNativePG Operator + +Kubernetes operator for managing PostgreSQL clusters with high availability. + +## Source + +- Helm chart: `cloudnative-pg` from https://cloudnative-pg.github.io/charts +- Documentation: https://cloudnative-pg.io/documentation/ + +## Deployment + +Managed via ArgoCD Application using Helm source (not kustomize). +The Application points directly to the upstream Helm repository. + +## ArgoCD CLI Commands + +```bash +# Check application status +argocd app get cloudnative-pg + +# Trigger a sync +argocd app sync cloudnative-pg + +# View deployment history +argocd app history cloudnative-pg +``` + +## Verification + +```bash +# Check operator pod is running +kubectl get pods -n cnpg-system + +# Check operator logs +kubectl logs -n cnpg-system -l app.kubernetes.io/name=cloudnative-pg + +# Check CRDs are installed +kubectl get crd | grep cnpg +``` + +## Files + +| File | Description | +|------|-------------| +| `values.yaml` | Helm values for customization | +| `README.md` | This file | + +## Notes + +- The operator is deployed to `cnpg-system` namespace +- PostgreSQL clusters are created separately using the `Cluster` CRD (see Step 7) +- No secrets required for the operator itself diff --git a/argocd/manifests/cloudnative-pg/values.yaml b/argocd/manifests/cloudnative-pg/values.yaml new file mode 100644 index 0000000..607895e --- /dev/null +++ b/argocd/manifests/cloudnative-pg/values.yaml @@ -0,0 +1,4 @@ +# CloudNativePG Helm values +# See: https://github.com/cloudnative-pg/charts/tree/main/charts/cloudnative-pg + +# Using defaults for now - customize as needed diff --git a/argocd/manifests/databases/README.md b/argocd/manifests/databases/README.md new file mode 100644 index 0000000..7696217 --- /dev/null +++ b/argocd/manifests/databases/README.md @@ -0,0 +1,97 @@ +# Database Manifests + +PostgreSQL clusters managed by CloudNativePG operator. + +## blumeops-pg + +Single-instance PostgreSQL cluster for blumeops services. + +### Configuration + +- **Instances**: 1 (single-node for minikube) +- **Storage**: 10Gi on `standard` storage class +- **Initial database**: `miniflux` owned by `miniflux` user + +### Users/Roles + +| User | Role | Purpose | Password Source | +|----------|-------------|----------------------------------|------------------------------------| +| postgres | superuser | CNPG internal (avoid using) | `blumeops-pg-superuser` secret | +| miniflux | app owner | Owns miniflux database | `blumeops-pg-app` secret | +| eblume | superuser | Admin access (matches brew pg) | `blumeops-pg-eblume` secret (manual) | + +### Manual Secret Setup + +Before deploying, create the eblume password secret: + +```bash +# Create namespace first +kubectl create namespace databases + +# Apply eblume password from 1Password +op inject -i argocd/manifests/databases/secret-eblume.yaml.tpl | kubectl apply -f - +``` + +The `miniflux` user password is auto-generated by CloudNativePG and stored in `blumeops-pg-app`. + +### Connection Information + +After the cluster is healthy: + +```bash +# Connect via Tailscale (temporary hostname during migration) +psql -h k8s-pg.tail8d86e.ts.net -U eblume -W -d miniflux + +# Or with password from 1Password +PGPASSWORD=$(op --vault blumeops item get guxu3j7ajhjyey6xxl2ovsl2ui --fields password --reveal) \ + psql -h k8s-pg.tail8d86e.ts.net -U eblume -d miniflux + +# Get miniflux app credentials (for applications) +kubectl -n databases get secret blumeops-pg-app -o jsonpath='{.data.uri}' | base64 -d + +# Get postgres superuser credentials (emergency only) +kubectl -n databases get secret blumeops-pg-superuser -o jsonpath='{.data.password}' | base64 -d +``` + +### Connecting via kubectl port-forward + +Alternative if Tailscale service is unavailable: + +```bash +# Terminal 1: Port-forward to the primary +kubectl -n databases port-forward svc/blumeops-pg-rw 5432:5432 + +# Terminal 2: Connect as eblume +PGPASSWORD=$(op --vault blumeops item get guxu3j7ajhjyey6xxl2ovsl2ui --fields password --reveal) \ + psql -h localhost -U eblume -d miniflux +``` + +### Status + +```bash +# Check cluster health +kubectl -n databases get cluster blumeops-pg + +# Check pods +kubectl -n databases get pods -l cnpg.io/cluster=blumeops-pg + +# Check managed roles status +kubectl -n databases get cluster blumeops-pg -o jsonpath='{.status.managedRolesStatus}' | jq + +# Operator logs +kubectl -n databases logs -l cnpg.io/cluster=blumeops-pg +``` + +## Tailscale Exposure + +### Current: Temporary Service + +`k8s-pg.tail8d86e.ts.net` - LoadBalancer service for testing during migration. + +### Phase 4: Production Service + +After miniflux migrates to k8s, the `pg.tail8d86e.ts.net` Tailscale service will switch +from brew PostgreSQL (indri) to this k8s cluster. At that point: +1. Delete `service-tailscale.yaml` (the `k8s-pg` service) +2. Update/create a service with `tailscale.com/hostname: "pg"` +3. Verify the orphaned `k8s-pg` device is removed from tailnet diff --git a/argocd/manifests/databases/blumeops-pg.yaml b/argocd/manifests/databases/blumeops-pg.yaml new file mode 100644 index 0000000..93d30a2 --- /dev/null +++ b/argocd/manifests/databases/blumeops-pg.yaml @@ -0,0 +1,52 @@ +# PostgreSQL Cluster for blumeops services +# Managed by CloudNativePG operator +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: blumeops-pg + namespace: databases +spec: + instances: 1 + + storage: + size: 10Gi + storageClass: standard + + # Bootstrap creates initial database and owner + bootstrap: + initdb: + database: miniflux + owner: miniflux + + # Managed roles - additional users beyond the bootstrap owner + managed: + roles: + # eblume superuser for admin access (matches current brew pg setup) + - name: eblume + login: true + superuser: true + createdb: true + createrole: true + passwordSecret: + name: blumeops-pg-eblume + + # Resource limits for minikube environment + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "1Gi" + cpu: "500m" + + # PostgreSQL configuration + postgresql: + parameters: + max_connections: "50" + shared_buffers: "128MB" + password_encryption: "scram-sha-256" + pg_hba: + # Allow all users to connect from any IP with password auth + # Network security is handled by Tailscale + - host all all 0.0.0.0/0 scram-sha-256 + - host all all ::/0 scram-sha-256 diff --git a/argocd/manifests/databases/kustomization.yaml b/argocd/manifests/databases/kustomization.yaml new file mode 100644 index 0000000..a115143 --- /dev/null +++ b/argocd/manifests/databases/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: databases + +resources: + - blumeops-pg.yaml + - service-tailscale.yaml diff --git a/argocd/manifests/databases/secret-eblume.yaml.tpl b/argocd/manifests/databases/secret-eblume.yaml.tpl new file mode 100644 index 0000000..481bd96 --- /dev/null +++ b/argocd/manifests/databases/secret-eblume.yaml.tpl @@ -0,0 +1,13 @@ +# Template for eblume superuser password +# Apply with: op inject -i secret-eblume.yaml.tpl | kubectl apply -f - +# +# Uses the same 1Password item as the brew PostgreSQL setup on indri +apiVersion: v1 +kind: Secret +metadata: + name: blumeops-pg-eblume + namespace: databases +type: kubernetes.io/basic-auth +stringData: + username: eblume + password: {{ op://vg6xf6vvfmoh5hqjjhlhbeoaie/guxu3j7ajhjyey6xxl2ovsl2ui/password }} diff --git a/argocd/manifests/databases/service-tailscale.yaml b/argocd/manifests/databases/service-tailscale.yaml new file mode 100644 index 0000000..06d3e5d --- /dev/null +++ b/argocd/manifests/databases/service-tailscale.yaml @@ -0,0 +1,22 @@ +# Tailscale LoadBalancer for PostgreSQL access +# Temporary service for testing during migration (k8s-pg.tail8d86e.ts.net) +# Will be replaced by pg.tail8d86e.ts.net in Phase 4 +apiVersion: v1 +kind: Service +metadata: + name: blumeops-pg-tailscale + namespace: databases + annotations: + tailscale.com/hostname: "k8s-pg" + tailscale.com/proxy-class: "crio-compat" +spec: + type: LoadBalancer + loadBalancerClass: tailscale + selector: + cnpg.io/cluster: blumeops-pg + role: primary + ports: + - name: postgresql + port: 5432 + targetPort: 5432 + protocol: TCP diff --git a/argocd/manifests/tailscale-operator/README.md b/argocd/manifests/tailscale-operator/README.md new file mode 100644 index 0000000..6365ffe --- /dev/null +++ b/argocd/manifests/tailscale-operator/README.md @@ -0,0 +1,90 @@ +# Tailscale Kubernetes Operator + +Manifests for the Tailscale Kubernetes Operator, managed via ArgoCD. + +## Source + +- `operator.yaml` - Static manifest from https://github.com/tailscale/tailscale/tree/main/cmd/k8s-operator/deploy/manifests +- Secret block removed from `operator.yaml` - managed separately via `secret.yaml.tpl` +- Image reference changed to fully-qualified `docker.io/tailscale/k8s-operator:stable` for CRI-O compatibility + +## Prerequisites + +1. OAuth client in Tailscale admin console with: + - Devices: Core (Read & Write) - tag: `tag:k8s-operator` + - Auth Keys: Read & Write + - Services: Write +2. ACL with `tag:k8s-operator` owning `tag:k8s` (so operator can tag resources it creates) + +## Manual Bootstrap (Before ArgoCD) + +Tailscale operator must be deployed before ArgoCD since ArgoCD uses Tailscale for ingress. + +```bash +# 1. Create namespace +kubectl create namespace tailscale + +# 2. Apply OAuth secret (uses 1Password) +op inject -i argocd/manifests/tailscale-operator/secret.yaml.tpl | kubectl apply -f - + +# 3. Apply manifests via kustomize +kubectl apply -k argocd/manifests/tailscale-operator/ +``` + +## Ongoing Management (After ArgoCD) + +Once ArgoCD is running, the operator is managed by the `tailscale-operator` ArgoCD Application. +ArgoCD pulls manifests from forge and applies them automatically. + +## ArgoCD CLI Commands + +```bash +# Check application status +argocd app get tailscale-operator + +# Trigger a sync (pull latest from forge and apply) +argocd app sync tailscale-operator + +# Preview what would change without applying +argocd app diff tailscale-operator + +# View deployment history +argocd app history tailscale-operator + +# Hard refresh (clear cache and re-fetch from git) +argocd app get tailscale-operator --hard-refresh +``` + +## Verification + +```bash +# Check operator pod is running +kubectl get pods -n tailscale + +# Check operator logs +kubectl logs -n tailscale -l app.kubernetes.io/name=operator +``` + +## Files + +| File | Description | +|------|-------------| +| `kustomization.yaml` | Kustomize configuration for all manifests | +| `operator.yaml` | Operator deployment, CRDs, RBAC (secret removed) | +| `proxyclass.yaml` | ProxyClass with fully-qualified images for CRI-O | +| `dnsconfig.yaml` | DNSConfig for cluster-to-tailnet name resolution | +| `egress-forge.yaml` | Egress proxy for accessing forge on indri | +| `secret.yaml.tpl` | 1Password template for OAuth credentials (manual) | +| `README.md` | This file | + +## Notes + +- **TODO:** The OAuth secret (`operator-oauth`) is not managed by ArgoCD and must be applied + manually. Future improvement: integrate with a secrets operator (e.g., External Secrets). +- Services using the Tailscale LoadBalancer must reference the ProxyClass: + ```yaml + annotations: + tailscale.com/proxy-class: "crio-compat" + ``` +- The egress proxy for forge targets `indri.tail8d86e.ts.net` directly (not `forge.tail8d86e.ts.net`) + because Tailscale Serve hostnames are virtual and only work via the Tailscale client. diff --git a/argocd/manifests/tailscale-operator/dnsconfig.yaml b/argocd/manifests/tailscale-operator/dnsconfig.yaml new file mode 100644 index 0000000..867d3dd --- /dev/null +++ b/argocd/manifests/tailscale-operator/dnsconfig.yaml @@ -0,0 +1,16 @@ +# DNSConfig for resolving MagicDNS names from within the cluster +# Deploys a nameserver that resolves ts.net names to egress proxy IPs +# +# Requires CoreDNS/kube-dns configuration to forward ts.net queries. +# See: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress +--- +apiVersion: tailscale.com/v1alpha1 +kind: DNSConfig +metadata: + name: ts-dns + namespace: tailscale +spec: + nameserver: + image: + repo: docker.io/tailscale/k8s-nameserver + tag: stable diff --git a/argocd/manifests/tailscale-operator/egress-forge.yaml b/argocd/manifests/tailscale-operator/egress-forge.yaml new file mode 100644 index 0000000..fec1752 --- /dev/null +++ b/argocd/manifests/tailscale-operator/egress-forge.yaml @@ -0,0 +1,20 @@ +# Egress proxy to expose Forgejo (forge) to the cluster +# Forge runs on indri:3001, exposed via Tailscale Serve as forge.tail8d86e.ts.net +# We target indri directly since egress can't reach Tailscale Serve hostnames +# +# See: https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress +--- +apiVersion: v1 +kind: Service +metadata: + name: forge + namespace: tailscale + annotations: + tailscale.com/tailnet-fqdn: indri.tail8d86e.ts.net + tailscale.com/proxy-class: "crio-compat" +spec: + type: ExternalName + externalName: placeholder + ports: + - port: 3001 + targetPort: 3001 diff --git a/argocd/manifests/tailscale-operator/kustomization.yaml b/argocd/manifests/tailscale-operator/kustomization.yaml new file mode 100644 index 0000000..f0517ad --- /dev/null +++ b/argocd/manifests/tailscale-operator/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: tailscale + +resources: + - operator.yaml + - proxyclass.yaml + - dnsconfig.yaml + - egress-forge.yaml + +# Note: OAuth secret (operator-oauth) is NOT included here. +# It must be manually applied before deploying - see README.md diff --git a/argocd/manifests/tailscale-operator/operator.yaml b/argocd/manifests/tailscale-operator/operator.yaml new file mode 100644 index 0000000..1383956 --- /dev/null +++ b/argocd/manifests/tailscale-operator/operator.yaml @@ -0,0 +1,5386 @@ +# Copyright (c) Tailscale Inc & AUTHORS +# SPDX-License-Identifier: BSD-3-Clause + +apiVersion: v1 +kind: Namespace +metadata: + name: tailscale +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: operator + namespace: tailscale +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: proxies + namespace: tailscale +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.0 + name: connectors.tailscale.com +spec: + group: tailscale.com + names: + kind: Connector + listKind: ConnectorList + plural: connectors + shortNames: + - cn + singular: connector + scope: Cluster + versions: + - additionalPrinterColumns: + - description: CIDR ranges exposed to tailnet by a subnet router defined via this Connector instance. + jsonPath: .status.subnetRoutes + name: SubnetRoutes + type: string + - description: Whether this Connector instance defines an exit node. + jsonPath: .status.isExitNode + name: IsExitNode + type: string + - description: Whether this Connector instance is an app connector. + jsonPath: .status.isAppConnector + name: IsAppConnector + type: string + - description: Status of the deployed Connector resources. + jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + Connector defines a Tailscale node that will be deployed in the cluster. The + node can be configured to act as a Tailscale subnet router and/or a Tailscale + exit node. + Connector is a cluster-scoped resource. + More info: + https://tailscale.com/kb/1441/kubernetes-operator-connector + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + ConnectorSpec describes the desired Tailscale component. + More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + appConnector: + description: |- + AppConnector defines whether the Connector device should act as a Tailscale app connector. A Connector that is + configured as an app connector cannot be a subnet router or an exit node. If this field is unset, the + Connector does not act as an app connector. + Note that you will need to manually configure the permissions and the domains for the app connector via the + Admin panel. + Note also that the main tested and supported use case of this config option is to deploy an app connector on + Kubernetes to access SaaS applications available on the public internet. Using the app connector to expose + cluster workloads or other internal workloads to tailnet might work, but this is not a use case that we have + tested or optimised for. + If you are using the app connector to access SaaS applications because you need a predictable egress IP that + can be whitelisted, it is also your responsibility to ensure that cluster traffic from the connector flows + via that predictable IP, for example by enforcing that cluster egress traffic is routed via an egress NAT + device with a static IP address. + https://tailscale.com/kb/1281/app-connectors + properties: + routes: + description: |- + Routes are optional preconfigured routes for the domains routed via the app connector. + If not set, routes for the domains will be discovered dynamically. + If set, the app connector will immediately be able to route traffic using the preconfigured routes, but may + also dynamically discover other routes. + https://tailscale.com/kb/1332/apps-best-practices#preconfiguration + items: + format: cidr + type: string + minItems: 1 + type: array + type: object + exitNode: + description: |- + ExitNode defines whether the Connector device should act as a Tailscale exit node. Defaults to false. + This field is mutually exclusive with the appConnector field. + https://tailscale.com/kb/1103/exit-nodes + type: boolean + hostname: + description: |- + Hostname is the tailnet hostname that should be assigned to the + Connector node. If unset, hostname defaults to -connector. Hostname can contain lower case letters, numbers and + dashes, it must not start or end with a dash and must be between 2 + and 63 characters long. This field should only be used when creating a connector + with an unspecified number of replicas, or a single replica. + pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$ + type: string + hostnamePrefix: + description: |- + HostnamePrefix specifies the hostname prefix for each + replica. Each device will have the integer number + from its StatefulSet pod appended to this prefix to form the full hostname. + HostnamePrefix can contain lower case letters, numbers and dashes, it + must not start with a dash and must be between 1 and 62 characters long. + pattern: ^[a-z0-9][a-z0-9-]{0,61}$ + type: string + proxyClass: + description: |- + ProxyClass is the name of the ProxyClass custom resource that + contains configuration options that should be applied to the + resources created for this Connector. If unset, the operator will + create resources with the default configuration. + type: string + replicas: + description: |- + Replicas specifies how many devices to create. Set this to enable + high availability for app connectors, subnet routers, or exit nodes. + https://tailscale.com/kb/1115/high-availability. Defaults to 1. + format: int32 + minimum: 0 + type: integer + subnetRouter: + description: |- + SubnetRouter defines subnet routes that the Connector device should + expose to tailnet as a Tailscale subnet router. + https://tailscale.com/kb/1019/subnets/ + If this field is unset, the device does not get configured as a Tailscale subnet router. + This field is mutually exclusive with the appConnector field. + properties: + advertiseRoutes: + description: |- + AdvertiseRoutes refer to CIDRs that the subnet router should make + available. Route values must be strings that represent a valid IPv4 + or IPv6 CIDR range. Values can be Tailscale 4via6 subnet routes. + https://tailscale.com/kb/1201/4via6-subnets/ + items: + format: cidr + type: string + minItems: 1 + type: array + required: + - advertiseRoutes + type: object + tags: + description: |- + Tags that the Tailscale node will be tagged with. + Defaults to [tag:k8s]. + To autoapprove the subnet routes or exit node defined by a Connector, + you can configure Tailscale ACLs to give these tags the necessary + permissions. + See https://tailscale.com/kb/1337/acl-syntax#autoapprovers. + If you specify custom tags here, you must also make the operator an owner of these tags. + See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. + Tags cannot be changed once a Connector node has been created. + Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. + items: + pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ + type: string + type: array + type: object + x-kubernetes-validations: + - message: A Connector needs to have at least one of exit node, subnet router or app connector configured. + rule: has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector) + - message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields. + rule: '!((has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true)) && has(self.appConnector))' + - message: The hostname field cannot be specified when replicas is greater than 1. + rule: '!(has(self.hostname) && has(self.replicas) && self.replicas > 1)' + - message: The hostname and hostnamePrefix fields are mutually exclusive. + rule: '!(has(self.hostname) && has(self.hostnamePrefix))' + status: + description: |- + ConnectorStatus describes the status of the Connector. This is set + and managed by the Tailscale operator. + properties: + conditions: + description: |- + List of status conditions to indicate the status of the Connector. + Known condition types are `ConnectorReady`. + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + devices: + description: Devices contains information on each device managed by the Connector resource. + items: + properties: + hostname: + description: |- + Hostname is the fully qualified domain name of the Connector replica. + If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the + node. + type: string + tailnetIPs: + description: |- + TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6) + assigned to the Connector replica. + items: + type: string + type: array + type: object + type: array + hostname: + description: |- + Hostname is the fully qualified domain name of the Connector node. + If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the + node. When using multiple replicas, this field will be populated with the + first replica's hostname. Use the Hostnames field for the full list + of hostnames. + type: string + isAppConnector: + description: IsAppConnector is set to true if the Connector acts as an app connector. + type: boolean + isExitNode: + description: IsExitNode is set to true if the Connector acts as an exit node. + type: boolean + subnetRoutes: + description: |- + SubnetRoutes are the routes currently exposed to tailnet via this + Connector instance. + type: string + tailnetIPs: + description: |- + TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6) + assigned to the Connector node. + items: + type: string + type: array + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.0 + name: dnsconfigs.tailscale.com +spec: + group: tailscale.com + names: + kind: DNSConfig + listKind: DNSConfigList + plural: dnsconfigs + shortNames: + - dc + singular: dnsconfig + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Service IP address of the nameserver + jsonPath: .status.nameserver.ip + name: NameserverIP + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + DNSConfig can be deployed to cluster to make a subset of Tailscale MagicDNS + names resolvable by cluster workloads. Use this if: A) you need to refer to + tailnet services, exposed to cluster via Tailscale Kubernetes operator egress + proxies by the MagicDNS names of those tailnet services (usually because the + services run over HTTPS) + B) you have exposed a cluster workload to the tailnet using Tailscale Ingress + and you also want to refer to the workload from within the cluster over the + Ingress's MagicDNS name (usually because you have some callback component + that needs to use the same URL as that used by a non-cluster client on + tailnet). + When a DNSConfig is applied to a cluster, Tailscale Kubernetes operator will + deploy a nameserver for ts.net DNS names and automatically populate it with records + for any Tailscale egress or Ingress proxies deployed to that cluster. + Currently you must manually update your cluster DNS configuration to add the + IP address of the deployed nameserver as a ts.net stub nameserver. + Instructions for how to do it: + https://kubernetes.io/docs/tasks/administer-cluster/dns-custom-nameservers/#configuration-of-stub-domain-and-upstream-nameserver-using-coredns (for CoreDNS), + https://cloud.google.com/kubernetes-engine/docs/how-to/kube-dns (for kube-dns). + Tailscale Kubernetes operator will write the address of a Service fronting + the nameserver to dsnconfig.status.nameserver.ip. + DNSConfig is a singleton - you must not create more than one. + NB: if you want cluster workloads to be able to refer to Tailscale Ingress + using its MagicDNS name, you must also annotate the Ingress resource with + tailscale.com/experimental-forward-cluster-traffic-via-ingress annotation to + ensure that the proxy created for the Ingress listens on its Pod IP address. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + Spec describes the desired DNS configuration. + More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + nameserver: + description: |- + Configuration for a nameserver that can resolve ts.net DNS names + associated with in-cluster proxies for Tailscale egress Services and + Tailscale Ingresses. The operator will always deploy this nameserver + when a DNSConfig is applied. + properties: + image: + description: Nameserver image. Defaults to tailscale/k8s-nameserver:unstable. + properties: + repo: + description: Repo defaults to tailscale/k8s-nameserver. + type: string + tag: + description: Tag defaults to unstable. + type: string + type: object + pod: + description: Pod configuration. + properties: + tolerations: + description: If specified, applies tolerations to the pods deployed by the DNSConfig resource. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + replicas: + description: Replicas specifies how many Pods to create. Defaults to 1. + format: int32 + minimum: 0 + type: integer + service: + description: Service configuration. + properties: + clusterIP: + description: ClusterIP sets the static IP of the service used by the nameserver. + type: string + type: object + type: object + required: + - nameserver + type: object + status: + description: |- + Status describes the status of the DNSConfig. This is set + and managed by the Tailscale operator. + properties: + conditions: + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + nameserver: + description: Nameserver describes the status of nameserver cluster resources. + properties: + ip: + description: |- + IP is the ClusterIP of the Service fronting the deployed ts.net nameserver. + Currently, you must manually update your cluster DNS config to add + this address as a stub nameserver for ts.net for cluster workloads to be + able to resolve MagicDNS names associated with egress or Ingress + proxies. + The IP address will change if you delete and recreate the DNSConfig. + type: string + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.0 + name: proxyclasses.tailscale.com +spec: + group: tailscale.com + names: + kind: ProxyClass + listKind: ProxyClassList + plural: proxyclasses + singular: proxyclass + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Status of the ProxyClass. + jsonPath: .status.conditions[?(@.type == "ProxyClassReady")].reason + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + ProxyClass describes a set of configuration parameters that can be applied to + proxy resources created by the Tailscale Kubernetes operator. + To apply a given ProxyClass to resources created for a tailscale Ingress or + Service, use tailscale.com/proxy-class= label. To apply a + given ProxyClass to resources created for a Connector, use + connector.spec.proxyClass field. + ProxyClass is a cluster scoped resource. + More info: + https://tailscale.com/kb/1445/kubernetes-operator-customization#cluster-resource-customization-using-proxyclass-custom-resource + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + Specification of the desired state of the ProxyClass resource. + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + metrics: + description: |- + Configuration for proxy metrics. Metrics are currently not supported + for egress proxies and for Ingress proxies that have been configured + with tailscale.com/experimental-forward-cluster-traffic-via-ingress + annotation. Note that the metrics are currently considered unstable + and will likely change in breaking ways in the future - we only + recommend that you use those for debugging purposes. + properties: + enable: + description: |- + Setting enable to true will make the proxy serve Tailscale metrics + at :9002/metrics. + A metrics Service named -metrics will also be created in the operator's namespace and will + serve the metrics at :9002/metrics. + + In 1.78.x and 1.80.x, this field also serves as the default value for + .spec.statefulSet.pod.tailscaleContainer.debug.enable. From 1.82.0, both + fields will independently default to false. + + Defaults to false. + type: boolean + serviceMonitor: + description: |- + Enable to create a Prometheus ServiceMonitor for scraping the proxy's Tailscale metrics. + The ServiceMonitor will select the metrics Service that gets created when metrics are enabled. + The ingested metrics for each Service monitor will have labels to identify the proxy: + ts_proxy_type: ingress_service|ingress_resource|connector|proxygroup + ts_proxy_parent_name: name of the parent resource (i.e name of the Connector, Tailscale Ingress, Tailscale Service or ProxyGroup) + ts_proxy_parent_namespace: namespace of the parent resource (if the parent resource is not cluster scoped) + job: ts__[]_ + properties: + enable: + description: If Enable is set to true, a Prometheus ServiceMonitor will be created. Enable can only be set to true if metrics are enabled. + type: boolean + labels: + additionalProperties: + maxLength: 63 + pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$ + type: string + description: |- + Labels to add to the ServiceMonitor. + Labels must be valid Kubernetes labels. + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set + type: object + required: + - enable + type: object + required: + - enable + type: object + x-kubernetes-validations: + - message: ServiceMonitor can only be enabled if metrics are enabled + rule: '!(has(self.serviceMonitor) && self.serviceMonitor.enable && !self.enable)' + statefulSet: + description: |- + Configuration parameters for the proxy's StatefulSet. Tailscale + Kubernetes operator deploys a StatefulSet for each of the user + configured proxies (Tailscale Ingress, Tailscale Service, Connector). + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations that will be added to the StatefulSet created for the proxy. + Any Annotations specified here will be merged with the default annotations + applied to the StatefulSet by the Tailscale Kubernetes operator as + well as any other annotations that might have been applied by other + actors. + Annotations must be valid Kubernetes annotations. + https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + type: object + labels: + additionalProperties: + maxLength: 63 + pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$ + type: string + description: |- + Labels that will be added to the StatefulSet created for the proxy. + Any labels specified here will be merged with the default labels + applied to the StatefulSet by the Tailscale Kubernetes operator as + well as any other labels that might have been applied by other + actors. + Label keys and values must be valid Kubernetes label keys and values. + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set + type: object + pod: + description: Configuration for the proxy Pod. + properties: + affinity: + description: |- + Proxy Pod's affinity rules. + By default, the Tailscale Kubernetes operator does not apply any affinity rules. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#affinity + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + annotations: + additionalProperties: + type: string + description: |- + Annotations that will be added to the proxy Pod. + Any annotations specified here will be merged with the default + annotations applied to the Pod by the Tailscale Kubernetes operator. + Annotations must be valid Kubernetes annotations. + https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + type: object + dnsConfig: + description: |- + DNSConfig defines DNS parameters for the proxy Pod in addition to those generated from DNSPolicy. + When DNSPolicy is set to "None", DNSConfig must be specified. + https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-config + properties: + nameservers: + description: |- + A list of DNS name server IP addresses. + This will be appended to the base nameservers generated from DNSPolicy. + Duplicated nameservers will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + options: + description: |- + A list of DNS resolver options. + This will be merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options given in Options + will override those that appear in the base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver options of a pod. + properties: + name: + description: |- + Name is this DNS resolver option's name. + Required. + type: string + value: + description: Value is this DNS resolver option's value. + type: string + type: object + type: array + x-kubernetes-list-type: atomic + searches: + description: |- + A list of DNS search domains for host-name lookup. + This will be appended to the base search paths generated from DNSPolicy. + Duplicated search paths will be removed. + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + dnsPolicy: + description: |- + DNSPolicy defines how DNS will be configured for the proxy Pod. + By default the Tailscale Kubernetes Operator does not set a DNS policy (uses cluster default). + https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy + enum: + - ClusterFirstWithHostNet + - ClusterFirst + - Default + - None + type: string + imagePullSecrets: + description: |- + Proxy Pod's image pull Secrets. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + labels: + additionalProperties: + maxLength: 63 + pattern: ^(([a-zA-Z0-9][-._a-zA-Z0-9]*)?[a-zA-Z0-9])?$ + type: string + description: |- + Labels that will be added to the proxy Pod. + Any labels specified here will be merged with the default labels + applied to the Pod by the Tailscale Kubernetes operator. + Label keys and values must be valid Kubernetes label keys and values. + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set + type: object + nodeName: + description: |- + Proxy Pod's node name. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + Proxy Pod's node selector. + By default Tailscale Kubernetes operator does not apply any node + selector. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling + type: object + priorityClassName: + description: |- + PriorityClassName for the proxy Pod. + By default Tailscale Kubernetes operator does not apply any priority class. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling + type: string + securityContext: + description: |- + Proxy Pod's security context. + By default Tailscale Kubernetes operator does not apply any Pod + security context. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context-2 + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + tailscaleContainer: + description: Configuration for the proxy container running tailscale. + properties: + debug: + description: |- + Configuration for enabling extra debug information in the container. + Not recommended for production use. + properties: + enable: + description: |- + Enable tailscaled's HTTP pprof endpoints at :9001/debug/pprof/ + and internal debug metrics endpoint at :9001/debug/metrics, where + 9001 is a container port named "debug". The endpoints and their responses + may change in backwards incompatible ways in the future, and should not + be considered stable. + + In 1.78.x and 1.80.x, this setting will default to the value of + .spec.metrics.enable, and requests to the "metrics" port matching the + mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x, + this setting will default to false, and no requests will be proxied. + type: boolean + type: object + env: + description: |- + List of environment variables to set in the container. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#environment-variables + Note that environment variables provided here will take precedence + over Tailscale-specific environment variables set by the operator, + however running proxies with custom values for Tailscale environment + variables (i.e TS_USERSPACE) is not recommended and might break in + the future. + items: + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + pattern: ^[-._a-zA-Z][-._a-zA-Z0-9]*$ + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded using the previously defined + environment variables in the container and any service environment + variables. If a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single $, which + allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never + be expanded, regardless of whether the variable exists or not. Defaults + to "". + type: string + required: + - name + type: object + type: array + image: + description: |- + Container image name. By default images are pulled from docker.io/tailscale, + but the official images are also available at ghcr.io/tailscale. + + For all uses except on ProxyGroups of type "kube-apiserver", this image must + be either tailscale/tailscale, or an equivalent mirror of that image. + To apply to ProxyGroups of type "kube-apiserver", this image must be + tailscale/k8s-proxy or a mirror of that image. + + For "tailscale/tailscale"-based proxies, specifying image name here will + override any proxy image values specified via the Kubernetes operator's + Helm chart values or PROXY_IMAGE env var in the operator Deployment. + For "tailscale/k8s-proxy"-based proxies, there is currently no way to + configure your own default, and this field is the only way to use a + custom image. + + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image + type: string + imagePullPolicy: + description: |- + Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image + enum: + - Always + - Never + - IfNotPresent + type: string + resources: + description: |- + Container resource requirements. + By default Tailscale Kubernetes operator does not apply any resource + requirements. The amount of resources required wil depend on the + amount of resources the operator needs to parse, usage patterns and + cluster size. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + securityContext: + description: |- + Container security context. + Security context specified here will override the security context set by the operator. + By default the operator sets the Tailscale container and the Tailscale init container to privileged + for proxies created for Tailscale ingress and egress Service, Connector and ProxyGroup. + You can reduce the permissions of the Tailscale container to cap NET_ADMIN by + installing device plugin in your cluster and configuring the proxies tun device to be created + by the device plugin, see https://github.com/tailscale/tailscale/issues/10814#issuecomment-2479977752 + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + type: object + tailscaleInitContainer: + description: |- + Configuration for the proxy init container that enables forwarding. + Not valid to apply to ProxyGroups of type "kube-apiserver". + properties: + debug: + description: |- + Configuration for enabling extra debug information in the container. + Not recommended for production use. + properties: + enable: + description: |- + Enable tailscaled's HTTP pprof endpoints at :9001/debug/pprof/ + and internal debug metrics endpoint at :9001/debug/metrics, where + 9001 is a container port named "debug". The endpoints and their responses + may change in backwards incompatible ways in the future, and should not + be considered stable. + + In 1.78.x and 1.80.x, this setting will default to the value of + .spec.metrics.enable, and requests to the "metrics" port matching the + mux pattern /debug/ will be forwarded to the "debug" port. In 1.82.x, + this setting will default to false, and no requests will be proxied. + type: boolean + type: object + env: + description: |- + List of environment variables to set in the container. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#environment-variables + Note that environment variables provided here will take precedence + over Tailscale-specific environment variables set by the operator, + however running proxies with custom values for Tailscale environment + variables (i.e TS_USERSPACE) is not recommended and might break in + the future. + items: + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + pattern: ^[-._a-zA-Z][-._a-zA-Z0-9]*$ + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded using the previously defined + environment variables in the container and any service environment + variables. If a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single $, which + allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never + be expanded, regardless of whether the variable exists or not. Defaults + to "". + type: string + required: + - name + type: object + type: array + image: + description: |- + Container image name. By default images are pulled from docker.io/tailscale, + but the official images are also available at ghcr.io/tailscale. + + For all uses except on ProxyGroups of type "kube-apiserver", this image must + be either tailscale/tailscale, or an equivalent mirror of that image. + To apply to ProxyGroups of type "kube-apiserver", this image must be + tailscale/k8s-proxy or a mirror of that image. + + For "tailscale/tailscale"-based proxies, specifying image name here will + override any proxy image values specified via the Kubernetes operator's + Helm chart values or PROXY_IMAGE env var in the operator Deployment. + For "tailscale/k8s-proxy"-based proxies, there is currently no way to + configure your own default, and this field is the only way to use a + custom image. + + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image + type: string + imagePullPolicy: + description: |- + Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image + enum: + - Always + - Never + - IfNotPresent + type: string + resources: + description: |- + Container resource requirements. + By default Tailscale Kubernetes operator does not apply any resource + requirements. The amount of resources required wil depend on the + amount of resources the operator needs to parse, usage patterns and + cluster size. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + securityContext: + description: |- + Container security context. + Security context specified here will override the security context set by the operator. + By default the operator sets the Tailscale container and the Tailscale init container to privileged + for proxies created for Tailscale ingress and egress Service, Connector and ProxyGroup. + You can reduce the permissions of the Tailscale container to cap NET_ADMIN by + installing device plugin in your cluster and configuring the proxies tun device to be created + by the device plugin, see https://github.com/tailscale/tailscale/issues/10814#issuecomment-2479977752 + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + type: object + tolerations: + description: |- + Proxy Pod's tolerations. + By default Tailscale Kubernetes operator does not apply any + tolerations. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + description: |- + Proxy Pod's topology spread constraints. + By default Tailscale Kubernetes operator does not apply any topology spread constraints. + https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/ + items: + description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. + properties: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + description: |- + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. + + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + type: object + staticEndpoints: + description: |- + Configuration for 'static endpoints' on proxies in order to facilitate + direct connections from other devices on the tailnet. + See https://tailscale.com/kb/1445/kubernetes-operator-customization#static-endpoints. + properties: + nodePort: + description: The configuration for static endpoints using NodePort Services. + properties: + ports: + description: |- + The port ranges from which the operator will select NodePorts for the Services. + You must ensure that firewall rules allow UDP ingress traffic for these ports + to the node's external IPs. + The ports must be in the range of service node ports for the cluster (default `30000-32767`). + See https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport. + items: + properties: + endPort: + description: |- + endPort indicates that the range of ports from port to endPort if set, inclusive, + should be used. This field cannot be defined if the port field is not defined. + The endPort must be either unset, or equal or greater than port. + type: integer + port: + description: port represents a port selected to be used. This is a required field. + type: integer + required: + - port + type: object + minItems: 1 + type: array + selector: + additionalProperties: + type: string + description: |- + A selector which will be used to select the node's that will have their `ExternalIP`'s advertised + by the ProxyGroup as Static Endpoints. + type: object + required: + - ports + type: object + required: + - nodePort + type: object + tailscale: + description: |- + TailscaleConfig contains options to configure the tailscale-specific + parameters of proxies. + properties: + acceptRoutes: + description: |- + AcceptRoutes can be set to true to make the proxy instance accept + routes advertized by other nodes on the tailnet, such as subnet + routes. + This is equivalent of passing --accept-routes flag to a tailscale Linux client. + https://tailscale.com/kb/1019/subnets#use-your-subnet-routes-from-other-devices + Defaults to false. + type: boolean + type: object + useLetsEncryptStagingEnvironment: + description: |- + Set UseLetsEncryptStagingEnvironment to true to issue TLS + certificates for any HTTPS endpoints exposed to the tailnet from + LetsEncrypt's staging environment. + https://letsencrypt.org/docs/staging-environment/ + This setting only affects Tailscale Ingress resources. + By default Ingress TLS certificates are issued from LetsEncrypt's + production environment. + Changing this setting true -> false, will result in any + existing certs being re-issued from the production environment. + Changing this setting false (default) -> true, when certs have already + been provisioned from production environment will NOT result in certs + being re-issued from the staging environment before they need to be + renewed. + type: boolean + type: object + status: + description: |- + Status of the ProxyClass. This is set and managed automatically. + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + properties: + conditions: + description: |- + List of status conditions to indicate the status of the ProxyClass. + Known condition types are `ProxyClassReady`. + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.0 + name: proxygroups.tailscale.com +spec: + group: tailscale.com + names: + kind: ProxyGroup + listKind: ProxyGroupList + plural: proxygroups + shortNames: + - pg + singular: proxygroup + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Status of the deployed ProxyGroup resources. + jsonPath: .status.conditions[?(@.type == "ProxyGroupReady")].reason + name: Status + type: string + - description: URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if any. Only applies to ProxyGroups of type kube-apiserver. + jsonPath: .status.url + name: URL + type: string + - description: ProxyGroup type. + jsonPath: .spec.type + name: Type + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + ProxyGroup defines a set of Tailscale devices that will act as proxies. + Depending on spec.Type, it can be a group of egress, ingress, or kube-apiserver + proxies. In addition to running a highly available set of proxies, ingress + and egress ProxyGroups also allow for serving many annotated Services from a + single set of proxies to minimise resource consumption. + + For ingress and egress, use the tailscale.com/proxy-group annotation on a + Service to specify that the proxy should be implemented by a ProxyGroup + instead of a single dedicated proxy. + + More info: + * https://tailscale.com/kb/1438/kubernetes-operator-cluster-egress + * https://tailscale.com/kb/1439/kubernetes-operator-cluster-ingress + + For kube-apiserver, the ProxyGroup is a standalone resource. Use the + spec.kubeAPIServer field to configure options specific to the kube-apiserver + ProxyGroup type. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec describes the desired ProxyGroup instances. + properties: + hostnamePrefix: + description: |- + HostnamePrefix is the hostname prefix to use for tailnet devices created + by the ProxyGroup. Each device will have the integer number from its + StatefulSet pod appended to this prefix to form the full hostname. + HostnamePrefix can contain lower case letters, numbers and dashes, it + must not start with a dash and must be between 1 and 62 characters long. + pattern: ^[a-z0-9][a-z0-9-]{0,61}$ + type: string + kubeAPIServer: + description: |- + KubeAPIServer contains configuration specific to the kube-apiserver + ProxyGroup type. This field is only used when Type is set to "kube-apiserver". + properties: + hostname: + description: |- + Hostname is the hostname with which to expose the Kubernetes API server + proxies. Must be a valid DNS label no longer than 63 characters. If not + specified, the name of the ProxyGroup is used as the hostname. Must be + unique across the whole tailnet. + pattern: ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$ + type: string + mode: + description: |- + Mode to run the API server proxy in. Supported modes are auth and noauth. + In auth mode, requests from the tailnet proxied over to the Kubernetes + API server are additionally impersonated using the sender's tailnet identity. + If not specified, defaults to auth mode. + enum: + - auth + - noauth + type: string + type: object + proxyClass: + description: |- + ProxyClass is the name of the ProxyClass custom resource that contains + configuration options that should be applied to the resources created + for this ProxyGroup. If unset, and there is no default ProxyClass + configured, the operator will create resources with the default + configuration. + type: string + replicas: + description: |- + Replicas specifies how many replicas to create the StatefulSet with. + Defaults to 2. + format: int32 + minimum: 0 + type: integer + tags: + description: |- + Tags that the Tailscale devices will be tagged with. Defaults to [tag:k8s]. + If you specify custom tags here, make sure you also make the operator + an owner of these tags. + See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. + Tags cannot be changed once a ProxyGroup device has been created. + Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. + items: + pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ + type: string + type: array + type: + description: |- + Type of the ProxyGroup proxies. Supported types are egress, ingress, and kube-apiserver. + Type is immutable once a ProxyGroup is created. + enum: + - egress + - ingress + - kube-apiserver + type: string + x-kubernetes-validations: + - message: ProxyGroup type is immutable + rule: self == oldSelf + required: + - type + type: object + status: + description: |- + ProxyGroupStatus describes the status of the ProxyGroup resources. This is + set and managed by the Tailscale operator. + properties: + conditions: + description: |- + List of status conditions to indicate the status of the ProxyGroup + resources. Known condition types include `ProxyGroupReady` and + `ProxyGroupAvailable`. + + * `ProxyGroupReady` indicates all ProxyGroup resources are reconciled and + all expected conditions are true. + * `ProxyGroupAvailable` indicates that at least one proxy is ready to + serve traffic. + + For ProxyGroups of type kube-apiserver, there are two additional conditions: + + * `KubeAPIServerProxyConfigured` indicates that at least one API server + proxy is configured and ready to serve traffic. + * `KubeAPIServerProxyValid` indicates that spec.kubeAPIServer config is + valid. + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + devices: + description: List of tailnet devices associated with the ProxyGroup StatefulSet. + items: + properties: + hostname: + description: |- + Hostname is the fully qualified domain name of the device. + If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the + node. + type: string + staticEndpoints: + description: StaticEndpoints are user configured, 'static' endpoints by which tailnet peers can reach this device. + items: + type: string + type: array + tailnetIPs: + description: |- + TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6) + assigned to the device. + items: + type: string + type: array + required: + - hostname + type: object + type: array + x-kubernetes-list-map-keys: + - hostname + x-kubernetes-list-type: map + url: + description: |- + URL of the kube-apiserver proxy advertised by the ProxyGroup devices, if + any. Only applies to ProxyGroups of type kube-apiserver. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.0 + name: recorders.tailscale.com +spec: + group: tailscale.com + names: + kind: Recorder + listKind: RecorderList + plural: recorders + shortNames: + - rec + singular: recorder + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Status of the deployed Recorder resources. + jsonPath: .status.conditions[?(@.type == "RecorderReady")].reason + name: Status + type: string + - description: URL on which the UI is exposed if enabled. + jsonPath: .status.devices[?(@.url != "")].url + name: URL + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + Recorder defines a tsrecorder device for recording SSH sessions. By default, + it will store recordings in a local ephemeral volume. If you want to persist + recordings, you can configure an S3-compatible API for storage. + + More info: https://tailscale.com/kb/1484/kubernetes-operator-deploying-tsrecorder + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec describes the desired recorder instance. + properties: + enableUI: + description: |- + Set to true to enable the Recorder UI. The UI lists and plays recorded sessions. + The UI will be served at :443. Defaults to false. + Corresponds to --ui tsrecorder flag https://tailscale.com/kb/1246/tailscale-ssh-session-recording#deploy-a-recorder-node. + Required if S3 storage is not set up, to ensure that recordings are accessible. + type: boolean + replicas: + description: Replicas specifies how many instances of tsrecorder to run. Defaults to 1. + format: int32 + minimum: 0 + type: integer + statefulSet: + description: |- + Configuration parameters for the Recorder's StatefulSet. The operator + deploys a StatefulSet for each Recorder resource. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations that will be added to the StatefulSet created for the Recorder. + Any Annotations specified here will be merged with the default annotations + applied to the StatefulSet by the operator. + https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + type: object + labels: + additionalProperties: + type: string + description: |- + Labels that will be added to the StatefulSet created for the Recorder. + Any labels specified here will be merged with the default labels applied + to the StatefulSet by the operator. + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set + type: object + pod: + description: Configuration for pods created by the Recorder's StatefulSet. + properties: + affinity: + description: |- + Affinity rules for Recorder Pods. By default, the operator does not + apply any affinity rules. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#affinity + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + annotations: + additionalProperties: + type: string + description: |- + Annotations that will be added to Recorder Pods. Any annotations + specified here will be merged with the default annotations applied to + the Pod by the operator. + https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + type: object + container: + description: Configuration for the Recorder container running tailscale. + properties: + env: + description: |- + List of environment variables to set in the container. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#environment-variables + Note that environment variables provided here will take precedence + over Tailscale-specific environment variables set by the operator, + however running proxies with custom values for Tailscale environment + variables (i.e TS_USERSPACE) is not recommended and might break in + the future. + items: + properties: + name: + description: Name of the environment variable. Must be a C_IDENTIFIER. + pattern: ^[-._a-zA-Z][-._a-zA-Z0-9]*$ + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded using the previously defined + environment variables in the container and any service environment + variables. If a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single $, which + allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + produce the string literal "$(VAR_NAME)". Escaped references will never + be expanded, regardless of whether the variable exists or not. Defaults + to "". + type: string + required: + - name + type: object + type: array + image: + description: |- + Container image name including tag. Defaults to docker.io/tailscale/tsrecorder + with the same tag as the operator, but the official images are also + available at ghcr.io/tailscale/tsrecorder. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image + type: string + imagePullPolicy: + description: |- + Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#image + enum: + - Always + - Never + - IfNotPresent + type: string + resources: + description: |- + Container resource requirements. + By default, the operator does not apply any resource requirements. The + amount of resources required wil depend on the volume of recordings sent. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + securityContext: + description: |- + Container security context. By default, the operator does not apply any + container security context. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context + properties: + allowPrivilegeEscalation: + description: |- + AllowPrivilegeEscalation controls whether a process can gain more + privileges than its parent process. This bool directly controls if + the no_new_privs flag will be set on the container process. + AllowPrivilegeEscalation is true always when the container is: + 1) run as Privileged + 2) has CAP_SYS_ADMIN + Note that this field cannot be set when spec.os.name is windows. + type: boolean + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by this container. If set, this profile + overrides the pod's appArmorProfile. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + capabilities: + description: |- + The capabilities to add/drop when running containers. + Defaults to the default set of capabilities granted by the container runtime. + Note that this field cannot be set when spec.os.name is windows. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities type + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + description: |- + Run container in privileged mode. + Processes in privileged containers are essentially equivalent to root on the host. + Defaults to false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + procMount: + description: |- + procMount denotes the type of proc mount to use for the containers. + The default value is Default which uses the container runtime defaults for + readonly paths and masked paths. + This requires the ProcMountType feature flag to be enabled. + Note that this field cannot be set when spec.os.name is windows. + type: string + readOnlyRootFilesystem: + description: |- + Whether this container has a read-only root filesystem. + Default is false. + Note that this field cannot be set when spec.os.name is windows. + type: boolean + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxOptions: + description: |- + The SELinux context to be applied to the container. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by this container. If seccomp options are + provided at both the pod & container level, the container options + override the pod options. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options from the PodSecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + type: object + imagePullSecrets: + description: |- + Image pull Secrets for Recorder Pods. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#PodSpec + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + labels: + additionalProperties: + type: string + description: |- + Labels that will be added to Recorder Pods. Any labels specified here + will be merged with the default labels applied to the Pod by the operator. + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set + type: object + nodeSelector: + additionalProperties: + type: string + description: |- + Node selector rules for Recorder Pods. By default, the operator does + not apply any node selector rules. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling + type: object + securityContext: + description: |- + Security context for Recorder Pods. By default, the operator does not + apply any Pod security context. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#security-context-2 + properties: + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies to the container. + type: string + role: + description: Role is a SELinux role label that applies to the container. + type: string + type: + description: Type is a SELinux type label that applies to the container. + type: string + user: + description: User is a SELinux user label that applies to the container. + type: string + type: object + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object + type: object + serviceAccount: + description: |- + Config for the ServiceAccount to create for the Recorder's StatefulSet. + By default, the operator will create a ServiceAccount with the same + name as the Recorder resource. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#service-account + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations to add to the ServiceAccount. + https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set + + You can use this to add IAM roles to the ServiceAccount (IRSA) instead of + providing static S3 credentials in a Secret. + https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html + + For example: + eks.amazonaws.com/role-arn: arn:aws:iam:::role/ + type: object + name: + description: |- + Name of the ServiceAccount to create. Defaults to the name of the + Recorder resource. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#service-account + maxLength: 253 + pattern: ^[a-z0-9]([a-z0-9-.]{0,61}[a-z0-9])?$ + type: string + type: object + tolerations: + description: |- + Tolerations for Recorder Pods. By default, the operator does not apply + any tolerations. + https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#scheduling + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + type: object + storage: + description: |- + Configure where to store session recordings. By default, recordings will + be stored in a local ephemeral volume, and will not be persisted past the + lifetime of a specific pod. + properties: + s3: + description: |- + Configure an S3-compatible API for storage. Required if the UI is not + enabled, to ensure that recordings are accessible. + properties: + bucket: + description: |- + Bucket name to write to. The bucket is expected to be used solely for + recordings, as there is no stable prefix for written object names. + type: string + credentials: + description: |- + Configure environment variable credentials for managing objects in the + configured bucket. If not set, tsrecorder will try to acquire credentials + first from the file system and then the STS API. + properties: + secret: + description: |- + Use a Kubernetes Secret from the operator's namespace as the source of + credentials. + properties: + name: + description: |- + The name of a Kubernetes Secret in the operator's namespace that contains + credentials for writing to the configured bucket. Each key-value pair + from the secret's data will be mounted as an environment variable. It + should include keys for AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY if + using a static access key. + type: string + type: object + type: object + endpoint: + description: S3-compatible endpoint, e.g. s3.us-east-1.amazonaws.com. + type: string + type: object + type: object + tags: + description: |- + Tags that the Tailscale device will be tagged with. Defaults to [tag:k8s]. + If you specify custom tags here, make sure you also make the operator + an owner of these tags. + See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. + Tags cannot be changed once a Recorder node has been created. + Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. + items: + pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ + type: string + type: array + type: object + x-kubernetes-validations: + - message: S3 storage must be used when deploying multiple Recorder replicas + rule: '!(self.replicas > 1 && (!has(self.storage) || !has(self.storage.s3)))' + status: + description: |- + RecorderStatus describes the status of the recorder. This is set + and managed by the Tailscale operator. + properties: + conditions: + description: |- + List of status conditions to indicate the status of the Recorder. + Known condition types are `RecorderReady`. + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + devices: + description: List of tailnet devices associated with the Recorder StatefulSet. + items: + properties: + hostname: + description: |- + Hostname is the fully qualified domain name of the device. + If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the + node. + type: string + tailnetIPs: + description: |- + TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6) + assigned to the device. + items: + type: string + type: array + url: + description: |- + URL where the UI is available if enabled for replaying recordings. This + will be an HTTPS MagicDNS URL. You must be connected to the same tailnet + as the recorder to access it. + type: string + required: + - hostname + type: object + type: array + x-kubernetes-list-map-keys: + - hostname + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: tailscale-operator +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + - services + - services/status + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + - ingresses/status + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch + - apiGroups: + - tailscale.com + resources: + - connectors + - connectors/status + - proxyclasses + - proxyclasses/status + - proxygroups + - proxygroups/status + verbs: + - get + - list + - watch + - update + - apiGroups: + - tailscale.com + resources: + - dnsconfigs + - dnsconfigs/status + verbs: + - get + - list + - watch + - update + - apiGroups: + - tailscale.com + resources: + - recorders + - recorders/status + verbs: + - get + - list + - watch + - update + - apiGroups: + - apiextensions.k8s.io + resourceNames: + - servicemonitors.monitoring.coreos.com + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: tailscale-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: tailscale-operator +subjects: + - kind: ServiceAccount + name: operator + namespace: tailscale +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: operator + namespace: tailscale +rules: + - apiGroups: + - "" + resources: + - secrets + - serviceaccounts + - configmaps + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - pods/status + verbs: + - update + - apiGroups: + - apps + resources: + - statefulsets + - deployments + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch + - create + - update + - deletecollection + - apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + verbs: + - get + - create + - patch + - update + - list + - watch + - deletecollection + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - get + - list + - update + - create + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: proxies + namespace: tailscale +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: operator + namespace: tailscale +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: operator +subjects: + - kind: ServiceAccount + name: operator + namespace: tailscale +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: proxies + namespace: tailscale +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: proxies +subjects: + - kind: ServiceAccount + name: proxies + namespace: tailscale +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: operator + namespace: tailscale +spec: + replicas: 1 + selector: + matchLabels: + app: operator + strategy: + type: Recreate + template: + metadata: + labels: + app: operator + spec: + containers: + - env: + - name: OPERATOR_INITIAL_TAGS + value: tag:k8s-operator + - name: OPERATOR_HOSTNAME + value: tailscale-operator + - name: OPERATOR_SECRET + value: operator + - name: OPERATOR_LOGGING + value: info + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OPERATOR_LOGIN_SERVER + value: null + - name: OPERATOR_INGRESS_CLASS_NAME + value: tailscale + - name: CLIENT_ID_FILE + value: /oauth/client_id + - name: CLIENT_SECRET_FILE + value: /oauth/client_secret + - name: PROXY_IMAGE + value: tailscale/tailscale:stable + - name: PROXY_TAGS + value: tag:k8s + - name: APISERVER_PROXY + value: "false" + - name: PROXY_FIREWALL_MODE + value: auto + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + image: docker.io/tailscale/k8s-operator:stable + imagePullPolicy: Always + name: operator + volumeMounts: + - mountPath: /oauth + name: oauth + readOnly: true + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: operator + volumes: + - name: oauth + secret: + secretName: operator-oauth +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + annotations: {} + name: tailscale +spec: + controller: tailscale.com/ts-ingress diff --git a/argocd/manifests/tailscale-operator/proxyclass.yaml b/argocd/manifests/tailscale-operator/proxyclass.yaml new file mode 100644 index 0000000..bb22ad7 --- /dev/null +++ b/argocd/manifests/tailscale-operator/proxyclass.yaml @@ -0,0 +1,29 @@ +# ProxyClass: crio-compat +# +# Why this exists: +# CRI-O (the container runtime used by minikube) cannot resolve short image +# names like "tailscale/tailscale:stable". It requires fully-qualified names +# with an explicit registry prefix (e.g., "docker.io/tailscale/tailscale:stable"). +# +# The Tailscale operator creates proxy pods (StatefulSets) for each LoadBalancer +# Service or Ingress. By default, these pods use short image names which fail +# on CRI-O with "ImageInspectError". +# +# Usage: +# Add this annotation to any Tailscale Service or Ingress: +# tailscale.com/proxy-class: "crio-compat" +# +# This tells the operator to use the fully-qualified image names defined below +# when creating the proxy pod for that resource. +--- +apiVersion: tailscale.com/v1alpha1 +kind: ProxyClass +metadata: + name: crio-compat +spec: + statefulSet: + pod: + tailscaleContainer: + image: docker.io/tailscale/tailscale:stable + tailscaleInitContainer: + image: docker.io/tailscale/tailscale:stable diff --git a/argocd/manifests/tailscale-operator/secret.yaml.tpl b/argocd/manifests/tailscale-operator/secret.yaml.tpl new file mode 100644 index 0000000..700bfc0 --- /dev/null +++ b/argocd/manifests/tailscale-operator/secret.yaml.tpl @@ -0,0 +1,14 @@ +# Tailscale Operator OAuth Secret +# This template is processed by `op inject` to resolve 1Password references. +# +# Usage: +# op inject -i secret.yaml.tpl | kubectl apply -f - +# +apiVersion: v1 +kind: Secret +metadata: + name: operator-oauth + namespace: tailscale +stringData: + client_id: "{{ op://vg6xf6vvfmoh5hqjjhlhbeoaie/2it22lavwgbxdskoaxanej354q/client-id }}" + client_secret: "{{ op://vg6xf6vvfmoh5hqjjhlhbeoaie/2it22lavwgbxdskoaxanej354q/client-secret }}" diff --git a/mise-tasks/indri-services-check b/mise-tasks/indri-services-check index 07f9feb..d669304 100755 --- a/mise-tasks/indri-services-check +++ b/mise-tasks/indri-services-check @@ -83,6 +83,14 @@ check_service "minikube" "ssh indri 'minikube status --format={{.Host}} | grep - check_service "k8s-apiserver (indri)" "ssh indri 'kubectl get --raw /healthz'" check_service "k8s-apiserver (remote)" "kubectl --kubeconfig=$HOME/.kube/minikube-indri/config.yml --context=minikube-indri get --raw /healthz" +echo "" +echo "Kubernetes workloads (via Tailscale):" +check_http "ArgoCD" "https://argocd.tail8d86e.ts.net/healthz" +# k8s PostgreSQL - check TCP connection (no auth needed for pg_isready) +check_service "k8s-pg" "pg_isready -h k8s-pg.tail8d86e.ts.net -p 5432" +# ArgoCD apps sync status +check_service "ArgoCD apps synced" "kubectl --context=minikube-indri get applications -n argocd -o jsonpath='{.items[*].status.sync.status}' | grep -v OutOfSync" + echo "" if [ $FAILED -eq 0 ]; then echo -e "${GREEN}All services healthy!${NC}" diff --git a/plans/k8s-migration/00_overview.md b/plans/k8s-migration/00_overview.md new file mode 100644 index 0000000..514206b --- /dev/null +++ b/plans/k8s-migration/00_overview.md @@ -0,0 +1,149 @@ +# Blumeops Minikube Migration Plan + +This plan details a phased migration of blumeops services from direct hosting on indri (Mac Mini M1) to a minikube cluster, while maintaining critical infrastructure services outside of Kubernetes. + +## Phases + +| Phase | Name | Status | Description | +|-------|------|--------|-------------| +| 0 | [Foundation](P0_foundation.complete.md) | Complete | Container registry + minikube cluster | +| 1 | [K8s Infrastructure](P1_k8s_infrastructure.md) | In Progress | Tailscale operator, ArgoCD, CloudNativePG, PostgreSQL cluster | +| 2 | [Grafana](P2_grafana.md) | Pending | Migrate Grafana (pilot) via ArgoCD | +| 3 | [PostgreSQL](P3_postgresql.md) | Pending | Data migration to k8s PostgreSQL | +| 4 | [Miniflux](P4_miniflux.md) | Pending | Migrate Miniflux via ArgoCD | +| 5 | [devpi](P5_devpi.md) | Pending | Migrate devpi via ArgoCD | +| 6 | [Kiwix](P6_kiwix.md) | Pending | Migrate Kiwix via ArgoCD | +| 7 | [Forgejo](P7_forgejo.md) | Pending | Migrate Forgejo (highest risk) via ArgoCD | +| 8 | [Woodpecker](P8_woodpecker.md) | Pending | Deploy CI/CD via ArgoCD | +| 9 | [Cleanup](P9_cleanup.md) | Pending | Remove deprecated services | + +## Architecture Overview + +### Services Staying on Indri (Outside K8s) +| Service | Reason | +|---------|--------| +| **Zot Registry** (NEW) | Avoid circular dependency - k8s needs images to start | +| **Prometheus** | Observability backbone must survive k8s failures | +| **Loki** | Log aggregation backbone | +| **Borgmatic** | Backup system | +| **Grafana-alloy** | Metrics/logs collector on host | +| **Plex** | Until Jellyfin replacement | +| **Transmission** | Downloads for kiwix ZIM files | + +### Services Moving to K8s +| Service | Complexity | Dependencies | +|---------|------------|--------------| +| Grafana | LOW | Phase 1 | +| Kiwix | LOW | Phase 1 | +| Miniflux | MEDIUM | PostgreSQL | +| devpi | MEDIUM | Registry | +| PostgreSQL | HIGH | Phase 1 | +| Forgejo | HIGH | PostgreSQL | +| Woodpecker CI | MEDIUM | Forgejo | + +## Technical Decisions + +### Container Registry: Zot +- OCI-native, lightweight +- Native support for proxying multiple registries (Docker Hub, GHCR, Quay) +- Built from source at `~/code/3rd/zot` (not in homebrew) +- Binary: `~/code/3rd/zot/bin/zot-darwin-arm64` +- Config: `~/.config/zot/config.json` +- Data: `~/zot/` + +### Minikube Driver: Podman +- Rootless containers for better security +- Lighter than full VM (QEMU) +- Uses existing container ecosystem +- `minikube start --driver=podman --container-runtime=cri-o` + +### PostgreSQL: CloudNativePG Operator +- Production-grade operator +- Built-in backup/restore +- Prometheus metrics +- PITR support + +### K8s Service Exposure: Tailscale Operator +- `loadBalancerClass: tailscale` on Services +- Automatic TLS and MagicDNS names +- ACL-controlled access + +### LaunchAgent Requirements (Critical) +LaunchAgents do NOT get homebrew on PATH. All commands must use **absolute paths**: +- `/Users/erichblume/code/3rd/zot/bin/zot-darwin-arm64` for zot (built from source) +- `/opt/homebrew/opt/mise/bin/mise x --` for mise-managed tools +- `/opt/homebrew/opt/postgresql@18/bin/pg_dump` for postgres tools + +This applies to all mcquack LaunchAgents (zot, devpi, kiwix, borgmatic, metrics collectors). +`brew services` handles this automatically but those aren't tracked in ansible. + +### Backup Strategy + +Borgmatic remains on indri (outside k8s), writing to sifaka NAS via SMB at `/Volumes/backups`. This ensures backups continue even if k8s is down. + +| Service | Backup Approach | +|---------|-----------------| +| **Zot Registry** | No backup needed - pull-through cache is re-fetchable, private images rebuilt from source control | +| **Minikube** | No backup of cluster state - declarative manifests in git, can recreate | +| **PostgreSQL (k8s)** | CloudNativePG scheduled backups to sifaka (Phase 1) | +| **Grafana (k8s)** | Dashboards in ansible source control, no runtime backup needed | +| **Miniflux (k8s)** | Database backed up via CloudNativePG | +| **Forgejo (k8s)** | Git repos are distributed, config in ansible; data dir backed up via borgmatic before migration | +| **devpi (k8s)** | Private packages backed up, PyPI cache re-fetchable | +| **Kiwix (k8s)** | ZIM files re-downloadable via torrent, no backup needed | + +**Borgmatic config changes:** None required for Phase 0. Future phases may add k8s PV paths if needed. + +--- + +## Critical Files + +| File | Purpose | +|------|---------| +| `ansible/playbooks/indri.yml` | Main playbook - add k8s roles, remove migrated services | +| `ansible/roles/tailscale_serve/defaults/main.yml` | Transition services to Tailscale operator | +| `pulumi/policy.hujson` | Add tags: k8s, registry, ci | +| `ansible/roles/borgmatic/defaults/main.yml` | Update PostgreSQL endpoint | +| `mise-tasks/indri-services-check` | Add k8s health checks | + +## New Directory Structure + +``` +ansible/ + k8s/ + operators/ + tailscale-operator.yaml + cloudnative-pg.yaml + databases/ + blumeops-pg.yaml + apps/ + grafana/ + miniflux/ + forgejo/ + devpi/ + kiwix/ + woodpecker/ + roles/ + zot/ # NEW + podman/ # NEW + minikube/ # NEW +``` + +## Risk Mitigation + +- **Circular dependency prevention**: Zot registry runs outside k8s +- **Observability**: Prometheus/Loki stay on indri +- **Data loss prevention**: borgmatic + manual backups before each phase +- **Recovery**: Can manually push images, restore from backups + +## Container Images (All ARM64) + +| Service | Image | +|---------|-------| +| Miniflux | `ghcr.io/miniflux/miniflux:latest` | +| Forgejo | `codeberg.org/forgejo/forgejo:10` | +| Grafana | `grafana/grafana:latest` | +| Kiwix | `ghcr.io/kiwix/kiwix-serve:3.8.1` | +| Woodpecker | `woodpeckerci/woodpecker-server` | + +Note: Zot runs as a native binary on indri (built from source at `~/code/3rd/zot`), not as a container. diff --git a/plans/k8s-migration.md b/plans/k8s-migration/P0_foundation.complete.md similarity index 72% rename from plans/k8s-migration.md rename to plans/k8s-migration/P0_foundation.complete.md index 39a2583..934e83a 100644 --- a/plans/k8s-migration.md +++ b/plans/k8s-migration/P0_foundation.complete.md @@ -1,91 +1,12 @@ -# Blumeops Minikube Migration Plan - -This plan details a phased migration of blumeops services from direct hosting on indri (Mac Mini M1) to a minikube cluster, while maintaining critical infrastructure services outside of Kubernetes. - -## Architecture Overview - -### Services Staying on Indri (Outside K8s) -| Service | Reason | -|---------|--------| -| **Zot Registry** (NEW) | Avoid circular dependency - k8s needs images to start | -| **Prometheus** | Observability backbone must survive k8s failures | -| **Loki** | Log aggregation backbone | -| **Borgmatic** | Backup system | -| **Grafana-alloy** | Metrics/logs collector on host | -| **Plex** | Until Jellyfin replacement | -| **Transmission** | Downloads for kiwix ZIM files | - -### Services Moving to K8s -| Service | Complexity | Dependencies | -|---------|------------|--------------| -| Grafana | LOW | Phase 1 | -| Kiwix | LOW | Phase 1 | -| Miniflux | MEDIUM | PostgreSQL | -| devpi | MEDIUM | Registry | -| PostgreSQL | HIGH | Phase 1 | -| Forgejo | HIGH | PostgreSQL | -| Woodpecker CI | MEDIUM | Forgejo | - -## Technical Decisions - -### Container Registry: Zot -- OCI-native, lightweight -- Native support for proxying multiple registries (Docker Hub, GHCR, Quay) -- Built from source at `~/code/3rd/zot` (not in homebrew) -- Binary: `~/code/3rd/zot/bin/zot-darwin-arm64` -- Config: `~/.config/zot/config.json` -- Data: `~/zot/` - -### Minikube Driver: Podman -- Rootless containers for better security -- Lighter than full VM (QEMU) -- Uses existing container ecosystem -- `minikube start --driver=podman --container-runtime=cri-o` - -### PostgreSQL: CloudNativePG Operator -- Production-grade operator -- Built-in backup/restore -- Prometheus metrics -- PITR support - -### K8s Service Exposure: Tailscale Operator -- `loadBalancerClass: tailscale` on Services -- Automatic TLS and MagicDNS names -- ACL-controlled access - -### LaunchAgent Requirements (Critical) -LaunchAgents do NOT get homebrew on PATH. All commands must use **absolute paths**: -- `/Users/erichblume/code/3rd/zot/bin/zot-darwin-arm64` for zot (built from source) -- `/opt/homebrew/opt/mise/bin/mise x --` for mise-managed tools -- `/opt/homebrew/opt/postgresql@18/bin/pg_dump` for postgres tools - -This applies to all mcquack LaunchAgents (zot, devpi, kiwix, borgmatic, metrics collectors). -`brew services` handles this automatically but those aren't tracked in ansible. - -### Backup Strategy - -Borgmatic remains on indri (outside k8s), writing to sifaka NAS via SMB at `/Volumes/backups`. This ensures backups continue even if k8s is down. - -| Service | Backup Approach | -|---------|-----------------| -| **Zot Registry** | No backup needed - pull-through cache is re-fetchable, private images rebuilt from source control | -| **Minikube** | No backup of cluster state - declarative manifests in git, can recreate | -| **PostgreSQL (k8s)** | CloudNativePG scheduled backups to sifaka (Phase 1) | -| **Grafana (k8s)** | Dashboards in ansible source control, no runtime backup needed | -| **Miniflux (k8s)** | Database backed up via CloudNativePG | -| **Forgejo (k8s)** | Git repos are distributed, config in ansible; data dir backed up via borgmatic before migration | -| **devpi (k8s)** | Private packages backed up, PyPI cache re-fetchable | -| **Kiwix (k8s)** | ZIM files re-downloadable via torrent, no backup needed | - -**Borgmatic config changes:** None required for Phase 0. Future phases may add k8s PV paths if needed. - ---- - -## Phase 0: Foundation +# Phase 0: Foundation (Complete) **Goal**: Container registry + minikube cluster without disrupting existing services -### Important: Tailscale Service Creation Order +**Status**: Complete + +--- + +## Important: Tailscale Service Creation Order > **WARNING**: You MUST create services in the Tailscale admin console BEFORE running `tailscale serve` commands via ansible. If you run `tailscale serve --service svc:foo` before the service exists in the admin console, the local config will be in a bad state. > @@ -97,7 +18,7 @@ Borgmatic remains on indri (outside k8s), writing to sifaka NAS via SMB at `/Vol --- -### Step 0.1: Update Pulumi ACLs (BEFORE Tailscale serve) +## Step 0.1: Update Pulumi ACLs (BEFORE Tailscale serve) **Files to modify:** - `pulumi/policy.hujson` @@ -135,7 +56,7 @@ mise run tailnet-up # Apply changes --- -### Step 0.2: Create Tailscale Services in Admin Console (MANUAL) +## Step 0.2: Create Tailscale Services in Admin Console (MANUAL) > **CRITICAL**: Do this BEFORE running any ansible that calls `tailscale serve` @@ -155,7 +76,7 @@ tailscale status | grep registry --- -### Step 0.3: Create Zot Registry Ansible Role +## Step 0.3: Create Zot Registry Ansible Role **Note:** Zot is NOT in homebrew (no formula or tap). Clone to `~/code/3rd/` on indri and build from source (requires Go). @@ -330,7 +251,7 @@ ssh indri 'curl -s http://localhost:5000/v2/_catalog' --- -### Step 0.4: Add Zot to Tailscale Serve +## Step 0.4: Add Zot to Tailscale Serve **Files to modify:** - `ansible/roles/tailscale_serve/defaults/main.yml` @@ -368,7 +289,7 @@ curl -s https://registry.tail8d86e.ts.net/v2/_catalog --- -### Step 0.5: Create Zot Metrics Role +## Step 0.5: Create Zot Metrics Role **New files:** ``` @@ -418,7 +339,7 @@ curl -s "http://indri:9090/api/v1/query?query=zot_up" | jq '.data.result[0].valu --- -### Step 0.6: Add Zot Log Collection to Alloy +## Step 0.6: Add Zot Log Collection to Alloy **Files to modify:** - `ansible/roles/alloy/defaults/main.yml` @@ -445,7 +366,7 @@ mise run provision-indri -- --tags alloy --- -### Step 0.7: Update indri-services-check Script +## Step 0.7: Update indri-services-check Script **Files to modify:** - `mise-tasks/indri-services-check` @@ -480,7 +401,7 @@ mise run indri-services-check --- -### Step 0.8: Install and Configure Podman on Indri +## Step 0.8: Install and Configure Podman on Indri **New files:** ``` @@ -534,7 +455,7 @@ ssh indri 'podman run --rm hello-world' --- -### Step 0.9: Install and Configure Minikube +## Step 0.9: Install and Configure Minikube **New files:** ``` @@ -604,7 +525,7 @@ ssh indri 'kubectl get nodes' --- -### Step 0.10: Configure Kubeconfig on Gilbert +## Step 0.10: Configure Kubeconfig on Gilbert **Goal**: Enable `kubectl` and `k9s` on gilbert to connect to the minikube cluster running on indri. @@ -718,7 +639,7 @@ Rather than copying private keys between machines, credentials are stored in 1Pa --- -### Step 0.11: Add Minikube to indri-services-check +## Step 0.11: Add Minikube to indri-services-check **Files to modify:** - `mise-tasks/indri-services-check` @@ -748,7 +669,7 @@ mise run indri-services-check --- -### Step 0.12: Create Zettelkasten Documentation +## Step 0.12: Create Zettelkasten Documentation **New files:** - `~/code/personal/zk/zot.md` @@ -940,7 +861,7 @@ podman machine start --- -### Step 0.13: Update Main Playbook +## Step 0.13: Update Main Playbook **Files to modify:** - `ansible/playbooks/indri.yml` @@ -964,7 +885,232 @@ podman machine start --- -### Phase 0 Verification Checklist +## Step 0.14: Expose K8s API as Tailscale Service (Added Post-Completion) + +> **Note**: This step was added after Phase 0 was otherwise complete, to provide a stable, named endpoint for the Kubernetes API server. + +**Goal**: Expose the minikube API server as `k8s.tail8d86e.ts.net` instead of using `indri:`. + +**Current state:** +- Minikube API server on port 39535 (dynamic, could change on cluster recreation) +- Accessed via `https://indri:39535` +- Certificate SANs include "indri" + +**Target state:** +- Stable Tailscale service at `k8s.tail8d86e.ts.net:443` +- Fixed API server port (6443, the k8s standard) +- Certificate SANs include both hostnames for compatibility + +--- + +### Step 0.14.1: Update Pulumi ACLs + +**Files to modify:** +- `pulumi/policy.hujson` +- `pulumi/__main__.py` + +**Changes to policy.hujson:** + +1. Add tag to `tagOwners`: +```hujson +"tag:k8s-api": ["autogroup:admin", "tag:blumeops"], +``` + +2. Update Erich's test case accept list to include k8s-api: +```hujson +"accept": ["tag:grafana:443", "tag:kiwix:443", "tag:feed:443", "tag:loki:3100", "tag:pg:5432", "tag:homelab:22", "tag:registry:443", "tag:k8s-api:443"], +``` + +3. Update Allison's deny list: +```hujson +"deny": ["tag:grafana:443", "tag:loki:3100", "tag:nas:445", "tag:registry:443", "tag:k8s-api:443"], +``` + +**Changes to __main__.py:** +- Add `"tag:k8s-api"` to indri's DeviceTags + +**Testing:** +```bash +mise run tailnet-preview # Review changes +mise run tailnet-up # Apply changes +``` + +--- + +### Step 0.14.2: Create Tailscale Service in Admin Console (MANUAL) + +> **CRITICAL**: Do this BEFORE running ansible that calls `tailscale serve` + +1. Go to https://login.tailscale.com/admin/services +2. Create service `k8s` with: + - Port: 443 (TCP) + - Host: indri + +--- + +### Step 0.14.3: Recreate Minikube Cluster + +The cluster needs to be recreated to: +1. Add `k8s.tail8d86e.ts.net` to the API server certificate SANs +2. Fix the API server port to 6443 (standard k8s port) + +**On indri:** +```bash +# Stop and delete existing cluster +minikube stop +minikube delete + +# Recreate with new settings +minikube start \ + --driver=podman \ + --container-runtime=cri-o \ + --cpus=4 --memory=7800 --disk-size=200g \ + --apiserver-names=k8s.tail8d86e.ts.net,indri \ + --apiserver-port=6443 \ + --listen-address=0.0.0.0 + +# Verify certificate SANs include both names +kubectl config view --minify -o jsonpath="{.clusters[0].cluster.server}" +# Expected: https://127.0.0.1:6443 or similar + +# Verify cluster is running +minikube status +kubectl get nodes +``` + +**Update ansible role defaults** (`ansible/roles/minikube/defaults/main.yml`): +```yaml +minikube_apiserver_names: + - k8s.tail8d86e.ts.net + - indri +minikube_apiserver_port: 6443 +``` + +--- + +### Step 0.14.4: Add K8s Service to Tailscale Serve + +**Files to modify:** +- `ansible/roles/tailscale_serve/defaults/main.yml` + +**Add to services list:** +```yaml +- name: svc:k8s + tcp: + port: 443 + upstream: tcp://localhost:6443 +``` + +**Note:** Using TCP passthrough (not HTTPS termination) because k8s uses mTLS authentication. + +**Deploy:** +```bash +mise run provision-indri -- --tags tailscale-serve +``` + +--- + +### Step 0.14.5: Update 1Password Credentials + +After cluster recreation, the client certificates have changed. + +**On indri, get the new credentials:** +```bash +# Display new certificates (copy to 1Password) +cat ~/.minikube/profiles/minikube/client.crt +cat ~/.minikube/profiles/minikube/client.key +cat ~/.minikube/ca.crt +``` + +**In 1Password** (vault: `vg6xf6vvfmoh5hqjjhlhbeoaie`, item: `3jo4f2hnzvwfmamudfsbbbec7e`): +- Update `client-cert` field with new certificate +- Update `client-key` field with new key +- Update `ca-cert` field with new CA certificate + +--- + +### Step 0.14.6: Update Kubeconfig on Gilbert + +**Update CA certificate:** +```bash +# Fetch new CA cert from 1Password +op --vault vg6xf6vvfmoh5hqjjhlhbeoaie item get 3jo4f2hnzvwfmamudfsbbbec7e --fields ca-cert | sed 's/^"//; s/"$//' > ~/.kube/minikube-indri/ca.crt +``` + +**Update kubeconfig** (`~/.kube/minikube-indri/config.yml`): +```yaml +clusters: +- cluster: + certificate-authority: /Users/eblume/.kube/minikube-indri/ca.crt + server: https://k8s.tail8d86e.ts.net # Changed from https://indri:39535 + name: minikube-indri +``` + +**Verification:** +```bash +# Test connection via new hostname +kubectl --context=minikube-indri get nodes + +# Test via abbreviation +ki get nodes +``` + +--- + +### Step 0.14.7: Update Documentation + +**Files to update:** +- `~/code/personal/zk/minikube.md` - Update API server URL and port info +- `~/code/personal/zk/1767747119-YCPO.md` - Update Services table and Port Map + +**Changes to blumeops card:** + +1. Update Services table: + | **Kubernetes** | https://k8s.tail8d86e.ts.net | Minikube cluster | [[minikube]] | + +2. Update Port Map: + | 6443 | K8s API | HTTPS/TCP | 0.0.0.0 | Minikube API server (via Tailscale) | + +3. Add `tag:k8s-api` to Device Tags table + +--- + +### Step 0.14.8: Update indri-services-check + +**Files to modify:** +- `mise-tasks/indri-services-check` + +**Changes:** +```bash +# Update remote k8s check to use new URL +check_service "k8s-apiserver (remote)" "kubectl --kubeconfig=$HOME/.kube/minikube-indri/config.yml --context=minikube-indri get --raw /healthz" +# (No change needed - uses kubeconfig which now points to k8s.tail8d86e.ts.net) +``` + +--- + +### Step 0.14 Verification + +```bash +# 1. Service health check +mise run indri-services-check +# All services should be OK + +# 2. Test k8s access via Tailscale hostname +curl -k https://k8s.tail8d86e.ts.net/healthz +# Expected: ok (or certificate error if mTLS required - that's fine) + +# 3. kubectl via Tailscale +ki get nodes +ki get namespaces + +# 4. k9s via Tailscale +k9i +``` + +--- + +## Phase 0 Verification Checklist Run after completing all steps: @@ -1005,7 +1151,7 @@ k9s --- -### Phase 0 Rollback +## Phase 0 Rollback If something goes wrong: @@ -1040,232 +1186,7 @@ rm ~/code/personal/zk/{zot,minikube}.md --- -### Step 0.14: Expose K8s API as Tailscale Service (Added Post-Completion) - -> **Note**: This step was added after Phase 0 was otherwise complete, to provide a stable, named endpoint for the Kubernetes API server. - -**Goal**: Expose the minikube API server as `k8s.tail8d86e.ts.net` instead of using `indri:`. - -**Current state:** -- Minikube API server on port 39535 (dynamic, could change on cluster recreation) -- Accessed via `https://indri:39535` -- Certificate SANs include "indri" - -**Target state:** -- Stable Tailscale service at `k8s.tail8d86e.ts.net:443` -- Fixed API server port (6443, the k8s standard) -- Certificate SANs include both hostnames for compatibility - ---- - -#### Step 0.14.1: Update Pulumi ACLs - -**Files to modify:** -- `pulumi/policy.hujson` -- `pulumi/__main__.py` - -**Changes to policy.hujson:** - -1. Add tag to `tagOwners`: -```hujson -"tag:k8s-api": ["autogroup:admin", "tag:blumeops"], -``` - -2. Update Erich's test case accept list to include k8s-api: -```hujson -"accept": ["tag:grafana:443", "tag:kiwix:443", "tag:feed:443", "tag:loki:3100", "tag:pg:5432", "tag:homelab:22", "tag:registry:443", "tag:k8s-api:443"], -``` - -3. Update Allison's deny list: -```hujson -"deny": ["tag:grafana:443", "tag:loki:3100", "tag:nas:445", "tag:registry:443", "tag:k8s-api:443"], -``` - -**Changes to __main__.py:** -- Add `"tag:k8s-api"` to indri's DeviceTags - -**Testing:** -```bash -mise run tailnet-preview # Review changes -mise run tailnet-up # Apply changes -``` - ---- - -#### Step 0.14.2: Create Tailscale Service in Admin Console (MANUAL) - -> **CRITICAL**: Do this BEFORE running ansible that calls `tailscale serve` - -1. Go to https://login.tailscale.com/admin/services -2. Create service `k8s` with: - - Port: 443 (TCP) - - Host: indri - ---- - -#### Step 0.14.3: Recreate Minikube Cluster - -The cluster needs to be recreated to: -1. Add `k8s.tail8d86e.ts.net` to the API server certificate SANs -2. Fix the API server port to 6443 (standard k8s port) - -**On indri:** -```bash -# Stop and delete existing cluster -minikube stop -minikube delete - -# Recreate with new settings -minikube start \ - --driver=podman \ - --container-runtime=cri-o \ - --cpus=4 --memory=7800 --disk-size=200g \ - --apiserver-names=k8s.tail8d86e.ts.net,indri \ - --apiserver-port=6443 \ - --listen-address=0.0.0.0 - -# Verify certificate SANs include both names -kubectl config view --minify -o jsonpath="{.clusters[0].cluster.server}" -# Expected: https://127.0.0.1:6443 or similar - -# Verify cluster is running -minikube status -kubectl get nodes -``` - -**Update ansible role defaults** (`ansible/roles/minikube/defaults/main.yml`): -```yaml -minikube_apiserver_names: - - k8s.tail8d86e.ts.net - - indri -minikube_apiserver_port: 6443 -``` - ---- - -#### Step 0.14.4: Add K8s Service to Tailscale Serve - -**Files to modify:** -- `ansible/roles/tailscale_serve/defaults/main.yml` - -**Add to services list:** -```yaml -- name: svc:k8s - tcp: - port: 443 - upstream: tcp://localhost:6443 -``` - -**Note:** Using TCP passthrough (not HTTPS termination) because k8s uses mTLS authentication. - -**Deploy:** -```bash -mise run provision-indri -- --tags tailscale-serve -``` - ---- - -#### Step 0.14.5: Update 1Password Credentials - -After cluster recreation, the client certificates have changed. - -**On indri, get the new credentials:** -```bash -# Display new certificates (copy to 1Password) -cat ~/.minikube/profiles/minikube/client.crt -cat ~/.minikube/profiles/minikube/client.key -cat ~/.minikube/ca.crt -``` - -**In 1Password** (vault: `vg6xf6vvfmoh5hqjjhlhbeoaie`, item: `3jo4f2hnzvwfmamudfsbbbec7e`): -- Update `client-cert` field with new certificate -- Update `client-key` field with new key -- Update `ca-cert` field with new CA certificate - ---- - -#### Step 0.14.6: Update Kubeconfig on Gilbert - -**Update CA certificate:** -```bash -# Fetch new CA cert from 1Password -op --vault vg6xf6vvfmoh5hqjjhlhbeoaie item get 3jo4f2hnzvwfmamudfsbbbec7e --fields ca-cert | sed 's/^"//; s/"$//' > ~/.kube/minikube-indri/ca.crt -``` - -**Update kubeconfig** (`~/.kube/minikube-indri/config.yml`): -```yaml -clusters: -- cluster: - certificate-authority: /Users/eblume/.kube/minikube-indri/ca.crt - server: https://k8s.tail8d86e.ts.net # Changed from https://indri:39535 - name: minikube-indri -``` - -**Verification:** -```bash -# Test connection via new hostname -kubectl --context=minikube-indri get nodes - -# Test via abbreviation -ki get nodes -``` - ---- - -#### Step 0.14.7: Update Documentation - -**Files to update:** -- `~/code/personal/zk/minikube.md` - Update API server URL and port info -- `~/code/personal/zk/1767747119-YCPO.md` - Update Services table and Port Map - -**Changes to blumeops card:** - -1. Update Services table: - | **Kubernetes** | https://k8s.tail8d86e.ts.net | Minikube cluster | [[minikube]] | - -2. Update Port Map: - | 6443 | K8s API | HTTPS/TCP | 0.0.0.0 | Minikube API server (via Tailscale) | - -3. Add `tag:k8s-api` to Device Tags table - ---- - -#### Step 0.14.8: Update indri-services-check - -**Files to modify:** -- `mise-tasks/indri-services-check` - -**Changes:** -```bash -# Update remote k8s check to use new URL -check_service "k8s-apiserver (remote)" "kubectl --kubeconfig=$HOME/.kube/minikube-indri/config.yml --context=minikube-indri get --raw /healthz" -# (No change needed - uses kubeconfig which now points to k8s.tail8d86e.ts.net) -``` - ---- - -#### Step 0.14 Verification - -```bash -# 1. Service health check -mise run indri-services-check -# All services should be OK - -# 2. Test k8s access via Tailscale hostname -curl -k https://k8s.tail8d86e.ts.net/healthz -# Expected: ok (or certificate error if mTLS required - that's fine) - -# 3. kubectl via Tailscale -ki get nodes -ki get namespaces - -# 4. k9s via Tailscale -k9i -``` - ---- - -### Phase 0 Follow-up: Grafana Dashboards +## Phase 0 Follow-up: Grafana Dashboards After Phase 0 is running and stable, create monitoring dashboards: @@ -1282,7 +1203,7 @@ After Phase 0 is running and stable, create monitoring dashboards: --- -### New Files Summary +## New Files Summary | File | Purpose | |------|---------| @@ -1293,7 +1214,7 @@ After Phase 0 is running and stable, create monitoring dashboards: | `~/code/personal/zk/zot.md` | Zot management documentation | | `~/code/personal/zk/minikube.md` | Minikube management documentation | -### Modified Files Summary +## Modified Files Summary | File | Changes | |------|---------| @@ -1302,365 +1223,3 @@ After Phase 0 is running and stable, create monitoring dashboards: | `ansible/roles/tailscale_serve/defaults/main.yml` | Add svc:registry | | `ansible/roles/alloy/templates/config.alloy.j2` | Add zot log collection | | `mise-tasks/indri-services-check` | Add zot and k8s checks | - ---- - -## Phase 1: Kubernetes Infrastructure - -**Goal**: Tailscale operator + CloudNativePG operator - -### Steps - -1. **Update Pulumi ACLs for k8s workloads** - - Add `tag:k8s` to `pulumi/policy.hujson` - this tag is for k8s workloads that need to access other services (e.g., Woodpecker CI pushing to registry). - - **Changes to tagOwners:** - ```hujson - "tag:k8s": ["autogroup:admin", "tag:blumeops"], - ``` - - **Add grant for k8s→registry access:** - ```hujson - // k8s workloads (e.g., Woodpecker CI) can push/pull from registry - { - "src": ["tag:k8s"], - "dst": ["tag:registry"], - "ip": ["tcp:443"], - }, - ``` - - **Add test case:** - ```hujson - { - "src": "tag:k8s", - "accept": ["tag:registry:443"], - }, - ``` - - ```bash - mise run tailnet-preview && mise run tailnet-up - ``` - -2. **Create Tailscale OAuth client** - - Scopes: Devices Core, Auth Keys, Services write - - Tag: `tag:k8s-operator` - - Store in 1Password - -3. **Deploy Tailscale Kubernetes Operator** - ```bash - helm repo add tailscale https://pkgs.tailscale.com/helmcharts - helm install tailscale-operator tailscale/tailscale-operator \ - --namespace tailscale-system --create-namespace \ - --set oauth.clientId=$CLIENT_ID \ - --set oauth.clientSecret=$CLIENT_SECRET - ``` - -4. **Deploy CloudNativePG operator** - ```bash - kubectl apply -f https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.24/releases/cnpg-1.24.0.yaml - ``` - -5. **Create PostgreSQL cluster** - ```yaml - apiVersion: postgresql.cnpg.io/v1 - kind: Cluster - metadata: - name: blumeops-pg - namespace: databases - spec: - instances: 1 - storage: - size: 10Gi - storageClass: standard - monitoring: - enablePodMonitor: true - ``` - -6. **Update Alloy config** - - Add kubernetes_sd_configs for k8s metrics - - Scrape operator metrics - -### New Files -- `ansible/k8s/operators/` - Operator manifests -- `ansible/k8s/databases/` - PostgreSQL cluster - -### Verification -```bash -kubectl get pods -n tailscale-system -kubectl get pods -n cnpg-system -kubectl get cluster -n databases -``` - ---- - -## Phase 2: Grafana Migration (Pilot) - -**Goal**: Migrate Grafana as lowest-risk pilot service - -### Steps - -1. **Deploy Grafana via Helm** - - Copy datasource config from existing role - - Copy dashboards from `ansible/roles/grafana/files/dashboards/` - - Point to indri Prometheus/Loki (http://indri:9090, http://indri:3100) - -2. **Configure Tailscale LoadBalancer** - ```yaml - service: - type: LoadBalancer - loadBalancerClass: tailscale - ``` - -3. **Verify all dashboards work** - -4. **Update tailscale_serve** - remove grafana entry - -5. **Stop brew grafana**: `brew services stop grafana` - -### Verification -- https://grafana.tail8d86e.ts.net loads -- All dashboards functional - ---- - -## Phase 3: PostgreSQL Migration - -**Goal**: Migrate miniflux database to CloudNativePG - -### Steps - -1. **Create databases and users in k8s PostgreSQL** - - miniflux database/user - - borgmatic read-only user - -2. **Export from brew PostgreSQL** - ```bash - pg_dump -h localhost -U miniflux miniflux > miniflux_backup.sql - ``` - -3. **Expose k8s PostgreSQL via Tailscale** - - Service with `loadBalancerClass: tailscale` - - Tag: `svc:pg-k8s` - -4. **Import data** - ```bash - psql -h pg-k8s.tail8d86e.ts.net -U miniflux miniflux < miniflux_backup.sql - ``` - -5. **Update borgmatic config** - - Change hostname to k8s PostgreSQL - -6. **Verify data integrity** - -### Rollback -Keep brew PostgreSQL running until Phase 4 verified - ---- - -## Phase 4: Miniflux Migration - -**Goal**: Migrate Miniflux to k8s - -### Steps - -1. **Deploy Miniflux** - ```yaml - image: ghcr.io/miniflux/miniflux:latest - env: - DATABASE_URL: from secret - RUN_MIGRATIONS: "1" - ``` - -2. **Configure Tailscale LoadBalancer** - tag: `svc:feed` - -3. **Update Alloy log collection** - add k8s namespace - -4. **Verify**: login, feeds refresh, API works - -5. **Stop brew miniflux**: `brew services stop miniflux` - ---- - -## Phase 5: devpi Migration - -**Goal**: Migrate devpi to k8s - -### Steps - -1. **Build devpi container** - - Dockerfile with devpi-server + devpi-web - - Push to local Zot registry - -2. **Deploy as StatefulSet** - - PVC for data (50Gi) - - Migrate existing data (excluding PyPI cache) - -3. **Configure Tailscale LoadBalancer** - tag: `svc:pypi` - -4. **Update pip.conf on gilbert** - -5. **Stop mcquack devpi** - ---- - -## Phase 6: Kiwix Migration - -**Goal**: Migrate kiwix-serve to k8s - -### Steps - -1. **Create NFS/hostPath PV for ZIM files** - - Point to transmission download directory - - ReadOnlyMany access - -2. **Deploy Kiwix** - ```yaml - image: ghcr.io/kiwix/kiwix-serve:3.8.1 - args: ["/data/*.zim"] - ``` - -3. **Configure Tailscale LoadBalancer** - tag: `svc:kiwix` - -4. **Stop mcquack kiwix-serve** - ---- - -## Phase 7: Forgejo Migration (Highest Risk) - -**Goal**: Migrate Forgejo to k8s - -### Pre-Migration Checklist -- [ ] Full borgmatic backup verified -- [ ] Manual backup of `/opt/homebrew/var/forgejo` -- [ ] Document SSH keys and webhooks - -### Steps - -1. **Deploy Forgejo via Helm** - ```bash - helm install forgejo forgejo/forgejo \ - --namespace forgejo --create-namespace - ``` - -2. **Migrate data** - - Stop brew forgejo - - Copy data to PVC - - Start k8s forgejo - -3. **Configure Tailscale services** - - HTTPS 443 via LoadBalancer - - SSH port 22 (TCP proxy) - -4. **Verify all repositories accessible** - -### Rollback -Restore brew forgejo and tailscale serve config - ---- - -## Phase 8: CI/CD (Woodpecker) - -**Goal**: Deploy Woodpecker CI integrated with Forgejo - -### Steps - -1. **Create Forgejo OAuth application** - - Callback: https://ci.tail8d86e.ts.net/authorize - - Store in 1Password - -2. **Deploy Woodpecker Server + Agent** - -3. **Configure Tailscale LoadBalancer** - tag: `svc:ci` - -4. **Test pipeline** - create `.woodpecker.yaml` in test repo - ---- - -## Phase 9: Cleanup - -**Goal**: Remove deprecated services, harden system - -### Steps - -1. **Stop/remove unused brew services** - - postgresql@18, grafana, miniflux, forgejo - -2. **Update ansible playbook** - - Remove migrated service roles - - Add k8s deployment references - -3. **Configure Velero backups** (optional) - - Install with MinIO on sifaka - - Schedule daily cluster backups - -4. **Update zk documentation** - - New architecture - - Runbooks - - DR procedures - ---- - -## Critical Files - -| File | Purpose | -|------|---------| -| `ansible/playbooks/indri.yml` | Main playbook - add k8s roles, remove migrated services | -| `ansible/roles/tailscale_serve/defaults/main.yml` | Transition services to Tailscale operator | -| `pulumi/policy.hujson` | Add tags: k8s, registry, ci | -| `ansible/roles/borgmatic/defaults/main.yml` | Update PostgreSQL endpoint | -| `mise-tasks/indri-services-check` | Add k8s health checks | - -## New Directory Structure - -``` -ansible/ - k8s/ - operators/ - tailscale-operator.yaml - cloudnative-pg.yaml - databases/ - blumeops-pg.yaml - apps/ - grafana/ - miniflux/ - forgejo/ - devpi/ - kiwix/ - woodpecker/ - roles/ - zot/ # NEW - podman/ # NEW - minikube/ # NEW -``` - -## Risk Mitigation - -- **Circular dependency prevention**: Zot registry runs outside k8s -- **Observability**: Prometheus/Loki stay on indri -- **Data loss prevention**: borgmatic + manual backups before each phase -- **Recovery**: Can manually push images, restore from backups - -## Container Images (All ARM64) - -| Service | Image | -|---------|-------| -| Miniflux | `ghcr.io/miniflux/miniflux:latest` | -| Forgejo | `codeberg.org/forgejo/forgejo:10` | -| Grafana | `grafana/grafana:latest` | -| Kiwix | `ghcr.io/kiwix/kiwix-serve:3.8.1` | -| Woodpecker | `woodpeckerci/woodpecker-server` | - -Note: Zot runs as a native binary on indri (built from source at `~/code/3rd/zot`), not as a container. - ---- - -## Plan Completion - -When all phases are complete and verified: - -```bash -# Move plan to completed directory with completion date -git mv plans/k8s-migration.md plans/completed/k8s-migration.$(date +%Y-%m-%d).md -git commit -m "Complete k8s migration plan" -``` diff --git a/plans/k8s-migration/P1_k8s_infrastructure.md b/plans/k8s-migration/P1_k8s_infrastructure.md new file mode 100644 index 0000000..9e02286 --- /dev/null +++ b/plans/k8s-migration/P1_k8s_infrastructure.md @@ -0,0 +1,657 @@ +# Phase 1: Kubernetes Infrastructure + +**Goal**: Tailscale operator, ArgoCD, CloudNativePG operator, PostgreSQL cluster + +**Status**: In Progress + +**Prerequisites**: [Phase 0](P0_foundation.complete.md) complete + +--- + +## Overview + +Phase 1 establishes the k8s control plane infrastructure: +1. **Tailscale operator** - Exposes services on the tailnet +2. **ArgoCD** - GitOps continuous delivery +3. **CloudNativePG** - PostgreSQL operator +4. **PostgreSQL cluster** - Database for future app migrations + +The deployment follows a bootstrap pattern: +- First two components deployed via `kubectl apply -k` (no GitOps yet) +- ArgoCD then takes over management of all components including itself +- All subsequent deployments use ArgoCD + +--- + +## Kubernetes Tags Overview + +| Tag | Purpose | Applied To | +|-----|---------|------------| +| `tag:k8s-api` | Controls access to the K8s API server | indri (Phase 0.14) | +| `tag:k8s-operator` | Identifies the Tailscale K8s Operator | OAuth client for operator | +| `tag:k8s` | Default tag for operator-managed resources | Proxies, services, ingresses created by operator | + +**Ownership chain**: `tag:k8s-operator` must own `tag:k8s` so the operator can assign that tag to devices it creates. + +--- + +## PostgreSQL Migration Strategy + +The k8s PostgreSQL cluster will eventually replace the brew PostgreSQL on indri. + +| Phase | `pg.tail8d86e.ts.net` points to | Miniflux connects to | +|-------|--------------------------------|---------------------| +| Current | brew PostgreSQL (indri) | `pg.tail8d86e.ts.net` | +| Phase 1 | brew PostgreSQL (indri) | `pg.tail8d86e.ts.net` (no change) | +| Phase 4 | brew PostgreSQL (indri) | k8s PG (internal, after miniflux migrates to k8s) | +| Post-Phase 4 | k8s PostgreSQL | k8s PG (internal) | +| Cleanup | k8s PostgreSQL | k8s PG (internal) | + +This allows zero-downtime migration - the Tailscale service switches after apps are migrated. + +--- + +## Steps + +### 1. Update Pulumi ACLs for k8s workloads ✓ + +**Status**: Complete + +Added to `pulumi/policy.hujson`: +- `tag:k8s-operator` - for the operator OAuth client +- `tag:k8s` - for operator-managed resources (owned by `tag:k8s-operator`) +- Grant for `tag:k8s` → `tag:registry` access + +--- + +### 2. Create Tailscale OAuth client ✓ + +**Status**: Complete + +OAuth client stored in 1Password (vault: `vg6xf6vvfmoh5hqjjhlhbeoaie`, item: `2it22lavwgbxdskoaxanej354q`) + +**Configuration used:** +- Tags: `tag:k8s-operator` +- Devices write scope tag: `tag:k8s` +- Scopes: Devices Core (R/W), Auth Keys (R/W), Services (Write) + +--- + +### 3. Deploy Tailscale Kubernetes Operator (Bootstrap) + +Deploy via `kubectl apply -k` - will be migrated to ArgoCD management in Step 5. + +**Setup manifests directory:** +```bash +mkdir -p argocd/manifests/tailscale-operator +cd argocd/manifests/tailscale-operator + +# Download static manifest from Tailscale repo +curl -sL https://raw.githubusercontent.com/tailscale/tailscale/main/cmd/k8s-operator/deploy/manifests/operator.yaml -o operator.yaml + +# Download CRDs +curl -sL https://raw.githubusercontent.com/tailscale/tailscale/main/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml -o crds/connectors.yaml +curl -sL https://raw.githubusercontent.com/tailscale/tailscale/main/cmd/k8s-operator/deploy/crds/tailscale.com_proxyclasses.yaml -o crds/proxyclasses.yaml +# ... (other CRDs as needed) +``` + +**Create kustomization.yaml:** +```yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: tailscale-system +resources: + - operator.yaml +secretGenerator: + - name: operator-oauth + namespace: tailscale-system + literals: + - client_id=PLACEHOLDER + - client_secret=PLACEHOLDER +generatorOptions: + disableNameSuffixHash: true +``` + +**Deploy:** +```bash +# Get credentials from 1Password and create secret manually (kustomize secretGenerator is for reference) +CLIENT_ID=$(op --vault vg6xf6vvfmoh5hqjjhlhbeoaie item get 2it22lavwgbxdskoaxanej354q --fields client-id --reveal) +CLIENT_SECRET=$(op --vault vg6xf6vvfmoh5hqjjhlhbeoaie item get 2it22lavwgbxdskoaxanej354q --fields client-secret --reveal) + +kubectl create namespace tailscale-system +kubectl create secret generic operator-oauth \ + --namespace tailscale-system \ + --from-literal=client_id=$CLIENT_ID \ + --from-literal=client_secret=$CLIENT_SECRET + +# Apply operator manifests +kubectl apply -k argocd/manifests/tailscale-operator/ +``` + +**Verification:** +```bash +kubectl get pods -n tailscale-system +# Expected: operator pod Running + +kubectl logs -n tailscale-system -l app.kubernetes.io/name=tailscale-operator +``` + +--- + +### 4. Deploy ArgoCD + +Deploy ArgoCD and expose via Tailscale as `argocd.tail8d86e.ts.net`. + +**Prerequisites:** +- Add `tag:argocd` to Pulumi ACLs +- Create Tailscale service `argocd` in admin console + +**Setup manifests:** +```bash +mkdir -p argocd/manifests/argocd + +# Download ArgoCD install manifest +curl -sL https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml -o argocd/manifests/argocd/install.yaml +``` + +**Create kustomization.yaml:** +```yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: argocd +resources: + - install.yaml + - service-tailscale.yaml # LoadBalancer for Tailscale exposure +``` + +**Create service-tailscale.yaml:** +```yaml +apiVersion: v1 +kind: Service +metadata: + name: argocd-server-tailscale + namespace: argocd + annotations: + tailscale.com/hostname: "argocd" +spec: + type: LoadBalancer + loadBalancerClass: tailscale + selector: + app.kubernetes.io/name: argocd-server + ports: + - name: https + port: 443 + targetPort: 8080 +``` + +**Deploy:** +```bash +kubectl create namespace argocd +kubectl apply -k argocd/manifests/argocd/ +``` + +**Get initial admin password:** +```bash +kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d +``` + +**Verification:** +- https://argocd.tail8d86e.ts.net loads +- Can login with admin / + +**Post-setup:** +1. Change admin password, store in 1Password +2. Configure git repo connection to `github.com/eblume/blumeops` (public, no auth needed) + - Note: Using GitHub mirror since ArgoCD can't easily reach forge without additional networking + +--- + +### 5. Migrate Tailscale Operator to ArgoCD + +Create ArgoCD Application to manage the Tailscale operator. + +**Create argocd/apps/tailscale-operator.yaml:** +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: tailscale-operator + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/eblume/blumeops.git + targetRevision: main + path: argocd/manifests/tailscale-operator + destination: + server: https://kubernetes.default.svc + namespace: tailscale-system + syncPolicy: + automated: + prune: true + selfHeal: true +``` + +**Apply:** +```bash +kubectl apply -f argocd/apps/tailscale-operator.yaml +``` + +**Note on secrets:** The OAuth secret was created manually in Step 3. For GitOps, consider: +- Sealed Secrets +- External Secrets Operator +- SOPS + +For now, the secret remains manually managed outside of ArgoCD. + +--- + +### 6. Deploy CloudNativePG via ArgoCD + +**Setup manifests:** +```bash +mkdir -p argocd/manifests/cloudnative-pg + +# Download CNPG operator manifest +curl -sL https://raw.githubusercontent.com/cloudnative-pg/cloudnative-pg/release-1.24/releases/cnpg-1.24.0.yaml -o argocd/manifests/cloudnative-pg/operator.yaml +``` + +**Create kustomization.yaml:** +```yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - operator.yaml +``` + +**Create ArgoCD Application (argocd/apps/cloudnative-pg.yaml):** +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: cloudnative-pg + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/eblume/blumeops.git + targetRevision: main + path: argocd/manifests/cloudnative-pg + destination: + server: https://kubernetes.default.svc + namespace: cnpg-system + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true +``` + +**Apply:** +```bash +kubectl apply -f argocd/apps/cloudnative-pg.yaml +``` + +**Verification:** +```bash +kubectl get pods -n cnpg-system +# Expected: cnpg-controller-manager Running +``` + +--- + +### 7. Create PostgreSQL Cluster via ArgoCD + +Create the database cluster. **Not exposed via Tailscale yet** - internal only until apps migrate. + +**Create argocd/manifests/databases/blumeops-pg.yaml:** +```yaml +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: blumeops-pg + namespace: databases +spec: + instances: 1 + storage: + size: 10Gi + storageClass: standard + monitoring: + enablePodMonitor: true + bootstrap: + initdb: + database: miniflux + owner: miniflux +``` + +**Create kustomization.yaml:** +```yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: databases +resources: + - blumeops-pg.yaml +``` + +**Create ArgoCD Application (argocd/apps/blumeops-pg.yaml):** +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: blumeops-pg + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/eblume/blumeops.git + targetRevision: main + path: argocd/manifests/databases + destination: + server: https://kubernetes.default.svc + namespace: databases + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true +``` + +**Apply:** +```bash +kubectl apply -f argocd/apps/blumeops-pg.yaml +``` + +**Verification:** +```bash +kubectl get cluster -n databases +# Expected: blumeops-pg with STATUS "Cluster in healthy state" + +kubectl get pods -n databases +# Expected: blumeops-pg-1 Running + +# Get connection secret +kubectl -n databases get secret blumeops-pg-app -o jsonpath='{.data.uri}' | base64 -d +``` + +--- + +### 8. Create App-of-Apps Root Application + +Once all components are deployed, create a root application to manage all apps. + +**Create argocd/apps/root.yaml:** +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: root + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/eblume/blumeops.git + targetRevision: main + path: argocd/apps + destination: + server: https://kubernetes.default.svc + namespace: argocd + syncPolicy: + automated: + prune: true + selfHeal: true +``` + +**Apply:** +```bash +kubectl apply -f argocd/apps/root.yaml +``` + +Now ArgoCD manages itself and all other applications via the app-of-apps pattern. + +--- + +## New Files Summary + +``` +argocd/ + apps/ + root.yaml # App-of-apps root + tailscale-operator.yaml # Tailscale operator app + cloudnative-pg.yaml # CNPG operator app + blumeops-pg.yaml # PostgreSQL cluster app + manifests/ + tailscale-operator/ + kustomization.yaml + operator.yaml + argocd/ + kustomization.yaml + install.yaml + service-tailscale.yaml + cloudnative-pg/ + kustomization.yaml + operator.yaml + databases/ + kustomization.yaml + blumeops-pg.yaml +``` + +--- + +## Pulumi ACL Updates Required + +Add to `pulumi/policy.hujson`: +```hujson +"tag:argocd": ["autogroup:admin", "tag:blumeops"], +``` + +Add to Erich's test accept list: +```hujson +"accept": [..., "tag:argocd:443"], +``` + +Add to Allison's deny list: +```hujson +"deny": [..., "tag:argocd:443"], +``` + +--- + +## Verification Checklist + +```bash +# 1. Tailscale operator running +kubectl get pods -n tailscale-system + +# 2. ArgoCD accessible +curl -k https://argocd.tail8d86e.ts.net/healthz + +# 3. CloudNativePG operator running +kubectl get pods -n cnpg-system + +# 4. PostgreSQL cluster healthy +kubectl get cluster -n databases + +# 5. All ArgoCD apps synced +kubectl get applications -n argocd +# All should show STATUS: Synced, HEALTH: Healthy +``` + +--- + +## Rollback + +```bash +# Remove ArgoCD apps (will cascade delete managed resources) +kubectl delete application -n argocd root +kubectl delete application -n argocd blumeops-pg +kubectl delete application -n argocd cloudnative-pg +kubectl delete application -n argocd tailscale-operator + +# Remove ArgoCD +kubectl delete -k argocd/manifests/argocd/ +kubectl delete namespace argocd + +# Remove namespaces +kubectl delete namespace databases +kubectl delete namespace cnpg-system +kubectl delete namespace tailscale-system + +# Revert ACL changes +git checkout pulumi/policy.hujson +mise run tailnet-up +``` + +--- + +## Implementation Notes (Deviations from Plan) + +*Added during implementation for retrospective review* + +### Git Source: Forge Instead of GitHub + +**Plan**: Use GitHub mirror (`github.com/eblume/blumeops`) +**Actual**: Use internal Forgejo (`ssh://forgejo@indri.tail8d86e.ts.net:2200/eblume/blumeops.git`) + +**Why**: User preference to use internal infrastructure, accepting circular dependency for later. + +**Required changes**: +- Deploy key added to forge for ArgoCD SSH access +- Repository secret `repo-forge` with SSH private key from 1Password +- Discovered: `op read` requires `?ssh-format=openssh` query parameter for ArgoCD-compatible key format +- Egress proxy service to reach forge from cluster (targets `indri.tail8d86e.ts.net` not `forge.tail8d86e.ts.net` due to Tailscale Serve limitation) +- DNSConfig CRD for cluster-to-tailnet MagicDNS resolution +- ACL grant: `tag:k8s` → `tag:homelab` on ports 3001 (HTTP) and 2200 (SSH) + +### ArgoCD Exposure: Ingress Instead of LoadBalancer + +**Plan**: LoadBalancer service with `tailscale.com/hostname` annotation +**Actual**: Tailscale Ingress with Let's Encrypt TLS termination + +**Why**: Ingress provides automatic TLS certificates and is the recommended approach. + +**File**: `argocd/manifests/argocd/service-tailscale.yaml` uses `kind: Ingress` with `ingressClassName: tailscale` + +### Namespace: `tailscale` Instead of `tailscale-system` + +**Plan**: `tailscale-system` namespace +**Actual**: `tailscale` namespace + +**Why**: Matches upstream Tailscale operator defaults. + +### Sync Policy: Manual Instead of Automated + +**Plan**: `syncPolicy.automated` with prune and selfHeal +**Actual**: Manual sync policy for workload apps; auto-sync only for app-of-apps + +**Why**: User preference for explicit control over deployments during initial migration phase. + +**Pattern**: +- `apps.yaml` (app-of-apps): auto-sync to pick up new Application manifests +- All workload apps: manual sync requires `argocd app sync ` + +### CloudNativePG: Helm Chart Instead of Raw Manifest + +**Plan**: Download raw CNPG manifest +**Actual**: Multi-source Application using official Helm chart from `https://cloudnative-pg.github.io/charts` + +**Why**: Helm chart is the officially supported distribution method. + +**Additional fix**: Required `ServerSideApply=true` sync option due to large CRD exceeding annotation size limit. + +### App-of-Apps: Named `apps` Instead of `root` + +**Plan**: `argocd/apps/root.yaml` +**Actual**: `argocd/apps/apps.yaml` with Application named `apps` + +**Why**: Clearer naming; `apps` manages apps, `argocd` manages itself. + +### ArgoCD Self-Management Added + +**Plan**: Not explicitly planned +**Actual**: `argocd/apps/argocd.yaml` Application for ArgoCD self-management + +**Why**: Standard GitOps pattern - ArgoCD manages its own deployment after bootstrap. + +### CRI-O Registry Mirror for Zot + +**Plan**: Not in original plan +**Actual**: Configured CRI-O to use zot as pull-through cache for docker.io, ghcr.io, quay.io + +**Why**: Reduces external bandwidth, speeds up pulls, avoids rate limits. + +**Implementation**: Ansible `minikube` role applies `/etc/containers/registries.conf.d/zot-mirror.conf` inside minikube VM using stable hostname `host.containers.internal:5050`. + +### ProxyClass for CRI-O Image Compatibility + +**Plan**: Not mentioned +**Actual**: Required `ProxyClass` with fully-qualified image paths (`docker.io/tailscale/...`) + +**Why**: CRI-O requires fully-qualified image references; default Tailscale operator uses short names. + +### Actual File Structure + +``` +argocd/ + apps/ + apps.yaml # App-of-apps (auto-sync) + argocd.yaml # ArgoCD self-management (manual sync) + tailscale-operator.yaml # Tailscale operator (manual sync) + cloudnative-pg.yaml # CNPG operator via Helm (manual sync) + manifests/ + tailscale-operator/ + kustomization.yaml + operator.yaml + proxyclass.yaml # CRI-O compatibility + dnsconfig.yaml # Cluster-to-tailnet DNS + egress-forge.yaml # Egress proxy for forge + secret.yaml.tpl # OAuth secret template (manual) + README.md + argocd/ + kustomization.yaml # Uses remote base from upstream + service-tailscale.yaml # Ingress (not LoadBalancer) + argocd-cmd-params-cm.yaml # Disable HTTPS redirect + repo-forge-secret.yaml.tpl # SSH key template (manual) + README.md + cloudnative-pg/ + values.yaml # Helm values (currently minimal) + README.md +``` + +### Bootstrap Commands (Actual) + +```bash +# 1. Create namespaces +kubectl create namespace tailscale +kubectl create namespace argocd + +# 2. Apply secrets (manual, uses 1Password) +op inject -i argocd/manifests/tailscale-operator/secret.yaml.tpl | kubectl apply -f - + +PRIV_KEY=$(op read "op://vg6xf6vvfmoh5hqjjhlhbeoaie/csjncynh6htjvnh2l2da65y32q/private key?ssh-format=openssh")$'\n' && \ +kubectl create secret generic repo-forge -n argocd \ + --from-literal=type=git \ + --from-literal=url='ssh://forgejo@indri.tail8d86e.ts.net:2200/eblume/blumeops.git' \ + --from-literal=insecure=true \ + --from-literal=sshPrivateKey="$PRIV_KEY" && \ +kubectl label secret repo-forge -n argocd argocd.argoproj.io/secret-type=repository + +# 3. Bootstrap tailscale-operator +kubectl apply -k argocd/manifests/tailscale-operator/ + +# 4. Bootstrap ArgoCD +kubectl apply -k argocd/manifests/argocd/ + +# 5. Login and change password +argocd login argocd.tail8d86e.ts.net --username admin --grpc-web +argocd account update-password + +# 6. Apply ArgoCD Applications +kubectl apply -f argocd/apps/argocd.yaml +kubectl apply -f argocd/apps/apps.yaml + +# 7. Sync workloads +argocd app sync tailscale-operator +argocd app sync cloudnative-pg +``` diff --git a/plans/k8s-migration/P2_grafana.md b/plans/k8s-migration/P2_grafana.md new file mode 100644 index 0000000..58bcb2b --- /dev/null +++ b/plans/k8s-migration/P2_grafana.md @@ -0,0 +1,52 @@ +# Phase 2: Grafana Migration (Pilot) + +**Goal**: Migrate Grafana as lowest-risk pilot service + +**Status**: Pending + +**Prerequisites**: [Phase 1](P1_k8s_infrastructure.md) complete + +--- + +## Steps + +### 1. Deploy Grafana via Helm + +- Copy datasource config from existing role +- Copy dashboards from `ansible/roles/grafana/files/dashboards/` +- Point to indri Prometheus/Loki (http://indri:9090, http://indri:3100) + +--- + +### 2. Configure Tailscale LoadBalancer + +```yaml +service: + type: LoadBalancer + loadBalancerClass: tailscale +``` + +--- + +### 3. Verify all dashboards work + +--- + +### 4. Update tailscale_serve + +Remove grafana entry from `ansible/roles/tailscale_serve/defaults/main.yml` + +--- + +### 5. Stop brew grafana + +```bash +brew services stop grafana +``` + +--- + +## Verification + +- https://grafana.tail8d86e.ts.net loads +- All dashboards functional diff --git a/plans/k8s-migration/P3_postgresql.md b/plans/k8s-migration/P3_postgresql.md new file mode 100644 index 0000000..4e14f54 --- /dev/null +++ b/plans/k8s-migration/P3_postgresql.md @@ -0,0 +1,55 @@ +# Phase 3: PostgreSQL Migration + +**Goal**: Migrate miniflux database to CloudNativePG + +**Status**: Pending + +**Prerequisites**: [Phase 2](P2_grafana.md) complete + +--- + +## Steps + +### 1. Create databases and users in k8s PostgreSQL + +- miniflux database/user +- borgmatic read-only user + +--- + +### 2. Export from brew PostgreSQL + +```bash +pg_dump -h localhost -U miniflux miniflux > miniflux_backup.sql +``` + +--- + +### 3. Expose k8s PostgreSQL via Tailscale + +- Service with `loadBalancerClass: tailscale` +- Tag: `svc:pg-k8s` + +--- + +### 4. Import data + +```bash +psql -h pg-k8s.tail8d86e.ts.net -U miniflux miniflux < miniflux_backup.sql +``` + +--- + +### 5. Update borgmatic config + +- Change hostname to k8s PostgreSQL + +--- + +### 6. Verify data integrity + +--- + +## Rollback + +Keep brew PostgreSQL running until Phase 4 verified diff --git a/plans/k8s-migration/P4_miniflux.md b/plans/k8s-migration/P4_miniflux.md new file mode 100644 index 0000000..c4a31a0 --- /dev/null +++ b/plans/k8s-migration/P4_miniflux.md @@ -0,0 +1,48 @@ +# Phase 4: Miniflux Migration + +**Goal**: Migrate Miniflux to k8s + +**Status**: Pending + +**Prerequisites**: [Phase 3](P3_postgresql.md) complete + +--- + +## Steps + +### 1. Deploy Miniflux + +```yaml +image: ghcr.io/miniflux/miniflux:latest +env: + DATABASE_URL: from secret + RUN_MIGRATIONS: "1" +``` + +--- + +### 2. Configure Tailscale LoadBalancer + +Tag: `svc:feed` + +--- + +### 3. Update Alloy log collection + +Add k8s namespace + +--- + +### 4. Verify + +- Login works +- Feeds refresh +- API works + +--- + +### 5. Stop brew miniflux + +```bash +brew services stop miniflux +``` diff --git a/plans/k8s-migration/P5_devpi.md b/plans/k8s-migration/P5_devpi.md new file mode 100644 index 0000000..7f24b7b --- /dev/null +++ b/plans/k8s-migration/P5_devpi.md @@ -0,0 +1,37 @@ +# Phase 5: devpi Migration + +**Goal**: Migrate devpi to k8s + +**Status**: Pending + +**Prerequisites**: [Phase 4](P4_miniflux.md) complete + +--- + +## Steps + +### 1. Build devpi container + +- Dockerfile with devpi-server + devpi-web +- Push to local Zot registry + +--- + +### 2. Deploy as StatefulSet + +- PVC for data (50Gi) +- Migrate existing data (excluding PyPI cache) + +--- + +### 3. Configure Tailscale LoadBalancer + +Tag: `svc:pypi` + +--- + +### 4. Update pip.conf on gilbert + +--- + +### 5. Stop mcquack devpi diff --git a/plans/k8s-migration/P6_kiwix.md b/plans/k8s-migration/P6_kiwix.md new file mode 100644 index 0000000..378a33e --- /dev/null +++ b/plans/k8s-migration/P6_kiwix.md @@ -0,0 +1,35 @@ +# Phase 6: Kiwix Migration + +**Goal**: Migrate kiwix-serve to k8s + +**Status**: Pending + +**Prerequisites**: [Phase 5](P5_devpi.md) complete + +--- + +## Steps + +### 1. Create NFS/hostPath PV for ZIM files + +- Point to transmission download directory +- ReadOnlyMany access + +--- + +### 2. Deploy Kiwix + +```yaml +image: ghcr.io/kiwix/kiwix-serve:3.8.1 +args: ["/data/*.zim"] +``` + +--- + +### 3. Configure Tailscale LoadBalancer + +Tag: `svc:kiwix` + +--- + +### 4. Stop mcquack kiwix-serve diff --git a/plans/k8s-migration/P7_forgejo.md b/plans/k8s-migration/P7_forgejo.md new file mode 100644 index 0000000..3cfa4f1 --- /dev/null +++ b/plans/k8s-migration/P7_forgejo.md @@ -0,0 +1,51 @@ +# Phase 7: Forgejo Migration (Highest Risk) + +**Goal**: Migrate Forgejo to k8s + +**Status**: Pending + +**Prerequisites**: [Phase 6](P6_kiwix.md) complete + +--- + +## Pre-Migration Checklist + +- [ ] Full borgmatic backup verified +- [ ] Manual backup of `/opt/homebrew/var/forgejo` +- [ ] Document SSH keys and webhooks + +--- + +## Steps + +### 1. Deploy Forgejo via Helm + +```bash +helm install forgejo forgejo/forgejo \ + --namespace forgejo --create-namespace +``` + +--- + +### 2. Migrate data + +- Stop brew forgejo +- Copy data to PVC +- Start k8s forgejo + +--- + +### 3. Configure Tailscale services + +- HTTPS 443 via LoadBalancer +- SSH port 22 (TCP proxy) + +--- + +### 4. Verify all repositories accessible + +--- + +## Rollback + +Restore brew forgejo and tailscale serve config diff --git a/plans/k8s-migration/P8_woodpecker.md b/plans/k8s-migration/P8_woodpecker.md new file mode 100644 index 0000000..904398e --- /dev/null +++ b/plans/k8s-migration/P8_woodpecker.md @@ -0,0 +1,32 @@ +# Phase 8: CI/CD (Woodpecker) + +**Goal**: Deploy Woodpecker CI integrated with Forgejo + +**Status**: Pending + +**Prerequisites**: [Phase 7](P7_forgejo.md) complete + +--- + +## Steps + +### 1. Create Forgejo OAuth application + +- Callback: https://ci.tail8d86e.ts.net/authorize +- Store in 1Password + +--- + +### 2. Deploy Woodpecker Server + Agent + +--- + +### 3. Configure Tailscale LoadBalancer + +Tag: `svc:ci` + +--- + +### 4. Test pipeline + +Create `.woodpecker.yaml` in test repo diff --git a/plans/k8s-migration/P9_cleanup.md b/plans/k8s-migration/P9_cleanup.md new file mode 100644 index 0000000..9178b01 --- /dev/null +++ b/plans/k8s-migration/P9_cleanup.md @@ -0,0 +1,52 @@ +# Phase 9: Cleanup + +**Goal**: Remove deprecated services, harden system + +**Status**: Pending + +**Prerequisites**: [Phase 8](P8_woodpecker.md) complete + +--- + +## Steps + +### 1. Stop/remove unused brew services + +- postgresql@18 +- grafana +- miniflux +- forgejo + +--- + +### 2. Update ansible playbook + +- Remove migrated service roles +- Add k8s deployment references + +--- + +### 3. Configure Velero backups (optional) + +- Install with MinIO on sifaka +- Schedule daily cluster backups + +--- + +### 4. Update zk documentation + +- New architecture +- Runbooks +- DR procedures + +--- + +## Plan Completion + +When all phases are complete and verified: + +```bash +# Rename this folder to indicate completion +git mv plans/k8s-migration plans/k8s-migration.complete +git commit -m "Complete k8s migration plan" +``` diff --git a/pulumi/policy.hujson b/pulumi/policy.hujson index 4ce2ab0..c575037 100644 --- a/pulumi/policy.hujson +++ b/pulumi/policy.hujson @@ -59,6 +59,21 @@ "dst": ["tag:nas"], "ip": ["*"], }, + + // --- Kubernetes workloads --- + // k8s workloads (e.g., Woodpecker CI) can push/pull from registry + { + "src": ["tag:k8s"], + "dst": ["tag:registry"], + "ip": ["tcp:443"], + }, + // k8s workloads (e.g., ArgoCD) can access forge on indri for GitOps + // HTTP on 3001, SSH on 2200 + { + "src": ["tag:k8s"], + "dst": ["tag:homelab"], + "ip": ["tcp:3001", "tcp:2200"], + }, ], // ============== SSH Access ============== @@ -103,6 +118,8 @@ "tag:feed": ["autogroup:admin", "tag:blumeops"], "tag:registry": ["autogroup:admin", "tag:blumeops"], "tag:k8s-api": ["autogroup:admin", "tag:blumeops"], + "tag:k8s-operator": ["autogroup:admin", "tag:blumeops"], + "tag:k8s": ["autogroup:admin", "tag:blumeops", "tag:k8s-operator"], }, // ============== ACL Tests ============== @@ -123,5 +140,10 @@ "src": "tag:homelab", "accept": ["tag:homelab:22", "tag:nas:445"], }, + // K8s workloads can reach registry and forge (on indri:3001 HTTP, :2200 SSH) + { + "src": "tag:k8s", + "accept": ["tag:registry:443", "tag:homelab:3001", "tag:homelab:2200"], + }, ], }