From d05d2fbaff3478e6fd6edcdf54afdf4e79e58017 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Mon, 23 Feb 2026 18:07:18 -0800 Subject: [PATCH] C2: Upgrade Grafana to 12.x with Nix container and Kustomize (#260) ## Summary Mikado chain to upgrade Grafana from 11.4.0 (Helm chart) to 12.x with: - Home-built Nix container image (`forge.ops.eblu.me/eblume/grafana`) - Kustomize manifests replacing the Helm chart - Single-source ArgoCD app ## Chain Goal: `upgrade-grafana` Leaves: `build-grafana-container`, `kustomize-grafana-deployment` Track with: `mise run docs-mikado upgrade-grafana` ## Test plan - [ ] Container builds successfully via Nix - [ ] Container pushed to registry - [ ] Kustomize manifests produce equivalent resources to current Helm - [ ] Pod runs, UI loads, OIDC works, datasources healthy - [ ] `mise run services-check` passes Reviewed-on: https://forge.ops.eblu.me/eblume/blumeops/pulls/260 --- argocd/apps/grafana.yaml | 19 +- argocd/manifests/grafana/configmap.yaml | 98 ++++++++++ argocd/manifests/grafana/deployment.yaml | 176 ++++++++++++++++++ argocd/manifests/grafana/kustomization.yaml | 12 ++ argocd/manifests/grafana/pvc.yaml | 14 ++ argocd/manifests/grafana/rbac.yaml | 54 ++++++ argocd/manifests/grafana/service.yaml | 18 ++ argocd/manifests/grafana/serviceaccount.yaml | 9 + argocd/manifests/grafana/values.yaml | 108 ----------- containers/grafana/Dockerfile | 65 +++++++ docs/changelog.d/upgrade-grafana.feature.md | 1 + .../how-to/grafana/build-grafana-container.md | 34 ++-- .../grafana/kustomize-grafana-deployment.md | 43 ++--- docs/how-to/grafana/upgrade-grafana.md | 63 ++----- service-versions.yaml | 4 +- 15 files changed, 510 insertions(+), 208 deletions(-) create mode 100644 argocd/manifests/grafana/configmap.yaml create mode 100644 argocd/manifests/grafana/deployment.yaml create mode 100644 argocd/manifests/grafana/kustomization.yaml create mode 100644 argocd/manifests/grafana/pvc.yaml create mode 100644 argocd/manifests/grafana/rbac.yaml create mode 100644 argocd/manifests/grafana/service.yaml create mode 100644 argocd/manifests/grafana/serviceaccount.yaml delete mode 100644 argocd/manifests/grafana/values.yaml create mode 100644 containers/grafana/Dockerfile create mode 100644 docs/changelog.d/upgrade-grafana.feature.md diff --git a/argocd/apps/grafana.yaml b/argocd/apps/grafana.yaml index ec9262e..3a2cdd0 100644 --- a/argocd/apps/grafana.yaml +++ b/argocd/apps/grafana.yaml @@ -1,7 +1,5 @@ # Grafana - Dashboards & Observability # -# Chart mirrored from https://github.com/grafana/helm-charts to forge -# # Before syncing, create the admin password secret: # kubectl create namespace monitoring # op inject -i argocd/manifests/grafana-config/secret-admin.yaml.tpl | kubectl apply -f - @@ -12,19 +10,10 @@ metadata: namespace: argocd spec: project: default - sources: - # Helm chart from forge mirror (SSH via egress) - - repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/grafana-helm-charts.git - targetRevision: grafana-8.8.2 - path: charts/grafana - helm: - releaseName: grafana - valueFiles: - - $values/argocd/manifests/grafana/values.yaml - # Values from our git repo - - repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/blumeops.git - targetRevision: main - ref: values + source: + repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/blumeops.git + targetRevision: main + path: argocd/manifests/grafana destination: server: https://kubernetes.default.svc namespace: monitoring diff --git a/argocd/manifests/grafana/configmap.yaml b/argocd/manifests/grafana/configmap.yaml new file mode 100644 index 0000000..f0c00a7 --- /dev/null +++ b/argocd/manifests/grafana/configmap.yaml @@ -0,0 +1,98 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana + namespace: monitoring + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana +data: + grafana.ini: | + [analytics] + check_for_updates = false + reporting_enabled = false + + [auth.generic_oauth] + allow_sign_up = true + api_url = https://authentik.ops.eblu.me/application/o/userinfo/ + auth_url = https://authentik.ops.eblu.me/application/o/authorize/ + auto_login = false + client_id = grafana + client_secret = $__env{GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET} + enabled = true + name = Authentik + role_attribute_path = 'Admin' + scopes = openid profile email + token_url = https://authentik.ops.eblu.me/application/o/token/ + + [log] + mode = console + + [paths] + data = /var/lib/grafana/ + logs = /var/log/grafana + plugins = /var/lib/grafana/plugins + provisioning = /etc/grafana/provisioning + + [security] + allow_embedding = false + + [server] + root_url = https://grafana.ops.eblu.me + + datasources.yaml: | + apiVersion: 1 + datasources: + - access: proxy + editable: false + isDefault: true + name: Prometheus + orgId: 1 + type: prometheus + uid: prometheus + url: http://prometheus.monitoring.svc.cluster.local:9090 + - access: proxy + editable: false + name: Loki + orgId: 1 + type: loki + uid: loki + url: http://loki.monitoring.svc.cluster.local:3100 + - access: proxy + database: teslamate + editable: false + jsonData: + connMaxLifetime: 14400 + maxIdleConns: 2 + maxOpenConns: 5 + sslmode: disable + name: TeslaMate + orgId: 1 + secureJsonData: + password: $TESLAMATE_DB_PASSWORD + type: postgres + uid: TeslaMate + url: blumeops-pg-rw.databases.svc.cluster.local:5432 + user: teslamate +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-config-dashboards + namespace: monitoring + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana +data: + provider.yaml: | + apiVersion: 1 + providers: + - name: 'sidecarProvider' + orgId: 1 + type: file + disableDeletion: false + allowUiUpdates: false + updateIntervalSeconds: 30 + options: + foldersFromFilesStructure: true + path: /tmp/dashboards diff --git a/argocd/manifests/grafana/deployment.yaml b/argocd/manifests/grafana/deployment.yaml new file mode 100644 index 0000000..2adcdf8 --- /dev/null +++ b/argocd/manifests/grafana/deployment.yaml @@ -0,0 +1,176 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: monitoring + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + strategy: + type: RollingUpdate + template: + metadata: + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + annotations: + kubectl.kubernetes.io/default-container: grafana + spec: + automountServiceAccountToken: true + serviceAccountName: grafana + securityContext: + fsGroup: 472 + runAsGroup: 472 + runAsNonRoot: true + runAsUser: 472 + initContainers: + - name: init-chown-data + image: docker.io/library/busybox:1.31.1 + imagePullPolicy: IfNotPresent + command: ["chown", "-R", "472:472", "/var/lib/grafana"] + securityContext: + runAsNonRoot: false + runAsUser: 0 + capabilities: + add: ["CHOWN"] + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: storage + mountPath: /var/lib/grafana + containers: + # Dashboard sidecar - watches ConfigMaps with grafana_dashboard=1 + - name: grafana-sc-dashboard + image: quay.io/kiwigrid/k8s-sidecar:1.28.0 + imagePullPolicy: IfNotPresent + env: + - name: METHOD + value: WATCH + - name: LABEL + value: grafana_dashboard + - name: LABEL_VALUE + value: "1" + - name: FOLDER + value: /tmp/dashboards + - name: RESOURCE + value: both + - name: FOLDER_ANNOTATION + value: grafana_folder + - name: REQ_USERNAME + valueFrom: + secretKeyRef: + name: grafana-admin + key: admin-user + - name: REQ_PASSWORD + valueFrom: + secretKeyRef: + name: grafana-admin + key: admin-password + - name: REQ_URL + value: http://localhost:3000/api/admin/provisioning/dashboards/reload + - name: REQ_METHOD + value: POST + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: sc-dashboard-volume + mountPath: /tmp/dashboards + # Grafana + - name: grafana + image: registry.ops.eblu.me/blumeops/grafana:v12.3.3-09ac36b + imagePullPolicy: IfNotPresent + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: grafana-admin + key: admin-user + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: grafana-admin + key: admin-password + - name: GF_PATHS_DATA + value: /var/lib/grafana/ + - name: GF_PATHS_LOGS + value: /var/log/grafana + - name: GF_PATHS_PLUGINS + value: /var/lib/grafana/plugins + - name: GF_PATHS_PROVISIONING + value: /etc/grafana/provisioning + envFrom: + - secretRef: + name: grafana-teslamate-datasource + optional: true + - secretRef: + name: grafana-authentik-oauth + optional: true + ports: + - name: http + containerPort: 3000 + protocol: TCP + livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + failureThreshold: 10 + readinessProbe: + httpGet: + path: /api/health + port: 3000 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: config + mountPath: /etc/grafana/grafana.ini + subPath: grafana.ini + - name: config + mountPath: /etc/grafana/provisioning/datasources/datasources.yaml + subPath: datasources.yaml + - name: storage + mountPath: /var/lib/grafana + - name: sc-dashboard-volume + mountPath: /tmp/dashboards + - name: sc-dashboard-provider + mountPath: /etc/grafana/provisioning/dashboards/sc-dashboardproviders.yaml + subPath: provider.yaml + volumes: + - name: config + configMap: + name: grafana + - name: storage + persistentVolumeClaim: + claimName: grafana + - name: sc-dashboard-volume + emptyDir: {} + - name: sc-dashboard-provider + configMap: + name: grafana-config-dashboards diff --git a/argocd/manifests/grafana/kustomization.yaml b/argocd/manifests/grafana/kustomization.yaml new file mode 100644 index 0000000..2707d42 --- /dev/null +++ b/argocd/manifests/grafana/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: monitoring + +resources: + - serviceaccount.yaml + - configmap.yaml + - pvc.yaml + - deployment.yaml + - service.yaml + - rbac.yaml diff --git a/argocd/manifests/grafana/pvc.yaml b/argocd/manifests/grafana/pvc.yaml new file mode 100644 index 0000000..e119e3a --- /dev/null +++ b/argocd/manifests/grafana/pvc.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: grafana + namespace: monitoring + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/argocd/manifests/grafana/rbac.yaml b/argocd/manifests/grafana/rbac.yaml new file mode 100644 index 0000000..d0d0c843 --- /dev/null +++ b/argocd/manifests/grafana/rbac.yaml @@ -0,0 +1,54 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: grafana-clusterrole + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana +rules: + - apiGroups: [""] + resources: ["configmaps", "secrets"] + verbs: ["get", "watch", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: grafana-clusterrolebinding + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: grafana-clusterrole +subjects: + - kind: ServiceAccount + name: grafana + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: grafana + namespace: monitoring + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana +rules: [] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: grafana + namespace: monitoring + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: grafana +subjects: + - kind: ServiceAccount + name: grafana + namespace: monitoring diff --git a/argocd/manifests/grafana/service.yaml b/argocd/manifests/grafana/service.yaml new file mode 100644 index 0000000..eea02f1 --- /dev/null +++ b/argocd/manifests/grafana/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: monitoring + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana +spec: + type: ClusterIP + ports: + - name: http + port: 80 + targetPort: 3000 + protocol: TCP + selector: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana diff --git a/argocd/manifests/grafana/serviceaccount.yaml b/argocd/manifests/grafana/serviceaccount.yaml new file mode 100644 index 0000000..4a1363e --- /dev/null +++ b/argocd/manifests/grafana/serviceaccount.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: grafana + namespace: monitoring + labels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana +automountServiceAccountToken: false diff --git a/argocd/manifests/grafana/values.yaml b/argocd/manifests/grafana/values.yaml deleted file mode 100644 index e9e2207..0000000 --- a/argocd/manifests/grafana/values.yaml +++ /dev/null @@ -1,108 +0,0 @@ -# Grafana Helm values for blumeops -# Chart: https://github.com/grafana/helm-charts/tree/main/charts/grafana - -# Admin credentials from pre-created secret -# Secret must exist before deploying - see grafana-config/README.md -admin: - existingSecret: grafana-admin - userKey: admin-user - passwordKey: admin-password - -# Environment variables from secrets (for datasource credentials) -envFromSecrets: - - name: grafana-teslamate-datasource - optional: true - - name: grafana-authentik-oauth - optional: true - -# Persistence with PVC for SQLite database -persistence: - enabled: true - type: pvc - size: 1Gi - accessModes: - - ReadWriteOnce - -# Grafana configuration via grafana.ini -grafana.ini: - server: - root_url: https://grafana.ops.eblu.me - security: - # Embedding disabled - iframe approach didn't work well for Homepage - allow_embedding: false - auth.generic_oauth: - enabled: true - name: Authentik - client_id: grafana - client_secret: $__env{GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET} - scopes: openid profile email - auth_url: https://authentik.ops.eblu.me/application/o/authorize/ - token_url: https://authentik.ops.eblu.me/application/o/token/ - api_url: https://authentik.ops.eblu.me/application/o/userinfo/ - allow_sign_up: true - role_attribute_path: "'Admin'" - auto_login: false - analytics: - check_for_updates: false - reporting_enabled: false - -# Datasources - point to k8s-internal services -datasources: - datasources.yaml: - apiVersion: 1 - datasources: - - name: Prometheus - type: prometheus - access: proxy - orgId: 1 - uid: prometheus - url: http://prometheus.monitoring.svc.cluster.local:9090 - isDefault: true - editable: false - - name: Loki - type: loki - access: proxy - orgId: 1 - uid: loki - url: http://loki.monitoring.svc.cluster.local:3100 - editable: false - - name: TeslaMate - type: postgres - access: proxy - orgId: 1 - uid: TeslaMate - url: blumeops-pg-rw.databases.svc.cluster.local:5432 - database: teslamate - user: teslamate - editable: false - jsonData: - sslmode: disable - maxOpenConns: 5 - maxIdleConns: 2 - connMaxLifetime: 14400 - secureJsonData: - password: $TESLAMATE_DB_PASSWORD - -# Dashboard provisioning - sidecar watches for ConfigMaps with label -sidecar: - dashboards: - enabled: true - label: grafana_dashboard - labelValue: "1" - folderAnnotation: grafana_folder - provider: - foldersFromFilesStructure: true - -# Service configuration (Ingress will handle external access) -service: - type: ClusterIP - port: 80 - -# Resource limits for minikube -resources: - requests: - memory: "128Mi" - cpu: "100m" - limits: - memory: "512Mi" - cpu: "500m" diff --git a/containers/grafana/Dockerfile b/containers/grafana/Dockerfile new file mode 100644 index 0000000..f89adda --- /dev/null +++ b/containers/grafana/Dockerfile @@ -0,0 +1,65 @@ +ARG CONTAINER_APP_VERSION=12.3.3 + +FROM alpine:3.22 + +ARG TARGETPLATFORM +ARG CONTAINER_APP_VERSION +ARG GRAFANA_VERSION=${CONTAINER_APP_VERSION} + +RUN set -e && \ + apk --no-cache add dumb-init curl && \ + # Detect architecture + if [ -n "$TARGETPLATFORM" ]; then \ + echo "TARGETPLATFORM: $TARGETPLATFORM"; \ + case "$TARGETPLATFORM" in \ + linux/arm64*) ARCH="arm64" ;; \ + linux/amd64*) ARCH="amd64" ;; \ + *) ARCH="" ;; \ + esac; \ + else \ + echo "TARGETPLATFORM not set, detecting from uname..."; \ + UNAME_ARCH=$(uname -m); \ + echo "uname -m: $UNAME_ARCH"; \ + case "$UNAME_ARCH" in \ + aarch64|arm64) ARCH="arm64" ;; \ + x86_64) ARCH="amd64" ;; \ + *) ARCH="" ;; \ + esac; \ + fi && \ + if [ -z "$ARCH" ]; then \ + echo "ERROR: Unsupported architecture"; \ + exit 1; \ + fi && \ + url="https://dl.grafana.com/oss/release/grafana-${GRAFANA_VERSION}.linux-${ARCH}.tar.gz" && \ + echo "URL: $url" && \ + curl -fSL "$url" | tar -xz -C /tmp && \ + mv /tmp/grafana-${GRAFANA_VERSION} /usr/share/grafana && \ + apk del curl + +# Standard Grafana paths +RUN mkdir -p /etc/grafana /var/lib/grafana /var/log/grafana && \ + cp /usr/share/grafana/conf/defaults.ini /etc/grafana/grafana.ini && \ + cp /usr/share/grafana/conf/defaults.ini /etc/grafana/defaults.ini + +# UID 472 matches official Grafana image for PVC compatibility +RUN adduser -D -u 472 -h /usr/share/grafana grafana && \ + chown -R grafana:grafana /usr/share/grafana /etc/grafana /var/lib/grafana /var/log/grafana + +ENV PATH="/usr/share/grafana/bin:$PATH" + +USER grafana +WORKDIR /usr/share/grafana +EXPOSE 3000 + +LABEL org.opencontainers.image.title="Grafana" +LABEL org.opencontainers.image.description="Grafana OSS observability platform" +LABEL org.opencontainers.image.source="https://github.com/grafana/grafana" + +ENTRYPOINT ["/usr/bin/dumb-init", "--"] +CMD ["grafana", "server", \ + "--homepath=/usr/share/grafana", \ + "--config=/etc/grafana/grafana.ini", \ + "cfg:default.paths.data=/var/lib/grafana", \ + "cfg:default.paths.logs=/var/log/grafana", \ + "cfg:default.paths.plugins=/var/lib/grafana/plugins", \ + "cfg:default.paths.provisioning=/etc/grafana/provisioning"] diff --git a/docs/changelog.d/upgrade-grafana.feature.md b/docs/changelog.d/upgrade-grafana.feature.md new file mode 100644 index 0000000..cbd7e7c --- /dev/null +++ b/docs/changelog.d/upgrade-grafana.feature.md @@ -0,0 +1 @@ +Upgrade Grafana from 11.4.0 to 12.3.3 with home-built container image and Kustomize manifests, replacing the Helm chart deployment. diff --git a/docs/how-to/grafana/build-grafana-container.md b/docs/how-to/grafana/build-grafana-container.md index ccec654..0c073ce 100644 --- a/docs/how-to/grafana/build-grafana-container.md +++ b/docs/how-to/grafana/build-grafana-container.md @@ -1,6 +1,5 @@ --- title: Build Grafana Container -status: active modified: 2026-02-23 tags: - how-to @@ -10,25 +9,32 @@ tags: # Build Grafana Container -Build a home-built Grafana 12.x container image and publish to the forge registry. +Home-built Grafana container image published to `registry.ops.eblu.me/blumeops/grafana`. -## Context +## How It Works -Grafana currently uses the upstream `docker.io/grafana/grafana:11.4.0` image via the Helm chart. Per supply-chain policy, this should be replaced with a locally built image pushed to `forge.ops.eblu.me/eblume/grafana`. +The Dockerfile at `containers/grafana/Dockerfile` downloads the official Grafana OSS tarball for the target architecture (arm64/amd64), installs it into Alpine, and sets up standard paths. -## Steps +To build and push a new version: -1. Add a Grafana container build to Dagger (or Nix, following existing patterns) -2. Base on the official Grafana source or use a Nix derivation -3. Tag and push to `forge.ops.eblu.me/eblume/grafana:` -4. Add to `mise run container-list` inventory +```fish +# Update version in Dockerfile +# ARG CONTAINER_APP_VERSION=12.3.3 -## Reference +mise run container-build-and-release grafana +``` -- Follow [[build-container-image]] for the standard container build workflow -- See existing container builds in `.dagger/` for patterns -- The k8s-sidecar image (`quay.io/kiwigrid/k8s-sidecar`) is a secondary concern — address after the main Grafana image +## Gotchas + +- **Tarball directory name:** Extracts to `grafana-` (e.g. `grafana-12.3.3`), *not* `grafana-v`. +- **Binary PATH:** The binary lives at `bin/grafana` inside the extracted directory. The Dockerfile sets `ENV PATH="/usr/share/grafana/bin:$PATH"`. +- **UID 472:** Matches the official Grafana image for PVC ownership compatibility. + +## Future Work + +The k8s-sidecar image (`quay.io/kiwigrid/k8s-sidecar`) is still pulled from upstream. Replace with a home-built image when prioritized. ## Related -- [[upgrade-grafana]] — Goal card +- [[upgrade-grafana]] — Migration context +- [[build-container-image]] — Standard container build workflow diff --git a/docs/how-to/grafana/kustomize-grafana-deployment.md b/docs/how-to/grafana/kustomize-grafana-deployment.md index f25d14d..eb30ae9 100644 --- a/docs/how-to/grafana/kustomize-grafana-deployment.md +++ b/docs/how-to/grafana/kustomize-grafana-deployment.md @@ -1,6 +1,5 @@ --- title: Kustomize Grafana Deployment -status: active modified: 2026-02-23 tags: - how-to @@ -9,35 +8,27 @@ tags: # Kustomize Grafana Deployment -Convert Grafana from a Helm chart deployment to plain Kustomize manifests. +Grafana is deployed via plain Kustomize manifests in `argocd/manifests/grafana/`, replacing the previous Helm chart. -## Context +## Manifest Structure -Grafana is currently deployed via ArgoCD using a Helm chart (`grafana-8.8.2`) from a forge mirror. The chart produces: Deployment, Service, PVC, ConfigMaps (grafana.ini, datasources), RBAC resources, and a sidecar container for dashboard provisioning. +| File | Purpose | +|------|---------| +| `kustomization.yaml` | Resource list | +| `deployment.yaml` | Grafana container + k8s-sidecar for dashboards | +| `service.yaml` | ClusterIP on port 80 → 3000 | +| `pvc.yaml` | 1Gi SQLite storage | +| `configmap.yaml` | `grafana.ini` and datasource provisioning | +| `serviceaccount.yaml` | Service account | +| `rbac.yaml` | ClusterRole/RoleBinding for sidecar ConfigMap access | -## Steps +## Key Details -1. Template the current Helm chart to see what it produces: - ```fish - # From the forge mirror, or use argocd app manifests - argocd app manifests grafana > /tmp/grafana-helm-output.yaml - ``` -2. Create Kustomize equivalents in `argocd/manifests/grafana/`: - - `kustomization.yaml` - - `deployment.yaml` — Grafana container + k8s-sidecar container - - `service.yaml` - - `pvc.yaml` — Reuse existing 1Gi PVC - - `configmap.yaml` — grafana.ini and datasource provisioning - - `rbac.yaml` — ClusterRole, ClusterRoleBinding, Role, RoleBinding -3. Update `argocd/apps/grafana.yaml` to use a single kustomize source instead of the Helm multi-source -4. Remove the Helm values.yaml (replaced by the kustomize manifests) - -## Notes - -- The existing PVC must not be deleted during the transition — ensure the kustomize PVC matches the existing one's name -- The sidecar (`quay.io/kiwigrid/k8s-sidecar`) should also be replaced with a home-built image eventually, but is lower priority — focus on Grafana itself first -- Preserve all existing config: Authentik OIDC, datasources, dashboard sidecar labels +- **PVC name must remain `grafana`** — changing it would create a new volume and lose the SQLite DB +- **Sidecar** watches ConfigMaps with label `grafana_dashboard=1` and reloads dashboards via the Grafana API +- **Secrets** come from ExternalSecrets (`grafana-admin`, `grafana-authentik-oauth`, `grafana-teslamate-datasource`) managed by the `grafana-config` ArgoCD app ## Related -- [[upgrade-grafana]] — Goal card +- [[upgrade-grafana]] — Migration context +- [[grafana]] — Service reference card diff --git a/docs/how-to/grafana/upgrade-grafana.md b/docs/how-to/grafana/upgrade-grafana.md index 4044150..6371ee1 100644 --- a/docs/how-to/grafana/upgrade-grafana.md +++ b/docs/how-to/grafana/upgrade-grafana.md @@ -1,6 +1,5 @@ --- title: Upgrade Grafana -status: active requires: - kustomize-grafana-deployment - build-grafana-container @@ -13,60 +12,38 @@ tags: # Upgrade Grafana -Upgrade Grafana from 11.4.0 (Helm chart 8.8.2) to 12.x, converting from Helm to Kustomize with a home-built container image. +Upgraded Grafana from 11.4.0 (Helm chart) to 12.3.3, converting from Helm to Kustomize with a home-built container image. -## Current State +## What Changed -| Property | Value | -|----------|-------| -| **Helm chart** | `grafana-8.8.2` (from forge mirror of `grafana/helm-charts`) | -| **Grafana app** | 11.4.0 | -| **Deployment** | Helm via ArgoCD multi-source | -| **Namespace** | `monitoring` | -| **Storage** | SQLite on 1Gi PVC | +- **Image:** `docker.io/grafana/grafana:11.4.0` → `registry.ops.eblu.me/blumeops/grafana:v12.3.3` +- **Deployment:** Helm multi-source (chart + values) → single Kustomize directory +- **ArgoCD app:** Simplified to one source pointing at `argocd/manifests/grafana/` -Datasources: [[prometheus]], [[loki]], PostgreSQL (TeslaMate). Dashboard ConfigMaps provisioned via sidecar. - -## Target State - -- Grafana 12.x running from a home-built container (`forge.ops.eblu.me/eblume/grafana`) -- Kustomize manifests in `argocd/manifests/grafana/` (no Helm chart dependency) -- ArgoCD app simplified to a single source (kustomize directory) -- All existing datasources, dashboards, and Authentik OIDC intact +All existing datasources ([[prometheus]], [[loki]], TeslaMate), dashboard ConfigMaps, and Authentik OIDC were preserved without changes. ## Grafana 12 Breaking Changes -- **Angular plugin removal:** All AngularJS panels force-migrated to React. Our dashboards already use only React panels — no action needed. -- **Datasource UID format enforcement:** UIDs must be alphanumeric + dash/underscore, ≤40 chars. Our UIDs (`prometheus`, `loki`, `TeslaMate`) are compliant. -- **Annotation table migration:** Full rewrite of the `annotation` table (adds `dashboard_uid` column). Small SQLite DB — should be fast. PVC is disposable if anything goes wrong. +None affected us: -Overall risk: **Low.** +- **Angular plugin removal** — our dashboards already used React panels +- **Datasource UID format enforcement** — our UIDs were already compliant +- **Annotation table migration** — completed automatically on the small SQLite DB -## Execution +## How to Repeat -Once both prerequisites are complete: +To upgrade Grafana again in the future: -1. Update `argocd/apps/grafana.yaml` to point at the kustomize directory (single source, remove Helm multi-source) -2. Update `argocd/manifests/grafana/` with the kustomize manifests using the home-built image -3. Deploy on branch, verify with checklist below -4. Update `service-versions.yaml` to the new version and today's date +1. Update `CONTAINER_APP_VERSION` in `containers/grafana/Dockerfile` +2. Build and push via `mise run container-build-and-release grafana` +3. Update the image tag in `argocd/manifests/grafana/deployment.yaml` +4. Update `service-versions.yaml` +5. Sync: `argocd app sync grafana` -The SQLite PVC is disposable — dashboards are provisioned from ConfigMaps and datasources from config. No backup needed. - -## Verification Checklist - -- [ ] Pod running: `kubectl --context=minikube-indri -n monitoring get pods -l app.kubernetes.io/name=grafana` -- [ ] UI loads at `https://grafana.ops.eblu.me` -- [ ] Admin login works -- [ ] Authentik OIDC login works -- [ ] Datasources healthy: Prometheus, Loki, TeslaMate (Settings → Datasources → Test each) -- [ ] Key dashboards render: macOS System, Services Health, TeslaMate Overview -- [ ] Sidecar loaded all dashboard ConfigMaps -- [ ] `mise run services-check` passes -- [ ] No errors in pod logs +The SQLite PVC is disposable — dashboards come from ConfigMaps and datasources from config. ## Related - [[grafana]] — Service reference card -- [[build-grafana-container]] — Prereq: build the container image -- [[kustomize-grafana-deployment]] — Prereq: create kustomize manifests +- [[build-grafana-container]] — Building the container image +- [[kustomize-grafana-deployment]] — Kustomize manifest structure diff --git a/service-versions.yaml b/service-versions.yaml index 318ff82..92e4d65 100644 --- a/service-versions.yaml +++ b/service-versions.yaml @@ -83,9 +83,9 @@ services: - name: grafana type: argocd last-reviewed: 2026-02-23 - current-version: "v11.4.0" + current-version: "12.3.3" upstream-source: https://github.com/grafana/grafana/releases - notes: Helm chart 8.8.2; Mikado chain to upgrade to 12.x with kustomize + notes: Home-built container; upgrading to 12.x via Mikado chain - name: cloudnative-pg type: argocd