From 3fc06cda889358ae92546ead09ee697ae1b0acd6 Mon Sep 17 00:00:00 2001 From: Erich Blume Date: Thu, 5 Mar 2026 10:03:17 -0800 Subject: [PATCH] Add Tempo manifests and ArgoCD Application Deploys Grafana Tempo 2.10.1 on minikube-indri for distributed trace storage. Includes OTLP receivers (gRPC + HTTP), local filesystem storage with 7d retention, and metrics_generator that remote-writes span-metrics to Prometheus. Two Tailscale Ingresses: tempo (query API) and tempo-otlp (OTLP HTTP receiver for cross-cluster trace ingestion). Co-Authored-By: Claude Opus 4.6 --- argocd/apps/tempo.yaml | 17 +++++ .../tempo/ingress-tailscale-otlp.yaml | 27 +++++++ argocd/manifests/tempo/ingress-tailscale.yaml | 26 +++++++ argocd/manifests/tempo/kustomization.yaml | 19 +++++ argocd/manifests/tempo/service.yaml | 22 ++++++ argocd/manifests/tempo/statefulset.yaml | 70 +++++++++++++++++++ argocd/manifests/tempo/tempo.yaml | 53 ++++++++++++++ 7 files changed, 234 insertions(+) create mode 100644 argocd/apps/tempo.yaml create mode 100644 argocd/manifests/tempo/ingress-tailscale-otlp.yaml create mode 100644 argocd/manifests/tempo/ingress-tailscale.yaml create mode 100644 argocd/manifests/tempo/kustomization.yaml create mode 100644 argocd/manifests/tempo/service.yaml create mode 100644 argocd/manifests/tempo/statefulset.yaml create mode 100644 argocd/manifests/tempo/tempo.yaml diff --git a/argocd/apps/tempo.yaml b/argocd/apps/tempo.yaml new file mode 100644 index 0000000..b04d297 --- /dev/null +++ b/argocd/apps/tempo.yaml @@ -0,0 +1,17 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: tempo + namespace: argocd +spec: + project: default + source: + repoURL: ssh://forgejo@forge.ops.eblu.me:2222/eblume/blumeops.git + targetRevision: main + path: argocd/manifests/tempo + destination: + server: https://kubernetes.default.svc + namespace: monitoring + syncPolicy: + syncOptions: + - CreateNamespace=true diff --git a/argocd/manifests/tempo/ingress-tailscale-otlp.yaml b/argocd/manifests/tempo/ingress-tailscale-otlp.yaml new file mode 100644 index 0000000..ed65113 --- /dev/null +++ b/argocd/manifests/tempo/ingress-tailscale-otlp.yaml @@ -0,0 +1,27 @@ +# Tailscale Ingress for Tempo OTLP HTTP receiver +# Used by ringtail Alloy to push traces across tailnet +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: tempo-otlp-tailscale + namespace: monitoring + annotations: + tailscale.com/funnel: "false" + tailscale.com/proxy-group: "ingress" + tailscale.com/tags: "tag:k8s" + gethomepage.dev/enabled: "false" +spec: + ingressClassName: tailscale + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: tempo + port: + number: 4318 + tls: + - hosts: + - tempo-otlp diff --git a/argocd/manifests/tempo/ingress-tailscale.yaml b/argocd/manifests/tempo/ingress-tailscale.yaml new file mode 100644 index 0000000..660d77a --- /dev/null +++ b/argocd/manifests/tempo/ingress-tailscale.yaml @@ -0,0 +1,26 @@ +# Tailscale Ingress for Tempo query API +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: tempo-tailscale + namespace: monitoring + annotations: + tailscale.com/funnel: "false" + tailscale.com/proxy-group: "ingress" + tailscale.com/tags: "tag:k8s" + gethomepage.dev/enabled: "false" +spec: + ingressClassName: tailscale + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: tempo + port: + number: 3200 + tls: + - hosts: + - tempo diff --git a/argocd/manifests/tempo/kustomization.yaml b/argocd/manifests/tempo/kustomization.yaml new file mode 100644 index 0000000..68a209c --- /dev/null +++ b/argocd/manifests/tempo/kustomization.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: monitoring + +resources: + - statefulset.yaml + - service.yaml + - ingress-tailscale.yaml + - ingress-tailscale-otlp.yaml + +images: + - name: grafana/tempo + newTag: "2.10.1" + +configMapGenerator: + - name: tempo-config + files: + - tempo.yaml diff --git a/argocd/manifests/tempo/service.yaml b/argocd/manifests/tempo/service.yaml new file mode 100644 index 0000000..37b25df --- /dev/null +++ b/argocd/manifests/tempo/service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: tempo + namespace: monitoring +spec: + selector: + app: tempo + ports: + - name: http + port: 3200 + targetPort: 3200 + - name: grpc + port: 9095 + targetPort: 9095 + - name: otlp-grpc + port: 4317 + targetPort: 4317 + - name: otlp-http + port: 4318 + targetPort: 4318 + type: ClusterIP diff --git a/argocd/manifests/tempo/statefulset.yaml b/argocd/manifests/tempo/statefulset.yaml new file mode 100644 index 0000000..7975347 --- /dev/null +++ b/argocd/manifests/tempo/statefulset.yaml @@ -0,0 +1,70 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: tempo + namespace: monitoring +spec: + serviceName: tempo + replicas: 1 + selector: + matchLabels: + app: tempo + template: + metadata: + labels: + app: tempo + spec: + securityContext: + fsGroup: 10001 + runAsNonRoot: true + runAsUser: 10001 + containers: + - name: tempo + image: grafana/tempo + args: + - -config.file=/etc/tempo/tempo.yaml + ports: + - name: http + containerPort: 3200 + - name: grpc + containerPort: 9095 + - name: otlp-grpc + containerPort: 4317 + - name: otlp-http + containerPort: 4318 + volumeMounts: + - name: config + mountPath: /etc/tempo + - name: data + mountPath: /var/tempo + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "1Gi" + cpu: "500m" + livenessProbe: + httpGet: + path: /ready + port: 3200 + initialDelaySeconds: 45 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /ready + port: 3200 + initialDelaySeconds: 10 + periodSeconds: 5 + volumes: + - name: config + configMap: + name: tempo-config + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi diff --git a/argocd/manifests/tempo/tempo.yaml b/argocd/manifests/tempo/tempo.yaml new file mode 100644 index 0000000..da26cbe --- /dev/null +++ b/argocd/manifests/tempo/tempo.yaml @@ -0,0 +1,53 @@ +stream_over_http_enabled: true + +server: + http_listen_port: 3200 + grpc_listen_port: 9095 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + http: + endpoint: "0.0.0.0:4318" + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + +compactor: + compaction: + block_retention: 168h # 7 days + +metrics_generator: + registry: + external_labels: + source: tempo + storage: + path: /var/tempo/generator/wal + remote_write: + - url: http://prometheus.monitoring.svc.cluster.local:9090/api/v1/write + send_exemplars: true + processor: + span_metrics: + dimensions: + - service.name + - http.method + - http.status_code + - http.target + service_graphs: + dimensions: + - service.name + +overrides: + defaults: + metrics_generator: + processors: + - span-metrics + - service-graphs