Erich Blume
9c789a1868
All checks were successful
Deploy Fly.io Proxy / deploy (push) Successful in 1m19s
## Summary - Remove `match_all = true` from `flyio_nginx_cache_requests_total` in Alloy so the metric only counts requests that go through the proxy cache (excludes health checks with empty `cache_status`) - Change dashboard queries from `rate(...[5m])` to `increase(...[$__range])` — aggregates over the full dashboard time window instead of a 5-minute sliding window, giving meaningful ratios for low-traffic static sites - Add null/NaN value mapping to show "No traffic" in neutral color instead of blank/red ## Root cause Health check requests from Fly.io hit the default nginx server block (no `proxy_cache`), producing entries with empty `upstream_cache_status`. With `match_all = true`, these were counted in the cache metric, diluting the Fly.io dashboard ratio. For APM dashboards, `rate()[5m]` on low-traffic sites with 24h cache validity almost always returns either all-HITs (100%) or no data (blank → red background). ## Deployment - Fly.io proxy redeploy needed for Alloy config change - ArgoCD sync for dashboard ConfigMap changes ## Test plan - [ ] Redeploy Fly.io proxy - [ ] Sync grafana-config in ArgoCD - [ ] Verify CV APM cache hit ratio shows a real percentage (not 100%) - [ ] Verify Docs APM shows "No traffic" in neutral color when idle, real ratio when visited - [ ] Verify Fly.io proxy dashboard cache ratio excludes health checks Reviewed-on: https://forge.ops.eblu.me/eblume/blumeops/pulls/177
263 lines
11 KiB
YAML
263 lines
11 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: grafana-dashboard-cv-apm
|
|
namespace: monitoring
|
|
labels:
|
|
grafana_dashboard: "1"
|
|
data:
|
|
cv-apm.json: |
|
|
{
|
|
"annotations": { "list": [] },
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 1,
|
|
"id": null,
|
|
"links": [],
|
|
"panels": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "req/s",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 20,
|
|
"gradientMode": "none",
|
|
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": { "type": "linear" },
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": { "group": "A", "mode": "normal" },
|
|
"thresholdsStyle": { "mode": "off" }
|
|
},
|
|
"mappings": [],
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"unit": "reqps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 8, "w": 16, "x": 0, "y": 0 },
|
|
"id": 1,
|
|
"options": {
|
|
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true },
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
},
|
|
"targets": [
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (status) (rate(flyio_nginx_http_requests_total{host=\"cv.eblu.me\"}[5m]))", "legendFormat": "{{status}}", "refId": "A" }
|
|
],
|
|
"title": "Request Rate by Status",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "thresholds" },
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.05 }] },
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 4, "w": 8, "x": 16, "y": 0 },
|
|
"id": 2,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
|
"textMode": "auto"
|
|
},
|
|
"targets": [
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_http_requests_total{host=\"cv.eblu.me\",status=~\"5..\"}[5m])) / sum(rate(flyio_nginx_http_requests_total{host=\"cv.eblu.me\"}[5m]))", "refId": "A" }
|
|
],
|
|
"title": "Error Rate (5xx)",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "thresholds" },
|
|
"mappings": [{ "type": "special", "options": { "match": "null+nan", "result": { "text": "No traffic", "color": "text" } } }],
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 0.8 }] },
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 4 },
|
|
"id": 3,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "none",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
|
"textMode": "auto"
|
|
},
|
|
"targets": [
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(increase(flyio_nginx_cache_requests_total{host=\"cv.eblu.me\",cache_status=\"HIT\"}[$__range])) / sum(increase(flyio_nginx_cache_requests_total{host=\"cv.eblu.me\"}[$__range]))", "refId": "A" }
|
|
],
|
|
"title": "Cache Hit Ratio",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "thresholds" },
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"unit": "reqps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 4 },
|
|
"id": 4,
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
|
"textMode": "auto"
|
|
},
|
|
"targets": [
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_http_requests_total{host=\"cv.eblu.me\"}[5m]))", "refId": "A" }
|
|
],
|
|
"title": "Current RPS",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "seconds",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": { "type": "linear" },
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": { "group": "A", "mode": "none" },
|
|
"thresholdsStyle": { "mode": "off" }
|
|
},
|
|
"mappings": [],
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
|
|
"id": 5,
|
|
"options": {
|
|
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true },
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
},
|
|
"targets": [
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(flyio_nginx_http_request_duration_seconds_bucket{host=\"cv.eblu.me\"}[5m])))", "legendFormat": "p50", "refId": "A" },
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.90, sum by (le) (rate(flyio_nginx_http_request_duration_seconds_bucket{host=\"cv.eblu.me\"}[5m])))", "legendFormat": "p90", "refId": "B" },
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(flyio_nginx_http_request_duration_seconds_bucket{host=\"cv.eblu.me\"}[5m])))", "legendFormat": "p99", "refId": "C" }
|
|
],
|
|
"title": "Latency Percentiles",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 20,
|
|
"gradientMode": "none",
|
|
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": { "type": "linear" },
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": { "group": "A", "mode": "none" },
|
|
"thresholdsStyle": { "mode": "off" }
|
|
},
|
|
"mappings": [],
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"unit": "Bps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
|
|
"id": 6,
|
|
"options": {
|
|
"legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true },
|
|
"tooltip": { "mode": "single", "sort": "none" }
|
|
},
|
|
"targets": [
|
|
{ "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(flyio_nginx_http_response_bytes_total{host=\"cv.eblu.me\"}[5m]))", "legendFormat": "Bandwidth", "refId": "A" }
|
|
],
|
|
"title": "Bandwidth",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": { "type": "loki", "uid": "loki" },
|
|
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 },
|
|
"id": 7,
|
|
"options": {
|
|
"dedupStrategy": "none",
|
|
"enableLogDetails": true,
|
|
"prettifyLogMessage": false,
|
|
"showCommonLabels": false,
|
|
"showLabels": false,
|
|
"showTime": true,
|
|
"sortOrder": "Descending",
|
|
"wrapLogMessage": false
|
|
},
|
|
"targets": [
|
|
{ "datasource": { "type": "loki", "uid": "loki" }, "expr": "{instance=\"flyio-proxy\", job=\"flyio-nginx\"} |= \"cv.eblu.me\" | json | line_format \"{{.client_ip}} {{.request_method}} {{.request_uri}} {{.status}} cache={{.upstream_cache_status}} {{.request_time}}s\"", "refId": "A" }
|
|
],
|
|
"title": "Recent Access Logs",
|
|
"type": "logs"
|
|
}
|
|
],
|
|
"refresh": "30s",
|
|
"schemaVersion": 38,
|
|
"tags": ["cv", "flyio", "apm"],
|
|
"templating": { "list": [] },
|
|
"time": { "from": "now-6h", "to": "now" },
|
|
"timepicker": {},
|
|
"timezone": "",
|
|
"title": "CV APM",
|
|
"uid": "cv-apm",
|
|
"version": 1,
|
|
"weekStart": ""
|
|
}
|