2026-01-15 12:24:13 -08:00
|
|
|
{
|
|
|
|
|
"annotations": {
|
|
|
|
|
"list": []
|
|
|
|
|
},
|
|
|
|
|
"editable": true,
|
|
|
|
|
"fiscalYearStartMonth": 0,
|
|
|
|
|
"graphTooltip": 0,
|
|
|
|
|
"id": null,
|
|
|
|
|
"links": [],
|
|
|
|
|
"panels": [
|
|
|
|
|
{
|
|
|
|
|
"datasource": {
|
|
|
|
|
"type": "prometheus",
|
|
|
|
|
"uid": "prometheus"
|
|
|
|
|
},
|
|
|
|
|
"fieldConfig": {
|
|
|
|
|
"defaults": {
|
|
|
|
|
"color": {
|
|
|
|
|
"mode": "thresholds"
|
|
|
|
|
},
|
|
|
|
|
"mappings": [],
|
|
|
|
|
"thresholds": {
|
|
|
|
|
"mode": "absolute",
|
|
|
|
|
"steps": [
|
|
|
|
|
{ "color": "green", "value": null },
|
|
|
|
|
{ "color": "yellow", "value": 1073741824 },
|
|
|
|
|
{ "color": "red", "value": 5368709120 }
|
|
|
|
|
]
|
|
|
|
|
},
|
|
|
|
|
"unit": "bytes"
|
|
|
|
|
},
|
|
|
|
|
"overrides": []
|
|
|
|
|
},
|
|
|
|
|
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
|
|
|
|
|
"id": 1,
|
|
|
|
|
"options": {
|
|
|
|
|
"colorMode": "value",
|
|
|
|
|
"graphMode": "none",
|
|
|
|
|
"justifyMode": "auto",
|
|
|
|
|
"orientation": "auto",
|
|
|
|
|
"reduceOptions": {
|
|
|
|
|
"calcs": ["lastNotNull"],
|
|
|
|
|
"fields": "",
|
|
|
|
|
"values": false
|
|
|
|
|
},
|
|
|
|
|
"textMode": "auto"
|
|
|
|
|
},
|
|
|
|
|
"pluginVersion": "10.0.0",
|
|
|
|
|
"targets": [
|
|
|
|
|
{
|
|
|
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
2026-01-15 13:56:11 -08:00
|
|
|
"expr": "sum(loki_ingester_chunk_stored_bytes_total)",
|
2026-01-15 12:24:13 -08:00
|
|
|
"refId": "A"
|
|
|
|
|
}
|
|
|
|
|
],
|
2026-01-15 13:56:11 -08:00
|
|
|
"title": "Total Stored",
|
2026-01-15 12:24:13 -08:00
|
|
|
"type": "stat"
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"datasource": {
|
|
|
|
|
"type": "prometheus",
|
|
|
|
|
"uid": "prometheus"
|
|
|
|
|
},
|
|
|
|
|
"fieldConfig": {
|
|
|
|
|
"defaults": {
|
|
|
|
|
"color": {
|
|
|
|
|
"mode": "thresholds"
|
|
|
|
|
},
|
|
|
|
|
"mappings": [],
|
|
|
|
|
"thresholds": {
|
|
|
|
|
"mode": "absolute",
|
2026-01-16 19:33:02 -08:00
|
|
|
"steps": [{ "color": "green", "value": null }]
|
2026-01-15 12:24:13 -08:00
|
|
|
},
|
|
|
|
|
"unit": "short"
|
|
|
|
|
},
|
|
|
|
|
"overrides": []
|
|
|
|
|
},
|
|
|
|
|
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
|
|
|
|
|
"id": 2,
|
|
|
|
|
"options": {
|
|
|
|
|
"colorMode": "value",
|
|
|
|
|
"graphMode": "none",
|
|
|
|
|
"justifyMode": "auto",
|
|
|
|
|
"orientation": "auto",
|
|
|
|
|
"reduceOptions": {
|
|
|
|
|
"calcs": ["lastNotNull"],
|
|
|
|
|
"fields": "",
|
|
|
|
|
"values": false
|
|
|
|
|
},
|
|
|
|
|
"textMode": "auto"
|
|
|
|
|
},
|
|
|
|
|
"pluginVersion": "10.0.0",
|
|
|
|
|
"targets": [
|
|
|
|
|
{
|
|
|
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
|
|
|
"expr": "sum(loki_ingester_memory_chunks)",
|
|
|
|
|
"refId": "A"
|
|
|
|
|
}
|
|
|
|
|
],
|
|
|
|
|
"title": "Active Chunks",
|
|
|
|
|
"type": "stat"
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"datasource": {
|
|
|
|
|
"type": "prometheus",
|
|
|
|
|
"uid": "prometheus"
|
|
|
|
|
},
|
|
|
|
|
"fieldConfig": {
|
|
|
|
|
"defaults": {
|
|
|
|
|
"color": {
|
|
|
|
|
"mode": "thresholds"
|
|
|
|
|
},
|
|
|
|
|
"mappings": [],
|
|
|
|
|
"thresholds": {
|
|
|
|
|
"mode": "absolute",
|
2026-01-16 19:33:02 -08:00
|
|
|
"steps": [{ "color": "green", "value": null }]
|
2026-01-15 12:24:13 -08:00
|
|
|
},
|
|
|
|
|
"unit": "short"
|
|
|
|
|
},
|
|
|
|
|
"overrides": []
|
|
|
|
|
},
|
|
|
|
|
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
|
|
|
|
|
"id": 3,
|
|
|
|
|
"options": {
|
|
|
|
|
"colorMode": "value",
|
|
|
|
|
"graphMode": "none",
|
|
|
|
|
"justifyMode": "auto",
|
|
|
|
|
"orientation": "auto",
|
|
|
|
|
"reduceOptions": {
|
|
|
|
|
"calcs": ["lastNotNull"],
|
|
|
|
|
"fields": "",
|
|
|
|
|
"values": false
|
|
|
|
|
},
|
|
|
|
|
"textMode": "auto"
|
|
|
|
|
},
|
|
|
|
|
"pluginVersion": "10.0.0",
|
|
|
|
|
"targets": [
|
|
|
|
|
{
|
|
|
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
|
|
|
"expr": "sum(loki_ingester_streams_created_total) - sum(loki_ingester_streams_removed_total)",
|
|
|
|
|
"refId": "A"
|
|
|
|
|
}
|
|
|
|
|
],
|
|
|
|
|
"title": "Active Streams",
|
|
|
|
|
"type": "stat"
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"datasource": {
|
|
|
|
|
"type": "prometheus",
|
|
|
|
|
"uid": "prometheus"
|
|
|
|
|
},
|
|
|
|
|
"fieldConfig": {
|
|
|
|
|
"defaults": {
|
|
|
|
|
"color": {
|
|
|
|
|
"mode": "thresholds"
|
|
|
|
|
},
|
|
|
|
|
"mappings": [],
|
|
|
|
|
"thresholds": {
|
|
|
|
|
"mode": "absolute",
|
|
|
|
|
"steps": [
|
|
|
|
|
{ "color": "green", "value": null },
|
|
|
|
|
{ "color": "yellow", "value": 0.5 },
|
|
|
|
|
{ "color": "red", "value": 0.9 }
|
|
|
|
|
]
|
|
|
|
|
},
|
|
|
|
|
"unit": "percentunit"
|
|
|
|
|
},
|
|
|
|
|
"overrides": []
|
|
|
|
|
},
|
|
|
|
|
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
|
|
|
|
|
"id": 4,
|
|
|
|
|
"options": {
|
|
|
|
|
"colorMode": "value",
|
|
|
|
|
"graphMode": "none",
|
|
|
|
|
"justifyMode": "auto",
|
|
|
|
|
"orientation": "auto",
|
|
|
|
|
"reduceOptions": {
|
|
|
|
|
"calcs": ["lastNotNull"],
|
|
|
|
|
"fields": "",
|
|
|
|
|
"values": false
|
|
|
|
|
},
|
|
|
|
|
"textMode": "auto"
|
|
|
|
|
},
|
|
|
|
|
"pluginVersion": "10.0.0",
|
|
|
|
|
"targets": [
|
|
|
|
|
{
|
|
|
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
|
|
|
"expr": "1 - (go_memstats_heap_idle_bytes{job=\"loki\"} / go_memstats_heap_sys_bytes{job=\"loki\"})",
|
|
|
|
|
"refId": "A"
|
|
|
|
|
}
|
|
|
|
|
],
|
|
|
|
|
"title": "Heap Usage",
|
|
|
|
|
"type": "stat"
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"datasource": {
|
|
|
|
|
"type": "prometheus",
|
|
|
|
|
"uid": "prometheus"
|
|
|
|
|
},
|
|
|
|
|
"fieldConfig": {
|
|
|
|
|
"defaults": {
|
|
|
|
|
"color": {
|
|
|
|
|
"mode": "palette-classic"
|
|
|
|
|
},
|
|
|
|
|
"custom": {
|
|
|
|
|
"axisBorderShow": false,
|
|
|
|
|
"axisCenteredZero": false,
|
|
|
|
|
"axisColorMode": "text",
|
|
|
|
|
"axisLabel": "",
|
|
|
|
|
"axisPlacement": "auto",
|
|
|
|
|
"barAlignment": 0,
|
|
|
|
|
"drawStyle": "line",
|
|
|
|
|
"fillOpacity": 10,
|
|
|
|
|
"gradientMode": "none",
|
|
|
|
|
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
|
|
|
|
"insertNulls": false,
|
|
|
|
|
"lineInterpolation": "linear",
|
|
|
|
|
"lineWidth": 1,
|
|
|
|
|
"pointSize": 5,
|
|
|
|
|
"scaleDistribution": { "type": "linear" },
|
|
|
|
|
"showPoints": "never",
|
|
|
|
|
"spanNulls": false,
|
|
|
|
|
"stacking": { "group": "A", "mode": "none" },
|
|
|
|
|
"thresholdsStyle": { "mode": "off" }
|
|
|
|
|
},
|
|
|
|
|
"mappings": [],
|
|
|
|
|
"thresholds": {
|
|
|
|
|
"mode": "absolute",
|
|
|
|
|
"steps": [{ "color": "green", "value": null }]
|
|
|
|
|
},
|
|
|
|
|
"unit": "Bps"
|
|
|
|
|
},
|
|
|
|
|
"overrides": []
|
|
|
|
|
},
|
|
|
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
|
|
|
|
|
"id": 5,
|
|
|
|
|
"options": {
|
2026-01-16 19:33:02 -08:00
|
|
|
"legend": {
|
|
|
|
|
"calcs": ["mean", "max"],
|
|
|
|
|
"displayMode": "table",
|
|
|
|
|
"placement": "bottom",
|
|
|
|
|
"showLegend": true
|
|
|
|
|
},
|
2026-01-15 12:24:13 -08:00
|
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
|
|
|
},
|
|
|
|
|
"pluginVersion": "10.0.0",
|
|
|
|
|
"targets": [
|
|
|
|
|
{
|
|
|
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
|
|
|
"expr": "rate(loki_distributor_bytes_received_total[5m])",
|
|
|
|
|
"legendFormat": "Bytes Received",
|
|
|
|
|
"refId": "A"
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
|
|
|
"expr": "rate(loki_ingester_chunk_stored_bytes_total[5m])",
|
|
|
|
|
"legendFormat": "Bytes Stored",
|
|
|
|
|
"refId": "B"
|
|
|
|
|
}
|
|
|
|
|
],
|
|
|
|
|
"title": "Ingestion Rate",
|
|
|
|
|
"type": "timeseries"
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"datasource": {
|
|
|
|
|
"type": "prometheus",
|
|
|
|
|
"uid": "prometheus"
|
|
|
|
|
},
|
|
|
|
|
"fieldConfig": {
|
|
|
|
|
"defaults": {
|
|
|
|
|
"color": {
|
|
|
|
|
"mode": "palette-classic"
|
|
|
|
|
},
|
|
|
|
|
"custom": {
|
|
|
|
|
"axisBorderShow": false,
|
|
|
|
|
"axisCenteredZero": false,
|
|
|
|
|
"axisColorMode": "text",
|
|
|
|
|
"axisLabel": "",
|
|
|
|
|
"axisPlacement": "auto",
|
|
|
|
|
"barAlignment": 0,
|
|
|
|
|
"drawStyle": "line",
|
|
|
|
|
"fillOpacity": 10,
|
|
|
|
|
"gradientMode": "none",
|
|
|
|
|
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
|
|
|
|
"insertNulls": false,
|
|
|
|
|
"lineInterpolation": "linear",
|
|
|
|
|
"lineWidth": 1,
|
|
|
|
|
"pointSize": 5,
|
|
|
|
|
"scaleDistribution": { "type": "linear" },
|
|
|
|
|
"showPoints": "never",
|
|
|
|
|
"spanNulls": false,
|
|
|
|
|
"stacking": { "group": "A", "mode": "none" },
|
|
|
|
|
"thresholdsStyle": { "mode": "off" }
|
|
|
|
|
},
|
|
|
|
|
"mappings": [],
|
|
|
|
|
"thresholds": {
|
|
|
|
|
"mode": "absolute",
|
|
|
|
|
"steps": [{ "color": "green", "value": null }]
|
|
|
|
|
},
|
|
|
|
|
"unit": "short"
|
|
|
|
|
},
|
|
|
|
|
"overrides": []
|
|
|
|
|
},
|
|
|
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
|
|
|
|
|
"id": 6,
|
|
|
|
|
"options": {
|
2026-01-16 19:33:02 -08:00
|
|
|
"legend": {
|
|
|
|
|
"calcs": ["mean", "max"],
|
|
|
|
|
"displayMode": "table",
|
|
|
|
|
"placement": "bottom",
|
|
|
|
|
"showLegend": true
|
|
|
|
|
},
|
2026-01-15 12:24:13 -08:00
|
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
|
|
|
},
|
|
|
|
|
"pluginVersion": "10.0.0",
|
|
|
|
|
"targets": [
|
|
|
|
|
{
|
|
|
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
|
|
|
"expr": "rate(loki_distributor_lines_received_total[5m])",
|
|
|
|
|
"legendFormat": "Lines/sec",
|
|
|
|
|
"refId": "A"
|
|
|
|
|
}
|
|
|
|
|
],
|
|
|
|
|
"title": "Log Lines Rate",
|
|
|
|
|
"type": "timeseries"
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"datasource": {
|
|
|
|
|
"type": "prometheus",
|
|
|
|
|
"uid": "prometheus"
|
|
|
|
|
},
|
|
|
|
|
"fieldConfig": {
|
|
|
|
|
"defaults": {
|
|
|
|
|
"color": {
|
|
|
|
|
"mode": "palette-classic"
|
|
|
|
|
},
|
|
|
|
|
"custom": {
|
|
|
|
|
"axisBorderShow": false,
|
|
|
|
|
"axisCenteredZero": false,
|
|
|
|
|
"axisColorMode": "text",
|
|
|
|
|
"axisLabel": "",
|
|
|
|
|
"axisPlacement": "auto",
|
|
|
|
|
"barAlignment": 0,
|
|
|
|
|
"drawStyle": "line",
|
|
|
|
|
"fillOpacity": 10,
|
|
|
|
|
"gradientMode": "none",
|
|
|
|
|
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
|
|
|
|
"insertNulls": false,
|
|
|
|
|
"lineInterpolation": "linear",
|
|
|
|
|
"lineWidth": 1,
|
|
|
|
|
"pointSize": 5,
|
|
|
|
|
"scaleDistribution": { "type": "linear" },
|
|
|
|
|
"showPoints": "never",
|
|
|
|
|
"spanNulls": false,
|
|
|
|
|
"stacking": { "group": "A", "mode": "none" },
|
|
|
|
|
"thresholdsStyle": { "mode": "off" }
|
|
|
|
|
},
|
|
|
|
|
"mappings": [],
|
|
|
|
|
"thresholds": {
|
|
|
|
|
"mode": "absolute",
|
|
|
|
|
"steps": [{ "color": "green", "value": null }]
|
|
|
|
|
},
|
|
|
|
|
"unit": "bytes"
|
|
|
|
|
},
|
|
|
|
|
"overrides": []
|
|
|
|
|
},
|
|
|
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
|
|
|
|
|
"id": 7,
|
|
|
|
|
"options": {
|
2026-01-16 19:33:02 -08:00
|
|
|
"legend": {
|
|
|
|
|
"calcs": ["lastNotNull"],
|
|
|
|
|
"displayMode": "table",
|
|
|
|
|
"placement": "bottom",
|
|
|
|
|
"showLegend": true
|
|
|
|
|
},
|
2026-01-15 12:24:13 -08:00
|
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
|
|
|
},
|
|
|
|
|
"pluginVersion": "10.0.0",
|
|
|
|
|
"targets": [
|
|
|
|
|
{
|
|
|
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
2026-01-15 13:56:11 -08:00
|
|
|
"expr": "sum(loki_ingester_chunk_stored_bytes_total)",
|
|
|
|
|
"legendFormat": "Total Stored",
|
2026-01-15 12:24:13 -08:00
|
|
|
"refId": "A"
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
|
|
|
"expr": "go_memstats_heap_inuse_bytes{job=\"loki\"}",
|
|
|
|
|
"legendFormat": "Heap In Use",
|
|
|
|
|
"refId": "B"
|
|
|
|
|
}
|
|
|
|
|
],
|
2026-01-15 13:56:11 -08:00
|
|
|
"title": "Storage Over Time",
|
2026-01-15 12:24:13 -08:00
|
|
|
"type": "timeseries"
|
|
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
"datasource": {
|
|
|
|
|
"type": "prometheus",
|
|
|
|
|
"uid": "prometheus"
|
|
|
|
|
},
|
|
|
|
|
"fieldConfig": {
|
|
|
|
|
"defaults": {
|
|
|
|
|
"color": {
|
|
|
|
|
"mode": "palette-classic"
|
|
|
|
|
},
|
|
|
|
|
"custom": {
|
|
|
|
|
"axisBorderShow": false,
|
|
|
|
|
"axisCenteredZero": false,
|
|
|
|
|
"axisColorMode": "text",
|
|
|
|
|
"axisLabel": "",
|
|
|
|
|
"axisPlacement": "auto",
|
|
|
|
|
"barAlignment": 0,
|
|
|
|
|
"drawStyle": "line",
|
|
|
|
|
"fillOpacity": 10,
|
|
|
|
|
"gradientMode": "none",
|
|
|
|
|
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
|
|
|
|
"insertNulls": false,
|
|
|
|
|
"lineInterpolation": "linear",
|
|
|
|
|
"lineWidth": 1,
|
|
|
|
|
"pointSize": 5,
|
|
|
|
|
"scaleDistribution": { "type": "linear" },
|
|
|
|
|
"showPoints": "never",
|
|
|
|
|
"spanNulls": false,
|
|
|
|
|
"stacking": { "group": "A", "mode": "none" },
|
|
|
|
|
"thresholdsStyle": { "mode": "off" }
|
|
|
|
|
},
|
|
|
|
|
"mappings": [],
|
|
|
|
|
"thresholds": {
|
|
|
|
|
"mode": "absolute",
|
|
|
|
|
"steps": [{ "color": "green", "value": null }]
|
|
|
|
|
},
|
|
|
|
|
"unit": "short"
|
|
|
|
|
},
|
|
|
|
|
"overrides": []
|
|
|
|
|
},
|
|
|
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
|
|
|
|
|
"id": 8,
|
|
|
|
|
"options": {
|
2026-01-16 19:33:02 -08:00
|
|
|
"legend": {
|
|
|
|
|
"calcs": ["sum"],
|
|
|
|
|
"displayMode": "table",
|
|
|
|
|
"placement": "bottom",
|
|
|
|
|
"showLegend": true
|
|
|
|
|
},
|
2026-01-15 12:24:13 -08:00
|
|
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
|
|
|
|
},
|
|
|
|
|
"pluginVersion": "10.0.0",
|
|
|
|
|
"targets": [
|
|
|
|
|
{
|
|
|
|
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
|
|
|
|
"expr": "increase(loki_ingester_chunk_stored_bytes_total[24h])",
|
|
|
|
|
"legendFormat": "Bytes Stored (24h)",
|
|
|
|
|
"refId": "A"
|
|
|
|
|
}
|
|
|
|
|
],
|
|
|
|
|
"title": "Storage Growth (24h rolling)",
|
|
|
|
|
"type": "timeseries"
|
|
|
|
|
}
|
|
|
|
|
],
|
|
|
|
|
"refresh": "1m",
|
|
|
|
|
"schemaVersion": 38,
|
|
|
|
|
"tags": ["loki", "logs"],
|
|
|
|
|
"templating": {
|
|
|
|
|
"list": []
|
|
|
|
|
},
|
|
|
|
|
"time": {
|
|
|
|
|
"from": "now-24h",
|
|
|
|
|
"to": "now"
|
|
|
|
|
},
|
|
|
|
|
"timepicker": {},
|
|
|
|
|
"timezone": "",
|
|
|
|
|
"title": "Loki",
|
|
|
|
|
"uid": "loki-homelab",
|
|
|
|
|
"version": 1,
|
|
|
|
|
"weekStart": ""
|
|
|
|
|
}
|
