diff --git a/manifests/monitoring/loki-stack/kustomization.yaml b/manifests/monitoring/loki-stack/kustomization.yaml new file mode 100644 index 00000000..d6d43382 --- /dev/null +++ b/manifests/monitoring/loki-stack/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: monitoring +resources: + - repository.yaml + - release.yaml diff --git a/manifests/monitoring/loki-stack/release.yaml b/manifests/monitoring/loki-stack/release.yaml new file mode 100644 index 00000000..fac56914 --- /dev/null +++ b/manifests/monitoring/loki-stack/release.yaml @@ -0,0 +1,40 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: loki-stack +spec: + interval: 5m + dependsOn: + - name: kube-prometheus-stack + chart: + spec: + version: "2.x" + chart: loki-stack + sourceRef: + kind: HelmRepository + name: grafana-charts + interval: 60m + # https://github.com/grafana/helm-charts/blob/main/charts/loki-stack/values.yaml + # https://github.com/grafana/helm-charts/blob/main/charts/loki/values.yaml + values: + grafana: + enabled: false + sidecar: + datasources: + enabled: true + maxLines: 1000 + promtail: + enabled: true + loki: + enabled: true + isDefault: false + serviceMonitor: + enabled: true + additionalLabels: + app.kubernetes.io/part-of: kube-prometheus-stack + config: + chunk_store_config: + max_look_back_period: 0s + table_manager: + retention_deletes_enabled: true + retention_period: 12h diff --git a/manifests/monitoring/loki-stack/repository.yaml b/manifests/monitoring/loki-stack/repository.yaml new file mode 100644 index 00000000..49f58cdd --- /dev/null +++ b/manifests/monitoring/loki-stack/repository.yaml @@ -0,0 +1,7 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: grafana-charts +spec: + interval: 120m0s + url: https://grafana.github.io/helm-charts diff --git a/manifests/monitoring/monitoring-config/dashboards/logs.json b/manifests/monitoring/monitoring-config/dashboards/logs.json new file mode 100644 index 00000000..4d9f58da --- /dev/null +++ b/manifests/monitoring/monitoring-config/dashboards/logs.json @@ -0,0 +1,332 @@ +{ + "__inputs": [ + { + "name": "DS_LOKI", + "label": "Loki", + "description": "", + "type": "datasource", + "pluginId": "loki", + "pluginName": "Loki" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "iconColor": "red", + "name": "flux events", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + "flux" + ], + "type": "tags" + } + } + ] + }, + "description": "Flux logs collected from Kubernetes, stored in Loki", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 29, + "iteration": 1653748775696, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": "${DS_LOKI}", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": "${DS_LOKI}", + "expr": "sum(count_over_time({namespace=~\"$namespace\", stream=~\"$stream\", app =~\"$controller\"} | json | __error__!=\"JSONParserErr\" | level=~\"$level\" |= \"$query\" [$__interval]))", + "instant": false, + "legendFormat": "Log count", + "range": true, + "refId": "A" + } + ], + "type": "timeseries" + }, + { + "datasource": "${DS_LOKI}", + "description": "Logs from services running in Kubernetes", + "gridPos": { + "h": 25, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 2, + "options": { + "dedupStrategy": "numbers", + "enableLogDetails": false, + "prettifyLogMessage": true, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": "${DS_LOKI}", + "expr": "{namespace=~\"$namespace\", stream=~\"$stream\", app =~\"$controller\"} | json | __error__!=\"JSONParserErr\" | level=~\"$level\" |= \"$query\"", + "refId": "A" + } + ], + "type": "logs" + } + ], + "refresh": "10s", + "schemaVersion": 36, + "style": "light", + "tags": [ + "flux" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "description": "String to search for", + "hide": 0, + "label": "Search Query", + "name": "query", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + }, + { + "allValue": "info|error", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "multi": false, + "name": "level", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "info", + "value": "info" + }, + { + "selected": false, + "text": "error", + "value": "error" + } + ], + "query": "info,error", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "${DS_LOKI}", + "definition": "label_values(app)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "controller", + "options": [], + "query": "label_values(app)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "flux-system" + ], + "value": [ + "flux-system" + ] + }, + "datasource": "${DS_LOKI}", + "definition": "label_values(namespace)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "namespace", + "options": [], + "query": "label_values(namespace)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "${DS_LOKI}", + "definition": "label_values(stream)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "stream", + "options": [], + "query": "label_values(stream)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Loki", + "value": "Loki" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "DS_LOKI", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Flux Logs", + "uid": "flux-logs", + "version": 2 +} diff --git a/manifests/monitoring/monitoring-config/kustomization.yaml b/manifests/monitoring/monitoring-config/kustomization.yaml index 9f036a57..f35f7dbb 100644 --- a/manifests/monitoring/monitoring-config/kustomization.yaml +++ b/manifests/monitoring/monitoring-config/kustomization.yaml @@ -8,6 +8,7 @@ configMapGenerator: files: - dashboards/control-plane.json - dashboards/cluster.json + - dashboards/logs.json options: labels: grafana_dashboard: "1"