monitoring: Add Grafana Loki HR and Flux logs dashboard

- add loki-stack HelmRelease to install Loki and Promtail in the monitoring namespace
- make the loki-stack HelmRelease depend on kube-prometheus-stack to install Loki's datasource and service monitors in the correct order
- add a Grafana dashboard for displaying and filtering the Flux controllers JSON logs

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>

manifests/monitoring/loki-stack/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: monitoring
+resources:
+  - repository.yaml
+  - release.yaml

manifests/monitoring/loki-stack/release.yaml

@@ -0,0 +1,40 @@
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: loki-stack
+spec:
+  interval: 5m
+  dependsOn:
+    - name: kube-prometheus-stack
+  chart:
+    spec:
+      version: "2.x"
+      chart: loki-stack
+      sourceRef:
+        kind: HelmRepository
+        name: grafana-charts
+      interval: 60m
+  # https://github.com/grafana/helm-charts/blob/main/charts/loki-stack/values.yaml
+  # https://github.com/grafana/helm-charts/blob/main/charts/loki/values.yaml
+  values:
+    grafana:
+      enabled: false
+      sidecar:
+        datasources:
+          enabled: true
+          maxLines: 1000
+    promtail:
+      enabled: true
+    loki:
+      enabled: true
+      isDefault: false
+      serviceMonitor:
+        enabled: true
+        additionalLabels:
+          app.kubernetes.io/part-of: kube-prometheus-stack
+      config:
+        chunk_store_config:
+          max_look_back_period: 0s
+        table_manager:
+          retention_deletes_enabled: true
+          retention_period: 12h

manifests/monitoring/loki-stack/repository.yaml

@@ -0,0 +1,7 @@
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: grafana-charts
+spec:
+  interval: 120m0s
+  url: https://grafana.github.io/helm-charts