Merge pull request #148 from fluxcd/monitoring-stack

Add monitoring stack and dashboards

docs/guides/installation.md

@@ -328,3 +328,30 @@ tk create helmrelease sealed-secrets \
 --chart-name=sealed-secrets \
 --chart-version="^1.10.0"
 ```
+
+### Monitoring with Prometheus and Grafana
+
+The GitOps Toolkit comes with an optional monitoring stack.
+You can install the stack in the `gitops-system` namespace with:
+
+```yaml
+kustomize build github.com/fluxcd/toolkit/manifests/monitoring?ref=master | kubectl apply -f-
+```
+
+The monitoring stack is composed of:
+
+* Prometheus server - collects metrics from the toolkit controllers and stores them for 2h
+* Grafana dashboards - displays the control plane resource usage and reconciliation stats
+
+![](../_files/cp-dashboard-p1.png)
+
+![](../_files/cp-dashboard-p2.png)
+
+If you wish to use your own Prometheus and Grafana instances, then you can import the dashboards from
+[GitHub](https://github.com/fluxcd/toolkit/tree/master/manifests/monitoring/grafana/dashboards).
+
+!!! hint
+    Note that the toolkit controllers expose the `/metrics` endpoint on port `8080`.
+    When using Prometheus Operator you should create `PodMonitor` objects to configure scraping.
+    When Prometheus is running outside of the `gitops-system` namespace, you have to create a network policy
+    that allows traffic on port `8080` from the namespace where Prometheus is deployed.

manifests/monitoring/grafana/datasources.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-datasources
+  namespace: gitops-system
+data:
+  datasources.yaml: |-
+    apiVersion: 1
+    deleteDatasources:
+      - name: prometheus
+    datasources:
+    - name: prometheus
+      type: prometheus
+      access: proxy
+      url: http://prometheus:9090
+      isDefault: true
+      editable: true
+      version: 1

manifests/monitoring/grafana/deployment.yaml

@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grafana
+  labels:
+    app: grafana
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: grafana
+  template:
+    metadata:
+      labels:
+        app: grafana
+      annotations:
+        prometheus.io/scrape: 'false'
+    spec:
+      containers:
+        - name: grafana
+          image: "grafana/grafana:7.1.1"
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 3000
+              protocol: TCP
+          env:
+            - name: GF_PATHS_PROVISIONING
+              value: /etc/grafana/provisioning/
+            - name: GF_AUTH_BASIC_ENABLED
+              value: "false"
+            - name: GF_AUTH_ANONYMOUS_ENABLED
+              value: "true"
+            - name: GF_AUTH_ANONYMOUS_ORG_ROLE
+              value: Admin
+            - name: GF_DEFAULT_THEME
+              value: "Light"
+          volumeMounts:
+            - name: grafana
+              mountPath: /var/lib/grafana
+            - name: dashboards
+              mountPath: /etc/grafana/dashboards
+            - name: datasources
+              mountPath: /etc/grafana/provisioning/datasources
+            - name: providers
+              mountPath: /etc/grafana/provisioning/dashboards
+          resources:
+            {}
+      volumes:
+        - name: grafana
+          emptyDir: {}
+        - name: dashboards
+          configMap:
+            name: grafana-dashboards
+        - name: providers
+          configMap:
+            name: grafana-providers
+        - name: datasources
+          configMap:
+            name: grafana-datasources

manifests/monitoring/grafana/kustomization.yaml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: gitops-system
+resources:
+  - service.yaml
+  - deployment.yaml
+  - providers.yaml
+  - datasources.yaml
+configMapGenerator:
+  - name: grafana-dashboards
+    files:
+      - dashboards/control-plane.json
+

manifests/monitoring/grafana/providers.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-providers
+  namespace: gitops-system
+data:
+  providers.yaml: |+
+    apiVersion: 1
+    providers:
+    - name: 'default'
+      orgId: 1
+      folder: ''
+      type: file
+      disableDeletion: false
+      editable: true
+      options:
+        path: /etc/grafana/dashboards

manifests/monitoring/grafana/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana
+  namespace: gitops-system
+  labels:
+    app: grafana
+spec:
+  type: ClusterIP
+  ports:
+    - port: 3000
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app: grafana

manifests/monitoring/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: gitops-system
+resources:
+- prometheus
+- grafana

manifests/monitoring/prometheus/account.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus
+  namespace: gitops-system

manifests/monitoring/prometheus/deployment.yaml

@@ -0,0 +1,52 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus
+  namespace: gitops-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: prometheus
+  template:
+    metadata:
+      labels:
+        app: prometheus
+      annotations:
+        appmesh.k8s.aws/sidecarInjectorWebhook: disabled
+        sidecar.istio.io/inject: "false"
+    spec:
+      serviceAccountName: prometheus
+      containers:
+          - name: prometheus
+            image: prom/prometheus:v2.20.0
+            imagePullPolicy: IfNotPresent
+            args:
+              - '--storage.tsdb.retention=2h'
+              - '--config.file=/etc/prometheus/prometheus.yml'
+            ports:
+              - containerPort: 9090
+                name: http
+            livenessProbe:
+              httpGet:
+                path: /-/healthy
+                port: 9090
+            readinessProbe:
+              httpGet:
+                path: /-/ready
+                port: 9090
+            resources:
+              requests:
+                cpu: 10m
+                memory: 128Mi
+            volumeMounts:
+              - name: config-volume
+                mountPath: /etc/prometheus
+              - name: data-volume
+                mountPath: /prometheus/data
+      volumes:
+        - name: config-volume
+          configMap:
+            name: prometheus
+        - name: data-volume
+          emptyDir: {}

manifests/monitoring/prometheus/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: gitops-system
+resources:
+  - account.yaml
+  - rbac.yaml
+  - service.yaml
+  - deployment.yaml
+configMapGenerator:
+  - name: prometheus
+    files:
+      - prometheus.yml

manifests/monitoring/prometheus/prometheus.yml

@@ -0,0 +1,52 @@
+global:
+  scrape_interval: 10s
+scrape_configs:
+
+# Kubernetes API
+- job_name: kubernetes-apiserver
+  kubernetes_sd_configs:
+  - role: endpoints
+    namespaces:
+      names:
+      - default
+  scheme: https
+  tls_config:
+    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    insecure_skip_verify: true
+  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+  relabel_configs:
+  - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
+    action: keep
+    regex: kubernetes;https
+
+# Kubernetes pods
+- job_name: kubernetes-pods
+  kubernetes_sd_configs:
+  - role: pod
+  relabel_configs:
+  - action: keep
+    regex: true
+    source_labels:
+    - __meta_kubernetes_pod_annotation_prometheus_io_scrape
+  - action: replace
+    regex: (.+)
+    source_labels:
+    - __meta_kubernetes_pod_annotation_prometheus_io_path
+    target_label: __metrics_path__
+  - action: replace
+    regex: ([^:]+)(?::\d+)?;(\d+)
+    replacement: $1:$2
+    source_labels:
+    - __address__
+    - __meta_kubernetes_pod_annotation_prometheus_io_port
+    target_label: __address__
+  - action: labelmap
+    regex: __meta_kubernetes_pod_label_(.+)
+  - action: replace
+    source_labels:
+    - __meta_kubernetes_namespace
+    target_label: kubernetes_namespace
+  - action: replace
+    source_labels:
+    - __meta_kubernetes_pod_name
+    target_label: kubernetes_pod_name

manifests/monitoring/prometheus/rbac.yaml

@@ -0,0 +1,32 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus-gitops-system
+rules:
+  - apiGroups: [""]
+    resources:
+      - nodes
+      - services
+      - endpoints
+      - pods
+      - nodes/proxy
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources:
+      - configmaps
+    verbs: ["get"]
+  - nonResourceURLs: ["/metrics"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus-gitops-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus-gitops-system
+subjects:
+- kind: ServiceAccount
+  name: prometheus
+  namespace: gitops-system

manifests/monitoring/prometheus/service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus
+  namespace: gitops-system
+spec:
+  selector:
+    app: prometheus
+  ports:
+    - name: http
+      protocol: TCP
+      port: 9090