Merge pull request #148 from fluxcd/monitoring-stack

Add monitoring stack and dashboards

CONTRIBUTING.md

@@ -18,16 +18,39 @@ organization.
 
 ## Communications
 
-The project uses Slack: To join the conversation, simply join the
+For realtime communications we use Slack: To join the conversation, simply
-[CNCF](https://slack.cncf.io/) Slack workspace and use the
+join the [CNCF](https://slack.cncf.io/) Slack workspace and use the
 [#flux-dev](https://cloud-native.slack.com/messages/flux-dev/) channel.
 
-The developers use a mailing list to discuss development as well.
+To discuss ideas and specifications we use [Github
-Simply subscribe to [flux-dev on cncf.io](https://lists.cncf.io/g/cncf-flux-dev)
+Discussions](https://github.com/fluxcd/toolkit/discussions).
-to join the conversation (this will also add an invitation to your
+
-Google calendar for our [Flux
+For announcements we use a mailing list as well. Simply subscribe to
+[flux-dev on cncf.io](https://lists.cncf.io/g/cncf-flux-dev)
+to join the conversation (there you can also add calendar invites
+to your Google calendar for our [Flux
 meeting](https://docs.google.com/document/d/1l_M0om0qUEN_NNiGgpqJ2tvsF2iioHkaARDeh6b70B0/edit#)).
 
+## Understanding the GitOps Toolkit
+
+If you are entirely new to the GitOps Toolkit,
+you might want to take a look at the [introductory talk and demo](https://www.youtube.com/watch?v=qQBtSkgl7tI).
+
+This project is composed of:
+
+- [/f/toolkit](https://github.com/fluxcd/toolkit): The GitOps Toolkit CLI
+- [/f/source-manager](https://github.com/fluxcd/source-controller): Kubernetes operator for managing sources
+- [/f/kustomize-controller](https://github.com/fluxcd/kustomize-controller): Kubernetes operator for building GitOps pipelines with Kustomize
+- [/f/helm-controller](https://github.com/fluxcd/helm-controller): Kubernetes operator for building GitOps pipelines with Helm
+- [/f/notification-controller](https://github.com/fluxcd/notification-controller): Kubernetes operator for handling inbound and outbound events
+
+### Understanding the code
+
+To get started with developing controllers, you might want to review
+[our guide](https://toolkit.fluxcd.io/dev-guides/source-watcher/) which
+walks you through writing a short and concise controller that watches out
+for source changes.
+
 ### How to run the test suite
 
 You can run the unit tests by simply doing
@@ -66,16 +89,3 @@ For the GitOps Toolkit controllers we prefer the following rules for good commit
 
 The [following article](https://chris.beams.io/posts/git-commit/#seven-rules)
 has some more helpful advice on documenting your work.
-
-## Understanding the GitOps Toolkit
-
-If you are entirely new to the GitOps Toolkit,
-you might want to take a look at the [introductory talk and demo](https://www.youtube.com/watch?v=qQBtSkgl7tI).
-
-This project is composed of:
-
-- [/f/toolkit](https://github.com/fluxcd/toolkit): The GitOps Toolkit CLI
-- [/f/source-manager](https://github.com/fluxcd/source-controller): Kubernetes operator for managing sources
-- [/f/kustomize-controller](https://github.com/fluxcd/kustomize-controller): Kubernetes operator for building GitOps pipelines with Kustomize
-- [/f/helm-controller](https://github.com/fluxcd/helm-controller): Kubernetes operator for building GitOps pipelines with Helm
-- [/f/notification-controller](https://github.com/fluxcd/notification-controller): Kubernetes operator for handling inbound and outbound events

cmd/tk/create_source_helm.go

@@ -49,6 +49,13 @@ For private Helm repositories, the basic authentication credentials are stored i
     --url=https://stefanprodan.github.io/podinfo \
     --username=username \
     --password=password
+
+  # Create a source from a Helm repository using TLS authentication
+  tk create source helm podinfo \
+    --url=https://stefanprodan.github.io/podinfo \
+    --cert-file=./cert.crt \
+    --key-file=./key.crt \
+    --ca-file=./ca.crt
 `,
 	RunE: createSourceHelmCmdRun,
 }
@@ -57,12 +64,18 @@ var (
 	sourceHelmURL      string
 	sourceHelmUsername string
 	sourceHelmPassword string
+	sourceHelmCertFile string
+	sourceHelmKeyFile  string
+	sourceHelmCAFile   string
 )
 
 func init() {
 	createSourceHelmCmd.Flags().StringVar(&sourceHelmURL, "url", "", "Helm repository address")
 	createSourceHelmCmd.Flags().StringVarP(&sourceHelmUsername, "username", "u", "", "basic authentication username")
 	createSourceHelmCmd.Flags().StringVarP(&sourceHelmPassword, "password", "p", "", "basic authentication password")
+	createSourceHelmCmd.Flags().StringVar(&sourceHelmCertFile, "cert-file", "", "TLS authentication cert file path")
+	createSourceHelmCmd.Flags().StringVar(&sourceHelmKeyFile, "key-file", "", "TLS authentication key file path")
+	createSourceHelmCmd.Flags().StringVar(&sourceHelmCAFile, "ca-file", "", "TLS authentication CA file path")
 
 	createSourceCmd.AddCommand(createSourceHelmCmd)
 }
@@ -113,35 +126,52 @@ func createSourceHelmCmdRun(cmd *cobra.Command, args []string) error {
 		return exportHelmRepository(helmRepository)
 	}
 
-	withAuth := false
+	logger.Generatef("generating source")
+
+	secret := corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secretName,
+			Namespace: namespace,
+		},
+		StringData: map[string]string{},
+	}
+
 	if sourceHelmUsername != "" && sourceHelmPassword != "" {
-		logger.Actionf("applying secret with basic auth credentials")
+		secret.StringData["username"] = sourceHelmUsername
-		secret := corev1.Secret{
+		secret.StringData["password"] = sourceHelmPassword
-			ObjectMeta: metav1.ObjectMeta{
+	}
-				Name:      secretName,
+
-				Namespace: namespace,
+	if sourceHelmCertFile != "" && sourceHelmKeyFile != "" {
-			},
+		cert, err := ioutil.ReadFile(sourceHelmCertFile)
-			StringData: map[string]string{
+		if err != nil {
-				"username": sourceHelmUsername,
+			return fmt.Errorf("failed to read repository cert file '%s': %w", sourceHelmCertFile, err)
-				"password": sourceHelmPassword,
-			},
 		}
+		secret.StringData["certFile"] = string(cert)
+
+		key, err := ioutil.ReadFile(sourceHelmKeyFile)
+		if err != nil {
+			return fmt.Errorf("failed to read repository key file '%s': %w", sourceHelmKeyFile, err)
+		}
+		secret.StringData["keyFile"] = string(key)
+	}
+
+	if sourceHelmCAFile != "" {
+		ca, err := ioutil.ReadFile(sourceHelmCAFile)
+		if err != nil {
+			return fmt.Errorf("failed to read repository CA file '%s': %w", sourceHelmCAFile, err)
+		}
+		secret.StringData["caFile"] = string(ca)
+	}
+
+	if len(secret.StringData) > 0 {
+		logger.Actionf("applying secret with repository credentials")
 		if err := upsertSecret(ctx, kubeClient, secret); err != nil {
 			return err
 		}
-		withAuth = true
-	}
-
-	if withAuth {
-		logger.Successf("authentication configured")
-	}
-
-	logger.Generatef("generating source")
-
-	if withAuth {
 		helmRepository.Spec.SecretRef = &corev1.LocalObjectReference{
 			Name: secretName,
 		}
+		logger.Successf("authentication configured")
 	}
 
 	logger.Actionf("applying source")

docs/cmd/tk_create_source_helm.md

@@ -26,15 +26,25 @@ tk create source helm [name] [flags]
     --username=username \
     --password=password
 
+  # Create a source from a Helm repository using TLS authentication
+  tk create source helm podinfo \
+    --url=https://stefanprodan.github.io/podinfo \
+    --cert-file=./cert.crt \
+    --key-file=./key.crt \
+    --ca-file=./ca.crt
+
 ```
 
 ### Options
 
 ```
-  -h, --help              help for helm
+      --ca-file string     TLS authentication CA file path
-  -p, --password string   basic authentication password
+      --cert-file string   TLS authentication cert file path
-      --url string        Helm repository address
+  -h, --help               help for helm
-  -u, --username string   basic authentication username
+      --key-file string    TLS authentication key file path
+  -p, --password string    basic authentication password
+      --url string         Helm repository address
+  -u, --username string    basic authentication username
 ```
 
 ### Options inherited from parent commands

docs/guides/helmreleases.md

@@ -12,7 +12,7 @@ The helm-controller is part of the default toolkit installation.
 To follow this guide you'll need a Kubernetes cluster with the GitOps 
 toolkit controllers installed on it.
 Please see the [get started guide](../get-started/index.md)
-or the [install command docs](../cmd/tk_install.md).
+or the [installation guide](installation.md).
 
 ## Define a Helm repository
 

docs/guides/installation.md

@@ -0,0 +1,357 @@
+# Installation 
+
+This guide walks you through setting up the GitOps Toolkit 
+to manage one or more Kubernetes clusters.
+
+## Prerequisites
+
+You will need a Kubernetes cluster version **1.16** or newer
+and kubectl version **1.18** or newer.
+
+Install the toolkit CLI with:
+
+```sh
+curl -s https://toolkit.fluxcd.io/install.sh | sudo bash
+```
+
+The install script downloads the tk binary to `/usr/local/bin`.
+Binaries for macOS and Linux AMD64 are available for download on the 
+[release page](https://github.com/fluxcd/toolkit/releases).
+
+Verify that your cluster satisfies the prerequisites with:
+
+```sh
+tk check --pre
+```
+
+## Bootstrap
+
+Using the `tk bootstrap` command you can install the toolkit on a Kubernetes cluster 
+and configure it to manage itself from a Git repository.
+
+The bootstrap creates a Git repository if one doesn't exist and
+commits the toolkit components manifests to the master branch.
+Then it configures the target cluster to synchronize with that
+repository by setting up SSH deploy keys.
+
+If the toolkit components are present on the cluster,
+the bootstrap command will perform an upgrade if needed.
+The bootstrap is idempotent, it's safe to run the command as many times as you want.
+
+You can choose what components to install and for which cluster with:
+
+```sh
+tk bootstrap <GIT-PROVIDER> \
+  --components=source-controller,kustomize-controller,helm-controller,notification-controller \
+  --path=my-cluster \
+  --version=latest
+```
+
+If you wish to install a specific version, use the toolkit 
+[release tag](https://github.com/fluxcd/toolkit/releases) e.g. `--version=v0.0.14`.
+
+With `--path` you can configure the directory which will be used to reconcile the target cluster.
+To control multiple clusters from the same Git repository, you have to set a unique path per
+cluster e.g. `staging-cluster` and `production-cluster`:
+
+```sh
+├── staging-cluster # <- path=staging-cluster
+│   └── gitops-system # <- namespace dir generated by bootstrap
+│       ├── toolkit-components.yaml
+│       ├── toolkit-kustomization.yaml
+│       └── toolkit-source.yaml
+└── production-cluster # <- path=production-cluster
+    └── gitops-system
+``` 
+
+### GitHub and GitHub Enterprise
+
+Generate a [personal access token](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line)
+that can create repositories by checking all permissions under `repo`.
+
+Export your GitHub personal access token as an environment variable:
+
+```sh
+export GITHUB_TOKEN=<your-token>
+```
+
+Run the bootstrap for a repository on your personal GitHub account:
+
+```sh
+tk bootstrap github \
+  --owner=my-github-username \
+  --repository=my-repository \
+  --path=my-cluster \
+  --personal
+```
+
+Run the bootstrap for a repository owned by a GitHub organization:
+
+```sh
+tk bootstrap github \
+  --owner=my-github-organization \
+  --repository=my-repository \
+  --team=team1-slug \
+  --team=team2-slug \
+  --path=my-cluster
+```
+
+When you specify a list of teams, those teams will be granted maintainer access to the repository.
+
+To run the bootstrap for a repository hosted on GitHub Enterprise, you have to specify your GitHub hostname:
+
+```sh
+tk bootstrap github \
+  --hostname=my-github-enterprise.com \
+  --owner=my-github-organization \
+  --repository=my-repository \
+  --path=my-cluster
+```
+
+### GitLab and GitLab Enterprise
+
+Generate a [personal access token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html)
+that grants complete read/write access to the GitLab API.
+
+Export your GitLab personal access token as an environment variable:
+
+```sh
+export GITLAB_TOKEN=<your-token>
+```
+
+Run the bootstrap for a repository on your personal GitLab account:
+
+```sh
+tk bootstrap gitlab \
+  --owner=my-gitlab-username \
+  --repository=my-repository \
+  --path=my-cluster \
+  --personal
+```
+
+Run the bootstrap for a repository owned by a GitLab group:
+
+```sh
+tk bootstrap gitlab \
+  --owner=my-gitlab-group \
+  --repository=my-repository \
+  --path=my-cluster
+```
+
+To run the bootstrap for a repository hosted on GitLab on-prem or enterprise, you have to specify your GitLab hostname:
+
+```sh
+tk bootstrap gitlab \
+  --hostname=my-gitlab.com \
+  --owner=my-gitlab-group \
+  --repository=my-repository \
+  --path=my-cluster
+```
+
+### Generic Git Server
+
+For other Git providers such as Bitbucket, Gogs, Gitea, etc you can manually setup the repository and the deploy key.
+
+Create a Git repository and clone it locally:
+
+```sh
+git clone ssh://<host>/<org>/my-repository
+cd my-repository
+```
+
+Create a directory inside the repository:
+
+```sh
+mkdir -p ./my-cluster/gitops-system
+```
+
+Generate the toolkit manifests with:
+
+```sh
+tk install --version=latest \
+  --export > ./my-cluster/gitops-system/toolkit-components.yaml
+```
+
+If your cluster must pull images from a private container registry, first you should pull
+the toolkit images from Docker Hub and push them to your registry, for example:
+
+```sh
+docker pull fluxcd/source-controller:v0.0.7
+docker tag fluxcd/source-controller:v0.0.7 registry.internal/fluxcd/source-controller:v0.0.7
+docker push registry.internal/fluxcd/source-controller:v0.0.7
+```
+
+Create the pull secret in the `gitops-system` namespace:
+
+```sh
+kubectl create ns gitops-system
+
+kubectl -n gitops-system create secret generic regcred \
+    --from-file=.dockerconfigjson=/.docker/config.json \
+    --type=kubernetes.io/dockerconfigjson
+```
+
+Set your registry domain, and the pull secret when generating the manifests:
+
+```sh
+tk install --version=latest \
+  --registry=registry.internal/fluxcd \
+  --image-pull-secret=regcred \
+  --export > ./my-cluster/gitops-system/toolkit-components.yaml
+```
+
+Commit and push the manifest to the master branch:
+
+```sh
+git add -A && git commit -m "add toolkit manifests" && git push
+```
+
+Apply the manifests on your cluster:
+
+```sh
+kubectl apply -f ./my-cluster/gitops-system/toolkit-components.yaml
+```
+
+Verify that the toolkit controllers have started:
+
+```sh
+tk check
+```
+
+Create a `GitRepository` object on your cluster by specifying the SSH address of your repo:
+
+```sh
+tk create source git gitops-system \
+  --url= ssh://<host>/<org>/my-repository \
+  --ssh-key-algorithm=ecdsa \
+  --ssh-ecdsa-curve=p521 \
+  --branch=master \
+  --interval=1m
+```
+
+You will be prompted to add a deploy key to your repository.
+If you don't specify the SSH algorithm, then tk will generate an RSA 2048 bits key.
+
+If your Git server supports basic auth, you can set the URL to HTTPS and specify the credentials with:
+
+```sh
+tk create source git gitops-system \
+  --url=https://<host>/<org>/my-repository \
+  --username=my-username \
+  --password=my-password \
+  --branch=master \
+  --interval=1m
+```
+
+Create a `Kustomization` object on your cluster:
+
+```sh
+tk create kustomization gitops-system \
+  --source=gitops-system \
+  --path="./my-cluster" \
+  --prune=true \
+  --interval=10m
+```
+
+Export both objects, commit and push the manifests to Git:
+
+```sh
+tk export source git gitops-system \
+  > ./my-cluster/gitops-system/toolkit-source.yaml
+
+tk export kustomization gitops-system \
+  > ./my-cluster/gitops-system/toolkit-kustomization.yaml
+
+git add -A && git commit -m "add toolkit reconciliation" && git push
+```
+
+To upgrade the toolkit to a newer version, run the install command and commit the changes:
+
+```sh
+tk install --version=latest \
+  --export > ./my-cluster/gitops-system/toolkit-components.yaml
+
+git add -A && git commit -m "update toolkit" && git push
+```
+
+The source-controller will pull the changes on the cluster, then the kustomize-controller
+will perform a rolling update of all toolkit components including itself.
+
+## Dev install
+
+For testing purposes you can install the toolkit without storing its manifests in a Git repository.
+
+Here is the equivalent to `fluxctl install`:
+
+```sh
+tk install \
+  --components=source-controller,kustomize-controller
+```
+
+Then you can register Git repositories and reconcile them on your cluster:
+
+```sh
+tk create source git podinfo \
+  --url=https://github.com/stefanprodan/podinfo \
+  --tag-semver=">=4.0.0" \
+  --interval=1m
+
+tk create kustomization podinfo-default \
+  --source=podinfo \
+  --path="./kustomize" \
+  --prune=true \
+  --validation=client \
+  --interval=10m \
+  --health-check="Deployment/podinfo.default" \
+  --health-check-timeout=2m
+```
+
+Here is the equivalent to `helm install helm-operator`:
+
+```sh
+tk install \
+  --components=source-controller,kustomize-controller,helm-controller
+```
+
+Then you can register Helm repositories and create Helm releases:
+
+```sh
+tk create source helm stable \
+--interval=1h \
+--url=https://kubernetes-charts.storage.googleapis.com
+
+tk create helmrelease sealed-secrets \
+--interval=1h \
+--release-name=sealed-secrets \
+--target-namespace=gitops-system \
+--source=stable \
+--chart-name=sealed-secrets \
+--chart-version="^1.10.0"
+```
+
+### Monitoring with Prometheus and Grafana
+
+The GitOps Toolkit comes with an optional monitoring stack.
+You can install the stack in the `gitops-system` namespace with:
+
+```yaml
+kustomize build github.com/fluxcd/toolkit/manifests/monitoring?ref=master | kubectl apply -f-
+```
+
+The monitoring stack is composed of:
+
+* Prometheus server - collects metrics from the toolkit controllers and stores them for 2h
+* Grafana dashboards - displays the control plane resource usage and reconciliation stats
+
+![](../_files/cp-dashboard-p1.png)
+
+![](../_files/cp-dashboard-p2.png)
+
+If you wish to use your own Prometheus and Grafana instances, then you can import the dashboards from
+[GitHub](https://github.com/fluxcd/toolkit/tree/master/manifests/monitoring/grafana/dashboards).
+
+!!! hint
+    Note that the toolkit controllers expose the `/metrics` endpoint on port `8080`.
+    When using Prometheus Operator you should create `PodMonitor` objects to configure scraping.
+    When Prometheus is running outside of the `gitops-system` namespace, you have to create a network policy
+    that allows traffic on port `8080` from the namespace where Prometheus is deployed.

docs/guides/notifications.md

@@ -8,7 +8,10 @@ of an app was deployed and if the deployment is healthy.
 
 ## Prerequisites
 
-* [Get started guide](../get-started/index.md)
+To follow this guide you'll need a Kubernetes cluster with the GitOps 
+toolkit controllers installed on it.
+Please see the [get started guide](../get-started/index.md)
+or the [installation guide](installation.md).
 
 The GitOps toolkit controllers emit Kubernetes events whenever a resource status changes.
 You can use the [notification-controller](../components/notification/controller.md)

docs/guides/sealed-secrets.md

@@ -11,7 +11,7 @@ nobody else can obtain the original secret, even if they have access to the Git
 To follow this guide you'll need a Kubernetes cluster with the GitOps 
 toolkit controllers installed on it.
 Please see the [get started guide](../get-started/index.md)
-or the [install command docs](../cmd/tk_install.md).
+or the [installation guide](installation.md).
 
 The sealed-secrets controller comes with a companion CLI tool called kubeseal.
 With kubeseal you can create SealedSecret custom resources in YAML format

docs/guides/webhook-receivers.md

@@ -11,7 +11,7 @@ GitOps pipelines that react to external events.
 To follow this guide you'll need a Kubernetes cluster with the GitOps 
 toolkit controllers installed on it.
 Please see the [get started guide](../get-started/index.md)
-or the [install command docs](../cmd/tk_install.md).
+or the [installation guide](installation.md).
 
 The [notification controller](../components/notification/controller.md)
 can handle events coming from external systems

manifests/bases/notification-controller/kustomization.yaml

@@ -1,5 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- github.com/fluxcd/notification-controller/config//crd?ref=v0.0.6
+- github.com/fluxcd/notification-controller/config//crd?ref=v0.0.7
-- github.com/fluxcd/notification-controller/config//manager?ref=v0.0.6
+- github.com/fluxcd/notification-controller/config//manager?ref=v0.0.7

manifests/monitoring/grafana/datasources.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-datasources
+  namespace: gitops-system
+data:
+  datasources.yaml: |-
+    apiVersion: 1
+    deleteDatasources:
+      - name: prometheus
+    datasources:
+    - name: prometheus
+      type: prometheus
+      access: proxy
+      url: http://prometheus:9090
+      isDefault: true
+      editable: true
+      version: 1

manifests/monitoring/grafana/deployment.yaml

@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grafana
+  labels:
+    app: grafana
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: grafana
+  template:
+    metadata:
+      labels:
+        app: grafana
+      annotations:
+        prometheus.io/scrape: 'false'
+    spec:
+      containers:
+        - name: grafana
+          image: "grafana/grafana:7.1.1"
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 3000
+              protocol: TCP
+          env:
+            - name: GF_PATHS_PROVISIONING
+              value: /etc/grafana/provisioning/
+            - name: GF_AUTH_BASIC_ENABLED
+              value: "false"
+            - name: GF_AUTH_ANONYMOUS_ENABLED
+              value: "true"
+            - name: GF_AUTH_ANONYMOUS_ORG_ROLE
+              value: Admin
+            - name: GF_DEFAULT_THEME
+              value: "Light"
+          volumeMounts:
+            - name: grafana
+              mountPath: /var/lib/grafana
+            - name: dashboards
+              mountPath: /etc/grafana/dashboards
+            - name: datasources
+              mountPath: /etc/grafana/provisioning/datasources
+            - name: providers
+              mountPath: /etc/grafana/provisioning/dashboards
+          resources:
+            {}
+      volumes:
+        - name: grafana
+          emptyDir: {}
+        - name: dashboards
+          configMap:
+            name: grafana-dashboards
+        - name: providers
+          configMap:
+            name: grafana-providers
+        - name: datasources
+          configMap:
+            name: grafana-datasources

manifests/monitoring/grafana/kustomization.yaml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: gitops-system
+resources:
+  - service.yaml
+  - deployment.yaml
+  - providers.yaml
+  - datasources.yaml
+configMapGenerator:
+  - name: grafana-dashboards
+    files:
+      - dashboards/control-plane.json
+

manifests/monitoring/grafana/providers.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-providers
+  namespace: gitops-system
+data:
+  providers.yaml: |+
+    apiVersion: 1
+    providers:
+    - name: 'default'
+      orgId: 1
+      folder: ''
+      type: file
+      disableDeletion: false
+      editable: true
+      options:
+        path: /etc/grafana/dashboards

manifests/monitoring/grafana/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana
+  namespace: gitops-system
+  labels:
+    app: grafana
+spec:
+  type: ClusterIP
+  ports:
+    - port: 3000
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app: grafana

manifests/monitoring/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: gitops-system
+resources:
+- prometheus
+- grafana

manifests/monitoring/prometheus/account.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: prometheus
+  namespace: gitops-system

manifests/monitoring/prometheus/deployment.yaml

@@ -0,0 +1,52 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus
+  namespace: gitops-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: prometheus
+  template:
+    metadata:
+      labels:
+        app: prometheus
+      annotations:
+        appmesh.k8s.aws/sidecarInjectorWebhook: disabled
+        sidecar.istio.io/inject: "false"
+    spec:
+      serviceAccountName: prometheus
+      containers:
+          - name: prometheus
+            image: prom/prometheus:v2.20.0
+            imagePullPolicy: IfNotPresent
+            args:
+              - '--storage.tsdb.retention=2h'
+              - '--config.file=/etc/prometheus/prometheus.yml'
+            ports:
+              - containerPort: 9090
+                name: http
+            livenessProbe:
+              httpGet:
+                path: /-/healthy
+                port: 9090
+            readinessProbe:
+              httpGet:
+                path: /-/ready
+                port: 9090
+            resources:
+              requests:
+                cpu: 10m
+                memory: 128Mi
+            volumeMounts:
+              - name: config-volume
+                mountPath: /etc/prometheus
+              - name: data-volume
+                mountPath: /prometheus/data
+      volumes:
+        - name: config-volume
+          configMap:
+            name: prometheus
+        - name: data-volume
+          emptyDir: {}

manifests/monitoring/prometheus/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: gitops-system
+resources:
+  - account.yaml
+  - rbac.yaml
+  - service.yaml
+  - deployment.yaml
+configMapGenerator:
+  - name: prometheus
+    files:
+      - prometheus.yml

manifests/monitoring/prometheus/prometheus.yml

@@ -0,0 +1,52 @@
+global:
+  scrape_interval: 10s
+scrape_configs:
+
+# Kubernetes API
+- job_name: kubernetes-apiserver
+  kubernetes_sd_configs:
+  - role: endpoints
+    namespaces:
+      names:
+      - default
+  scheme: https
+  tls_config:
+    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    insecure_skip_verify: true
+  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+  relabel_configs:
+  - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
+    action: keep
+    regex: kubernetes;https
+
+# Kubernetes pods
+- job_name: kubernetes-pods
+  kubernetes_sd_configs:
+  - role: pod
+  relabel_configs:
+  - action: keep
+    regex: true
+    source_labels:
+    - __meta_kubernetes_pod_annotation_prometheus_io_scrape
+  - action: replace
+    regex: (.+)
+    source_labels:
+    - __meta_kubernetes_pod_annotation_prometheus_io_path
+    target_label: __metrics_path__
+  - action: replace
+    regex: ([^:]+)(?::\d+)?;(\d+)
+    replacement: $1:$2
+    source_labels:
+    - __address__
+    - __meta_kubernetes_pod_annotation_prometheus_io_port
+    target_label: __address__
+  - action: labelmap
+    regex: __meta_kubernetes_pod_label_(.+)
+  - action: replace
+    source_labels:
+    - __meta_kubernetes_namespace
+    target_label: kubernetes_namespace
+  - action: replace
+    source_labels:
+    - __meta_kubernetes_pod_name
+    target_label: kubernetes_pod_name

manifests/monitoring/prometheus/rbac.yaml

@@ -0,0 +1,32 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: prometheus-gitops-system
+rules:
+  - apiGroups: [""]
+    resources:
+      - nodes
+      - services
+      - endpoints
+      - pods
+      - nodes/proxy
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources:
+      - configmaps
+    verbs: ["get"]
+  - nonResourceURLs: ["/metrics"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus-gitops-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: prometheus-gitops-system
+subjects:
+- kind: ServiceAccount
+  name: prometheus
+  namespace: gitops-system

manifests/monitoring/prometheus/service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus
+  namespace: gitops-system
+spec:
+  selector:
+    app: prometheus
+  ports:
+    - name: http
+      protocol: TCP
+      port: 9090

mkdocs.yml

@@ -45,6 +45,7 @@ nav:
   - Introduction: index.md
   - Get Started: get-started/index.md
   - Guides:
+      - Installation: guides/installation.md
       - Manage Helm Releases: guides/helmreleases.md
       - Setup Notifications: guides/notifications.md
       - Setup Webhook Receivers: guides/webhook-receivers.md