Publish manifests as release assets

Add export option to tk install

.github/workflows/release.yaml

@@ -14,7 +14,7 @@ jobs:
       - name: Unshallow
         run: git fetch --prune --unshallow
       - name: Setup Go
-        uses: actions/setup-go@v2-beta
+        uses: actions/setup-go@v2
         with:
           go-version: 1.14.x
       - name: Download release notes utility
@@ -25,10 +25,60 @@ jobs:
         run: |
           echo 'CHANGELOG' > /tmp/release.txt
           github-release-notes -org fluxcd -repo toolkit -since-latest-release >> /tmp/release.txt
+      - name: Setup Kustomize
+        uses: ./.github/actions/kustomize
+      - name: Generate manifests tarball
+        run: |
+          mkdir -p ./output
+          files=""
+
+          # build controllers
+          for controller in ./manifests/bases/*/; do
+              output_path="./output/$(basename $controller).yaml"
+              echo "building $controller to $output_path"
+
+              kustomize build $controller > $output_path
+              files+=" $(basename $output_path)"
+          done
+
+          # build rbac
+          rbac_path="./manifests/rbac"
+          rbac_output_path="./output/rbac.yaml"
+          echo "building $rbac_path to $rbac_output_path"
+          kustomize build $rbac_path > $rbac_output_path
+          files+=" $(basename $rbac_output_path)"
+
+          # build policies
+          policies_path="./manifests/policies"
+          policies_output_path="./output/policies.yaml"
+          echo "building $policies_path to $policies_output_path"
+          kustomize build $policies_path > $policies_output_path
+          files+=" $(basename $policies_output_path)"
+
+          # create tarball
+          cd ./output && tar -cvzf manifests.tar.gz $files
+      - name: Create release
+        id: create_release
+        uses: actions/create-release@latest
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: ${{ github.ref }}
+      - name: Upload artifacts
+        id: upload-release-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./output/manifests.tar.gz
+          asset_name: manifests.tar.gz
+          asset_content_type: application/gzip
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v1
         with:
           version: latest
-          args: release --release-notes=/tmp/release.txt
+          args: release --release-notes=/tmp/release.txt --skip-validate
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -14,3 +14,4 @@
 # Dependency directories (remove the comment below to include it)
 # vendor/
 bin/
+output/

cmd/tk/bootstrap.go

@@ -141,7 +141,6 @@ func generateSyncManifests(url, name, namespace, targetPath, tmpDir string, inte
 	}
 
 	gvk = kustomizev1.GroupVersion.WithKind("Kustomization")
-	emptyAPIGroup := ""
 	kustomization := kustomizev1.Kustomization{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       gvk.Kind,
@@ -157,9 +156,8 @@ func generateSyncManifests(url, name, namespace, targetPath, tmpDir string, inte
 			},
 			Path:  fmt.Sprintf("./%s", strings.TrimPrefix(targetPath, "./")),
 			Prune: true,
-			SourceRef: corev1.TypedLocalObjectReference{
+			SourceRef: kustomizev1.CrossNamespaceObjectReference{
-				APIGroup: &emptyAPIGroup,
+				Kind: sourcev1.GitRepositoryKind,
-				Kind:     "GitRepository",
 				Name: name,
 			},
 		},

cmd/tk/create_kustomization.go

@@ -125,7 +125,6 @@ func createKsCmdRun(cmd *cobra.Command, args []string) error {
 		logger.Generatef("generating kustomization")
 	}
 
-	emptyAPIGroup := ""
 	kustomization := kustomizev1.Kustomization{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
@@ -138,9 +137,8 @@ func createKsCmdRun(cmd *cobra.Command, args []string) error {
 			},
 			Path:  ksPath,
 			Prune: ksPrune,
-			SourceRef: corev1.TypedLocalObjectReference{
+			SourceRef: kustomizev1.CrossNamespaceObjectReference{
-				APIGroup: &emptyAPIGroup,
+				Kind: sourcev1.GitRepositoryKind,
-				Kind:     "GitRepository",
 				Name: ksSource,
 			},
 			Suspend:    false,

cmd/tk/install.go

@@ -24,6 +24,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/spf13/cobra"
 	"sigs.k8s.io/kustomize/api/filesys"
@@ -43,11 +44,15 @@ If a previous version is installed, then an in-place upgrade will be performed.`
 
   # Dry-run install with manifests preview 
   tk install --dry-run --verbose
+
+  # Write install manifests to file 
+  tk install --export > gitops-system.yaml
 `,
 	RunE: installCmdRun,
 }
 
 var (
+	installExport        bool
 	installDryRun        bool
 	installManifestsPath string
 	installVersion       string
@@ -55,6 +60,8 @@ var (
 )
 
 func init() {
+	installCmd.Flags().BoolVar(&installExport, "export", false,
+		"write the install manifests to stdout and exit")
 	installCmd.Flags().BoolVarP(&installDryRun, "dry-run", "", false,
 		"only print the object that would be applied")
 	installCmd.Flags().StringVarP(&installVersion, "version", "v", defaultVersion,
@@ -84,7 +91,9 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
 	}
 	defer os.RemoveAll(tmpDir)
 
+	if !installExport {
 		logger.Generatef("generating manifests")
+	}
 	if kustomizePath == "" {
 		err = genInstallManifests(installVersion, namespace, installComponents, tmpDir)
 		if err != nil {
@@ -104,6 +113,12 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
 	} else {
 		if verbose {
 			fmt.Print(yaml)
+		} else if installExport {
+			fmt.Println("---")
+			fmt.Println("# GitOps Toolkit revision", installVersion, time.Now().Format(time.RFC3339))
+			fmt.Print(yaml)
+			fmt.Println("---")
+			return nil
 		}
 	}
 	logger.Successf("manifests build completed")

docs/cmd/tk_install.md

@@ -23,6 +23,9 @@ tk install [flags]
   # Dry-run install with manifests preview 
   tk install --dry-run --verbose
 
+  # Write install manifests to file 
+  tk install --export > gitops-system.yaml
+
 ```
 
 ### Options
@@ -30,6 +33,7 @@ tk install [flags]
 ```
       --components strings   list of components, accepts comma-separated values (default [source-controller,kustomize-controller,helm-controller,notification-controller])
       --dry-run              only print the object that would be applied
+      --export               write the install manifests to stdout and exit
   -h, --help                 help for install
       --manifests string     path to the manifest directory, dev only
   -v, --version string       toolkit tag or branch (default "master")

docs/guides/helmreleases.md

@@ -1,4 +1,4 @@
-# Managing Helm releases
+# Manage Helm Releases
 
 The [helm-controller](../components/helm/controller.md) allows you to
 declaratively manage Helm chart releases with Kubernetes manifests.
@@ -92,3 +92,101 @@ helm-controller.
     for finer grain control over how Helm actions are performed.
     See the [`HelmRelease` CRD docs](../components/helm/helmreleases.md)
     for more details.
+
+## Configure notifications
+
+The default toolkit installation configures the helm-controller to
+broadcast events to the [notification-controller](../components/notification/controller.md).
+
+To receive the events as notifications, a `Provider` needs to be setup
+first as described in the [notifications guide](notifications.md#define-a-provider).
+Once you have set up the `Provider`, create a new `Alert` resource in
+the `gitops-system` to start receiving notifications about the Helm
+release:
+
+```yaml
+apiVersion: notification.fluxcd.io/v1alpha1
+  kind: Alert
+  metadata:
+    generation: 2
+    name: helm-podinfo
+    namespace: gitops-system
+  spec:
+    providerRef:
+      name: slack
+    eventSeverity: info
+    eventSources:
+    - kind: HelmRepository
+      name: podinfo
+    - kind: HelmChart
+      name: default-podinfo
+    - kind: HelmRelease
+      name: podinfo
+      namespace: default
+```
+
+![helm-controller alerts](../diagrams/helm-controller-alerts.png)
+
+## Configure webhook receivers
+
+When using semver ranges for Helm releases, you may want to trigger an update
+as soon as a new chart version is published to your Helm repository.
+In order to notify source-controller about a chart update,
+you can [setup webhook receivers](webhook-receivers.md).
+
+First generate a random string and create a secret with a `token` field:
+
+```sh
+TOKEN=$(head -c 12 /dev/urandom | shasum | cut -d ' ' -f1)
+echo $TOKEN
+
+kubectl -n gitops-system create secret generic webhook-token \	
+--from-literal=token=$TOKEN
+```
+
+When using [Harbor](https://goharbor.io/) as your Helm repository, you can define a receiver with:
+
+```yaml
+apiVersion: notification.fluxcd.io/v1alpha1
+kind: Receiver
+metadata:
+  name: helm-podinfo
+  namespace: gitops-system
+spec:
+  type: harbor
+  secretRef:
+    name: webhook-token
+  resources:
+    - kind: HelmRepository
+      name: podinfo
+```
+
+The notification-controller generates a unique URL using the provided token and the receiver name/namespace.
+
+Find the URL with:
+
+```console
+$ kubectl -n gitops-system get receiver/helm-podinfo
+
+NAME           READY   STATUS
+helm-podinfo   True    Receiver initialised with URL: /hook/bed6d00b5555b1603e1f59b94d7fdbca58089cb5663633fb83f2815dc626d92b
+```
+
+Log in to the Harbor interface, go to Projects, select a project, and select Webhooks.
+Fill the form with:
+
+* Endpoint URL: compose the address using the receiver LB and the generated URL `http://<LoadBalancerAddress>/<ReceiverURL>`
+* Auth Header: use the `token` string
+
+With the above settings, when you upload a chart, the following happens:
+
+* Harbor sends the chart push event to the receiver address
+* Notification controller validates the authenticity of the payload using the auth header
+* Source controller is notified about the changes
+* Source controller pulls the changes into the cluster and updates the `HelmChart` version
+* Helm controller is notified about the version change and upgrades the release
+
+!!! hint "Note"
+    Besides Harbor, you can define receivers for **GitHub**, **GitLab**, **Bitbucket**
+    and any other system that supports webhooks e.g. Jenkins, CircleCI, etc.
+    See the [Receiver CRD docs](../components/notification/receiver.md) for more details.

go.mod

@@ -4,8 +4,8 @@ go 1.14
 
 require (
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/fluxcd/helm-controller v0.0.1-alpha.1
+	github.com/fluxcd/helm-controller v0.0.1-alpha.2
-	github.com/fluxcd/kustomize-controller v0.0.3
+	github.com/fluxcd/kustomize-controller v0.0.4
 	github.com/fluxcd/pkg v0.0.3
 	github.com/fluxcd/source-controller v0.0.5
 	github.com/golang/protobuf v1.4.2 // indirect
@@ -20,7 +20,7 @@ require (
 	k8s.io/apimachinery v0.18.4
 	k8s.io/client-go v0.18.4
 	sigs.k8s.io/controller-runtime v0.6.0
-	sigs.k8s.io/kustomize/api v0.5.0
+	sigs.k8s.io/kustomize/api v0.5.1
 	sigs.k8s.io/yaml v1.2.0
 )
 

go.sum

@@ -170,10 +170,10 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwC
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fluxcd/helm-controller v0.0.1-alpha.1 h1:stbyNQdAhHqjL2i+Oi85A5cceOnatt20/fcNc0taeHA=
+github.com/fluxcd/helm-controller v0.0.1-alpha.2 h1:X5ycFeobLnv8EHVWC8hvw0fT6QZhbmM8y2ShM1XF4mk=
-github.com/fluxcd/helm-controller v0.0.1-alpha.1/go.mod h1:ya+BGrB1Md5DaxQIQmXwf6qNaydZZoa3EZJGpqFFu/Y=
+github.com/fluxcd/helm-controller v0.0.1-alpha.2/go.mod h1:ya+BGrB1Md5DaxQIQmXwf6qNaydZZoa3EZJGpqFFu/Y=
-github.com/fluxcd/kustomize-controller v0.0.3 h1:oJJo+GNMfmuxg9M/TXkNui2AtV+oxaX/ZNI+33AuJHU=
+github.com/fluxcd/kustomize-controller v0.0.4 h1:QZNygZXWeM5nOn0iRmzOaaoXvQuZ+YZhx+QgZjuX2z0=
-github.com/fluxcd/kustomize-controller v0.0.3/go.mod h1:mOBo09pl4BKZP0DPFoijgsjReMNFLSznYMuKbwqqpOU=
+github.com/fluxcd/kustomize-controller v0.0.4/go.mod h1:JU62CExAWDlITSfLabTe7ilsq1DnJB3yaffv80nYlVs=
 github.com/fluxcd/pkg v0.0.3 h1:yhjtpGtD9LxFo8JtwTuUxJyFcX2wSSb0TPptIEpGSmA=
 github.com/fluxcd/pkg v0.0.3/go.mod h1:rtlppQU+9DNikyDZptLdOeTf+wBvQQiQQ/J113FPoeU=
 github.com/fluxcd/source-controller v0.0.5 h1:flUhdZh00/+jzqf9/sSRUMrewp+VlNFDITnoQC7pO4A=
@@ -1002,10 +1002,10 @@ sigs.k8s.io/controller-runtime v0.6.0 h1:Fzna3DY7c4BIP6KwfSlrfnj20DJ+SeMBK8HSFvO
 sigs.k8s.io/controller-runtime v0.6.0/go.mod h1:CpYf5pdNY/B352A1TFLAS2JVSlnGQ5O2cftPHndTroo=
 sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0=
 sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU=
-sigs.k8s.io/kustomize/api v0.5.0 h1:LT/Yxm6J8M9BJoLxfYIwGWamvuGSPQl7ELQXdlOR/fY=
+sigs.k8s.io/kustomize/api v0.5.1 h1:iHGTs5LcnJGqHstUSxWD/kX6XZgmd82x79LLlZwDU0I=
-sigs.k8s.io/kustomize/api v0.5.0/go.mod h1:OND6MoJxF5BoS3IiAqDjPKdI7c+oTSaxuXKbljPbwNU=
+sigs.k8s.io/kustomize/api v0.5.1/go.mod h1:LGqJ9ZWOnWDqlECqrFgNUyEqSJc6ooA9ZiWZ4KFZv+I=
-sigs.k8s.io/kustomize/kyaml v0.3.4 h1:RhxnabYltv4mdD5+I7pIaJtae+eaTn4TiZqPT7K+C7A=
+sigs.k8s.io/kustomize/kyaml v0.4.1 h1:NEqA/35upoAjb+I5vh1ODUqxoX4DOrezeQa9BhhG5Co=
-sigs.k8s.io/kustomize/kyaml v0.3.4/go.mod h1:XJL84E6sOFeNrQ7CADiemc1B0EjIxHo3OhW4o1aJYNw=
+sigs.k8s.io/kustomize/kyaml v0.4.1/go.mod h1:XJL84E6sOFeNrQ7CADiemc1B0EjIxHo3OhW4o1aJYNw=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=

manifests/bases/helm-controller/kustomization.yaml

@@ -1,8 +1,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- github.com/fluxcd/helm-controller/config//crd?ref=v0.0.1-alpha.1
+- github.com/fluxcd/helm-controller/config//crd?ref=v0.0.1-alpha.2
-- github.com/fluxcd/helm-controller/config//manager?ref=v0.0.1-alpha.1
+- github.com/fluxcd/helm-controller/config//manager?ref=v0.0.1-alpha.2
 patchesJson6902:
 - target:
     group: apps

manifests/bases/kustomize-controller/kustomization.yaml

@@ -1,8 +1,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- github.com/fluxcd/kustomize-controller/config//crd?ref=v0.0.2
+- github.com/fluxcd/kustomize-controller/config//crd?ref=v0.0.4
-- github.com/fluxcd/kustomize-controller/config//manager?ref=v0.0.2
+- github.com/fluxcd/kustomize-controller/config//manager?ref=v0.0.4
 patchesJson6902:
 - target:
     group: apps

manifests/bases/notification-controller/kustomization.yaml

@@ -1,5 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- github.com/fluxcd/notification-controller/config//crd?ref=v0.0.3
+- github.com/fluxcd/notification-controller/config//crd?ref=v0.0.4
-- github.com/fluxcd/notification-controller/config//manager?ref=v0.0.3
+- github.com/fluxcd/notification-controller/config//manager?ref=v0.0.4

manifests/bases/source-controller/kustomization.yaml

@@ -1,8 +1,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- github.com/fluxcd/source-controller/config//crd?ref=v0.0.2
+- github.com/fluxcd/source-controller/config//crd?ref=v0.0.5
-- github.com/fluxcd/source-controller/config//manager?ref=v0.0.2
+- github.com/fluxcd/source-controller/config//manager?ref=v0.0.5
 patchesJson6902:
 - target:
     group: apps

mkdocs.yml

@@ -40,7 +40,7 @@ nav:
   - Introduction: index.md
   - Get Started: get-started/index.md
   - Guides:
-      - Managing Helm releases: guides/helmreleases.md
+      - Manage Helm Releases: guides/helmreleases.md
       - Setup Notifications: guides/notifications.md
       - Setup Webhook Receivers: guides/webhook-receivers.md
   - Toolkit Components: