Merge pull request #108 from fluxcd/link-to-image-update-discussion

Link to image update discussion

.github/workflows/release.yaml

@@ -14,7 +14,7 @@ jobs:
       - name: Unshallow
         run: git fetch --prune --unshallow
       - name: Setup Go
-        uses: actions/setup-go@v2-beta
+        uses: actions/setup-go@v2
         with:
           go-version: 1.14.x
       - name: Download release notes utility
@@ -25,10 +25,60 @@ jobs:
         run: |
           echo 'CHANGELOG' > /tmp/release.txt
           github-release-notes -org fluxcd -repo toolkit -since-latest-release >> /tmp/release.txt
+      - name: Setup Kustomize
+        uses: ./.github/actions/kustomize
+      - name: Generate manifests tarball
+        run: |
+          mkdir -p ./output
+          files=""
+
+          # build controllers
+          for controller in ./manifests/bases/*/; do
+              output_path="./output/$(basename $controller).yaml"
+              echo "building $controller to $output_path"
+
+              kustomize build $controller > $output_path
+              files+=" $(basename $output_path)"
+          done
+
+          # build rbac
+          rbac_path="./manifests/rbac"
+          rbac_output_path="./output/rbac.yaml"
+          echo "building $rbac_path to $rbac_output_path"
+          kustomize build $rbac_path > $rbac_output_path
+          files+=" $(basename $rbac_output_path)"
+
+          # build policies
+          policies_path="./manifests/policies"
+          policies_output_path="./output/policies.yaml"
+          echo "building $policies_path to $policies_output_path"
+          kustomize build $policies_path > $policies_output_path
+          files+=" $(basename $policies_output_path)"
+
+          # create tarball
+          cd ./output && tar -cvzf manifests.tar.gz $files
+      - name: Create release
+        id: create_release
+        uses: actions/create-release@latest
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: ${{ github.ref }}
+      - name: Upload artifacts
+        id: upload-release-asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./output/manifests.tar.gz
+          asset_name: manifests.tar.gz
+          asset_content_type: application/gzip
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v1
         with:
           version: latest
-          args: release --release-notes=/tmp/release.txt
+          args: release --release-notes=/tmp/release.txt --skip-validate
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -13,4 +13,5 @@
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
-bin/
+bin/
+output/

cmd/tk/bootstrap.go

@@ -58,7 +58,7 @@ const (
 
 func init() {
 	bootstrapCmd.PersistentFlags().StringVarP(&bootstrapVersion, "version", "v", defaultVersion,
-		"toolkit tag or branch")
+		"toolkit version")
 	bootstrapCmd.PersistentFlags().StringSliceVar(&bootstrapComponents, "components", defaultComponents,
 		"list of components, accepts comma-separated values")
 

cmd/tk/install.go

@@ -19,11 +19,14 @@ package main
 import (
 	"context"
 	"fmt"
+	"github.com/fluxcd/pkg/untar"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/spf13/cobra"
 	"sigs.k8s.io/kustomize/api/filesys"
@@ -36,18 +39,22 @@ var installCmd = &cobra.Command{
 	Long: `The install command deploys the toolkit components in the specified namespace.
 If a previous version is installed, then an in-place upgrade will be performed.`,
 	Example: `  # Install the latest version in the gitops-systems namespace
-  tk install --version=master --namespace=gitops-systems
+  tk install --version=latest --namespace=gitops-systems
 
   # Dry-run install for a specific version and a series of components
-  tk install --dry-run --version=0.0.1 --components="source-controller,kustomize-controller"
+  tk install --dry-run --version=v0.0.7 --components="source-controller,kustomize-controller"
 
   # Dry-run install with manifests preview 
   tk install --dry-run --verbose
+
+  # Write install manifests to file 
+  tk install --export > gitops-system.yaml
 `,
 	RunE: installCmdRun,
 }
 
 var (
+	installExport        bool
 	installDryRun        bool
 	installManifestsPath string
 	installVersion       string
@@ -55,10 +62,12 @@ var (
 )
 
 func init() {
+	installCmd.Flags().BoolVar(&installExport, "export", false,
+		"write the install manifests to stdout and exit")
 	installCmd.Flags().BoolVarP(&installDryRun, "dry-run", "", false,
 		"only print the object that would be applied")
 	installCmd.Flags().StringVarP(&installVersion, "version", "v", defaultVersion,
-		"toolkit tag or branch")
+		"toolkit version")
 	installCmd.Flags().StringSliceVar(&installComponents, "components", defaultComponents,
 		"list of components, accepts comma-separated values")
 	installCmd.Flags().StringVarP(&installManifestsPath, "manifests", "", "",
@@ -84,7 +93,9 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
 	}
 	defer os.RemoveAll(tmpDir)
 
-	logger.Generatef("generating manifests")
+	if !installExport {
+		logger.Generatef("generating manifests")
+	}
 	if kustomizePath == "" {
 		err = genInstallManifests(installVersion, namespace, installComponents, tmpDir)
 		if err != nil {
@@ -104,6 +115,12 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
 	} else {
 		if verbose {
 			fmt.Print(yaml)
+		} else if installExport {
+			fmt.Println("---")
+			fmt.Println("# GitOps Toolkit revision", installVersion, time.Now().Format(time.RFC3339))
+			fmt.Print(yaml)
+			fmt.Println("---")
+			return nil
 		}
 	}
 	logger.Successf("manifests build completed")
@@ -174,10 +191,10 @@ transformers:
   - labels.yaml
 resources:
   - namespace.yaml
+  - policies.yaml
   - roles
-  - github.com/fluxcd/toolkit/manifests/policies?ref={{$version}}
 {{- range .Components }}
-  - github.com/fluxcd/toolkit/manifests/bases/{{.}}?ref={{$version}}
+  - {{.}}.yaml
 {{- end }}
 `
 
@@ -185,10 +202,44 @@ var kustomizationRolesTmpl = `---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - github.com/fluxcd/toolkit/manifests/rbac?ref={{.Version}}
+  - rbac.yaml
 nameSuffix: -{{.Namespace}}
 `
 
+func downloadManifests(version string, tmpDir string) error {
+	ghURL := "https://github.com/fluxcd/toolkit/releases/latest/download/manifests.tar.gz"
+	if strings.HasPrefix(version, "v") {
+		ghURL = fmt.Sprintf("https://github.com/fluxcd/toolkit/releases/download/%s/manifests.tar.gz", version)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	req, err := http.NewRequest("GET", ghURL, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create HTTP request for %s, error: %w", ghURL, err)
+	}
+
+	// download
+	resp, err := http.DefaultClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return fmt.Errorf("failed to download artifact from %s, error: %w", ghURL, err)
+	}
+	defer resp.Body.Close()
+
+	// check response
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("faild to download artifact from %s, status: %s", ghURL, resp.Status)
+	}
+
+	// extract
+	if _, err = untar.Untar(resp.Body, tmpDir); err != nil {
+		return fmt.Errorf("faild to untar manifests from %s, error: %w", ghURL, err)
+	}
+
+	return nil
+}
+
 func genInstallManifests(version string, namespace string, components []string, tmpDir string) error {
 	model := struct {
 		Version    string
@@ -200,6 +251,10 @@ func genInstallManifests(version string, namespace string, components []string,
 		Components: components,
 	}
 
+	if err := downloadManifests(version, tmpDir); err != nil {
+		return err
+	}
+
 	if err := utils.execTemplate(model, namespaceTmpl, path.Join(tmpDir, "namespace.yaml")); err != nil {
 		return fmt.Errorf("generate namespace failed: %w", err)
 	}
@@ -220,6 +275,10 @@ func genInstallManifests(version string, namespace string, components []string,
 		return fmt.Errorf("generate roles failed: %w", err)
 	}
 
+	if err := utils.copyFile(filepath.Join(tmpDir, "rbac.yaml"), filepath.Join(tmpDir, "roles/rbac.yaml")); err != nil {
+		return fmt.Errorf("generate rbac failed: %w", err)
+	}
+
 	return nil
 }
 

cmd/tk/main.go

@@ -105,7 +105,7 @@ var (
 
 var (
 	defaultComponents = []string{"source-controller", "kustomize-controller", "helm-controller", "notification-controller"}
-	defaultVersion    = "master"
+	defaultVersion    = "latest"
 	defaultNamespace  = "gitops-system"
 )
 

cmd/tk/utils.go

@@ -143,3 +143,23 @@ func (*Utils) writeFile(content, filename string) error {
 
 	return file.Sync()
 }
+
+func (*Utils) copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		return err
+	}
+	return out.Close()
+}

docs/cmd/tk_bootstrap.md

@@ -11,7 +11,7 @@ The bootstrap sub-commands bootstrap the toolkit components on the targeted Git
 ```
       --components strings   list of components, accepts comma-separated values (default [source-controller,kustomize-controller,helm-controller,notification-controller])
   -h, --help                 help for bootstrap
-  -v, --version string       toolkit tag or branch (default "master")
+  -v, --version string       toolkit version (default "latest")
 ```
 
 ### Options inherited from parent commands

docs/cmd/tk_bootstrap_github.md

@@ -59,7 +59,7 @@ tk bootstrap github [flags]
       --namespace string     the namespace scope for this operation (default "gitops-system")
       --timeout duration     timeout for this operation (default 5m0s)
       --verbose              print generated objects
-  -v, --version string       toolkit tag or branch (default "master")
+  -v, --version string       toolkit version (default "latest")
 ```
 
 ### SEE ALSO

docs/cmd/tk_bootstrap_gitlab.md

@@ -55,7 +55,7 @@ tk bootstrap gitlab [flags]
       --namespace string     the namespace scope for this operation (default "gitops-system")
       --timeout duration     timeout for this operation (default 5m0s)
       --verbose              print generated objects
-  -v, --version string       toolkit tag or branch (default "master")
+  -v, --version string       toolkit version (default "latest")
 ```
 
 ### SEE ALSO

docs/cmd/tk_install.md

@@ -15,14 +15,17 @@ tk install [flags]
 
 ```
   # Install the latest version in the gitops-systems namespace
-  tk install --version=master --namespace=gitops-systems
+  tk install --version=latest --namespace=gitops-systems
 
   # Dry-run install for a specific version and a series of components
-  tk install --dry-run --version=0.0.1 --components="source-controller,kustomize-controller"
+  tk install --dry-run --version=v0.0.7 --components="source-controller,kustomize-controller"
 
   # Dry-run install with manifests preview 
   tk install --dry-run --verbose
 
+  # Write install manifests to file 
+  tk install --export > gitops-system.yaml
+
 ```
 
 ### Options
@@ -30,9 +33,10 @@ tk install [flags]
 ```
       --components strings   list of components, accepts comma-separated values (default [source-controller,kustomize-controller,helm-controller,notification-controller])
       --dry-run              only print the object that would be applied
+      --export               write the install manifests to stdout and exit
   -h, --help                 help for install
       --manifests string     path to the manifest directory, dev only
-  -v, --version string       toolkit tag or branch (default "master")
+  -v, --version string       toolkit version (default "latest")
 ```
 
 ### Options inherited from parent commands

docs/guides/helmreleases.md

@@ -1,4 +1,4 @@
-# Manage Helm releases
+# Manage Helm Releases
 
 The [helm-controller](../components/helm/controller.md) allows you to
 declaratively manage Helm chart releases with Kubernetes manifests.
@@ -93,7 +93,7 @@ helm-controller.
     See the [`HelmRelease` CRD docs](../components/helm/helmreleases.md)
     for more details.
 
-## Receive notifications
+## Configure notifications
 
 The default toolkit installation configures the helm-controller to
 broadcast events to the [notification-controller](../components/notification/controller.md).
@@ -126,3 +126,67 @@ apiVersion: notification.fluxcd.io/v1alpha1
 ```
 
 ![helm-controller alerts](../diagrams/helm-controller-alerts.png)
+
+## Configure webhook receivers
+
+When using semver ranges for Helm releases, you may want to trigger an update
+as soon as a new chart version is published to your Helm repository.
+In order to notify source-controller about a chart update,
+you can [setup webhook receivers](webhook-receivers.md).
+
+First generate a random string and create a secret with a `token` field:
+
+```sh
+TOKEN=$(head -c 12 /dev/urandom | shasum | cut -d ' ' -f1)
+echo $TOKEN
+
+kubectl -n gitops-system create secret generic webhook-token \	
+--from-literal=token=$TOKEN
+```
+
+When using [Harbor](https://goharbor.io/) as your Helm repository, you can define a receiver with:
+
+```yaml
+apiVersion: notification.fluxcd.io/v1alpha1
+kind: Receiver
+metadata:
+  name: helm-podinfo
+  namespace: gitops-system
+spec:
+  type: harbor
+  secretRef:
+    name: webhook-token
+  resources:
+    - kind: HelmRepository
+      name: podinfo
+```
+
+The notification-controller generates a unique URL using the provided token and the receiver name/namespace.
+
+Find the URL with:
+
+```console
+$ kubectl -n gitops-system get receiver/helm-podinfo
+
+NAME           READY   STATUS
+helm-podinfo   True    Receiver initialised with URL: /hook/bed6d00b5555b1603e1f59b94d7fdbca58089cb5663633fb83f2815dc626d92b
+```
+
+Log in to the Harbor interface, go to Projects, select a project, and select Webhooks.
+Fill the form with:
+
+* Endpoint URL: compose the address using the receiver LB and the generated URL `http://<LoadBalancerAddress>/<ReceiverURL>`
+* Auth Header: use the `token` string
+
+With the above settings, when you upload a chart, the following happens:
+
+* Harbor sends the chart push event to the receiver address
+* Notification controller validates the authenticity of the payload using the auth header
+* Source controller is notified about the changes
+* Source controller pulls the changes into the cluster and updates the `HelmChart` version
+* Helm controller is notified about the version change and upgrades the release
+
+!!! hint "Note"
+    Besides Harbor, you can define receivers for **GitHub**, **GitLab**, **Bitbucket**
+    and any other system that supports webhooks e.g. Jenkins, CircleCI, etc.
+    See the [Receiver CRD docs](../components/notification/receiver.md) for more details.

docs/roadmap/index.md

@@ -41,7 +41,7 @@ Tasks
 
 Goals
 
-- Offer a dedicated component that can replace Flux v1 image update feature
+- Offer components that can replace Flux v1 image update feature
 
 Non-Goals
 
@@ -49,11 +49,9 @@ Non-Goals
 
 Tasks
 
-- Design the Git push API
-- Implement Git push in source controller
-- Design the image scanning API
+- [Design the image scanning and automation API](https://github.com/fluxcd/toolkit/discussions/107)
 - Implement an image scanning controller
-- Design the manifests patching component
+- Design the automation component
 - Implement the image scan/patch/push workflow
 - Integrate the new components in the toolkit assembler
 - Create a migration guide from Flux annotations
@@ -69,13 +67,19 @@ Goals
 Non-Goals
 
 - Migrate users that are using Helm v2
-- Migrate users that are using Helm charts from Git
+
+Stretch-Goals
+
+- [Migrate users that are using Helm charts from Git](https://github.com/fluxcd/toolkit/discussions/75#discussioncomment-38589)
 
 Tasks
 
+- ~~Implement a Helm controller for Helm v3 covering all the current release options~~
+- Discuss and design Helm releases based on source API:
+    + [Providing values from sources](https://github.com/fluxcd/toolkit/discussions/100)
+    + [Conditional remediation on failed Helm actions](https://github.com/fluxcd/toolkit/discussions/102)
+    + [Support running Helm test actions on an interval](https://github.com/fluxcd/toolkit/discussions/103)
 - Review the Helm release, chart and repository APIs
-- Design Helm releases based on source API
-- Implement a Helm controller for Helm v3 covering all the current release options
-- Implement events in Helm controller
+- ~~Implement events in Helm controller~~
 - Implement Prometheus metrics in Helm controller
 - Create a migration guide for Helm Operator users

mkdocs.yml

@@ -40,7 +40,7 @@ nav:
   - Introduction: index.md
   - Get Started: get-started/index.md
   - Guides:
-      - Manage Helm releases: guides/helmreleases.md
+      - Manage Helm Releases: guides/helmreleases.md
       - Setup Notifications: guides/notifications.md
       - Setup Webhook Receivers: guides/webhook-receivers.md
   - Toolkit Components: