Use semver manifests as kustomize base

Instead of cloning the components repositories to download the base manifests, we build them in CI at release time and download them in tk install/bootstrap based on the provided semver. This speeds up the manifests generation from minutes to milliseconds.

cmd/tk/bootstrap.go

@@ -58,7 +58,7 @@ const (
 
 func init() {
 	bootstrapCmd.PersistentFlags().StringVarP(&bootstrapVersion, "version", "v", defaultVersion,
-		"toolkit tag or branch")
+		"toolkit version")
 	bootstrapCmd.PersistentFlags().StringSliceVar(&bootstrapComponents, "components", defaultComponents,
 		"list of components, accepts comma-separated values")
 

cmd/tk/install.go

@@ -19,7 +19,9 @@ package main
 import (
 	"context"
 	"fmt"
+	"github.com/fluxcd/pkg/untar"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -37,10 +39,10 @@ var installCmd = &cobra.Command{
 	Long: `The install command deploys the toolkit components in the specified namespace.
 If a previous version is installed, then an in-place upgrade will be performed.`,
 	Example: `  # Install the latest version in the gitops-systems namespace
-  tk install --version=master --namespace=gitops-systems
+  tk install --version=latest --namespace=gitops-systems
 
   # Dry-run install for a specific version and a series of components
-  tk install --dry-run --version=0.0.1 --components="source-controller,kustomize-controller"
+  tk install --dry-run --version=v0.0.7 --components="source-controller,kustomize-controller"
 
   # Dry-run install with manifests preview 
   tk install --dry-run --verbose
@@ -65,7 +67,7 @@ func init() {
 	installCmd.Flags().BoolVarP(&installDryRun, "dry-run", "", false,
 		"only print the object that would be applied")
 	installCmd.Flags().StringVarP(&installVersion, "version", "v", defaultVersion,
-		"toolkit tag or branch")
+		"toolkit version")
 	installCmd.Flags().StringSliceVar(&installComponents, "components", defaultComponents,
 		"list of components, accepts comma-separated values")
 	installCmd.Flags().StringVarP(&installManifestsPath, "manifests", "", "",
@@ -189,10 +191,10 @@ transformers:
   - labels.yaml
 resources:
   - namespace.yaml
+  - policies.yaml
   - roles
-  - github.com/fluxcd/toolkit/manifests/policies?ref={{$version}}
 {{- range .Components }}
-  - github.com/fluxcd/toolkit/manifests/bases/{{.}}?ref={{$version}}
+  - {{.}}.yaml
 {{- end }}
 `
 
@@ -200,10 +202,44 @@ var kustomizationRolesTmpl = `---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - github.com/fluxcd/toolkit/manifests/rbac?ref={{.Version}}
+  - rbac.yaml
 nameSuffix: -{{.Namespace}}
 `
 
+func downloadManifests(version string, tmpDir string) error {
+	ghURL := "https://github.com/fluxcd/toolkit/releases/latest/download/manifests.tar.gz"
+	if strings.HasPrefix(version, "v") {
+		ghURL = fmt.Sprintf("https://github.com/fluxcd/toolkit/releases/download/%s/manifests.tar.gz", version)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	req, err := http.NewRequest("GET", ghURL, nil)
+	if err != nil {
+		return fmt.Errorf("failed to create HTTP request for %s, error: %w", ghURL, err)
+	}
+
+	// download
+	resp, err := http.DefaultClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return fmt.Errorf("failed to download artifact from %s, error: %w", ghURL, err)
+	}
+	defer resp.Body.Close()
+
+	// check response
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("faild to download artifact from %s, status: %s", ghURL, resp.Status)
+	}
+
+	// extract
+	if _, err = untar.Untar(resp.Body, tmpDir); err != nil {
+		return fmt.Errorf("faild to untar manifests from %s, error: %w", ghURL, err)
+	}
+
+	return nil
+}
+
 func genInstallManifests(version string, namespace string, components []string, tmpDir string) error {
 	model := struct {
 		Version    string
@@ -215,6 +251,10 @@ func genInstallManifests(version string, namespace string, components []string,
 		Components: components,
 	}
 
+	if err := downloadManifests(version, tmpDir); err != nil {
+		return err
+	}
+
 	if err := utils.execTemplate(model, namespaceTmpl, path.Join(tmpDir, "namespace.yaml")); err != nil {
 		return fmt.Errorf("generate namespace failed: %w", err)
 	}
@@ -235,6 +275,10 @@ func genInstallManifests(version string, namespace string, components []string,
 		return fmt.Errorf("generate roles failed: %w", err)
 	}
 
+	if err := utils.copyFile(filepath.Join(tmpDir, "rbac.yaml"), filepath.Join(tmpDir, "roles/rbac.yaml")); err != nil {
+		return fmt.Errorf("generate rbac failed: %w", err)
+	}
+
 	return nil
 }
 

cmd/tk/main.go

@@ -105,7 +105,7 @@ var (
 
 var (
 	defaultComponents = []string{"source-controller", "kustomize-controller", "helm-controller", "notification-controller"}
-	defaultVersion    = "master"
+	defaultVersion    = "latest"
 	defaultNamespace  = "gitops-system"
 )
 

cmd/tk/utils.go

@@ -143,3 +143,23 @@ func (*Utils) writeFile(content, filename string) error {
 
 	return file.Sync()
 }
+
+func (*Utils) copyFile(src, dst string) error {
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+
+	out, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		return err
+	}
+	return out.Close()
+}

docs/cmd/tk_bootstrap.md

@@ -11,7 +11,7 @@ The bootstrap sub-commands bootstrap the toolkit components on the targeted Git
 ```
       --components strings   list of components, accepts comma-separated values (default [source-controller,kustomize-controller,helm-controller,notification-controller])
   -h, --help                 help for bootstrap
-  -v, --version string       toolkit tag or branch (default "master")
+  -v, --version string       toolkit version (default "latest")
 ```
 
 ### Options inherited from parent commands

docs/cmd/tk_bootstrap_github.md

@@ -59,7 +59,7 @@ tk bootstrap github [flags]
       --namespace string     the namespace scope for this operation (default "gitops-system")
       --timeout duration     timeout for this operation (default 5m0s)
       --verbose              print generated objects
-  -v, --version string       toolkit tag or branch (default "master")
+  -v, --version string       toolkit version (default "latest")
 ```
 
 ### SEE ALSO

docs/cmd/tk_bootstrap_gitlab.md

@@ -55,7 +55,7 @@ tk bootstrap gitlab [flags]
       --namespace string     the namespace scope for this operation (default "gitops-system")
       --timeout duration     timeout for this operation (default 5m0s)
       --verbose              print generated objects
-  -v, --version string       toolkit tag or branch (default "master")
+  -v, --version string       toolkit version (default "latest")
 ```
 
 ### SEE ALSO

docs/cmd/tk_install.md

@@ -15,10 +15,10 @@ tk install [flags]
 
 ```
   # Install the latest version in the gitops-systems namespace
-  tk install --version=master --namespace=gitops-systems
+  tk install --version=latest --namespace=gitops-systems
 
   # Dry-run install for a specific version and a series of components
-  tk install --dry-run --version=0.0.1 --components="source-controller,kustomize-controller"
+  tk install --dry-run --version=v0.0.7 --components="source-controller,kustomize-controller"
 
   # Dry-run install with manifests preview 
   tk install --dry-run --verbose
@@ -36,7 +36,7 @@ tk install [flags]
       --export               write the install manifests to stdout and exit
   -h, --help                 help for install
       --manifests string     path to the manifest directory, dev only
-  -v, --version string       toolkit tag or branch (default "master")
+  -v, --version string       toolkit version (default "latest")
 ```
 
 ### Options inherited from parent commands