From b92cbcd7e72e8ad505bb1bae0389998fd7e21552 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Fri, 17 Jul 2020 13:10:19 +0300 Subject: [PATCH] Use semver manifests as kustomize base Instead of cloning the components repositories to download the base manifests, we build them in CI at release time and download them in tk install/bootstrap based on the provided semver. This speeds up the manifests generation from minutes to milliseconds. --- cmd/tk/bootstrap.go | 2 +- cmd/tk/install.go | 56 +++++++++++++++++++++++++++++---- cmd/tk/main.go | 2 +- cmd/tk/utils.go | 20 ++++++++++++ docs/cmd/tk_bootstrap.md | 2 +- docs/cmd/tk_bootstrap_github.md | 2 +- docs/cmd/tk_bootstrap_gitlab.md | 2 +- docs/cmd/tk_install.md | 6 ++-- 8 files changed, 78 insertions(+), 14 deletions(-) diff --git a/cmd/tk/bootstrap.go b/cmd/tk/bootstrap.go index 82aa1b28..9cc5261b 100644 --- a/cmd/tk/bootstrap.go +++ b/cmd/tk/bootstrap.go @@ -58,7 +58,7 @@ const ( func init() { bootstrapCmd.PersistentFlags().StringVarP(&bootstrapVersion, "version", "v", defaultVersion, - "toolkit tag or branch") + "toolkit version") bootstrapCmd.PersistentFlags().StringSliceVar(&bootstrapComponents, "components", defaultComponents, "list of components, accepts comma-separated values") diff --git a/cmd/tk/install.go b/cmd/tk/install.go index 2e929dd5..5817dc9c 100644 --- a/cmd/tk/install.go +++ b/cmd/tk/install.go @@ -19,7 +19,9 @@ package main import ( "context" "fmt" + "github.com/fluxcd/pkg/untar" "io/ioutil" + "net/http" "os" "path" "path/filepath" @@ -37,10 +39,10 @@ var installCmd = &cobra.Command{ Long: `The install command deploys the toolkit components in the specified namespace. If a previous version is installed, then an in-place upgrade will be performed.`, Example: ` # Install the latest version in the gitops-systems namespace - tk install --version=master --namespace=gitops-systems + tk install --version=latest --namespace=gitops-systems # Dry-run install for a specific version and a series of components - tk install --dry-run --version=0.0.1 --components="source-controller,kustomize-controller" + tk install --dry-run --version=v0.0.7 --components="source-controller,kustomize-controller" # Dry-run install with manifests preview tk install --dry-run --verbose @@ -65,7 +67,7 @@ func init() { installCmd.Flags().BoolVarP(&installDryRun, "dry-run", "", false, "only print the object that would be applied") installCmd.Flags().StringVarP(&installVersion, "version", "v", defaultVersion, - "toolkit tag or branch") + "toolkit version") installCmd.Flags().StringSliceVar(&installComponents, "components", defaultComponents, "list of components, accepts comma-separated values") installCmd.Flags().StringVarP(&installManifestsPath, "manifests", "", "", @@ -189,10 +191,10 @@ transformers: - labels.yaml resources: - namespace.yaml + - policies.yaml - roles - - github.com/fluxcd/toolkit/manifests/policies?ref={{$version}} {{- range .Components }} - - github.com/fluxcd/toolkit/manifests/bases/{{.}}?ref={{$version}} + - {{.}}.yaml {{- end }} ` @@ -200,10 +202,44 @@ var kustomizationRolesTmpl = `--- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - github.com/fluxcd/toolkit/manifests/rbac?ref={{.Version}} + - rbac.yaml nameSuffix: -{{.Namespace}} ` +func downloadManifests(version string, tmpDir string) error { + ghURL := "https://github.com/fluxcd/toolkit/releases/latest/download/manifests.tar.gz" + if strings.HasPrefix(version, "v") { + ghURL = fmt.Sprintf("https://github.com/fluxcd/toolkit/releases/download/%s/manifests.tar.gz", version) + } + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + req, err := http.NewRequest("GET", ghURL, nil) + if err != nil { + return fmt.Errorf("failed to create HTTP request for %s, error: %w", ghURL, err) + } + + // download + resp, err := http.DefaultClient.Do(req.WithContext(ctx)) + if err != nil { + return fmt.Errorf("failed to download artifact from %s, error: %w", ghURL, err) + } + defer resp.Body.Close() + + // check response + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("faild to download artifact from %s, status: %s", ghURL, resp.Status) + } + + // extract + if _, err = untar.Untar(resp.Body, tmpDir); err != nil { + return fmt.Errorf("faild to untar manifests from %s, error: %w", ghURL, err) + } + + return nil +} + func genInstallManifests(version string, namespace string, components []string, tmpDir string) error { model := struct { Version string @@ -215,6 +251,10 @@ func genInstallManifests(version string, namespace string, components []string, Components: components, } + if err := downloadManifests(version, tmpDir); err != nil { + return err + } + if err := utils.execTemplate(model, namespaceTmpl, path.Join(tmpDir, "namespace.yaml")); err != nil { return fmt.Errorf("generate namespace failed: %w", err) } @@ -235,6 +275,10 @@ func genInstallManifests(version string, namespace string, components []string, return fmt.Errorf("generate roles failed: %w", err) } + if err := utils.copyFile(filepath.Join(tmpDir, "rbac.yaml"), filepath.Join(tmpDir, "roles/rbac.yaml")); err != nil { + return fmt.Errorf("generate rbac failed: %w", err) + } + return nil } diff --git a/cmd/tk/main.go b/cmd/tk/main.go index cc52f434..4b01667f 100644 --- a/cmd/tk/main.go +++ b/cmd/tk/main.go @@ -105,7 +105,7 @@ var ( var ( defaultComponents = []string{"source-controller", "kustomize-controller", "helm-controller", "notification-controller"} - defaultVersion = "master" + defaultVersion = "latest" defaultNamespace = "gitops-system" ) diff --git a/cmd/tk/utils.go b/cmd/tk/utils.go index 433bcbac..81012877 100644 --- a/cmd/tk/utils.go +++ b/cmd/tk/utils.go @@ -143,3 +143,23 @@ func (*Utils) writeFile(content, filename string) error { return file.Sync() } + +func (*Utils) copyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, in) + if err != nil { + return err + } + return out.Close() +} diff --git a/docs/cmd/tk_bootstrap.md b/docs/cmd/tk_bootstrap.md index 3ca6eb23..6ae349f1 100644 --- a/docs/cmd/tk_bootstrap.md +++ b/docs/cmd/tk_bootstrap.md @@ -11,7 +11,7 @@ The bootstrap sub-commands bootstrap the toolkit components on the targeted Git ``` --components strings list of components, accepts comma-separated values (default [source-controller,kustomize-controller,helm-controller,notification-controller]) -h, --help help for bootstrap - -v, --version string toolkit tag or branch (default "master") + -v, --version string toolkit version (default "latest") ``` ### Options inherited from parent commands diff --git a/docs/cmd/tk_bootstrap_github.md b/docs/cmd/tk_bootstrap_github.md index e4ebd723..df7c34f4 100644 --- a/docs/cmd/tk_bootstrap_github.md +++ b/docs/cmd/tk_bootstrap_github.md @@ -59,7 +59,7 @@ tk bootstrap github [flags] --namespace string the namespace scope for this operation (default "gitops-system") --timeout duration timeout for this operation (default 5m0s) --verbose print generated objects - -v, --version string toolkit tag or branch (default "master") + -v, --version string toolkit version (default "latest") ``` ### SEE ALSO diff --git a/docs/cmd/tk_bootstrap_gitlab.md b/docs/cmd/tk_bootstrap_gitlab.md index 933ec721..5b0cda8b 100644 --- a/docs/cmd/tk_bootstrap_gitlab.md +++ b/docs/cmd/tk_bootstrap_gitlab.md @@ -55,7 +55,7 @@ tk bootstrap gitlab [flags] --namespace string the namespace scope for this operation (default "gitops-system") --timeout duration timeout for this operation (default 5m0s) --verbose print generated objects - -v, --version string toolkit tag or branch (default "master") + -v, --version string toolkit version (default "latest") ``` ### SEE ALSO diff --git a/docs/cmd/tk_install.md b/docs/cmd/tk_install.md index 39becb37..d42c8aa7 100644 --- a/docs/cmd/tk_install.md +++ b/docs/cmd/tk_install.md @@ -15,10 +15,10 @@ tk install [flags] ``` # Install the latest version in the gitops-systems namespace - tk install --version=master --namespace=gitops-systems + tk install --version=latest --namespace=gitops-systems # Dry-run install for a specific version and a series of components - tk install --dry-run --version=0.0.1 --components="source-controller,kustomize-controller" + tk install --dry-run --version=v0.0.7 --components="source-controller,kustomize-controller" # Dry-run install with manifests preview tk install --dry-run --verbose @@ -36,7 +36,7 @@ tk install [flags] --export write the install manifests to stdout and exit -h, --help help for install --manifests string path to the manifest directory, dev only - -v, --version string toolkit tag or branch (default "master") + -v, --version string toolkit version (default "latest") ``` ### Options inherited from parent commands