From 37f5587085236988dde8acf3fad215a44d87d98b Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Thu, 11 Feb 2021 15:20:19 +0200 Subject: [PATCH] Allow Flux to be deployed on tainted Kubernetes nodes Signed-off-by: Stefan Prodan --- cmd/flux/bootstrap.go | 4 ++++ cmd/flux/install.go | 13 ++++++++++--- docs/cmd/flux.md | 2 +- docs/cmd/flux_bootstrap.md | 1 + docs/cmd/flux_bootstrap_github.md | 1 + docs/cmd/flux_bootstrap_gitlab.md | 1 + docs/cmd/flux_install.md | 10 +++++++--- docs/guides/installation.md | 4 ++++ pkg/manifestgen/install/install_test.go | 5 +++++ pkg/manifestgen/install/options.go | 1 + pkg/manifestgen/install/templates.go | 7 +++++++ 11 files changed, 42 insertions(+), 7 deletions(-) diff --git a/cmd/flux/bootstrap.go b/cmd/flux/bootstrap.go index b31befd6..b81e3576 100644 --- a/cmd/flux/bootstrap.go +++ b/cmd/flux/bootstrap.go @@ -60,6 +60,7 @@ type bootstrapFlags struct { requiredComponents []string tokenAuth bool clusterDomain string + tolerationKeys []string } const ( @@ -91,6 +92,8 @@ func init() { bootstrapCmd.PersistentFlags().Var(&bootstrapArgs.logLevel, "log-level", bootstrapArgs.logLevel.Description()) bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.manifestsPath, "manifests", "", "path to the manifest directory") bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.clusterDomain, "cluster-domain", rootArgs.defaults.ClusterDomain, "internal cluster domain") + bootstrapCmd.PersistentFlags().StringSliceVar(&bootstrapArgs.tolerationKeys, "toleration-keys", nil, + "list of toleration keys used to schedule the components pods onto nodes with matching taints") bootstrapCmd.PersistentFlags().MarkHidden("manifests") bootstrapCmd.PersistentFlags().MarkDeprecated("arch", "multi-arch container image is now available for AMD64, ARMv7 and ARM64") rootCmd.AddCommand(bootstrapCmd) @@ -138,6 +141,7 @@ func generateInstallManifests(targetPath, namespace, tmpDir string, localManifes Timeout: rootArgs.timeout, TargetPath: targetPath, ClusterDomain: bootstrapArgs.clusterDomain, + TolerationKeys: bootstrapArgs.tolerationKeys, } if localManifests == "" { diff --git a/cmd/flux/install.go b/cmd/flux/install.go index b0d301eb..5218055f 100644 --- a/cmd/flux/install.go +++ b/cmd/flux/install.go @@ -34,15 +34,18 @@ import ( var installCmd = &cobra.Command{ Use: "install", - Short: "Install the toolkit components", - Long: `The install command deploys the toolkit components in the specified namespace. + Short: "Install or upgrade Flux", + Long: `The install command deploys Flux in the specified namespace. If a previous version is installed, then an in-place upgrade will be performed.`, Example: ` # Install the latest version in the flux-system namespace flux install --version=latest --namespace=flux-system - # Dry-run install for a specific version and a series of components + # Install a specific version and a series of components flux install --dry-run --version=v0.0.7 --components="source-controller,kustomize-controller" + # Install Flux onto tainted Kubernetes nodes + flux install --toleration-keys=node.kubernetes.io/dedicated-to-flux + # Dry-run install with manifests preview flux install --dry-run --verbose @@ -66,6 +69,7 @@ var ( installArch flags.Arch installLogLevel = flags.LogLevel(rootArgs.defaults.LogLevel) installClusterDomain string + installTolerationKeys []string ) func init() { @@ -91,6 +95,8 @@ func init() { installCmd.Flags().BoolVar(&installNetworkPolicy, "network-policy", rootArgs.defaults.NetworkPolicy, "deny ingress access to the toolkit controllers from other namespaces using network policies") installCmd.Flags().StringVar(&installClusterDomain, "cluster-domain", rootArgs.defaults.ClusterDomain, "internal cluster domain") + installCmd.Flags().StringSliceVar(&installTolerationKeys, "toleration-keys", nil, + "list of toleration keys used to schedule the components pods onto nodes with matching taints") installCmd.Flags().MarkHidden("manifests") installCmd.Flags().MarkDeprecated("arch", "multi-arch container image is now available for AMD64, ARMv7 and ARM64") rootCmd.AddCommand(installCmd) @@ -130,6 +136,7 @@ func installCmdRun(cmd *cobra.Command, args []string) error { ManifestFile: fmt.Sprintf("%s.yaml", rootArgs.namespace), Timeout: rootArgs.timeout, ClusterDomain: installClusterDomain, + TolerationKeys: installTolerationKeys, } if installManifestsPath == "" { diff --git a/docs/cmd/flux.md b/docs/cmd/flux.md index 34b9ea85..6686573d 100644 --- a/docs/cmd/flux.md +++ b/docs/cmd/flux.md @@ -84,7 +84,7 @@ Command line utility for assembling Kubernetes CD pipelines the GitOps way. * [flux delete](flux_delete.md) - Delete sources and resources * [flux export](flux_export.md) - Export resources in YAML format * [flux get](flux_get.md) - Get sources and resources -* [flux install](flux_install.md) - Install the toolkit components +* [flux install](flux_install.md) - Install or upgrade Flux * [flux reconcile](flux_reconcile.md) - Reconcile sources and resources * [flux resume](flux_resume.md) - Resume suspended resources * [flux suspend](flux_suspend.md) - Suspend resources diff --git a/docs/cmd/flux_bootstrap.md b/docs/cmd/flux_bootstrap.md index 465edd96..6ee474cb 100644 --- a/docs/cmd/flux_bootstrap.md +++ b/docs/cmd/flux_bootstrap.md @@ -19,6 +19,7 @@ The bootstrap sub-commands bootstrap the toolkit components on the targeted Git --network-policy deny ingress access to the toolkit controllers from other namespaces using network policies (default true) --registry string container registry where the toolkit images are published (default "ghcr.io/fluxcd") --token-auth when enabled, the personal access token will be used instead of SSH deploy key + --toleration-keys strings list of toleration keys used to schedule the components pods onto nodes with matching taints -v, --version string toolkit version (default "latest") --watch-all-namespaces watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true) ``` diff --git a/docs/cmd/flux_bootstrap_github.md b/docs/cmd/flux_bootstrap_github.md index 54c2fa67..ba185717 100644 --- a/docs/cmd/flux_bootstrap_github.md +++ b/docs/cmd/flux_bootstrap_github.md @@ -74,6 +74,7 @@ flux bootstrap github [flags] --registry string container registry where the toolkit images are published (default "ghcr.io/fluxcd") --timeout duration timeout for this operation (default 5m0s) --token-auth when enabled, the personal access token will be used instead of SSH deploy key + --toleration-keys strings list of toleration keys used to schedule the components pods onto nodes with matching taints --verbose print generated objects -v, --version string toolkit version (default "latest") --watch-all-namespaces watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true) diff --git a/docs/cmd/flux_bootstrap_gitlab.md b/docs/cmd/flux_bootstrap_gitlab.md index a718b648..5eb0c965 100644 --- a/docs/cmd/flux_bootstrap_gitlab.md +++ b/docs/cmd/flux_bootstrap_gitlab.md @@ -70,6 +70,7 @@ flux bootstrap gitlab [flags] --registry string container registry where the toolkit images are published (default "ghcr.io/fluxcd") --timeout duration timeout for this operation (default 5m0s) --token-auth when enabled, the personal access token will be used instead of SSH deploy key + --toleration-keys strings list of toleration keys used to schedule the components pods onto nodes with matching taints --verbose print generated objects -v, --version string toolkit version (default "latest") --watch-all-namespaces watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true) diff --git a/docs/cmd/flux_install.md b/docs/cmd/flux_install.md index 4765e8e0..faad5878 100644 --- a/docs/cmd/flux_install.md +++ b/docs/cmd/flux_install.md @@ -1,10 +1,10 @@ ## flux install -Install the toolkit components +Install or upgrade Flux ### Synopsis -The install command deploys the toolkit components in the specified namespace. +The install command deploys Flux in the specified namespace. If a previous version is installed, then an in-place upgrade will be performed. ``` @@ -17,9 +17,12 @@ flux install [flags] # Install the latest version in the flux-system namespace flux install --version=latest --namespace=flux-system - # Dry-run install for a specific version and a series of components + # Install a specific version and a series of components flux install --dry-run --version=v0.0.7 --components="source-controller,kustomize-controller" + # Install Flux onto tainted Kubernetes nodes + flux install --toleration-keys=node.kubernetes.io/dedicated-to-flux + # Dry-run install with manifests preview flux install --dry-run --verbose @@ -41,6 +44,7 @@ flux install [flags] --log-level logLevel log level, available options are: (debug, info, error) (default info) --network-policy deny ingress access to the toolkit controllers from other namespaces using network policies (default true) --registry string container registry where the toolkit images are published (default "ghcr.io/fluxcd") + --toleration-keys strings list of toleration keys used to schedule the components pods onto nodes with matching taints -v, --version string toolkit version (default "latest") --watch-all-namespaces watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true) ``` diff --git a/docs/guides/installation.md b/docs/guides/installation.md index c4f9bece..152335f2 100644 --- a/docs/guides/installation.md +++ b/docs/guides/installation.md @@ -70,6 +70,10 @@ flux bootstrap \ If you wish to install a specific version, use the Flux [release tag](https://github.com/fluxcd/flux2/releases) e.g. `--version=v0.2.0`. +If you wish to deploy the Flux components onto +[tainted Kubernetes nodes](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/), +you can specify the toleration keys with `--toleration-keys=node.kubernetes.io/dedicated-to-flux`. + With `--path` you can configure the directory which will be used to reconcile the target cluster. To control multiple clusters from the same Git repository, you have to set a unique path per cluster e.g. `clusters/staging` and `clusters/production`: diff --git a/pkg/manifestgen/install/install_test.go b/pkg/manifestgen/install/install_test.go index 4fa6daf9..834b26d4 100644 --- a/pkg/manifestgen/install/install_test.go +++ b/pkg/manifestgen/install/install_test.go @@ -24,6 +24,7 @@ import ( func TestGenerate(t *testing.T) { opts := MakeDefaultOptions() + opts.TolerationKeys = []string{"node.kubernetes.io/controllers"} output, err := Generate(opts) if err != nil { t.Fatal(err) @@ -36,5 +37,9 @@ func TestGenerate(t *testing.T) { } } + if !strings.Contains(output.Content, opts.TolerationKeys[0]) { + t.Errorf("toleration key '%s' not found", opts.TolerationKeys[0]) + } + fmt.Println(output) } diff --git a/pkg/manifestgen/install/options.go b/pkg/manifestgen/install/options.go index 0a15f821..a456007b 100644 --- a/pkg/manifestgen/install/options.go +++ b/pkg/manifestgen/install/options.go @@ -35,6 +35,7 @@ type Options struct { Timeout time.Duration TargetPath string ClusterDomain string + TolerationKeys []string } func MakeDefaultOptions() Options { diff --git a/pkg/manifestgen/install/templates.go b/pkg/manifestgen/install/templates.go index 316435d1..1dcec5c9 100644 --- a/pkg/manifestgen/install/templates.go +++ b/pkg/manifestgen/install/templates.go @@ -137,6 +137,13 @@ spec: imagePullSecrets: - name: {{.ImagePullSecret}} {{- end }} +{{ if gt (len .TolerationKeys) 0 }} + tolerations: +{{- range $i, $key := .TolerationKeys }} + - key: "{{$key}}" + operator: "Exists" +{{- end }} +{{- end }} ` var labelsTmpl = `---