Allow Flux to be deployed on tainted Kubernetes nodes

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>

cmd/flux/bootstrap.go

@@ -60,6 +60,7 @@ type bootstrapFlags struct {
 	requiredComponents []string
 	tokenAuth          bool
 	clusterDomain      string
+	tolerationKeys     []string
 }
 
 const (
@@ -91,6 +92,8 @@ func init() {
 	bootstrapCmd.PersistentFlags().Var(&bootstrapArgs.logLevel, "log-level", bootstrapArgs.logLevel.Description())
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.manifestsPath, "manifests", "", "path to the manifest directory")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.clusterDomain, "cluster-domain", rootArgs.defaults.ClusterDomain, "internal cluster domain")
+	bootstrapCmd.PersistentFlags().StringSliceVar(&bootstrapArgs.tolerationKeys, "toleration-keys", nil,
+		"list of toleration keys used to schedule the components pods onto nodes with matching taints")
 	bootstrapCmd.PersistentFlags().MarkHidden("manifests")
 	bootstrapCmd.PersistentFlags().MarkDeprecated("arch", "multi-arch container image is now available for AMD64, ARMv7 and ARM64")
 	rootCmd.AddCommand(bootstrapCmd)
@@ -138,6 +141,7 @@ func generateInstallManifests(targetPath, namespace, tmpDir string, localManifes
 		Timeout:                rootArgs.timeout,
 		TargetPath:             targetPath,
 		ClusterDomain:          bootstrapArgs.clusterDomain,
+		TolerationKeys:         bootstrapArgs.tolerationKeys,
 	}
 
 	if localManifests == "" {

cmd/flux/install.go

@@ -34,15 +34,18 @@ import (
 
 var installCmd = &cobra.Command{
 	Use:   "install",
-	Short: "Install the toolkit components",
+	Short: "Install or upgrade Flux",
-	Long: `The install command deploys the toolkit components in the specified namespace.
+	Long: `The install command deploys Flux in the specified namespace.
 If a previous version is installed, then an in-place upgrade will be performed.`,
 	Example: `  # Install the latest version in the flux-system namespace
   flux install --version=latest --namespace=flux-system
 
-  # Dry-run install for a specific version and a series of components
+  # Install a specific version and a series of components
   flux install --dry-run --version=v0.0.7 --components="source-controller,kustomize-controller"
 
+  # Install Flux onto tainted Kubernetes nodes
+  flux install --toleration-keys=node.kubernetes.io/dedicated-to-flux
+
   # Dry-run install with manifests preview
   flux install --dry-run --verbose
 
@@ -66,6 +69,7 @@ var (
 	installArch               flags.Arch
 	installLogLevel           = flags.LogLevel(rootArgs.defaults.LogLevel)
 	installClusterDomain      string
+	installTolerationKeys     []string
 )
 
 func init() {
@@ -91,6 +95,8 @@ func init() {
 	installCmd.Flags().BoolVar(&installNetworkPolicy, "network-policy", rootArgs.defaults.NetworkPolicy,
 		"deny ingress access to the toolkit controllers from other namespaces using network policies")
 	installCmd.Flags().StringVar(&installClusterDomain, "cluster-domain", rootArgs.defaults.ClusterDomain, "internal cluster domain")
+	installCmd.Flags().StringSliceVar(&installTolerationKeys, "toleration-keys", nil,
+		"list of toleration keys used to schedule the components pods onto nodes with matching taints")
 	installCmd.Flags().MarkHidden("manifests")
 	installCmd.Flags().MarkDeprecated("arch", "multi-arch container image is now available for AMD64, ARMv7 and ARM64")
 	rootCmd.AddCommand(installCmd)
@@ -130,6 +136,7 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
 		ManifestFile:           fmt.Sprintf("%s.yaml", rootArgs.namespace),
 		Timeout:                rootArgs.timeout,
 		ClusterDomain:          installClusterDomain,
+		TolerationKeys:         installTolerationKeys,
 	}
 
 	if installManifestsPath == "" {

docs/cmd/flux.md

@@ -84,7 +84,7 @@ Command line utility for assembling Kubernetes CD pipelines the GitOps way.
 * [flux delete](flux_delete.md)	 - Delete sources and resources
 * [flux export](flux_export.md)	 - Export resources in YAML format
 * [flux get](flux_get.md)	 - Get sources and resources
-* [flux install](flux_install.md)	 - Install the toolkit components
+* [flux install](flux_install.md)	 - Install or upgrade Flux
 * [flux reconcile](flux_reconcile.md)	 - Reconcile sources and resources
 * [flux resume](flux_resume.md)	 - Resume suspended resources
 * [flux suspend](flux_suspend.md)	 - Suspend resources

docs/cmd/flux_bootstrap.md

@@ -19,6 +19,7 @@ The bootstrap sub-commands bootstrap the toolkit components on the targeted Git
       --network-policy             deny ingress access to the toolkit controllers from other namespaces using network policies (default true)
       --registry string            container registry where the toolkit images are published (default "ghcr.io/fluxcd")
       --token-auth                 when enabled, the personal access token will be used instead of SSH deploy key
+      --toleration-keys strings    list of toleration keys used to schedule the components pods onto nodes with matching taints
   -v, --version string             toolkit version (default "latest")
       --watch-all-namespaces       watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true)
 ```

docs/cmd/flux_bootstrap_github.md

@@ -74,6 +74,7 @@ flux bootstrap github [flags]
       --registry string            container registry where the toolkit images are published (default "ghcr.io/fluxcd")
       --timeout duration           timeout for this operation (default 5m0s)
       --token-auth                 when enabled, the personal access token will be used instead of SSH deploy key
+      --toleration-keys strings    list of toleration keys used to schedule the components pods onto nodes with matching taints
       --verbose                    print generated objects
   -v, --version string             toolkit version (default "latest")
       --watch-all-namespaces       watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true)

docs/cmd/flux_bootstrap_gitlab.md

@@ -70,6 +70,7 @@ flux bootstrap gitlab [flags]
       --registry string            container registry where the toolkit images are published (default "ghcr.io/fluxcd")
       --timeout duration           timeout for this operation (default 5m0s)
       --token-auth                 when enabled, the personal access token will be used instead of SSH deploy key
+      --toleration-keys strings    list of toleration keys used to schedule the components pods onto nodes with matching taints
       --verbose                    print generated objects
   -v, --version string             toolkit version (default "latest")
       --watch-all-namespaces       watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true)

docs/cmd/flux_install.md

@@ -1,10 +1,10 @@
 ## flux install
 
-Install the toolkit components
+Install or upgrade Flux
 
 ### Synopsis
 
-The install command deploys the toolkit components in the specified namespace.
+The install command deploys Flux in the specified namespace.
 If a previous version is installed, then an in-place upgrade will be performed.
 
 ```
@@ -17,9 +17,12 @@ flux install [flags]
   # Install the latest version in the flux-system namespace
   flux install --version=latest --namespace=flux-system
 
-  # Dry-run install for a specific version and a series of components
+  # Install a specific version and a series of components
   flux install --dry-run --version=v0.0.7 --components="source-controller,kustomize-controller"
 
+  # Install Flux onto tainted Kubernetes nodes
+  flux install --toleration-keys=node.kubernetes.io/dedicated-to-flux
+
   # Dry-run install with manifests preview
   flux install --dry-run --verbose
 
@@ -41,6 +44,7 @@ flux install [flags]
       --log-level logLevel         log level, available options are: (debug, info, error) (default info)
       --network-policy             deny ingress access to the toolkit controllers from other namespaces using network policies (default true)
       --registry string            container registry where the toolkit images are published (default "ghcr.io/fluxcd")
+      --toleration-keys strings    list of toleration keys used to schedule the components pods onto nodes with matching taints
   -v, --version string             toolkit version (default "latest")
       --watch-all-namespaces       watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true)
 ```

docs/guides/installation.md

@@ -70,6 +70,10 @@ flux bootstrap <GIT-PROVIDER> \
 If you wish to install a specific version, use the Flux
 [release tag](https://github.com/fluxcd/flux2/releases) e.g. `--version=v0.2.0`.
 
+If you wish to deploy the Flux components onto
+[tainted Kubernetes nodes](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/),
+you can specify the toleration keys with `--toleration-keys=node.kubernetes.io/dedicated-to-flux`.
+
 With `--path` you can configure the directory which will be used to reconcile the target cluster.
 To control multiple clusters from the same Git repository, you have to set a unique path per
 cluster e.g. `clusters/staging` and `clusters/production`:

pkg/manifestgen/install/install_test.go

@@ -24,6 +24,7 @@ import (
 
 func TestGenerate(t *testing.T) {
 	opts := MakeDefaultOptions()
+	opts.TolerationKeys = []string{"node.kubernetes.io/controllers"}
 	output, err := Generate(opts)
 	if err != nil {
 		t.Fatal(err)
@@ -36,5 +37,9 @@ func TestGenerate(t *testing.T) {
 		}
 	}
 
+	if !strings.Contains(output.Content, opts.TolerationKeys[0]) {
+		t.Errorf("toleration key '%s' not found", opts.TolerationKeys[0])
+	}
+
 	fmt.Println(output)
 }

pkg/manifestgen/install/options.go

@@ -35,6 +35,7 @@ type Options struct {
 	Timeout                time.Duration
 	TargetPath             string
 	ClusterDomain          string
+	TolerationKeys         []string
 }
 
 func MakeDefaultOptions() Options {

pkg/manifestgen/install/templates.go

@@ -137,6 +137,13 @@ spec:
       imagePullSecrets:
        - name: {{.ImagePullSecret}}
 {{- end }}
+{{ if gt (len .TolerationKeys) 0 }}
+      tolerations:
+{{- range $i, $key := .TolerationKeys }}
+       - key: "{{$key}}"
+         operator: "Exists"
+{{- end }}
+{{- end }}
 `
 
 var labelsTmpl = `---