Refactor manifests generation

- introduce manifestgen pkg, to be consumed by the CLI and Terraform provider
- consolidate defaults in manifestgen/install pkg
- introduce Manifest as the returning type of manifest generation
- add helper function to Manifest for writing multi-doc YAMLs on disk

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>

cmd/gotk/bootstrap.go

@@ -19,10 +19,7 @@ package main
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"net/url"
-	"os"
-	"path"
 	"path/filepath"
 	"time"
 
@@ -38,8 +35,8 @@ import (
 
 	"github.com/fluxcd/toolkit/internal/flags"
 	"github.com/fluxcd/toolkit/internal/utils"
-	"github.com/fluxcd/toolkit/pkg/install"
-	"github.com/fluxcd/toolkit/pkg/sync"
+	"github.com/fluxcd/toolkit/pkg/manifestgen/install"
+	"github.com/fluxcd/toolkit/pkg/manifestgen/sync"
 )
 
 var bootstrapCmd = &cobra.Command{
@@ -53,12 +50,12 @@ var (
 	bootstrapComponents         []string
 	bootstrapRegistry           string
 	bootstrapImagePullSecret    string
-	bootstrapArch               flags.Arch = "amd64"
 	bootstrapBranch             string
 	bootstrapWatchAllNamespaces bool
 	bootstrapNetworkPolicy      bool
-	bootstrapLogLevel           flags.LogLevel = "info"
 	bootstrapManifestsPath      string
+	bootstrapArch               = flags.Arch(defaults.Arch)
+	bootstrapLogLevel           = flags.LogLevel(defaults.LogLevel)
 	bootstrapRequiredComponents = []string{"source-controller", "kustomize-controller"}
 )
 
@@ -67,9 +64,9 @@ const (
 )
 
 func init() {
-	bootstrapCmd.PersistentFlags().StringVarP(&bootstrapVersion, "version", "v", defaultVersion,
+	bootstrapCmd.PersistentFlags().StringVarP(&bootstrapVersion, "version", "v", defaults.Version,
 		"toolkit version")
-	bootstrapCmd.PersistentFlags().StringSliceVar(&bootstrapComponents, "components", defaultComponents,
+	bootstrapCmd.PersistentFlags().StringSliceVar(&bootstrapComponents, "components", defaults.Components,
 		"list of components, accepts comma-separated values")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapRegistry, "registry", "ghcr.io/fluxcd",
 		"container registry where the toolkit images are published")
@@ -110,31 +107,27 @@ func generateInstallManifests(targetPath, namespace, tmpDir string, localManifes
 		WatchAllNamespaces:     bootstrapWatchAllNamespaces,
 		NetworkPolicy:          bootstrapNetworkPolicy,
 		LogLevel:               bootstrapLogLevel.String(),
-		NotificationController: defaultNotification,
-		ManifestsFile:          fmt.Sprintf("%s.yaml", namespace),
+		NotificationController: defaults.NotificationController,
+		ManifestFile:           defaults.ManifestFile,
 		Timeout:                timeout,
 		TargetPath:             targetPath,
 	}
 
 	if localManifests == "" {
-		opts.BaseURL = install.MakeDefaultOptions().BaseURL
+		opts.BaseURL = defaults.BaseURL
 	}
 
-	manifestPath, content, err := install.Generate(opts)
+	output, err := install.Generate(opts)
 	if err != nil {
 		return "", fmt.Errorf("generating install manifests failed: %w", err)
 	}
 
-	filePath := path.Join(tmpDir, manifestPath)
-	if err := os.MkdirAll(path.Dir(manifestPath), os.ModePerm); err != nil {
-		return "", fmt.Errorf("creating manifest dir failed: %w", err)
-	}
-
-	if err := ioutil.WriteFile(filePath, []byte(content), os.ModePerm); err != nil {
+	if filePath, err := output.WriteFile(tmpDir); err != nil {
 		return "", fmt.Errorf("generating install manifests failed: %w", err)
+	} else {
+		return filePath, nil
 	}
 
-	return filePath, nil
 }
 
 func applyInstallManifests(ctx context.Context, manifestPath string, components []string) error {
@@ -154,23 +147,22 @@ func applyInstallManifests(ctx context.Context, manifestPath string, components
 
 func generateSyncManifests(url, branch, name, namespace, targetPath, tmpDir string, interval time.Duration) error {
 	opts := sync.Options{
-		Name:       name,
-		Namespace:  namespace,
-		URL:        url,
-		Branch:     branch,
-		Interval:   interval,
-		TargetPath: targetPath,
+		Name:         name,
+		Namespace:    namespace,
+		URL:          url,
+		Branch:       branch,
+		Interval:     interval,
+		TargetPath:   targetPath,
+		ManifestFile: sync.MakeDefaultOptions().ManifestFile,
 	}
 
-	output, err := sync.Generate(opts)
+	manifest, err := sync.Generate(opts)
 	if err != nil {
 		return fmt.Errorf("generating install manifests failed: %w", err)
 	}
 
-	for _, v := range output {
-		if err := utils.WriteFile(v["content"], filepath.Join(tmpDir, v["file_path"])); err != nil {
-			return err
-		}
+	if _, err := manifest.WriteFile(tmpDir); err != nil {
+		return err
 	}
 
 	if err := utils.GenerateKustomizationYaml(filepath.Join(tmpDir, targetPath, namespace)); err != nil {

cmd/gotk/check.go

@@ -57,7 +57,7 @@ type kubectlVersion struct {
 func init() {
 	checkCmd.Flags().BoolVarP(&checkPre, "pre", "", false,
 		"only run pre-installation checks")
-	checkCmd.Flags().StringSliceVar(&checkComponents, "components", defaultComponents,
+	checkCmd.Flags().StringSliceVar(&checkComponents, "components", defaults.Components,
 		"list of components, accepts comma-separated values")
 	rootCmd.AddCommand(checkCmd)
 }

cmd/gotk/install.go

@@ -21,14 +21,14 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/spf13/cobra"
 
 	"github.com/fluxcd/toolkit/internal/flags"
 	"github.com/fluxcd/toolkit/internal/utils"
-	"github.com/fluxcd/toolkit/pkg/install"
+	"github.com/fluxcd/toolkit/pkg/manifestgen/install"
 )
 
 var installCmd = &cobra.Command{
@@ -59,10 +59,10 @@ var (
 	installComponents         []string
 	installRegistry           string
 	installImagePullSecret    string
-	installArch               flags.Arch = "amd64"
 	installWatchAllNamespaces bool
 	installNetworkPolicy      bool
-	installLogLevel           flags.LogLevel = "info"
+	installArch               = flags.Arch(defaults.Arch)
+	installLogLevel           = flags.LogLevel(defaults.LogLevel)
 )
 
 func init() {
@@ -70,21 +70,21 @@ func init() {
 		"write the install manifests to stdout and exit")
 	installCmd.Flags().BoolVarP(&installDryRun, "dry-run", "", false,
 		"only print the object that would be applied")
-	installCmd.Flags().StringVarP(&installVersion, "version", "v", defaultVersion,
+	installCmd.Flags().StringVarP(&installVersion, "version", "v", defaults.Version,
 		"toolkit version")
-	installCmd.Flags().StringSliceVar(&installComponents, "components", defaultComponents,
+	installCmd.Flags().StringSliceVar(&installComponents, "components", defaults.Components,
 		"list of components, accepts comma-separated values")
 	installCmd.Flags().StringVar(&installManifestsPath, "manifests", "", "path to the manifest directory")
 	installCmd.Flags().MarkHidden("manifests")
-	installCmd.Flags().StringVar(&installRegistry, "registry", "ghcr.io/fluxcd",
+	installCmd.Flags().StringVar(&installRegistry, "registry", defaults.Registry,
 		"container registry where the toolkit images are published")
 	installCmd.Flags().StringVar(&installImagePullSecret, "image-pull-secret", "",
 		"Kubernetes secret name used for pulling the toolkit images from a private registry")
 	installCmd.Flags().Var(&installArch, "arch", installArch.Description())
-	installCmd.Flags().BoolVar(&installWatchAllNamespaces, "watch-all-namespaces", true,
+	installCmd.Flags().BoolVar(&installWatchAllNamespaces, "watch-all-namespaces", defaults.WatchAllNamespaces,
 		"watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed")
 	installCmd.Flags().Var(&installLogLevel, "log-level", installLogLevel.Description())
-	installCmd.Flags().BoolVar(&installNetworkPolicy, "network-policy", true,
+	installCmd.Flags().BoolVar(&installNetworkPolicy, "network-policy", defaults.NetworkPolicy,
 		"deny ingress access to the toolkit controllers from other namespaces using network policies")
 	rootCmd.AddCommand(installCmd)
 }
@@ -114,8 +114,8 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
 		WatchAllNamespaces:     installWatchAllNamespaces,
 		NetworkPolicy:          installNetworkPolicy,
 		LogLevel:               installLogLevel.String(),
-		NotificationController: defaultNotification,
-		ManifestsFile:          fmt.Sprintf("%s.yaml", namespace),
+		NotificationController: defaults.NotificationController,
+		ManifestFile:           fmt.Sprintf("%s.yaml", namespace),
 		Timeout:                timeout,
 	}
 
@@ -123,23 +123,22 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
 		opts.BaseURL = install.MakeDefaultOptions().BaseURL
 	}
 
-	_, content, err := install.Generate(opts)
+	manifest, err := install.Generate(opts)
 	if err != nil {
 		return fmt.Errorf("install failed: %w", err)
 	}
 
-	manifest := path.Join(tmpDir, fmt.Sprintf("%s.yaml", namespace))
-	if err := ioutil.WriteFile(manifest, []byte(content), os.ModePerm); err != nil {
+	if _, err := manifest.WriteFile(tmpDir); err != nil {
 		return fmt.Errorf("install failed: %w", err)
 	}
 
 	if verbose {
-		fmt.Print(content)
+		fmt.Print(manifest.Content)
 	} else if installExport {
 		fmt.Println("---")
 		fmt.Println("# GitOps Toolkit revision", installVersion)
 		fmt.Println("# Components:", strings.Join(installComponents, ","))
-		fmt.Print(content)
+		fmt.Print(manifest.Content)
 		fmt.Println("---")
 		return nil
 	}
@@ -151,7 +150,7 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
 		applyOutput = utils.ModeOS
 	}
 
-	kubectlArgs := []string{"apply", "-f", manifest}
+	kubectlArgs := []string{"apply", "-f", filepath.Join(tmpDir, manifest.Path)}
 	if installDryRun {
 		args = append(args, "--dry-run=client")
 		applyOutput = utils.ModeOS

cmd/gotk/main.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"github.com/fluxcd/toolkit/pkg/manifestgen/install"
 	"log"
 	"os"
 	"path/filepath"
@@ -100,17 +101,11 @@ var (
 	verbose      bool
 	pollInterval                = 2 * time.Second
 	logger       gotklog.Logger = printLogger{}
-)
-
-var (
-	defaultComponents   = []string{"source-controller", "kustomize-controller", "helm-controller", "notification-controller"}
-	defaultVersion      = "latest"
-	defaultNamespace    = "gotk-system"
-	defaultNotification = "notification-controller"
+	defaults                    = install.MakeDefaultOptions()
 )
 
 func init() {
-	rootCmd.PersistentFlags().StringVarP(&namespace, "namespace", "n", defaultNamespace, "the namespace scope for this operation")
+	rootCmd.PersistentFlags().StringVarP(&namespace, "namespace", "n", defaults.Namespace, "the namespace scope for this operation")
 	rootCmd.PersistentFlags().DurationVar(&timeout, "timeout", 5*time.Minute, "timeout for this operation")
 	rootCmd.PersistentFlags().BoolVar(&verbose, "verbose", false, "print generated objects")
 }