Refactor manifests generation

- introduce manifestgen pkg, to be consumed by the CLI and Terraform provider
- consolidate defaults in manifestgen/install pkg
- introduce Manifest as the returning type of manifest generation
- add helper function to Manifest for writing multi-doc YAMLs on disk

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>

cmd/gotk/bootstrap.go

@@ -19,10 +19,7 @@ package main
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"net/url"
-	"os"
-	"path"
 	"path/filepath"
 	"time"
 
@@ -38,8 +35,8 @@ import (
 
 	"github.com/fluxcd/toolkit/internal/flags"
 	"github.com/fluxcd/toolkit/internal/utils"
-	"github.com/fluxcd/toolkit/pkg/install"
+	"github.com/fluxcd/toolkit/pkg/manifestgen/install"
-	"github.com/fluxcd/toolkit/pkg/sync"
+	"github.com/fluxcd/toolkit/pkg/manifestgen/sync"
 )
 
 var bootstrapCmd = &cobra.Command{
@@ -53,12 +50,12 @@ var (
 	bootstrapComponents         []string
 	bootstrapRegistry           string
 	bootstrapImagePullSecret    string
-	bootstrapArch               flags.Arch = "amd64"
 	bootstrapBranch             string
 	bootstrapWatchAllNamespaces bool
 	bootstrapNetworkPolicy      bool
-	bootstrapLogLevel           flags.LogLevel = "info"
 	bootstrapManifestsPath      string
+	bootstrapArch               = flags.Arch(defaults.Arch)
+	bootstrapLogLevel           = flags.LogLevel(defaults.LogLevel)
 	bootstrapRequiredComponents = []string{"source-controller", "kustomize-controller"}
 )
 
@@ -67,9 +64,9 @@ const (
 )
 
 func init() {
-	bootstrapCmd.PersistentFlags().StringVarP(&bootstrapVersion, "version", "v", defaultVersion,
+	bootstrapCmd.PersistentFlags().StringVarP(&bootstrapVersion, "version", "v", defaults.Version,
 		"toolkit version")
-	bootstrapCmd.PersistentFlags().StringSliceVar(&bootstrapComponents, "components", defaultComponents,
+	bootstrapCmd.PersistentFlags().StringSliceVar(&bootstrapComponents, "components", defaults.Components,
 		"list of components, accepts comma-separated values")
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapRegistry, "registry", "ghcr.io/fluxcd",
 		"container registry where the toolkit images are published")
@@ -110,31 +107,27 @@ func generateInstallManifests(targetPath, namespace, tmpDir string, localManifes
 		WatchAllNamespaces:     bootstrapWatchAllNamespaces,
 		NetworkPolicy:          bootstrapNetworkPolicy,
 		LogLevel:               bootstrapLogLevel.String(),
-		NotificationController: defaultNotification,
+		NotificationController: defaults.NotificationController,
-		ManifestsFile:          fmt.Sprintf("%s.yaml", namespace),
+		ManifestFile:           defaults.ManifestFile,
 		Timeout:                timeout,
 		TargetPath:             targetPath,
 	}
 
 	if localManifests == "" {
-		opts.BaseURL = install.MakeDefaultOptions().BaseURL
+		opts.BaseURL = defaults.BaseURL
 	}
 
-	manifestPath, content, err := install.Generate(opts)
+	output, err := install.Generate(opts)
 	if err != nil {
 		return "", fmt.Errorf("generating install manifests failed: %w", err)
 	}
 
-	filePath := path.Join(tmpDir, manifestPath)
+	if filePath, err := output.WriteFile(tmpDir); err != nil {
-	if err := os.MkdirAll(path.Dir(manifestPath), os.ModePerm); err != nil {
-		return "", fmt.Errorf("creating manifest dir failed: %w", err)
-	}
-
-	if err := ioutil.WriteFile(filePath, []byte(content), os.ModePerm); err != nil {
 		return "", fmt.Errorf("generating install manifests failed: %w", err)
+	} else {
+		return filePath, nil
 	}
 
-	return filePath, nil
 }
 
 func applyInstallManifests(ctx context.Context, manifestPath string, components []string) error {
@@ -160,18 +153,17 @@ func generateSyncManifests(url, branch, name, namespace, targetPath, tmpDir stri
 		Branch:       branch,
 		Interval:     interval,
 		TargetPath:   targetPath,
+		ManifestFile: sync.MakeDefaultOptions().ManifestFile,
 	}
 
-	output, err := sync.Generate(opts)
+	manifest, err := sync.Generate(opts)
 	if err != nil {
 		return fmt.Errorf("generating install manifests failed: %w", err)
 	}
 
-	for _, v := range output {
+	if _, err := manifest.WriteFile(tmpDir); err != nil {
-		if err := utils.WriteFile(v["content"], filepath.Join(tmpDir, v["file_path"])); err != nil {
 		return err
 	}
-	}
 
 	if err := utils.GenerateKustomizationYaml(filepath.Join(tmpDir, targetPath, namespace)); err != nil {
 		return err

cmd/gotk/check.go

@@ -57,7 +57,7 @@ type kubectlVersion struct {
 func init() {
 	checkCmd.Flags().BoolVarP(&checkPre, "pre", "", false,
 		"only run pre-installation checks")
-	checkCmd.Flags().StringSliceVar(&checkComponents, "components", defaultComponents,
+	checkCmd.Flags().StringSliceVar(&checkComponents, "components", defaults.Components,
 		"list of components, accepts comma-separated values")
 	rootCmd.AddCommand(checkCmd)
 }

cmd/gotk/install.go

@@ -21,14 +21,14 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/spf13/cobra"
 
 	"github.com/fluxcd/toolkit/internal/flags"
 	"github.com/fluxcd/toolkit/internal/utils"
-	"github.com/fluxcd/toolkit/pkg/install"
+	"github.com/fluxcd/toolkit/pkg/manifestgen/install"
 )
 
 var installCmd = &cobra.Command{
@@ -59,10 +59,10 @@ var (
 	installComponents         []string
 	installRegistry           string
 	installImagePullSecret    string
-	installArch               flags.Arch = "amd64"
 	installWatchAllNamespaces bool
 	installNetworkPolicy      bool
-	installLogLevel           flags.LogLevel = "info"
+	installArch               = flags.Arch(defaults.Arch)
+	installLogLevel           = flags.LogLevel(defaults.LogLevel)
 )
 
 func init() {
@@ -70,21 +70,21 @@ func init() {
 		"write the install manifests to stdout and exit")
 	installCmd.Flags().BoolVarP(&installDryRun, "dry-run", "", false,
 		"only print the object that would be applied")
-	installCmd.Flags().StringVarP(&installVersion, "version", "v", defaultVersion,
+	installCmd.Flags().StringVarP(&installVersion, "version", "v", defaults.Version,
 		"toolkit version")
-	installCmd.Flags().StringSliceVar(&installComponents, "components", defaultComponents,
+	installCmd.Flags().StringSliceVar(&installComponents, "components", defaults.Components,
 		"list of components, accepts comma-separated values")
 	installCmd.Flags().StringVar(&installManifestsPath, "manifests", "", "path to the manifest directory")
 	installCmd.Flags().MarkHidden("manifests")
-	installCmd.Flags().StringVar(&installRegistry, "registry", "ghcr.io/fluxcd",
+	installCmd.Flags().StringVar(&installRegistry, "registry", defaults.Registry,
 		"container registry where the toolkit images are published")
 	installCmd.Flags().StringVar(&installImagePullSecret, "image-pull-secret", "",
 		"Kubernetes secret name used for pulling the toolkit images from a private registry")
 	installCmd.Flags().Var(&installArch, "arch", installArch.Description())
-	installCmd.Flags().BoolVar(&installWatchAllNamespaces, "watch-all-namespaces", true,
+	installCmd.Flags().BoolVar(&installWatchAllNamespaces, "watch-all-namespaces", defaults.WatchAllNamespaces,
 		"watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed")
 	installCmd.Flags().Var(&installLogLevel, "log-level", installLogLevel.Description())
-	installCmd.Flags().BoolVar(&installNetworkPolicy, "network-policy", true,
+	installCmd.Flags().BoolVar(&installNetworkPolicy, "network-policy", defaults.NetworkPolicy,
 		"deny ingress access to the toolkit controllers from other namespaces using network policies")
 	rootCmd.AddCommand(installCmd)
 }
@@ -114,8 +114,8 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
 		WatchAllNamespaces:     installWatchAllNamespaces,
 		NetworkPolicy:          installNetworkPolicy,
 		LogLevel:               installLogLevel.String(),
-		NotificationController: defaultNotification,
+		NotificationController: defaults.NotificationController,
-		ManifestsFile:          fmt.Sprintf("%s.yaml", namespace),
+		ManifestFile:           fmt.Sprintf("%s.yaml", namespace),
 		Timeout:                timeout,
 	}
 
@@ -123,23 +123,22 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
 		opts.BaseURL = install.MakeDefaultOptions().BaseURL
 	}
 
-	_, content, err := install.Generate(opts)
+	manifest, err := install.Generate(opts)
 	if err != nil {
 		return fmt.Errorf("install failed: %w", err)
 	}
 
-	manifest := path.Join(tmpDir, fmt.Sprintf("%s.yaml", namespace))
+	if _, err := manifest.WriteFile(tmpDir); err != nil {
-	if err := ioutil.WriteFile(manifest, []byte(content), os.ModePerm); err != nil {
 		return fmt.Errorf("install failed: %w", err)
 	}
 
 	if verbose {
-		fmt.Print(content)
+		fmt.Print(manifest.Content)
 	} else if installExport {
 		fmt.Println("---")
 		fmt.Println("# GitOps Toolkit revision", installVersion)
 		fmt.Println("# Components:", strings.Join(installComponents, ","))
-		fmt.Print(content)
+		fmt.Print(manifest.Content)
 		fmt.Println("---")
 		return nil
 	}
@@ -151,7 +150,7 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
 		applyOutput = utils.ModeOS
 	}
 
-	kubectlArgs := []string{"apply", "-f", manifest}
+	kubectlArgs := []string{"apply", "-f", filepath.Join(tmpDir, manifest.Path)}
 	if installDryRun {
 		args = append(args, "--dry-run=client")
 		applyOutput = utils.ModeOS

cmd/gotk/main.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"github.com/fluxcd/toolkit/pkg/manifestgen/install"
 	"log"
 	"os"
 	"path/filepath"
@@ -100,17 +101,11 @@ var (
 	verbose      bool
 	pollInterval                = 2 * time.Second
 	logger       gotklog.Logger = printLogger{}
-)
+	defaults                    = install.MakeDefaultOptions()
-
-var (
-	defaultComponents   = []string{"source-controller", "kustomize-controller", "helm-controller", "notification-controller"}
-	defaultVersion      = "latest"
-	defaultNamespace    = "gotk-system"
-	defaultNotification = "notification-controller"
 )
 
 func init() {
-	rootCmd.PersistentFlags().StringVarP(&namespace, "namespace", "n", defaultNamespace, "the namespace scope for this operation")
+	rootCmd.PersistentFlags().StringVarP(&namespace, "namespace", "n", defaults.Namespace, "the namespace scope for this operation")
 	rootCmd.PersistentFlags().DurationVar(&timeout, "timeout", 5*time.Minute, "timeout for this operation")
 	rootCmd.PersistentFlags().BoolVar(&verbose, "verbose", false, "print generated objects")
 }

pkg/manifestgen/doc.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 The Flux CD contributors.
+Copyright 2020 The Flux authors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,19 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package sync
+// Package manifestgen generates Kubernetes manifests for gotk install
-
+// and the Git source and Kustomization manifests for gotk bootstrap.
-import (
+package manifestgen
-	"fmt"
-	"testing"
-)
-
-func TestGenerate(t *testing.T) {
-	opts := MakeDefaultOptions()
-	output, err := Generate(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	fmt.Println(output)
-}

pkg/manifestgen/install/install.go

@@ -23,45 +23,50 @@ import (
 	"os"
 	"path"
 	"strings"
+
+	"github.com/fluxcd/toolkit/pkg/manifestgen"
 )
 
 // Generate returns the install manifests as a multi-doc YAML.
 // The manifests are built from a GitHub release or from a
 // Kustomize overlay if the supplied Options.BaseURL is a local path.
-func Generate(options Options) (string, string, error) {
+func Generate(options Options) (*manifestgen.Manifest, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), options.Timeout)
 	defer cancel()
 
 	tmpDir, err := ioutil.TempDir("", options.Namespace)
 	if err != nil {
-		return "", "", fmt.Errorf("temp dir error: %w", err)
+		return nil, fmt.Errorf("temp dir error: %w", err)
 	}
 	defer os.RemoveAll(tmpDir)
 
-	output := path.Join(tmpDir, options.ManifestsFile)
+	output := path.Join(tmpDir, options.ManifestFile)
 
 	if !strings.HasPrefix(options.BaseURL, "http") {
 		if err := build(options.BaseURL, output); err != nil {
-			return "", "", err
+			return nil, err
 		}
 	} else {
 		if err := fetch(ctx, options.BaseURL, options.Version, tmpDir); err != nil {
-			return "", "", err
+			return nil, err
 		}
 
 		if err := generate(tmpDir, options); err != nil {
-			return "", "", err
+			return nil, err
 		}
 
 		if err := build(tmpDir, output); err != nil {
-			return "", "", err
+			return nil, err
 		}
 	}
 
 	content, err := ioutil.ReadFile(output)
 	if err != nil {
-		return "", "", err
+		return nil, err
 	}
 
-	return path.Join(options.TargetPath, options.Namespace, options.ManifestsFile), string(content), nil
+	return &manifestgen.Manifest{
+		Path:    path.Join(options.TargetPath, options.Namespace, options.ManifestFile),
+		Content: string(content),
+	}, nil
 }

pkg/manifestgen/install/install_test.go

@@ -24,17 +24,17 @@ import (
 
 func TestGenerate(t *testing.T) {
 	opts := MakeDefaultOptions()
-	_, output, err := Generate(opts)
+	output, err := Generate(opts)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	for _, component := range opts.Components {
 		img := fmt.Sprintf("%s/%s", opts.Registry, component)
-		if !strings.Contains(string(output), img) {
+		if !strings.Contains(output.Content, img) {
 			t.Errorf("component image '%s' not found", img)
 		}
 	}
 
-	fmt.Println(string(output))
+	fmt.Println(output)
 }

pkg/manifestgen/install/options.go

@@ -31,7 +31,7 @@ type Options struct {
 	NetworkPolicy          bool
 	LogLevel               string
 	NotificationController string
-	ManifestsFile          string
+	ManifestFile           string
 	Timeout                time.Duration
 	TargetPath             string
 }
@@ -50,7 +50,7 @@ func MakeDefaultOptions() Options {
 		LogLevel:               "info",
 		BaseURL:                "https://github.com/fluxcd/toolkit/releases",
 		NotificationController: "notification-controller",
-		ManifestsFile:          "toolkit-components.yaml",
+		ManifestFile:           "gotk-components.yaml",
 		Timeout:                time.Minute,
 		TargetPath:             "",
 	}

pkg/manifestgen/manifest.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2020 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package manifestgen
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+)
+
+// Manifest holds the data of a multi-doc YAML
+type Manifest struct {
+	// Relative path to the YAML file
+	Path string
+	// Content in YAML format
+	Content string
+}
+
+// WriteFile writes the YAML content to a file inside the the root path.
+// If the file does not exist, WriteFile creates it with permissions perm,
+// otherwise WriteFile overwrites the file, without changing permissions.
+func (m *Manifest) WriteFile(rootDir string) (string, error) {
+	if err := os.MkdirAll(path.Join(rootDir, filepath.Dir(m.Path)), os.ModePerm); err != nil {
+		return "", fmt.Errorf("unable to create dir, error: %w", err)
+	}
+
+	filePath := path.Join(rootDir, m.Path)
+	if err := ioutil.WriteFile(filePath, []byte(m.Content), os.ModePerm); err != nil {
+		return "", fmt.Errorf("unable to write file, error: %w", err)
+	}
+
+	return filePath, nil
+}

pkg/manifestgen/sync/options.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2020 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sync
+
+import "time"
+
+type Options struct {
+	Interval     time.Duration
+	URL          string
+	Name         string
+	Namespace    string
+	Branch       string
+	TargetPath   string
+	ManifestFile string
+}
+
+func MakeDefaultOptions() Options {
+	return Options{
+		Interval:     1 * time.Minute,
+		URL:          "",
+		Name:         "gotk-system",
+		Namespace:    "gotk-system",
+		Branch:       "main",
+		ManifestFile: "gotk-sync.yaml",
+		TargetPath:   "",
+	}
+}

pkg/manifestgen/sync/sync.go

@@ -1,6 +1,23 @@
+/*
+Copyright 2020 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package sync
 
 import (
+	"bytes"
 	"fmt"
 	"path/filepath"
 	"strings"
@@ -12,16 +29,11 @@ import (
 
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
-)
 
-const (
+	"github.com/fluxcd/toolkit/pkg/manifestgen"
-	bootstrapSourceManifest        = "toolkit-source.yaml"
-	bootstrapKustomizationManifest = "toolkit-kustomization.yaml"
 )
 
-func Generate(options Options) ([]map[string]string, error) {
+func Generate(options Options) (*manifestgen.Manifest, error) {
-	files := []map[string]string{}
-
 	gvk := sourcev1.GroupVersion.WithKind(sourcev1.GitRepositoryKind)
 	gitRepository := sourcev1.GitRepository{
 		TypeMeta: metav1.TypeMeta{
@@ -51,8 +63,6 @@ func Generate(options Options) ([]map[string]string, error) {
 		return nil, err
 	}
 
-	files = append(files, map[string]string{"file_path": filepath.Join(options.TargetPath, options.Namespace, bootstrapSourceManifest), "content": string(gitData)})
-
 	gvk = kustomizev1.GroupVersion.WithKind(kustomizev1.KustomizationKind)
 	kustomization := kustomizev1.Kustomization{
 		TypeMeta: metav1.TypeMeta{
@@ -82,7 +92,14 @@ func Generate(options Options) ([]map[string]string, error) {
 		return nil, err
 	}
 
-	files = append(files, map[string]string{"file_path": filepath.Join(options.TargetPath, options.Namespace, bootstrapKustomizationManifest), "content": string(ksData)})
+	return &manifestgen.Manifest{
+		Path:    filepath.Join(options.TargetPath, options.Namespace, options.ManifestFile),
+		Content: fmt.Sprintf("---\n%s---\n%s", resourceToString(gitData), resourceToString(ksData)),
+	}, nil
+}
 
-	return files, nil
+func resourceToString(data []byte) string {
+	data = bytes.Replace(data, []byte("  creationTimestamp: null\n"), []byte(""), 1)
+	data = bytes.Replace(data, []byte("status: {}\n"), []byte(""), 1)
+	return string(data)
 }

pkg/manifestgen/sync/sync_test.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2020 The Flux CD contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sync
+
+import (
+	"fmt"
+	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1"
+	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
+	"strings"
+	"testing"
+)
+
+func TestGenerate(t *testing.T) {
+	opts := MakeDefaultOptions()
+	output, err := Generate(opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, apiVersion := range []string{sourcev1.GroupVersion.String(), kustomizev1.GroupVersion.String()} {
+		if !strings.Contains(output.Content, apiVersion) {
+			t.Errorf("apiVersion '%s' not found", apiVersion)
+		}
+	}
+
+	fmt.Println(output.Content)
+}

pkg/sync/options.go

@@ -1,23 +0,0 @@
-package sync
-
-import "time"
-
-type Options struct {
-	Interval   time.Duration
-	URL        string
-	Name       string
-	Namespace  string
-	Branch     string
-	TargetPath string
-}
-
-func MakeDefaultOptions() Options {
-	return Options{
-		Interval:   1 * time.Minute,
-		URL:        "",
-		Name:       "gotk-system",
-		Namespace:  "gotk-system",
-		Branch:     "main",
-		TargetPath: "",
-	}
-}