Refactor manifests generation

- introduce manifestgen pkg, to be consumed by the CLI and Terraform provider
- consolidate defaults in manifestgen/install pkg
- introduce Manifest as the returning type of manifest generation
- add helper function to Manifest for writing multi-doc YAMLs on disk

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>

pkg/manifestgen/doc.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 The Flux CD contributors.
+Copyright 2020 The Flux authors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,19 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package sync
-
-import (
-	"fmt"
-	"testing"
-)
-
-func TestGenerate(t *testing.T) {
-	opts := MakeDefaultOptions()
-	output, err := Generate(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	fmt.Println(output)
-}
+// Package manifestgen generates Kubernetes manifests for gotk install
+// and the Git source and Kustomization manifests for gotk bootstrap.
+package manifestgen

pkg/manifestgen/install/install.go

@@ -23,45 +23,50 @@ import (
 	"os"
 	"path"
 	"strings"
+
+	"github.com/fluxcd/toolkit/pkg/manifestgen"
 )
 
 // Generate returns the install manifests as a multi-doc YAML.
 // The manifests are built from a GitHub release or from a
 // Kustomize overlay if the supplied Options.BaseURL is a local path.
-func Generate(options Options) (string, string, error) {
+func Generate(options Options) (*manifestgen.Manifest, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), options.Timeout)
 	defer cancel()
 
 	tmpDir, err := ioutil.TempDir("", options.Namespace)
 	if err != nil {
-		return "", "", fmt.Errorf("temp dir error: %w", err)
+		return nil, fmt.Errorf("temp dir error: %w", err)
 	}
 	defer os.RemoveAll(tmpDir)
 
-	output := path.Join(tmpDir, options.ManifestsFile)
+	output := path.Join(tmpDir, options.ManifestFile)
 
 	if !strings.HasPrefix(options.BaseURL, "http") {
 		if err := build(options.BaseURL, output); err != nil {
-			return "", "", err
+			return nil, err
 		}
 	} else {
 		if err := fetch(ctx, options.BaseURL, options.Version, tmpDir); err != nil {
-			return "", "", err
+			return nil, err
 		}
 
 		if err := generate(tmpDir, options); err != nil {
-			return "", "", err
+			return nil, err
 		}
 
 		if err := build(tmpDir, output); err != nil {
-			return "", "", err
+			return nil, err
 		}
 	}
 
 	content, err := ioutil.ReadFile(output)
 	if err != nil {
-		return "", "", err
+		return nil, err
 	}
 
-	return path.Join(options.TargetPath, options.Namespace, options.ManifestsFile), string(content), nil
+	return &manifestgen.Manifest{
+		Path:    path.Join(options.TargetPath, options.Namespace, options.ManifestFile),
+		Content: string(content),
+	}, nil
 }

pkg/manifestgen/install/install_test.go

@@ -24,17 +24,17 @@ import (
 
 func TestGenerate(t *testing.T) {
 	opts := MakeDefaultOptions()
-	_, output, err := Generate(opts)
+	output, err := Generate(opts)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	for _, component := range opts.Components {
 		img := fmt.Sprintf("%s/%s", opts.Registry, component)
-		if !strings.Contains(string(output), img) {
+		if !strings.Contains(output.Content, img) {
 			t.Errorf("component image '%s' not found", img)
 		}
 	}
 
-	fmt.Println(string(output))
+	fmt.Println(output)
 }

pkg/manifestgen/install/options.go

@@ -31,7 +31,7 @@ type Options struct {
 	NetworkPolicy          bool
 	LogLevel               string
 	NotificationController string
-	ManifestsFile          string
+	ManifestFile           string
 	Timeout                time.Duration
 	TargetPath             string
 }
@@ -50,7 +50,7 @@ func MakeDefaultOptions() Options {
 		LogLevel:               "info",
 		BaseURL:                "https://github.com/fluxcd/toolkit/releases",
 		NotificationController: "notification-controller",
-		ManifestsFile:          "toolkit-components.yaml",
+		ManifestFile:           "gotk-components.yaml",
 		Timeout:                time.Minute,
 		TargetPath:             "",
 	}

pkg/manifestgen/manifest.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2020 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package manifestgen
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+)
+
+// Manifest holds the data of a multi-doc YAML
+type Manifest struct {
+	// Relative path to the YAML file
+	Path string
+	// Content in YAML format
+	Content string
+}
+
+// WriteFile writes the YAML content to a file inside the the root path.
+// If the file does not exist, WriteFile creates it with permissions perm,
+// otherwise WriteFile overwrites the file, without changing permissions.
+func (m *Manifest) WriteFile(rootDir string) (string, error) {
+	if err := os.MkdirAll(path.Join(rootDir, filepath.Dir(m.Path)), os.ModePerm); err != nil {
+		return "", fmt.Errorf("unable to create dir, error: %w", err)
+	}
+
+	filePath := path.Join(rootDir, m.Path)
+	if err := ioutil.WriteFile(filePath, []byte(m.Content), os.ModePerm); err != nil {
+		return "", fmt.Errorf("unable to write file, error: %w", err)
+	}
+
+	return filePath, nil
+}

pkg/manifestgen/sync/options.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2020 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sync
+
+import "time"
+
+type Options struct {
+	Interval     time.Duration
+	URL          string
+	Name         string
+	Namespace    string
+	Branch       string
+	TargetPath   string
+	ManifestFile string
+}
+
+func MakeDefaultOptions() Options {
+	return Options{
+		Interval:     1 * time.Minute,
+		URL:          "",
+		Name:         "gotk-system",
+		Namespace:    "gotk-system",
+		Branch:       "main",
+		ManifestFile: "gotk-sync.yaml",
+		TargetPath:   "",
+	}
+}

pkg/manifestgen/sync/sync.go

@@ -1,6 +1,23 @@
+/*
+Copyright 2020 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package sync
 
 import (
+	"bytes"
 	"fmt"
 	"path/filepath"
 	"strings"
@@ -12,16 +29,11 @@ import (
 
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
+
+	"github.com/fluxcd/toolkit/pkg/manifestgen"
 )
 
-const (
-	bootstrapSourceManifest        = "toolkit-source.yaml"
-	bootstrapKustomizationManifest = "toolkit-kustomization.yaml"
-)
-
-func Generate(options Options) ([]map[string]string, error) {
-	files := []map[string]string{}
-
+func Generate(options Options) (*manifestgen.Manifest, error) {
 	gvk := sourcev1.GroupVersion.WithKind(sourcev1.GitRepositoryKind)
 	gitRepository := sourcev1.GitRepository{
 		TypeMeta: metav1.TypeMeta{
@@ -51,8 +63,6 @@ func Generate(options Options) ([]map[string]string, error) {
 		return nil, err
 	}
 
-	files = append(files, map[string]string{"file_path": filepath.Join(options.TargetPath, options.Namespace, bootstrapSourceManifest), "content": string(gitData)})
-
 	gvk = kustomizev1.GroupVersion.WithKind(kustomizev1.KustomizationKind)
 	kustomization := kustomizev1.Kustomization{
 		TypeMeta: metav1.TypeMeta{
@@ -82,7 +92,14 @@ func Generate(options Options) ([]map[string]string, error) {
 		return nil, err
 	}
 
-	files = append(files, map[string]string{"file_path": filepath.Join(options.TargetPath, options.Namespace, bootstrapKustomizationManifest), "content": string(ksData)})
-
-	return files, nil
+	return &manifestgen.Manifest{
+		Path:    filepath.Join(options.TargetPath, options.Namespace, options.ManifestFile),
+		Content: fmt.Sprintf("---\n%s---\n%s", resourceToString(gitData), resourceToString(ksData)),
+	}, nil
+}
+
+func resourceToString(data []byte) string {
+	data = bytes.Replace(data, []byte("  creationTimestamp: null\n"), []byte(""), 1)
+	data = bytes.Replace(data, []byte("status: {}\n"), []byte(""), 1)
+	return string(data)
 }

pkg/manifestgen/sync/sync_test.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2020 The Flux CD contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sync
+
+import (
+	"fmt"
+	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1"
+	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
+	"strings"
+	"testing"
+)
+
+func TestGenerate(t *testing.T) {
+	opts := MakeDefaultOptions()
+	output, err := Generate(opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, apiVersion := range []string{sourcev1.GroupVersion.String(), kustomizev1.GroupVersion.String()} {
+		if !strings.Contains(output.Content, apiVersion) {
+			t.Errorf("apiVersion '%s' not found", apiVersion)
+		}
+	}
+
+	fmt.Println(output.Content)
+}

pkg/sync/options.go

@@ -1,23 +0,0 @@
-package sync
-
-import "time"
-
-type Options struct {
-	Interval   time.Duration
-	URL        string
-	Name       string
-	Namespace  string
-	Branch     string
-	TargetPath string
-}
-
-func MakeDefaultOptions() Options {
-	return Options{
-		Interval:   1 * time.Minute,
-		URL:        "",
-		Name:       "gotk-system",
-		Namespace:  "gotk-system",
-		Branch:     "main",
-		TargetPath: "",
-	}
-}