Refactor manifests generation

- introduce manifestgen pkg, to be consumed by the CLI and Terraform provider
- consolidate defaults in manifestgen/install pkg
- introduce Manifest as the returning type of manifest generation
- add helper function to Manifest for writing multi-doc YAMLs on disk

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>
pull/387/head
Stefan Prodan 4 years ago
parent 41d4e7e15b
commit 9bc250d027
No known key found for this signature in database
GPG Key ID: 3299AEB0E4085BAF

@ -19,10 +19,7 @@ package main
import (
"context"
"fmt"
"io/ioutil"
"net/url"
"os"
"path"
"path/filepath"
"time"
@ -38,8 +35,8 @@ import (
"github.com/fluxcd/toolkit/internal/flags"
"github.com/fluxcd/toolkit/internal/utils"
"github.com/fluxcd/toolkit/pkg/install"
"github.com/fluxcd/toolkit/pkg/sync"
"github.com/fluxcd/toolkit/pkg/manifestgen/install"
"github.com/fluxcd/toolkit/pkg/manifestgen/sync"
)
var bootstrapCmd = &cobra.Command{
@ -53,12 +50,12 @@ var (
bootstrapComponents []string
bootstrapRegistry string
bootstrapImagePullSecret string
bootstrapArch flags.Arch = "amd64"
bootstrapBranch string
bootstrapWatchAllNamespaces bool
bootstrapNetworkPolicy bool
bootstrapLogLevel flags.LogLevel = "info"
bootstrapManifestsPath string
bootstrapArch = flags.Arch(defaults.Arch)
bootstrapLogLevel = flags.LogLevel(defaults.LogLevel)
bootstrapRequiredComponents = []string{"source-controller", "kustomize-controller"}
)
@ -67,9 +64,9 @@ const (
)
func init() {
bootstrapCmd.PersistentFlags().StringVarP(&bootstrapVersion, "version", "v", defaultVersion,
bootstrapCmd.PersistentFlags().StringVarP(&bootstrapVersion, "version", "v", defaults.Version,
"toolkit version")
bootstrapCmd.PersistentFlags().StringSliceVar(&bootstrapComponents, "components", defaultComponents,
bootstrapCmd.PersistentFlags().StringSliceVar(&bootstrapComponents, "components", defaults.Components,
"list of components, accepts comma-separated values")
bootstrapCmd.PersistentFlags().StringVar(&bootstrapRegistry, "registry", "ghcr.io/fluxcd",
"container registry where the toolkit images are published")
@ -110,31 +107,27 @@ func generateInstallManifests(targetPath, namespace, tmpDir string, localManifes
WatchAllNamespaces: bootstrapWatchAllNamespaces,
NetworkPolicy: bootstrapNetworkPolicy,
LogLevel: bootstrapLogLevel.String(),
NotificationController: defaultNotification,
ManifestsFile: fmt.Sprintf("%s.yaml", namespace),
NotificationController: defaults.NotificationController,
ManifestFile: defaults.ManifestFile,
Timeout: timeout,
TargetPath: targetPath,
}
if localManifests == "" {
opts.BaseURL = install.MakeDefaultOptions().BaseURL
opts.BaseURL = defaults.BaseURL
}
manifestPath, content, err := install.Generate(opts)
output, err := install.Generate(opts)
if err != nil {
return "", fmt.Errorf("generating install manifests failed: %w", err)
}
filePath := path.Join(tmpDir, manifestPath)
if err := os.MkdirAll(path.Dir(manifestPath), os.ModePerm); err != nil {
return "", fmt.Errorf("creating manifest dir failed: %w", err)
}
if err := ioutil.WriteFile(filePath, []byte(content), os.ModePerm); err != nil {
if filePath, err := output.WriteFile(tmpDir); err != nil {
return "", fmt.Errorf("generating install manifests failed: %w", err)
} else {
return filePath, nil
}
return filePath, nil
}
func applyInstallManifests(ctx context.Context, manifestPath string, components []string) error {
@ -154,23 +147,22 @@ func applyInstallManifests(ctx context.Context, manifestPath string, components
func generateSyncManifests(url, branch, name, namespace, targetPath, tmpDir string, interval time.Duration) error {
opts := sync.Options{
Name: name,
Namespace: namespace,
URL: url,
Branch: branch,
Interval: interval,
TargetPath: targetPath,
Name: name,
Namespace: namespace,
URL: url,
Branch: branch,
Interval: interval,
TargetPath: targetPath,
ManifestFile: sync.MakeDefaultOptions().ManifestFile,
}
output, err := sync.Generate(opts)
manifest, err := sync.Generate(opts)
if err != nil {
return fmt.Errorf("generating install manifests failed: %w", err)
}
for _, v := range output {
if err := utils.WriteFile(v["content"], filepath.Join(tmpDir, v["file_path"])); err != nil {
return err
}
if _, err := manifest.WriteFile(tmpDir); err != nil {
return err
}
if err := utils.GenerateKustomizationYaml(filepath.Join(tmpDir, targetPath, namespace)); err != nil {

@ -57,7 +57,7 @@ type kubectlVersion struct {
func init() {
checkCmd.Flags().BoolVarP(&checkPre, "pre", "", false,
"only run pre-installation checks")
checkCmd.Flags().StringSliceVar(&checkComponents, "components", defaultComponents,
checkCmd.Flags().StringSliceVar(&checkComponents, "components", defaults.Components,
"list of components, accepts comma-separated values")
rootCmd.AddCommand(checkCmd)
}

@ -21,14 +21,14 @@ import (
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"github.com/spf13/cobra"
"github.com/fluxcd/toolkit/internal/flags"
"github.com/fluxcd/toolkit/internal/utils"
"github.com/fluxcd/toolkit/pkg/install"
"github.com/fluxcd/toolkit/pkg/manifestgen/install"
)
var installCmd = &cobra.Command{
@ -59,10 +59,10 @@ var (
installComponents []string
installRegistry string
installImagePullSecret string
installArch flags.Arch = "amd64"
installWatchAllNamespaces bool
installNetworkPolicy bool
installLogLevel flags.LogLevel = "info"
installArch = flags.Arch(defaults.Arch)
installLogLevel = flags.LogLevel(defaults.LogLevel)
)
func init() {
@ -70,21 +70,21 @@ func init() {
"write the install manifests to stdout and exit")
installCmd.Flags().BoolVarP(&installDryRun, "dry-run", "", false,
"only print the object that would be applied")
installCmd.Flags().StringVarP(&installVersion, "version", "v", defaultVersion,
installCmd.Flags().StringVarP(&installVersion, "version", "v", defaults.Version,
"toolkit version")
installCmd.Flags().StringSliceVar(&installComponents, "components", defaultComponents,
installCmd.Flags().StringSliceVar(&installComponents, "components", defaults.Components,
"list of components, accepts comma-separated values")
installCmd.Flags().StringVar(&installManifestsPath, "manifests", "", "path to the manifest directory")
installCmd.Flags().MarkHidden("manifests")
installCmd.Flags().StringVar(&installRegistry, "registry", "ghcr.io/fluxcd",
installCmd.Flags().StringVar(&installRegistry, "registry", defaults.Registry,
"container registry where the toolkit images are published")
installCmd.Flags().StringVar(&installImagePullSecret, "image-pull-secret", "",
"Kubernetes secret name used for pulling the toolkit images from a private registry")
installCmd.Flags().Var(&installArch, "arch", installArch.Description())
installCmd.Flags().BoolVar(&installWatchAllNamespaces, "watch-all-namespaces", true,
installCmd.Flags().BoolVar(&installWatchAllNamespaces, "watch-all-namespaces", defaults.WatchAllNamespaces,
"watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed")
installCmd.Flags().Var(&installLogLevel, "log-level", installLogLevel.Description())
installCmd.Flags().BoolVar(&installNetworkPolicy, "network-policy", true,
installCmd.Flags().BoolVar(&installNetworkPolicy, "network-policy", defaults.NetworkPolicy,
"deny ingress access to the toolkit controllers from other namespaces using network policies")
rootCmd.AddCommand(installCmd)
}
@ -114,8 +114,8 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
WatchAllNamespaces: installWatchAllNamespaces,
NetworkPolicy: installNetworkPolicy,
LogLevel: installLogLevel.String(),
NotificationController: defaultNotification,
ManifestsFile: fmt.Sprintf("%s.yaml", namespace),
NotificationController: defaults.NotificationController,
ManifestFile: fmt.Sprintf("%s.yaml", namespace),
Timeout: timeout,
}
@ -123,23 +123,22 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
opts.BaseURL = install.MakeDefaultOptions().BaseURL
}
_, content, err := install.Generate(opts)
manifest, err := install.Generate(opts)
if err != nil {
return fmt.Errorf("install failed: %w", err)
}
manifest := path.Join(tmpDir, fmt.Sprintf("%s.yaml", namespace))
if err := ioutil.WriteFile(manifest, []byte(content), os.ModePerm); err != nil {
if _, err := manifest.WriteFile(tmpDir); err != nil {
return fmt.Errorf("install failed: %w", err)
}
if verbose {
fmt.Print(content)
fmt.Print(manifest.Content)
} else if installExport {
fmt.Println("---")
fmt.Println("# GitOps Toolkit revision", installVersion)
fmt.Println("# Components:", strings.Join(installComponents, ","))
fmt.Print(content)
fmt.Print(manifest.Content)
fmt.Println("---")
return nil
}
@ -151,7 +150,7 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
applyOutput = utils.ModeOS
}
kubectlArgs := []string{"apply", "-f", manifest}
kubectlArgs := []string{"apply", "-f", filepath.Join(tmpDir, manifest.Path)}
if installDryRun {
args = append(args, "--dry-run=client")
applyOutput = utils.ModeOS

@ -17,6 +17,7 @@ limitations under the License.
package main
import (
"github.com/fluxcd/toolkit/pkg/manifestgen/install"
"log"
"os"
"path/filepath"
@ -100,17 +101,11 @@ var (
verbose bool
pollInterval = 2 * time.Second
logger gotklog.Logger = printLogger{}
)
var (
defaultComponents = []string{"source-controller", "kustomize-controller", "helm-controller", "notification-controller"}
defaultVersion = "latest"
defaultNamespace = "gotk-system"
defaultNotification = "notification-controller"
defaults = install.MakeDefaultOptions()
)
func init() {
rootCmd.PersistentFlags().StringVarP(&namespace, "namespace", "n", defaultNamespace, "the namespace scope for this operation")
rootCmd.PersistentFlags().StringVarP(&namespace, "namespace", "n", defaults.Namespace, "the namespace scope for this operation")
rootCmd.PersistentFlags().DurationVar(&timeout, "timeout", 5*time.Minute, "timeout for this operation")
rootCmd.PersistentFlags().BoolVar(&verbose, "verbose", false, "print generated objects")
}

@ -1,5 +1,5 @@
/*
Copyright 2020 The Flux CD contributors.
Copyright 2020 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -14,19 +14,6 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package sync
import (
"fmt"
"testing"
)
func TestGenerate(t *testing.T) {
opts := MakeDefaultOptions()
output, err := Generate(opts)
if err != nil {
t.Fatal(err)
}
fmt.Println(output)
}
// Package manifestgen generates Kubernetes manifests for gotk install
// and the Git source and Kustomization manifests for gotk bootstrap.
package manifestgen

@ -23,45 +23,50 @@ import (
"os"
"path"
"strings"
"github.com/fluxcd/toolkit/pkg/manifestgen"
)
// Generate returns the install manifests as a multi-doc YAML.
// The manifests are built from a GitHub release or from a
// Kustomize overlay if the supplied Options.BaseURL is a local path.
func Generate(options Options) (string, string, error) {
func Generate(options Options) (*manifestgen.Manifest, error) {
ctx, cancel := context.WithTimeout(context.Background(), options.Timeout)
defer cancel()
tmpDir, err := ioutil.TempDir("", options.Namespace)
if err != nil {
return "", "", fmt.Errorf("temp dir error: %w", err)
return nil, fmt.Errorf("temp dir error: %w", err)
}
defer os.RemoveAll(tmpDir)
output := path.Join(tmpDir, options.ManifestsFile)
output := path.Join(tmpDir, options.ManifestFile)
if !strings.HasPrefix(options.BaseURL, "http") {
if err := build(options.BaseURL, output); err != nil {
return "", "", err
return nil, err
}
} else {
if err := fetch(ctx, options.BaseURL, options.Version, tmpDir); err != nil {
return "", "", err
return nil, err
}
if err := generate(tmpDir, options); err != nil {
return "", "", err
return nil, err
}
if err := build(tmpDir, output); err != nil {
return "", "", err
return nil, err
}
}
content, err := ioutil.ReadFile(output)
if err != nil {
return "", "", err
return nil, err
}
return path.Join(options.TargetPath, options.Namespace, options.ManifestsFile), string(content), nil
return &manifestgen.Manifest{
Path: path.Join(options.TargetPath, options.Namespace, options.ManifestFile),
Content: string(content),
}, nil
}

@ -24,17 +24,17 @@ import (
func TestGenerate(t *testing.T) {
opts := MakeDefaultOptions()
_, output, err := Generate(opts)
output, err := Generate(opts)
if err != nil {
t.Fatal(err)
}
for _, component := range opts.Components {
img := fmt.Sprintf("%s/%s", opts.Registry, component)
if !strings.Contains(string(output), img) {
if !strings.Contains(output.Content, img) {
t.Errorf("component image '%s' not found", img)
}
}
fmt.Println(string(output))
fmt.Println(output)
}

@ -31,7 +31,7 @@ type Options struct {
NetworkPolicy bool
LogLevel string
NotificationController string
ManifestsFile string
ManifestFile string
Timeout time.Duration
TargetPath string
}
@ -50,7 +50,7 @@ func MakeDefaultOptions() Options {
LogLevel: "info",
BaseURL: "https://github.com/fluxcd/toolkit/releases",
NotificationController: "notification-controller",
ManifestsFile: "toolkit-components.yaml",
ManifestFile: "gotk-components.yaml",
Timeout: time.Minute,
TargetPath: "",
}

@ -0,0 +1,49 @@
/*
Copyright 2020 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package manifestgen
import (
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
)
// Manifest holds the data of a multi-doc YAML
type Manifest struct {
// Relative path to the YAML file
Path string
// Content in YAML format
Content string
}
// WriteFile writes the YAML content to a file inside the the root path.
// If the file does not exist, WriteFile creates it with permissions perm,
// otherwise WriteFile overwrites the file, without changing permissions.
func (m *Manifest) WriteFile(rootDir string) (string, error) {
if err := os.MkdirAll(path.Join(rootDir, filepath.Dir(m.Path)), os.ModePerm); err != nil {
return "", fmt.Errorf("unable to create dir, error: %w", err)
}
filePath := path.Join(rootDir, m.Path)
if err := ioutil.WriteFile(filePath, []byte(m.Content), os.ModePerm); err != nil {
return "", fmt.Errorf("unable to write file, error: %w", err)
}
return filePath, nil
}

@ -0,0 +1,41 @@
/*
Copyright 2020 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sync
import "time"
type Options struct {
Interval time.Duration
URL string
Name string
Namespace string
Branch string
TargetPath string
ManifestFile string
}
func MakeDefaultOptions() Options {
return Options{
Interval: 1 * time.Minute,
URL: "",
Name: "gotk-system",
Namespace: "gotk-system",
Branch: "main",
ManifestFile: "gotk-sync.yaml",
TargetPath: "",
}
}

@ -1,6 +1,23 @@
/*
Copyright 2020 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sync
import (
"bytes"
"fmt"
"path/filepath"
"strings"
@ -12,16 +29,11 @@ import (
kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
)
const (
bootstrapSourceManifest = "toolkit-source.yaml"
bootstrapKustomizationManifest = "toolkit-kustomization.yaml"
"github.com/fluxcd/toolkit/pkg/manifestgen"
)
func Generate(options Options) ([]map[string]string, error) {
files := []map[string]string{}
func Generate(options Options) (*manifestgen.Manifest, error) {
gvk := sourcev1.GroupVersion.WithKind(sourcev1.GitRepositoryKind)
gitRepository := sourcev1.GitRepository{
TypeMeta: metav1.TypeMeta{
@ -51,8 +63,6 @@ func Generate(options Options) ([]map[string]string, error) {
return nil, err
}
files = append(files, map[string]string{"file_path": filepath.Join(options.TargetPath, options.Namespace, bootstrapSourceManifest), "content": string(gitData)})
gvk = kustomizev1.GroupVersion.WithKind(kustomizev1.KustomizationKind)
kustomization := kustomizev1.Kustomization{
TypeMeta: metav1.TypeMeta{
@ -82,7 +92,14 @@ func Generate(options Options) ([]map[string]string, error) {
return nil, err
}
files = append(files, map[string]string{"file_path": filepath.Join(options.TargetPath, options.Namespace, bootstrapKustomizationManifest), "content": string(ksData)})
return &manifestgen.Manifest{
Path: filepath.Join(options.TargetPath, options.Namespace, options.ManifestFile),
Content: fmt.Sprintf("---\n%s---\n%s", resourceToString(gitData), resourceToString(ksData)),
}, nil
}
return files, nil
func resourceToString(data []byte) string {
data = bytes.Replace(data, []byte(" creationTimestamp: null\n"), []byte(""), 1)
data = bytes.Replace(data, []byte("status: {}\n"), []byte(""), 1)
return string(data)
}

@ -0,0 +1,41 @@
/*
Copyright 2020 The Flux CD contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sync
import (
"fmt"
kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
"strings"
"testing"
)
func TestGenerate(t *testing.T) {
opts := MakeDefaultOptions()
output, err := Generate(opts)
if err != nil {
t.Fatal(err)
}
for _, apiVersion := range []string{sourcev1.GroupVersion.String(), kustomizev1.GroupVersion.String()} {
if !strings.Contains(output.Content, apiVersion) {
t.Errorf("apiVersion '%s' not found", apiVersion)
}
}
fmt.Println(output.Content)
}

@ -1,23 +0,0 @@
package sync
import "time"
type Options struct {
Interval time.Duration
URL string
Name string
Namespace string
Branch string
TargetPath string
}
func MakeDefaultOptions() Options {
return Options{
Interval: 1 * time.Minute,
URL: "",
Name: "gotk-system",
Namespace: "gotk-system",
Branch: "main",
TargetPath: "",
}
}
Loading…
Cancel
Save