From b74638c25c62c7b4e95a2614384b81cd96042e3e Mon Sep 17 00:00:00 2001 From: Soule BA Date: Tue, 4 Apr 2023 18:04:55 +0200 Subject: [PATCH] Add the possibility to ignore files with build and diff ks If implemented, user will be able to ignore files when using `build kustomization` and `diff kustomization` both with .sourceignore and `ignore-paths` flag. Signed-off-by: Soule BA --- cmd/flux/build_kustomization.go | 13 ++++++++++- cmd/flux/build_kustomization_test.go | 6 +++++ cmd/flux/diff_kustomization.go | 18 ++++++++++++--- .../build-kustomization/ignore/.sourceignore | 2 ++ .../build-kustomization/ignore/configmap.yaml | 6 +++++ .../ignore/not_deployable/ignore_svc.yaml | 17 ++++++++++++++ .../build-kustomization/ignore/secret.yaml | 7 ++++++ .../podinfo-with-ignore-result.yaml | 23 +++++++++++++++++++ go.mod | 2 +- go.sum | 4 ++-- internal/build/build.go | 18 +++++++++++++-- 11 files changed, 107 insertions(+), 9 deletions(-) create mode 100644 cmd/flux/testdata/build-kustomization/ignore/.sourceignore create mode 100644 cmd/flux/testdata/build-kustomization/ignore/configmap.yaml create mode 100644 cmd/flux/testdata/build-kustomization/ignore/not_deployable/ignore_svc.yaml create mode 100644 cmd/flux/testdata/build-kustomization/ignore/secret.yaml create mode 100644 cmd/flux/testdata/build-kustomization/podinfo-with-ignore-result.yaml diff --git a/cmd/flux/build_kustomization.go b/cmd/flux/build_kustomization.go index ee29edc7..ea0517c0 100644 --- a/cmd/flux/build_kustomization.go +++ b/cmd/flux/build_kustomization.go @@ -46,7 +46,14 @@ flux build kustomization my-app --path ./path/to/local/manifests --kustomization # Build in dry-run mode without connecting to the cluster. # Note that variable substitutions from Secrets and ConfigMaps are skipped in dry-run mode. -flux build kustomization my-app --path ./path/to/local/manifests --kustomization-file ./path/to/local/my-app.yaml --dry-run`, +flux build kustomization my-app --path ./path/to/local/manifests \ + --kustomization-file ./path/to/local/my-app.yaml \ + --dry-run + +# Exclude files by providing a comma separated list of entries that follow the .gitignore pattern fromat. +flux build kustomization my-app --path ./path/to/local/manifests \ + --kustomization-file ./path/to/local/my-app.yaml \ + --ignore-paths "/to_ignore/**/*.yaml,ignore.yaml"`, ValidArgsFunction: resourceNamesCompletionFunc(kustomizev1.GroupVersion.WithKind(kustomizev1.KustomizationKind)), RunE: buildKsCmdRun, } @@ -54,6 +61,7 @@ flux build kustomization my-app --path ./path/to/local/manifests --kustomization type buildKsFlags struct { kustomizationFile string path string + ignorePaths []string dryRun bool } @@ -62,6 +70,7 @@ var buildKsArgs buildKsFlags func init() { buildKsCmd.Flags().StringVar(&buildKsArgs.path, "path", "", "Path to the manifests location.") buildKsCmd.Flags().StringVar(&buildKsArgs.kustomizationFile, "kustomization-file", "", "Path to the Flux Kustomization YAML file.") + buildKsCmd.Flags().StringSliceVar(&buildKsArgs.ignorePaths, "ignore-paths", nil, "set paths to ignore in .gitignore format") buildKsCmd.Flags().BoolVar(&buildKsArgs.dryRun, "dry-run", false, "Dry run mode.") buildCmd.AddCommand(buildKsCmd) } @@ -97,12 +106,14 @@ func buildKsCmdRun(cmd *cobra.Command, args []string) (err error) { build.WithKustomizationFile(buildKsArgs.kustomizationFile), build.WithDryRun(buildKsArgs.dryRun), build.WithNamespace(*kubeconfigArgs.Namespace), + build.WithIgnore(buildKsArgs.ignorePaths), ) } else { builder, err = build.NewBuilder(name, buildKsArgs.path, build.WithClientConfig(kubeconfigArgs, kubeclientOptions), build.WithTimeout(rootArgs.timeout), build.WithKustomizationFile(buildKsArgs.kustomizationFile), + build.WithIgnore(buildKsArgs.ignorePaths), ) } diff --git a/cmd/flux/build_kustomization_test.go b/cmd/flux/build_kustomization_test.go index 5f134384..d1d06817 100644 --- a/cmd/flux/build_kustomization_test.go +++ b/cmd/flux/build_kustomization_test.go @@ -63,6 +63,12 @@ func TestBuildKustomization(t *testing.T) { resultFile: "./testdata/build-kustomization/podinfo-with-var-substitution-result.yaml", assertFunc: "assertGoldenTemplateFile", }, + { + name: "build ignore", + args: "build kustomization podinfo --path ./testdata/build-kustomization/ignore --ignore-paths \"!configmap.yaml,!secret.yaml\"", + resultFile: "./testdata/build-kustomization/podinfo-with-ignore-result.yaml", + assertFunc: "assertGoldenTemplateFile", + }, } tmpl := map[string]string{ diff --git a/cmd/flux/diff_kustomization.go b/cmd/flux/diff_kustomization.go index 25ca0097..0c4968eb 100644 --- a/cmd/flux/diff_kustomization.go +++ b/cmd/flux/diff_kustomization.go @@ -37,7 +37,13 @@ Exit status: 0 No differences were found. 1 Differences were found. >1 diff fail flux diff kustomization my-app --path ./path/to/local/manifests # Preview using a local flux kustomization file -flux diff kustomization my-app --path ./path/to/local/manifests --kustomization-file ./path/to/local/my-app.yaml`, +flux diff kustomization my-app --path ./path/to/local/manifests \ + --kustomization-file ./path/to/local/my-app.yaml + +# Exclude files by providing a comma separated list of entries that follow the .gitignore pattern fromat. +flux diff kustomization my-app --path ./path/to/local/manifests \ + --kustomization-file ./path/to/local/my-app.yaml \ + --ignore-paths "/to_ignore/**/*.yaml,ignore.yaml"`, ValidArgsFunction: resourceNamesCompletionFunc(kustomizev1.GroupVersion.WithKind(kustomizev1.KustomizationKind)), RunE: diffKsCmdRun, } @@ -45,6 +51,7 @@ flux diff kustomization my-app --path ./path/to/local/manifests --kustomization- type diffKsFlags struct { kustomizationFile string path string + ignorePaths []string progressBar bool } @@ -53,6 +60,7 @@ var diffKsArgs diffKsFlags func init() { diffKsCmd.Flags().StringVar(&diffKsArgs.path, "path", "", "Path to a local directory that matches the specified Kustomization.spec.path.") diffKsCmd.Flags().BoolVar(&diffKsArgs.progressBar, "progress-bar", true, "Boolean to set the progress bar. The default value is true.") + diffKsCmd.Flags().StringSliceVar(&diffKsArgs.ignorePaths, "ignore-paths", nil, "set paths to ignore in .gitignore format") diffKsCmd.Flags().StringVar(&diffKsArgs.kustomizationFile, "kustomization-file", "", "Path to the Flux Kustomization YAML file.") diffCmd.AddCommand(diffKsCmd) } @@ -86,12 +94,16 @@ func diffKsCmdRun(cmd *cobra.Command, args []string) error { build.WithClientConfig(kubeconfigArgs, kubeclientOptions), build.WithTimeout(rootArgs.timeout), build.WithKustomizationFile(diffKsArgs.kustomizationFile), - build.WithProgressBar()) + build.WithProgressBar(), + build.WithIgnore(diffKsArgs.ignorePaths), + ) } else { builder, err = build.NewBuilder(name, diffKsArgs.path, build.WithClientConfig(kubeconfigArgs, kubeclientOptions), build.WithTimeout(rootArgs.timeout), - build.WithKustomizationFile(diffKsArgs.kustomizationFile)) + build.WithKustomizationFile(diffKsArgs.kustomizationFile), + build.WithIgnore(diffKsArgs.ignorePaths), + ) } if err != nil { diff --git a/cmd/flux/testdata/build-kustomization/ignore/.sourceignore b/cmd/flux/testdata/build-kustomization/ignore/.sourceignore new file mode 100644 index 00000000..12bc52ca --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/ignore/.sourceignore @@ -0,0 +1,2 @@ +# exclude all +/* diff --git a/cmd/flux/testdata/build-kustomization/ignore/configmap.yaml b/cmd/flux/testdata/build-kustomization/ignore/configmap.yaml new file mode 100644 index 00000000..7f58221a --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/ignore/configmap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +data: + var: test +kind: ConfigMap +metadata: + name: configmap_ignore diff --git a/cmd/flux/testdata/build-kustomization/ignore/not_deployable/ignore_svc.yaml b/cmd/flux/testdata/build-kustomization/ignore/not_deployable/ignore_svc.yaml new file mode 100644 index 00000000..43fa3984 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/ignore/not_deployable/ignore_svc.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: do_not_deploy +spec: + type: ClusterIP + selector: + app: podinfo + ports: + - name: http + port: 9898 + protocol: TCP + targetPort: http + - port: 9999 + targetPort: grpc + protocol: TCP + name: grpc diff --git a/cmd/flux/testdata/build-kustomization/ignore/secret.yaml b/cmd/flux/testdata/build-kustomization/ignore/secret.yaml new file mode 100644 index 00000000..65cc9d0c --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/ignore/secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +data: + token: KipTT1BTKio= +kind: Secret +metadata: + name: secret_ignore +type: Opaque diff --git a/cmd/flux/testdata/build-kustomization/podinfo-with-ignore-result.yaml b/cmd/flux/testdata/build-kustomization/podinfo-with-ignore-result.yaml new file mode 100644 index 00000000..671d7d20 --- /dev/null +++ b/cmd/flux/testdata/build-kustomization/podinfo-with-ignore-result.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +data: + var: test +kind: ConfigMap +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: configmap_ignore + namespace: default +--- +apiVersion: v1 +data: + token: KipTT1BTKio= +kind: Secret +metadata: + labels: + kustomize.toolkit.fluxcd.io/name: podinfo + kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }} + name: secret_ignore + namespace: default +type: Opaque +--- diff --git a/go.mod b/go.mod index 13298e85..c0e4cd27 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/fluxcd/pkg/apis/meta v1.0.0 github.com/fluxcd/pkg/git v0.11.0 github.com/fluxcd/pkg/git/gogit v0.8.1 - github.com/fluxcd/pkg/kustomize v1.1.0 + github.com/fluxcd/pkg/kustomize v1.1.1 github.com/fluxcd/pkg/oci v0.22.0 github.com/fluxcd/pkg/runtime v0.35.0 github.com/fluxcd/pkg/sourceignore v0.3.3 diff --git a/go.sum b/go.sum index 6e53362d..c3894065 100644 --- a/go.sum +++ b/go.sum @@ -222,8 +222,8 @@ github.com/fluxcd/pkg/git v0.11.0/go.mod h1:VHRVlrZMHNoWBlaSAWxlGH6Vwlb9VRazUhPU github.com/fluxcd/pkg/git/gogit v0.8.1 h1:Q3EV2WBX6HiXSmsHyrwFzwl82gO4ZtFwb675iQPWwVc= github.com/fluxcd/pkg/git/gogit v0.8.1/go.mod h1:5M27gCl0gyo6l+ht9HwZSzimPY3LahKVIJ7/1vCCctg= github.com/fluxcd/pkg/gittestserver v0.8.1 h1:FMqnZBuS/11+9NhtLv9UAg+wm/v0Nf+hHeUOi2wJR3Q= -github.com/fluxcd/pkg/kustomize v1.1.0 h1:4qoTJBCtlg9RbO6YUwTNV3SttvXRIZxkjRRJE+jbsKk= -github.com/fluxcd/pkg/kustomize v1.1.0/go.mod h1:i+Z9iPAoSz28oH0FmDI73iqZ3oXZxQR2O3HfhdsWhfo= +github.com/fluxcd/pkg/kustomize v1.1.1 h1:hYFJGi+fiaecY4gXvx52fumlvDEq/1RdFbaev67oBhE= +github.com/fluxcd/pkg/kustomize v1.1.1/go.mod h1:i+Z9iPAoSz28oH0FmDI73iqZ3oXZxQR2O3HfhdsWhfo= github.com/fluxcd/pkg/oci v0.22.0 h1:6QRvCj1YXGEGXHyVkmKiBvYxsE0sEjUrpFknM513MbQ= github.com/fluxcd/pkg/oci v0.22.0/go.mod h1:y0jUgMqb6ionfX+8AjhnoG8D6hSSx4elhtrQ7Uo0WzI= github.com/fluxcd/pkg/runtime v0.35.0 h1:9PYLcul8qdfLYQArcYpHe/QuMqyhAGGFN9F7uY/QVX4= diff --git a/internal/build/build.go b/internal/build/build.go index dd09594a..5f6e21e5 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -25,6 +25,7 @@ import ( "fmt" "io" "os" + "strings" "sync" "time" @@ -71,6 +72,7 @@ type Builder struct { namespace string resourcesPath string kustomizationFile string + ignore []string // mu is used to synchronize access to the kustomization file mu sync.Mutex action kustomize.Action @@ -156,6 +158,14 @@ func WithDryRun(dryRun bool) BuilderOptionFunc { } } +// WithIgnore sets ignore field +func WithIgnore(ignore []string) BuilderOptionFunc { + return func(b *Builder) error { + b.ignore = ignore + return nil + } +} + // NewBuilder returns a new Builder // It takes a kustomization name and a path to the resources // It also takes a list of BuilderOptionFunc to configure the builder @@ -326,9 +336,13 @@ func (b *Builder) generate(kustomization kustomizev1.Kustomization, dirPath stri if err != nil { return "", err } - gen := kustomize.NewGeneratorWithIgnore("", "", unstructured.Unstructured{Object: data}) - // acuire the lock + // a scanner will be used down the line to parse the list + // so we have to make sure to unclude newlines + ignoreList := strings.Join(b.ignore, "\n") + gen := kustomize.NewGeneratorWithIgnore("", ignoreList, unstructured.Unstructured{Object: data}) + + // acquire the lock b.mu.Lock() defer b.mu.Unlock()