Add e2e tests for build/diff kustomization

Signed-off-by: Soule BA <soule@weave.works>

cmd/flux/build_kustomization.go

@@ -64,7 +64,7 @@ func buildKsCmdRun(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("invalid resource path %q", buildKsArgs.path)
 	}
 
-	builder, err := kustomization.NewBuilder(rootArgs.kubeconfig, rootArgs.kubecontext, rootArgs.namespace, name, buildKsArgs.path, kustomization.WithTimeout(rootArgs.timeout))
+	builder, err := kustomization.NewBuilder(kubeconfigArgs, name, buildKsArgs.path, kustomization.WithTimeout(rootArgs.timeout))
 	if err != nil {
 		return err
 	}

cmd/flux/build_kustomization_test.go

@@ -0,0 +1,83 @@
+//go:build unit
+// +build unit
+
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"testing"
+)
+
+func setup(t *testing.T, tmpl map[string]string) {
+	t.Helper()
+	testEnv.CreateObjectFile("./testdata/build-kustomization/podinfo-source.yaml", tmpl, t)
+	testEnv.CreateObjectFile("./testdata/build-kustomization/podinfo-kustomization.yaml", tmpl, t)
+}
+
+func TestBuildKustomization(t *testing.T) {
+	tests := []struct {
+		name       string
+		args       string
+		resultFile string
+		assertFunc string
+	}{
+		{
+			name:       "no args",
+			args:       "build kustomization podinfo",
+			resultFile: "invalid resource path \"\"",
+			assertFunc: "assertError",
+		},
+		{
+			name:       "build podinfo",
+			args:       "build kustomization podinfo --path ./testdata/build-kustomization/podinfo",
+			resultFile: "./testdata/build-kustomization/podinfo-result.yaml",
+			assertFunc: "assertGoldenTemplateFile",
+		},
+		{
+			name:       "build podinfo without service",
+			args:       "build kustomization podinfo --path ./testdata/build-kustomization/delete-service",
+			resultFile: "./testdata/build-kustomization/podinfo-without-service-result.yaml",
+			assertFunc: "assertGoldenTemplateFile",
+		},
+	}
+
+	tmpl := map[string]string{
+		"fluxns": allocateNamespace("flux-system"),
+	}
+	setup(t, tmpl)
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var assert assertFunc
+
+			switch tt.assertFunc {
+			case "assertGoldenTemplateFile":
+				assert = assertGoldenTemplateFile(tt.resultFile, tmpl)
+			case "assertError":
+				assert = assertError(tt.resultFile)
+			}
+
+			cmd := cmdTestCase{
+				args:   tt.args + " -n " + tmpl["fluxns"],
+				assert: assert,
+			}
+
+			cmd.runTestCmd(t)
+		})
+	}
+}

cmd/flux/diff_kustomization.go

@@ -62,16 +62,18 @@ func diffKsCmdRun(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("invalid resource path %q", diffKsArgs.path)
 	}
 
-	builder, err := kustomization.NewBuilder(rootArgs.kubeconfig, rootArgs.kubecontext, rootArgs.namespace, name, diffKsArgs.path, kustomization.WithTimeout(rootArgs.timeout))
+	builder, err := kustomization.NewBuilder(kubeconfigArgs, name, diffKsArgs.path, kustomization.WithTimeout(rootArgs.timeout))
 	if err != nil {
 		return err
 	}
 
-	err = builder.Diff()
+	output, err := builder.Diff()
 	if err != nil {
 		return err
 	}
 
+	cmd.Print(output)
+
 	return nil
 
 }

cmd/flux/diff_kustomization_test.go

@@ -0,0 +1,129 @@
+//go:build unit
+// +build unit
+
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/fluxcd/flux2/internal/kustomization"
+	"github.com/fluxcd/pkg/ssa"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+func TestDiffKustomization(t *testing.T) {
+	tests := []struct {
+		name       string
+		args       string
+		objectFile string
+		assert     assertFunc
+	}{
+		{
+			name:       "no args",
+			args:       "diff kustomization podinfo",
+			objectFile: "",
+			assert:     assertError("invalid resource path \"\""),
+		},
+		{
+			name:       "diff nothing deployed",
+			args:       "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo",
+			objectFile: "",
+			assert:     assertGoldenFile("./testdata/diff-kustomization/nothing-is-deployed.golden"),
+		},
+		{
+			name:       "diff with a deployment object",
+			args:       "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo",
+			objectFile: "./testdata/diff-kustomization/deployment.yaml",
+			assert:     assertGoldenFile("./testdata/diff-kustomization/diff-with-deployment.golden"),
+		},
+		{
+			name:       "diff with a drifted service object",
+			args:       "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo",
+			objectFile: "./testdata/diff-kustomization/service.yaml",
+			assert:     assertGoldenFile("./testdata/diff-kustomization/diff-with-drifted-service.golden"),
+		},
+		{
+			name:       "diff with a drifted secret object",
+			args:       "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo",
+			objectFile: "./testdata/diff-kustomization/secret.yaml",
+			assert:     assertGoldenFile("./testdata/diff-kustomization/diff-with-drifted-secret.golden"),
+		},
+		{
+			name:       "diff with a drifted key in sops secret object",
+			args:       "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo",
+			objectFile: "./testdata/diff-kustomization/key-sops-secret.yaml",
+			assert:     assertGoldenFile("./testdata/diff-kustomization/diff-with-drifted-key-sops-secret.golden"),
+		},
+		{
+			name:       "diff with a drifted value in sops secret object",
+			args:       "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo",
+			objectFile: "./testdata/diff-kustomization/value-sops-secret.yaml",
+			assert:     assertGoldenFile("./testdata/diff-kustomization/diff-with-drifted-value-sops-secret.golden"),
+		},
+	}
+
+	tmpl := map[string]string{
+		"fluxns": allocateNamespace("flux-system"),
+	}
+
+	b, _ := kustomization.NewBuilder(kubeconfigArgs, "podinfo", "")
+
+	resourceManager, err := b.Manager()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	setup(t, tmpl)
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.objectFile != "" {
+				resourceManager.ApplyAll(context.Background(), createObjectFromFile(tt.objectFile, tmpl, t), ssa.DefaultApplyOptions())
+			}
+			cmd := cmdTestCase{
+				args:   tt.args + " -n " + tmpl["fluxns"],
+				assert: tt.assert,
+			}
+			cmd.runTestCmd(t)
+			if tt.objectFile != "" {
+				testEnv.DeleteObjectFile(tt.objectFile, tmpl, t)
+			}
+		})
+	}
+}
+
+func createObjectFromFile(objectFile string, templateValues map[string]string, t *testing.T) []*unstructured.Unstructured {
+	buf, err := os.ReadFile(objectFile)
+	if err != nil {
+		t.Fatalf("Error reading file '%s': %v", objectFile, err)
+	}
+	content, err := executeTemplate(string(buf), templateValues)
+	if err != nil {
+		t.Fatalf("Error evaluating template file '%s': '%v'", objectFile, err)
+	}
+	clientObjects, err := readYamlObjects(strings.NewReader(content))
+	if err != nil {
+		t.Fatalf("Error decoding yaml file '%s': %v", objectFile, err)
+	}
+
+	return clientObjects
+}

cmd/flux/main_test.go

@@ -49,8 +49,8 @@ func allocateNamespace(prefix string) string {
 	return fmt.Sprintf("%s-%d", prefix, id)
 }
 
-func readYamlObjects(rdr io.Reader) ([]unstructured.Unstructured, error) {
-	objects := []unstructured.Unstructured{}
+func readYamlObjects(rdr io.Reader) ([]*unstructured.Unstructured, error) {
+	objects := []*unstructured.Unstructured{}
 	reader := k8syaml.NewYAMLReader(bufio.NewReader(rdr))
 	for {
 		doc, err := reader.Read()
@@ -65,7 +65,7 @@ func readYamlObjects(rdr io.Reader) ([]unstructured.Unstructured, error) {
 		if err != nil {
 			return nil, err
 		}
-		objects = append(objects, *unstructuredObj)
+		objects = append(objects, unstructuredObj)
 	}
 	return objects, nil
 }
@@ -96,7 +96,7 @@ func (m *testEnvKubeManager) CreateObjectFile(objectFile string, templateValues
 	}
 }
 
-func (m *testEnvKubeManager) CreateObjects(clientObjects []unstructured.Unstructured, t *testing.T) error {
+func (m *testEnvKubeManager) CreateObjects(clientObjects []*unstructured.Unstructured, t *testing.T) error {
 	for _, obj := range clientObjects {
 		// First create the object then set its status if present in the
 		// yaml file. Make a copy first since creating an object may overwrite
@@ -107,7 +107,7 @@ func (m *testEnvKubeManager) CreateObjects(clientObjects []unstructured.Unstruct
 			return err
 		}
 		obj.SetResourceVersion(createObj.GetResourceVersion())
-		err = m.client.Status().Update(context.Background(), &obj)
+		err = m.client.Status().Update(context.Background(), obj)
 		if err != nil {
 			return err
 		}
@@ -115,6 +115,36 @@ func (m *testEnvKubeManager) CreateObjects(clientObjects []unstructured.Unstruct
 	return nil
 }
 
+func (m *testEnvKubeManager) DeleteObjectFile(objectFile string, templateValues map[string]string, t *testing.T) {
+	buf, err := os.ReadFile(objectFile)
+	if err != nil {
+		t.Fatalf("Error reading file '%s': %v", objectFile, err)
+	}
+	content, err := executeTemplate(string(buf), templateValues)
+	if err != nil {
+		t.Fatalf("Error evaluating template file '%s': '%v'", objectFile, err)
+	}
+	clientObjects, err := readYamlObjects(strings.NewReader(content))
+	if err != nil {
+		t.Fatalf("Error decoding yaml file '%s': %v", objectFile, err)
+	}
+	err = m.DeleteObjects(clientObjects, t)
+	if err != nil {
+		t.Logf("Error deleting test objects: '%v'", err)
+	}
+}
+
+func (m *testEnvKubeManager) DeleteObjects(clientObjects []*unstructured.Unstructured, t *testing.T) error {
+	for _, obj := range clientObjects {
+		err := m.client.Delete(context.Background(), obj)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (m *testEnvKubeManager) Stop() error {
 	if m.testEnv == nil {
 		return fmt.Errorf("do nothing because testEnv is nil")

cmd/flux/testdata/build-kustomization/delete-service/deployment.yaml

@@ -0,0 +1,74 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: podinfo
+spec:
+  minReadySeconds: 3
+  revisionHistoryLimit: 5
+  progressDeadlineSeconds: 60
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 0
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: podinfo
+  template:
+    metadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9797"
+      labels:
+        app: podinfo
+    spec:
+      containers:
+      - name: podinfod
+        image: ghcr.io/stefanprodan/podinfo:6.0.3
+        imagePullPolicy: IfNotPresent
+        ports:
+        - name: http
+          containerPort: 9898
+          protocol: TCP
+        - name: http-metrics
+          containerPort: 9797
+          protocol: TCP
+        - name: grpc
+          containerPort: 9999
+          protocol: TCP
+        command:
+        - ./podinfo
+        - --port=9898
+        - --port-metrics=9797
+        - --grpc-port=9999
+        - --grpc-service-name=podinfo
+        - --level=info
+        - --random-delay=false
+        - --random-error=false
+        env:
+        - name: PODINFO_UI_COLOR
+          value: "#34577c"
+        livenessProbe:
+          exec:
+            command:
+            - podcli
+            - check
+            - http
+            - localhost:9898/healthz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        readinessProbe:
+          exec:
+            command:
+            - podcli
+            - check
+            - http
+            - localhost:9898/readyz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        resources:
+          limits:
+            cpu: 2000m
+            memory: 512Mi
+          requests:
+            cpu: 100m
+            memory: 64Mi

cmd/flux/testdata/build-kustomization/delete-service/hpa.yaml

@@ -0,0 +1,20 @@
+apiVersion: autoscaling/v2beta2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: podinfo
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  minReplicas: 2
+  maxReplicas: 4
+  metrics:
+  - type: Resource
+    resource:
+      name: cpu
+      target:
+        type: Utilization
+        # scale up if usage is above
+        # 99% of the requested CPU (100m)
+        averageUtilization: 99

cmd/flux/testdata/build-kustomization/delete-service/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ./deployment.yaml
+- ./hpa.yaml

cmd/flux/testdata/build-kustomization/podinfo-kustomization.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: podinfo
+  namespace: {{ .fluxns }}
+spec:
+  interval: 5m0s
+  path: ./kustomize
+  force: true
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: podinfo
+  targetNamespace: default

cmd/flux/testdata/build-kustomization/podinfo-result.yaml

@@ -0,0 +1,148 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    kustomize.toolkit.fluxcd.io/name: podinfo
+    kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }}
+  name: podinfo
+  namespace: default
+spec:
+  minReadySeconds: 3
+  progressDeadlineSeconds: 60
+  revisionHistoryLimit: 5
+  selector:
+    matchLabels:
+      app: podinfo
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 0
+    type: RollingUpdate
+  template:
+    metadata:
+      annotations:
+        prometheus.io/port: "9797"
+        prometheus.io/scrape: "true"
+      labels:
+        app: podinfo
+    spec:
+      containers:
+      - command:
+        - ./podinfo
+        - --port=9898
+        - --port-metrics=9797
+        - --grpc-port=9999
+        - --grpc-service-name=podinfo
+        - --level=info
+        - --random-delay=false
+        - --random-error=false
+        env:
+        - name: PODINFO_UI_COLOR
+          value: '#34577c'
+        image: ghcr.io/stefanprodan/podinfo:6.0.10
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          exec:
+            command:
+            - podcli
+            - check
+            - http
+            - localhost:9898/healthz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        name: podinfod
+        ports:
+        - containerPort: 9898
+          name: http
+          protocol: TCP
+        - containerPort: 9797
+          name: http-metrics
+          protocol: TCP
+        - containerPort: 9999
+          name: grpc
+          protocol: TCP
+        readinessProbe:
+          exec:
+            command:
+            - podcli
+            - check
+            - http
+            - localhost:9898/readyz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        resources:
+          limits:
+            cpu: 2000m
+            memory: 512Mi
+          requests:
+            cpu: 100m
+            memory: 64Mi
+---
+apiVersion: autoscaling/v2beta2
+kind: HorizontalPodAutoscaler
+metadata:
+  labels:
+    kustomize.toolkit.fluxcd.io/name: podinfo
+    kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }}
+  name: podinfo
+  namespace: default
+spec:
+  maxReplicas: 4
+  metrics:
+  - resource:
+      name: cpu
+      target:
+        averageUtilization: 99
+        type: Utilization
+    type: Resource
+  minReplicas: 2
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    kustomize.toolkit.fluxcd.io/name: podinfo
+    kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }}
+  name: podinfo
+  namespace: default
+spec:
+  ports:
+  - name: http
+    port: 9898
+    protocol: TCP
+    targetPort: http
+  - name: grpc
+    port: 9999
+    protocol: TCP
+    targetPort: grpc
+  selector:
+    app: podinfo
+  type: ClusterIP
+---
+apiVersion: v1
+data:
+  token: KipTT1BTKio=
+kind: Secret
+metadata:
+  labels:
+    kustomize.toolkit.fluxcd.io/name: podinfo
+    kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }}
+  name: podinfo-token-77t89m9b67
+  namespace: default
+type: Opaque
+---
+apiVersion: v1
+data:
+  password: MWYyZDFlMmU2N2Rm
+  username: YWRtaW4=
+kind: Secret
+metadata:
+  labels:
+    kustomize.toolkit.fluxcd.io/name: podinfo
+    kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }}
+  name: db-user-pass-bkbd782d2c
+  namespace: default
+type: Opaque

cmd/flux/testdata/build-kustomization/podinfo-source.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ .fluxns }}
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta1
+kind: GitRepository
+metadata:
+  name: podinfo
+  namespace: {{ .fluxns }}
+spec:
+  interval: 30s
+  ref:
+    branch: master
+  url: https://github.com/stefanprodan/podinfo

cmd/flux/testdata/build-kustomization/podinfo-without-service-result.yaml

@@ -0,0 +1,101 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    kustomize.toolkit.fluxcd.io/name: podinfo
+    kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }}
+  name: podinfo
+  namespace: default
+spec:
+  minReadySeconds: 3
+  progressDeadlineSeconds: 60
+  revisionHistoryLimit: 5
+  selector:
+    matchLabels:
+      app: podinfo
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 0
+    type: RollingUpdate
+  template:
+    metadata:
+      annotations:
+        prometheus.io/port: "9797"
+        prometheus.io/scrape: "true"
+      labels:
+        app: podinfo
+    spec:
+      containers:
+      - command:
+        - ./podinfo
+        - --port=9898
+        - --port-metrics=9797
+        - --grpc-port=9999
+        - --grpc-service-name=podinfo
+        - --level=info
+        - --random-delay=false
+        - --random-error=false
+        env:
+        - name: PODINFO_UI_COLOR
+          value: '#34577c'
+        image: ghcr.io/stefanprodan/podinfo:6.0.3
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          exec:
+            command:
+            - podcli
+            - check
+            - http
+            - localhost:9898/healthz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        name: podinfod
+        ports:
+        - containerPort: 9898
+          name: http
+          protocol: TCP
+        - containerPort: 9797
+          name: http-metrics
+          protocol: TCP
+        - containerPort: 9999
+          name: grpc
+          protocol: TCP
+        readinessProbe:
+          exec:
+            command:
+            - podcli
+            - check
+            - http
+            - localhost:9898/readyz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        resources:
+          limits:
+            cpu: 2000m
+            memory: 512Mi
+          requests:
+            cpu: 100m
+            memory: 64Mi
+---
+apiVersion: autoscaling/v2beta2
+kind: HorizontalPodAutoscaler
+metadata:
+  labels:
+    kustomize.toolkit.fluxcd.io/name: podinfo
+    kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }}
+  name: podinfo
+  namespace: default
+spec:
+  maxReplicas: 4
+  metrics:
+  - resource:
+      name: cpu
+      target:
+        averageUtilization: 99
+        type: Utilization
+    type: Resource
+  minReplicas: 2
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo

cmd/flux/testdata/build-kustomization/podinfo/deployment.yaml

@@ -0,0 +1,74 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: podinfo
+spec:
+  minReadySeconds: 3
+  revisionHistoryLimit: 5
+  progressDeadlineSeconds: 60
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 0
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: podinfo
+  template:
+    metadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9797"
+      labels:
+        app: podinfo
+    spec:
+      containers:
+      - name: podinfod
+        image: ghcr.io/stefanprodan/podinfo:6.0.10
+        imagePullPolicy: IfNotPresent
+        ports:
+        - name: http
+          containerPort: 9898
+          protocol: TCP
+        - name: http-metrics
+          containerPort: 9797
+          protocol: TCP
+        - name: grpc
+          containerPort: 9999
+          protocol: TCP
+        command:
+        - ./podinfo
+        - --port=9898
+        - --port-metrics=9797
+        - --grpc-port=9999
+        - --grpc-service-name=podinfo
+        - --level=info
+        - --random-delay=false
+        - --random-error=false
+        env:
+        - name: PODINFO_UI_COLOR
+          value: "#34577c"
+        livenessProbe:
+          exec:
+            command:
+            - podcli
+            - check
+            - http
+            - localhost:9898/healthz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        readinessProbe:
+          exec:
+            command:
+            - podcli
+            - check
+            - http
+            - localhost:9898/readyz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        resources:
+          limits:
+            cpu: 2000m
+            memory: 512Mi
+          requests:
+            cpu: 100m
+            memory: 64Mi

cmd/flux/testdata/build-kustomization/podinfo/hpa.yaml

@@ -0,0 +1,20 @@
+apiVersion: autoscaling/v2beta2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: podinfo
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  minReplicas: 2
+  maxReplicas: 4
+  metrics:
+  - type: Resource
+    resource:
+      name: cpu
+      target:
+        type: Utilization
+        # scale up if usage is above
+        # 99% of the requested CPU (100m)
+        averageUtilization: 99

cmd/flux/testdata/build-kustomization/podinfo/kustomization.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ./deployment.yaml
+- ./hpa.yaml
+- ./service.yaml
+secretGenerator:
+ - files:
+   - token=token.encrypted
+   name: podinfo-token
+ - literals:
+   - username=admin
+   - password=1f2d1e2e67df
+   name: db-user-pass

cmd/flux/testdata/build-kustomization/podinfo/service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: podinfo
+spec:
+  type: ClusterIP
+  selector:
+    app: podinfo
+  ports:
+    - name: http
+      port: 9898
+      protocol: TCP
+      targetPort: http
+    - port: 9999
+      targetPort: grpc
+      protocol: TCP
+      name: grpc

cmd/flux/testdata/build-kustomization/podinfo/token.encrypted

@@ -0,0 +1,20 @@
+ {
+     "data": "ENC[AES256_GCM,data:oBe5PlPmfQCUUc4sqKImjw==,iv:MLLEW15QC9kRdVVagJnzLCSk0xZGWIpAeTfHzyxT10g=,tag:K3GkBCGS+ut4Tpk6ndb0CA==,type:str]",
+     "sops": {
+         "kms": null,
+         "gcp_kms": null,
+         "azure_kv": null,
+         "hc_vault": null,
+         "age": [
+             {
+                 "recipient": "age10la2ge0wtvx3qr7datqf7rs4yngxszdal927fs9rukamr8u2pshsvtz7ce",
+                 "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+                                                                                                                IFgyNTUxOSA1L2RpZHRrK1FSVmYrd1Va\nY0hxWFQzSDBsT1k3WjNtYmU1QmliaDJycXlNCnF1YjdNOThVbVNvMG9rNS9ZUXZw\nMnV0bnRUMGNtejFPbzM4U2UzWkszeVkKLS0tIGJ6UGhxMUV3YmVJTHlJSUJpRVRZ\nVjd0RVRadU8wekxXTHIrYUplYkN2aEEK0I/   MCEtXRk+b/N2G1JF3vHQT24dShWYD\nw+JIUSA3aLf2sv0zr2MdUEdVWBJoM8nT4D4xVbBORD+669W+9nDeSw==\n-----END AGE ENCRYPTED FILE-----\n"
+             }
+         ],
+         "lastmodified": "2021-11-26T16:34:51Z",
+         "mac": "ENC[AES256_GCM,data:COGzf5YCHNNP6z4JaEKrjN3M8f5+Q1uKUKTMHwj388/ICmLyi2sSrTmj7PP+X7M9jTVwa8wVgYTpNLiVJx+LcxqvIXM0Tyo+/Cu1zrfao98aiACP8+TSEDiFQNtEus23H+d/X1hqMwRHDI3kQ+                      6scgEGnqY57r3RDSA3E8EhHr4=,iv:LxitVIYm8srZVqFueJh9loClA44Y2Z3XAVYmxesMmOg=,tag:Y8qFD8UGlDfwNSv7xlcn6A==,type:str]",
+         "pgp": null,
+         "unencrypted_suffix": "_unencrypted",
+         "version": "3.7.1"
+     }
+ }

cmd/flux/testdata/diff-kustomization/deployment.yaml

@@ -0,0 +1,78 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+      kustomize.toolkit.fluxcd.io/name: podinfo
+      kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }}
+  name: podinfo
+  namespace: default
+spec:
+  minReadySeconds: 3
+  revisionHistoryLimit: 5
+  progressDeadlineSeconds: 60
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 0
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: podinfo
+  template:
+    metadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9797"
+      labels:
+        app: podinfo
+    spec:
+      containers:
+      - name: podinfod
+        image: ghcr.io/stefanprodan/podinfo:6.0.10
+        imagePullPolicy: IfNotPresent
+        ports:
+        - name: http
+          containerPort: 9898
+          protocol: TCP
+        - name: http-metrics
+          containerPort: 9797
+          protocol: TCP
+        - name: grpc
+          containerPort: 9999
+          protocol: TCP
+        command:
+        - ./podinfo
+        - --port=9898
+        - --port-metrics=9797
+        - --grpc-port=9999
+        - --grpc-service-name=podinfo
+        - --level=info
+        - --random-delay=false
+        - --random-error=false
+        env:
+        - name: PODINFO_UI_COLOR
+          value: "#34577c"
+        livenessProbe:
+          exec:
+            command:
+            - podcli
+            - check
+            - http
+            - localhost:9898/healthz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        readinessProbe:
+          exec:
+            command:
+            - podcli
+            - check
+            - http
+            - localhost:9898/readyz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        resources:
+          limits:
+            cpu: 2000m
+            memory: 512Mi
+          requests:
+            cpu: 100m
+            memory: 64Mi

cmd/flux/testdata/diff-kustomization/diff-with-deployment.golden

@@ -0,0 +1,4 @@
+► HorizontalPodAutoscaler/default/podinfo created
+► Service/default/podinfo created
+► Secret/default/podinfo-token-77t89m9b67 created
+► Secret/default/db-user-pass-bkbd782d2c created

cmd/flux/testdata/diff-kustomization/diff-with-drifted-key-sops-secret.golden

@@ -0,0 +1,10 @@
+► Deployment/default/podinfo created
+► HorizontalPodAutoscaler/default/podinfo created
+► Service/default/podinfo created
+► Secret/default/podinfo-token-77t89m9b67 drifted
+
+data
+  - one map entry removed:     + one map entry added:
+    drift-key: "*****"           token: "*****"
+
+► Secret/default/db-user-pass-bkbd782d2c created

cmd/flux/testdata/diff-kustomization/diff-with-drifted-secret.golden

@@ -0,0 +1,11 @@
+► Deployment/default/podinfo created
+► HorizontalPodAutoscaler/default/podinfo created
+► Service/default/podinfo created
+► Secret/default/podinfo-token-77t89m9b67 created
+► Secret/default/db-user-pass-bkbd782d2c drifted
+
+data.password
+  ± value change
+    - ******
+    + *****
+

cmd/flux/testdata/diff-kustomization/diff-with-drifted-service.golden

@@ -0,0 +1,16 @@
+► Deployment/default/podinfo created
+► HorizontalPodAutoscaler/default/podinfo created
+► Service/default/podinfo drifted
+
+spec.ports
+  ⇆ order changed
+    - http, grpc
+    + grpc, http
+
+spec.ports.http.port
+  ± value change
+    - 9899
+    + 9898
+
+► Secret/default/podinfo-token-77t89m9b67 created
+► Secret/default/db-user-pass-bkbd782d2c created

cmd/flux/testdata/diff-kustomization/diff-with-drifted-value-sops-secret.golden

@@ -0,0 +1,4 @@
+► Deployment/default/podinfo created
+► HorizontalPodAutoscaler/default/podinfo created
+► Service/default/podinfo created
+► Secret/default/db-user-pass-bkbd782d2c created

cmd/flux/testdata/diff-kustomization/key-sops-secret.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+data:
+  drift-key: bXktc2VjcmV0LXRva2VuCg==
+kind: Secret
+metadata:
+  labels:
+      kustomize.toolkit.fluxcd.io/name: podinfo
+      kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }}
+  name: podinfo-token-77t89m9b67
+  namespace: default
+type: Opaque

cmd/flux/testdata/diff-kustomization/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ./deployment.yaml
+- ./hpa.yaml
+- ./service.yaml
+secretGenerator:
+ - literals:
+   - username=admin
+   - password=1f2d1e2e67df
+   name: secret-basic-auth

cmd/flux/testdata/diff-kustomization/nothing-is-deployed.golden

@@ -0,0 +1,5 @@
+► Deployment/default/podinfo created
+► HorizontalPodAutoscaler/default/podinfo created
+► Service/default/podinfo created
+► Secret/default/podinfo-token-77t89m9b67 created
+► Secret/default/db-user-pass-bkbd782d2c created

cmd/flux/testdata/diff-kustomization/secret.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+data:
+  password: cGFzc3dvcmQK
+  username: YWRtaW4=
+kind: Secret
+metadata:
+  labels:
+      kustomize.toolkit.fluxcd.io/name: podinfo
+      kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }}
+  name: db-user-pass-bkbd782d2c
+  namespace: default
+type: Opaque

cmd/flux/testdata/diff-kustomization/service.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+      kustomize.toolkit.fluxcd.io/name: podinfo
+      kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }}
+  name: podinfo
+  namespace: default
+spec:
+  type: ClusterIP
+  selector:
+    app: podinfo
+  ports:
+    - name: http
+      port: 9899
+      protocol: TCP
+      targetPort: http
+    - port: 9999
+      targetPort: grpc
+      protocol: TCP
+      name: grpc

cmd/flux/testdata/diff-kustomization/value-sops-secret.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+data:
+  token: ZHJpZnQtdmFsdWUK
+kind: Secret
+metadata:
+  labels:
+      kustomize.toolkit.fluxcd.io/name: podinfo
+      kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }}
+  name: podinfo-token-77t89m9b67
+  namespace: default
+type: Opaque

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/gonvenience/ytbx v1.4.2
 	github.com/google/go-cmp v0.5.6
 	github.com/google/go-containerregistry v0.2.0
+	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-retryablehttp v0.7.0 // indirect
 	github.com/homeport/dyff v1.4.6
 	github.com/lucasb-eyer/go-colorful v1.2.0
@@ -100,7 +101,6 @@ require (
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect

internal/kustomization/build.go

@@ -21,23 +21,26 @@ import (
 	"context"
 	"encoding/base64"
 	"fmt"
-	"os"
-	"path/filepath"
 	"time"
 
 	"github.com/fluxcd/flux2/internal/utils"
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2"
 	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
-	"sigs.k8s.io/kustomize/api/konfig"
 	"sigs.k8s.io/kustomize/api/resmap"
 	"sigs.k8s.io/kustomize/api/resource"
 	"sigs.k8s.io/kustomize/kyaml/filesys"
 )
 
-const mask string = "**SOPS**"
+const (
+	controllerName         = "kustomize-controller"
+	controllerGroup        = "kustomize.toolkit.fluxcd.io"
+	mask            string = "**SOPS**"
+)
 
 var defaultTimeout = 80 * time.Second
 
@@ -65,17 +68,13 @@ func WithTimeout(timeout time.Duration) BuilderOptionFunc {
 
 // NewBuilder returns a new Builder
 // to dp : create functional options
-func NewBuilder(kubeconfig string, kubecontext string, namespace, name, resources string, opts ...BuilderOptionFunc) (*Builder, error) {
-	kubeClient, err := utils.KubeClient(kubeconfig, kubecontext)
+func NewBuilder(rcg *genericclioptions.ConfigFlags, name, resources string, opts ...BuilderOptionFunc) (*Builder, error) {
+	kubeClient, err := utils.KubeClient(rcg)
 	if err != nil {
 		return nil, err
 	}
 
-	cfg, err := utils.KubeConfig(kubeconfig, kubecontext)
-	if err != nil {
-		return nil, err
-	}
-	restMapper, err := apiutil.NewDynamicRESTMapper(cfg)
+	restMapper, err := rcg.ToRESTMapper()
 	if err != nil {
 		return nil, err
 	}
@@ -84,7 +83,7 @@ func NewBuilder(kubeconfig string, kubecontext string, namespace, name, resource
 		client:        kubeClient,
 		restMapper:    restMapper,
 		name:          name,
-		namespace:     namespace,
+		namespace:     *rcg.Namespace,
 		resourcesPath: resources,
 	}
 
@@ -134,57 +133,70 @@ func (b *Builder) Build() ([]byte, error) {
 	return resources, nil
 }
 
-func (b *Builder) build() (resmap.ResMap, error) {
+func (b *Builder) build() (m resmap.ResMap, err error) {
 	ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
 	defer cancel()
 
 	// Get the kustomization object
 	k, err := b.getKustomization(ctx)
 	if err != nil {
-		return nil, err
+		return
 	}
 
+	// store the kustomization object
+	b.kustomization = k
+
 	// generate kustomization.yaml if needed
-	saved, err := b.generate(*k, b.resourcesPath)
-	if err != nil {
-		return nil, fmt.Errorf("failed to generate kustomization.yaml: %w", err)
+	action, er := b.generate(*k, b.resourcesPath)
+	if er != nil {
+		errf := CleanDirectory(b.resourcesPath, action)
+		err = fmt.Errorf("failed to generate kustomization.yaml: %w", fmt.Errorf("%v %v", er, errf))
+		return
 	}
 
+	defer func() {
+		errf := CleanDirectory(b.resourcesPath, action)
+		if err == nil {
+			err = errf
+		}
+	}()
+
 	// build the kustomization
-	m, err := b.do(ctx, *k, b.resourcesPath)
+	m, err = b.do(ctx, *k, b.resourcesPath)
 	if err != nil {
-		return nil, err
+		return
 	}
 
-	// make sure secrets are masked
 	for _, res := range m.Resources() {
-		err := trimSopsData(res)
+		// set owner labels
+		err = b.setOwnerLabels(res)
 		if err != nil {
-			return nil, err
+			return
 		}
-	}
 
-	// store the kustomization object
-	b.kustomization = k
-
-	// overwrite the kustomization.yaml to make sure it's clean
-	err = overwrite(saved, b.resourcesPath)
-	if err != nil {
-		return nil, fmt.Errorf("failed to restore kustomization.yaml: %w", err)
+		// make sure secrets are masked
+		err = trimSopsData(res)
+		if err != nil {
+			return
+		}
 	}
 
-	return m, nil
+	return
 
 }
 
-func (b *Builder) generate(kustomization kustomizev1.Kustomization, dirPath string) ([]byte, error) {
-	gen := NewGenerator(&kustomizeImpl{kustomization})
-	return gen.WriteFile(dirPath)
+func (b *Builder) generate(kustomization kustomizev1.Kustomization, dirPath string) (action, error) {
+	data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&kustomization)
+	if err != nil {
+		return "", err
+	}
+	gen := NewGenerator(unstructured.Unstructured{Object: data})
+	return gen.WriteFile(dirPath, WithSaveOriginalKustomization())
 }
 
 func (b *Builder) do(ctx context.Context, kustomization kustomizev1.Kustomization, dirPath string) (resmap.ResMap, error) {
 	fs := filesys.MakeFsOnDisk()
-	m, err := buildKustomization(fs, dirPath)
+	m, err := BuildKustomization(fs, dirPath)
 	if err != nil {
 		return nil, fmt.Errorf("kustomize build failed: %w", err)
 	}
@@ -192,7 +204,11 @@ func (b *Builder) do(ctx context.Context, kustomization kustomizev1.Kustomizatio
 	for _, res := range m.Resources() {
 		// run variable substitutions
 		if kustomization.Spec.PostBuild != nil {
-			outRes, err := substituteVariables(ctx, b.client, &kustomizeImpl{kustomization}, res)
+			data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&kustomization)
+			if err != nil {
+				return nil, err
+			}
+			outRes, err := SubstituteVariables(ctx, b.client, unstructured.Unstructured{Object: data}, res)
 			if err != nil {
 				return nil, fmt.Errorf("var substitution failed for '%s': %w", res.GetName(), err)
 			}
@@ -209,6 +225,20 @@ func (b *Builder) do(ctx context.Context, kustomization kustomizev1.Kustomizatio
 	return m, nil
 }
 
+func (b *Builder) setOwnerLabels(res *resource.Resource) error {
+	labels := res.GetLabels()
+
+	labels[controllerGroup+"/name"] = b.kustomization.GetName()
+	labels[controllerGroup+"/namespace"] = b.kustomization.GetNamespace()
+
+	err := res.SetLabels(labels)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func trimSopsData(res *resource.Resource) error {
 	// sopsMess is the base64 encoded mask
 	sopsMess := base64.StdEncoding.EncodeToString([]byte(mask))
@@ -233,12 +263,3 @@ func trimSopsData(res *resource.Resource) error {
 
 	return nil
 }
-
-func overwrite(saved []byte, dirPath string) error {
-	kfile := filepath.Join(dirPath, konfig.DefaultKustomizationFileName())
-	err := os.WriteFile(kfile, saved, 0644)
-	if err != nil {
-		return fmt.Errorf("failed to overwrite kustomization.yaml: %w", err)
-	}
-	return nil
-}

internal/kustomization/diff.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/base64"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"sort"
@@ -17,6 +18,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/homeport/dyff/pkg/dyff"
 	"github.com/lucasb-eyer/go-colorful"
+	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"sigs.k8s.io/cli-utils/pkg/kstatus/polling"
@@ -24,12 +26,7 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
-const (
-	controllerName  = "kustomize-controller"
-	controllerGroup = "kustomize.toolkit.fluxcd.io"
-)
-
-func (b *Builder) manager() (*ssa.ResourceManager, error) {
+func (b *Builder) Manager() (*ssa.ResourceManager, error) {
 	statusPoller := polling.NewStatusPoller(b.client, b.restMapper)
 	owner := ssa.Owner{
 		Field: controllerName,
@@ -39,20 +36,21 @@ func (b *Builder) manager() (*ssa.ResourceManager, error) {
 	return ssa.NewResourceManager(b.client, statusPoller, owner), nil
 }
 
-func (b *Builder) Diff() error {
+func (b *Builder) Diff() (string, error) {
+	output := strings.Builder{}
 	res, err := b.Build()
 	if err != nil {
-		return err
+		return "", err
 	}
 	// convert the build result into Kubernetes unstructured objects
 	objects, err := ssa.ReadObjects(bytes.NewReader(res))
 	if err != nil {
-		return err
+		return "", err
 	}
 
-	resourceManager, err := b.manager()
+	resourceManager, err := b.Manager()
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	resourceManager.SetOwnerLabels(objects, b.kustomization.GetName(), b.kustomization.GetNamespace())
@@ -61,7 +59,7 @@ func (b *Builder) Diff() error {
 	defer cancel()
 
 	if err := ssa.SetNativeKindsDefaults(objects); err != nil {
-		return err
+		return "", err
 	}
 
 	// create an inventory of objects to be reconciled
@@ -69,10 +67,10 @@ func (b *Builder) Diff() error {
 	for _, obj := range objects {
 		change, liveObject, mergedObject, err := resourceManager.Diff(ctx, obj)
 		if err != nil {
-			if b.kustomization.Spec.Force && strings.Contains(err.Error(), "immutable") {
-				writeString(fmt.Sprintf("► %s created", obj.GetName()), bunt.Green)
+			if b.kustomization.Spec.Force && isImmutableError(err) {
+				output.WriteString(writeString(fmt.Sprintf("► %s created\n", obj.GetName()), bunt.Green))
 			} else {
-				writeString(fmt.Sprint(`✗`, err), bunt.Red)
+				output.WriteString(writeString(fmt.Sprint(`✗`, err), bunt.Red))
 			}
 			continue
 		}
@@ -84,20 +82,20 @@ func (b *Builder) Diff() error {
 		}
 
 		if change.Action == string(ssa.CreatedAction) {
-			writeString(fmt.Sprintf("► %s created", change.Subject), bunt.Green)
+			output.WriteString(writeString(fmt.Sprintf("► %s created\n", change.Subject), bunt.Green))
 		}
 
 		if change.Action == string(ssa.ConfiguredAction) {
-			writeString(fmt.Sprintf("► %s drifted", change.Subject), bunt.WhiteSmoke)
+			output.WriteString(writeString(fmt.Sprintf("► %s drifted\n", change.Subject), bunt.WhiteSmoke))
 			liveFile, mergedFile, tmpDir, err := writeYamls(liveObject, mergedObject)
 			if err != nil {
-				return err
+				return "", err
 			}
 			defer cleanupDir(tmpDir)
 
-			err = diff(liveFile, mergedFile)
+			err = diff(liveFile, mergedFile, &output)
 			if err != nil {
-				return err
+				return "", err
 			}
 		}
 
@@ -109,15 +107,15 @@ func (b *Builder) Diff() error {
 		if oldStatus.Inventory != nil {
 			diffObjects, err := diffInventory(oldStatus.Inventory, newInventory)
 			if err != nil {
-				return err
+				return "", err
 			}
 			for _, object := range diffObjects {
-				writeString(fmt.Sprintf("► %s deleted", ssa.FmtUnstructured(object)), bunt.OrangeRed)
+				output.WriteString(writeString(fmt.Sprintf("► %s deleted\n", ssa.FmtUnstructured(object)), bunt.OrangeRed))
 			}
 		}
 	}
 
-	return nil
+	return output.String(), nil
 }
 
 func writeYamls(liveObject, mergedObject *unstructured.Unstructured) (string, string, string, error) {
@@ -141,19 +139,19 @@ func writeYamls(liveObject, mergedObject *unstructured.Unstructured) (string, st
 	return liveFile, mergedFile, tmpDir, nil
 }
 
-func writeString(t string, color colorful.Color) {
-	fmt.Println(bunt.Style(
+func writeString(t string, color colorful.Color) string {
+	return bunt.Style(
 		t,
 		bunt.EachLine(),
 		bunt.Foreground(color),
-	))
+	)
 }
 
 func cleanupDir(dir string) error {
 	return os.RemoveAll(dir)
 }
 
-func diff(liveFile, mergedFile string) error {
+func diff(liveFile, mergedFile string, output io.Writer) error {
 	from, to, err := ytbx.LoadFiles(liveFile, mergedFile)
 	if err != nil {
 		return fmt.Errorf("failed to load input files: %w", err)
@@ -172,7 +170,7 @@ func diff(liveFile, mergedFile string) error {
 		OmitHeader: true,
 	}
 
-	if err := reportWriter.WriteReport(os.Stdout); err != nil {
+	if err := reportWriter.WriteReport(output); err != nil {
 		return fmt.Errorf("failed to print report: %w", err)
 	}
 
@@ -285,3 +283,12 @@ func addObjectsToInventory(inv *kustomizev1.ResourceInventory, entry *ssa.Change
 
 	return nil
 }
+
+func isImmutableError(err error) bool {
+	// Detect immutability like kubectl does
+	// https://github.com/kubernetes/kubectl/blob/8165f83007/pkg/cmd/apply/patcher.go#L201
+	if errors.IsConflict(err) || errors.IsInvalid(err) {
+		return true
+	}
+	return false
+}

internal/kustomization/kustomization.go

@@ -16,69 +16,9 @@ limitations under the License.
 
 package kustomization
 
-import (
-	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2"
-	"github.com/fluxcd/pkg/apis/kustomize"
-	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-)
-
-// Kustomize defines the methods to retrieve the kustomization information
-// TO DO @souleb: move this to fluxcd/pkg along with generator and varsub
-type Kustomize interface {
-	client.Object
-	GetTargetNamespace() string
-	GetPatches() []kustomize.Patch
-	GetPatchesStrategicMerge() []apiextensionsv1.JSON
-	GetPatchesJSON6902() []kustomize.JSON6902Patch
-	GetImages() []kustomize.Image
-	GetSubstituteFrom() []SubstituteReference
-	GetSubstitute() map[string]string
-}
-
 // SubstituteReference contains a reference to a resource containing
 // the variables name and value.
 type SubstituteReference struct {
 	Kind string `json:"kind"`
 	Name string `json:"name"`
 }
-
-// TO DO @souleb: this is a temporary hack to get the kustomize object
-// from the kustomize controller.
-// At some point we should remove this and have the kustomize controller implement
-// the Kustomize interface.
-type kustomizeImpl struct {
-	kustomizev1.Kustomization
-}
-
-func (k *kustomizeImpl) GetTargetNamespace() string {
-	return k.Spec.TargetNamespace
-}
-
-func (k *kustomizeImpl) GetPatches() []kustomize.Patch {
-	return k.Spec.Patches
-}
-
-func (k *kustomizeImpl) GetPatchesStrategicMerge() []apiextensionsv1.JSON {
-	return k.Spec.PatchesStrategicMerge
-}
-
-func (k *kustomizeImpl) GetPatchesJSON6902() []kustomize.JSON6902Patch {
-	return k.Spec.PatchesJSON6902
-}
-
-func (k *kustomizeImpl) GetImages() []kustomize.Image {
-	return k.Spec.Images
-}
-
-func (k *kustomizeImpl) GetSubstituteFrom() []SubstituteReference {
-	refs := make([]SubstituteReference, 0, len(k.Spec.PostBuild.SubstituteFrom))
-	for _, s := range k.Spec.PostBuild.SubstituteFrom {
-		refs = append(refs, SubstituteReference(s))
-	}
-	return refs
-}
-
-func (k *kustomizeImpl) GetSubstitute() map[string]string {
-	return k.Spec.PostBuild.Substitute
-}

internal/kustomization/kustomization_generator.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2021 The Flux authors
+Copyright 2022 The Flux authors
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,11 +19,15 @@ package kustomization
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
 	"sync"
 
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	"sigs.k8s.io/kustomize/api/konfig"
 	"sigs.k8s.io/kustomize/api/krusty"
 	"sigs.k8s.io/kustomize/api/provider"
@@ -33,32 +37,74 @@ import (
 	"sigs.k8s.io/yaml"
 
 	"github.com/fluxcd/pkg/apis/kustomize"
+	"github.com/hashicorp/go-multierror"
+)
+
+const (
+	specField                 = "spec"
+	targetNSField             = "targetNamespace"
+	patchesField              = "patches"
+	patchesSMField            = "patchesStrategicMerge"
+	patchesJson6902Field      = "patchesJson6902"
+	imagesField               = "images"
+	originalKustomizationFile = "kustomization.yaml.original"
+)
+
+type action string
+
+const (
+	createdAction   action = "created"
+	unchangedAction action = "unchanged"
 )
 
 type KustomizeGenerator struct {
-	kustomization Kustomize
+	kustomization unstructured.Unstructured
 }
 
-func NewGenerator(kustomization Kustomize) *KustomizeGenerator {
+type SavingOptions func(dirPath, file string, action action) error
+
+func NewGenerator(kustomization unstructured.Unstructured) *KustomizeGenerator {
 	return &KustomizeGenerator{
 		kustomization: kustomization,
 	}
 }
 
+func WithSaveOriginalKustomization() SavingOptions {
+	return func(dirPath, kfile string, action action) error {
+		// copy the original kustomization.yaml to the directory if we did not create it
+		if action != createdAction {
+			if err := copyFile(kfile, filepath.Join(dirPath, originalKustomizationFile)); err != nil {
+				errf := CleanDirectory(dirPath, action)
+				return fmt.Errorf("%v %v", err, errf)
+			}
+		}
+		return nil
+	}
+}
+
 // WriteFile generates a kustomization.yaml in the given directory if it does not exist.
 // It apply the flux kustomize resources to the kustomization.yaml and then write the
 // updated kustomization.yaml to the directory.
-// It returns the original kustomization.yaml.
-func (kg *KustomizeGenerator) WriteFile(dirPath string) ([]byte, error) {
-	if err := kg.generateKustomization(dirPath); err != nil {
-		return nil, err
+// It returns an action that indicates if the kustomization.yaml was created or not.
+// It is the caller responsability to clean up the directory by use the provided function CleanDirectory.
+// example:
+// err := CleanDirectory(dirPath, action)
+// if err != nil {
+// 	log.Fatal(err)
+// }
+func (kg *KustomizeGenerator) WriteFile(dirPath string, opts ...SavingOptions) (action, error) {
+	action, err := kg.generateKustomization(dirPath)
+	if err != nil {
+		errf := CleanDirectory(dirPath, action)
+		return action, fmt.Errorf("%v %v", err, errf)
 	}
 
 	kfile := filepath.Join(dirPath, konfig.DefaultKustomizationFileName())
 
 	data, err := os.ReadFile(kfile)
 	if err != nil {
-		return nil, err
+		errf := CleanDirectory(dirPath, action)
+		return action, fmt.Errorf("%w %s", err, errf)
 	}
 
 	kus := kustypes.Kustomization{
@@ -69,36 +115,67 @@ func (kg *KustomizeGenerator) WriteFile(dirPath string) ([]byte, error) {
 	}
 
 	if err := yaml.Unmarshal(data, &kus); err != nil {
-		return nil, err
+		errf := CleanDirectory(dirPath, action)
+		return action, fmt.Errorf("%v %v", err, errf)
 	}
 
-	if kg.kustomization.GetTargetNamespace() != "" {
-		kus.Namespace = kg.kustomization.GetTargetNamespace()
+	tg, ok, err := kg.getNestedString(specField, targetNSField)
+	if err != nil {
+		errf := CleanDirectory(dirPath, action)
+		return action, fmt.Errorf("%v %v", err, errf)
+	}
+	if ok {
+		kus.Namespace = tg
+	}
+
+	patches, err := kg.getPatches()
+	if err != nil {
+		errf := CleanDirectory(dirPath, action)
+		return action, fmt.Errorf("unable to get patches: %w", fmt.Errorf("%v %v", err, errf))
 	}
 
-	for _, m := range kg.kustomization.GetPatches() {
+	for _, p := range patches {
 		kus.Patches = append(kus.Patches, kustypes.Patch{
-			Patch:  m.Patch,
-			Target: adaptSelector(&m.Target),
+			Patch:  p.Patch,
+			Target: adaptSelector(&p.Target),
 		})
 	}
 
-	for _, m := range kg.kustomization.GetPatchesStrategicMerge() {
-		kus.PatchesStrategicMerge = append(kus.PatchesStrategicMerge, kustypes.PatchStrategicMerge(m.Raw))
+	patchesSM, err := kg.getPatchesStrategicMerge()
+	if err != nil {
+		errf := CleanDirectory(dirPath, action)
+		return action, fmt.Errorf("unable to get patchesStrategicMerge: %w", fmt.Errorf("%v %v", err, errf))
+	}
+
+	for _, p := range patchesSM {
+		kus.PatchesStrategicMerge = append(kus.PatchesStrategicMerge, kustypes.PatchStrategicMerge(p.Raw))
 	}
 
-	for _, m := range kg.kustomization.GetPatchesJSON6902() {
-		patch, err := json.Marshal(m.Patch)
+	patchesJSON, err := kg.getPatchesJson6902()
+	if err != nil {
+		errf := CleanDirectory(dirPath, action)
+		return action, fmt.Errorf("unable to get patchesJson6902: %w", fmt.Errorf("%v %v", err, errf))
+	}
+
+	for _, p := range patchesJSON {
+		patch, err := json.Marshal(p.Patch)
 		if err != nil {
-			return nil, err
+			errf := CleanDirectory(dirPath, action)
+			return action, fmt.Errorf("%v %v", err, errf)
 		}
 		kus.PatchesJson6902 = append(kus.PatchesJson6902, kustypes.Patch{
 			Patch:  string(patch),
-			Target: adaptSelector(&m.Target),
+			Target: adaptSelector(&p.Target),
 		})
 	}
 
-	for _, image := range kg.kustomization.GetImages() {
+	images, err := kg.getImages()
+	if err != nil {
+		errf := CleanDirectory(dirPath, action)
+		return action, fmt.Errorf("unable to get images: %w", fmt.Errorf("%v %v", err, errf))
+	}
+
+	for _, image := range images {
 		newImage := kustypes.Image{
 			Name:    image.Name,
 			NewName: image.NewName,
@@ -112,13 +189,141 @@ func (kg *KustomizeGenerator) WriteFile(dirPath string) ([]byte, error) {
 	}
 
 	manifest, err := yaml.Marshal(kus)
+	if err != nil {
+		errf := CleanDirectory(dirPath, action)
+		return action, fmt.Errorf("%v %v", err, errf)
+	}
+
+	// copy the original kustomization.yaml to the directory if we did not create it
+	for _, opt := range opts {
+		if err := opt(dirPath, kfile, action); err != nil {
+			return action, fmt.Errorf("failed to save original kustomization.yaml: %w", err)
+		}
+	}
+
+	err = os.WriteFile(kfile, manifest, os.ModePerm)
+	if err != nil {
+		errf := CleanDirectory(dirPath, action)
+		return action, fmt.Errorf("%v %v", err, errf)
+	}
+
+	return action, nil
+}
+
+func (kg *KustomizeGenerator) getPatches() ([]kustomize.Patch, error) {
+	patches, ok, err := kg.getNestedSlice(specField, patchesField)
+	if err != nil {
+		return nil, err
+	}
+
+	var resultErr error
+	if ok {
+		res := make([]kustomize.Patch, 0, len(patches))
+		for k, p := range patches {
+			patch, ok := p.(map[string]interface{})
+			if !ok {
+				err := fmt.Errorf("unable to convert patch %d to map[string]interface{}", k)
+				resultErr = multierror.Append(resultErr, err)
+			}
+			var kpatch kustomize.Patch
+			err = runtime.DefaultUnstructuredConverter.FromUnstructured(patch, &kpatch)
+			if err != nil {
+				resultErr = multierror.Append(resultErr, err)
+			}
+			res = append(res, kpatch)
+		}
+		return res, resultErr
+	}
+
+	return nil, resultErr
+
+}
+
+func (kg *KustomizeGenerator) getPatchesStrategicMerge() ([]apiextensionsv1.JSON, error) {
+	patches, ok, err := kg.getNestedSlice(specField, patchesSMField)
+	if err != nil {
+		return nil, err
+	}
+
+	var resultErr error
+	if ok {
+		res := make([]apiextensionsv1.JSON, 0, len(patches))
+		for k, p := range patches {
+			patch, ok := p.(map[string]interface{})
+			if !ok {
+				err := fmt.Errorf("unable to convert patch %d to map[string]interface{}", k)
+				resultErr = multierror.Append(resultErr, err)
+			}
+			var kpatch apiextensionsv1.JSON
+			err = runtime.DefaultUnstructuredConverter.FromUnstructured(patch, &kpatch)
+			if err != nil {
+				resultErr = multierror.Append(resultErr, err)
+			}
+			res = append(res, kpatch)
+		}
+		return res, resultErr
+	}
+
+	return nil, resultErr
+
+}
+
+func (kg *KustomizeGenerator) getPatchesJson6902() ([]kustomize.JSON6902Patch, error) {
+	patches, ok, err := kg.getNestedSlice(specField, patchesJson6902Field)
+	if err != nil {
+		return nil, err
+	}
+
+	var resultErr error
+	if ok {
+		res := make([]kustomize.JSON6902Patch, 0, len(patches))
+		for k, p := range patches {
+			patch, ok := p.(map[string]interface{})
+			if !ok {
+				err := fmt.Errorf("unable to convert patch %d to map[string]interface{}", k)
+				resultErr = multierror.Append(resultErr, err)
+			}
+			var kpatch kustomize.JSON6902Patch
+			err = runtime.DefaultUnstructuredConverter.FromUnstructured(patch, &kpatch)
+			if err != nil {
+				resultErr = multierror.Append(resultErr, err)
+			}
+			res = append(res, kpatch)
+		}
+		return res, resultErr
+	}
+
+	return nil, resultErr
+
+}
+
+func (kg *KustomizeGenerator) getImages() ([]kustomize.Image, error) {
+	img, ok, err := kg.getNestedSlice(specField, imagesField)
 	if err != nil {
 		return nil, err
 	}
 
-	os.WriteFile(kfile, manifest, 0644)
+	var resultErr error
+	if ok {
+		res := make([]kustomize.Image, 0, len(img))
+		for k, i := range img {
+			im, ok := i.(map[string]interface{})
+			if !ok {
+				err := fmt.Errorf("unable to convert patch %d to map[string]interface{}", k)
+				resultErr = multierror.Append(resultErr, err)
+			}
+			var image kustomize.Image
+			err = runtime.DefaultUnstructuredConverter.FromUnstructured(im, &image)
+			if err != nil {
+				resultErr = multierror.Append(resultErr, err)
+			}
+			res = append(res, image)
+		}
+		return res, resultErr
+	}
+
+	return nil, resultErr
 
-	return data, nil
 }
 
 func checkKustomizeImageExists(images []kustypes.Image, imageName string) (bool, int) {
@@ -131,14 +336,32 @@ func checkKustomizeImageExists(images []kustypes.Image, imageName string) (bool,
 	return false, -1
 }
 
-func (kg *KustomizeGenerator) generateKustomization(dirPath string) error {
+func (kg *KustomizeGenerator) getNestedString(fields ...string) (string, bool, error) {
+	val, ok, err := unstructured.NestedString(kg.kustomization.Object, fields...)
+	if err != nil {
+		return "", ok, err
+	}
+
+	return val, ok, nil
+}
+
+func (kg *KustomizeGenerator) getNestedSlice(fields ...string) ([]interface{}, bool, error) {
+	val, ok, err := unstructured.NestedSlice(kg.kustomization.Object, fields...)
+	if err != nil {
+		return nil, ok, err
+	}
+
+	return val, ok, nil
+}
+
+func (kg *KustomizeGenerator) generateKustomization(dirPath string) (action, error) {
 	fs := filesys.MakeFsOnDisk()
 
 	// Determine if there already is a Kustomization file at the root,
 	// as this means we do not have to generate one.
 	for _, kfilename := range konfig.RecognizedKustomizationFileNames() {
 		if kpath := filepath.Join(dirPath, kfilename); fs.Exists(kpath) && !fs.IsDir(kpath) {
-			return nil
+			return unchangedAction, nil
 		}
 	}
 
@@ -186,18 +409,18 @@ func (kg *KustomizeGenerator) generateKustomization(dirPath string) error {
 
 	abs, err := filepath.Abs(dirPath)
 	if err != nil {
-		return err
+		return unchangedAction, err
 	}
 
 	files, err := scan(abs)
 	if err != nil {
-		return err
+		return unchangedAction, err
 	}
 
 	kfile := filepath.Join(dirPath, konfig.DefaultKustomizationFileName())
 	f, err := fs.Create(kfile)
 	if err != nil {
-		return err
+		return unchangedAction, err
 	}
 	f.Close()
 
@@ -216,10 +439,12 @@ func (kg *KustomizeGenerator) generateKustomization(dirPath string) error {
 	kus.Resources = resources
 	kd, err := yaml.Marshal(kus)
 	if err != nil {
-		return err
+		// delete the kustomization file
+		errf := CleanDirectory(dirPath, createdAction)
+		return unchangedAction, fmt.Errorf("%v %v", err, errf)
 	}
 
-	return os.WriteFile(kfile, kd, os.ModePerm)
+	return createdAction, os.WriteFile(kfile, kd, os.ModePerm)
 }
 
 func adaptSelector(selector *kustomize.Selector) (output *kustypes.Selector) {
@@ -239,10 +464,10 @@ func adaptSelector(selector *kustomize.Selector) (output *kustypes.Selector) {
 // TODO: remove mutex when kustomize fixes the concurrent map read/write panic
 var kustomizeBuildMutex sync.Mutex
 
-// buildKustomization wraps krusty.MakeKustomizer with the following settings:
+// BuildKustomization wraps krusty.MakeKustomizer with the following settings:
 // - load files from outside the kustomization.yaml root
 // - disable plugins except for the builtin ones
-func buildKustomization(fs filesys.FileSystem, dirPath string) (resmap.ResMap, error) {
+func BuildKustomization(fs filesys.FileSystem, dirPath string) (resmap.ResMap, error) {
 	// temporary workaround for concurrent map read and map write bug
 	// https://github.com/kubernetes-sigs/kustomize/issues/3659
 	kustomizeBuildMutex.Lock()
@@ -256,3 +481,52 @@ func buildKustomization(fs filesys.FileSystem, dirPath string) (resmap.ResMap, e
 	k := krusty.MakeKustomizer(buildOptions)
 	return k.Run(fs, dirPath)
 }
+
+// CleanDirectory removes the kustomization.yaml file from the given directory.
+func CleanDirectory(dirPath string, action action) error {
+	kfile := filepath.Join(dirPath, konfig.DefaultKustomizationFileName())
+	originalFile := filepath.Join(dirPath, originalKustomizationFile)
+
+	// restore old file if it exists
+	if _, err := os.Stat(originalFile); err == nil {
+		err := os.Rename(originalFile, kfile)
+		if err != nil {
+			return fmt.Errorf("failed to cleanup repository: %w", err)
+		}
+	}
+
+	if action == createdAction {
+		return os.Remove(kfile)
+	}
+
+	return nil
+}
+
+// copyFile copies the contents of the file named src to the file named
+// by dst. The file will be created if it does not already exist or else trucnated.
+func copyFile(src, dst string) (err error) {
+	in, err := os.Open(src)
+	if err != nil {
+		return
+	}
+
+	defer in.Close()
+
+	out, err := os.Create(dst)
+	if err != nil {
+		return
+	}
+
+	defer func() {
+		errf := out.Close()
+		if err == nil {
+			err = errf
+		}
+	}()
+
+	if _, err = io.Copy(out, in); err != nil {
+		return
+	}
+
+	return
+}

internal/kustomization/kustomization_varsub.go

@@ -23,7 +23,10 @@ import (
 	"strings"
 
 	"github.com/drone/envsubst"
+	"github.com/hashicorp/go-multierror"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/kustomize/api/resource"
@@ -37,13 +40,13 @@ const (
 	DisabledValue = "disabled"
 )
 
-// substituteVariables replaces the vars with their values in the specified resource.
+// SubstituteVariables replaces the vars with their values in the specified resource.
 // If a resource is labeled or annotated with
 // 'kustomize.toolkit.fluxcd.io/substitute: disabled' the substitution is skipped.
-func substituteVariables(
+func SubstituteVariables(
 	ctx context.Context,
 	kubeClient client.Client,
-	kustomization Kustomize,
+	kustomization unstructured.Unstructured,
 	res *resource.Resource) (*resource.Resource, error) {
 	resData, err := res.AsYAML()
 	if err != nil {
@@ -56,10 +59,46 @@ func substituteVariables(
 		return nil, nil
 	}
 
+	// load vars from ConfigMaps and Secrets data keys
+	vars, err := loadVars(ctx, kubeClient, kustomization)
+	if err != nil {
+		return nil, err
+	}
+
+	// load in-line vars (overrides the ones from resources)
+	substitute, ok, err := unstructured.NestedStringMap(kustomization.Object, "spec", "postBuild", "substitute")
+	if err != nil {
+		return nil, err
+	}
+	if ok {
+		for k, v := range substitute {
+			vars[k] = strings.Replace(v, "\n", "", -1)
+		}
+	}
+
+	// run bash variable substitutions
+	if len(vars) > 0 {
+		jsonData, err := varSubstitution(resData, vars)
+		if err != nil {
+			return nil, fmt.Errorf("YAMLToJSON: %w", err)
+		}
+		err = res.UnmarshalJSON(jsonData)
+		if err != nil {
+			return nil, fmt.Errorf("UnmarshalJSON: %w", err)
+		}
+	}
+
+	return res, nil
+}
+
+func loadVars(ctx context.Context, kubeClient client.Client, kustomization unstructured.Unstructured) (map[string]string, error) {
 	vars := make(map[string]string)
+	substituteFrom, err := getSubstituteFrom(kustomization)
+	if err != nil {
+		return nil, fmt.Errorf("unable to get subsituteFrom: %w", err)
+	}
 
-	// load vars from ConfigMaps and Secrets data keys
-	for _, reference := range kustomization.GetSubstituteFrom() {
+	for _, reference := range substituteFrom {
 		namespacedName := types.NamespacedName{Namespace: kustomization.GetNamespace(), Name: reference.Name}
 		switch reference.Kind {
 		case "ConfigMap":
@@ -81,39 +120,57 @@ func substituteVariables(
 		}
 	}
 
-	// load in-line vars (overrides the ones from resources)
-	if kustomization.GetSubstitute() != nil {
-		for k, v := range kustomization.GetSubstitute() {
-			vars[k] = strings.Replace(v, "\n", "", -1)
+	return vars, nil
+}
+
+func varSubstitution(data []byte, vars map[string]string) ([]byte, error) {
+	r, _ := regexp.Compile(varsubRegex)
+	for v := range vars {
+		if !r.MatchString(v) {
+			return nil, fmt.Errorf("'%s' var name is invalid, must match '%s'", v, varsubRegex)
 		}
 	}
 
-	// run bash variable substitutions
-	if len(vars) > 0 {
-		r, _ := regexp.Compile(varsubRegex)
-		for v := range vars {
-			if !r.MatchString(v) {
-				return nil, fmt.Errorf("'%s' var name is invalid, must match '%s'", v, varsubRegex)
-			}
-		}
+	output, err := envsubst.Eval(string(data), func(s string) string {
+		return vars[s]
+	})
+	if err != nil {
+		return nil, fmt.Errorf("variable substitution failed: %w", err)
+	}
 
-		output, err := envsubst.Eval(string(resData), func(s string) string {
-			return vars[s]
-		})
-		if err != nil {
-			return nil, fmt.Errorf("variable substitution failed: %w", err)
-		}
+	jsonData, err := yaml.YAMLToJSON([]byte(output))
+	if err != nil {
+		return nil, fmt.Errorf("YAMLToJSON: %w", err)
+	}
 
-		jsonData, err := yaml.YAMLToJSON([]byte(output))
-		if err != nil {
-			return nil, fmt.Errorf("YAMLToJSON: %w", err)
-		}
+	return jsonData, nil
+}
 
-		err = res.UnmarshalJSON(jsonData)
-		if err != nil {
-			return nil, fmt.Errorf("UnmarshalJSON: %w", err)
+func getSubstituteFrom(kustomization unstructured.Unstructured) ([]SubstituteReference, error) {
+	substituteFrom, ok, err := unstructured.NestedSlice(kustomization.Object, "spec", "postBuild", "substituteFrom")
+	if err != nil {
+		return nil, err
+	}
+
+	var resultErr error
+	if ok {
+		res := make([]SubstituteReference, 0, len(substituteFrom))
+		for k, s := range substituteFrom {
+			sub, ok := s.(map[string]interface{})
+			if !ok {
+				err := fmt.Errorf("unable to convert patch %d to map[string]interface{}", k)
+				resultErr = multierror.Append(resultErr, err)
+			}
+			var substitute SubstituteReference
+			err = runtime.DefaultUnstructuredConverter.FromUnstructured(sub, &substitute)
+			if err != nil {
+				resultErr = multierror.Append(resultErr, err)
+			}
+			res = append(res, substitute)
 		}
+		return res, nil
 	}
 
-	return res, nil
+	return nil, resultErr
+
 }