Recursively build and diff Kustomizations

Signed-off-by: Boris Kreitchman <bkreitch@gmail.com>

internal/build/build.go

@@ -25,11 +25,13 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/theckman/yacspin"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -51,13 +53,14 @@ import (
 )
 
 const (
-	controllerName      = "kustomize-controller"
-	controllerGroup     = "kustomize.toolkit.fluxcd.io"
-	mask                = "**SOPS**"
-	dockercfgSecretType = "kubernetes.io/dockerconfigjson"
-	typeField           = "type"
-	dataField           = "data"
-	stringDataField     = "stringData"
+	controllerName       = "kustomize-controller"
+	controllerGroup      = "kustomize.toolkit.fluxcd.io"
+	mask                 = "**SOPS**"
+	dockercfgSecretType  = "kubernetes.io/dockerconfigjson"
+	typeField            = "type"
+	dataField            = "data"
+	stringDataField      = "stringData"
+	spinnerDryRunMessage = "running dry-run"
 )
 
 var defaultTimeout = 80 * time.Second
@@ -81,6 +84,10 @@ type Builder struct {
 	spinner       *yacspin.Spinner
 	dryRun        bool
 	strictSubst   bool
+	recursive     bool
+	localSources  map[string]string
+	// diff needs to handle kustomizations one by one
+	singleKustomization bool
 }
 
 // BuilderOptionFunc is a function that configures a Builder
@@ -110,7 +117,7 @@ func WithProgressBar() BuilderOptionFunc {
 			CharSet:         yacspin.CharSets[59],
 			Suffix:          "Kustomization diffing...",
 			SuffixAutoColon: true,
-			Message:         "running dry-run",
+			Message:         spinnerDryRunMessage,
 			StopCharacter:   "✓",
 			StopColors:      []string{"fgGreen"},
 		}
@@ -175,6 +182,55 @@ func WithIgnore(ignore []string) BuilderOptionFunc {
 	}
 }
 
+// WithRecursive sets the recursive field
+func WithRecursive(recursive bool) BuilderOptionFunc {
+	return func(b *Builder) error {
+		b.recursive = recursive
+		return nil
+	}
+}
+
+// WithLocalSources sets the local sources field
+func WithLocalSources(localSources map[string]string) BuilderOptionFunc {
+	return func(b *Builder) error {
+		b.localSources = localSources
+		return nil
+	}
+}
+
+// WithSingleKustomization sets the single kustomization field to true
+func WithSingleKustomization() BuilderOptionFunc {
+	return func(b *Builder) error {
+		b.singleKustomization = true
+		return nil
+	}
+}
+
+// withClientConfigFrom copies client and restMapper fields
+func withClientConfigFrom(in *Builder) BuilderOptionFunc {
+	return func(b *Builder) error {
+		b.client = in.client
+		b.restMapper = in.restMapper
+		return nil
+	}
+}
+
+// withClientConfigFrom copies spinner field
+func withSpinnerFrom(in *Builder) BuilderOptionFunc {
+	return func(b *Builder) error {
+		b.spinner = in.spinner
+		return nil
+	}
+}
+
+// withKustomization sets the kustomization field
+func withKustomization(k *kustomizev1.Kustomization) BuilderOptionFunc {
+	return func(b *Builder) error {
+		b.kustomization = k
+		return nil
+	}
+}
+
 // NewBuilder returns a new Builder
 // It takes a kustomization name and a path to the resources
 // It also takes a list of BuilderOptionFunc to configure the builder
@@ -269,6 +325,27 @@ func (b *Builder) Build() ([]*unstructured.Unstructured, error) {
 		ssautil.SetCommonMetadata(objects, m.Labels, m.Annotations)
 	}
 
+	if b.recursive && !b.singleKustomization {
+		var objectsToAdd []*unstructured.Unstructured
+		for _, obj := range objects {
+			if isKustomization(obj) {
+				k, err := toKustomization(obj)
+				if err != nil {
+					return nil, err
+				}
+
+				if !kustomizationsEqual(k, b.kustomization) {
+					subObjects, err := b.kustomizationBuild(k)
+					if err != nil {
+						return nil, err
+					}
+					objectsToAdd = append(objectsToAdd, subObjects...)
+				}
+			}
+		}
+		objects = append(objects, objectsToAdd...)
+	}
+
 	return objects, nil
 }
 
@@ -281,7 +358,11 @@ func (b *Builder) build() (m resmap.ResMap, err error) {
 	if !b.dryRun {
 		liveKus, err = b.getKustomization(ctx)
 		if err != nil {
-			return nil, fmt.Errorf("failed to get kustomization object: %w", err)
+			if !apierrors.IsNotFound(err) || b.kustomization == nil {
+				return nil, fmt.Errorf("failed to get kustomization object: %w", err)
+			}
+			// use provided Kustomization
+			liveKus = b.kustomization
 		}
 	}
 	k, err := b.resolveKustomization(liveKus)
@@ -334,6 +415,46 @@ func (b *Builder) build() (m resmap.ResMap, err error) {
 
 }
 
+func (b *Builder) kustomizationBuild(k *kustomizev1.Kustomization) ([]*unstructured.Unstructured, error) {
+	resourcesPath, err := b.kustomizationPath(k)
+	if err != nil {
+		return nil, err
+	}
+
+	subBuilder, err := NewBuilder(k.Name, resourcesPath,
+		// use same client
+		withClientConfigFrom(b),
+		// kustomization will be used if there is no live kustomization
+		withKustomization(k),
+		WithTimeout(b.timeout),
+		WithNamespace(k.Namespace),
+		WithIgnore(b.ignore),
+		WithStrictSubstitute(b.strictSubst),
+		WithRecursive(b.recursive),
+		WithLocalSources(b.localSources),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return subBuilder.Build()
+}
+
+func (b *Builder) kustomizationPath(k *kustomizev1.Kustomization) (string, error) {
+	sourceRef := k.Spec.SourceRef.DeepCopy()
+	if sourceRef.Namespace == "" {
+		sourceRef.Namespace = k.Namespace
+	}
+
+	sourceKey := sourceRef.String()
+	localPath, ok := b.localSources[sourceKey]
+	if !ok {
+		return "", fmt.Errorf("cannot get local path for %s of kustomization %s", sourceKey, k.Name)
+	}
+
+	return filepath.Join(localPath, k.Spec.Path), nil
+}
+
 func (b *Builder) unMarshallKustomization() (*kustomizev1.Kustomization, error) {
 	data, err := os.ReadFile(b.kustomizationFile)
 	if err != nil {
@@ -423,6 +544,28 @@ func (b *Builder) do(ctx context.Context, kustomization kustomizev1.Kustomizatio
 	return m, nil
 }
 
+func isKustomization(object *unstructured.Unstructured) bool {
+	return strings.HasPrefix(object.GetAPIVersion(), kustomizev1.GroupVersion.Group+"/") &&
+		object.GetKind() == kustomizev1.KustomizationKind
+}
+
+func toKustomization(object *unstructured.Unstructured) (*kustomizev1.Kustomization, error) {
+	obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(object)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert to unstructured: %w", err)
+	}
+	k := &kustomizev1.Kustomization{}
+	err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj, k)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert to kustomization: %w", err)
+	}
+	return k, nil
+}
+
+func kustomizationsEqual(k1 *kustomizev1.Kustomization, k2 *kustomizev1.Kustomization) bool {
+	return k1.Name == k2.Name && k1.Namespace == k2.Namespace
+}
+
 func (b *Builder) setOwnerLabels(res *resource.Resource) error {
 	labels := res.GetLabels()
 
@@ -583,12 +726,7 @@ func (b *Builder) Cancel() error {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
-	err := b.stopSpinner()
-	if err != nil {
-		return err
-	}
-
-	err = kustomize.CleanDirectory(b.resourcesPath, b.action)
+	err := kustomize.CleanDirectory(b.resourcesPath, b.action)
 	if err != nil {
 		return err
 	}
@@ -596,7 +734,7 @@ func (b *Builder) Cancel() error {
 	return nil
 }
 
-func (b *Builder) startSpinner() error {
+func (b *Builder) StartSpinner() error {
 	if b.spinner == nil {
 		return nil
 	}
@@ -609,7 +747,7 @@ func (b *Builder) startSpinner() error {
 	return nil
 }
 
-func (b *Builder) stopSpinner() error {
+func (b *Builder) StopSpinner() error {
 	if b.spinner == nil {
 		return nil
 	}

internal/build/build_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/fluxcd/pkg/apis/meta"
 	"github.com/google/go-cmp/cmp"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"sigs.k8s.io/kustomize/api/resource"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
@@ -361,3 +362,242 @@ func Test_ResolveKustomization(t *testing.T) {
 		})
 	}
 }
+
+func Test_isKustomization(t *testing.T) {
+	tests := []struct {
+		name     string
+		expected bool
+		object   *unstructured.Unstructured
+	}{
+		{
+			name: "flux kustomization",
+			object: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "kustomize.toolkit.fluxcd.io/v1",
+					"kind":       "Kustomization",
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "other kustomization",
+			object: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "kustomize.config.k8s.io/v1beta1",
+					"kind":       "Kustomization",
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "wrong kind",
+			object: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "kustomize.toolkit.fluxcd.io/v1",
+					"kind":       "ConfigMap",
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "wrong object",
+			object: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "v1",
+					"kind":       "ConfigMap",
+				},
+			},
+			expected: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual := isKustomization(tt.object)
+			if actual != tt.expected {
+				t.Fatalf("got '%v', want '%v'", actual, tt.expected)
+			}
+		})
+	}
+}
+
+func Test_kustomizationsEqual(t *testing.T) {
+	tests := []struct {
+		name           string
+		kustomization1 *kustomizev1.Kustomization
+		kustomization2 *kustomizev1.Kustomization
+		expected       bool
+	}{
+		{
+			name: "equal",
+			kustomization1: &kustomizev1.Kustomization{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "podinfo",
+					Namespace: "flux-system",
+				},
+			},
+			kustomization2: &kustomizev1.Kustomization{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "podinfo",
+					Namespace: "flux-system",
+				},
+			},
+			expected: true,
+		},
+		{
+			name: "wrong name",
+			kustomization1: &kustomizev1.Kustomization{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "podinfo",
+					Namespace: "flux-system",
+				},
+			},
+			kustomization2: &kustomizev1.Kustomization{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "my-app",
+					Namespace: "flux-system",
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "wrong namespace",
+			kustomization1: &kustomizev1.Kustomization{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "podinfo",
+					Namespace: "flux-system",
+				},
+			},
+			kustomization2: &kustomizev1.Kustomization{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "podinfo",
+					Namespace: "my-ns",
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "wrong name and namespace",
+			kustomization1: &kustomizev1.Kustomization{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "podinfo",
+					Namespace: "flux-system",
+				},
+			},
+			kustomization2: &kustomizev1.Kustomization{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "my-app",
+					Namespace: "my-ns",
+				},
+			},
+			expected: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual := kustomizationsEqual(tt.kustomization1, tt.kustomization2)
+			if actual != tt.expected {
+				t.Fatalf("got '%v', want '%v'", actual, tt.expected)
+			}
+		})
+	}
+}
+
+func Test_kustomizationPath(t *testing.T) {
+	tests := []struct {
+		name          string
+		kustomization *kustomizev1.Kustomization
+		expected      string
+		wantErr       bool
+		errString     string
+	}{
+		{
+			name: "full repo",
+			kustomization: &kustomizev1.Kustomization{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "my-app",
+					Namespace: "flux-system",
+				},
+				Spec: kustomizev1.KustomizationSpec{
+					Path: "my-path",
+					SourceRef: kustomizev1.CrossNamespaceSourceReference{
+						Kind:      "GitRepository",
+						Name:      "my-repo",
+						Namespace: "flux-system",
+					},
+				},
+			},
+			expected: "path/to/local/git/my-path",
+		},
+		{
+			name: "repo without namespace",
+			kustomization: &kustomizev1.Kustomization{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "my-app",
+					Namespace: "flux-system",
+				},
+				Spec: kustomizev1.KustomizationSpec{
+					Path: "my-path",
+					SourceRef: kustomizev1.CrossNamespaceSourceReference{
+						Kind:      "GitRepository",
+						Name:      "my-repo",
+						Namespace: "",
+					},
+				},
+			},
+			expected: "path/to/local/git/my-path",
+		},
+		{
+			name: "repo not found",
+			kustomization: &kustomizev1.Kustomization{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "my-app",
+					Namespace: "flux-system",
+				},
+				Spec: kustomizev1.KustomizationSpec{
+					Path: "my-path",
+					SourceRef: kustomizev1.CrossNamespaceSourceReference{
+						Kind:      "GitRepository",
+						Name:      "my-repo",
+						Namespace: "my-ns",
+					},
+				},
+			},
+			wantErr:   true,
+			errString: "cannot get local path",
+		},
+	}
+
+	b := &Builder{
+		name:      "podinfo",
+		namespace: "flux-system",
+		localSources: map[string]string{
+			"GitRepository/flux-system/my-repo": "./path/to/local/git",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			actual, err := b.kustomizationPath(tt.kustomization)
+			if !tt.wantErr {
+				if err != nil {
+					t.Fatalf("unexpected err '%s'", err)
+				}
+
+				if actual != tt.expected {
+					t.Errorf("got '%v', want '%v'", actual, tt.expected)
+				}
+			} else {
+				if err == nil {
+					t.Fatal("expected error but got nil")
+				}
+
+				if !strings.Contains(err.Error(), tt.errString) {
+					t.Errorf("expected error '%s' to contain string '%s'", err.Error(), tt.errString)
+				}
+			}
+
+		})
+	}
+}

internal/build/diff.go

@@ -57,6 +57,22 @@ func (b *Builder) Manager() (*ssa.ResourceManager, error) {
 }
 
 func (b *Builder) Diff() (string, bool, error) {
+	err := b.StartSpinner()
+	if err != nil {
+		return "", false, err
+	}
+
+	output, createdOrDrifted, diffErr := b.diff()
+
+	err = b.StopSpinner()
+	if err != nil {
+		return "", false, err
+	}
+
+	return output, createdOrDrifted, diffErr
+}
+
+func (b *Builder) diff() (string, bool, error) {
 	output := strings.Builder{}
 	createdOrDrifted := false
 	objects, err := b.Build()
@@ -77,11 +93,6 @@ func (b *Builder) Diff() (string, bool, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
 	defer cancel()
 
-	err = b.startSpinner()
-	if err != nil {
-		return "", false, err
-	}
-
 	var diffErrs []error
 	// create an inventory of objects to be reconciled
 	newInventory := newInventory()
@@ -127,6 +138,30 @@ func (b *Builder) Diff() (string, bool, error) {
 		}
 
 		addObjectsToInventory(newInventory, change)
+
+		if b.recursive && isKustomization(obj) && change.Action != ssa.CreatedAction {
+			k, err := toKustomization(obj)
+			if err != nil {
+				return "", createdOrDrifted, err
+			}
+
+			if !kustomizationsEqual(k, b.kustomization) {
+				subOutput, subCreatedOrDrifted, err := b.kustomizationDiff(k)
+				if err != nil {
+					diffErrs = append(diffErrs, err)
+				}
+				if subCreatedOrDrifted {
+					createdOrDrifted = true
+					output.WriteString(bunt.Sprint(fmt.Sprintf("📁 %s changed\n", ssautil.FmtUnstructured(obj))))
+					output.WriteString(subOutput)
+				}
+
+				// finished with Kustomization diff
+				if b.spinner != nil {
+					b.spinner.Message(spinnerDryRunMessage)
+				}
+			}
+		}
 	}
 
 	if b.spinner != nil {
@@ -149,12 +184,43 @@ func (b *Builder) Diff() (string, bool, error) {
 		}
 	}
 
-	err = b.stopSpinner()
-	if err != nil {
-		return "", createdOrDrifted, err
+	return output.String(), createdOrDrifted, errors.Reduce(errors.Flatten(errors.NewAggregate(diffErrs)))
+}
+
+func (b *Builder) kustomizationDiff(kustomization *kustomizev1.Kustomization) (string, bool, error) {
+	if b.spinner != nil {
+		b.spinner.Message(fmt.Sprintf("%s in %s", spinnerDryRunMessage, kustomization.Name))
 	}
 
-	return output.String(), createdOrDrifted, errors.Reduce(errors.Flatten(errors.NewAggregate(diffErrs)))
+	sourceRef := kustomization.Spec.SourceRef.DeepCopy()
+	if sourceRef.Namespace == "" {
+		sourceRef.Namespace = kustomization.Namespace
+	}
+
+	sourceKey := sourceRef.String()
+	localPath, ok := b.localSources[sourceKey]
+	if !ok {
+		return "", false, fmt.Errorf("cannot get local path for %s of kustomization %s", sourceKey, kustomization.Name)
+	}
+
+	resourcesPath := filepath.Join(localPath, kustomization.Spec.Path)
+	subBuilder, err := NewBuilder(kustomization.Name, resourcesPath,
+		// use same client and spinner
+		withClientConfigFrom(b),
+		withSpinnerFrom(b),
+		WithTimeout(b.timeout),
+		WithNamespace(kustomization.Namespace),
+		WithIgnore(b.ignore),
+		WithStrictSubstitute(b.strictSubst),
+		WithRecursive(b.recursive),
+		WithLocalSources(b.localSources),
+		WithSingleKustomization(),
+	)
+	if err != nil {
+		return "", false, err
+	}
+
+	return subBuilder.diff()
 }
 
 func writeYamls(liveObject, mergedObject *unstructured.Unstructured) (string, string, string, error) {