Merge pull request #2418 from fluxcd/fix-bootstrap

Fix bootstrap: Reset schema cache after applying CRDs

cmd/flux/bootstrap_bitbucket_server.go

@@ -254,6 +254,7 @@ func bootstrapBServerCmdRun(cmd *cobra.Command, args []string) error {
 		bootstrap.WithKubeconfig(kubeconfigArgs),
 		bootstrap.WithLogger(logger),
 		bootstrap.WithCABundle(caBundle),
+		bootstrap.WithGitCommitSigning(bootstrapArgs.gpgKeyRingPath, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID),
 	}
 	if bootstrapArgs.sshHostname != "" {
 		bootstrapOpts = append(bootstrapOpts, bootstrap.WithSSHHostname(bootstrapArgs.sshHostname))

cmd/flux/bootstrap_github.go

@@ -243,6 +243,7 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error {
 		bootstrap.WithKubeconfig(kubeconfigArgs),
 		bootstrap.WithLogger(logger),
 		bootstrap.WithCABundle(caBundle),
+		bootstrap.WithGitCommitSigning(bootstrapArgs.gpgKeyRingPath, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID),
 	}
 	if bootstrapArgs.sshHostname != "" {
 		bootstrapOpts = append(bootstrapOpts, bootstrap.WithSSHHostname(bootstrapArgs.sshHostname))

cmd/flux/bootstrap_gitlab.go

@@ -257,6 +257,7 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error {
 		bootstrap.WithKubeconfig(kubeconfigArgs),
 		bootstrap.WithLogger(logger),
 		bootstrap.WithCABundle(caBundle),
+		bootstrap.WithGitCommitSigning(bootstrapArgs.gpgKeyRingPath, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID),
 	}
 	if bootstrapArgs.sshHostname != "" {
 		bootstrapOpts = append(bootstrapOpts, bootstrap.WithSSHHostname(bootstrapArgs.sshHostname))

cmd/flux/create_image_update.go

@@ -49,25 +49,40 @@ mentioned in YAMLs in a git repository.`,
     --push-branch=image-updates \
     --author-name=flux \
     --author-email=flux@example.com \
-    --commit-template="{{range .Updated.Images}}{{println .}}{{end}}"`,
+    --commit-template="{{range .Updated.Images}}{{println .}}{{end}}"
+
+  # Configure image updates for a Git repository in a different namespace
+  flux create image update apps \
+    --namespace=apps \
+    --git-repo-ref=flux-system \
+    --git-repo-namespace=flux-system \
+    --git-repo-path="./clusters/my-cluster" \
+    --checkout-branch=main \
+    --push-branch=image-updates \
+    --author-name=flux \
+    --author-email=flux@example.com \
+    --commit-template="{{range .Updated.Images}}{{println .}}{{end}}"
+`,
 	RunE: createImageUpdateRun,
 }
 
 type imageUpdateFlags struct {
-	gitRepoRef     string
-	gitRepoPath    string
-	checkoutBranch string
-	pushBranch     string
-	commitTemplate string
-	authorName     string
-	authorEmail    string
+	gitRepoName      string
+	gitRepoNamespace string
+	gitRepoPath      string
+	checkoutBranch   string
+	pushBranch       string
+	commitTemplate   string
+	authorName       string
+	authorEmail      string
 }
 
 var imageUpdateArgs = imageUpdateFlags{}
 
 func init() {
 	flags := createImageUpdateCmd.Flags()
-	flags.StringVar(&imageUpdateArgs.gitRepoRef, "git-repo-ref", "", "the name of a GitRepository resource with details of the upstream Git repository")
+	flags.StringVar(&imageUpdateArgs.gitRepoName, "git-repo-ref", "", "the name of a GitRepository resource with details of the upstream Git repository")
+	flags.StringVar(&imageUpdateArgs.gitRepoNamespace, "git-repo-namespace", "", "the namespace of the GitRepository resource, defaults to the ImageUpdateAutomation namespace")
 	flags.StringVar(&imageUpdateArgs.gitRepoPath, "git-repo-path", "", "path to the directory containing the manifests to be updated, defaults to the repository root")
 	flags.StringVar(&imageUpdateArgs.checkoutBranch, "checkout-branch", "", "the branch to checkout")
 	flags.StringVar(&imageUpdateArgs.pushBranch, "push-branch", "", "the branch to push commits to, defaults to the checkout branch if not specified")
@@ -84,7 +99,7 @@ func createImageUpdateRun(cmd *cobra.Command, args []string) error {
 	}
 	objectName := args[0]
 
-	if imageUpdateArgs.gitRepoRef == "" {
+	if imageUpdateArgs.gitRepoName == "" {
 		return fmt.Errorf("a reference to a GitRepository is required (--git-repo-ref)")
 	}
 
@@ -113,8 +128,9 @@ func createImageUpdateRun(cmd *cobra.Command, args []string) error {
 		},
 		Spec: autov1.ImageUpdateAutomationSpec{
 			SourceRef: autov1.CrossNamespaceSourceReference{
-				Kind: sourcev1.GitRepositoryKind,
-				Name: imageUpdateArgs.gitRepoRef,
+				Kind:      sourcev1.GitRepositoryKind,
+				Name:      imageUpdateArgs.gitRepoName,
+				Namespace: imageUpdateArgs.gitRepoNamespace,
 			},
 
 			GitSpec: &autov1.GitSpec{

cmd/flux/diff.go

@@ -23,7 +23,7 @@ import (
 var diffCmd = &cobra.Command{
 	Use:   "diff",
 	Short: "Diff a flux resource",
-	Long:  "The diff command is used to do a server-side dry-run on flux resources, then output the diff.",
+	Long:  "The diff command is used to do a server-side dry-run on flux resources, then prints the diff.",
 }
 
 func init() {

cmd/flux/diff_kustomization.go

@@ -31,8 +31,9 @@ var diffKsCmd = &cobra.Command{
 	Use:     "kustomization",
 	Aliases: []string{"ks"},
 	Short:   "Diff Kustomization",
-	Long:    `The diff command does a build, then it performs a server-side dry-run and output the diff.`,
-	Example: `# Preview changes local changes as they were applied on the cluster
+	Long: `The diff command does a build, then it performs a server-side dry-run and prints the diff.
+Exit status: 0 No differences were found. 1 Differences were found. >1 diff failed with an error.`,
+	Example: `# Preview local changes as they were applied on the cluster
 flux diff kustomization my-app --path ./path/to/local/manifests`,
 	ValidArgsFunction: resourceNamesCompletionFunc(kustomizev1.GroupVersion.WithKind(kustomizev1.KustomizationKind)),
 	RunE:              diffKsCmdRun,
@@ -56,16 +57,16 @@ func diffKsCmdRun(cmd *cobra.Command, args []string) error {
 	name := args[0]
 
 	if diffKsArgs.path == "" {
-		return fmt.Errorf("invalid resource path %q", diffKsArgs.path)
+		return &RequestError{StatusCode: 2, Err: fmt.Errorf("invalid resource path %q", diffKsArgs.path)}
 	}
 
 	if fs, err := os.Stat(diffKsArgs.path); err != nil || !fs.IsDir() {
-		return fmt.Errorf("invalid resource path %q", diffKsArgs.path)
+		return &RequestError{StatusCode: 2, Err: fmt.Errorf("invalid resource path %q", diffKsArgs.path)}
 	}
 
 	builder, err := build.NewBuilder(kubeconfigArgs, name, diffKsArgs.path, build.WithTimeout(rootArgs.timeout))
 	if err != nil {
-		return err
+		return &RequestError{StatusCode: 2, Err: err}
 	}
 
 	// create a signal channel
@@ -74,13 +75,18 @@ func diffKsCmdRun(cmd *cobra.Command, args []string) error {
 
 	errChan := make(chan error)
 	go func() {
-		output, err := builder.Diff()
+		output, hasChanged, err := builder.Diff()
 		if err != nil {
-			errChan <- err
+			errChan <- &RequestError{StatusCode: 2, Err: err}
 		}
 
 		cmd.Print(output)
-		errChan <- nil
+
+		if hasChanged {
+			errChan <- &RequestError{StatusCode: 1, Err: fmt.Errorf("identified at least one change, exiting with non-zero exit code")}
+		} else {
+			errChan <- nil
+		}
 	}()
 
 	select {

cmd/flux/diff_kustomization_test.go

@@ -79,6 +79,18 @@ func TestDiffKustomization(t *testing.T) {
 			objectFile: "./testdata/diff-kustomization/value-sops-secret.yaml",
 			assert:     assertGoldenFile("./testdata/diff-kustomization/diff-with-drifted-value-sops-secret.golden"),
 		},
+		{
+			name:       "diff with a sops dockerconfigjson secret object",
+			args:       "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo",
+			objectFile: "./testdata/diff-kustomization/dockerconfigjson-sops-secret.yaml",
+			assert:     assertGoldenFile("./testdata/diff-kustomization/diff-with-dockerconfigjson-sops-secret.golden"),
+		},
+		{
+			name:       "diff with a sops stringdata secret object",
+			args:       "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo",
+			objectFile: "./testdata/diff-kustomization/stringdata-sops-secret.yaml",
+			assert:     assertGoldenFile("./testdata/diff-kustomization/diff-with-stringdata-sops-secret.golden"),
+		},
 	}
 
 	tmpl := map[string]string{

cmd/flux/main.go

@@ -105,6 +105,16 @@ type rootFlags struct {
 	defaults     install.Options
 }
 
+// RequestError is a custom error type that wraps an error returned by the flux api.
+type RequestError struct {
+	StatusCode int
+	Err        error
+}
+
+func (r *RequestError) Error() string {
+	return r.Err.Error()
+}
+
 var rootArgs = NewRootFlags()
 var kubeconfigArgs = genericclioptions.NewConfigFlags(false)
 
@@ -143,6 +153,17 @@ func NewRootFlags() rootFlags {
 func main() {
 	log.SetFlags(0)
 	if err := rootCmd.Execute(); err != nil {
+
+		if err, ok := err.(*RequestError); ok {
+			if err.StatusCode == 1 {
+				logger.Warningf("%v", err)
+			} else {
+				logger.Failuref("%v", err)
+			}
+
+			os.Exit(err.StatusCode)
+		}
+
 		logger.Failuref("%v", err)
 		os.Exit(1)
 	}

cmd/flux/main_test.go

@@ -325,6 +325,12 @@ type cmdTestCase struct {
 
 func (cmd *cmdTestCase) runTestCmd(t *testing.T) {
 	actual, testErr := executeCommand(cmd.args)
+
+	// If the cmd error is a change, discard it
+	if isChangeError(testErr) {
+		testErr = nil
+	}
+
 	if assertErr := cmd.assert(actual, testErr); assertErr != nil {
 		t.Error(assertErr)
 	}
@@ -366,3 +372,12 @@ func resetCmdArgs() {
 	getArgs = GetFlags{}
 	secretGitArgs = NewSecretGitFlags()
 }
+
+func isChangeError(err error) bool {
+	if reqErr, ok := err.(*RequestError); ok {
+		if strings.Contains(err.Error(), "identified at least one change, exiting with non-zero exit code") && reqErr.StatusCode == 1 {
+			return true
+		}
+	}
+	return false
+}

cmd/flux/resume.go

@@ -48,6 +48,7 @@ func init() {
 
 type resumable interface {
 	adapter
+	copyable
 	statusable
 	setUnsuspended()
 	successMessage() string
@@ -97,10 +98,13 @@ func (resume resumeCommand) run(cmd *cobra.Command, args []string) error {
 
 	for i := 0; i < resume.list.len(); i++ {
 		logger.Actionf("resuming %s %s in %s namespace", resume.humanKind, resume.list.resumeItem(i).asClientObject().GetName(), *kubeconfigArgs.Namespace)
-		resume.list.resumeItem(i).setUnsuspended()
-		if err := kubeClient.Update(ctx, resume.list.resumeItem(i).asClientObject()); err != nil {
+		obj := resume.list.resumeItem(i)
+		patch := client.MergeFrom(obj.deepCopyClientObject())
+		obj.setUnsuspended()
+		if err := kubeClient.Patch(ctx, obj.asClientObject(), patch); err != nil {
 			return err
 		}
+
 		logger.Successf("%s resumed", resume.humanKind)
 
 		namespacedName := types.NamespacedName{

cmd/flux/suspend.go

@@ -46,6 +46,7 @@ func init() {
 
 type suspendable interface {
 	adapter
+	copyable
 	isSuspended() bool
 	setSuspended()
 }
@@ -94,8 +95,11 @@ func (suspend suspendCommand) run(cmd *cobra.Command, args []string) error {
 
 	for i := 0; i < suspend.list.len(); i++ {
 		logger.Actionf("suspending %s %s in %s namespace", suspend.humanKind, suspend.list.item(i).asClientObject().GetName(), *kubeconfigArgs.Namespace)
-		suspend.list.item(i).setSuspended()
-		if err := kubeClient.Update(ctx, suspend.list.item(i).asClientObject()); err != nil {
+
+		obj := suspend.list.item(i)
+		patch := client.MergeFrom(obj.deepCopyClientObject())
+		obj.setSuspended()
+		if err := kubeClient.Patch(ctx, obj.asClientObject(), patch); err != nil {
 			return err
 		}
 		logger.Successf("%s suspended", suspend.humanKind)

cmd/flux/testdata/build-kustomization/podinfo-result.yaml

@@ -123,6 +123,31 @@ spec:
   type: ClusterIP
 ---
 apiVersion: v1
+data:
+  .dockerconfigjson: eyJtYXNrIjoiKipTT1BTKioifQ==
+kind: Secret
+metadata:
+  labels:
+    kustomize.toolkit.fluxcd.io/name: podinfo
+    kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }}
+  name: docker-secret
+  namespace: default
+type: kubernetes.io/dockerconfigjson
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    kustomize.toolkit.fluxcd.io/name: podinfo
+    kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }}
+  name: secret-basic-auth-stringdata
+  namespace: default
+stringData:
+  password: KipTT1BTKio=
+  username: KipTT1BTKio=
+type: kubernetes.io/basic-auth
+---
+apiVersion: v1
 data:
   token: KipTT1BTKio=
 kind: Secret

cmd/flux/testdata/build-kustomization/podinfo/dockerconfigjson-sops-secret.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+data:
+  .dockerconfigjson: ENC[AES256_GCM,data:KHCFH3hNnc+PMfWLFEPjebf3W4z4WXbGFAANRZyZC+07z7wlrTALJM6rn8YslW4tMAWCoAYxblC5WRCszTy0h9rw0U/RGOv5H0qCgnNg/FILFUqhwo9pNfrUH+MEP4M9qxxbLKZwObpHUE7DUsKx1JYAxsI=,iv:q48lqUbUQD+0cbYcjNMZMJLRdGHi78ZmDhNAT2th9tg=,tag:QRI2SZZXQrAcdql3R5AH2g==,type:str]
+kind: Secret
+metadata:
+  name: docker-secret
+type: kubernetes.io/dockerconfigjson
+sops:
+  kms: []
+  gcp_kms: []
+  azure_kv: []
+  hc_vault: []
+  age:
+    - recipient: age10la2ge0wtvx3qr7datqf7rs4yngxszdal927fs9rukamr8u2pshsvtz7ce
+      enc: |
+        -----BEGIN AGE ENCRYPTED FILE-----
+        YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA3eU1CTEJhVXZ4eEVYYkVV
+        OU90TEcrR2pYckttN0pBanJoSUZWSW1RQXlRCkUydFJ3V1NZUTBuVFF0aC9GUEcw
+        bUdhNjJWTkoyL1FUVi9Dc1dxUDBkM0UKLS0tIE1sQXkwcWdGaEFuY0RHQTVXM0J6
+        dWpJcThEbW15V3dXYXpPZklBdW1Hd1kKoIAdmGNPrEctV8h1w8KuvQ5S+BGmgqN9
+        MgpNmUhJjWhgcQpb5BRYpQesBOgU5TBGK7j58A6DMDKlSiYZsdQchQ==
+        -----END AGE ENCRYPTED FILE-----
+  lastmodified: "2022-02-03T16:03:17Z"
+  mac: ENC[AES256_GCM,data:AHdYSawajwgAFwlmDN1IPNmT9vWaYKzyVIra2d6sPcjTbZ8/p+VRSRpVm4XZFFsaNnW5AUJaouwXnKYDTmJDXKlr/rQcu9kXqsssQgdzcXaA6l5uJlgsnml8ba7J3OK+iEKMax23mwQEx2EUskCd9ENOwFDkunP02sxqDNOz20k=,iv:8F5OamHt3fAVorf6p+SoIrWoqkcATSGWVoM0EK87S4M=,tag:E1mxXnc7wWkEX5BxhpLtng==,type:str]
+  pgp: []
+  encrypted_regex: ^(data|stringData)$
+  version: 3.7.1

cmd/flux/testdata/build-kustomization/podinfo/kustomization.yaml

@@ -4,6 +4,8 @@ resources:
 - ./deployment.yaml
 - ./hpa.yaml
 - ./service.yaml
+- ./dockerconfigjson-sops-secret.yaml
+- ./stringdata-secret.yaml
 secretGenerator:
  - files:
    - token=token.encrypted

cmd/flux/testdata/build-kustomization/podinfo/stringdata-secret.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: secret-basic-auth-stringdata
+type: kubernetes.io/basic-auth
+stringData:
+    username: ENC[AES256_GCM,data:uKiQR48=,iv:jh2lgyAVu7igJAgoJsnOGhjxFyvUAa9lvT21u3hhqpU=,tag:zXM2JEpk3ZEH7WfkcWXXkw==,type:str]
+    password: ENC[AES256_GCM,data:PyhZmNhy929JGQ==,iv:PBqPaJmSw21+kn4gIlg5VdjLNZyf613z5RUTCesBoVw=,tag:Hjc7DsuUrtsz7PYPdNkL3g==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age10la2ge0wtvx3qr7datqf7rs4yngxszdal927fs9rukamr8u2pshsvtz7ce
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJd0xxbDZhYjVoZzY4YWhK
+            d2NvMVgrSGRVUGhHRGg3R1FpVURnbmh1TDBzCjcwby85M3JaK09QVk0yZFNMb2NL
+            c2NQZW5hS1FhYlBHU0VoUzBVYzZYUUUKLS0tIEdaNEw2Y0VjVHpZc3pyYUtLVmJk
+            NmN3K2VLU0NiZ1d0VHBYbGlCM1lrNmMKeWz3yfFbMNE+ly21oLfc1XnDSPRmnlPP
+            wIs8lk/qrzVZ45C9GdWnnPeGZZiia46Yop9TxseUS8gCjJ6KCxJCAg==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2022-02-06T12:51:07Z"
+    mac: ENC[AES256_GCM,data:jtdzwj19uxdxvnmXg1HkAkDA6XlKMJOYFy7uLI5t/t11LwGop5Yeo7a4nQEEELehRx9J7B6U6NiySxAxBxWx5uW5vI5c8+069VV6dkiCIefnYSzuoIhQafjlFl1/KvH7VEjIWfHYuXF09v9PEKXkxEHUYDpS3QqQ3ymHRRI08pU=,  iv:xX3E7F+AM29Pm8G5oqxRfYu9E7tEBGIaHeCJYgrtFmc=,tag:MJPGusNvu05z939jg8PAwQ==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.7.1

cmd/flux/testdata/diff-kustomization/diff-with-deployment.golden

@@ -1,4 +1,6 @@
 ► HorizontalPodAutoscaler/default/podinfo created
 ► Service/default/podinfo created
+► Secret/default/docker-secret created
+► Secret/default/secret-basic-auth-stringdata created
 ► Secret/default/podinfo-token-77t89m9b67 created
 ► Secret/default/db-user-pass-bkbd782d2c created

cmd/flux/testdata/diff-kustomization/diff-with-dockerconfigjson-sops-secret.golden

@@ -0,0 +1,6 @@
+► Deployment/default/podinfo created
+► HorizontalPodAutoscaler/default/podinfo created
+► Service/default/podinfo created
+► Secret/default/secret-basic-auth-stringdata created
+► Secret/default/podinfo-token-77t89m9b67 created
+► Secret/default/db-user-pass-bkbd782d2c created

cmd/flux/testdata/diff-kustomization/diff-with-drifted-key-sops-secret.golden

@@ -1,6 +1,8 @@
 ► Deployment/default/podinfo created
 ► HorizontalPodAutoscaler/default/podinfo created
 ► Service/default/podinfo created
+► Secret/default/docker-secret created
+► Secret/default/secret-basic-auth-stringdata created
 ► Secret/default/podinfo-token-77t89m9b67 drifted
 
 data

cmd/flux/testdata/diff-kustomization/diff-with-drifted-secret.golden

@@ -1,6 +1,8 @@
 ► Deployment/default/podinfo created
 ► HorizontalPodAutoscaler/default/podinfo created
 ► Service/default/podinfo created
+► Secret/default/docker-secret created
+► Secret/default/secret-basic-auth-stringdata created
 ► Secret/default/podinfo-token-77t89m9b67 created
 ► Secret/default/db-user-pass-bkbd782d2c drifted
 

cmd/flux/testdata/diff-kustomization/diff-with-drifted-service.golden

@@ -7,5 +7,7 @@ spec.ports.http.port
     - 9899
     + 9898
 
+► Secret/default/docker-secret created
+► Secret/default/secret-basic-auth-stringdata created
 ► Secret/default/podinfo-token-77t89m9b67 created
 ► Secret/default/db-user-pass-bkbd782d2c created

cmd/flux/testdata/diff-kustomization/diff-with-drifted-value-sops-secret.golden

@@ -1,4 +1,6 @@
 ► Deployment/default/podinfo created
 ► HorizontalPodAutoscaler/default/podinfo created
 ► Service/default/podinfo created
+► Secret/default/docker-secret created
+► Secret/default/secret-basic-auth-stringdata created
 ► Secret/default/db-user-pass-bkbd782d2c created

cmd/flux/testdata/diff-kustomization/diff-with-stringdata-sops-secret.golden

@@ -0,0 +1,6 @@
+► Deployment/default/podinfo created
+► HorizontalPodAutoscaler/default/podinfo created
+► Service/default/podinfo created
+► Secret/default/docker-secret created
+► Secret/default/podinfo-token-77t89m9b67 created
+► Secret/default/db-user-pass-bkbd782d2c created

cmd/flux/testdata/diff-kustomization/dockerconfigjson-sops-secret.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+data:
+  .dockerconfigjson: eyJtYXNrIjoiKipTT1BTKioifQ==
+kind: Secret
+metadata:
+  labels:
+    kustomize.toolkit.fluxcd.io/name: podinfo
+    kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }}
+  name: docker-secret
+  namespace: default
+type: kubernetes.io/dockerconfigjson

cmd/flux/testdata/diff-kustomization/nothing-is-deployed.golden

@@ -1,5 +1,7 @@
 ► Deployment/default/podinfo created
 ► HorizontalPodAutoscaler/default/podinfo created
 ► Service/default/podinfo created
+► Secret/default/docker-secret created
+► Secret/default/secret-basic-auth-stringdata created
 ► Secret/default/podinfo-token-77t89m9b67 created
 ► Secret/default/db-user-pass-bkbd782d2c created

cmd/flux/testdata/diff-kustomization/stringdata-sops-secret.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    kustomize.toolkit.fluxcd.io/name: podinfo
+    kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }}
+  name: secret-basic-auth-stringdata
+  namespace: default
+stringData:
+  password: KipTT1BTKio=
+  username: KipTT1BTKio=
+type: kubernetes.io/basic-auth

cmd/flux/testdata/export/git-repo.yaml

@@ -11,6 +11,6 @@ spec:
     branch: main
   secretRef:
     name: flux-system
-  timeout: 20s
+  timeout: 1m0s
   url: ssh://git@github.com/example/repo
 

go.mod

@@ -10,23 +10,23 @@ require (
 	github.com/fluxcd/helm-controller/api v0.16.0
 	github.com/fluxcd/image-automation-controller/api v0.20.0
 	github.com/fluxcd/image-reflector-controller/api v0.16.0
-	github.com/fluxcd/kustomize-controller/api v0.20.0
+	github.com/fluxcd/kustomize-controller/api v0.20.2
 	github.com/fluxcd/notification-controller/api v0.21.0
 	github.com/fluxcd/pkg/apis/kustomize v0.3.1 // indirect
 	github.com/fluxcd/pkg/apis/meta v0.10.2
 	github.com/fluxcd/pkg/kustomize v0.0.2
 	github.com/fluxcd/pkg/runtime v0.12.4
-	github.com/fluxcd/pkg/ssa v0.12.0
+	github.com/fluxcd/pkg/ssa v0.13.0
 	github.com/fluxcd/pkg/ssh v0.3.1
 	github.com/fluxcd/pkg/untar v0.0.5
 	github.com/fluxcd/pkg/version v0.0.1
-	github.com/fluxcd/source-controller/api v0.21.1
+	github.com/fluxcd/source-controller/api v0.21.2
 	github.com/go-git/go-git/v5 v5.4.2
 	github.com/gonvenience/bunt v1.3.2
 	github.com/gonvenience/ytbx v1.4.2
 	github.com/google/go-cmp v0.5.6
 	github.com/google/go-containerregistry v0.2.0
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-retryablehttp v0.7.0 // indirect
 	github.com/homeport/dyff v1.4.6
 	github.com/lucasb-eyer/go-colorful v1.2.0
@@ -45,8 +45,8 @@ require (
 	k8s.io/kubectl v0.23.1
 	sigs.k8s.io/cli-utils v0.27.0
 	sigs.k8s.io/controller-runtime v0.11.0
-	sigs.k8s.io/kustomize/api v0.10.1
-	sigs.k8s.io/kustomize/kyaml v0.13.0
+	sigs.k8s.io/kustomize/api v0.11.2
+	sigs.k8s.io/kustomize/kyaml v0.13.3
 	sigs.k8s.io/yaml v1.3.0
 )
 

go.sum

@@ -230,8 +230,8 @@ github.com/fluxcd/image-automation-controller/api v0.20.0 h1:Z+lxqif0KwccsuNOBZq
 github.com/fluxcd/image-automation-controller/api v0.20.0/go.mod h1:XhLYccGUbmJvTTpJ1jAFKZHr2e1GNXy0T85ZBO50mik=
 github.com/fluxcd/image-reflector-controller/api v0.16.0 h1:1O1YdoK7LsJgWLyvfZTSbvQcUQCBcgJ573HA0arlQQY=
 github.com/fluxcd/image-reflector-controller/api v0.16.0/go.mod h1:OIe3mSXc3OwQiNbiQ9vNXWYtNif31hc7WAbZWlFUUnc=
-github.com/fluxcd/kustomize-controller/api v0.20.0 h1:Vw+2qCxeHMv0y1mfiBgVrMfcfFevBMrRfLEdfPTrb40=
-github.com/fluxcd/kustomize-controller/api v0.20.0/go.mod h1:5MdpzJVx8+KiDIRv37zLme992BAOCgE0v1n+NOgs1lo=
+github.com/fluxcd/kustomize-controller/api v0.20.2 h1:zqCvKGsNCL10WMfmjk2Sd526J6gv8ml027DwesFoZsc=
+github.com/fluxcd/kustomize-controller/api v0.20.2/go.mod h1:5MdpzJVx8+KiDIRv37zLme992BAOCgE0v1n+NOgs1lo=
 github.com/fluxcd/notification-controller/api v0.21.0 h1:D5B3TH5YtSww0SyvW1Ru5xWsh0MgHQanC/a1t3CvXq0=
 github.com/fluxcd/notification-controller/api v0.21.0/go.mod h1:gA9/j0kjh7VDuUC2Cubr9twxOdzb/0+ojcE9Lzsc9ug=
 github.com/fluxcd/pkg/apis/acl v0.0.3 h1:Lw0ZHdpnO4G7Zy9KjrzwwBmDZQuy4qEjaU/RvA6k1lc=
@@ -245,8 +245,8 @@ github.com/fluxcd/pkg/kustomize v0.0.2/go.mod h1:AFwnf3OqQmpTCuwCARTGpPRMBf0ZFJN
 github.com/fluxcd/pkg/runtime v0.12.3/go.mod h1:imJ2xYy/d4PbSinX2IefmZk+iS2c1P5fY0js8mCE4SM=
 github.com/fluxcd/pkg/runtime v0.12.4 h1:gA27RG/+adN2/7Qe03PB46Iwmye/MusPCpuS4zQ2fW0=
 github.com/fluxcd/pkg/runtime v0.12.4/go.mod h1:gspNvhAqodZgSmK1ZhMtvARBf/NGAlxmaZaIOHkJYsc=
-github.com/fluxcd/pkg/ssa v0.12.0 h1:7nF4UigU9Zk/9P/nbzUP3ah8IRC+BpB64O9iu5VnvEo=
-github.com/fluxcd/pkg/ssa v0.12.0/go.mod h1:S+qig7BTOxop0c134y8Yv8/iQST4Kt7S2xXiFkP4VMA=
+github.com/fluxcd/pkg/ssa v0.13.0 h1:LU4wf7dB9ksYdda0BEWNTBSTd68E5YwWxuPiPLAtw4Y=
+github.com/fluxcd/pkg/ssa v0.13.0/go.mod h1:XGVGjUaG152HGN6sSUj+VFK/Th5i5rj2XsXSDdlIMNU=
 github.com/fluxcd/pkg/ssh v0.3.1 h1:iQw07bkX2rScactX8WYv+uMDsufFOlg8M3fV2TNs244=
 github.com/fluxcd/pkg/ssh v0.3.1/go.mod h1:Sebfv4Um51PvomuYdMvKRncQW5dtKhZ5J5TA+wiHNSQ=
 github.com/fluxcd/pkg/untar v0.0.5 h1:UGI3Ch1UIEIaqQvMicmImL1s9npQa64DJ/ozqHKB7gk=
@@ -254,8 +254,8 @@ github.com/fluxcd/pkg/untar v0.0.5/go.mod h1:O6V9+rtl8c1mHBafgqFlJN6zkF1HS5SSYn7
 github.com/fluxcd/pkg/version v0.0.1 h1:/8asQoDXSThz3csiwi4Qo8Zb6blAxLXbtxNgeMJ9bCg=
 github.com/fluxcd/pkg/version v0.0.1/go.mod h1:WAF4FEEA9xyhngF8TDxg3UPu5fA1qhEYV8Pmi2Il01Q=
 github.com/fluxcd/source-controller/api v0.21.0/go.mod h1:Ab2qDmAUz6ZCp8UaHYLYzxyFrC1FQqEqjxiROb/Rdiw=
-github.com/fluxcd/source-controller/api v0.21.1 h1:7X39YQHmB1vmIBrHxU+YAqxwtdC9Zh+tdtMKREW3xiQ=
-github.com/fluxcd/source-controller/api v0.21.1/go.mod h1:Ab2qDmAUz6ZCp8UaHYLYzxyFrC1FQqEqjxiROb/Rdiw=
+github.com/fluxcd/source-controller/api v0.21.2 h1:J0S5NN4V8FPLrkSMXIUoUvj1X/RuTpVJSjIRF414wmc=
+github.com/fluxcd/source-controller/api v0.21.2/go.mod h1:Ab2qDmAUz6ZCp8UaHYLYzxyFrC1FQqEqjxiROb/Rdiw=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c=
@@ -1411,16 +1411,18 @@ sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNza
 sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y=
 sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY=
 sigs.k8s.io/kustomize/api v0.8.11/go.mod h1:a77Ls36JdfCWojpUqR6m60pdGY1AYFix4AH83nJtY1g=
-sigs.k8s.io/kustomize/api v0.10.1 h1:KgU7hfYoscuqag84kxtzKdEC3mKMb99DPI3a0eaV1d0=
 sigs.k8s.io/kustomize/api v0.10.1/go.mod h1:2FigT1QN6xKdcnGS2Ppp1uIWrtWN28Ms8A3OZUZhwr8=
+sigs.k8s.io/kustomize/api v0.11.2 h1:6YvCJHFDwsLwAX7zNHBxMZi3k7dGIXI8G9l0saYQI0E=
+sigs.k8s.io/kustomize/api v0.11.2/go.mod h1:GZuhith5YcqxIDe0GnRJNx5xxPTjlwaLTt/e+ChUtJA=
 sigs.k8s.io/kustomize/cmd/config v0.9.13/go.mod h1:7547FLF8W/lTaDf0BDqFTbZxM9zqwEJqCKN9sSR0xSs=
 sigs.k8s.io/kustomize/cmd/config v0.10.2/go.mod h1:K2aW7nXJ0AaT+VA/eO0/dzFLxmpFcTzudmAgDwPY1HQ=
 sigs.k8s.io/kustomize/kustomize/v4 v4.2.0/go.mod h1:MOkR6fmhwG7hEDRXBYELTi5GSFcLwfqwzTRHW3kv5go=
 sigs.k8s.io/kustomize/kustomize/v4 v4.4.1/go.mod h1:qOKJMMz2mBP+vcS7vK+mNz4HBLjaQSWRY22EF6Tb7Io=
 sigs.k8s.io/kustomize/kyaml v0.11.0/go.mod h1:GNMwjim4Ypgp/MueD3zXHLRJEjz7RvtPae0AwlvEMFM=
 sigs.k8s.io/kustomize/kyaml v0.12.0/go.mod h1:FTJxEZ86ScK184NpGSAQcfEqee0nul8oLCK30D47m4E=
-sigs.k8s.io/kustomize/kyaml v0.13.0 h1:9c+ETyNfSrVhxvphs+K2dzT3dh5oVPPEqPOE/cUpScY=
 sigs.k8s.io/kustomize/kyaml v0.13.0/go.mod h1:FTJxEZ86ScK184NpGSAQcfEqee0nul8oLCK30D47m4E=
+sigs.k8s.io/kustomize/kyaml v0.13.3 h1:tNNQIC+8cc+aXFTVg+RtQAOsjwUdYBZRAgYOVI3RBc4=
+sigs.k8s.io/kustomize/kyaml v0.13.3/go.mod h1:/ya3Gk4diiQzlE4mBh7wykyLRFZNvqlbh+JnwQ9Vhrc=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ=
 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=

internal/bootstrap/bootstrap_provider.go

@@ -275,7 +275,7 @@ func (b *GitProviderBootstrapper) reconcileOrgRepository(ctx context.Context) (g
 	subOrgs, repoName := splitSubOrganizationsFromRepositoryName(b.repositoryName)
 	orgRef, err := b.getOrganization(ctx, subOrgs)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create new Git repository for the organization %q: %w", orgRef.String(), err)
+		return nil, fmt.Errorf("failed to create new Git repository %q: %w", b.repositoryName, err)
 	}
 	repoRef := newOrgRepositoryRef(*orgRef, repoName)
 	repoInfo := newRepositoryInfo(b.description, b.defaultBranch, b.visibility)

internal/build/build.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"sync"
 	"time"
@@ -36,12 +37,17 @@ import (
 	"sigs.k8s.io/kustomize/api/resmap"
 	"sigs.k8s.io/kustomize/api/resource"
 	"sigs.k8s.io/kustomize/kyaml/filesys"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
 )
 
 const (
-	controllerName  = "kustomize-controller"
-	controllerGroup = "kustomize.toolkit.fluxcd.io"
-	mask            = "**SOPS**"
+	controllerName      = "kustomize-controller"
+	controllerGroup     = "kustomize.toolkit.fluxcd.io"
+	mask                = "**SOPS**"
+	dockercfgSecretType = "kubernetes.io/dockerconfigjson"
+	typeField           = "type"
+	dataField           = "data"
+	stringDataField     = "stringData"
 )
 
 var defaultTimeout = 80 * time.Second
@@ -182,7 +188,7 @@ func (b *Builder) build() (m resmap.ResMap, err error) {
 		}
 
 		// make sure secrets are masked
-		err = trimSopsData(res)
+		err = maskSopsData(res)
 		if err != nil {
 			return
 		}
@@ -256,26 +262,131 @@ func (b *Builder) setOwnerLabels(res *resource.Resource) error {
 	return nil
 }
 
-func trimSopsData(res *resource.Resource) error {
+func maskSopsData(res *resource.Resource) error {
 	// sopsMess is the base64 encoded mask
 	sopsMess := base64.StdEncoding.EncodeToString([]byte(mask))
 
 	if res.GetKind() == "Secret" {
+		// get both data and stringdata maps as a secret can have both
 		dataMap := res.GetDataMap()
-		for k, v := range dataMap {
-			data, err := base64.StdEncoding.DecodeString(v)
+		stringDataMap := getStringDataMap(res)
+		asYaml, err := res.AsYAML()
+		if err != nil {
+			return fmt.Errorf("failed to mask secret %s sops data: %w", res.GetName(), err)
+		}
+
+		// delete any sops data as we don't want to expose it
+		// assume that both data and stringdata are encrypted
+		if bytes.Contains(asYaml, []byte("sops:")) && bytes.Contains(asYaml, []byte("mac: ENC[")) {
+			// delete the sops object
+			res.PipeE(yaml.FieldClearer{Name: "sops"})
+
+			secretType, err := res.GetFieldValue(typeField)
 			if err != nil {
-				if _, ok := err.(base64.CorruptInputError); ok {
-					return fmt.Errorf("failed to decode secret data: %w", err)
-				}
+				return fmt.Errorf("failed to mask secret %s sops data: %w", res.GetName(), err)
 			}
 
-			if bytes.Contains(data, []byte("sops")) && bytes.Contains(data, []byte("ENC[")) {
-				dataMap[k] = sopsMess
+			if v, ok := secretType.(string); ok && v == dockercfgSecretType {
+				// if the secret is a json docker config secret, we need to mask the data with a json object
+				err := maskDockerconfigjsonSopsData(dataMap)
+				if err != nil {
+					return fmt.Errorf("failed to mask secret %s sops data: %w", res.GetName(), err)
+				}
+
+				err = maskDockerconfigjsonSopsData(stringDataMap)
+				if err != nil {
+					return fmt.Errorf("failed to mask secret %s sops data: %w", res.GetName(), err)
+				}
+
+			} else {
+				for k := range dataMap {
+					dataMap[k] = sopsMess
+				}
+
+				for k := range stringDataMap {
+					stringDataMap[k] = sopsMess
+				}
+			}
+		} else {
+			err := maskBase64EncryptedSopsData(dataMap, sopsMess)
+			if err != nil {
+				return fmt.Errorf("failed to mask secret %s sops data: %w", res.GetName(), err)
+			}
+
+			err = maskSopsDataInStringDataSecret(stringDataMap, sopsMess)
+			if err != nil {
+				return fmt.Errorf("failed to mask secret %s sops data: %w", res.GetName(), err)
 			}
 		}
 
+		// set the data and stringdata maps
 		res.SetDataMap(dataMap)
+
+		if len(stringDataMap) > 0 {
+			err = res.SetMapField(yaml.NewMapRNode(&stringDataMap), stringDataField)
+			if err != nil {
+				return fmt.Errorf("failed to mask secret %s sops data: %w", res.GetName(), err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func getStringDataMap(rn *resource.Resource) map[string]string {
+	n, err := rn.Pipe(yaml.Lookup(stringDataField))
+	if err != nil {
+		return nil
+	}
+	result := map[string]string{}
+	_ = n.VisitFields(func(node *yaml.MapNode) error {
+		result[yaml.GetValue(node.Key)] = yaml.GetValue(node.Value)
+		return nil
+	})
+	return result
+}
+
+func maskDockerconfigjsonSopsData(dataMap map[string]string) error {
+	sopsMess := struct {
+		Mask string `json:"mask"`
+	}{
+		Mask: mask,
+	}
+
+	maskJson, err := json.Marshal(sopsMess)
+	if err != nil {
+		return err
+	}
+
+	for k := range dataMap {
+		dataMap[k] = base64.StdEncoding.EncodeToString(maskJson)
+	}
+
+	return nil
+}
+
+func maskBase64EncryptedSopsData(dataMap map[string]string, mask string) error {
+	for k, v := range dataMap {
+		data, err := base64.StdEncoding.DecodeString(v)
+		if err != nil {
+			if _, ok := err.(base64.CorruptInputError); ok {
+				return err
+			}
+		}
+
+		if bytes.Contains(data, []byte("sops")) && bytes.Contains(data, []byte("ENC[")) {
+			dataMap[k] = mask
+		}
+	}
+
+	return nil
+}
+
+func maskSopsDataInStringDataSecret(stringDataMap map[string]string, mask string) error {
+	for k, v := range stringDataMap {
+		if bytes.Contains([]byte(v), []byte("sops")) && bytes.Contains([]byte(v), []byte("ENC[")) {
+			stringDataMap[k] = mask
+		}
 	}
 
 	return nil

internal/build/build_test.go

@@ -91,6 +91,45 @@ kind: Secret
 metadata:
   name: secret-basic-auth
 type: kubernetes.io/basic-auth
+`,
+		},
+		{
+			name: "secret sops secret",
+			yamlStr: `apiVersion: v1
+data:
+  .dockerconfigjson: ENC[AES256_GCM,data:KHCFH3hNnc+PMfWLFEPjebf3W4z4WXbGFAANRZyZC+07z7wlrTALJM6rn8YslW4tMAWCoAYxblC5WRCszTy0h9rw0U/RGOv5H0qCgnNg/FILFUqhwo9pNfrUH+MEP4M9qxxbLKZwObpHUE7DUsKx1JYAxsI=,iv:q48lqUbUQD+0cbYcjNMZMJLRdGHi78ZmDhNAT2th9tg=,tag:QRI2SZZXQrAcdql3R5AH2g==,type:str]
+kind: Secret
+metadata:
+  name: secret
+type: kubernetes.io/dockerconfigjson
+sops:
+  kms: []
+  gcp_kms: []
+  azure_kv: []
+  hc_vault: []
+  age:
+    - recipient: age10la2ge0wtvx3qr7datqf7rs4yngxszdal927fs9rukamr8u2pshsvtz7ce
+      enc: |
+        -----BEGIN AGE ENCRYPTED FILE-----
+        YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA3eU1CTEJhVXZ4eEVYYkVV
+        OU90TEcrR2pYckttN0pBanJoSUZWSW1RQXlRCkUydFJ3V1NZUTBuVFF0aC9GUEcw
+        bUdhNjJWTkoyL1FUVi9Dc1dxUDBkM0UKLS0tIE1sQXkwcWdGaEFuY0RHQTVXM0J6
+        dWpJcThEbW15V3dXYXpPZklBdW1Hd1kKoIAdmGNPrEctV8h1w8KuvQ5S+BGmgqN9
+        MgpNmUhJjWhgcQpb5BRYpQesBOgU5TBGK7j58A6DMDKlSiYZsdQchQ==
+        -----END AGE ENCRYPTED FILE-----
+  lastmodified: "2022-02-03T16:03:17Z"
+  mac: ENC[AES256_GCM,data:AHdYSawajwgAFwlmDN1IPNmT9vWaYKzyVIra2d6sPcjTbZ8/p+VRSRpVm4XZFFsaNnW5AUJaouwXnKYDTmJDXKlr/rQcu9kXqsssQgdzcXaA6l5uJlgsnml8ba7J3OK+iEKMax23mwQEx2EUskCd9ENOwFDkunP02sxqDNOz20k=,iv:8F5OamHt3fAVorf6p+SoIrWoqkcATSGWVoM0EK87S4M=,tag:E1mxXnc7wWkEX5BxhpLtng==,type:str]
+  pgp: []
+  encrypted_regex: ^(data|stringData)$
+  version: 3.7.1
+`,
+			expected: `apiVersion: v1
+data:
+  .dockerconfigjson: eyJtYXNrIjoiKipTT1BTKioifQ==
+kind: Secret
+metadata:
+  name: secret
+type: kubernetes.io/dockerconfigjson
 `,
 		},
 	}
@@ -103,7 +142,7 @@ type: kubernetes.io/basic-auth
 			}
 
 			resource := &resource.Resource{RNode: *r}
-			err = trimSopsData(resource)
+			err = maskSopsData(resource)
 			if err != nil {
 				t.Fatalf("unable to trim sops data: %v", err)
 			}

internal/build/diff.go

@@ -32,6 +32,7 @@ import (
 	"github.com/gonvenience/bunt"
 	"github.com/gonvenience/ytbx"
 	"github.com/google/go-cmp/cmp"
+	"github.com/hashicorp/go-multierror"
 	"github.com/homeport/dyff/pkg/dyff"
 	"github.com/lucasb-eyer/go-colorful"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -51,30 +52,32 @@ func (b *Builder) Manager() (*ssa.ResourceManager, error) {
 	return ssa.NewResourceManager(b.client, statusPoller, owner), nil
 }
 
-func (b *Builder) Diff() (string, error) {
+func (b *Builder) Diff() (string, bool, error) {
 	output := strings.Builder{}
+	createdOrDrifted := false
 	res, err := b.Build()
 	if err != nil {
-		return "", err
+		return "", createdOrDrifted, err
 	}
 	// convert the build result into Kubernetes unstructured objects
 	objects, err := ssa.ReadObjects(bytes.NewReader(res))
 	if err != nil {
-		return "", err
+		return "", createdOrDrifted, err
 	}
 
 	resourceManager, err := b.Manager()
 	if err != nil {
-		return "", err
+		return "", createdOrDrifted, err
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
 	defer cancel()
 
 	if err := ssa.SetNativeKindsDefaults(objects); err != nil {
-		return "", err
+		return "", createdOrDrifted, err
 	}
 
+	var diffErrs error
 	// create an inventory of objects to be reconciled
 	newInventory := newInventory()
 	for _, obj := range objects {
@@ -85,11 +88,8 @@ func (b *Builder) Diff() (string, error) {
 		}
 		change, liveObject, mergedObject, err := resourceManager.Diff(ctx, obj, diffOptions)
 		if err != nil {
-			if b.kustomization.Spec.Force && ssa.IsImmutableError(err) {
-				output.WriteString(writeString(fmt.Sprintf("► %s created\n", obj.GetName()), bunt.Green))
-			} else {
-				output.WriteString(writeString(fmt.Sprintf("✗ %v\n", err), bunt.Red))
-			}
+			// gather errors and continue, as we want to see all the diffs
+			diffErrs = multierror.Append(diffErrs, err)
 			continue
 		}
 
@@ -101,31 +101,34 @@ func (b *Builder) Diff() (string, error) {
 
 		if change.Action == string(ssa.CreatedAction) {
 			output.WriteString(writeString(fmt.Sprintf("► %s created\n", change.Subject), bunt.Green))
+			createdOrDrifted = true
 		}
 
 		if change.Action == string(ssa.ConfiguredAction) {
 			output.WriteString(writeString(fmt.Sprintf("► %s drifted\n", change.Subject), bunt.WhiteSmoke))
 			liveFile, mergedFile, tmpDir, err := writeYamls(liveObject, mergedObject)
 			if err != nil {
-				return "", err
+				return "", createdOrDrifted, err
 			}
 			defer cleanupDir(tmpDir)
 
 			err = diff(liveFile, mergedFile, &output)
 			if err != nil {
-				return "", err
+				return "", createdOrDrifted, err
 			}
+
+			createdOrDrifted = true
 		}
 
 		addObjectsToInventory(newInventory, change)
 	}
 
-	if b.kustomization.Spec.Prune {
+	if b.kustomization.Spec.Prune && diffErrs == nil {
 		oldStatus := b.kustomization.Status.DeepCopy()
 		if oldStatus.Inventory != nil {
 			diffObjects, err := diffInventory(oldStatus.Inventory, newInventory)
 			if err != nil {
-				return "", err
+				return "", createdOrDrifted, err
 			}
 			for _, object := range diffObjects {
 				output.WriteString(writeString(fmt.Sprintf("► %s deleted\n", ssa.FmtUnstructured(object)), bunt.OrangeRed))
@@ -133,7 +136,7 @@ func (b *Builder) Diff() (string, error) {
 		}
 	}
 
-	return output.String(), nil
+	return output.String(), createdOrDrifted, diffErrs
 }
 
 func writeYamls(liveObject, mergedObject *unstructured.Unstructured) (string, string, string, error) {
@@ -196,17 +199,31 @@ func diff(liveFile, mergedFile string, output io.Writer) error {
 }
 
 func diffSopsSecret(obj, liveObject, mergedObject *unstructured.Unstructured, change *ssa.ChangeSetEntry) {
-	data := obj.Object["data"]
-	for _, v := range data.(map[string]interface{}) {
+	// get both data and stringdata maps
+	data := obj.Object[dataField]
+	stringData := obj.Object[stringDataField]
+
+	if m, ok := data.(map[string]interface{}); ok && m != nil {
+		applySopsDiff(m, liveObject, mergedObject, change)
+	}
+
+	if m, ok := stringData.(map[string]interface{}); ok && m != nil {
+		applySopsDiff(m, liveObject, mergedObject, change)
+	}
+}
+
+func applySopsDiff(data map[string]interface{}, liveObject, mergedObject *unstructured.Unstructured, change *ssa.ChangeSetEntry) {
+	for _, v := range data {
 		v, err := base64.StdEncoding.DecodeString(v.(string))
 		if err != nil {
 			fmt.Println(err)
 		}
+
 		if bytes.Contains(v, []byte(mask)) {
 			if liveObject != nil && mergedObject != nil {
 				change.Action = string(ssa.UnchangedAction)
-				dataLive := liveObject.Object["data"].(map[string]interface{})
-				dataMerged := mergedObject.Object["data"].(map[string]interface{})
+				dataLive := liveObject.Object[dataField].(map[string]interface{})
+				dataMerged := mergedObject.Object[dataField].(map[string]interface{})
 				if cmp.Diff(keys(dataLive), keys(dataMerged)) != "" {
 					change.Action = string(ssa.ConfiguredAction)
 				}

internal/utils/apply.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/fluxcd/pkg/ssa"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -37,31 +38,12 @@ import (
 // Apply is the equivalent of 'kubectl apply --server-side -f'.
 // If the given manifest is a kustomization.yaml, then apply performs the equivalent of 'kubectl apply --server-side -k'.
 func Apply(ctx context.Context, rcg genericclioptions.RESTClientGetter, manifestPath string) (string, error) {
-	cfg, err := KubeConfig(rcg)
-	if err != nil {
-		return "", err
-	}
-	restMapper, err := rcg.ToRESTMapper()
-	if err != nil {
-		return "", err
-	}
-	kubeClient, err := client.New(cfg, client.Options{Mapper: restMapper})
-	if err != nil {
-		return "", err
-	}
-	kubePoller := polling.NewStatusPoller(kubeClient, restMapper, nil)
-
-	resourceManager := ssa.NewResourceManager(kubeClient, kubePoller, ssa.Owner{
-		Field: "flux",
-		Group: "fluxcd.io",
-	})
-
 	objs, err := readObjects(manifestPath)
 	if err != nil {
 		return "", err
 	}
 
-	if len(objs) < 1 {
+	if len(objs) == 0 {
 		return "", fmt.Errorf("no Kubernetes objects found at: %s", manifestPath)
 	}
 
@@ -69,11 +51,42 @@ func Apply(ctx context.Context, rcg genericclioptions.RESTClientGetter, manifest
 		return "", err
 	}
 
-	changeSet, err := resourceManager.ApplyAllStaged(ctx, objs, ssa.DefaultApplyOptions())
-	if err != nil {
+	changeSet := ssa.NewChangeSet()
+
+	// contains only CRDs and Namespaces
+	var stageOne []*unstructured.Unstructured
+
+	// contains all objects except for CRDs and Namespaces
+	var stageTwo []*unstructured.Unstructured
+
+	for _, u := range objs {
+		if ssa.IsClusterDefinition(u) {
+			stageOne = append(stageOne, u)
+		} else {
+			stageTwo = append(stageTwo, u)
+		}
+	}
+
+	if len(stageOne) > 0 {
+		cs, err := applySet(ctx, rcg, stageOne)
+		if err != nil {
+			return "", err
+		}
+		changeSet.Append(cs.Entries)
+	}
+
+	if err := waitForSet(rcg, changeSet); err != nil {
 		return "", err
 	}
 
+	if len(stageTwo) > 0 {
+		cs, err := applySet(ctx, rcg, stageTwo)
+		if err != nil {
+			return "", err
+		}
+		changeSet.Append(cs.Entries)
+	}
+
 	return changeSet.String(), nil
 }
 
@@ -98,3 +111,42 @@ func readObjects(manifestPath string) ([]*unstructured.Unstructured, error) {
 
 	return ssa.ReadObjects(bufio.NewReader(ms))
 }
+
+func newManager(rcg genericclioptions.RESTClientGetter) (*ssa.ResourceManager, error) {
+	cfg, err := KubeConfig(rcg)
+	if err != nil {
+		return nil, err
+	}
+	restMapper, err := rcg.ToRESTMapper()
+	if err != nil {
+		return nil, err
+	}
+	kubeClient, err := client.New(cfg, client.Options{Mapper: restMapper, Scheme: NewScheme()})
+	if err != nil {
+		return nil, err
+	}
+	kubePoller := polling.NewStatusPoller(kubeClient, restMapper, nil)
+
+	return ssa.NewResourceManager(kubeClient, kubePoller, ssa.Owner{
+		Field: "flux",
+		Group: "fluxcd.io",
+	}), nil
+
+}
+
+func applySet(ctx context.Context, rcg genericclioptions.RESTClientGetter, objects []*unstructured.Unstructured) (*ssa.ChangeSet, error) {
+	man, err := newManager(rcg)
+	if err != nil {
+		return nil, err
+	}
+
+	return man.ApplyAll(ctx, objects, ssa.DefaultApplyOptions())
+}
+
+func waitForSet(rcg genericclioptions.RESTClientGetter, changeSet *ssa.ChangeSet) error {
+	man, err := newManager(rcg)
+	if err != nil {
+		return err
+	}
+	return man.WaitForSet(changeSet.ToObjMetadataSet(), ssa.WaitOptions{Interval: 2 * time.Second, Timeout: time.Minute})
+}

manifests/bases/kustomize-controller/kustomization.yaml

@@ -1,8 +1,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- https://github.com/fluxcd/kustomize-controller/releases/download/v0.20.0/kustomize-controller.crds.yaml
-- https://github.com/fluxcd/kustomize-controller/releases/download/v0.20.0/kustomize-controller.deployment.yaml
+- https://github.com/fluxcd/kustomize-controller/releases/download/v0.20.2/kustomize-controller.crds.yaml
+- https://github.com/fluxcd/kustomize-controller/releases/download/v0.20.2/kustomize-controller.deployment.yaml
 - account.yaml
 patchesJson6902:
 - target:

manifests/bases/source-controller/kustomization.yaml

@@ -1,8 +1,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- https://github.com/fluxcd/source-controller/releases/download/v0.21.1/source-controller.crds.yaml
-- https://github.com/fluxcd/source-controller/releases/download/v0.21.1/source-controller.deployment.yaml
+- https://github.com/fluxcd/source-controller/releases/download/v0.21.2/source-controller.crds.yaml
+- https://github.com/fluxcd/source-controller/releases/download/v0.21.2/source-controller.deployment.yaml
 - account.yaml
 patchesJson6902:
 - target:

manifests/crds/kustomization.yaml

@@ -1,8 +1,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- https://github.com/fluxcd/source-controller/releases/download/v0.21.1/source-controller.crds.yaml
-- https://github.com/fluxcd/kustomize-controller/releases/download/v0.20.0/kustomize-controller.crds.yaml
+- https://github.com/fluxcd/source-controller/releases/download/v0.21.2/source-controller.crds.yaml
+- https://github.com/fluxcd/kustomize-controller/releases/download/v0.20.2/kustomize-controller.crds.yaml
 - https://github.com/fluxcd/helm-controller/releases/download/v0.16.0/helm-controller.crds.yaml
 - https://github.com/fluxcd/notification-controller/releases/download/v0.21.0/notification-controller.crds.yaml
 - https://github.com/fluxcd/image-reflector-controller/releases/download/v0.16.0/image-reflector-controller.crds.yaml

tests/azure/azure_test.go

@@ -483,7 +483,7 @@ func TestImageRepositoryACR(t *testing.T) {
 			Interval: metav1.Duration{
 				Duration: 1 * time.Minute,
 			},
-			SourceRef: automationv1beta1.SourceReference{
+			SourceRef: automationv1beta1.CrossNamespaceSourceReference{
 				Kind: "GitRepository",
 				Name: name,
 			},

tests/azure/go.mod

@@ -7,11 +7,11 @@ require (
 	github.com/fluxcd/helm-controller/api v0.16.0
 	github.com/fluxcd/image-automation-controller/api v0.20.0
 	github.com/fluxcd/image-reflector-controller/api v0.16.0
-	github.com/fluxcd/kustomize-controller/api v0.20.0
+	github.com/fluxcd/kustomize-controller/api v0.20.2
 	github.com/fluxcd/notification-controller/api v0.21.0
 	github.com/fluxcd/pkg/apis/meta v0.10.2
 	github.com/fluxcd/pkg/runtime v0.12.4
-	github.com/fluxcd/source-controller/api v0.21.1
+	github.com/fluxcd/source-controller/api v0.21.2
 	github.com/hashicorp/terraform-exec v0.14.0
 	github.com/libgit2/git2go/v31 v31.6.1
 	github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5

tests/azure/go.sum

@@ -204,8 +204,8 @@ github.com/fluxcd/image-automation-controller/api v0.20.0 h1:Z+lxqif0KwccsuNOBZq
 github.com/fluxcd/image-automation-controller/api v0.20.0/go.mod h1:XhLYccGUbmJvTTpJ1jAFKZHr2e1GNXy0T85ZBO50mik=
 github.com/fluxcd/image-reflector-controller/api v0.16.0 h1:1O1YdoK7LsJgWLyvfZTSbvQcUQCBcgJ573HA0arlQQY=
 github.com/fluxcd/image-reflector-controller/api v0.16.0/go.mod h1:OIe3mSXc3OwQiNbiQ9vNXWYtNif31hc7WAbZWlFUUnc=
-github.com/fluxcd/kustomize-controller/api v0.20.0 h1:Vw+2qCxeHMv0y1mfiBgVrMfcfFevBMrRfLEdfPTrb40=
-github.com/fluxcd/kustomize-controller/api v0.20.0/go.mod h1:5MdpzJVx8+KiDIRv37zLme992BAOCgE0v1n+NOgs1lo=
+github.com/fluxcd/kustomize-controller/api v0.20.2 h1:zqCvKGsNCL10WMfmjk2Sd526J6gv8ml027DwesFoZsc=
+github.com/fluxcd/kustomize-controller/api v0.20.2/go.mod h1:5MdpzJVx8+KiDIRv37zLme992BAOCgE0v1n+NOgs1lo=
 github.com/fluxcd/notification-controller/api v0.21.0 h1:D5B3TH5YtSww0SyvW1Ru5xWsh0MgHQanC/a1t3CvXq0=
 github.com/fluxcd/notification-controller/api v0.21.0/go.mod h1:gA9/j0kjh7VDuUC2Cubr9twxOdzb/0+ojcE9Lzsc9ug=
 github.com/fluxcd/pkg/apis/acl v0.0.3 h1:Lw0ZHdpnO4G7Zy9KjrzwwBmDZQuy4qEjaU/RvA6k1lc=
@@ -218,8 +218,8 @@ github.com/fluxcd/pkg/runtime v0.12.3/go.mod h1:imJ2xYy/d4PbSinX2IefmZk+iS2c1P5f
 github.com/fluxcd/pkg/runtime v0.12.4 h1:gA27RG/+adN2/7Qe03PB46Iwmye/MusPCpuS4zQ2fW0=
 github.com/fluxcd/pkg/runtime v0.12.4/go.mod h1:gspNvhAqodZgSmK1ZhMtvARBf/NGAlxmaZaIOHkJYsc=
 github.com/fluxcd/source-controller/api v0.21.0/go.mod h1:Ab2qDmAUz6ZCp8UaHYLYzxyFrC1FQqEqjxiROb/Rdiw=
-github.com/fluxcd/source-controller/api v0.21.1 h1:7X39YQHmB1vmIBrHxU+YAqxwtdC9Zh+tdtMKREW3xiQ=
-github.com/fluxcd/source-controller/api v0.21.1/go.mod h1:Ab2qDmAUz6ZCp8UaHYLYzxyFrC1FQqEqjxiROb/Rdiw=
+github.com/fluxcd/source-controller/api v0.21.2 h1:J0S5NN4V8FPLrkSMXIUoUvj1X/RuTpVJSjIRF414wmc=
+github.com/fluxcd/source-controller/api v0.21.2/go.mod h1:Ab2qDmAUz6ZCp8UaHYLYzxyFrC1FQqEqjxiROb/Rdiw=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c=