diff --git a/internal/build/build.go b/internal/build/build.go
index 17cfb65e..9a59c4bb 100644
--- a/internal/build/build.go
+++ b/internal/build/build.go
@@ -659,17 +659,10 @@ func kustomizationsEqual(k1 *kustomizev1.Kustomization, k2 *kustomizev1.Kustomiz
 }
 
 func (b *Builder) setOwnerLabels(res *resource.Resource) error {
-	labels := res.GetLabels()
-
-	labels[controllerGroup+"/name"] = b.kustomization.GetName()
-	labels[controllerGroup+"/namespace"] = b.kustomization.GetNamespace()
-
-	err := res.SetLabels(labels)
-	if err != nil {
+	if err := res.PipeE(yaml.SetLabel(controllerGroup+"/name", b.kustomization.GetName())); err != nil {
 		return err
 	}
-
-	return nil
+	return res.PipeE(yaml.SetLabel(controllerGroup+"/namespace", b.kustomization.GetNamespace()))
 }
 
 func maskSopsData(res *resource.Resource) error {
diff --git a/internal/build/build_test.go b/internal/build/build_test.go
index fa7ffff3..800bb1f4 100644
--- a/internal/build/build_test.go
+++ b/internal/build/build_test.go
@@ -840,3 +840,38 @@ resources:
 		t.Fatal("expected error when referencing resource outside cwd, got nil")
 	}
 }
+
+func Test_Build_preserveAllLabels(t *testing.T) {
+	b, err := NewBuilder("test-ks", "testdata/invalid-labels/resources",
+		WithDryRun(true),
+		WithNamespace("flux-system"),
+		WithKustomizationFile("testdata/invalid-labels/kustomization.yaml"),
+	)
+	if err != nil {
+		t.Fatalf("unable to create builder: %v", err)
+	}
+
+	objects, err := b.Build()
+	if err != nil {
+		t.Fatalf("build failed: %v", err)
+	}
+
+	if len(objects) == 0 {
+		t.Fatal("expected at least one object, got none")
+	}
+
+	rawLabels, _, _ := unstructured.NestedMap(objects[0].Object, "metadata", "labels")
+
+	if got, ok := rawLabels["invalid"]; !ok || got != int64(0) {
+		t.Errorf("expected label invalid=0, got %#v", got)
+	}
+	if got, ok := rawLabels["valid"]; !ok || got != "yes" {
+		t.Errorf("expected label valid=\"yes\", got %v", got)
+	}
+	if got := rawLabels[controllerGroup+"/name"]; got != "test-ks" {
+		t.Errorf("expected owner name label \"test-ks\", got %v", got)
+	}
+	if got := rawLabels[controllerGroup+"/namespace"]; got != "flux-system" {
+		t.Errorf("expected owner namespace label \"flux-system\", got %v", got)
+	}
+}
diff --git a/internal/build/testdata/invalid-labels/kustomization.yaml b/internal/build/testdata/invalid-labels/kustomization.yaml
new file mode 100644
index 00000000..2d4f52a6
--- /dev/null
+++ b/internal/build/testdata/invalid-labels/kustomization.yaml
@@ -0,0 +1,7 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: test-ks
+  namespace: flux-system
+spec:
+  path: "./resources"
diff --git a/internal/build/testdata/invalid-labels/resources/invalid-resource.yaml b/internal/build/testdata/invalid-labels/resources/invalid-resource.yaml
new file mode 100644
index 00000000..d0b19746
--- /dev/null
+++ b/internal/build/testdata/invalid-labels/resources/invalid-resource.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: invalid-resource
+  labels:
+    invalid: 0
+    valid: yes
diff --git a/internal/build/testdata/invalid-labels/resources/kustomization.yaml b/internal/build/testdata/invalid-labels/resources/kustomization.yaml
new file mode 100644
index 00000000..2024cb82
--- /dev/null
+++ b/internal/build/testdata/invalid-labels/resources/kustomization.yaml
@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - invalid-resource.yaml