monitoring: Configure KSM & cluster dashboard

Update kube-prometheus-stack helm release values to configure
kube-state-metrics and use kube-state-metrics to collect gotk resource
state metrics.

- Configure kube-state-metrics to run in custom resource state only
  mode. In this mode, it'll only watch custom resources. Also, pass
  empty collectors as extra args to prevent passing all the core
  resources to watch as an argument.
- Running kube-state-metrics in custom resource state only mode makes
  the default grafana dashboards of no use. Disable the default
  dashboards.
- Add kube-state-metrics configuration to provide RBAC permissions to it
  to allow listing and watching flux CRDs.
- Also, configure custom resource state for each of the flux custom
  resources using Info type metrics called `gotk_resource_info`. KSM
  issues a warning if an Info type object doesn't have `_info` suffix.
  These metrics have the value 1 always. This works well for the CRD
  state metrics as a zero value would mean that the resource doesn't
  exist, in which case, the resource is deleted.
- Update the cluster dashboard panels to use `gotk_resource_info` in the
  queries.
  - Only the following panels have been updated
    - Cluster Reconcilers
    - Failing Reconcilers
    - Cluster reconciliation readiness
    - Kubernetes Manifests Sources
    - Failing Sources
    - Source acquisition readiness
  - The panels have been updated such that it's work with static
  resources which don't have any status as well. By default, it assumes
  such static resources to be in a Ready state. Resources are seen as
  failed only when the ready value is false.
  - The queries have been updated to Instant type in order to show the
  current data, instead of the result of past 15 minutes. This shows
  more accurate resource data as the resource metrics change.
  - The Stat visualizers have been updated to have zero as the default
  value when there's no data. This is to prevent showing no data when
  there's no object. This was motivated by the behavior of the previous
  configuration which depended on stale metrics from controllers and
  deleted conditions to show zero value when objects get deleted. With
  the fixes in the controller metrics that removes stale metrics, this
  will no longer work. In order to show a zero value for these stats, a
  default is set.
  - The `$namespace` variable has been updated to refer to
  `exported_namespace` from `gotk_resource_info`.

Signed-off-by: Sunny <darkowlzz@protonmail.com>

cmd/flux/check.go

@@ -57,7 +57,7 @@ type checkFlags struct {
 }
 
 var kubernetesConstraints = []string{
-	">=1.25.0-0",
+	">=1.24.0-0",
 }
 
 var checkArgs checkFlags

cmd/flux/create_image_repository.go

@@ -101,7 +101,7 @@ func createImageRepositoryRun(cmd *cobra.Command, args []string) error {
 	var repo = imagev1.ImageRepository{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      objectName,
-			Namespace: GetDesiredNamespace(kubeconfigArgs),
+			Namespace: *kubeconfigArgs.Namespace,
 			Labels:    labels,
 		},
 		Spec: imagev1.ImageRepositorySpec{

cmd/flux/diff_kustomization_test.go

@@ -91,12 +91,6 @@ func TestDiffKustomization(t *testing.T) {
 			objectFile: "./testdata/diff-kustomization/stringdata-sops-secret.yaml",
 			assert:     assertGoldenFile("./testdata/diff-kustomization/diff-with-drifted-stringdata-sops-secret.golden"),
 		},
-		{
-			name:       "diff where kustomization file has multiple objects with the same name",
-			args:       "diff kustomization podinfo --path ./testdata/build-kustomization/podinfo --progress-bar=false --kustomization-file ./testdata/diff-kustomization/flux-kustomization-multiobj.yaml",
-			objectFile: "",
-			assert:     assertGoldenFile("./testdata/diff-kustomization/nothing-is-deployed.golden"),
-		},
 	}
 
 	tmpl := map[string]string{

cmd/flux/get.go

@@ -146,11 +146,9 @@ func (get getCommand) run(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	ns := GetDesiredNamespace(kubeconfigArgs)
-
 	var listOpts []client.ListOption
 	if !getArgs.allNamespaces {
-		listOpts = append(listOpts, client.InNamespace(ns))
+		listOpts = append(listOpts, client.InNamespace(*kubeconfigArgs.Namespace))
 	}
 
 	if len(args) > 0 {
@@ -192,12 +190,12 @@ func (get getCommand) run(cmd *cobra.Command, args []string) error {
 			logger.Failuref("%s object '%s' not found in %s namespace",
 				get.kind,
 				args[0],
-				namespaceNameOrAny(getArgs.allNamespaces, ns),
+				namespaceNameOrAny(getArgs.allNamespaces, *kubeconfigArgs.Namespace),
 			)
 		} else if !getAll {
 			logger.Failuref("no %s objects found in %s namespace",
 				get.kind,
-				namespaceNameOrAny(getArgs.allNamespaces, ns),
+				namespaceNameOrAny(getArgs.allNamespaces, *kubeconfigArgs.Namespace),
 			)
 		}
 		return nil

cmd/flux/install.go

@@ -146,7 +146,7 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
 	opts := install.Options{
 		BaseURL:                installArgs.manifestsPath,
 		Version:                installArgs.version,
-		Namespace:              GetDesiredNamespace(kubeconfigArgs),
+		Namespace:              *kubeconfigArgs.Namespace,
 		Components:             components,
 		Registry:               installArgs.registry,
 		ImagePullSecret:        installArgs.imagePullSecret,
@@ -181,7 +181,7 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
 	}
 
 	logger.Successf("manifests build completed")
-	logger.Actionf("installing components in %s namespace", opts.Namespace)
+	logger.Actionf("installing components in %s namespace", *kubeconfigArgs.Namespace)
 
 	applyOutput, err := utils.Apply(ctx, kubeconfigArgs, kubeclientOptions, tmpDir, filepath.Join(tmpDir, manifest.Path))
 	if err != nil {

cmd/flux/main.go

@@ -105,10 +105,6 @@ Command line utility for assembling Kubernetes CD pipelines the GitOps way.`,
 			return fmt.Errorf("error getting namespace: %w", err)
 		}
 
-		if ns == "" {
-			return nil
-		}
-
 		if e := validation.IsDNS1123Label(ns); len(e) > 0 {
 			return fmt.Errorf("namespace must be a valid DNS label: %q", ns)
 		}
@@ -144,6 +140,7 @@ func init() {
 	rootCmd.PersistentFlags().DurationVar(&rootArgs.timeout, "timeout", 5*time.Minute, "timeout for this operation")
 	rootCmd.PersistentFlags().BoolVar(&rootArgs.verbose, "verbose", false, "print generated objects")
 
+	configureDefaultNamespace()
 	kubeconfigArgs.APIServer = nil // prevent AddFlags from configuring --server flag
 	kubeconfigArgs.Timeout = nil   // prevent AddFlags from configuring --request-timeout flag, we have --timeout instead
 	kubeconfigArgs.AddFlags(rootCmd.PersistentFlags())
@@ -201,10 +198,8 @@ func main() {
 	}
 }
 
-func GetDesiredNamespace(cfg *genericclioptions.ConfigFlags) string {
-	if *cfg.Namespace != "" {
-		return *cfg.Namespace
-	}
+func configureDefaultNamespace() {
+	*kubeconfigArgs.Namespace = rootArgs.defaults.Namespace
 	fromEnv := os.Getenv("FLUX_SYSTEM_NAMESPACE")
 	if fromEnv != "" {
 		// namespace must be a valid DNS label. Assess against validation
@@ -212,28 +207,11 @@ func GetDesiredNamespace(cfg *genericclioptions.ConfigFlags) string {
 		// may not be actively provided by end-user.
 		if e := validation.IsDNS1123Label(fromEnv); len(e) > 0 {
 			logger.Warningf(" ignoring invalid FLUX_SYSTEM_NAMESPACE: %q", fromEnv)
-		} else {
-			return fromEnv
+			return
 		}
-	}
 
-	if _, has := os.LookupEnv("FLUX_NS_FOLLOW_KUBECONTEXT"); has {
-		rawCfg, err := cfg.ToRawKubeConfigLoader().RawConfig()
-		if err != nil {
-			logger.Warningf(" failed parsing kubeconfig, ignoring: %q", fromEnv)
-		} else {
-			ctx := *cfg.Context
-			if ctx == "" {
-				ctx = rawCfg.CurrentContext
-			}
-			ns := rawCfg.Contexts[ctx].Namespace
-			if ns != "" {
-				return ns
-			}
-		}
+		kubeconfigArgs.Namespace = &fromEnv
 	}
-
-	return rootArgs.defaults.Namespace
 }
 
 // readPasswordFromStdin reads a password from stdin and returns the input

cmd/flux/status.go

@@ -85,7 +85,7 @@ func buildComponentObjectRefs(components ...string) ([]object.ObjMetadata, error
 	var objRefs []object.ObjMetadata
 	for _, deployment := range components {
 		objRefs = append(objRefs, object.ObjMetadata{
-			Namespace: GetDesiredNamespace(kubeconfigArgs),
+			Namespace: *kubeconfigArgs.Namespace,
 			Name:      deployment,
 			GroupKind: schema.GroupKind{Group: "apps", Kind: "Deployment"},
 		})

cmd/flux/testdata/check/check_pre.golden

@@ -1,3 +1,3 @@
 ► checking prerequisites
-✔ Kubernetes {{ .serverVersion }} >=1.25.0-0
+✔ Kubernetes {{ .serverVersion }} >=1.24.0-0
 ✔ prerequisites checks passed

cmd/flux/testdata/diff-kustomization/flux-kustomization-multiobj.yaml

@@ -1,19 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: podinfo
-
----
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: podinfo
-spec:
-  interval: 5m0s
-  path: ./kustomize
-  force: true
-  prune: true
-  sourceRef:
-    kind: GitRepository
-    name: podinfo
-  targetNamespace: default

internal/build/build.go

@@ -333,7 +333,7 @@ func (b *Builder) unMarshallKustomization() (*kustomizev1.Kustomization, error)
 	k := &kustomizev1.Kustomization{}
 	decoder := k8syaml.NewYAMLOrJSONDecoder(bytes.NewBuffer(data), len(data))
 	// check for kustomization in yaml with the same name and namespace
-	for {
+	for !(k.Name == b.name && (k.Namespace == b.namespace || k.Namespace == "")) {
 		err = decoder.Decode(k)
 		if err != nil {
 			if err == io.EOF {
@@ -343,13 +343,6 @@ func (b *Builder) unMarshallKustomization() (*kustomizev1.Kustomization, error)
 				return nil, fmt.Errorf("failed to unmarshall kustomization file %s: %w", b.kustomizationFile, err)
 			}
 		}
-
-		if strings.HasPrefix(k.APIVersion, kustomizev1.GroupVersion.Group+"/") &&
-			k.Kind == kustomizev1.KustomizationKind &&
-			k.Name == b.name &&
-			(k.Namespace == b.namespace || k.Namespace == "") {
-			break
-		}
 	}
 	return k, nil
 }

internal/build/build_test.go

@@ -189,12 +189,6 @@ func Test_unMarshallKustomization(t *testing.T) {
 			wantErr:     true,
 			errString:   "failed find kustomization with name",
 		},
-		{
-			name:        "yaml containing other resource with same name as kustomization",
-			localKsFile: "testdata/local-kustomization/invalid-resource.yaml",
-			wantErr:     true,
-			errString:   "failed find kustomization with name",
-		},
 	}
 
 	b := &Builder{
@@ -330,10 +324,7 @@ func Test_ResolveKustomization(t *testing.T) {
 		},
 	}
 
-	b := &Builder{
-		name:      "podinfo",
-		namespace: "flux-system",
-	}
+	b := &Builder{}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			b.kustomizationFile = tt.localKsFile

internal/build/testdata/local-kustomization/invalid-resource.yaml

@@ -1,4 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: podinfo

manifests/monitoring/kube-prometheus-stack/release.yaml

@@ -6,7 +6,7 @@ spec:
   interval: 5m
   chart:
     spec:
-      version: "45.x"
+      version: "48.x"
       chart: kube-prometheus-stack
       sourceRef:
         kind: HelmRepository
@@ -31,6 +31,249 @@ spec:
         podMonitorSelector:
           matchLabels:
             app.kubernetes.io/component: monitoring
+    grafana:
+      defaultDashboardsEnabled: false
+    kube-state-metrics:
+      collectors: []
+      extraArgs:
+        - --custom-resource-state-only=true
+      rbac:
+        extraRules:
+          - apiGroups:
+              - source.toolkit.fluxcd.io
+              - kustomize.toolkit.fluxcd.io
+              - helm.toolkit.fluxcd.io
+              - image.toolkit.fluxcd.io
+              - notification.toolkit.fluxcd.io
+            resources:
+              - gitrepositories
+              - buckets
+              - helmrepositories
+              - helmcharts
+              - ocirepositories
+              - kustomizations
+              - helmreleases
+              - imagerepositories
+              - imagepolicies
+              - imageupdateautomations
+              - alerts
+              - providers
+              - receivers
+            verbs: ["list", "watch"]
+      customResourceState:
+        enabled: true
+        config:
+          spec:
+            resources:
+              - groupVersionKind:
+                  group: source.toolkit.fluxcd.io
+                  version: "v1"
+                  kind: GitRepository
+                metricNamePrefix: gotk
+                metrics:
+                  - name: "resource_info"
+                    help: "The current state of a GitOps Toolkit resource."
+                    each:
+                      type: Info
+                      info:
+                        labelsFromPath:
+                          name: [metadata, name]
+                    labelsFromPath:
+                      exported_namespace: [metadata, namespace]
+                      ready: [status, conditions, "[type=Ready]", status]
+              - groupVersionKind:
+                  group: source.toolkit.fluxcd.io
+                  version: "v1beta2"
+                  kind: Bucket
+                metricNamePrefix: gotk
+                metrics:
+                  - name: "resource_info"
+                    help: "The current state of a GitOps Toolkit resource."
+                    each:
+                      type: Info
+                      info:
+                        labelsFromPath:
+                          name: [metadata, name]
+                    labelsFromPath:
+                      exported_namespace: [metadata, namespace]
+                      ready: [status, conditions, "[type=Ready]", status]
+              - groupVersionKind:
+                  group: source.toolkit.fluxcd.io
+                  version: "v1beta2"
+                  kind: HelmRepository
+                metricNamePrefix: gotk
+                metrics:
+                  - name: "resource_info"
+                    help: "The current state of a GitOps Toolkit resource."
+                    each:
+                      type: Info
+                      info:
+                        labelsFromPath:
+                          name: [metadata, name]
+                    labelsFromPath:
+                      exported_namespace: [metadata, namespace]
+                      type: [spec, type]
+                      ready: [status, conditions, "[type=Ready]", status]
+              - groupVersionKind:
+                  group: source.toolkit.fluxcd.io
+                  version: "v1beta2"
+                  kind: HelmChart
+                metricNamePrefix: gotk
+                metrics:
+                  - name: "resource_info"
+                    help: "The current state of a GitOps Toolkit resource."
+                    each:
+                      type: Info
+                      info:
+                        labelsFromPath:
+                          name: [metadata, name]
+                    labelsFromPath:
+                      exported_namespace: [metadata, namespace]
+                      ready: [status, conditions, "[type=Ready]", status]
+              - groupVersionKind:
+                  group: source.toolkit.fluxcd.io
+                  version: "v1beta2"
+                  kind: OCIRepository
+                metricNamePrefix: gotk
+                metrics:
+                  - name: "resource_info"
+                    help: "The current state of a GitOps Toolkit resource."
+                    each:
+                      type: Info
+                      info:
+                        labelsFromPath:
+                          name: [metadata, name]
+                    labelsFromPath:
+                      exported_namespace: [metadata, namespace]
+                      ready: [status, conditions, "[type=Ready]", status]
+              - groupVersionKind:
+                  group: kustomize.toolkit.fluxcd.io
+                  version: "v1"
+                  kind: Kustomization
+                metricNamePrefix: gotk
+                metrics:
+                  - name: "resource_info"
+                    help: "The current state of a GitOps Toolkit resource."
+                    each:
+                      type: Info
+                      info:
+                        labelsFromPath:
+                          name: [metadata, name]
+                    labelsFromPath:
+                      exported_namespace: [metadata, namespace]
+                      ready: [status, conditions, "[type=Ready]", status]
+              - groupVersionKind:
+                  group: helm.toolkit.fluxcd.io
+                  version: "v2beta1"
+                  kind: HelmRelease
+                metricNamePrefix: gotk
+                metrics:
+                  - name: "resource_info"
+                    help: "The current state of a GitOps Toolkit resource."
+                    each:
+                      type: Info
+                      info:
+                        labelsFromPath:
+                          name: [metadata, name]
+                    labelsFromPath:
+                      exported_namespace: [metadata, namespace]
+                      ready: [status, conditions, "[type=Ready]", status]
+              - groupVersionKind:
+                  group: image.toolkit.fluxcd.io
+                  version: "v1beta2"
+                  kind: ImageRepository
+                metricNamePrefix: gotk
+                metrics:
+                  - name: "resource_info"
+                    help: "The current state of a GitOps Toolkit resource."
+                    each:
+                      type: Info
+                      info:
+                        labelsFromPath:
+                          name: [metadata, name]
+                    labelsFromPath:
+                      exported_namespace: [metadata, namespace]
+                      ready: [status, conditions, "[type=Ready]", status]
+              - groupVersionKind:
+                  group: image.toolkit.fluxcd.io
+                  version: "v1beta2"
+                  kind: ImagePolicy
+                metricNamePrefix: gotk
+                metrics:
+                  - name: "resource_info"
+                    help: "The current state of a GitOps Toolkit resource."
+                    each:
+                      type: Info
+                      info:
+                        labelsFromPath:
+                          name: [metadata, name]
+                    labelsFromPath:
+                      exported_namespace: [metadata, namespace]
+                      ready: [status, conditions, "[type=Ready]", status]
+              - groupVersionKind:
+                  group: image.toolkit.fluxcd.io
+                  version: "v1beta1"
+                  kind: ImageUpdateAutomation
+                metricNamePrefix: gotk
+                metrics:
+                  - name: "resource_info"
+                    help: "The current state of a GitOps Toolkit resource."
+                    each:
+                      type: Info
+                      info:
+                        labelsFromPath:
+                          name: [metadata, name]
+                    labelsFromPath:
+                      exported_namespace: [metadata, namespace]
+                      ready: [status, conditions, "[type=Ready]", status]
+              - groupVersionKind:
+                  group: notification.toolkit.fluxcd.io
+                  version: "v1beta2"
+                  kind: Alert
+                metricNamePrefix: gotk
+                metrics:
+                  - name: "resource_info"
+                    help: "The current state of a GitOps Toolkit resource."
+                    each:
+                      type: Info
+                      info:
+                        labelsFromPath:
+                          name: [metadata, name]
+                    labelsFromPath:
+                      exported_namespace: [metadata, namespace]
+                      ready: [status, conditions, "[type=Ready]", status]
+              - groupVersionKind:
+                  group: notification.toolkit.fluxcd.io
+                  version: "v1beta2"
+                  kind: Provider
+                metricNamePrefix: gotk
+                metrics:
+                  - name: "resource_info"
+                    help: "The current state of a GitOps Toolkit resource."
+                    each:
+                      type: Info
+                      info:
+                        labelsFromPath:
+                          name: [metadata, name]
+                    labelsFromPath:
+                      exported_namespace: [metadata, namespace]
+                      ready: [status, conditions, "[type=Ready]", status]
+              - groupVersionKind:
+                  group: notification.toolkit.fluxcd.io
+                  version: "v1"
+                  kind: Receiver
+                metricNamePrefix: gotk
+                metrics:
+                  - name: "resource_info"
+                    help: "The current state of a GitOps Toolkit resource."
+                    each:
+                      type: Info
+                      info:
+                        labelsFromPath:
+                          name: [metadata, name]
+                    labelsFromPath:
+                      exported_namespace: [metadata, namespace]
+                      ready: [status, conditions, "[type=Ready]", status]
   postRenderers:
     - kustomize:
         patches:

manifests/monitoring/monitoring-config/dashboards/cluster.json

@@ -30,18 +30,23 @@
     ]
   },
   "editable": true,
-  "gnetId": null,
+  "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "iteration": 1652337714814,
+  "id": 5,
   "links": [],
+  "liveNow": false,
   "panels": [
     {
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
       "description": "",
       "fieldConfig": {
         "defaults": {
           "decimals": 0,
           "mappings": [],
+          "noValue": "0",
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -81,28 +86,37 @@
         "text": {},
         "textMode": "value"
       },
-      "pluginVersion": "7.5.5",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
-          "exemplar": true,
-          "expr": "count(gotk_reconcile_condition{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",type=\"Ready\",status=\"True\",kind=~\"Kustomization|HelmRelease\"})\n-\nsum(gotk_reconcile_condition{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",type=\"Ready\",status=\"Deleted\",kind=~\"Kustomization|HelmRelease\"})",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"Kustomization|HelmRelease\"})",
+          "instant": true,
           "interval": "",
           "legendFormat": "",
+          "range": false,
           "refId": "A"
         }
       ],
-      "timeFrom": null,
-      "timeShift": null,
       "title": "Cluster Reconcilers",
       "type": "stat"
     },
     {
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
       "description": "",
       "fieldConfig": {
         "defaults": {
           "decimals": 0,
           "mappings": [],
+          "noValue": "0",
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -138,28 +152,37 @@
         "text": {},
         "textMode": "value"
       },
-      "pluginVersion": "7.5.5",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
-          "exemplar": true,
-          "expr": "sum(gotk_reconcile_condition{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",type=\"Ready\",status=\"False\",kind=~\"Kustomization|HelmRelease\"})",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"Kustomization|HelmRelease\", ready=\"False\"})",
+          "instant": true,
           "interval": "",
           "legendFormat": "",
+          "range": false,
           "refId": "A"
         }
       ],
-      "timeFrom": null,
-      "timeShift": null,
       "title": "Failing Reconcilers",
       "type": "stat"
     },
     {
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
       "description": "",
       "fieldConfig": {
         "defaults": {
           "decimals": 0,
           "mappings": [],
+          "noValue": "0",
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -199,28 +222,37 @@
         "text": {},
         "textMode": "value"
       },
-      "pluginVersion": "7.5.5",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
-          "exemplar": true,
-          "expr": "count(gotk_reconcile_condition{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",type=\"Ready\",status=\"True\",kind=~\"GitRepository|HelmRepository|Bucket\"})\n-\nsum(gotk_reconcile_condition{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",type=\"Ready\",status=\"Deleted\",kind=~\"GitRepository|HelmRepository|Bucket\"})",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"})",
+          "instant": true,
           "interval": "",
           "legendFormat": "",
+          "range": false,
           "refId": "A"
         }
       ],
-      "timeFrom": null,
-      "timeShift": null,
       "title": "Kubernetes Manifests Sources",
       "type": "stat"
     },
     {
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
       "description": "",
       "fieldConfig": {
         "defaults": {
           "decimals": 0,
           "mappings": [],
+          "noValue": "0",
           "thresholds": {
             "mode": "absolute",
             "steps": [
@@ -256,18 +288,23 @@
         "text": {},
         "textMode": "value"
       },
-      "pluginVersion": "7.5.5",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
-          "exemplar": true,
-          "expr": "sum(gotk_reconcile_condition{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",type=\"Ready\",status=\"False\",kind=~\"GitRepository|HelmRepository|Bucket\"})",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "count(gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\", ready=\"False\"})",
+          "instant": true,
           "interval": "",
           "legendFormat": "",
+          "range": false,
           "refId": "A"
         }
       ],
-      "timeFrom": null,
-      "timeShift": null,
       "title": "Failing Sources",
       "type": "stat"
     },
@@ -318,9 +355,10 @@
           "values": false
         },
         "showUnfilled": true,
-        "text": {}
+        "text": {},
+        "valueMode": "color"
       },
-      "pluginVersion": "7.5.5",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "exemplar": true,
@@ -330,8 +368,6 @@
           "refId": "A"
         }
       ],
-      "timeFrom": null,
-      "timeShift": null,
       "title": "Reconciler ops avg. duration",
       "type": "bargauge"
     },
@@ -382,20 +418,19 @@
           "values": false
         },
         "showUnfilled": true,
-        "text": {}
+        "text": {},
+        "valueMode": "color"
       },
-      "pluginVersion": "7.5.5",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
           "exemplar": true,
-          "expr": "  sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket\"}[5m])) by (kind)\n/\n  sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket\"}[5m])) by (kind)",
+          "expr": "  sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind)\n/\n  sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind)",
           "interval": "",
           "legendFormat": "{{kind}}",
           "refId": "A"
         }
       ],
-      "timeFrom": null,
-      "timeShift": null,
       "title": "Source ops avg. duration",
       "type": "bargauge"
     },
@@ -414,23 +449,33 @@
       "type": "row"
     },
     {
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
       "description": "",
       "fieldConfig": {
         "defaults": {
           "custom": {
-            "displayMode": "auto",
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
             "filterable": true,
             "inspect": false
           },
           "mappings": [
             {
               "options": {
-                "0": {
-                  "text": "Ready"
-                },
-                "1": {
+                "False": {
+                  "color": "red",
+                  "index": 1,
                   "text": "Not Ready"
+                },
+                "True": {
+                  "color": "blue",
+                  "index": 0,
+                  "text": "Ready"
                 }
               },
               "type": "value"
@@ -440,16 +485,8 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "blue",
+                "color": "transparent",
                 "value": null
-              },
-              {
-                "color": "blue",
-                "value": 0
-              },
-              {
-                "color": "red",
-                "value": 1
               }
             ]
           }
@@ -457,13 +494,16 @@
         "overrides": [
           {
             "matcher": {
-              "id": "byName",
-              "options": "Status"
+              "id": "byType",
+              "options": "string"
             },
             "properties": [
               {
-                "id": "custom.displayMode",
-                "value": "color-background"
+                "id": "custom.cellOptions",
+                "value": {
+                  "mode": "basic",
+                  "type": "color-background"
+                }
               }
             ]
           }
@@ -477,7 +517,9 @@
       },
       "id": 33,
       "options": {
+        "cellHeight": "sm",
         "footer": {
+          "countRows": false,
           "fields": "",
           "reducer": [
             "sum"
@@ -492,11 +534,16 @@
           }
         ]
       },
-      "pluginVersion": "7.5.5",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
           "exemplar": true,
-          "expr": "gotk_reconcile_condition{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",type=\"Ready\",status=\"False\",kind=~\"Kustomization|HelmRelease\"}",
+          "expr": "gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"Kustomization|HelmRelease\"}",
           "format": "table",
           "instant": true,
           "interval": "",
@@ -504,8 +551,6 @@
           "refId": "A"
         }
       ],
-      "timeFrom": null,
-      "timeShift": null,
       "title": "Cluster reconciliation readiness ",
       "transformations": [
         {
@@ -513,11 +558,16 @@
           "options": {
             "excludeByName": {
               "Time": true,
+              "Value": true,
               "__name__": true,
               "app": true,
               "container": true,
+              "customresource_group": true,
+              "customresource_kind": false,
+              "customresource_version": true,
               "endpoint": true,
               "exported_namespace": false,
+              "gotk_type": true,
               "instance": true,
               "job": true,
               "kubernetes_namespace": true,
@@ -525,16 +575,36 @@
               "namespace": true,
               "pod": true,
               "pod_template_hash": true,
+              "service": true,
               "status": true,
               "type": true
             },
-            "indexByName": {},
+            "indexByName": {
+              "Time": 0,
+              "Value": 15,
+              "__name__": 1,
+              "container": 2,
+              "customresource_group": 4,
+              "customresource_kind": 5,
+              "customresource_version": 6,
+              "endpoint": 7,
+              "exported_namespace": 3,
+              "instance": 8,
+              "job": 9,
+              "name": 10,
+              "namespace": 11,
+              "pod": 12,
+              "ready": 13,
+              "service": 14
+            },
             "renameByName": {
-              "Value": "Status",
+              "Value": "",
+              "customresource_kind": "Kind",
               "exported_namespace": "Namespace",
               "kind": "Kind",
               "name": "Name",
-              "namespace": "Operator Namespace"
+              "namespace": "Operator Namespace",
+              "ready": "Status"
             }
           }
         }
@@ -542,23 +612,36 @@
       "type": "table"
     },
     {
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
       "description": "",
       "fieldConfig": {
         "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
           "custom": {
-            "displayMode": "auto",
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
             "filterable": true,
             "inspect": false
           },
           "mappings": [
             {
               "options": {
-                "0": {
-                  "text": "Ready"
-                },
-                "1": {
+                "False": {
+                  "color": "red",
+                  "index": 1,
                   "text": "Not Ready"
+                },
+                "True": {
+                  "color": "blue",
+                  "index": 0,
+                  "text": "Ready"
                 }
               },
               "type": "value"
@@ -568,21 +651,28 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "blue",
+                "color": "transparent",
                 "value": null
-              },
-              {
-                "color": "blue",
-                "value": 0
-              },
-              {
-                "color": "red",
-                "value": 1
               }
             ]
           }
         },
         "overrides": [
+          {
+            "matcher": {
+              "id": "byType",
+              "options": "string"
+            },
+            "properties": [
+              {
+                "id": "custom.cellOptions",
+                "value": {
+                  "mode": "basic",
+                  "type": "color-background"
+                }
+              }
+            ]
+          },
           {
             "matcher": {
               "id": "byName",
@@ -590,8 +680,15 @@
             },
             "properties": [
               {
-                "id": "custom.displayMode",
-                "value": "color-background"
+                "id": "noValue",
+                "value": "Ready"
+              },
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "blue",
+                  "mode": "fixed"
+                }
               }
             ]
           }
@@ -605,7 +702,9 @@
       },
       "id": 34,
       "options": {
+        "cellHeight": "sm",
         "footer": {
+          "countRows": false,
           "fields": "",
           "reducer": [
             "sum"
@@ -620,11 +719,16 @@
           }
         ]
       },
-      "pluginVersion": "7.5.5",
+      "pluginVersion": "10.0.2",
       "targets": [
         {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
           "exemplar": true,
-          "expr": "gotk_reconcile_condition{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",type=\"Ready\",status=\"False\",kind=~\"GitRepository|HelmRepository|Bucket\"}",
+          "expr": "gotk_resource_info{exported_namespace=~\"$namespace\", customresource_kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}",
           "format": "table",
           "instant": true,
           "interval": "",
@@ -632,8 +736,6 @@
           "refId": "A"
         }
       ],
-      "timeFrom": null,
-      "timeShift": null,
       "title": "Source acquisition readiness ",
       "transformations": [
         {
@@ -641,11 +743,16 @@
           "options": {
             "excludeByName": {
               "Time": true,
+              "Value": true,
               "__name__": true,
               "app": true,
               "container": true,
+              "customresource_group": true,
+              "customresource_kind": false,
+              "customresource_version": true,
               "endpoint": true,
               "exported_namespace": false,
+              "gotk_type": true,
               "instance": true,
               "job": true,
               "kubernetes_namespace": true,
@@ -653,16 +760,37 @@
               "namespace": true,
               "pod": true,
               "pod_template_hash": true,
+              "ready": false,
+              "service": true,
               "status": true,
               "type": true
             },
-            "indexByName": {},
+            "indexByName": {
+              "Time": 0,
+              "Value": 15,
+              "__name__": 1,
+              "container": 2,
+              "customresource_group": 5,
+              "customresource_kind": 6,
+              "customresource_version": 7,
+              "endpoint": 8,
+              "exported_namespace": 4,
+              "instance": 9,
+              "job": 10,
+              "name": 11,
+              "namespace": 3,
+              "pod": 12,
+              "ready": 13,
+              "service": 14
+            },
             "renameByName": {
-              "Value": "Status",
+              "Value": "",
+              "customresource_kind": "Kind",
               "exported_namespace": "Namespace",
               "kind": "Kind",
               "name": "Name",
-              "namespace": "Operator Namespace"
+              "namespace": "Operator Namespace",
+              "ready": "Status"
             }
           }
         }
@@ -690,10 +818,6 @@
       "dashes": false,
       "datasource": "${DS_PROMETHEUS}",
       "description": "",
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": []
-      },
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -724,7 +848,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.5.5",
+      "pluginVersion": "10.0.2",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -743,9 +867,7 @@
         }
       ],
       "thresholds": [],
-      "timeFrom": null,
       "timeRegions": [],
-      "timeShift": null,
       "title": "Cluster reconciliation duration",
       "tooltip": {
         "shared": true,
@@ -754,33 +876,24 @@
       },
       "type": "graph",
       "xaxis": {
-        "buckets": null,
         "mode": "time",
-        "name": null,
         "show": true,
         "values": []
       },
       "yaxes": [
         {
           "format": "s",
-          "label": null,
           "logBase": 1,
-          "max": null,
-          "min": null,
           "show": true
         },
         {
           "format": "short",
-          "label": null,
           "logBase": 1,
-          "max": null,
-          "min": null,
           "show": true
         }
       ],
       "yaxis": {
-        "align": false,
-        "alignLevel": null
+        "align": false
       }
     },
     {
@@ -790,10 +903,6 @@
       "dashes": false,
       "datasource": "${DS_PROMETHEUS}",
       "description": "",
-      "fieldConfig": {
-        "defaults": {},
-        "overrides": []
-      },
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -824,7 +933,7 @@
         "alertThreshold": true
       },
       "percentage": false,
-      "pluginVersion": "7.5.5",
+      "pluginVersion": "10.0.2",
       "pointradius": 2,
       "points": false,
       "renderer": "flot",
@@ -835,7 +944,7 @@
       "targets": [
         {
           "exemplar": true,
-          "expr": "  sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket\"}[5m])) by (kind, name)\n/\n  sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket\"}[5m])) by (kind, name)",
+          "expr": "  sum(rate(gotk_reconcile_duration_seconds_sum{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind, name)\n/\n  sum(rate(gotk_reconcile_duration_seconds_count{namespace=~\"$operator_namespace\",exported_namespace=~\"$namespace\",kind=~\"GitRepository|HelmRepository|Bucket|OCIRepository\"}[5m])) by (kind, name)",
           "hide": false,
           "interval": "",
           "legendFormat": "{{kind}}/{{name}}",
@@ -843,9 +952,7 @@
         }
       ],
       "thresholds": [],
-      "timeFrom": null,
       "timeRegions": [],
-      "timeShift": null,
       "title": "Source acquisition duration",
       "tooltip": {
         "shared": true,
@@ -854,38 +961,29 @@
       },
       "type": "graph",
       "xaxis": {
-        "buckets": null,
         "mode": "time",
-        "name": null,
         "show": true,
         "values": []
       },
       "yaxes": [
         {
           "format": "s",
-          "label": null,
           "logBase": 1,
-          "max": null,
-          "min": null,
           "show": true
         },
         {
           "format": "short",
-          "label": null,
           "logBase": 1,
-          "max": null,
-          "min": null,
           "show": true
         }
       ],
       "yaxis": {
-        "align": false,
-        "alignLevel": null
+        "align": false
       }
     }
   ],
   "refresh": "30s",
-  "schemaVersion": 36,
+  "schemaVersion": 38,
   "style": "light",
   "tags": [
     "flux"
@@ -903,13 +1001,13 @@
             "$__all"
           ]
         },
-        "datasource": "$DS_PROMETHEUS",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "$DS_PROMETHEUS"
+        },
         "definition": "label_values(gotk_reconcile_condition, namespace)",
-        "description": null,
-        "error": null,
         "hide": 0,
         "includeAll": true,
-        "label": null,
         "multi": true,
         "name": "operator_namespace",
         "options": [],
@@ -928,10 +1026,8 @@
         "useTags": false
       },
       {
-        "allValue": null,
         "current": {
           "selected": true,
-          "tags": [],
           "text": [
             "All"
           ],
@@ -939,19 +1035,19 @@
             "$__all"
           ]
         },
-        "datasource": "$DS_PROMETHEUS",
-        "definition": "label_values(gotk_reconcile_condition, exported_namespace)",
-        "description": null,
-        "error": null,
+        "datasource": {
+          "type": "prometheus",
+          "uid": "$DS_PROMETHEUS"
+        },
+        "definition": "label_values(gotk_resource_info,exported_namespace)",
         "hide": 0,
         "includeAll": true,
-        "label": null,
         "multi": true,
         "name": "namespace",
         "options": [],
         "query": {
-          "query": "label_values(gotk_reconcile_condition, exported_namespace)",
-          "refId": "StandardVariableQuery"
+          "query": "label_values(gotk_resource_info,exported_namespace)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 2,
         "regex": "",
@@ -1000,7 +1096,9 @@
       "1d"
     ]
   },
+  "timezone": "",
   "title": "Flux Cluster Stats",
   "uid": "flux-cluster",
-  "version": 3
+  "version": 4,
+  "weekStart": ""
 }