Add registry cred Deployments/CronJobs for aws/gcp/azure via kustomize

Signed-off-by: leigh capili <leigh@null.net>

manifests/integrations/registry-credentials-sync/_base/kustomization.yaml

@@ -0,0 +1,20 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+commonLabels:
+  app: credentials-sync
+
+resources:
+- sync.yaml
+
+vars:
+- name: KUBE_SECRET
+  objref:
+    kind: ConfigMap
+    name: credentials-sync
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.KUBE_SECRET
+
+configurations:
+- kustomizeconfig.yaml

manifests/integrations/registry-credentials-sync/_base/kustomizeconfig.yaml

@@ -0,0 +1,3 @@
+varReference:
+- path: rules/resourceNames
+  kind: Role

manifests/integrations/registry-credentials-sync/_base/sync.yaml

@@ -0,0 +1,125 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: credentials-sync
+data:
+  # Patch this ConfigMap with additional values needed for your cloud
+  KUBE_SECRET: my-registry-token  # does not yet exist -- will be created in the same Namespace
+  SYNC_PERIOD: "3600"  # tokens expire; refresh faster than that
+
+---
+# This Deployment frequently fetches registry tokens and applies them as an imagePullSecret.
+# It's done as a 1-replica Deployment rather than a CronJob, because CronJob scheduling can
+# block cluster bootstraps and cold-reboots from obtaining registry tokens for a considerable time.
+# This deployment will immediately fetch a token, which reduces latency for working image updates.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  template:
+    spec:
+      serviceAccount: credentials-sync
+      containers:
+      - image: busybox  # override this with a cloud-specific image
+        name: sync
+        envFrom:
+        - configMapRef:
+            name: credentials-sync
+        env:
+        - name: RECONCILE_SH  # override this env var with a shell function in a kustomize patch
+          value: |-
+            reconcile() {
+              echo reconciling...
+            }
+        command:
+        - bash
+        - -ceu
+        - |-
+          # template reconcile() into the script
+          # env var is expanded by k8s before the pod starts
+          $(RECONCILE_SH)
+
+          apply-secret() {
+            /kbin/kubectl create secret docker-registry "${1}" \
+              --docker-server="${2}" \
+              --docker-username="${3}" \
+              --docker-password="${4}" \
+              --dry-run=client -o=yaml \
+              | grep -v "creationTimestamp:" \
+              | /kbin/kubectl apply -f -
+          }
+
+          pause_loop() {
+            sleep "${SYNC_PERIOD:-3600}" || true
+          }
+
+          graceful_exit() {
+            echo "Trapped signal -- $(date)"
+            job_ids="$(
+              jobs \
+                | grep "pause_loop" \
+                | cut -d] -f1 \
+                | tr [ %
+              )"
+            # shellcheck disable=SC2086
+            if [ "${job_ids}" ]; then
+              kill ${job_ids}
+            fi
+            wait
+            echo "Graceful exit -- $(date)"
+          }
+
+          trap graceful_exit INT TERM
+
+          echo "Loop started (period: ${SYNC_PERIOD} s) -- $(date)"
+          while true; do
+            reconcile & wait $!
+            pause_loop & wait $!
+          done
+        resources: {}
+
+
+# RBAC necessary for our Deployment to apply our imagePullSecret
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+rules:
+- apiGroups: [""]
+  resources:
+  - secrets
+  verbs:
+  - get
+  - create
+  - update
+  - patch
+  # # Lock this down to the specific Secret name  (Optional)
+  resourceNames:
+  - $(KUBE_SECRET)  # templated from kustomize vars referencing ConfigMap, also see kustomizeconfig.yaml
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+subjects:
+- kind: ServiceAccount
+  name: credentials-sync
+roleRef:
+  kind: Role
+  name: credentials-sync
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: credentials-sync
+  namespace: flux-system

manifests/integrations/registry-credentials-sync/_cronjobs/_base/kustomization.yaml

@@ -0,0 +1,20 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+commonLabels:
+  app: credentials-sync
+
+resources:
+- sync.yaml
+
+vars:
+- name: KUBE_SECRET
+  objref:
+    kind: ConfigMap
+    name: credentials-sync
+    apiVersion: v1
+  fieldref:
+    fieldpath: data.KUBE_SECRET
+
+configurations:
+- kustomizeconfig.yaml

manifests/integrations/registry-credentials-sync/_cronjobs/_base/kustomizeconfig.yaml

@@ -0,0 +1,3 @@
+varReference:
+- path: rules/resourceNames
+  kind: Role

manifests/integrations/registry-credentials-sync/_cronjobs/_base/sync.yaml

@@ -0,0 +1,101 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: credentials-sync
+data:
+  # Patch this ConfigMap with additional values needed for your cloud
+  KUBE_SECRET: my-registry-token  # does not yet exist -- will be created in the same Namespace
+
+---
+# This CronJob frequently fetches registry tokens and applies them as an imagePullSecret.
+# note: CronJob scheduling can block cluster bootstraps and cold-reboots from obtaining registry tokens for a considerable time.
+# To run the job immediately, do `kubectl create job --from=cronjob/credentials-sync -n flux-system credentials-sync-init`
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+spec:
+  suspend: false
+  schedule: 0 */6 * * *
+  failedJobsHistoryLimit: 1
+  successfulJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          serviceAccountName: credentials-sync
+          restartPolicy: Never
+          containers:
+          - image: busybox  # override this with a cloud-specific image
+            name: sync
+            envFrom:
+            - configMapRef:
+                name: credentials-sync
+            env:
+            - name: RECONCILE_SH  # override this env var with a shell function in a kustomize patch
+              value: |-
+                reconcile() {
+                  echo reconciling...
+                }
+            command:
+            - bash
+            - -ceu
+            - |-
+              # template reconcile() into the script
+              # env var is expanded by k8s before the pod starts
+              $(RECONCILE_SH)
+
+              apply-secret() {
+                /kbin/kubectl create secret docker-registry "${1}" \
+                  --docker-server="${2}" \
+                  --docker-username="${3}" \
+                  --docker-password="${4}" \
+                  --dry-run=client -o=yaml \
+                  | grep -v "creationTimestamp:" \
+                  | /kbin/kubectl apply -f -
+              }
+
+              reconcile
+            resources: {}
+
+
+# RBAC necessary for our Deployment to apply our imagePullSecret
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+rules:
+- apiGroups: [""]
+  resources:
+  - secrets
+  verbs:
+  - get
+  - create
+  - update
+  - patch
+  # # Lock this down to the specific Secret name  (Optional)
+  resourceNames:
+  - $(KUBE_SECRET)  # templated from kustomize vars referencing ConfigMap, also see kustomizeconfig.yaml
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+subjects:
+- kind: ServiceAccount
+  name: credentials-sync
+roleRef:
+  kind: Role
+  name: credentials-sync
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: credentials-sync
+  namespace: flux-system

manifests/integrations/registry-credentials-sync/_cronjobs/aws/config-patches.yaml

@@ -0,0 +1,52 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: credentials-sync
+data:
+  ECR_REGION: us-east-1  # set the region
+  ECR_REGISTRY: <account id>.dkr.ecr.<region>.amazonaws.com  # fill in the account id and region
+  KUBE_SECRET: ecr-credentials  # does not yet exist -- will be created in the same Namespace
+
+
+# Bind IRSA for the ServiceAccount 
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+  annotations:
+    eks.amazonaws.com/role-arn: <role arn>  # set the ARN for your role
+
+
+# Set the reconcile period
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+spec:
+  schedule: 0 */6 * * *  # every 6hrs -- ECR tokens expire every 12 hours; refresh faster than that
+
+
+## If not using IRSA, set the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables
+## Store these values in a Secret and load them in the container using envFrom.
+## For managing this secret via GitOps, consider using SOPS or SealedSecrets and add that manifest in a resource file for this kustomize build.
+##   https://toolkit.fluxcd.io/guides/mozilla-sops/
+##   https://toolkit.fluxcd.io/guides/sealed-secrets/
+# ---
+# apiVersion: apps/v1
+# kind: Deployment
+# metadata:
+#   name: credentials-sync
+#   namespace: flux-system
+# spec:
+#   template:
+#     spec:
+#       containers:
+#       - name: sync
+#         envFrom:
+#           secretRef:
+#             name: $(ECR_SECRET_NAME)  # uncomment the var for this in kustomization.yaml

manifests/integrations/registry-credentials-sync/_cronjobs/aws/kubectl-patch.yaml

@@ -0,0 +1,30 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+spec:
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          initContainers:
+          - image: bitnami/kubectl
+            name: copy-kubectl
+            # it's okay to do this because kubectl is a statically linked binary
+            command:
+            - sh
+            - -ceu
+            - cp $(which kubectl) /kbin/
+            resources: {}
+            volumeMounts:
+            - name: kbin
+              mountPath: /kbin
+          containers:
+          - name: sync
+            volumeMounts:
+            - name: kbin
+              mountPath: /kbin
+          volumes:
+          - name: kbin
+            emptyDir: {}

manifests/integrations/registry-credentials-sync/_cronjobs/aws/kustomization.yaml

@@ -0,0 +1,26 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namePrefix: ecr-
+commonLabels:
+  app: ecr-credentials-sync
+
+namespace: flux-system
+
+bases:
+- ../_base
+## If not using IRSA, consider creating the following file via SOPS or SealedSecrets
+# - encrypted-secret.yaml
+
+patchesStrategicMerge:
+- config-patches.yaml
+- kubectl-patch.yaml
+- reconcile-patch.yaml
+
+## uncomment if using encrypted-secret.yaml
+# vars:
+# - name: ECR_SECRET_NAME
+#   objref:
+#     kind: Secret
+#     name: credentials-sync
+#     apiVersion: v1

manifests/integrations/registry-credentials-sync/_cronjobs/aws/reconcile-patch.yaml

@@ -0,0 +1,29 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+spec:
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: sync
+            image: mcr.microsoft.com/azure-cli
+            env:
+            - name: RECONCILE_SH
+              value: |-
+                reconcile() {
+                  echo "Starting ECR token sync -- $(date)"
+                  echo "Logging into ECR: ${ECR_REGION} -- ${ECR_REGISTRY}"
+                  token="$(aws ecr get-login-password --region "${ECR_REGION}")"
+                  user="AWS"
+                  server="${ECR_REGISTRY}"
+
+                  echo "Creating secret: ${KUBE_SECRET}"
+                  apply-secret "${KUBE_SECRET}" "${token}" "${user}" "${server}"
+
+                  echo "Finished ECR token sync -- $(date)"
+                  echo
+                }

manifests/integrations/registry-credentials-sync/_cronjobs/azure/az-identity.yaml

@@ -0,0 +1,7 @@
+# This is a stub resource patched by config-patches.yaml, so that all config is visible in one file
+---
+apiVersion: aadpodidentity.k8s.io/v1
+kind: AzureIdentity
+metadata:
+  name: credentials-sync  # if this is changed, also change in config-patches.yaml
+  namespace: flux-system

manifests/integrations/registry-credentials-sync/_cronjobs/azure/config-patches.yaml

@@ -0,0 +1,41 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: credentials-sync
+data:
+  ACR_NAME: my-registry
+  KUBE_SECRET: acr-my-registry  # does not yet exist -- will be created in the same Namespace
+
+# Create an identity in Azure and assign it a role to pull from ACR  (note: the identity's resourceGroup should match the desired ACR):
+#     az identity create -n acr-sync
+#     az role assignment create --role AcrPull --assignee-object-id "$(az identity show -n acr-sync -o tsv --query principalId)"
+# Fetch the clientID and resourceID to configure the AzureIdentity spec below:
+#     az identity show -n acr-sync -otsv --query clientId
+#     az identity show -n acr-sync -otsv --query resourceId
+---
+apiVersion: aadpodidentity.k8s.io/v1
+kind: AzureIdentity
+metadata:
+  name: credentials-sync  # name must match the stub-resource in az-identity.yaml
+  namespace: flux-system
+spec:
+  clientID: 82d01fb0-7799-4d9d-92c7-21e7632c0000
+  resourceID: /subscriptions/873c7e7f-76cd-4805-ae86-b923850b0000/resourcegroups/stealthybox/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-sync
+  type: 0  # user-managed identity
+
+# Set the reconcile period + specify the pod-identity via the aadpodidbinding label
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+spec:
+  schedule: 0 * * * *  # ACR tokens expire every 3 hours; refresh faster than that
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            aadpodidbinding: $(AZ_IDENTITY_NAME)  # match the AzureIdentity name

manifests/integrations/registry-credentials-sync/_cronjobs/azure/kubectl-patch.yaml

@@ -0,0 +1,30 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+spec:
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          initContainers:
+          - image: bitnami/kubectl
+            name: copy-kubectl
+            # it's okay to do this because kubectl is a statically linked binary
+            command:
+            - sh
+            - -ceu
+            - cp $(which kubectl) /kbin/
+            resources: {}
+            volumeMounts:
+            - name: kbin
+              mountPath: /kbin
+          containers:
+          - name: sync
+            volumeMounts:
+            - name: kbin
+              mountPath: /kbin
+          volumes:
+          - name: kbin
+            emptyDir: {}

manifests/integrations/registry-credentials-sync/_cronjobs/azure/kustomization.yaml

@@ -0,0 +1,28 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namePrefix: acr-
+commonLabels:
+  app: acr-credentials-sync
+
+namespace: flux-system
+
+bases:
+- ../_base
+resources:
+- az-identity.yaml
+
+patchesStrategicMerge:
+- config-patches.yaml
+- kubectl-patch.yaml
+- reconcile-patch.yaml
+
+vars:
+- name: AZ_IDENTITY_NAME
+  objref:
+    kind: AzureIdentity
+    name: credentials-sync
+    apiVersion: aadpodidentity.k8s.io/v1
+
+configurations:
+- kustomizeconfig.yaml

manifests/integrations/registry-credentials-sync/_cronjobs/azure/kustomizeconfig.yaml

@@ -0,0 +1,3 @@
+varReference:
+- path: spec/jobTemplate/spec/template/metadata/labels
+  kind: Deployment

manifests/integrations/registry-credentials-sync/_cronjobs/azure/reconcile-patch.yaml

@@ -0,0 +1,37 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+spec:
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: sync
+            image: mcr.microsoft.com/azure-cli
+            env:
+            - name: RECONCILE_SH
+              value: |-
+                reconcile() {
+                  echo "Starting ACR token sync -- $(date)"
+                  echo "Logging into Azure"
+                  az login --identity
+                  echo "Logging into ACR: ${ACR_NAME}"
+                  output="$(az acr login --expose-token -o=tsv -n "${ACR_NAME}")"
+                  read token server <<< "${output}"
+                  user="00000000-0000-0000-0000-000000000000"
+
+                  echo "Creating secret: ${KUBE_SECRET}"
+                  /kbin/kubectl create secret docker-registry "${KUBE_SECRET}" \
+                    --docker-server="${server}" \
+                    --docker-username="00000000-0000-0000-0000-000000000000" \
+                    --docker-password="${token}" \
+                    --dry-run=client -o=yaml \
+                    | grep -v "creationTimestamp:" \
+                    | /kbin/kubectl apply -f -
+
+                  echo "Finished ACR token sync -- $(date)"
+                  echo
+                }

manifests/integrations/registry-credentials-sync/_cronjobs/gcp/config-patches.yaml

@@ -0,0 +1,28 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: credentials-sync
+data:
+  GCR_REGISTRY: gcr.io  # set the registry
+  KUBE_SECRET: gcr-credentials  # does not yet exist -- will be created in the same Namespace
+
+# Bind to the GCP service-account
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+  annotations:
+    iam.gke.io/gcp-service-account: <name>@<project-id>.iam.gserviceaccount.com # set the GCP service-account
+
+# Set the reconcile period
+---
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+spec:
+  schedule: 0,30 * * * *  # 30m interval -- GCR tokens expire every hour; refresh faster than that

manifests/integrations/registry-credentials-sync/_cronjobs/gcp/kustomization.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namePrefix: gcr-
+commonLabels:
+  app: gcr-credentials-sync
+
+namespace: flux-system
+
+bases:
+- ../_base
+
+patchesStrategicMerge:
+- config-patches.yaml
+- reconcile-patch.yaml

manifests/integrations/registry-credentials-sync/_cronjobs/gcp/reconcile-patch.yaml

@@ -0,0 +1,29 @@
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+spec:
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: sync
+            image: aws/aws-cli
+            env:
+            - name: RECONCILE_SH
+              value: |-
+                reconcile() {
+                  echo "Starting GCR token sync -- $(date)"
+                  echo "Logging into ECR: ${ECR_REGION} -- ${ECR_REGISTRY}"
+                  token="$(gcloud auth print-access-token)"
+                  user="oauth2accesstoken "
+                  server="${GCR_REGISTRY}"
+
+                  echo "Creating secret: ${KUBE_SECRET}"
+                  apply-secret "${KUBE_SECRET}" "${token}" "${user}" "${server}"
+
+                  echo "Finished GCR token sync -- $(date)"
+                  echo
+                }

manifests/integrations/registry-credentials-sync/aws/config-patches.yaml

@@ -0,0 +1,42 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: credentials-sync
+data:
+  ECR_REGION: us-east-1  # set the region
+  ECR_REGISTRY: <account id>.dkr.ecr.<region>.amazonaws.com  # fill in the account id and region
+  KUBE_SECRET: ecr-credentials  # does not yet exist -- will be created in the same Namespace
+  SYNC_PERIOD: "21600"  # 6hrs -- ECR tokens expire every 12 hours; refresh faster than that
+
+
+# Bind IRSA for the ServiceAccount 
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+  annotations:
+    eks.amazonaws.com/role-arn: <role arn>  # set the ARN for your role
+
+
+## If not using IRSA, set the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables
+## Store these values in a Secret and load them in the container using envFrom.
+## For managing this secret via GitOps, consider using SOPS or SealedSecrets and add that manifest in a resource file for this kustomize build.
+##   https://toolkit.fluxcd.io/guides/mozilla-sops/
+##   https://toolkit.fluxcd.io/guides/sealed-secrets/
+# ---
+# apiVersion: apps/v1
+# kind: Deployment
+# metadata:
+#   name: credentials-sync
+#   namespace: flux-system
+# spec:
+#   template:
+#     spec:
+#       containers:
+#       - name: sync
+#         envFrom:
+#           secretRef:
+#             name: $(ECR_SECRET_NAME)  # uncomment the var for this in kustomization.yaml

manifests/integrations/registry-credentials-sync/aws/kubectl-patch.yaml

@@ -0,0 +1,28 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+spec:
+  template:
+    spec:
+      initContainers:
+      - image: bitnami/kubectl
+        name: copy-kubectl
+        # it's okay to do this because kubectl is a statically linked binary
+        command:
+        - sh
+        - -ceu
+        - cp $(which kubectl) /kbin/
+        resources: {}
+        volumeMounts:
+        - name: kbin
+          mountPath: /kbin
+      containers:
+      - name: sync
+        volumeMounts:
+        - name: kbin
+          mountPath: /kbin
+      volumes:
+      - name: kbin
+        emptyDir: {}

manifests/integrations/registry-credentials-sync/aws/kustomization.yaml

@@ -0,0 +1,26 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namePrefix: ecr-
+commonLabels:
+  app: ecr-credentials-sync
+
+namespace: flux-system
+
+bases:
+- ../_base
+## If not using IRSA, consider creating the following file via SOPS or SealedSecrets
+# - encrypted-secret.yaml
+
+patchesStrategicMerge:
+- config-patches.yaml
+- kubectl-patch.yaml
+- reconcile-patch.yaml
+
+## uncomment if using encrypted-secret.yaml
+# vars:
+# - name: ECR_SECRET_NAME
+#   objref:
+#     kind: Secret
+#     name: credentials-sync
+#     apiVersion: v1

manifests/integrations/registry-credentials-sync/aws/reconcile-patch.yaml

@@ -0,0 +1,28 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+spec:
+  template:
+    spec:
+      containers:
+      - name: sync
+        image: aws/aws-cli
+        env:
+        - name: RECONCILE_SH
+          value: |-
+            reconcile() {
+              echo "Starting ECR token sync -- $(date)"
+              echo "Logging into ECR: ${ECR_REGION} -- ${ECR_REGISTRY}"
+              token="$(aws ecr get-login-password --region "${ECR_REGION}")"
+              user="AWS"
+              server="${ECR_REGISTRY}"
+
+              echo "Creating secret: ${KUBE_SECRET}"
+              apply-secret "${KUBE_SECRET}" "${token}" "${user}" "${server}"
+
+              echo "Finished ECR token sync -- $(date)"
+              echo
+            }

manifests/integrations/registry-credentials-sync/azure/az-identity.yaml

@@ -0,0 +1,7 @@
+# This is a stub resource patched by config-patches.yaml, so that all config is visible in one file
+---
+apiVersion: aadpodidentity.k8s.io/v1
+kind: AzureIdentity
+metadata:
+  name: credentials-sync  # if this is changed, also change in config-patches.yaml
+  namespace: flux-system

manifests/integrations/registry-credentials-sync/azure/config-patches.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: credentials-sync
+data:
+  ACR_NAME: my-registry
+  KUBE_SECRET: acr-my-registry  # does not yet exist -- will be created in the same Namespace
+  SYNC_PERIOD: "3600"  # ACR tokens expire every 3 hours; refresh faster than that
+
+# Create an identity in Azure and assign it a role to pull from ACR  (note: the identity's resourceGroup should match the desired ACR):
+#     az identity create -n acr-sync
+#     az role assignment create --role AcrPull --assignee-object-id "$(az identity show -n acr-sync -o tsv --query principalId)"
+# Fetch the clientID and resourceID to configure the AzureIdentity spec below:
+#     az identity show -n acr-sync -otsv --query clientId
+#     az identity show -n acr-sync -otsv --query resourceId
+---
+apiVersion: aadpodidentity.k8s.io/v1
+kind: AzureIdentity
+metadata:
+  name: credentials-sync  # name must match the stub-resource in az-identity.yaml
+  namespace: flux-system
+spec:
+  clientID: 82d01fb0-7799-4d9d-92c7-21e7632c0000
+  resourceID: /subscriptions/873c7e7f-76cd-4805-ae86-b923850b0000/resourcegroups/stealthybox/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-sync
+  type: 0  # user-managed identity
+
+# Specify the pod-identity via the aadpodidbinding label
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+spec:
+  template:
+    metadata:
+      labels:
+        aadpodidbinding: $(AZ_IDENTITY_NAME)  # match the AzureIdentity name

manifests/integrations/registry-credentials-sync/azure/kubectl-patch.yaml

@@ -0,0 +1,28 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+spec:
+  template:
+    spec:
+      initContainers:
+      - image: bitnami/kubectl
+        name: copy-kubectl
+        # it's okay to do this because kubectl is a statically linked binary
+        command:
+        - sh
+        - -ceu
+        - cp $(which kubectl) /kbin/
+        resources: {}
+        volumeMounts:
+        - name: kbin
+          mountPath: /kbin
+      containers:
+      - name: sync
+        volumeMounts:
+        - name: kbin
+          mountPath: /kbin
+      volumes:
+      - name: kbin
+        emptyDir: {}

manifests/integrations/registry-credentials-sync/azure/kustomization.yaml

@@ -0,0 +1,28 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namePrefix: acr-
+commonLabels:
+  app: acr-credentials-sync
+
+namespace: flux-system
+
+bases:
+- ../_base
+resources:
+- az-identity.yaml
+
+patchesStrategicMerge:
+- config-patches.yaml
+- kubectl-patch.yaml
+- reconcile-patch.yaml
+
+vars:
+- name: AZ_IDENTITY_NAME
+  objref:
+    kind: AzureIdentity
+    name: credentials-sync
+    apiVersion: aadpodidentity.k8s.io/v1
+
+configurations:
+- kustomizeconfig.yaml

manifests/integrations/registry-credentials-sync/azure/kustomizeconfig.yaml

@@ -0,0 +1,3 @@
+varReference:
+- path: spec/template/metadata/labels
+  kind: Deployment

manifests/integrations/registry-credentials-sync/azure/reconcile-patch.yaml

@@ -0,0 +1,30 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+spec:
+  template:
+    spec:
+      containers:
+      - name: sync
+        image: mcr.microsoft.com/azure-cli
+        env:
+        - name: RECONCILE_SH
+          value: |-
+            reconcile() {
+              echo "Starting ACR token sync -- $(date)"
+              echo "Logging into Azure"
+              az login --identity
+              echo "Logging into ACR: ${ACR_NAME}"
+              output="$(az acr login --expose-token -o=tsv -n "${ACR_NAME}")"
+              read token server <<< "${output}"
+              user="00000000-0000-0000-0000-000000000000"
+
+              echo "Creating secret: ${KUBE_SECRET}"
+              apply-secret "${KUBE_SECRET}" "${token}" "${user}" "${server}"
+
+              echo "Finished ECR token sync -- $(date)"
+              echo
+            }

manifests/integrations/registry-credentials-sync/gcp/config-patches.yaml

@@ -0,0 +1,20 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: credentials-sync
+data:
+  GCR_REGISTRY: gcr.io  # set the registry
+  KUBE_SECRET: gcr-credentials  # does not yet exist -- will be created in the same Namespace
+  SYNC_PERIOD: "1800"  # 30m -- GCR tokens expire every hour; refresh faster than that
+
+
+# Bind to the GCP service-account
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+  annotations:
+    iam.gke.io/gcp-service-account: <name>@<project-id>.iam.gserviceaccount.com # set the GCP service-account

manifests/integrations/registry-credentials-sync/gcp/kustomization.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namePrefix: gcr-
+commonLabels:
+  app: gcr-credentials-sync
+
+namespace: flux-system
+
+bases:
+- ../_base
+
+patchesStrategicMerge:
+- config-patches.yaml
+- reconcile-patch.yaml

manifests/integrations/registry-credentials-sync/gcp/reconcile-patch.yaml

@@ -0,0 +1,28 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: credentials-sync
+  namespace: flux-system
+spec:
+  template:
+    spec:
+      containers:
+      - name: sync
+        image: aws/aws-cli
+        env:
+        - name: RECONCILE_SH
+          value: |-
+            reconcile() {
+              echo "Starting GCR token sync -- $(date)"
+              echo "Logging into ECR: ${ECR_REGION} -- ${ECR_REGISTRY}"
+              token="$(gcloud auth print-access-token)"
+              user="oauth2accesstoken "
+              server="${GCR_REGISTRY}"
+
+              echo "Creating secret: ${KUBE_SECRET}"
+              apply-secret "${KUBE_SECRET}" "${token}" "${user}" "${server}"
+
+              echo "Finished GCR token sync -- $(date)"
+              echo
+            }