From 99825f2663cd8513691cf7156d50e7e9e9a291e5 Mon Sep 17 00:00:00 2001 From: leigh capili Date: Mon, 8 Feb 2021 01:14:53 -0700 Subject: [PATCH] Add registry cred Deployments/CronJobs for aws/gcp/azure via kustomize Signed-off-by: leigh capili --- .../_base/kustomization.yaml | 20 +++ .../_base/kustomizeconfig.yaml | 3 + .../registry-credentials-sync/_base/sync.yaml | 125 ++++++++++++++++++ .../_cronjobs/_base/kustomization.yaml | 20 +++ .../_cronjobs/_base/kustomizeconfig.yaml | 3 + .../_cronjobs/_base/sync.yaml | 101 ++++++++++++++ .../_cronjobs/aws/config-patches.yaml | 52 ++++++++ .../_cronjobs/aws/kubectl-patch.yaml | 30 +++++ .../_cronjobs/aws/kustomization.yaml | 26 ++++ .../_cronjobs/aws/reconcile-patch.yaml | 29 ++++ .../_cronjobs/azure/az-identity.yaml | 7 + .../_cronjobs/azure/config-patches.yaml | 41 ++++++ .../_cronjobs/azure/kubectl-patch.yaml | 30 +++++ .../_cronjobs/azure/kustomization.yaml | 28 ++++ .../_cronjobs/azure/kustomizeconfig.yaml | 3 + .../_cronjobs/azure/reconcile-patch.yaml | 37 ++++++ .../_cronjobs/gcp/config-patches.yaml | 28 ++++ .../_cronjobs/gcp/kustomization.yaml | 15 +++ .../_cronjobs/gcp/reconcile-patch.yaml | 29 ++++ .../aws/config-patches.yaml | 42 ++++++ .../aws/kubectl-patch.yaml | 28 ++++ .../aws/kustomization.yaml | 26 ++++ .../aws/reconcile-patch.yaml | 28 ++++ .../azure/az-identity.yaml | 7 + .../azure/config-patches.yaml | 39 ++++++ .../azure/kubectl-patch.yaml | 28 ++++ .../azure/kustomization.yaml | 28 ++++ .../azure/kustomizeconfig.yaml | 3 + .../azure/reconcile-patch.yaml | 30 +++++ .../gcp/config-patches.yaml | 20 +++ .../gcp/kustomization.yaml | 15 +++ .../gcp/reconcile-patch.yaml | 28 ++++ 32 files changed, 949 insertions(+) create mode 100644 manifests/integrations/registry-credentials-sync/_base/kustomization.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_base/kustomizeconfig.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_base/sync.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/_base/kustomization.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/_base/kustomizeconfig.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/_base/sync.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/aws/config-patches.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/aws/kubectl-patch.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/aws/kustomization.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/aws/reconcile-patch.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/azure/az-identity.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/azure/config-patches.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/azure/kubectl-patch.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/azure/kustomization.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/azure/kustomizeconfig.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/azure/reconcile-patch.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/gcp/config-patches.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/gcp/kustomization.yaml create mode 100644 manifests/integrations/registry-credentials-sync/_cronjobs/gcp/reconcile-patch.yaml create mode 100644 manifests/integrations/registry-credentials-sync/aws/config-patches.yaml create mode 100644 manifests/integrations/registry-credentials-sync/aws/kubectl-patch.yaml create mode 100644 manifests/integrations/registry-credentials-sync/aws/kustomization.yaml create mode 100644 manifests/integrations/registry-credentials-sync/aws/reconcile-patch.yaml create mode 100644 manifests/integrations/registry-credentials-sync/azure/az-identity.yaml create mode 100644 manifests/integrations/registry-credentials-sync/azure/config-patches.yaml create mode 100644 manifests/integrations/registry-credentials-sync/azure/kubectl-patch.yaml create mode 100644 manifests/integrations/registry-credentials-sync/azure/kustomization.yaml create mode 100644 manifests/integrations/registry-credentials-sync/azure/kustomizeconfig.yaml create mode 100644 manifests/integrations/registry-credentials-sync/azure/reconcile-patch.yaml create mode 100644 manifests/integrations/registry-credentials-sync/gcp/config-patches.yaml create mode 100644 manifests/integrations/registry-credentials-sync/gcp/kustomization.yaml create mode 100644 manifests/integrations/registry-credentials-sync/gcp/reconcile-patch.yaml diff --git a/manifests/integrations/registry-credentials-sync/_base/kustomization.yaml b/manifests/integrations/registry-credentials-sync/_base/kustomization.yaml new file mode 100644 index 00000000..c26a2c0a --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_base/kustomization.yaml @@ -0,0 +1,20 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +commonLabels: + app: credentials-sync + +resources: +- sync.yaml + +vars: +- name: KUBE_SECRET + objref: + kind: ConfigMap + name: credentials-sync + apiVersion: v1 + fieldref: + fieldpath: data.KUBE_SECRET + +configurations: +- kustomizeconfig.yaml diff --git a/manifests/integrations/registry-credentials-sync/_base/kustomizeconfig.yaml b/manifests/integrations/registry-credentials-sync/_base/kustomizeconfig.yaml new file mode 100644 index 00000000..61edffd4 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_base/kustomizeconfig.yaml @@ -0,0 +1,3 @@ +varReference: +- path: rules/resourceNames + kind: Role diff --git a/manifests/integrations/registry-credentials-sync/_base/sync.yaml b/manifests/integrations/registry-credentials-sync/_base/sync.yaml new file mode 100644 index 00000000..4efb25f0 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_base/sync.yaml @@ -0,0 +1,125 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync +data: + # Patch this ConfigMap with additional values needed for your cloud + KUBE_SECRET: my-registry-token # does not yet exist -- will be created in the same Namespace + SYNC_PERIOD: "3600" # tokens expire; refresh faster than that + +--- +# This Deployment frequently fetches registry tokens and applies them as an imagePullSecret. +# It's done as a 1-replica Deployment rather than a CronJob, because CronJob scheduling can +# block cluster bootstraps and cold-reboots from obtaining registry tokens for a considerable time. +# This deployment will immediately fetch a token, which reduces latency for working image updates. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync + namespace: flux-system +spec: + replicas: 1 + strategy: + type: Recreate + template: + spec: + serviceAccount: credentials-sync + containers: + - image: busybox # override this with a cloud-specific image + name: sync + envFrom: + - configMapRef: + name: credentials-sync + env: + - name: RECONCILE_SH # override this env var with a shell function in a kustomize patch + value: |- + reconcile() { + echo reconciling... + } + command: + - bash + - -ceu + - |- + # template reconcile() into the script + # env var is expanded by k8s before the pod starts + $(RECONCILE_SH) + + apply-secret() { + /kbin/kubectl create secret docker-registry "${1}" \ + --docker-server="${2}" \ + --docker-username="${3}" \ + --docker-password="${4}" \ + --dry-run=client -o=yaml \ + | grep -v "creationTimestamp:" \ + | /kbin/kubectl apply -f - + } + + pause_loop() { + sleep "${SYNC_PERIOD:-3600}" || true + } + + graceful_exit() { + echo "Trapped signal -- $(date)" + job_ids="$( + jobs \ + | grep "pause_loop" \ + | cut -d] -f1 \ + | tr [ % + )" + # shellcheck disable=SC2086 + if [ "${job_ids}" ]; then + kill ${job_ids} + fi + wait + echo "Graceful exit -- $(date)" + } + + trap graceful_exit INT TERM + + echo "Loop started (period: ${SYNC_PERIOD} s) -- $(date)" + while true; do + reconcile & wait $! + pause_loop & wait $! + done + resources: {} + + +# RBAC necessary for our Deployment to apply our imagePullSecret +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: credentials-sync + namespace: flux-system +rules: +- apiGroups: [""] + resources: + - secrets + verbs: + - get + - create + - update + - patch + # # Lock this down to the specific Secret name (Optional) + resourceNames: + - $(KUBE_SECRET) # templated from kustomize vars referencing ConfigMap, also see kustomizeconfig.yaml +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: credentials-sync + namespace: flux-system +subjects: +- kind: ServiceAccount + name: credentials-sync +roleRef: + kind: Role + name: credentials-sync + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: credentials-sync + namespace: flux-system diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/_base/kustomization.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/_base/kustomization.yaml new file mode 100644 index 00000000..c26a2c0a --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/_base/kustomization.yaml @@ -0,0 +1,20 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +commonLabels: + app: credentials-sync + +resources: +- sync.yaml + +vars: +- name: KUBE_SECRET + objref: + kind: ConfigMap + name: credentials-sync + apiVersion: v1 + fieldref: + fieldpath: data.KUBE_SECRET + +configurations: +- kustomizeconfig.yaml diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/_base/kustomizeconfig.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/_base/kustomizeconfig.yaml new file mode 100644 index 00000000..61edffd4 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/_base/kustomizeconfig.yaml @@ -0,0 +1,3 @@ +varReference: +- path: rules/resourceNames + kind: Role diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/_base/sync.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/_base/sync.yaml new file mode 100644 index 00000000..86330170 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/_base/sync.yaml @@ -0,0 +1,101 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync +data: + # Patch this ConfigMap with additional values needed for your cloud + KUBE_SECRET: my-registry-token # does not yet exist -- will be created in the same Namespace + +--- +# This CronJob frequently fetches registry tokens and applies them as an imagePullSecret. +# note: CronJob scheduling can block cluster bootstraps and cold-reboots from obtaining registry tokens for a considerable time. +# To run the job immediately, do `kubectl create job --from=cronjob/credentials-sync -n flux-system credentials-sync-init` +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync + namespace: flux-system +spec: + suspend: false + schedule: 0 */6 * * * + failedJobsHistoryLimit: 1 + successfulJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + serviceAccountName: credentials-sync + restartPolicy: Never + containers: + - image: busybox # override this with a cloud-specific image + name: sync + envFrom: + - configMapRef: + name: credentials-sync + env: + - name: RECONCILE_SH # override this env var with a shell function in a kustomize patch + value: |- + reconcile() { + echo reconciling... + } + command: + - bash + - -ceu + - |- + # template reconcile() into the script + # env var is expanded by k8s before the pod starts + $(RECONCILE_SH) + + apply-secret() { + /kbin/kubectl create secret docker-registry "${1}" \ + --docker-server="${2}" \ + --docker-username="${3}" \ + --docker-password="${4}" \ + --dry-run=client -o=yaml \ + | grep -v "creationTimestamp:" \ + | /kbin/kubectl apply -f - + } + + reconcile + resources: {} + + +# RBAC necessary for our Deployment to apply our imagePullSecret +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: credentials-sync + namespace: flux-system +rules: +- apiGroups: [""] + resources: + - secrets + verbs: + - get + - create + - update + - patch + # # Lock this down to the specific Secret name (Optional) + resourceNames: + - $(KUBE_SECRET) # templated from kustomize vars referencing ConfigMap, also see kustomizeconfig.yaml +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: credentials-sync + namespace: flux-system +subjects: +- kind: ServiceAccount + name: credentials-sync +roleRef: + kind: Role + name: credentials-sync + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: credentials-sync + namespace: flux-system diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/aws/config-patches.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/aws/config-patches.yaml new file mode 100644 index 00000000..fabaead4 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/aws/config-patches.yaml @@ -0,0 +1,52 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync +data: + ECR_REGION: us-east-1 # set the region + ECR_REGISTRY: .dkr.ecr..amazonaws.com # fill in the account id and region + KUBE_SECRET: ecr-credentials # does not yet exist -- will be created in the same Namespace + + +# Bind IRSA for the ServiceAccount +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: credentials-sync + namespace: flux-system + annotations: + eks.amazonaws.com/role-arn: # set the ARN for your role + + +# Set the reconcile period +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync + namespace: flux-system +spec: + schedule: 0 */6 * * * # every 6hrs -- ECR tokens expire every 12 hours; refresh faster than that + + +## If not using IRSA, set the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables +## Store these values in a Secret and load them in the container using envFrom. +## For managing this secret via GitOps, consider using SOPS or SealedSecrets and add that manifest in a resource file for this kustomize build. +## https://toolkit.fluxcd.io/guides/mozilla-sops/ +## https://toolkit.fluxcd.io/guides/sealed-secrets/ +# --- +# apiVersion: apps/v1 +# kind: Deployment +# metadata: +# name: credentials-sync +# namespace: flux-system +# spec: +# template: +# spec: +# containers: +# - name: sync +# envFrom: +# secretRef: +# name: $(ECR_SECRET_NAME) # uncomment the var for this in kustomization.yaml diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/aws/kubectl-patch.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/aws/kubectl-patch.yaml new file mode 100644 index 00000000..b4d83e22 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/aws/kubectl-patch.yaml @@ -0,0 +1,30 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync + namespace: flux-system +spec: + jobTemplate: + spec: + template: + spec: + initContainers: + - image: bitnami/kubectl + name: copy-kubectl + # it's okay to do this because kubectl is a statically linked binary + command: + - sh + - -ceu + - cp $(which kubectl) /kbin/ + resources: {} + volumeMounts: + - name: kbin + mountPath: /kbin + containers: + - name: sync + volumeMounts: + - name: kbin + mountPath: /kbin + volumes: + - name: kbin + emptyDir: {} diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/aws/kustomization.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/aws/kustomization.yaml new file mode 100644 index 00000000..11eea1b4 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/aws/kustomization.yaml @@ -0,0 +1,26 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: ecr- +commonLabels: + app: ecr-credentials-sync + +namespace: flux-system + +bases: +- ../_base +## If not using IRSA, consider creating the following file via SOPS or SealedSecrets +# - encrypted-secret.yaml + +patchesStrategicMerge: +- config-patches.yaml +- kubectl-patch.yaml +- reconcile-patch.yaml + +## uncomment if using encrypted-secret.yaml +# vars: +# - name: ECR_SECRET_NAME +# objref: +# kind: Secret +# name: credentials-sync +# apiVersion: v1 diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/aws/reconcile-patch.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/aws/reconcile-patch.yaml new file mode 100644 index 00000000..98264fed --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/aws/reconcile-patch.yaml @@ -0,0 +1,29 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync + namespace: flux-system +spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: sync + image: mcr.microsoft.com/azure-cli + env: + - name: RECONCILE_SH + value: |- + reconcile() { + echo "Starting ECR token sync -- $(date)" + echo "Logging into ECR: ${ECR_REGION} -- ${ECR_REGISTRY}" + token="$(aws ecr get-login-password --region "${ECR_REGION}")" + user="AWS" + server="${ECR_REGISTRY}" + + echo "Creating secret: ${KUBE_SECRET}" + apply-secret "${KUBE_SECRET}" "${token}" "${user}" "${server}" + + echo "Finished ECR token sync -- $(date)" + echo + } diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/azure/az-identity.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/az-identity.yaml new file mode 100644 index 00000000..c3c6be81 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/az-identity.yaml @@ -0,0 +1,7 @@ +# This is a stub resource patched by config-patches.yaml, so that all config is visible in one file +--- +apiVersion: aadpodidentity.k8s.io/v1 +kind: AzureIdentity +metadata: + name: credentials-sync # if this is changed, also change in config-patches.yaml + namespace: flux-system diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/azure/config-patches.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/config-patches.yaml new file mode 100644 index 00000000..a6428860 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/config-patches.yaml @@ -0,0 +1,41 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync +data: + ACR_NAME: my-registry + KUBE_SECRET: acr-my-registry # does not yet exist -- will be created in the same Namespace + +# Create an identity in Azure and assign it a role to pull from ACR (note: the identity's resourceGroup should match the desired ACR): +# az identity create -n acr-sync +# az role assignment create --role AcrPull --assignee-object-id "$(az identity show -n acr-sync -o tsv --query principalId)" +# Fetch the clientID and resourceID to configure the AzureIdentity spec below: +# az identity show -n acr-sync -otsv --query clientId +# az identity show -n acr-sync -otsv --query resourceId +--- +apiVersion: aadpodidentity.k8s.io/v1 +kind: AzureIdentity +metadata: + name: credentials-sync # name must match the stub-resource in az-identity.yaml + namespace: flux-system +spec: + clientID: 82d01fb0-7799-4d9d-92c7-21e7632c0000 + resourceID: /subscriptions/873c7e7f-76cd-4805-ae86-b923850b0000/resourcegroups/stealthybox/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-sync + type: 0 # user-managed identity + +# Set the reconcile period + specify the pod-identity via the aadpodidbinding label +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync + namespace: flux-system +spec: + schedule: 0 * * * * # ACR tokens expire every 3 hours; refresh faster than that + jobTemplate: + spec: + template: + metadata: + labels: + aadpodidbinding: $(AZ_IDENTITY_NAME) # match the AzureIdentity name diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/azure/kubectl-patch.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/kubectl-patch.yaml new file mode 100644 index 00000000..b4d83e22 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/kubectl-patch.yaml @@ -0,0 +1,30 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync + namespace: flux-system +spec: + jobTemplate: + spec: + template: + spec: + initContainers: + - image: bitnami/kubectl + name: copy-kubectl + # it's okay to do this because kubectl is a statically linked binary + command: + - sh + - -ceu + - cp $(which kubectl) /kbin/ + resources: {} + volumeMounts: + - name: kbin + mountPath: /kbin + containers: + - name: sync + volumeMounts: + - name: kbin + mountPath: /kbin + volumes: + - name: kbin + emptyDir: {} diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/azure/kustomization.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/kustomization.yaml new file mode 100644 index 00000000..1dd497e0 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/kustomization.yaml @@ -0,0 +1,28 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: acr- +commonLabels: + app: acr-credentials-sync + +namespace: flux-system + +bases: +- ../_base +resources: +- az-identity.yaml + +patchesStrategicMerge: +- config-patches.yaml +- kubectl-patch.yaml +- reconcile-patch.yaml + +vars: +- name: AZ_IDENTITY_NAME + objref: + kind: AzureIdentity + name: credentials-sync + apiVersion: aadpodidentity.k8s.io/v1 + +configurations: +- kustomizeconfig.yaml diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/azure/kustomizeconfig.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/kustomizeconfig.yaml new file mode 100644 index 00000000..22524c1d --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/kustomizeconfig.yaml @@ -0,0 +1,3 @@ +varReference: +- path: spec/jobTemplate/spec/template/metadata/labels + kind: Deployment diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/azure/reconcile-patch.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/reconcile-patch.yaml new file mode 100644 index 00000000..b36fa90c --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/azure/reconcile-patch.yaml @@ -0,0 +1,37 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync + namespace: flux-system +spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: sync + image: mcr.microsoft.com/azure-cli + env: + - name: RECONCILE_SH + value: |- + reconcile() { + echo "Starting ACR token sync -- $(date)" + echo "Logging into Azure" + az login --identity + echo "Logging into ACR: ${ACR_NAME}" + output="$(az acr login --expose-token -o=tsv -n "${ACR_NAME}")" + read token server <<< "${output}" + user="00000000-0000-0000-0000-000000000000" + + echo "Creating secret: ${KUBE_SECRET}" + /kbin/kubectl create secret docker-registry "${KUBE_SECRET}" \ + --docker-server="${server}" \ + --docker-username="00000000-0000-0000-0000-000000000000" \ + --docker-password="${token}" \ + --dry-run=client -o=yaml \ + | grep -v "creationTimestamp:" \ + | /kbin/kubectl apply -f - + + echo "Finished ACR token sync -- $(date)" + echo + } diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/gcp/config-patches.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/gcp/config-patches.yaml new file mode 100644 index 00000000..fdbb39d1 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/gcp/config-patches.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync +data: + GCR_REGISTRY: gcr.io # set the registry + KUBE_SECRET: gcr-credentials # does not yet exist -- will be created in the same Namespace + +# Bind to the GCP service-account +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: credentials-sync + namespace: flux-system + annotations: + iam.gke.io/gcp-service-account: @.iam.gserviceaccount.com # set the GCP service-account + +# Set the reconcile period +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync + namespace: flux-system +spec: + schedule: 0,30 * * * * # 30m interval -- GCR tokens expire every hour; refresh faster than that diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/gcp/kustomization.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/gcp/kustomization.yaml new file mode 100644 index 00000000..ea28e0b6 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/gcp/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: gcr- +commonLabels: + app: gcr-credentials-sync + +namespace: flux-system + +bases: +- ../_base + +patchesStrategicMerge: +- config-patches.yaml +- reconcile-patch.yaml diff --git a/manifests/integrations/registry-credentials-sync/_cronjobs/gcp/reconcile-patch.yaml b/manifests/integrations/registry-credentials-sync/_cronjobs/gcp/reconcile-patch.yaml new file mode 100644 index 00000000..5b5ced3f --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/_cronjobs/gcp/reconcile-patch.yaml @@ -0,0 +1,29 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync + namespace: flux-system +spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: sync + image: aws/aws-cli + env: + - name: RECONCILE_SH + value: |- + reconcile() { + echo "Starting GCR token sync -- $(date)" + echo "Logging into ECR: ${ECR_REGION} -- ${ECR_REGISTRY}" + token="$(gcloud auth print-access-token)" + user="oauth2accesstoken " + server="${GCR_REGISTRY}" + + echo "Creating secret: ${KUBE_SECRET}" + apply-secret "${KUBE_SECRET}" "${token}" "${user}" "${server}" + + echo "Finished GCR token sync -- $(date)" + echo + } diff --git a/manifests/integrations/registry-credentials-sync/aws/config-patches.yaml b/manifests/integrations/registry-credentials-sync/aws/config-patches.yaml new file mode 100644 index 00000000..69f12363 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/aws/config-patches.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync +data: + ECR_REGION: us-east-1 # set the region + ECR_REGISTRY: .dkr.ecr..amazonaws.com # fill in the account id and region + KUBE_SECRET: ecr-credentials # does not yet exist -- will be created in the same Namespace + SYNC_PERIOD: "21600" # 6hrs -- ECR tokens expire every 12 hours; refresh faster than that + + +# Bind IRSA for the ServiceAccount +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: credentials-sync + namespace: flux-system + annotations: + eks.amazonaws.com/role-arn: # set the ARN for your role + + +## If not using IRSA, set the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables +## Store these values in a Secret and load them in the container using envFrom. +## For managing this secret via GitOps, consider using SOPS or SealedSecrets and add that manifest in a resource file for this kustomize build. +## https://toolkit.fluxcd.io/guides/mozilla-sops/ +## https://toolkit.fluxcd.io/guides/sealed-secrets/ +# --- +# apiVersion: apps/v1 +# kind: Deployment +# metadata: +# name: credentials-sync +# namespace: flux-system +# spec: +# template: +# spec: +# containers: +# - name: sync +# envFrom: +# secretRef: +# name: $(ECR_SECRET_NAME) # uncomment the var for this in kustomization.yaml diff --git a/manifests/integrations/registry-credentials-sync/aws/kubectl-patch.yaml b/manifests/integrations/registry-credentials-sync/aws/kubectl-patch.yaml new file mode 100644 index 00000000..b054d7ce --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/aws/kubectl-patch.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync + namespace: flux-system +spec: + template: + spec: + initContainers: + - image: bitnami/kubectl + name: copy-kubectl + # it's okay to do this because kubectl is a statically linked binary + command: + - sh + - -ceu + - cp $(which kubectl) /kbin/ + resources: {} + volumeMounts: + - name: kbin + mountPath: /kbin + containers: + - name: sync + volumeMounts: + - name: kbin + mountPath: /kbin + volumes: + - name: kbin + emptyDir: {} diff --git a/manifests/integrations/registry-credentials-sync/aws/kustomization.yaml b/manifests/integrations/registry-credentials-sync/aws/kustomization.yaml new file mode 100644 index 00000000..11eea1b4 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/aws/kustomization.yaml @@ -0,0 +1,26 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: ecr- +commonLabels: + app: ecr-credentials-sync + +namespace: flux-system + +bases: +- ../_base +## If not using IRSA, consider creating the following file via SOPS or SealedSecrets +# - encrypted-secret.yaml + +patchesStrategicMerge: +- config-patches.yaml +- kubectl-patch.yaml +- reconcile-patch.yaml + +## uncomment if using encrypted-secret.yaml +# vars: +# - name: ECR_SECRET_NAME +# objref: +# kind: Secret +# name: credentials-sync +# apiVersion: v1 diff --git a/manifests/integrations/registry-credentials-sync/aws/reconcile-patch.yaml b/manifests/integrations/registry-credentials-sync/aws/reconcile-patch.yaml new file mode 100644 index 00000000..edac3a11 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/aws/reconcile-patch.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync + namespace: flux-system +spec: + template: + spec: + containers: + - name: sync + image: aws/aws-cli + env: + - name: RECONCILE_SH + value: |- + reconcile() { + echo "Starting ECR token sync -- $(date)" + echo "Logging into ECR: ${ECR_REGION} -- ${ECR_REGISTRY}" + token="$(aws ecr get-login-password --region "${ECR_REGION}")" + user="AWS" + server="${ECR_REGISTRY}" + + echo "Creating secret: ${KUBE_SECRET}" + apply-secret "${KUBE_SECRET}" "${token}" "${user}" "${server}" + + echo "Finished ECR token sync -- $(date)" + echo + } diff --git a/manifests/integrations/registry-credentials-sync/azure/az-identity.yaml b/manifests/integrations/registry-credentials-sync/azure/az-identity.yaml new file mode 100644 index 00000000..c3c6be81 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/azure/az-identity.yaml @@ -0,0 +1,7 @@ +# This is a stub resource patched by config-patches.yaml, so that all config is visible in one file +--- +apiVersion: aadpodidentity.k8s.io/v1 +kind: AzureIdentity +metadata: + name: credentials-sync # if this is changed, also change in config-patches.yaml + namespace: flux-system diff --git a/manifests/integrations/registry-credentials-sync/azure/config-patches.yaml b/manifests/integrations/registry-credentials-sync/azure/config-patches.yaml new file mode 100644 index 00000000..d386a497 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/azure/config-patches.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync +data: + ACR_NAME: my-registry + KUBE_SECRET: acr-my-registry # does not yet exist -- will be created in the same Namespace + SYNC_PERIOD: "3600" # ACR tokens expire every 3 hours; refresh faster than that + +# Create an identity in Azure and assign it a role to pull from ACR (note: the identity's resourceGroup should match the desired ACR): +# az identity create -n acr-sync +# az role assignment create --role AcrPull --assignee-object-id "$(az identity show -n acr-sync -o tsv --query principalId)" +# Fetch the clientID and resourceID to configure the AzureIdentity spec below: +# az identity show -n acr-sync -otsv --query clientId +# az identity show -n acr-sync -otsv --query resourceId +--- +apiVersion: aadpodidentity.k8s.io/v1 +kind: AzureIdentity +metadata: + name: credentials-sync # name must match the stub-resource in az-identity.yaml + namespace: flux-system +spec: + clientID: 82d01fb0-7799-4d9d-92c7-21e7632c0000 + resourceID: /subscriptions/873c7e7f-76cd-4805-ae86-b923850b0000/resourcegroups/stealthybox/providers/Microsoft.ManagedIdentity/userAssignedIdentities/acr-sync + type: 0 # user-managed identity + +# Specify the pod-identity via the aadpodidbinding label +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync + namespace: flux-system +spec: + template: + metadata: + labels: + aadpodidbinding: $(AZ_IDENTITY_NAME) # match the AzureIdentity name diff --git a/manifests/integrations/registry-credentials-sync/azure/kubectl-patch.yaml b/manifests/integrations/registry-credentials-sync/azure/kubectl-patch.yaml new file mode 100644 index 00000000..b054d7ce --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/azure/kubectl-patch.yaml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync + namespace: flux-system +spec: + template: + spec: + initContainers: + - image: bitnami/kubectl + name: copy-kubectl + # it's okay to do this because kubectl is a statically linked binary + command: + - sh + - -ceu + - cp $(which kubectl) /kbin/ + resources: {} + volumeMounts: + - name: kbin + mountPath: /kbin + containers: + - name: sync + volumeMounts: + - name: kbin + mountPath: /kbin + volumes: + - name: kbin + emptyDir: {} diff --git a/manifests/integrations/registry-credentials-sync/azure/kustomization.yaml b/manifests/integrations/registry-credentials-sync/azure/kustomization.yaml new file mode 100644 index 00000000..1dd497e0 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/azure/kustomization.yaml @@ -0,0 +1,28 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: acr- +commonLabels: + app: acr-credentials-sync + +namespace: flux-system + +bases: +- ../_base +resources: +- az-identity.yaml + +patchesStrategicMerge: +- config-patches.yaml +- kubectl-patch.yaml +- reconcile-patch.yaml + +vars: +- name: AZ_IDENTITY_NAME + objref: + kind: AzureIdentity + name: credentials-sync + apiVersion: aadpodidentity.k8s.io/v1 + +configurations: +- kustomizeconfig.yaml diff --git a/manifests/integrations/registry-credentials-sync/azure/kustomizeconfig.yaml b/manifests/integrations/registry-credentials-sync/azure/kustomizeconfig.yaml new file mode 100644 index 00000000..afd68fe5 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/azure/kustomizeconfig.yaml @@ -0,0 +1,3 @@ +varReference: +- path: spec/template/metadata/labels + kind: Deployment diff --git a/manifests/integrations/registry-credentials-sync/azure/reconcile-patch.yaml b/manifests/integrations/registry-credentials-sync/azure/reconcile-patch.yaml new file mode 100644 index 00000000..c6e1691d --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/azure/reconcile-patch.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync + namespace: flux-system +spec: + template: + spec: + containers: + - name: sync + image: mcr.microsoft.com/azure-cli + env: + - name: RECONCILE_SH + value: |- + reconcile() { + echo "Starting ACR token sync -- $(date)" + echo "Logging into Azure" + az login --identity + echo "Logging into ACR: ${ACR_NAME}" + output="$(az acr login --expose-token -o=tsv -n "${ACR_NAME}")" + read token server <<< "${output}" + user="00000000-0000-0000-0000-000000000000" + + echo "Creating secret: ${KUBE_SECRET}" + apply-secret "${KUBE_SECRET}" "${token}" "${user}" "${server}" + + echo "Finished ECR token sync -- $(date)" + echo + } diff --git a/manifests/integrations/registry-credentials-sync/gcp/config-patches.yaml b/manifests/integrations/registry-credentials-sync/gcp/config-patches.yaml new file mode 100644 index 00000000..dda354ce --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/gcp/config-patches.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync +data: + GCR_REGISTRY: gcr.io # set the registry + KUBE_SECRET: gcr-credentials # does not yet exist -- will be created in the same Namespace + SYNC_PERIOD: "1800" # 30m -- GCR tokens expire every hour; refresh faster than that + + +# Bind to the GCP service-account +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: credentials-sync + namespace: flux-system + annotations: + iam.gke.io/gcp-service-account: @.iam.gserviceaccount.com # set the GCP service-account diff --git a/manifests/integrations/registry-credentials-sync/gcp/kustomization.yaml b/manifests/integrations/registry-credentials-sync/gcp/kustomization.yaml new file mode 100644 index 00000000..ea28e0b6 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/gcp/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: gcr- +commonLabels: + app: gcr-credentials-sync + +namespace: flux-system + +bases: +- ../_base + +patchesStrategicMerge: +- config-patches.yaml +- reconcile-patch.yaml diff --git a/manifests/integrations/registry-credentials-sync/gcp/reconcile-patch.yaml b/manifests/integrations/registry-credentials-sync/gcp/reconcile-patch.yaml new file mode 100644 index 00000000..9c78e4f4 --- /dev/null +++ b/manifests/integrations/registry-credentials-sync/gcp/reconcile-patch.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync + namespace: flux-system +spec: + template: + spec: + containers: + - name: sync + image: aws/aws-cli + env: + - name: RECONCILE_SH + value: |- + reconcile() { + echo "Starting GCR token sync -- $(date)" + echo "Logging into ECR: ${ECR_REGION} -- ${ECR_REGISTRY}" + token="$(gcloud auth print-access-token)" + user="oauth2accesstoken " + server="${GCR_REGISTRY}" + + echo "Creating secret: ${KUBE_SECRET}" + apply-secret "${KUBE_SECRET}" "${token}" "${user}" "${server}" + + echo "Finished GCR token sync -- $(date)" + echo + }