From 0404790df9dbc1c3b074e61319af917df5a45467 Mon Sep 17 00:00:00 2001 From: Edvin Norling Date: Wed, 5 May 2021 13:40:37 +0200 Subject: [PATCH] How to automatically renew Azure eventhub To use JWT to communicate with Azure eventhub we need to renew the JWT credentials from time to time. This example yaml helps out with that * Supports both deployment and cronjob based renewal * static service principal * aad-pod-identity in azure Signed-off-by: Edvin Norling --- .../_base/kustomization.yaml | 27 ++++ .../_base/kustomizeconfig.yaml | 3 + .../eventhub-credentials-sync/_base/sync.yaml | 133 ++++++++++++++++++ .../_cronjobs/_base/kustomization.yaml | 27 ++++ .../_cronjobs/_base/kustomizeconfig.yaml | 3 + .../_cronjobs/_base/sync.yaml | 109 ++++++++++++++ .../_cronjobs/azure/az-identity.yaml | 16 +++ .../_cronjobs/azure/config-patches.yaml | 50 +++++++ .../_cronjobs/azure/kubectl-patch.yaml | 34 +++++ .../_cronjobs/azure/kustomization.yaml | 28 ++++ .../_cronjobs/azure/kustomizeconfig.yaml | 3 + .../_cronjobs/azure/reconcile-patch.yaml | 27 ++++ .../_cronjobs/generic/config-patches.yaml | 16 +++ .../_cronjobs/generic/kubectl-patch.yaml | 34 +++++ .../_cronjobs/generic/kustomization.yaml | 21 +++ .../_cronjobs/generic/kustomizeconfig.yaml | 3 + .../_cronjobs/generic/reconcile-patch.yaml | 42 ++++++ .../generic/secret-azure-credentials.yaml | 14 ++ .../azure/az-identity.yaml | 16 +++ .../azure/config-patches.yaml | 48 +++++++ .../azure/kubectl-patch.yaml | 32 +++++ .../azure/kustomization.yaml | 28 ++++ .../azure/kustomizeconfig.yaml | 3 + .../azure/reconcile-patch.yaml | 26 ++++ .../generic/config-patches.yaml | 17 +++ .../generic/kubectl-patch.yaml | 32 +++++ .../generic/kustomization.yaml | 21 +++ .../generic/kustomizeconfig.yaml | 3 + .../generic/reconcile-patch.yaml | 41 ++++++ .../generic/secret-azure-credentials.yaml | 14 ++ 30 files changed, 871 insertions(+) create mode 100644 manifests/integrations/eventhub-credentials-sync/_base/kustomization.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/_base/kustomizeconfig.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/_base/sync.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/_cronjobs/_base/kustomization.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/_cronjobs/_base/kustomizeconfig.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/_cronjobs/_base/sync.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/az-identity.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/config-patches.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/kubectl-patch.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/kustomization.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/kustomizeconfig.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/reconcile-patch.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/config-patches.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/kubectl-patch.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/kustomization.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/kustomizeconfig.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/reconcile-patch.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/secret-azure-credentials.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/azure/az-identity.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/azure/config-patches.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/azure/kubectl-patch.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/azure/kustomization.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/azure/kustomizeconfig.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/azure/reconcile-patch.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/generic/config-patches.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/generic/kubectl-patch.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/generic/kustomization.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/generic/kustomizeconfig.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/generic/reconcile-patch.yaml create mode 100644 manifests/integrations/eventhub-credentials-sync/generic/secret-azure-credentials.yaml diff --git a/manifests/integrations/eventhub-credentials-sync/_base/kustomization.yaml b/manifests/integrations/eventhub-credentials-sync/_base/kustomization.yaml new file mode 100644 index 00000000..dfd56766 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/_base/kustomization.yaml @@ -0,0 +1,27 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +commonLabels: + app: credentials-sync-eventhub + +resources: + - sync.yaml + +vars: + - name: KUBE_SECRET + objref: + kind: ConfigMap + name: credentials-sync-eventhub + apiVersion: v1 + fieldref: + fieldpath: data.KUBE_SECRET + - name: ADDRESS + objref: + kind: ConfigMap + name: credentials-sync-eventhub + apiVersion: v1 + fieldref: + fieldpath: data.ADDRESS + +configurations: + - kustomizeconfig.yaml diff --git a/manifests/integrations/eventhub-credentials-sync/_base/kustomizeconfig.yaml b/manifests/integrations/eventhub-credentials-sync/_base/kustomizeconfig.yaml new file mode 100644 index 00000000..61edffd4 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/_base/kustomizeconfig.yaml @@ -0,0 +1,3 @@ +varReference: +- path: rules/resourceNames + kind: Role diff --git a/manifests/integrations/eventhub-credentials-sync/_base/sync.yaml b/manifests/integrations/eventhub-credentials-sync/_base/sync.yaml new file mode 100644 index 00000000..62ea86f0 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/_base/sync.yaml @@ -0,0 +1,133 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync-eventhub +data: + # Patch this ConfigMap with additional values needed for your cloud + KUBE_SECRET: webhook-url # does not yet exist -- will be created in the same Namespace + ADDRESS: "fluxv2" # the Azure Event Hub name + SYNC_PERIOD: "3600" # tokens expire; refresh faster than that + +--- +# This Deployment frequently fetches registry tokens and applies them as an imagePullSecret. +# It's done as a 1-replica Deployment rather than a CronJob, because CronJob scheduling can +# block cluster bootstraps and cold-reboots from obtaining registry tokens for a considerable time. +# This deployment will immediately fetch a token, which reduces latency for working image updates. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync-eventhub + namespace: flux-system +spec: + replicas: 1 + strategy: + type: Recreate + template: + spec: + serviceAccountName: credentials-sync-eventhub + securityContext: + runAsNonRoot: true + runAsUser: 1001 + containers: + - image: busybox # override this with a cloud-specific image + name: sync + envFrom: + - configMapRef: + name: credentials-sync-eventhub + env: + - name: RECONCILE_SH # override this env var with a shell function in a kustomize patch + value: |- + reconcile() { + echo reconciling... + } + command: + - bash + - -ceu + - |- + # template reconcile() into the script + # env var is expanded by k8s before the pod starts + $(RECONCILE_SH) + + apply-secret() { + /kbin/kubectl create secret generic "${1}" \ + --from-literal=token="${2}" \ + --from-literal=address="${3}" \ + --dry-run=client -o=yaml \ + | grep -v "creationTimestamp:" \ + | /kbin/kubectl apply -f - + } + + pause_loop() { + sleep "${SYNC_PERIOD:-3600}" || true + } + + graceful_exit() { + echo "Trapped signal -- $(date)" + job_ids="$( + jobs \ + | grep "pause_loop" \ + | cut -d] -f1 \ + | tr [ % + )" + # shellcheck disable=SC2086 + if [ "${job_ids}" ]; then + kill ${job_ids} + fi + wait + echo "Graceful exit -- $(date)" + } + + trap graceful_exit INT TERM + + echo "Loop started (period: ${SYNC_PERIOD} s) -- $(date)" + while true; do + reconcile & wait $! + pause_loop & wait $! + done + resources: {} + volumeMounts: + - mountPath: /.azure + name: cache-volume + volumes: + - emptyDir: {} + name: cache-volume + +# RBAC necessary for our Deployment to apply our secret that will store the JWT token +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: credentials-sync-eventhub + namespace: flux-system +rules: + - apiGroups: [""] + resources: + - secrets + verbs: + - get + - create + - update + - patch + # # Lock this down to the specific Secret name (Optional) + #resourceNames: + # - $(KUBE_SECRET) # templated from kustomize vars referencing ConfigMap, also see kustomizeconfig.yaml +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: credentials-sync-eventhub + namespace: flux-system +subjects: + - kind: ServiceAccount + name: credentials-sync-eventhub +roleRef: + kind: Role + name: credentials-sync-eventhub + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: credentials-sync-eventhub + namespace: flux-system diff --git a/manifests/integrations/eventhub-credentials-sync/_cronjobs/_base/kustomization.yaml b/manifests/integrations/eventhub-credentials-sync/_cronjobs/_base/kustomization.yaml new file mode 100644 index 00000000..dfd56766 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/_cronjobs/_base/kustomization.yaml @@ -0,0 +1,27 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +commonLabels: + app: credentials-sync-eventhub + +resources: + - sync.yaml + +vars: + - name: KUBE_SECRET + objref: + kind: ConfigMap + name: credentials-sync-eventhub + apiVersion: v1 + fieldref: + fieldpath: data.KUBE_SECRET + - name: ADDRESS + objref: + kind: ConfigMap + name: credentials-sync-eventhub + apiVersion: v1 + fieldref: + fieldpath: data.ADDRESS + +configurations: + - kustomizeconfig.yaml diff --git a/manifests/integrations/eventhub-credentials-sync/_cronjobs/_base/kustomizeconfig.yaml b/manifests/integrations/eventhub-credentials-sync/_cronjobs/_base/kustomizeconfig.yaml new file mode 100644 index 00000000..61edffd4 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/_cronjobs/_base/kustomizeconfig.yaml @@ -0,0 +1,3 @@ +varReference: +- path: rules/resourceNames + kind: Role diff --git a/manifests/integrations/eventhub-credentials-sync/_cronjobs/_base/sync.yaml b/manifests/integrations/eventhub-credentials-sync/_cronjobs/_base/sync.yaml new file mode 100644 index 00000000..e7fd16a7 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/_cronjobs/_base/sync.yaml @@ -0,0 +1,109 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync-eventhub +data: + # Patch this ConfigMap with additional values needed for your cloud + KUBE_SECRET: webhook-url # does not yet exist -- will be created in the same Namespace + ADDRESS: "fluxv2" # the Azure Event Hub name + +--- +# This CronJob frequently fetches registry tokens and applies them as an imagePullSecret. +# note: CronJob scheduling can block cluster bootstraps and cold-reboots from obtaining registry tokens for a considerable time. +# To run the job immediately, do `kubectl create job --from=cronjob/credentials-sync-eventhub -n flux-system credentials-sync-eventhub-init` +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync-eventhub + namespace: flux-system +spec: + suspend: false + schedule: 0 */6 * * * + failedJobsHistoryLimit: 1 + successfulJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + serviceAccountName: credentials-sync-eventhub + securityContext: + runAsNonRoot: true + runAsUser: 1001 + restartPolicy: Never + containers: + - image: busybox # override this with a cloud-specific image + name: sync + envFrom: + - configMapRef: + name: credentials-sync-eventhub + env: + - name: RECONCILE_SH # override this env var with a shell function in a kustomize patch + value: |- + reconcile() { + echo reconciling... + } + command: + - bash + - -ceu + - |- + # template reconcile() into the script + # env var is expanded by k8s before the pod starts + $(RECONCILE_SH) + + apply-secret() { + /kbin/kubectl create secret generic "${1}" \ + --from-literal=token="${2}" \ + --from-literal=address="${3}" \ + --dry-run=client -o=yaml \ + | grep -v "creationTimestamp:" \ + | /kbin/kubectl apply -f - + } + + reconcile + resources: {} + volumeMounts: + - mountPath: /.azure + name: cache-volume + volumes: + - emptyDir: {} + name: cache-volume + +# RBAC necessary for our Deployment to apply our secret that will store the JWT token +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: credentials-sync-eventhub + namespace: flux-system +rules: + - apiGroups: [""] + resources: + - secrets + verbs: + - get + - create + - update + - patch + # # Lock this down to the specific Secret name (Optional) + #resourceNames: + # - $(KUBE_SECRET) # templated from kustomize vars referencing ConfigMap, also see kustomizeconfig.yaml +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: credentials-sync-eventhub + namespace: flux-system +subjects: + - kind: ServiceAccount + name: credentials-sync-eventhub +roleRef: + kind: Role + name: credentials-sync-eventhub + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: credentials-sync-eventhub + namespace: flux-system diff --git a/manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/az-identity.yaml b/manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/az-identity.yaml new file mode 100644 index 00000000..1591126b --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/az-identity.yaml @@ -0,0 +1,16 @@ +# This is a stub resource patched by config-patches.yaml, so that all config is visible in one file +--- +apiVersion: aadpodidentity.k8s.io/v1 +kind: AzureIdentity +metadata: + name: lab # if this is changed, also change in config-patches.yaml + namespace: flux-system +--- +apiVersion: aadpodidentity.k8s.io/v1 +kind: AzureIdentityBinding +metadata: + name: lab + namespace: flux-system +spec: + azureIdentity: lab + selector: lab diff --git a/manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/config-patches.yaml b/manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/config-patches.yaml new file mode 100644 index 00000000..3d0ffac4 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/config-patches.yaml @@ -0,0 +1,50 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync-eventhub +data: + KUBE_SECRET: webhook-url # does not yet exist -- will be created in the same Namespace + ADDRESS: "fluxv2" # the Azure Event Hub name + +# Create an identity in Azure and assign it a role to write to Azure Event Hub (note: the identity's resourceGroup should match the Azure Event Hub): +# az identity create -n eventhub-write +# az role assignment create --role eventhub --assignee-object-id "$(az identity show -n eventhub-write -o tsv --query principalId)" +# Fetch the clientID and resourceID to configure the AzureIdentity spec below: +# az identity show -n eventhub-write -otsv --query clientId +# az identity show -n eventhub-write -otsv --query resourceId +--- +apiVersion: aadpodidentity.k8s.io/v1 +kind: AzureIdentity +metadata: + name: lab + namespace: flux-system +spec: + clientID: 82d01fb0-7799-4d9d-92c7-21e7632c0000 + resourceID: /subscriptions/82d01fb0-7799-4d9d-92c7-21e7632c0000/resourceGroups/stealthybox/providers/Microsoft.ManagedIdentity/userAssignedIdentities/eventhub-write + type: 0 +--- +apiVersion: aadpodidentity.k8s.io/v1 +kind: AzureIdentityBinding +metadata: + name: lab + namespace: flux-system +spec: + azureIdentity: jwt-lab + selector: jwt-lab + +# Set the reconcile period + specify the pod-identity via the aadpodidbinding label +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync-eventhub + namespace: flux-system +spec: + schedule: 0 * * * * # JWT tokens expire every 24 hours; refresh faster than that + jobTemplate: + spec: + template: + metadata: + labels: + aadpodidbinding: $(AZ_IDENTITY_NAME) # match the AzureIdentity name diff --git a/manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/kubectl-patch.yaml b/manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/kubectl-patch.yaml new file mode 100644 index 00000000..d05c07e5 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/kubectl-patch.yaml @@ -0,0 +1,34 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync-eventhub + namespace: flux-system +spec: + jobTemplate: + spec: + template: + spec: + initContainers: + - image: bitnami/kubectl + securityContext: + privileged: false + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + name: copy-kubectl + # it's okay to do this because kubectl is a statically linked binary + command: + - sh + - -ceu + - cp $(which kubectl) /kbin/ + resources: {} + volumeMounts: + - name: kbin + mountPath: /kbin + containers: + - name: sync + volumeMounts: + - name: kbin + mountPath: /kbin + volumes: + - name: kbin + emptyDir: {} diff --git a/manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/kustomization.yaml b/manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/kustomization.yaml new file mode 100644 index 00000000..14a0d59f --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/kustomization.yaml @@ -0,0 +1,28 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: jwt- +commonLabels: + app: jwt-eventhub-credentials-sync + +namespace: flux-system + +bases: + - ../_base +resources: + - az-identity.yaml + +patchesStrategicMerge: + - config-patches.yaml + - kubectl-patch.yaml + - reconcile-patch.yaml + +vars: + - name: AZ_IDENTITY_NAME + objref: + kind: AzureIdentity + name: lab + apiVersion: aadpodidentity.k8s.io/v1 + +configurations: + - kustomizeconfig.yaml diff --git a/manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/kustomizeconfig.yaml b/manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/kustomizeconfig.yaml new file mode 100644 index 00000000..175f04a2 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/kustomizeconfig.yaml @@ -0,0 +1,3 @@ +varReference: + - path: spec/jobTemplate/spec/template/metadata/labels + kind: CronJob diff --git a/manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/reconcile-patch.yaml b/manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/reconcile-patch.yaml new file mode 100644 index 00000000..1e96e536 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/_cronjobs/azure/reconcile-patch.yaml @@ -0,0 +1,27 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync-eventhub + namespace: flux-system +spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: sync + image: mcr.microsoft.com/azure-cli + env: + - name: RECONCILE_SH + value: |- + reconcile() { + echo "Starting JWT token sync -- $(date)" + echo "Logging into Azure" + az login --identity + echo "Getting JWT token" + token=$(az account get-access-token --resource https://eventhubs.azure.net |jq -r .accessToken) + echo "Creating secret: ${KUBE_SECRET}" + apply-secret "${KUBE_SECRET}" ${token} "${ADDRESS}" + echo "Finished JWT token sync -- $(date)" + echo + } diff --git a/manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/config-patches.yaml b/manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/config-patches.yaml new file mode 100644 index 00000000..b2374ac0 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/config-patches.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync-eventhub + namespace: flux-system +data: + KUBE_SECRET: webhook-url # does not yet exist -- will be created in the same Namespace + ADDRESS: "fluxv2" # the Azure Event Hub name + +# Create an identity in Azure and assign it a role to write to Azure Event Hub (note: the identity's resourceGroup should match the Azure Event Hub): +# az identity create -n eventhub-write +# az role assignment create --role eventhub --assignee-object-id "$(az identity show -n eventhub-write -o tsv --query principalId)" +# Fetch the clientID and resourceID to configure the AzureIdentity spec below: +# az identity show -n eventhub-write -otsv --query clientId +# az identity show -n eventhub-write -otsv --query resourceId diff --git a/manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/kubectl-patch.yaml b/manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/kubectl-patch.yaml new file mode 100644 index 00000000..d05c07e5 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/kubectl-patch.yaml @@ -0,0 +1,34 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync-eventhub + namespace: flux-system +spec: + jobTemplate: + spec: + template: + spec: + initContainers: + - image: bitnami/kubectl + securityContext: + privileged: false + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + name: copy-kubectl + # it's okay to do this because kubectl is a statically linked binary + command: + - sh + - -ceu + - cp $(which kubectl) /kbin/ + resources: {} + volumeMounts: + - name: kbin + mountPath: /kbin + containers: + - name: sync + volumeMounts: + - name: kbin + mountPath: /kbin + volumes: + - name: kbin + emptyDir: {} diff --git a/manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/kustomization.yaml b/manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/kustomization.yaml new file mode 100644 index 00000000..109f3a07 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/kustomization.yaml @@ -0,0 +1,21 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: jwt- +commonLabels: + app: jwt-eventhub-credentials-sync + +namespace: flux-system + +bases: + - ../_base +resources: + - secret-azure-credentials.yaml + +patchesStrategicMerge: + - config-patches.yaml + - kubectl-patch.yaml + - reconcile-patch.yaml + +configurations: + - kustomizeconfig.yaml diff --git a/manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/kustomizeconfig.yaml b/manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/kustomizeconfig.yaml new file mode 100644 index 00000000..175f04a2 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/kustomizeconfig.yaml @@ -0,0 +1,3 @@ +varReference: + - path: spec/jobTemplate/spec/template/metadata/labels + kind: CronJob diff --git a/manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/reconcile-patch.yaml b/manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/reconcile-patch.yaml new file mode 100644 index 00000000..67eebfc9 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/reconcile-patch.yaml @@ -0,0 +1,42 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: credentials-sync-eventhub + namespace: flux-system +spec: + jobTemplate: + spec: + template: + spec: + containers: + - name: sync + image: mcr.microsoft.com/azure-cli + env: + - name: RECONCILE_SH + value: |- + reconcile() { + echo "Starting JWT token sync -- $(date)" + echo "Logging into Azure" + az login --service-principal -u ${AZURE_CLIENT_ID} -p ${AZURE_CLIENT_SECRET} --tenant ${AZURE_TENANT_ID} + echo "Getting JWT token" + token=$(az account get-access-token --resource https://eventhubs.azure.net |jq -r .accessToken) + echo "Creating secret: ${KUBE_SECRET}" + apply-secret "${KUBE_SECRET}" ${token} "${ADDRESS}" + echo "Finished JWT token sync -- $(date)" + echo + } + - name: AZURE_CLIENT_ID + valueFrom: + secretKeyRef: + name: azure-credentials + key: AZURE_CLIENT_ID + - name: AZURE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: azure-credentials + key: AZURE_CLIENT_SECRET + - name: AZURE_TENANT_ID + valueFrom: + secretKeyRef: + name: azure-credentials + key: AZURE_TENANT_ID diff --git a/manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/secret-azure-credentials.yaml b/manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/secret-azure-credentials.yaml new file mode 100644 index 00000000..8a6d8a2c --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/_cronjobs/generic/secret-azure-credentials.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +data: + AZURE_CLIENT_ID: MDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMA== + AZURE_CLIENT_SECRET: c28tbXVjaC1zZWNyZXQ= + AZURE_TENANT_ID: MDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMA== +kind: Secret +metadata: + name: azure-credentials + namespace: flux-system +type: Opaque +# This is just a example secret, you should never store secrets in git. +# One way forward can be to use sealed-secrets or SOPS +# https://fluxcd.io/docs/guides/sealed-secrets/ +# https://fluxcd.io/docs/guides/mozilla-sops/ diff --git a/manifests/integrations/eventhub-credentials-sync/azure/az-identity.yaml b/manifests/integrations/eventhub-credentials-sync/azure/az-identity.yaml new file mode 100644 index 00000000..1591126b --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/azure/az-identity.yaml @@ -0,0 +1,16 @@ +# This is a stub resource patched by config-patches.yaml, so that all config is visible in one file +--- +apiVersion: aadpodidentity.k8s.io/v1 +kind: AzureIdentity +metadata: + name: lab # if this is changed, also change in config-patches.yaml + namespace: flux-system +--- +apiVersion: aadpodidentity.k8s.io/v1 +kind: AzureIdentityBinding +metadata: + name: lab + namespace: flux-system +spec: + azureIdentity: lab + selector: lab diff --git a/manifests/integrations/eventhub-credentials-sync/azure/config-patches.yaml b/manifests/integrations/eventhub-credentials-sync/azure/config-patches.yaml new file mode 100644 index 00000000..c285ed2c --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/azure/config-patches.yaml @@ -0,0 +1,48 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync-eventhub +data: + KUBE_SECRET: webhook-url # does not yet exist -- will be created in the same Namespace + ADDRESS: "fluxv2" # the Azure Event Hub name + SYNC_PERIOD: "3600" # tokens expire; refresh faster than that + +# Create an identity in Azure and assign it a role to write to Azure Event Hub (note: the identity's resourceGroup should match the Azure Event Hub): +# az identity create -n eventhub-write +# az role assignment create --role eventhub --assignee-object-id "$(az identity show -n eventhub-write -o tsv --query principalId)" +# Fetch the clientID and resourceID to configure the AzureIdentity spec below: +# az identity show -n eventhub-write -otsv --query clientId +# az identity show -n eventhub-write -otsv --query resourceId +--- +apiVersion: aadpodidentity.k8s.io/v1 +kind: AzureIdentity +metadata: + name: lab + namespace: flux-system +spec: + clientID: 82d01fb0-7799-4d9d-92c7-21e7632c0000 + resourceID: /subscriptions/82d01fb0-7799-4d9d-92c7-21e7632c0000/resourceGroups/stealthybox/providers/Microsoft.ManagedIdentity/userAssignedIdentities/eventhub-write + type: 0 +--- +apiVersion: aadpodidentity.k8s.io/v1 +kind: AzureIdentityBinding +metadata: + name: lab + namespace: flux-system +spec: + azureIdentity: jwt-lab + selector: jwt-lab + +# Specify the pod-identity via the aadpodidbinding label +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync-eventhub + namespace: flux-system +spec: + template: + metadata: + labels: + aadpodidbinding: $(AZ_IDENTITY_NAME) # match the AzureIdentity name diff --git a/manifests/integrations/eventhub-credentials-sync/azure/kubectl-patch.yaml b/manifests/integrations/eventhub-credentials-sync/azure/kubectl-patch.yaml new file mode 100644 index 00000000..65226a0f --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/azure/kubectl-patch.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync-eventhub + namespace: flux-system +spec: + template: + spec: + initContainers: + - image: bitnami/kubectl + securityContext: + privileged: false + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + name: copy-kubectl + # it's okay to do this because kubectl is a statically linked binary + command: + - sh + - -ceu + - cp $(which kubectl) /kbin/ + resources: {} + volumeMounts: + - name: kbin + mountPath: /kbin + containers: + - name: sync + volumeMounts: + - name: kbin + mountPath: /kbin + volumes: + - name: kbin + emptyDir: {} diff --git a/manifests/integrations/eventhub-credentials-sync/azure/kustomization.yaml b/manifests/integrations/eventhub-credentials-sync/azure/kustomization.yaml new file mode 100644 index 00000000..14a0d59f --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/azure/kustomization.yaml @@ -0,0 +1,28 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: jwt- +commonLabels: + app: jwt-eventhub-credentials-sync + +namespace: flux-system + +bases: + - ../_base +resources: + - az-identity.yaml + +patchesStrategicMerge: + - config-patches.yaml + - kubectl-patch.yaml + - reconcile-patch.yaml + +vars: + - name: AZ_IDENTITY_NAME + objref: + kind: AzureIdentity + name: lab + apiVersion: aadpodidentity.k8s.io/v1 + +configurations: + - kustomizeconfig.yaml diff --git a/manifests/integrations/eventhub-credentials-sync/azure/kustomizeconfig.yaml b/manifests/integrations/eventhub-credentials-sync/azure/kustomizeconfig.yaml new file mode 100644 index 00000000..afd68fe5 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/azure/kustomizeconfig.yaml @@ -0,0 +1,3 @@ +varReference: +- path: spec/template/metadata/labels + kind: Deployment diff --git a/manifests/integrations/eventhub-credentials-sync/azure/reconcile-patch.yaml b/manifests/integrations/eventhub-credentials-sync/azure/reconcile-patch.yaml new file mode 100644 index 00000000..88d9086a --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/azure/reconcile-patch.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync-eventhub + namespace: flux-system +spec: + template: + spec: + containers: + - name: sync + image: mcr.microsoft.com/azure-cli + env: + - name: RECONCILE_SH + value: |- + reconcile() { + echo "Starting JWT token sync -- $(date)" + echo "Logging into Azure" + az login --identity + echo "Getting JWT token" + token=$(az account get-access-token --resource https://eventhubs.azure.net |jq -r .accessToken) + echo "Creating secret: ${KUBE_SECRET}" + apply-secret "${KUBE_SECRET}" ${token} "${ADDRESS}" + echo "Finished JWT token sync -- $(date)" + echo + } diff --git a/manifests/integrations/eventhub-credentials-sync/generic/config-patches.yaml b/manifests/integrations/eventhub-credentials-sync/generic/config-patches.yaml new file mode 100644 index 00000000..9c1ca794 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/generic/config-patches.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: credentials-sync-eventhub +data: + KUBE_SECRET: webhook-url # does not yet exist -- will be created in the same Namespace + ADDRESS: "fluxv2" # the Azure Event Hub name + SYNC_PERIOD: "3600" # tokens expire; refresh faster than that + +# Create an identity in Azure and assign it a role to write to Azure Event Hub (note: the identity's resourceGroup should match the Azure Event Hub): +# az identity create -n eventhub-write +# az role assignment create --role eventhub --assignee-object-id "$(az identity show -n eventhub-write -o tsv --query principalId)" +# Fetch the clientID and resourceID to configure the AzureIdentity spec below: +# az identity show -n eventhub-write -otsv --query clientId +# az identity show -n eventhub-write -otsv --query resourceId +# Specify the pod-identity via the aadpodidbinding label diff --git a/manifests/integrations/eventhub-credentials-sync/generic/kubectl-patch.yaml b/manifests/integrations/eventhub-credentials-sync/generic/kubectl-patch.yaml new file mode 100644 index 00000000..65226a0f --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/generic/kubectl-patch.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync-eventhub + namespace: flux-system +spec: + template: + spec: + initContainers: + - image: bitnami/kubectl + securityContext: + privileged: false + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + name: copy-kubectl + # it's okay to do this because kubectl is a statically linked binary + command: + - sh + - -ceu + - cp $(which kubectl) /kbin/ + resources: {} + volumeMounts: + - name: kbin + mountPath: /kbin + containers: + - name: sync + volumeMounts: + - name: kbin + mountPath: /kbin + volumes: + - name: kbin + emptyDir: {} diff --git a/manifests/integrations/eventhub-credentials-sync/generic/kustomization.yaml b/manifests/integrations/eventhub-credentials-sync/generic/kustomization.yaml new file mode 100644 index 00000000..109f3a07 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/generic/kustomization.yaml @@ -0,0 +1,21 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namePrefix: jwt- +commonLabels: + app: jwt-eventhub-credentials-sync + +namespace: flux-system + +bases: + - ../_base +resources: + - secret-azure-credentials.yaml + +patchesStrategicMerge: + - config-patches.yaml + - kubectl-patch.yaml + - reconcile-patch.yaml + +configurations: + - kustomizeconfig.yaml diff --git a/manifests/integrations/eventhub-credentials-sync/generic/kustomizeconfig.yaml b/manifests/integrations/eventhub-credentials-sync/generic/kustomizeconfig.yaml new file mode 100644 index 00000000..afd68fe5 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/generic/kustomizeconfig.yaml @@ -0,0 +1,3 @@ +varReference: +- path: spec/template/metadata/labels + kind: Deployment diff --git a/manifests/integrations/eventhub-credentials-sync/generic/reconcile-patch.yaml b/manifests/integrations/eventhub-credentials-sync/generic/reconcile-patch.yaml new file mode 100644 index 00000000..89953444 --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/generic/reconcile-patch.yaml @@ -0,0 +1,41 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: credentials-sync-eventhub + namespace: flux-system +spec: + template: + spec: + containers: + - name: sync + image: mcr.microsoft.com/azure-cli + env: + - name: RECONCILE_SH + value: |- + reconcile() { + echo "Starting JWT token sync -- $(date)" + echo "Logging into Azure" + az login --service-principal -u ${AZURE_CLIENT_ID} -p ${AZURE_CLIENT_SECRET} --tenant ${AZURE_TENANT_ID} + echo "Getting JWT token" + token=$(az account get-access-token --resource https://eventhubs.azure.net |jq -r .accessToken) + echo "Creating secret: ${KUBE_SECRET}" + apply-secret "${KUBE_SECRET}" ${token} "${ADDRESS}" + echo "Finished JWT token sync -- $(date)" + echo + } + - name: AZURE_CLIENT_ID + valueFrom: + secretKeyRef: + name: azure-credentials + key: AZURE_CLIENT_ID + - name: AZURE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: azure-credentials + key: AZURE_CLIENT_SECRET + - name: AZURE_TENANT_ID + valueFrom: + secretKeyRef: + name: azure-credentials + key: AZURE_TENANT_ID diff --git a/manifests/integrations/eventhub-credentials-sync/generic/secret-azure-credentials.yaml b/manifests/integrations/eventhub-credentials-sync/generic/secret-azure-credentials.yaml new file mode 100644 index 00000000..8a6d8a2c --- /dev/null +++ b/manifests/integrations/eventhub-credentials-sync/generic/secret-azure-credentials.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +data: + AZURE_CLIENT_ID: MDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMA== + AZURE_CLIENT_SECRET: c28tbXVjaC1zZWNyZXQ= + AZURE_TENANT_ID: MDAwMDAwMDAtMDAwMC0wMDAwLTAwMDAtMDAwMDAwMA== +kind: Secret +metadata: + name: azure-credentials + namespace: flux-system +type: Opaque +# This is just a example secret, you should never store secrets in git. +# One way forward can be to use sealed-secrets or SOPS +# https://fluxcd.io/docs/guides/sealed-secrets/ +# https://fluxcd.io/docs/guides/mozilla-sops/