Refactor ARM64 e2e test suite

- Migrate the GitHub runners to Equinix c3.large.arm64 instances
- Run the test suite on all supported Kubernetes versions
- Add multi-tenancy smoke test

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>

.github/workflows/e2e-arm64.yaml

@@ -3,7 +3,7 @@ name: e2e-arm64
 on:
   workflow_dispatch:
   push:
-    branches: [ main, update-components ]
+    branches: [ main, update-components, e2e-arm64* ]
 
 permissions:
   contents: read
@@ -13,6 +13,10 @@ jobs:
     # Hosted on Equinix
     # Docs: https://github.com/fluxcd/flux2/tree/main/.github/runners
     runs-on: [self-hosted, Linux, ARM64, equinix]
+    strategy:
+      matrix:
+        # Keep this list up-to-date with https://endoflife.date/kubernetes
+        KUBERNETES_VERSION: [ 1.23.13, 1.24.7, 1.25.3 ]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -23,16 +27,46 @@ jobs:
       - name: Prepare
         id: prep
         run: |
-          echo ::set-output name=CLUSTER::arm64-${GITHUB_SHA:0:7}-$(date +%s)
-          echo ::set-output name=CONTEXT::kind-arm64-${GITHUB_SHA:0:7}-$(date +%s)
+          ID=${GITHUB_SHA:0:7}-${{ matrix.KUBERNETES_VERSION }}-$(date +%s)
+          echo "CLUSTER=arm64-${ID}" >> $GITHUB_OUTPUT
       - name: Build
         run: |
           make build
       - name: Setup Kubernetes Kind
         run: |
-          kind create cluster --name ${{ steps.prep.outputs.CLUSTER }} --kubeconfig=/tmp/${{ steps.prep.outputs.CLUSTER }}
+          kind create cluster \
+          --wait 5m \
+          --name ${{ steps.prep.outputs.CLUSTER }} \
+          --kubeconfig=/tmp/${{ steps.prep.outputs.CLUSTER }} \
+          --image=kindest/node:v${{ matrix.KUBERNETES_VERSION }}
       - name: Run e2e tests
         run: TEST_KUBECONFIG=/tmp/${{ steps.prep.outputs.CLUSTER }} make e2e
+      - name: Run multi-tenancy tests
+        env:
+          KUBECONFIG: /tmp/${{ steps.prep.outputs.CLUSTER }}
+        run: |
+          ./bin/flux install
+          ./bin/flux create source git flux-system \
+          --interval=15m \
+          --url=https://github.com/fluxcd/flux2-multi-tenancy \
+          --branch=main \
+          --ignore-paths="./clusters/**/flux-system/"
+          ./bin/flux create kustomization flux-system \
+          --interval=15m \
+          --source=flux-system \
+          --path=./clusters/staging
+          kubectl -n flux-system wait kustomization/tenants --for=condition=ready --timeout=5m
+          kubectl -n apps wait kustomization/dev-team --for=condition=ready --timeout=1m
+          kubectl -n apps wait helmrelease/podinfo --for=condition=ready --timeout=1m
+      - name: Debug failure
+        if: failure()
+        env:
+          KUBECONFIG: /tmp/${{ steps.prep.outputs.CLUSTER }}
+        run: |
+          kubectl -n flux-system get all
+          kubectl -n flux-system describe po 
+          kubectl -n flux-system logs deploy/source-controller
+          kubectl -n flux-system logs deploy/kustomize-controller
       - name: Cleanup
         if: always()
         run: |