Add Azure E2E tests

Signed-off-by: Philip Laine <philip.laine@xenit.se>

.github/workflows/e2e-azure.yaml

@@ -0,0 +1,66 @@
+name: e2e-azure
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron:  '0 6 * * *'
+  push:
+    branches: [ azure/e2e ]
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Restore Go cache
+        uses: actions/cache@v1
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go1.16-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go1.16-
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16.x
+      - name: Install libgit2
+        run: |
+          sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 648ACFD622F3D138
+          sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 0E98404D386FA1D9
+          echo "deb http://deb.debian.org/debian unstable main" | sudo tee -a /etc/apt/sources.list
+          echo "deb-src http://deb.debian.org/debian unstable main" | sudo tee -a /etc/apt/sources.list
+          sudo apt-get update
+          sudo apt-get install -y --allow-downgrades libgit2-dev/unstable zlib1g-dev/unstable libssh2-1-dev/unstable libpcre3-dev/unstable
+      - name: Setup Flux CLI
+        run: |
+          make build
+          mkdir -p $HOME/.local/bin
+          mv ./bin/flux $HOME/.local/bin
+      - name: Setup SOPS
+        run: |
+          wget https://github.com/mozilla/sops/releases/download/v3.7.1/sops-v3.7.1.linux
+          chmod +x sops-v3.7.1.linux
+          mkdir -p $HOME/.local/bin
+          mv sops-v3.7.1.linux $HOME/.local/bin/sops
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v1
+        with:
+          terraform_version: 1.0.7
+          terraform_wrapper: false
+      - name: Setup Azure CLI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+      - name: Run Azure e2e tests
+        env:
+          ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
+          ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
+          ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
+          ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
+        run: |
+          echo $HOME
+          echo $PATH
+          ls $HOME/.local/bin
+          az login --service-principal -u ${ARM_CLIENT_ID} -p ${ARM_CLIENT_SECRET} -t ${ARM_TENANT_ID}
+          cd ./tests/azure
+          go test -timeout 60m .

tests/.gitignore

@@ -0,0 +1,40 @@
+
+# Created by https://www.toptal.com/developers/gitignore/api/terraform
+# Edit at https://www.toptal.com/developers/gitignore?templates=terraform
+
+### Terraform ###
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+
+# Exclude all .tfvars files, which are likely to contain sentitive data, such as
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
+# to change depending on the environment.
+#
+*.tfvars
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc
+
+# End of https://www.toptal.com/developers/gitignore/api/terraform

tests/azure/README.md

@@ -0,0 +1,51 @@
+# Azure E2E
+
+E2E tests for Azure are needed to mitigate introduction of new bugs in dependencies like libgit2. The goal is to verify that Flux integration with
+Azure services are actually working now and in the future.
+
+## Architecture
+
+The tests are run with the help of pre configured Azure subscriptions and Azure DevOps organization. Access to thse accounts are currently limited to
+Flux maintainers.
+* [Azure Subscription](https://portal.azure.com/#@weaveworksendtoend.onmicrosoft.com/resource/subscriptions/71e8dce4-9af6-405a-8e96-425f5d3c302b/overview)
+* [Azure DevOps organization](https://dev.azure.com/flux-azure/)
+
+All infrastructure is and should be created with Terraform. There are two separate Terraform states. All state should be configured to use remote
+state in Azure. They should all be placed in the [same container](https://portal.azure.com/#@weaveworksendtoend.onmicrosoft.com/resource/subscriptions/71e8dce4-9af6-405a-8e96-425f5d3c302b/resourceGroups/terraform-state/providers/Microsoft.Storage/storageAccounts/terraformstate0419/containersList)
+but use different keys.
+
+The [shared](./terraform/shared) Terraform creates long running cheaper infrastructure that is used across all tests. This includes a Key Vault which
+contains an ssh key and Azure DevOps Personal Access Token which cant be created automatically. It also includes an Azure Container Registry which the
+forked [podinfo](https://dev.azure.com/flux-azure/e2e/_git/podinfo) repository pushes an Helm Chart and Docker image to.
+
+The [aks](./terraform/aks) Terraform creates the AKS cluster and related resources to run the tests. It creates the AKS cluster, Azure DevOps
+repositories, Key Vault Key for Sops, and Azure EventHub. The resources should be created and destroyed before and after every test run. Currently
+the same state is reused between runs to make sure that resources are left running after each test run.
+
+## Tests
+
+Each test run is intiated by running `terraform apply` on the aks Terraform, it does this by using the library [terraform-exec](github.com/hashicorp/terraform-exec).
+It then reads the output of the Terraform to get credentials and ssh keys, this means that a lot of the communication with the Azure API is offest to
+Terraform instead of requiring it to be implemented in the test.
+
+The following tests are currently implemented:
+
+[x] Flux can be successfully installed on AKS using the CLI e.g.:
+[x] source-controller can clone Azure DevOps repositories (https+ssh)
+[x] image-reflector-controller can list tags from Azure Container Registry image repositories
+[x] kustomize-controller can decrypt secrets using SOPS and Azure Key Vault
+[x] image-automation-controller can create branches and push to Azure DevOps repositories (https+ssh)
+[x] notification-controller can send commit status to Azure DevOps
+[x] notification-controller can forward events to Azure Event Hub
+[x] source-controller can pull charts from Azure Container Registry Helm repositories
+
+## Give User Access
+
+There are a couple of steps required when adding a new user to get access to the Azure portal and Azure DevOps. To begin with add the new user to
+[Azure AD](https://portal.azure.com/#blade/Microsoft_AAD_IAM/UsersManagementMenuBlade/MsGraphUsers), and add the user to the [Azure AD group
+flux-contributors](https://portal.azure.com/#blade/Microsoft_AAD_IAM/GroupDetailsMenuBlade/Overview/groupId/24e0f3f6-6555-4d3d-99ab-414c869cab5d). The
+new users should now go through the invite process, and be able to sign in to both the Azure Portal and Azure DevOps.
+
+After the new user has signed into Azure DevOps you will need to modify the users permissions. This cannot be done before the user has signed in a
+first time. In the [organization users page](https://dev.azure.com/flux-azure/_settings/users) chose "Manage User" and set the "Access Level" to basic
+and make the user "Project Contributor" of the "e2e" Azure DevOps project.

tests/azure/azure_test.go

@@ -0,0 +1,883 @@
+package test
+
+import (
+	"bytes"
+	"context"
+	b64 "encoding/base64"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	eventhub "github.com/Azure/azure-event-hubs-go/v3"
+	"github.com/hashicorp/terraform-exec/tfexec"
+	"github.com/hashicorp/terraform-exec/tfinstall"
+	git2go "github.com/libgit2/git2go/v31"
+	"github.com/microsoft/azure-devops-go-api/azuredevops"
+	"github.com/microsoft/azure-devops-go-api/azuredevops/git"
+	"github.com/stretchr/testify/require"
+	giturls "github.com/whilp/git-urls"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	automationv1beta1 "github.com/fluxcd/image-automation-controller/api/v1beta1"
+	reflectorv1beta1 "github.com/fluxcd/image-reflector-controller/api/v1beta1"
+	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1"
+	notiv1beta1 "github.com/fluxcd/notification-controller/api/v1beta1"
+	"github.com/fluxcd/pkg/apis/meta"
+	"github.com/fluxcd/pkg/runtime/events"
+	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
+)
+
+type config struct {
+	kubeconfigPath string
+	kubeClient     client.Client
+
+	azdoPat               string
+	idRsa                 string
+	idRsaPub              string
+	knownHosts            string
+	fleetInfraRepository  repoConfig
+	applicationRepository repoConfig
+
+	fluxAzureSp spConfig
+	sopsId      string
+	acr         acrConfig
+	eventHubSas string
+}
+
+type spConfig struct {
+	tenantId     string
+	clientId     string
+	clientSecret string
+}
+
+type repoConfig struct {
+	http string
+	ssh  string
+}
+
+type acrConfig struct {
+	url      string
+	username string
+	password string
+}
+
+var cfg config
+
+func TestMain(m *testing.M) {
+	ctx := context.TODO()
+
+	log.Println("Setting up Azure test infrastructure")
+	execPath, err := tfinstall.Find(ctx, &whichTerraform{})
+	if err != nil {
+		log.Fatalf("terraform exec path not found: %v", err)
+	}
+	tf, err := tfexec.NewTerraform("./terraform/aks", execPath)
+	if err != nil {
+		log.Fatalf("could not create terraform instance: %v", err)
+	}
+	log.Println("Init Terraform")
+	err = tf.Init(ctx, tfexec.Upgrade(true))
+	if err != nil {
+		log.Fatalf("error running init: %v", err)
+	}
+	log.Println("Applying Terraform")
+	err = tf.Apply(ctx)
+	if err != nil {
+		log.Fatalf("error running apply: %v", err)
+	}
+	state, err := tf.Show(ctx)
+	if err != nil {
+		log.Fatalf("error running show: %v", err)
+	}
+	outputs := state.Values.Outputs
+	kubeconfig := outputs["aks_kube_config"].Value.(string)
+	aksHost := outputs["aks_host"].Value.(string)
+	aksCert := outputs["aks_client_certificate"].Value.(string)
+	aksKey := outputs["aks_client_key"].Value.(string)
+	aksCa := outputs["aks_cluster_ca_certificate"].Value.(string)
+	azdoPat := outputs["shared_pat"].Value.(string)
+	idRsa := outputs["shared_id_rsa"].Value.(string)
+	idRsaPub := outputs["shared_id_rsa_pub"].Value.(string)
+	fleetInfraRepository := outputs["fleet_infra_repository"].Value.(map[string]interface{})
+	applicationRepository := outputs["application_repository"].Value.(map[string]interface{})
+	fluxAzureSp := outputs["flux_azure_sp"].Value.(map[string]interface{})
+	sharedSopsId := outputs["sops_id"].Value.(string)
+	acr := outputs["acr"].Value.(map[string]interface{})
+	eventHubSas := outputs["event_hub_sas"].Value.(string)
+
+	log.Println("Creating Kubernetes client")
+	kubeconfigPath, kubeClient, err := getKubernetesCredentials(kubeconfig, aksHost, aksCert, aksKey, aksCa)
+	if err != nil {
+		log.Fatalf("error create Kubernetes client: %v", err)
+	}
+	defer os.RemoveAll(filepath.Dir(kubeconfigPath))
+
+	cfg = config{
+		kubeconfigPath: kubeconfigPath,
+		kubeClient:     kubeClient,
+		azdoPat:        azdoPat,
+		idRsa:          idRsa,
+		idRsaPub:       idRsaPub,
+		knownHosts:     "ssh.dev.azure.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7Hr1oTWqNqOlzGJOfGJ4NakVyIzf1rXYd4d7wo6jBlkLvCA4odBlL0mDUyZ0/QUfTTqeu+tm22gOsv+VrVTMk6vwRU75gY/y9ut5Mb3bR5BV58dKXyq9A9UeB5Cakehn5Zgm6x1mKoVyf+FFn26iYqXJRgzIZZcZ5V6hrE0Qg39kZm4az48o0AUbf6Sp4SLdvnuMa2sVNwHBboS7EJkm57XQPVU3/QpyNLHbWDdzwtrlS+ez30S3AdYhLKEOxAG8weOnyrtLJAUen9mTkol8oII1edf7mWWbWVf0nBmly21+nZcmCTISQBtdcyPaEno7fFQMDD26/s0lfKob4Kw8H",
+		fleetInfraRepository: repoConfig{
+			http: fleetInfraRepository["http"].(string),
+			ssh:  fleetInfraRepository["ssh"].(string),
+		},
+		applicationRepository: repoConfig{
+			http: applicationRepository["http"].(string),
+			ssh:  applicationRepository["ssh"].(string),
+		},
+		fluxAzureSp: spConfig{
+			tenantId:     fluxAzureSp["tenant_id"].(string),
+			clientId:     fluxAzureSp["client_id"].(string),
+			clientSecret: fluxAzureSp["client_secret"].(string),
+		},
+		sopsId: sharedSopsId,
+		acr: acrConfig{
+			url:      acr["url"].(string),
+			username: acr["username"].(string),
+			password: acr["password"].(string),
+		},
+		eventHubSas: eventHubSas,
+	}
+
+	err = installFlux(ctx, kubeClient, kubeconfigPath, cfg.fleetInfraRepository.http, azdoPat, cfg.fluxAzureSp)
+	if err != nil {
+		log.Fatalf("error installing Flux: %v", err)
+	}
+
+	log.Println("Running Azure e2e tests")
+	exitVal := m.Run()
+
+	log.Println("Tearing down Azure test infrastructure")
+	err = tf.Destroy(ctx)
+	if err != nil {
+		log.Fatalf("error running Show: %v", err)
+	}
+
+	os.Exit(exitVal)
+}
+
+func TestFluxInstallation(t *testing.T) {
+	ctx := context.TODO()
+	require.Eventually(t, func() bool {
+		err := verifyGitAndKustomization(ctx, cfg.kubeClient, "flux-system", "flux-system")
+		if err != nil {
+			return false
+		}
+		return true
+	}, 60*time.Second, 5*time.Second)
+}
+
+func TestAzureDevOpsCloning(t *testing.T) {
+	ctx := context.TODO()
+	branchName := "feature/branch"
+	tagName := "v1"
+
+	tests := []struct {
+		name      string
+		refType   string
+		cloneType string
+	}{
+		{
+			name:      "https-feature-branch",
+			refType:   "branch",
+			cloneType: "http",
+		},
+		{
+			name:      "https-v1",
+			refType:   "tag",
+			cloneType: "http",
+		},
+		{
+			name:      "ssh-feature-branch",
+			refType:   "branch",
+			cloneType: "ssh",
+		},
+		{
+			name:      "ssh-v1",
+			refType:   "tag",
+			cloneType: "ssh",
+		},
+	}
+
+	t.Log("Creating application sources")
+	repo, repoDir, err := getRepository(cfg.applicationRepository.http, branchName, true, cfg.azdoPat)
+	require.NoError(t, err)
+	for _, tt := range tests {
+		manifest := fmt.Sprintf(`
+      apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        name: foobar
+        namespace: %s
+    `, tt.name)
+		err = runCommand(ctx, repoDir, fmt.Sprintf("mkdir -p ./cloning-test/%s", tt.name))
+		require.NoError(t, err)
+		err = runCommand(ctx, repoDir, fmt.Sprintf("echo '%s' > ./cloning-test/%s/configmap.yaml", manifest, tt.name))
+		require.NoError(t, err)
+	}
+	err = commitAndPushAll(repo, branchName, cfg.azdoPat)
+	require.NoError(t, err)
+	err = createTagAndPush(repo, branchName, tagName, cfg.azdoPat)
+	require.NoError(t, err)
+
+	t.Log("Verifying application-gitops namespaces")
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ref := &sourcev1.GitRepositoryRef{
+				Branch: branchName,
+			}
+			if tt.refType == "tag" {
+				ref = &sourcev1.GitRepositoryRef{
+					Tag: tagName,
+				}
+			}
+			url := cfg.applicationRepository.http
+			secretData := map[string]string{
+				"username": "git",
+				"password": cfg.azdoPat,
+			}
+			if tt.cloneType == "ssh" {
+				url = cfg.applicationRepository.ssh
+				secretData = map[string]string{
+					"identity":     cfg.idRsa,
+					"identity.pub": cfg.idRsaPub,
+					"known_hosts":  cfg.knownHosts,
+				}
+			}
+
+			namespace := corev1.Namespace{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: tt.name,
+				},
+			}
+			_, err := controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, &namespace, func() error {
+				return nil
+			})
+			gitSecret := &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "git-credentials",
+					Namespace: namespace.Name,
+				},
+			}
+			_, err = controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, gitSecret, func() error {
+				gitSecret.StringData = secretData
+				return nil
+			})
+			source := &sourcev1.GitRepository{ObjectMeta: metav1.ObjectMeta{Name: tt.name, Namespace: namespace.Name}}
+			_, err = controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, source, func() error {
+				source.Spec = sourcev1.GitRepositorySpec{
+					GitImplementation: sourcev1.LibGit2Implementation,
+					Reference:         ref,
+					SecretRef: &meta.LocalObjectReference{
+						Name: gitSecret.Name,
+					},
+					URL: url,
+				}
+				return nil
+			})
+			require.NoError(t, err)
+			kustomization := &kustomizev1.Kustomization{ObjectMeta: metav1.ObjectMeta{Name: tt.name, Namespace: namespace.Name}}
+			_, err = controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, kustomization, func() error {
+				kustomization.Spec = kustomizev1.KustomizationSpec{
+					Path: fmt.Sprintf("./cloning-test/%s", tt.name),
+					SourceRef: kustomizev1.CrossNamespaceSourceReference{
+						Kind:      sourcev1.GitRepositoryKind,
+						Name:      tt.name,
+						Namespace: namespace.Name,
+					},
+					Interval: metav1.Duration{Duration: 1 * time.Minute},
+					Prune:    true,
+				}
+				return nil
+			})
+			require.NoError(t, err)
+
+			// Wait for configmap to be deployed
+			require.Eventually(t, func() bool {
+				err := verifyGitAndKustomization(ctx, cfg.kubeClient, namespace.Name, tt.name)
+				if err != nil {
+					return false
+				}
+				nn := types.NamespacedName{Name: "foobar", Namespace: namespace.Name}
+				cm := &corev1.ConfigMap{}
+				err = cfg.kubeClient.Get(ctx, nn, cm)
+				if err != nil {
+					return false
+				}
+				return true
+			}, 120*time.Second, 5*time.Second)
+		})
+	}
+}
+
+func TestImageRepositoryACR(t *testing.T) {
+	ctx := context.TODO()
+	name := "image-repository-acr"
+	repoUrl := cfg.applicationRepository.http
+	oldVersion := "1.0.0"
+	newVersion := "1.0.1"
+	manifest := fmt.Sprintf(`
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: podinfo
+      namespace: %s
+    spec:
+      selector:
+        matchLabels:
+          app: podinfo
+      template:
+        metadata:
+          labels:
+            app: podinfo
+        spec:
+          containers:
+          - name: podinfod
+            image: %s/container/podinfo:%s # {"$imagepolicy": "%s:podinfo"}
+            readinessProbe:
+              exec:
+                command:
+                - podcli
+                - check
+                - http
+                - localhost:9898/readyz
+              initialDelaySeconds: 5
+              timeoutSeconds: 5`, name, cfg.acr.url, oldVersion, name)
+
+	repo, repoDir, err := getRepository(repoUrl, name, true, cfg.azdoPat)
+	require.NoError(t, err)
+	err = addFile(repoDir, "podinfo.yaml", manifest)
+	require.NoError(t, err)
+	err = commitAndPushAll(repo, name, cfg.azdoPat)
+	require.NoError(t, err)
+
+	err = setupNamespace(ctx, cfg.kubeClient, repoUrl, cfg.azdoPat, name)
+	require.NoError(t, err)
+	acrSecret := corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "acr-docker", Namespace: name}}
+	_, err = controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, &acrSecret, func() error {
+		acrSecret.Type = corev1.SecretTypeDockerConfigJson
+		acrSecret.StringData = map[string]string{
+			".dockerconfigjson": fmt.Sprintf(`
+        {
+          "auths": {
+            "%s": {
+              "auth": "%s"
+            }
+          }
+        }
+        `, cfg.acr.url, b64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", cfg.acr.username, cfg.acr.password)))),
+		}
+		return nil
+	})
+	require.NoError(t, err)
+	imageRepository := reflectorv1beta1.ImageRepository{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "podinfo",
+			Namespace: name,
+		},
+	}
+	_, err = controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, &imageRepository, func() error {
+		imageRepository.Spec = reflectorv1beta1.ImageRepositorySpec{
+			Image: fmt.Sprintf("%s/container/podinfo", cfg.acr.url),
+			Interval: metav1.Duration{
+				Duration: 1 * time.Minute,
+			},
+			SecretRef: &meta.LocalObjectReference{
+				Name: acrSecret.Name,
+			},
+		}
+		return nil
+	})
+	require.NoError(t, err)
+	imagePolicy := reflectorv1beta1.ImagePolicy{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "podinfo",
+			Namespace: name,
+		},
+	}
+	_, err = controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, &imagePolicy, func() error {
+		imagePolicy.Spec = reflectorv1beta1.ImagePolicySpec{
+			ImageRepositoryRef: meta.LocalObjectReference{
+				Name: imageRepository.Name,
+			},
+			Policy: reflectorv1beta1.ImagePolicyChoice{
+				SemVer: &reflectorv1beta1.SemVerPolicy{
+					Range: "1.0.x",
+				},
+			},
+		}
+		return nil
+	})
+	require.NoError(t, err)
+	imageAutomation := automationv1beta1.ImageUpdateAutomation{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "podinfo",
+			Namespace: name,
+		},
+	}
+	_, err = controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, &imageAutomation, func() error {
+		imageAutomation.Spec = automationv1beta1.ImageUpdateAutomationSpec{
+			Interval: metav1.Duration{
+				Duration: 1 * time.Minute,
+			},
+			SourceRef: automationv1beta1.SourceReference{
+				Kind: "GitRepository",
+				Name: name,
+			},
+			GitSpec: &automationv1beta1.GitSpec{
+				Checkout: &automationv1beta1.GitCheckoutSpec{
+					Reference: sourcev1.GitRepositoryRef{
+						Branch: name,
+					},
+				},
+				Commit: automationv1beta1.CommitSpec{
+					Author: automationv1beta1.CommitUser{
+						Email: "imageautomation@example.com",
+						Name:  "imageautomation",
+					},
+				},
+			},
+		}
+		return nil
+	})
+	require.NoError(t, err)
+
+	// Wait for image repository to be ready
+	require.Eventually(t, func() bool {
+		_, repoDir, err := getRepository(repoUrl, name, false, cfg.azdoPat)
+		if err != nil {
+			return false
+		}
+		b, err := os.ReadFile(filepath.Join(repoDir, "podinfo.yaml"))
+		if err != nil {
+			return false
+		}
+		if bytes.Contains(b, []byte(newVersion)) == false {
+			return false
+		}
+		return true
+	}, 120*time.Second, 5*time.Second)
+}
+
+func TestKeyVaultSops(t *testing.T) {
+	ctx := context.TODO()
+	name := "key-vault-sops"
+	repoUrl := cfg.applicationRepository.http
+	secretYaml := `apiVersion: v1
+kind: Secret
+metadata:
+  name: "test"
+  namespace: "key-vault-sops"
+stringData:
+  foo: "bar"`
+
+	repo, tmpDir, err := getRepository(repoUrl, name, true, cfg.azdoPat)
+	err = runCommand(ctx, tmpDir, "mkdir -p ./key-vault-sops")
+	require.NoError(t, err)
+	err = runCommand(ctx, tmpDir, fmt.Sprintf("echo \"%s\" > ./key-vault-sops/secret.enc.yaml", secretYaml))
+	require.NoError(t, err)
+	err = runCommand(ctx, tmpDir, fmt.Sprintf("sops --encrypt --encrypted-regex '^(data|stringData)$' --azure-kv %s --in-place ./key-vault-sops/secret.enc.yaml", cfg.sopsId))
+	require.NoError(t, err)
+	err = commitAndPushAll(repo, name, cfg.azdoPat)
+	require.NoError(t, err)
+
+	err = setupNamespace(ctx, cfg.kubeClient, repoUrl, cfg.azdoPat, name)
+	require.NoError(t, err)
+
+	source := &sourcev1.GitRepository{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: name}}
+	require.Eventually(t, func() bool {
+		_, err = controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, source, func() error {
+			source.Spec = sourcev1.GitRepositorySpec{
+				GitImplementation: sourcev1.LibGit2Implementation,
+				Reference: &sourcev1.GitRepositoryRef{
+					Branch: name,
+				},
+				SecretRef: &meta.LocalObjectReference{
+					Name: "https-credentials",
+				},
+				URL: repoUrl,
+			}
+			return nil
+		})
+		if err != nil {
+			return false
+		}
+		return true
+	}, 10*time.Second, 1*time.Second)
+	kustomization := &kustomizev1.Kustomization{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: name}}
+	require.Eventually(t, func() bool {
+		_, err = controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, kustomization, func() error {
+			kustomization.Spec = kustomizev1.KustomizationSpec{
+				Path: "./key-vault-sops",
+				SourceRef: kustomizev1.CrossNamespaceSourceReference{
+					Kind:      sourcev1.GitRepositoryKind,
+					Name:      source.Name,
+					Namespace: source.Namespace,
+				},
+				Interval: metav1.Duration{Duration: 1 * time.Minute},
+				Prune:    true,
+				Decryption: &kustomizev1.Decryption{
+					Provider: "sops",
+				},
+			}
+			return nil
+		})
+		if err != nil {
+			return false
+		}
+		return true
+	}, 10*time.Second, 1*time.Second)
+
+	require.Eventually(t, func() bool {
+		nn := types.NamespacedName{Name: "test", Namespace: name}
+		secret := &corev1.Secret{}
+		err = cfg.kubeClient.Get(ctx, nn, secret)
+		if err != nil {
+			return false
+		}
+		return true
+	}, 120*time.Second, 5*time.Second)
+}
+
+func TestAzureDevOpsCommitStatus(t *testing.T) {
+	ctx := context.TODO()
+	name := "commit-status"
+	repoUrl := cfg.applicationRepository.http
+	manifest := fmt.Sprintf(`
+    apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: foobar
+      namespace: %s
+  `, name)
+
+	repo, repoDir, err := getRepository(repoUrl, name, true, cfg.azdoPat)
+	require.NoError(t, err)
+	err = addFile(repoDir, "configmap.yaml", manifest)
+	require.NoError(t, err)
+	err = commitAndPushAll(repo, name, cfg.azdoPat)
+	require.NoError(t, err)
+
+	err = setupNamespace(ctx, cfg.kubeClient, repoUrl, cfg.azdoPat, name)
+	require.NoError(t, err)
+
+	kustomization := &kustomizev1.Kustomization{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: name}}
+	require.Eventually(t, func() bool {
+		_, err = controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, kustomization, func() error {
+			kustomization.Spec.HealthChecks = []meta.NamespacedObjectKindReference{
+				{
+					APIVersion: "v1",
+					Kind:       "ConfigMap",
+					Name:       "foobar",
+					Namespace:  name,
+				},
+			}
+			return nil
+		})
+		if err != nil {
+			return false
+		}
+		return true
+	}, 10*time.Second, 1*time.Second)
+
+	require.Eventually(t, func() bool {
+		err := verifyGitAndKustomization(ctx, cfg.kubeClient, name, name)
+		if err != nil {
+			return false
+		}
+		return true
+	}, 10*time.Second, 1*time.Second)
+
+	secret := corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "azuredevops-token",
+			Namespace: name,
+		},
+	}
+	_, err = controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, &secret, func() error {
+		secret.StringData = map[string]string{
+			"token": cfg.azdoPat,
+		}
+		return nil
+	})
+	provider := notiv1beta1.Provider{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "azuredevops",
+			Namespace: name,
+		},
+	}
+	_, err = controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, &provider, func() error {
+		provider.Spec = notiv1beta1.ProviderSpec{
+			Type:    "azuredevops",
+			Address: repoUrl,
+			SecretRef: &meta.LocalObjectReference{
+				Name: "azuredevops-token",
+			},
+		}
+		return nil
+	})
+	require.NoError(t, err)
+	alert := notiv1beta1.Alert{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "azuredevops",
+			Namespace: name,
+		},
+	}
+	_, err = controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, &alert, func() error {
+		alert.Spec = notiv1beta1.AlertSpec{
+			ProviderRef: meta.LocalObjectReference{
+				Name: provider.Name,
+			},
+			EventSources: []notiv1beta1.CrossNamespaceObjectReference{
+				{
+					Kind:      "Kustomization",
+					Name:      name,
+					Namespace: name,
+				},
+			},
+		}
+		return nil
+	})
+	require.NoError(t, err)
+
+	u, err := giturls.Parse(repoUrl)
+	require.NoError(t, err)
+	id := strings.TrimLeft(u.Path, "/")
+	id = strings.TrimSuffix(id, ".git")
+	comp := strings.Split(id, "/")
+	orgUrl := fmt.Sprintf("%s://%s/%v", u.Scheme, u.Host, comp[0])
+	project := comp[1]
+	repoId := comp[3]
+	branch, err := repo.LookupBranch(name, git2go.BranchLocal)
+	require.NoError(t, err)
+	commit, err := repo.LookupCommit(branch.Target())
+	rev := commit.Id().String()
+	connection := azuredevops.NewPatConnection(orgUrl, cfg.azdoPat)
+	client, err := git.NewClient(ctx, connection)
+	require.NoError(t, err)
+	getArgs := git.GetStatusesArgs{
+		Project:      &project,
+		RepositoryId: &repoId,
+		CommitId:     &rev,
+	}
+	require.Eventually(t, func() bool {
+		statuses, err := client.GetStatuses(ctx, getArgs)
+		if err != nil {
+			return false
+		}
+		if len(*statuses) != 1 {
+			return false
+		}
+		return true
+	}, 500*time.Second, 5*time.Second)
+}
+
+func TestEventHubNotification(t *testing.T) {
+	ctx := context.TODO()
+	name := "event-hub"
+
+	// Start listening to eventhub with latest offset
+	hub, err := eventhub.NewHubFromConnectionString(cfg.eventHubSas)
+	require.NoError(t, err)
+	c := make(chan string, 10)
+	handler := func(ctx context.Context, event *eventhub.Event) error {
+		c <- string(event.Data)
+		return nil
+	}
+	runtimeInfo, err := hub.GetRuntimeInformation(ctx)
+	require.NoError(t, err)
+	require.Equal(t, 1, len(runtimeInfo.PartitionIDs))
+	listenerHandler, err := hub.Receive(ctx, runtimeInfo.PartitionIDs[0], handler, eventhub.ReceiveWithLatestOffset())
+	require.NoError(t, err)
+
+	// Setup Flux resources
+	repoUrl := cfg.applicationRepository.http
+	manifest := fmt.Sprintf(`
+    apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: foobar
+      namespace: %s
+  `, name)
+
+	repo, repoDir, err := getRepository(repoUrl, name, true, cfg.azdoPat)
+	require.NoError(t, err)
+	err = addFile(repoDir, "configmap.yaml", manifest)
+	require.NoError(t, err)
+	err = commitAndPushAll(repo, name, cfg.azdoPat)
+	require.NoError(t, err)
+
+	err = setupNamespace(ctx, cfg.kubeClient, repoUrl, cfg.azdoPat, name)
+	require.NoError(t, err)
+
+	secret := corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: name,
+		},
+	}
+	_, err = controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, &secret, func() error {
+		secret.StringData = map[string]string{
+			"address": cfg.eventHubSas,
+		}
+		return nil
+	})
+	provider := notiv1beta1.Provider{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: name,
+		},
+	}
+	_, err = controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, &provider, func() error {
+		provider.Spec = notiv1beta1.ProviderSpec{
+			Type:    "azureeventhub",
+			Address: repoUrl,
+			SecretRef: &meta.LocalObjectReference{
+				Name: name,
+			},
+		}
+		return nil
+	})
+	require.NoError(t, err)
+	alert := notiv1beta1.Alert{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: name,
+		},
+	}
+	_, err = controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, &alert, func() error {
+		alert.Spec = notiv1beta1.AlertSpec{
+			ProviderRef: meta.LocalObjectReference{
+				Name: provider.Name,
+			},
+			EventSources: []notiv1beta1.CrossNamespaceObjectReference{
+				{
+					Kind:      "Kustomization",
+					Name:      name,
+					Namespace: name,
+				},
+			},
+		}
+		return nil
+	})
+	require.NoError(t, err)
+
+	kustomization := &kustomizev1.Kustomization{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: name}}
+	require.Eventually(t, func() bool {
+		_, err := controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, kustomization, func() error {
+			kustomization.Spec.HealthChecks = []meta.NamespacedObjectKindReference{
+				{
+					APIVersion: "v1",
+					Kind:       "ConfigMap",
+					Name:       "foobar",
+					Namespace:  name,
+				},
+			}
+			return nil
+		})
+		if err != nil {
+			return false
+		}
+		return true
+	}, 10*time.Second, 1*time.Second)
+
+	require.Eventually(t, func() bool {
+		err := verifyGitAndKustomization(ctx, cfg.kubeClient, name, name)
+		if err != nil {
+			return false
+		}
+		return true
+	}, 60*time.Second, 5*time.Second)
+
+	// Wait to read even from event hub
+	require.Eventually(t, func() bool {
+		select {
+		case eventJson := <-c:
+			event := &events.Event{}
+			err := json.Unmarshal([]byte(eventJson), event)
+			if err != nil {
+				fmt.Println(err)
+				return false
+			}
+			if event.InvolvedObject.Kind != "Kustomization" {
+				return false
+			}
+			if event.Message != "Health check passed" {
+				return false
+			}
+			return true
+		default:
+			return false
+		}
+		//}, 700*time.Second, 10*time.Second)
+	}, 60*time.Second, 1*time.Second)
+	err = listenerHandler.Close(ctx)
+	require.NoError(t, err)
+	err = hub.Close(ctx)
+	require.NoError(t, err)
+}
+
+// TODO: Enable when source-controller supports Helm charts from OCI sources.
+/*func TestACRHelmRelease(t *testing.T) {
+	ctx := context.TODO()
+
+	// Create namespace for test
+	namespace := corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "acr-helm-release",
+		},
+	}
+	err := kubeClient.Create(ctx, &namespace)
+	require.NoError(t, err)
+	defer func() {
+		kubeClient.Delete(ctx, &namespace)
+		require.NoError(t, err)
+	}()
+
+	// Copy ACR credentials to new namespace
+	acrNn := types.NamespacedName{
+		Name:      "acr-helm",
+		Namespace: "flux-system",
+	}
+	acrSecret := corev1.Secret{}
+	err = kubeClient.Get(ctx, acrNn, &acrSecret)
+	require.NoError(t, err)
+	acrSecret.ObjectMeta = metav1.ObjectMeta{
+		Name:      acrSecret.Name,
+		Namespace: namespace.Name,
+	}
+	err = kubeClient.Create(ctx, &acrSecret)
+	require.NoError(t, err)
+
+	// Create HelmRepository and wait for it to sync
+	helmRepository := sourcev1.HelmRepository{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "acr",
+			Namespace: namespace.Name,
+		},
+		Spec: sourcev1.HelmRepositorySpec{
+			URL: "https://acrappsoarfish.azurecr.io/helm/podinfo",
+			Interval: metav1.Duration{
+				Duration: 5 * time.Minute,
+			},
+			SecretRef: &meta.LocalObjectReference{
+				Name: acrSecret.Name,
+			},
+			PassCredentials: true,
+		},
+	}
+	err = kubeClient.Create(ctx, &helmRepository)
+	require.NoError(t, err)
+}*/

tests/azure/go.mod

@@ -0,0 +1,24 @@
+module github.com/fluxcd/flux2/tests/azure
+
+go 1.16
+
+require (
+	github.com/Azure/azure-event-hubs-go/v3 v3.3.13
+	github.com/fluxcd/helm-controller/api v0.11.2
+	github.com/fluxcd/image-automation-controller/api v0.14.1
+	github.com/fluxcd/image-reflector-controller/api v0.11.1
+	github.com/fluxcd/kustomize-controller/api v0.14.1
+	github.com/fluxcd/notification-controller/api v0.16.0
+	github.com/fluxcd/pkg/apis/meta v0.10.1
+	github.com/fluxcd/pkg/runtime v0.12.1
+	github.com/fluxcd/source-controller/api v0.15.4
+	github.com/hashicorp/terraform-exec v0.14.0
+	github.com/libgit2/git2go/v31 v31.6.1
+	github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5
+	github.com/stretchr/testify v1.7.0
+	github.com/whilp/git-urls v1.0.0
+	k8s.io/api v0.22.2
+	k8s.io/apimachinery v0.22.2
+	k8s.io/client-go v0.22.2
+	sigs.k8s.io/controller-runtime v0.10.1
+)

tests/azure/terraform.go

@@ -0,0 +1,19 @@
+package test
+
+import (
+	"context"
+	"os/exec"
+	"strings"
+)
+
+type whichTerraform struct{}
+
+func (w *whichTerraform) ExecPath(ctx context.Context) (string, error) {
+	cmd := exec.CommandContext(ctx, "which", "terraform")
+	output, err := cmd.Output()
+	if err != nil {
+		return "", err
+	}
+	path := strings.TrimSuffix(string(output), "\n")
+	return path, nil
+}

tests/azure/terraform/aks/.terraform.lock.hcl

@@ -0,0 +1,78 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/azuread" {
+  version     = "1.6.0"
+  constraints = "1.6.0"
+  hashes = [
+    "h1:BlO53mX+Y2W//YqlCKvoxzofegFQk636XlKtmZYH0PY=",
+    "zh:0db70045a464d325fdb3d71809f0467844c3e2fcf1349e568bc51ad5035c99d9",
+    "zh:3629f1d7b4eba48d744b24c7cf7fe878d5ef5910a36b525507bd3d588010ccec",
+    "zh:5a73a45b6d1ff353810cc9b00d7c90a2fb328ba0a9ef3d24392b1500fb98741a",
+    "zh:7a6a9c390cf1bf752321abb8d0643c9f623e8c2ad871dfb378d64c9d90fada2d",
+    "zh:7d6de55d326b046dabc16bd7b655f008ff780c36ffc884b139a7c7da37b446d5",
+    "zh:8d725c618396ccae290e411296c892e08e776c3e9e5a82b0ef1f633a917146ec",
+    "zh:a206d1d8042bf66ca12b97334bbd6fcdf12fd6131f8cb4547c82b9fa7a701612",
+    "zh:b03ab4ff07dcb5ed8be8b0619c6ec9fb0da0c83594ccb0a1bff72f346083b530",
+    "zh:b6131f9d438b340a4016c770b569139ec7ac2532358a8ab783234e8c93d141d5",
+    "zh:ce9372d38e9e62accfd54f4669753000d3dcbae4b45686d74630eb63eb879f37",
+    "zh:df9a607c333d464d8bdeb248b1ff41e493c1d0661453a1e1ce396b89952a74ee",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/azurerm" {
+  version     = "2.76.0"
+  constraints = "2.76.0"
+  hashes = [
+    "h1:kF+u0s0DPnE5gMKhzQACWRUIdwZG1Ax4atXt9hk1J8M=",
+    "zh:137eb7c07d3d3c9fe123e74381c108c4442efba9fc051faa2ca603503ff2840f",
+    "zh:142a354dffd59a1d6b7f1614ab66a468ace3636d95933589a8d704ee8dbc4ea6",
+    "zh:4c343b4da8b86e4213c1b11f73337cec73a55b1fa95a0e0e0c79f34597d37cc3",
+    "zh:75d3109d48726fdbaad840d2fa294ec3362b32a3628c261af00f5c5608427521",
+    "zh:7b1e78c144c6ad2beebc798abb9e76c725bf34ced41df36dc0120a0f2426e801",
+    "zh:981235b01c3d4acf94c78cdd96624fd01d0a3622bc06b5c62aef3e788f1481c3",
+    "zh:bad819efae7293ce371409e1ed34197c3e879f61d3e44893af0ce68e6aaffde7",
+    "zh:c8008967722929deccfec9695754ae55028ce12311c321ae7a7c753dde162a44",
+    "zh:d38513d1138864269b2ff333b08a64a7949630d489f18e660630bbaff3b7ebb8",
+    "zh:e1f64d2d91b5f5cba6a9c5d35278a4918d332d7385a87f8e3466aaadb782a90f",
+    "zh:e93a377a1e823df69718686703b07f1712046eeb742006022e982f2e8a594161",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/random" {
+  version = "3.1.0"
+  hashes = [
+    "h1:BZMEPucF+pbu9gsPk0G0BHx7YP04+tKdq2MrRDF1EDM=",
+    "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc",
+    "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626",
+    "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff",
+    "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2",
+    "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992",
+    "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427",
+    "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc",
+    "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f",
+    "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b",
+    "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7",
+    "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a",
+  ]
+}
+
+provider "registry.terraform.io/microsoft/azuredevops" {
+  version     = "0.1.7"
+  constraints = "0.1.7"
+  hashes = [
+    "h1:AWNWqJ3XhlKp3xdJF+3WKdK1zVoCFYInQvi06exsBzg=",
+    "zh:0c024992f2282ef73d4829e487ec8482dd98e9272b903f2e5979f5f62567ee4e",
+    "zh:47fef8f57dfdca6aebe5a907b4866880007512019d9bec29805fc83501412309",
+    "zh:692736c501c6b987a4a74c69fb7702a54969180706d1f67eff13e6ed2a0f9fec",
+    "zh:6c3c4339206f5dcbc9d10fb2fe343652e7e14255223dcece5bf79ef9030858ef",
+    "zh:77dfc63377b8d8fe24cbbe479ead18bfd1c7ded067fd694b6532434d6305ad31",
+    "zh:93dba26dbade208a1cba43333f104a64252ca2404636ab033702da29648bfaaa",
+    "zh:952d28b3e6c137de9b8700d2b748e5a4a2aa53ed07005f0f7abdd66b84cc63fe",
+    "zh:a7b8238b8b2f04ad2d720a207377bfc2066d54b1d9d7285f2535afc43ff80fdb",
+    "zh:bb23d8fc3cdd3c01d7620dadb2ba7b724706f2112d7738e135d1be1455682f5e",
+    "zh:cb4da640beb5fc59296479c201a03351789496c04aaa57ae1530a7aac9095b92",
+    "zh:ede6fb7ab598081fdddac56d470bae14448271dfd43a645bc02d136643391ebe",
+    "zh:fd8291e6dc9118323a744660326a0f11de2a475c4a358e50f480feed1f3bb080",
+  ]
+}

tests/azure/terraform/aks/aks.tf

@@ -0,0 +1,37 @@
+resource "azurerm_kubernetes_cluster" "this" {
+  name                = "aks-${local.name_suffix}"
+  location            = azurerm_resource_group.this.location
+  resource_group_name = azurerm_resource_group.this.name
+
+  dns_prefix = "aks${local.name_suffix}"
+
+  default_node_pool {
+    name            = "default"
+    node_count      = 2
+    vm_size         = "Standard_B2s"
+    os_disk_size_gb = 30
+  }
+
+  identity {
+    type = "SystemAssigned"
+  }
+
+  role_based_access_control {
+    enabled = true
+  }
+
+  network_profile {
+    network_plugin = "kubenet"
+    network_policy = "calico"
+  }
+
+  tags = {
+    environment = "e2e"
+  }
+}
+
+resource "azurerm_role_assignment" "aks_acr_pull" {
+  scope                = data.azurerm_container_registry.shared.id
+  role_definition_name = "AcrPull"
+  principal_id         = azurerm_kubernetes_cluster.this.kubelet_identity[0].object_id
+}

tests/azure/terraform/aks/azuredevops.tf

@@ -0,0 +1,21 @@
+data "azuredevops_project" "e2e" {
+  name = "e2e"
+}
+
+resource "azuredevops_git_repository" "fleet_infra" {
+  project_id = data.azuredevops_project.e2e.id
+  name       = "fleet-infra-${local.name_suffix}"
+  default_branch = "refs/heads/main"
+  initialization {
+    init_type = "Clean"
+  }
+}
+
+resource "azuredevops_git_repository" "application" {
+  project_id = data.azuredevops_project.e2e.id
+  name       = "application-${local.name_suffix}"
+  default_branch = "refs/heads/main"
+  initialization {
+    init_type = "Clean"
+  }
+}

tests/azure/terraform/aks/event-hub.tf

@@ -0,0 +1,26 @@
+resource "azurerm_eventhub_namespace" "this" {
+  name = "ehns-${local.name_suffix}"
+  location            = azurerm_resource_group.this.location
+  resource_group_name = azurerm_resource_group.this.name
+  sku                 = "Standard"
+  capacity            = 1
+}
+
+
+resource "azurerm_eventhub" "this" {
+  name = "eh-${local.name_suffix}"
+  namespace_name      = azurerm_eventhub_namespace.this.name
+  resource_group_name = azurerm_resource_group.this.name
+  partition_count     = 1
+  message_retention   = 1
+}
+
+resource "azurerm_eventhub_authorization_rule" "this" {
+  name                = "flux"
+  resource_group_name = azurerm_resource_group.this.name
+  namespace_name      = azurerm_eventhub_namespace.this.name
+  eventhub_name       = azurerm_eventhub.this.name
+  listen              = true
+  send                = true
+  manage              = false
+}

tests/azure/terraform/aks/keyvault.tf

@@ -0,0 +1,37 @@
+resource "azurerm_key_vault" "this" {
+  name                = "kv-${random_pet.suffix.id}"
+  resource_group_name = azurerm_resource_group.this.name
+  location            = azurerm_resource_group.this.location
+  tenant_id           = data.azurerm_client_config.current.tenant_id
+  sku_name            = "standard"
+}
+
+resource "azurerm_key_vault_access_policy" "sops_write" {
+  key_vault_id = azurerm_key_vault.this.id
+  tenant_id = data.azurerm_client_config.current.tenant_id
+  object_id = data.azurerm_client_config.current.object_id
+
+  key_permissions = [
+    "Encrypt",
+    "Decrypt",
+    "Create",
+    "Delete",
+    "Purge",
+    "Get",
+    "List",
+  ]
+}
+
+resource "azurerm_key_vault_key" "sops" {
+  depends_on = [azurerm_key_vault_access_policy.sops_write]
+
+  name         = "sops"
+  key_vault_id = azurerm_key_vault.this.id
+  key_type     = "RSA"
+  key_size     = 2048
+
+  key_opts = [
+    "decrypt",
+    "encrypt",
+  ]
+}

tests/azure/terraform/aks/main.tf

@@ -0,0 +1,52 @@
+terraform {
+  backend "azurerm" {
+    resource_group_name  = "terraform-state"
+    storage_account_name = "terraformstate0419"
+    container_name       = "aks-tfstate"
+    key                  = "prod.terraform.tfstate"
+  }
+
+  required_version = "1.0.7"
+
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "2.76.0"
+    }
+    azuread = {
+      source = "hashicorp/azuread"
+      version = "1.6.0"
+    }
+    azuredevops = {
+      source = "microsoft/azuredevops"
+      version = "0.1.7"
+    }
+  }
+}
+
+provider "azurerm" {
+  features {}
+}
+
+provider "azuredevops" {
+  org_service_url = "https://dev.azure.com/${local.azure_devops_org}"
+  personal_access_token = data.azurerm_key_vault_secret.shared_pat.value
+}
+
+data "azurerm_client_config" "current" {}
+
+resource "random_pet" "suffix" {}
+
+locals {
+  azure_devops_org = "flux-azure"
+  name_suffix = "e2e-${random_pet.suffix.id}"
+}
+
+resource "azurerm_resource_group" "this" {
+  name     = "rg-${local.name_suffix}"
+  location = "West Europe"
+
+  tags = {
+    environment = "e2e"
+  }
+}

tests/azure/terraform/aks/outputs.tf

@@ -0,0 +1,76 @@
+output "aks_kube_config" {
+  sensitive = true
+  value = azurerm_kubernetes_cluster.this.kube_config_raw
+}
+
+output "aks_host" {
+  value = azurerm_kubernetes_cluster.this.kube_config[0].host
+}
+
+output "aks_client_certificate" {
+  value = base64decode(azurerm_kubernetes_cluster.this.kube_config[0].client_certificate)
+}
+
+output "aks_client_key" {
+  value = base64decode(azurerm_kubernetes_cluster.this.kube_config[0].client_key)
+}
+
+output "aks_cluster_ca_certificate" {
+  value = base64decode(azurerm_kubernetes_cluster.this.kube_config[0].cluster_ca_certificate)
+}
+
+output "shared_pat" {
+  sensitive = true
+  value = data.azurerm_key_vault_secret.shared_pat.value
+}
+
+output "shared_id_rsa" {
+  sensitive = true
+  value = data.azurerm_key_vault_secret.shared_id_rsa.value
+}
+
+output "shared_id_rsa_pub" {
+  sensitive = true
+  value = data.azurerm_key_vault_secret.shared_id_rsa_pub.value
+}
+
+output "fleet_infra_repository" {
+  value = {
+    http = azuredevops_git_repository.fleet_infra.remote_url
+    ssh = "ssh://git@ssh.dev.azure.com/v3/${local.azure_devops_org}/${azuredevops_git_repository.fleet_infra.project_id}/${azuredevops_git_repository.fleet_infra.name}"
+  }
+}
+
+output "application_repository" {
+  value = {
+    http = azuredevops_git_repository.application.remote_url
+    ssh = "ssh://git@ssh.dev.azure.com/v3/${local.azure_devops_org}/${azuredevops_git_repository.application.project_id}/${azuredevops_git_repository.application.name}"
+  }
+}
+
+output "flux_azure_sp" {
+  value = {
+    tenant_id = data.azurerm_client_config.current.tenant_id
+    client_id = azuread_service_principal.flux.application_id
+    client_secret = azuread_service_principal_password.flux.value
+  }
+  sensitive = true
+}
+
+output "event_hub_sas" {
+  value = azurerm_eventhub_authorization_rule.this.primary_connection_string
+  sensitive = true
+}
+
+output "sops_id" {
+  value = azurerm_key_vault_key.sops.id
+}
+
+output "acr" {
+  value = {
+    url = data.azurerm_container_registry.shared.login_server
+    username = azuread_service_principal.flux.application_id
+    password = azuread_service_principal_password.flux.value
+  }
+  sensitive = true
+}

tests/azure/terraform/aks/service-principal.tf

@@ -0,0 +1,52 @@
+resource "azuread_application" "flux" {
+  display_name = "flux-${local.name_suffix}"
+
+  required_resource_access {
+    resource_app_id = "00000003-0000-0000-c000-000000000000"
+
+    resource_access {
+      id   = "df021288-bdef-4463-88db-98f22de89214"
+      type = "Role"
+    }
+  }
+
+  required_resource_access {
+    resource_app_id = "00000002-0000-0000-c000-000000000000"
+
+    resource_access {
+      id   = "1cda74f2-2616-4834-b122-5cb1b07f8a59"
+      type = "Role"
+    }
+    resource_access {
+      id   = "78c8a3c8-a07e-4b9e-af1b-b5ccab50a175"
+      type = "Role"
+    }
+  }
+}
+
+resource "azuread_service_principal" "flux" {
+  application_id = azuread_application.flux.application_id
+}
+
+resource "azuread_service_principal_password" "flux" {
+  service_principal_id = azuread_service_principal.flux.object_id
+}
+
+resource "azurerm_role_assignment" "acr" {
+  scope                = data.azurerm_container_registry.shared.id
+  role_definition_name = "AcrPull"
+  principal_id         = azuread_service_principal.flux.object_id
+}
+
+resource "azurerm_key_vault_access_policy" "sops_decrypt" {
+  key_vault_id = azurerm_key_vault.this.id
+  tenant_id    = data.azurerm_client_config.current.tenant_id
+  object_id    = azuread_service_principal.flux.object_id
+
+  key_permissions = [
+    "Encrypt",
+    "Decrypt",
+    "Get",
+    "List",
+  ]
+}

tests/azure/terraform/aks/shared.tf

@@ -0,0 +1,32 @@
+locals {
+  shared_suffix = "oarfish"
+}
+
+data "azurerm_resource_group" "shared" {
+  name = "e2e-shared"
+}
+
+data "azurerm_container_registry" "shared" {
+  name                = "acrapps${local.shared_suffix}"
+  resource_group_name = data.azurerm_resource_group.shared.name
+}
+
+data "azurerm_key_vault" "shared" {
+  resource_group_name = data.azurerm_resource_group.shared.name
+  name                = "kv-credentials-${local.shared_suffix}"
+}
+
+data "azurerm_key_vault_secret" "shared_pat" {
+  key_vault_id = data.azurerm_key_vault.shared.id
+  name         = "pat"
+}
+
+data "azurerm_key_vault_secret" "shared_id_rsa" {
+  key_vault_id = data.azurerm_key_vault.shared.id
+  name         = "id-rsa"
+}
+
+data "azurerm_key_vault_secret" "shared_id_rsa_pub" {
+  key_vault_id = data.azurerm_key_vault.shared.id
+  name         = "id-rsa-pub"
+}

tests/azure/terraform/shared/.terraform.lock.hcl

@@ -0,0 +1,58 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/azuread" {
+  version     = "1.6.0"
+  constraints = "1.6.0"
+  hashes = [
+    "h1:BlO53mX+Y2W//YqlCKvoxzofegFQk636XlKtmZYH0PY=",
+    "zh:0db70045a464d325fdb3d71809f0467844c3e2fcf1349e568bc51ad5035c99d9",
+    "zh:3629f1d7b4eba48d744b24c7cf7fe878d5ef5910a36b525507bd3d588010ccec",
+    "zh:5a73a45b6d1ff353810cc9b00d7c90a2fb328ba0a9ef3d24392b1500fb98741a",
+    "zh:7a6a9c390cf1bf752321abb8d0643c9f623e8c2ad871dfb378d64c9d90fada2d",
+    "zh:7d6de55d326b046dabc16bd7b655f008ff780c36ffc884b139a7c7da37b446d5",
+    "zh:8d725c618396ccae290e411296c892e08e776c3e9e5a82b0ef1f633a917146ec",
+    "zh:a206d1d8042bf66ca12b97334bbd6fcdf12fd6131f8cb4547c82b9fa7a701612",
+    "zh:b03ab4ff07dcb5ed8be8b0619c6ec9fb0da0c83594ccb0a1bff72f346083b530",
+    "zh:b6131f9d438b340a4016c770b569139ec7ac2532358a8ab783234e8c93d141d5",
+    "zh:ce9372d38e9e62accfd54f4669753000d3dcbae4b45686d74630eb63eb879f37",
+    "zh:df9a607c333d464d8bdeb248b1ff41e493c1d0661453a1e1ce396b89952a74ee",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/azurerm" {
+  version     = "2.76.0"
+  constraints = "2.76.0"
+  hashes = [
+    "h1:kF+u0s0DPnE5gMKhzQACWRUIdwZG1Ax4atXt9hk1J8M=",
+    "zh:137eb7c07d3d3c9fe123e74381c108c4442efba9fc051faa2ca603503ff2840f",
+    "zh:142a354dffd59a1d6b7f1614ab66a468ace3636d95933589a8d704ee8dbc4ea6",
+    "zh:4c343b4da8b86e4213c1b11f73337cec73a55b1fa95a0e0e0c79f34597d37cc3",
+    "zh:75d3109d48726fdbaad840d2fa294ec3362b32a3628c261af00f5c5608427521",
+    "zh:7b1e78c144c6ad2beebc798abb9e76c725bf34ced41df36dc0120a0f2426e801",
+    "zh:981235b01c3d4acf94c78cdd96624fd01d0a3622bc06b5c62aef3e788f1481c3",
+    "zh:bad819efae7293ce371409e1ed34197c3e879f61d3e44893af0ce68e6aaffde7",
+    "zh:c8008967722929deccfec9695754ae55028ce12311c321ae7a7c753dde162a44",
+    "zh:d38513d1138864269b2ff333b08a64a7949630d489f18e660630bbaff3b7ebb8",
+    "zh:e1f64d2d91b5f5cba6a9c5d35278a4918d332d7385a87f8e3466aaadb782a90f",
+    "zh:e93a377a1e823df69718686703b07f1712046eeb742006022e982f2e8a594161",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/random" {
+  version = "3.1.0"
+  hashes = [
+    "h1:BZMEPucF+pbu9gsPk0G0BHx7YP04+tKdq2MrRDF1EDM=",
+    "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc",
+    "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626",
+    "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff",
+    "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2",
+    "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992",
+    "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427",
+    "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc",
+    "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f",
+    "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b",
+    "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7",
+    "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a",
+  ]
+}

tests/azure/terraform/shared/acr.tf

@@ -0,0 +1,6 @@
+resource "azurerm_container_registry" "this" {
+  name                = "acrapps${random_pet.suffix.id}"
+  resource_group_name = azurerm_resource_group.this.name
+  location            = azurerm_resource_group.this.location
+  sku                 = "standard"
+}

tests/azure/terraform/shared/keyvault.tf

@@ -0,0 +1,43 @@
+resource "azurerm_key_vault" "this" {
+  name                = "kv-credentials-${random_pet.suffix.id}"
+  resource_group_name = azurerm_resource_group.this.name
+  location            = azurerm_resource_group.this.location
+  tenant_id           = data.azurerm_client_config.current.tenant_id
+  sku_name            = "standard"
+}
+
+resource "azurerm_key_vault_access_policy" "admin" {
+  key_vault_id = azurerm_key_vault.this.id
+  tenant_id = data.azurerm_client_config.current.tenant_id
+  object_id = data.azurerm_client_config.current.object_id
+
+  key_permissions = [
+    "Backup",
+    "Create",
+    "Decrypt",
+    "Delete",
+    "Encrypt",
+    "Get",
+    "Import",
+    "List",
+    "Purge",
+    "Recover",
+    "Restore",
+    "Sign",
+    "UnwrapKey",
+    "Update",
+    "Verify",
+    "WrapKey",
+  ]
+
+  secret_permissions = [
+    "Backup",
+    "Delete",
+    "Get",
+    "List",
+    "Purge",
+    "Recover",
+    "Restore",
+    "Set",
+  ]
+}

tests/azure/terraform/shared/main.tf

@@ -0,0 +1,39 @@
+terraform {
+  backend "azurerm" {
+    resource_group_name  = "terraform-state"
+    storage_account_name = "terraformstate0419"
+    container_name       = "shared-tfstate"
+    key                  = "prod.terraform.tfstate"
+  }
+
+  required_version = "1.0.7"
+
+  required_providers {
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "2.76.0"
+    }
+    azuread = {
+      source = "hashicorp/azuread"
+      version = "1.6.0"
+    }
+  }
+}
+
+provider "azurerm" {
+  features {}
+}
+
+resource "random_pet" "suffix" {
+  length    = 1
+  separator = ""
+}
+
+data "azurerm_client_config" "current" {}
+
+data "azurerm_subscription" "current" {}
+
+resource "azurerm_resource_group" "this" {
+  name     = "e2e-shared"
+  location = "West Europe"
+}

tests/azure/terraform/shared/outputs.tf

@@ -0,0 +1,18 @@
+output "azure_devops_sp" {
+  value = {
+    client_id = azuread_service_principal.azure_devops.application_id
+    client_secret = azuread_application_password.azure_devops.value
+  }
+  sensitive = true
+}
+
+output "github_sp" {
+  value = {
+    tenant_id = data.azurerm_client_config.current.tenant_id
+    subscription_id = data.azurerm_client_config.current.subscription_id
+    client_id = azuread_service_principal.github.application_id
+    client_secret = azuread_application_password.github.value
+  }
+  sensitive = true
+}
+

tests/azure/terraform/shared/service-principal.tf

@@ -0,0 +1,105 @@
+# Service Principal used by Azure DevOps to push OCI and Helm Charts
+resource "azuread_application" "azure_devops" {
+  display_name = "azure-devops-${random_pet.suffix.id}"
+
+  required_resource_access {
+    resource_app_id = "00000003-0000-0000-c000-000000000000"
+
+    resource_access {
+      id   = "df021288-bdef-4463-88db-98f22de89214"
+      type = "Role"
+    }
+  }
+
+  required_resource_access {
+    resource_app_id = "00000002-0000-0000-c000-000000000000"
+
+    resource_access {
+      id   = "1cda74f2-2616-4834-b122-5cb1b07f8a59"
+      type = "Role"
+    }
+    resource_access {
+      id   = "78c8a3c8-a07e-4b9e-af1b-b5ccab50a175"
+      type = "Role"
+    }
+  }
+}
+
+resource "azuread_application_password" "azure_devops" {
+  display_name = "password"
+  application_object_id = azuread_application.azure_devops.object_id
+}
+
+resource "azuread_service_principal" "azure_devops" {
+  application_id = azuread_application.azure_devops.application_id
+}
+
+resource "azurerm_role_assignment" "azure_devops_acr" {
+  scope                = azurerm_container_registry.this.id
+  role_definition_name = "Contributor"
+  principal_id         = azuread_service_principal.azure_devops.object_id
+}
+
+# Service Principal that is used to run the tests in GitHub Actions
+resource "azuread_application" "github" {
+  display_name = "github-${random_pet.suffix.id}"
+
+  required_resource_access {
+    resource_app_id = "00000003-0000-0000-c000-000000000000"
+
+    resource_access {
+      id   = "df021288-bdef-4463-88db-98f22de89214"
+      type = "Role"
+    }
+  }
+
+  required_resource_access {
+    resource_app_id = "00000002-0000-0000-c000-000000000000"
+
+    resource_access {
+      id   = "1cda74f2-2616-4834-b122-5cb1b07f8a59"
+      type = "Role"
+    }
+    resource_access {
+      id   = "78c8a3c8-a07e-4b9e-af1b-b5ccab50a175"
+      type = "Role"
+    }
+  }
+}
+
+resource "azuread_application_password" "github" {
+  display_name = "password"
+  application_object_id = azuread_application.github.object_id
+}
+
+resource "azuread_service_principal" "github" {
+  application_id = azuread_application.github.application_id
+}
+
+data "azurerm_storage_account" "terraform_state" {
+  resource_group_name = "terraform-state"
+  name                = "terraformstate0419"
+}
+
+resource "azurerm_role_assignment" "github_resource_group" {
+  scope              = data.azurerm_subscription.current.id
+  role_definition_name = "Contributor"
+  principal_id       = azuread_service_principal.github.object_id
+}
+
+resource "azurerm_role_assignment" "github_acr" {
+  scope                = azurerm_container_registry.this.id
+  role_definition_name = "Owner"
+  principal_id         = azuread_service_principal.github.object_id
+}
+
+resource "azurerm_key_vault_access_policy" "github_keyvault_secret_read" {
+  key_vault_id = azurerm_key_vault.this.id
+  tenant_id = data.azurerm_client_config.current.tenant_id
+  object_id = azuread_service_principal.github.object_id
+
+  secret_permissions = [
+    "Get",
+    "List",
+  ]
+}

tests/azure/util_test.go

@@ -0,0 +1,461 @@
+package test
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+
+	git2go "github.com/libgit2/git2go/v31"
+
+	corev1 "k8s.io/api/core/v1"
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	helmv2beta1 "github.com/fluxcd/helm-controller/api/v2beta1"
+	automationv1beta1 "github.com/fluxcd/image-automation-controller/api/v1beta1"
+	reflectorv1beta1 "github.com/fluxcd/image-reflector-controller/api/v1beta1"
+	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1"
+	notiv1beta1 "github.com/fluxcd/notification-controller/api/v1beta1"
+	"github.com/fluxcd/pkg/apis/meta"
+	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
+)
+
+const defaultBranch = "main"
+
+// getKubernetesCredentials returns a path to a kubeconfig file and a kube client instance.
+func getKubernetesCredentials(kubeconfig, aksHost, aksCert, aksKey, aksCa string) (string, client.Client, error) {
+	tmpDir, err := ioutil.TempDir("", "*-azure-e2e")
+	if err != nil {
+		return "", nil, err
+	}
+	kubeconfigPath := fmt.Sprintf("%s/kubeconfig", tmpDir)
+	os.WriteFile(kubeconfigPath, []byte(kubeconfig), 0750)
+	kubeCfg := &rest.Config{
+		Host: aksHost,
+		TLSClientConfig: rest.TLSClientConfig{
+			CertData: []byte(aksCert),
+			KeyData:  []byte(aksKey),
+			CAData:   []byte(aksCa),
+		},
+	}
+	err = sourcev1.AddToScheme(scheme.Scheme)
+	if err != nil {
+		return "", nil, err
+	}
+	err = kustomizev1.AddToScheme(scheme.Scheme)
+	if err != nil {
+		return "", nil, err
+	}
+	err = helmv2beta1.AddToScheme(scheme.Scheme)
+	if err != nil {
+		return "", nil, err
+	}
+	err = reflectorv1beta1.AddToScheme(scheme.Scheme)
+	if err != nil {
+		return "", nil, err
+	}
+	err = automationv1beta1.AddToScheme(scheme.Scheme)
+	if err != nil {
+		return "", nil, err
+	}
+	err = notiv1beta1.AddToScheme(scheme.Scheme)
+	if err != nil {
+		return "", nil, err
+	}
+	kubeClient, err := client.New(kubeCfg, client.Options{Scheme: scheme.Scheme})
+	if err != nil {
+		return "", nil, err
+	}
+	return kubeconfigPath, kubeClient, nil
+}
+
+// installFlux adds the core Flux components to the cluster specified in the kubeconfig file.
+func installFlux(ctx context.Context, kubeClient client.Client, kubeconfigPath, repoUrl, azdoPat string, sp spConfig) error {
+	namespace := corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "flux-system",
+		},
+	}
+	_, err := controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, &namespace, func() error {
+		return nil
+	})
+	httpsCredentials := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "https-credentials", Namespace: "flux-system"}}
+	_, err = controllerutil.CreateOrUpdate(ctx, kubeClient, httpsCredentials, func() error {
+		httpsCredentials.StringData = map[string]string{
+			"username": "git",
+			"password": azdoPat,
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	azureSp := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "azure-sp", Namespace: "flux-system"}}
+	_, err = controllerutil.CreateOrUpdate(ctx, kubeClient, azureSp, func() error {
+		azureSp.StringData = map[string]string{
+			"AZURE_TENANT_ID":     sp.tenantId,
+			"AZURE_CLIENT_ID":     sp.clientId,
+			"AZURE_CLIENT_SECRET": sp.clientSecret,
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	// Install Flux and push files to git repository
+	repo, repoDir, err := getRepository(repoUrl, defaultBranch, true, azdoPat)
+	if err != nil {
+		return err
+	}
+	err = runCommand(ctx, repoDir, "mkdir -p ./clusters/e2e/flux-system")
+	if err != nil {
+		return err
+	}
+	err = runCommand(ctx, repoDir, "flux install --components-extra=\"image-reflector-controller,image-automation-controller\" --export > ./clusters/e2e/flux-system/gotk-components.yaml")
+	if err != nil {
+		return err
+	}
+	err = runCommand(ctx, repoDir, fmt.Sprintf("flux create source git flux-system --git-implementation=libgit2 --url=%s --branch=%s --secret-ref=https-credentials --interval=1m  --export > ./clusters/e2e/flux-system/gotk-sync.yaml", repoUrl, defaultBranch))
+	if err != nil {
+		return err
+	}
+	err = runCommand(ctx, repoDir, "flux create kustomization flux-system --source=flux-system --path='./clusters/e2e' --prune=true --interval=1m --export >> ./clusters/e2e/flux-system/gotk-sync.yaml")
+	if err != nil {
+		return err
+	}
+	kustomizeYaml := `
+  resources:
+    - gotk-components.yaml
+    - gotk-sync.yaml
+  patchesStrategicMerge:
+    - |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: kustomize-controller
+        namespace: flux-system
+      spec:
+        template:
+          spec:
+            containers:
+            - name: manager
+              envFrom:
+              - secretRef:
+                  name: azure-sp
+    - |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: source-controller
+        namespace: flux-system
+      spec:
+        template:
+          spec:
+            containers:
+            - name: manager
+              envFrom:
+              - secretRef:
+                  name: azure-sp
+  `
+	err = runCommand(ctx, repoDir, fmt.Sprintf("echo \"%s\" > ./clusters/e2e/flux-system/kustomization.yaml", kustomizeYaml))
+	if err != nil {
+		return err
+	}
+	err = commitAndPushAll(repo, defaultBranch, azdoPat)
+	if err != nil {
+		return err
+	}
+	// Need to apply CRDs first to make sure that the sync resources will apply properly
+	err = runCommand(ctx, repoDir, fmt.Sprintf("kubectl --kubeconfig=%s apply -f ./clusters/e2e/flux-system/gotk-components.yaml", kubeconfigPath))
+	if err != nil {
+		return err
+	}
+	err = runCommand(ctx, repoDir, fmt.Sprintf("kubectl --kubeconfig=%s apply -k ./clusters/e2e/flux-system/", kubeconfigPath))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func runCommand(ctx context.Context, dir, command string) error {
+	timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
+	defer cancel()
+	cmd := exec.CommandContext(timeoutCtx, "bash", "-c", command)
+	cmd.Dir = dir
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failure to run command %s: %v", string(output), err)
+	}
+	return nil
+}
+
+// verifyGitAndKustomization checks that the gitrespository and kustomization combination are working properly.
+func verifyGitAndKustomization(ctx context.Context, kubeClient client.Client, namespace, name string) error {
+	nn := types.NamespacedName{
+		Name:      name,
+		Namespace: namespace,
+	}
+	source := &sourcev1.GitRepository{}
+	err := kubeClient.Get(ctx, nn, source)
+	if err != nil {
+		return err
+	}
+	if apimeta.IsStatusConditionPresentAndEqual(source.Status.Conditions, meta.ReadyCondition, metav1.ConditionTrue) == false {
+		return fmt.Errorf("source condition not ready")
+	}
+	kustomization := &kustomizev1.Kustomization{}
+	err = kubeClient.Get(ctx, nn, kustomization)
+	if err != nil {
+		return err
+	}
+	if apimeta.IsStatusConditionPresentAndEqual(kustomization.Status.Conditions, meta.ReadyCondition, metav1.ConditionTrue) == false {
+		return fmt.Errorf("kustomization condition not ready")
+	}
+	return nil
+}
+
+func setupNamespace(ctx context.Context, kubeClient client.Client, repoUrl, password, name string) error {
+	namespace := corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+	}
+	_, err := controllerutil.CreateOrUpdate(ctx, kubeClient, &namespace, func() error {
+		return nil
+	})
+	secret := corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "https-credentials",
+			Namespace: name,
+		},
+	}
+	_, err = controllerutil.CreateOrUpdate(ctx, kubeClient, &secret, func() error {
+		secret.StringData = map[string]string{
+			"username": "git",
+			"password": password,
+		}
+		return nil
+	})
+	source := &sourcev1.GitRepository{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace.Name}}
+	_, err = controllerutil.CreateOrUpdate(ctx, kubeClient, source, func() error {
+		source.Spec = sourcev1.GitRepositorySpec{
+			Interval: metav1.Duration{
+				Duration: 1 * time.Minute,
+			},
+			GitImplementation: sourcev1.LibGit2Implementation,
+			Reference: &sourcev1.GitRepositoryRef{
+				Branch: name,
+			},
+			SecretRef: &meta.LocalObjectReference{
+				Name: "https-credentials",
+			},
+			URL: repoUrl,
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	kustomization := &kustomizev1.Kustomization{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace.Name}}
+	_, err = controllerutil.CreateOrUpdate(ctx, kubeClient, kustomization, func() error {
+		kustomization.Spec = kustomizev1.KustomizationSpec{
+			SourceRef: kustomizev1.CrossNamespaceSourceReference{
+				Kind:      sourcev1.GitRepositoryKind,
+				Name:      source.Name,
+				Namespace: source.Namespace,
+			},
+			Interval: metav1.Duration{
+				Duration: 1 * time.Minute,
+			},
+			Prune: true,
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func getRepository(url, branchName string, overrideBranch bool, password string) (*git2go.Repository, string, error) {
+	checkoutBranch := defaultBranch
+	if overrideBranch == false {
+		checkoutBranch = branchName
+	}
+
+	tmpDir, err := ioutil.TempDir("", "*-repository")
+	if err != nil {
+		return nil, "", err
+	}
+	repo, err := git2go.Clone(url, tmpDir, &git2go.CloneOptions{
+		FetchOptions: &git2go.FetchOptions{
+			RemoteCallbacks: git2go.RemoteCallbacks{
+				CredentialsCallback: credentialCallback("git", password),
+			},
+		},
+		CheckoutBranch: checkoutBranch,
+	})
+	if err != nil {
+		return nil, "", err
+	}
+	// Nothing to do further if correct branch is checked out
+	if checkoutBranch == branchName {
+		return repo, tmpDir, nil
+	}
+	head, err := repo.Head()
+	if err != nil {
+		return nil, "", err
+	}
+	headCommit, err := repo.LookupCommit(head.Target())
+	if err != nil {
+		return nil, "", err
+	}
+	_, err = repo.CreateBranch(branchName, headCommit, true)
+	if err != nil {
+		return nil, "", err
+	}
+	return repo, tmpDir, nil
+}
+
+func addFile(dir, path, content string) error {
+	err := os.WriteFile(filepath.Join(dir, path), []byte(content), 0777)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func commitAndPushAll(repo *git2go.Repository, branchName, password string) error {
+	idx, err := repo.Index()
+	if err != nil {
+		return err
+	}
+	err = idx.AddAll([]string{}, git2go.IndexAddDefault, nil)
+	if err != nil {
+		return err
+	}
+	treeId, err := idx.WriteTree()
+	if err != nil {
+		return err
+	}
+	err = idx.Write()
+	if err != nil {
+		return err
+	}
+	tree, err := repo.LookupTree(treeId)
+	if err != nil {
+		return err
+	}
+	branch, err := repo.LookupBranch(branchName, git2go.BranchLocal)
+	if err != nil {
+		return err
+	}
+	commitTarget, err := repo.LookupCommit(branch.Target())
+	if err != nil {
+		return err
+	}
+	sig := &git2go.Signature{
+		Name:  "git",
+		Email: "test@example.com",
+		When:  time.Now(),
+	}
+	_, err = repo.CreateCommit(fmt.Sprintf("refs/heads/%s", branchName), sig, sig, "add file", tree, commitTarget)
+	if err != nil {
+		return err
+	}
+	origin, err := repo.Remotes.Lookup("origin")
+	if err != nil {
+		return err
+	}
+	err = origin.Push([]string{fmt.Sprintf("+refs/heads/%s", branchName)}, &git2go.PushOptions{
+		RemoteCallbacks: git2go.RemoteCallbacks{
+			CredentialsCallback: credentialCallback("git", password),
+		},
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func createTagAndPush(repo *git2go.Repository, branchName, tag, password string) error {
+	branch, err := repo.LookupBranch(branchName, git2go.BranchAll)
+	if err != nil {
+		return err
+	}
+	commit, err := repo.LookupCommit(branch.Target())
+	if err != nil {
+		return err
+	}
+
+	tags, err := repo.Tags.List()
+	if err != nil {
+		return err
+	}
+	for _, existingTag := range tags {
+		if existingTag == tag {
+			err = repo.Tags.Remove(tag)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	sig := &git2go.Signature{
+		Name:  "git",
+		Email: "test@example.com",
+		When:  time.Now(),
+	}
+	_, err = repo.Tags.Create(tag, commit, sig, "create tag")
+	if err != nil {
+		return err
+	}
+	origin, err := repo.Remotes.Lookup("origin")
+	if err != nil {
+		return err
+	}
+	err = origin.Push([]string{fmt.Sprintf("+refs/tags/%s", tag)}, &git2go.PushOptions{
+		RemoteCallbacks: git2go.RemoteCallbacks{
+			CredentialsCallback: credentialCallback("git", password),
+		},
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func credentialCallback(username, password string) git2go.CredentialsCallback {
+	return func(url string, usernameFromURL string, allowedTypes git2go.CredType) (*git2go.Cred, error) {
+		cred, err := git2go.NewCredentialUserpassPlaintext(username, password)
+		if err != nil {
+			return nil, err
+		}
+		return cred, nil
+	}
+}
+
+func getTestManifest(namespace string) string {
+	return fmt.Sprintf(`
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: %s
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: foobar
+  namespace: %s
+`, namespace, namespace)
+}