flux2/tests/azure/util_test.go

package test

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	git2go "github.com/libgit2/git2go/v31"

	corev1 "k8s.io/api/core/v1"
	apimeta "k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/rest"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

	helmv2beta1 "github.com/fluxcd/helm-controller/api/v2beta1"
	automationv1beta1 "github.com/fluxcd/image-automation-controller/api/v1beta1"
	reflectorv1beta1 "github.com/fluxcd/image-reflector-controller/api/v1beta1"
	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2"
	notiv1beta1 "github.com/fluxcd/notification-controller/api/v1beta1"
	"github.com/fluxcd/pkg/apis/meta"
	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
)

const defaultBranch = "main"

// getKubernetesCredentials returns a path to a kubeconfig file and a kube client instance.
func getKubernetesCredentials(kubeconfig, aksHost, aksCert, aksKey, aksCa string) (string, client.Client, error) {
	tmpDir, err := ioutil.TempDir("", "*-azure-e2e")
	if err != nil {
		return "", nil, err
	}
	kubeconfigPath := fmt.Sprintf("%s/kubeconfig", tmpDir)
	os.WriteFile(kubeconfigPath, []byte(kubeconfig), 0750)
	kubeCfg := &rest.Config{
		Host: aksHost,
		TLSClientConfig: rest.TLSClientConfig{
			CertData: []byte(aksCert),
			KeyData:  []byte(aksKey),
			CAData:   []byte(aksCa),
		},
	}
	err = sourcev1.AddToScheme(scheme.Scheme)
	if err != nil {
		return "", nil, err
	}
	err = kustomizev1.AddToScheme(scheme.Scheme)
	if err != nil {
		return "", nil, err
	}
	err = helmv2beta1.AddToScheme(scheme.Scheme)
	if err != nil {
		return "", nil, err
	}
	err = reflectorv1beta1.AddToScheme(scheme.Scheme)
	if err != nil {
		return "", nil, err
	}
	err = automationv1beta1.AddToScheme(scheme.Scheme)
	if err != nil {
		return "", nil, err
	}
	err = notiv1beta1.AddToScheme(scheme.Scheme)
	if err != nil {
		return "", nil, err
	}
	kubeClient, err := client.New(kubeCfg, client.Options{Scheme: scheme.Scheme})
	if err != nil {
		return "", nil, err
	}
	return kubeconfigPath, kubeClient, nil
}

// installFlux adds the core Flux components to the cluster specified in the kubeconfig file.
func installFlux(ctx context.Context, kubeClient client.Client, kubeconfigPath, repoUrl, azdoPat string, sp spConfig) error {
	namespace := corev1.Namespace{
		ObjectMeta: metav1.ObjectMeta{
			Name: "flux-system",
		},
	}
	_, err := controllerutil.CreateOrUpdate(ctx, cfg.kubeClient, &namespace, func() error {
		return nil
	})
	httpsCredentials := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "https-credentials", Namespace: "flux-system"}}
	_, err = controllerutil.CreateOrUpdate(ctx, kubeClient, httpsCredentials, func() error {
		httpsCredentials.StringData = map[string]string{
			"username": "git",
			"password": azdoPat,
		}
		return nil
	})
	if err != nil {
		return err
	}
	azureSp := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "azure-sp", Namespace: "flux-system"}}
	_, err = controllerutil.CreateOrUpdate(ctx, kubeClient, azureSp, func() error {
		azureSp.StringData = map[string]string{
			"AZURE_TENANT_ID":     sp.tenantId,
			"AZURE_CLIENT_ID":     sp.clientId,
			"AZURE_CLIENT_SECRET": sp.clientSecret,
		}
		return nil
	})
	if err != nil {
		return err
	}

	// Install Flux and push files to git repository
	repo, repoDir, err := getRepository(repoUrl, defaultBranch, true, azdoPat)
	if err != nil {
		return err
	}
	err = runCommand(ctx, repoDir, "mkdir -p ./clusters/e2e/flux-system")
	if err != nil {
		return err
	}
	err = runCommand(ctx, repoDir, "flux install --components-extra=\"image-reflector-controller,image-automation-controller\" --export > ./clusters/e2e/flux-system/gotk-components.yaml")
	if err != nil {
		return err
	}
	err = runCommand(ctx, repoDir, fmt.Sprintf("flux create source git flux-system --git-implementation=libgit2 --url=%s --branch=%s --secret-ref=https-credentials --interval=1m  --export > ./clusters/e2e/flux-system/gotk-sync.yaml", repoUrl, defaultBranch))
	if err != nil {
		return err
	}
	err = runCommand(ctx, repoDir, "flux create kustomization flux-system --source=flux-system --path='./clusters/e2e' --prune=true --interval=1m --export >> ./clusters/e2e/flux-system/gotk-sync.yaml")
	if err != nil {
		return err
	}
	kustomizeYaml := `
  resources:
    - gotk-components.yaml
    - gotk-sync.yaml
  patchesStrategicMerge:
    - |-
      apiVersion: apps/v1
      kind: Deployment
      metadata:
        name: kustomize-controller
        namespace: flux-system
      spec:
        template:
          spec:
            containers:
            - name: manager
              envFrom:
              - secretRef:
                  name: azure-sp
    - |-
      apiVersion: apps/v1
      kind: Deployment
      metadata:
        name: source-controller
        namespace: flux-system
      spec:
        template:
          spec:
            containers:
            - name: manager
              envFrom:
              - secretRef:
                  name: azure-sp
  `
	err = runCommand(ctx, repoDir, fmt.Sprintf("echo \"%s\" > ./clusters/e2e/flux-system/kustomization.yaml", kustomizeYaml))
	if err != nil {
		return err
	}
	err = commitAndPushAll(repo, defaultBranch, azdoPat)
	if err != nil {
		return err
	}
	// Need to apply CRDs first to make sure that the sync resources will apply properly
	err = runCommand(ctx, repoDir, fmt.Sprintf("kubectl --kubeconfig=%s apply -f ./clusters/e2e/flux-system/gotk-components.yaml", kubeconfigPath))
	if err != nil {
		return err
	}
	err = runCommand(ctx, repoDir, fmt.Sprintf("kubectl --kubeconfig=%s apply -k ./clusters/e2e/flux-system/", kubeconfigPath))
	if err != nil {
		return err
	}
	return nil
}

func runCommand(ctx context.Context, dir, command string) error {
	timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
	defer cancel()
	cmd := exec.CommandContext(timeoutCtx, "bash", "-c", command)
	cmd.Dir = dir
	output, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("failure to run command %s: %v", string(output), err)
	}
	return nil
}

// verifyGitAndKustomization checks that the gitrespository and kustomization combination are working properly.
func verifyGitAndKustomization(ctx context.Context, kubeClient client.Client, namespace, name string) error {
	nn := types.NamespacedName{
		Name:      name,
		Namespace: namespace,
	}
	source := &sourcev1.GitRepository{}
	err := kubeClient.Get(ctx, nn, source)
	if err != nil {
		return err
	}
	if apimeta.IsStatusConditionPresentAndEqual(source.Status.Conditions, meta.ReadyCondition, metav1.ConditionTrue) == false {
		return fmt.Errorf("source condition not ready")
	}
	kustomization := &kustomizev1.Kustomization{}
	err = kubeClient.Get(ctx, nn, kustomization)
	if err != nil {
		return err
	}
	if apimeta.IsStatusConditionPresentAndEqual(kustomization.Status.Conditions, meta.ReadyCondition, metav1.ConditionTrue) == false {
		return fmt.Errorf("kustomization condition not ready")
	}
	return nil
}

func setupNamespace(ctx context.Context, kubeClient client.Client, repoUrl, password, name string) error {
	namespace := corev1.Namespace{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
		},
	}
	_, err := controllerutil.CreateOrUpdate(ctx, kubeClient, &namespace, func() error {
		return nil
	})
	secret := corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "https-credentials",
			Namespace: name,
		},
	}
	_, err = controllerutil.CreateOrUpdate(ctx, kubeClient, &secret, func() error {
		secret.StringData = map[string]string{
			"username": "git",
			"password": password,
		}
		return nil
	})
	source := &sourcev1.GitRepository{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace.Name}}
	_, err = controllerutil.CreateOrUpdate(ctx, kubeClient, source, func() error {
		source.Spec = sourcev1.GitRepositorySpec{
			Interval: metav1.Duration{
				Duration: 1 * time.Minute,
			},
			GitImplementation: sourcev1.LibGit2Implementation,
			Reference: &sourcev1.GitRepositoryRef{
				Branch: name,
			},
			SecretRef: &meta.LocalObjectReference{
				Name: "https-credentials",
			},
			URL: repoUrl,
		}
		return nil
	})
	if err != nil {
		return err
	}
	kustomization := &kustomizev1.Kustomization{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace.Name}}
	_, err = controllerutil.CreateOrUpdate(ctx, kubeClient, kustomization, func() error {
		kustomization.Spec = kustomizev1.KustomizationSpec{
			SourceRef: kustomizev1.CrossNamespaceSourceReference{
				Kind:      sourcev1.GitRepositoryKind,
				Name:      source.Name,
				Namespace: source.Namespace,
			},
			Interval: metav1.Duration{
				Duration: 1 * time.Minute,
			},
			Prune: true,
		}
		return nil
	})
	if err != nil {
		return err
	}
	return nil
}

func getRepository(url, branchName string, overrideBranch bool, password string) (*git2go.Repository, string, error) {
	checkoutBranch := defaultBranch
	if overrideBranch == false {
		checkoutBranch = branchName
	}

	tmpDir, err := ioutil.TempDir("", "*-repository")
	if err != nil {
		return nil, "", err
	}
	repo, err := git2go.Clone(url, tmpDir, &git2go.CloneOptions{
		FetchOptions: &git2go.FetchOptions{
			RemoteCallbacks: git2go.RemoteCallbacks{
				CredentialsCallback: credentialCallback("git", password),
			},
		},
		CheckoutBranch: checkoutBranch,
	})
	if err != nil {
		return nil, "", err
	}
	// Nothing to do further if correct branch is checked out
	if checkoutBranch == branchName {
		return repo, tmpDir, nil
	}
	head, err := repo.Head()
	if err != nil {
		return nil, "", err
	}
	headCommit, err := repo.LookupCommit(head.Target())
	if err != nil {
		return nil, "", err
	}
	_, err = repo.CreateBranch(branchName, headCommit, true)
	if err != nil {
		return nil, "", err
	}
	return repo, tmpDir, nil
}

func addFile(dir, path, content string) error {
	err := os.WriteFile(filepath.Join(dir, path), []byte(content), 0777)
	if err != nil {
		return err
	}
	return nil
}

func commitAndPushAll(repo *git2go.Repository, branchName, password string) error {
	idx, err := repo.Index()
	if err != nil {
		return err
	}
	err = idx.AddAll([]string{}, git2go.IndexAddDefault, nil)
	if err != nil {
		return err
	}
	treeId, err := idx.WriteTree()
	if err != nil {
		return err
	}
	err = idx.Write()
	if err != nil {
		return err
	}
	tree, err := repo.LookupTree(treeId)
	if err != nil {
		return err
	}
	branch, err := repo.LookupBranch(branchName, git2go.BranchLocal)
	if err != nil {
		return err
	}
	commitTarget, err := repo.LookupCommit(branch.Target())
	if err != nil {
		return err
	}
	sig := &git2go.Signature{
		Name:  "git",
		Email: "test@example.com",
		When:  time.Now(),
	}
	_, err = repo.CreateCommit(fmt.Sprintf("refs/heads/%s", branchName), sig, sig, "add file", tree, commitTarget)
	if err != nil {
		return err
	}
	origin, err := repo.Remotes.Lookup("origin")
	if err != nil {
		return err
	}
	err = origin.Push([]string{fmt.Sprintf("+refs/heads/%s", branchName)}, &git2go.PushOptions{
		RemoteCallbacks: git2go.RemoteCallbacks{
			CredentialsCallback: credentialCallback("git", password),
		},
	})
	if err != nil {
		return err
	}
	return nil
}

func createTagAndPush(repo *git2go.Repository, branchName, tag, password string) error {
	branch, err := repo.LookupBranch(branchName, git2go.BranchAll)
	if err != nil {
		return err
	}
	commit, err := repo.LookupCommit(branch.Target())
	if err != nil {
		return err
	}

	tags, err := repo.Tags.List()
	if err != nil {
		return err
	}
	for _, existingTag := range tags {
		if existingTag == tag {
			err = repo.Tags.Remove(tag)
			if err != nil {
				return err
			}
		}
	}

	sig := &git2go.Signature{
		Name:  "git",
		Email: "test@example.com",
		When:  time.Now(),
	}
	_, err = repo.Tags.Create(tag, commit, sig, "create tag")
	if err != nil {
		return err
	}
	origin, err := repo.Remotes.Lookup("origin")
	if err != nil {
		return err
	}
	err = origin.Push([]string{fmt.Sprintf("+refs/tags/%s", tag)}, &git2go.PushOptions{
		RemoteCallbacks: git2go.RemoteCallbacks{
			CredentialsCallback: credentialCallback("git", password),
		},
	})
	if err != nil {
		return err
	}
	return nil
}

func credentialCallback(username, password string) git2go.CredentialsCallback {
	return func(url string, usernameFromURL string, allowedTypes git2go.CredType) (*git2go.Cred, error) {
		cred, err := git2go.NewCredentialUserpassPlaintext(username, password)
		if err != nil {
			return nil, err
		}
		return cred, nil
	}
}

func getTestManifest(namespace string) string {
	return fmt.Sprintf(`
apiVersion: v1
kind: Namespace
metadata:
  name: %s
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: foobar
  namespace: %s
`, namespace, namespace)
}