implement testEnv for e2e tests

Signed-off-by: Chanwit Kaewkasi <chanwit@gmail.com>

cmd/flux/main_test.go

@@ -4,17 +4,23 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/fluxcd/flux2/internal/utils"
 	"github.com/google/go-cmp/cmp"
 	"github.com/mattn/go-shellwords"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	k8syaml "k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/client-go/tools/clientcmd"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
 )
 
 func TestMain(m *testing.M) {
@@ -49,17 +55,18 @@ func readYamlObjects(objectFile string) ([]client.Object, error) {
 }
 
 // A KubeManager that can create objects that are subject to a test.
-type fakeKubeManager struct {
-	fakeClient client.WithWatch
+type testEnvKubeManager struct {
+	client  client.WithWatch
+	testEnv *envtest.Environment
 }
 
-func (m *fakeKubeManager) NewClient(kubeconfig string, kubecontext string) (client.WithWatch, error) {
-	return m.fakeClient, nil
+func (m *testEnvKubeManager) NewClient(kubeconfig string, kubecontext string) (client.WithWatch, error) {
+	return m.client, nil
 }
 
-func (m *fakeKubeManager) CreateObjects(clientObjects []client.Object) error {
+func (m *testEnvKubeManager) CreateObjects(clientObjects []client.Object) error {
 	for _, obj := range clientObjects {
-		err := m.fakeClient.Create(context.Background(), obj)
+		err := m.client.Create(context.Background(), obj)
 		if err != nil {
 			return err
 		}
@@ -67,32 +74,92 @@ func (m *fakeKubeManager) CreateObjects(clientObjects []client.Object) error {
 	return nil
 }
 
-func NewFakeKubeManager() *fakeKubeManager {
-	c := fakeclient.NewClientBuilder().WithScheme(utils.NewScheme()).Build()
-	return &fakeKubeManager{
-		fakeClient: c,
+func (m *testEnvKubeManager) Stop() error {
+	if m.testEnv == nil {
+		return fmt.Errorf("do nothing because testEnv is nil")
 	}
+	return m.testEnv.Stop()
 }
 
-// Run the command and return the captured output.
-func executeCommand(cmd string) (string, error) {
-	args, err := shellwords.Parse(cmd)
-	if err != nil {
-		return "", err
+func NewTestEnvKubeManager(testClusterMode TestClusterMode) (*testEnvKubeManager, error) {
+	switch testClusterMode {
+	case FakeClusterMode:
+		c := fakeclient.NewClientBuilder().WithScheme(utils.NewScheme()).Build()
+		return &testEnvKubeManager{
+			client: c,
+		}, nil
+	case TestEnvClusterMode:
+		useExistingCluster := false
+		testEnv := &envtest.Environment{
+			UseExistingCluster: &useExistingCluster,
+		}
+		cfg, err := testEnv.Start()
+		if err != nil {
+			return nil, err
+		}
+		user, err := testEnv.ControlPlane.AddUser(envtest.User{
+			Name:   "envtest-admin",
+			Groups: []string{"system:masters"},
+		}, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		kubeConfig, err := user.KubeConfig()
+		if err != nil {
+			return nil, err
+		}
+
+		tmpFilename := filepath.Join("/tmp", "kubeconfig-"+time.Nanosecond.String())
+		ioutil.WriteFile(tmpFilename, kubeConfig, 0644)
+		rootArgs.kubeconfig = tmpFilename
+		k8sClient, err := client.NewWithWatch(cfg, client.Options{})
+		if err != nil {
+			return nil, err
+		}
+		return &testEnvKubeManager{
+			testEnv: testEnv,
+			client:  k8sClient,
+		}, nil
+	case ExistingClusterMode:
+		// TEST_KUBECONFIG is mandatory to prevent destroying a current cluster accidentally.
+		testKubeConfig := os.Getenv("TEST_KUBECONFIG")
+		if testKubeConfig == "" {
+			return nil, fmt.Errorf("environment variable TEST_KUBECONFIG is required to run tests against an existing cluster")
+		}
+		rootArgs.kubeconfig = testKubeConfig
+
+		useExistingCluster := true
+		config, err := clientcmd.BuildConfigFromFlags("", testKubeConfig)
+		testEnv := &envtest.Environment{
+			UseExistingCluster: &useExistingCluster,
+			Config:             config,
+		}
+		cfg, err := testEnv.Start()
+		if err != nil {
+			return nil, err
+		}
+		k8sClient, err := client.NewWithWatch(cfg, client.Options{})
+		if err != nil {
+			return nil, err
+		}
+		return &testEnvKubeManager{
+			testEnv: testEnv,
+			client:  k8sClient,
+		}, nil
 	}
 
-	buf := new(bytes.Buffer)
-
-	rootCmd.SetOut(buf)
-	rootCmd.SetErr(buf)
-	rootCmd.SetArgs(args)
-
-	_, err = rootCmd.ExecuteC()
-	result := buf.String()
-
-	return result, err
+	return nil, nil
 }
 
+type TestClusterMode int
+
+const (
+	FakeClusterMode = TestClusterMode(iota + 1)
+	TestEnvClusterMode
+	ExistingClusterMode
+)
+
 // Structure used for each test to load objects into kubernetes, run
 // commands and assert on the expected output.
 type cmdTestCase struct {
@@ -106,11 +173,20 @@ type cmdTestCase struct {
 	goldenFile string
 	// Filename that contains yaml objects to load into Kubernetes
 	objectFile string
+	// TestClusterMode to bootstrap and testing, default to Fake
+	testClusterMode TestClusterMode
 }
 
 func (cmd *cmdTestCase) runTestCmd(t *testing.T) {
-	km := NewFakeKubeManager()
-	rootCtx.kubeManager = km
+	km, err := NewTestEnvKubeManager(cmd.testClusterMode)
+	if err != nil {
+		t.Fatalf("Error creating kube manager: '%v'", err)
+	}
+
+	if km != nil {
+		rootCtx.kubeManager = km
+		defer km.Stop()
+	}
 
 	if cmd.objectFile != "" {
 		clientObjects, err := readYamlObjects(cmd.objectFile)
@@ -149,6 +225,27 @@ func (cmd *cmdTestCase) runTestCmd(t *testing.T) {
 	}
 }
 
+// Run the command and return the captured output.
+func executeCommand(cmd string) (string, error) {
+	args, err := shellwords.Parse(cmd)
+	if err != nil {
+		return "", err
+	}
+
+	buf := new(bytes.Buffer)
+
+	rootCmd.SetOut(buf)
+	rootCmd.SetErr(buf)
+	rootCmd.SetArgs(args)
+
+	logger.stderr = rootCmd.ErrOrStderr()
+
+	_, err = rootCmd.ExecuteC()
+	result := buf.String()
+
+	return result, err
+}
+
 func TestVersion(t *testing.T) {
 	cmd := cmdTestCase{
 		args:        "--version",