implement e2e tests for install manifests

Signed-off-by: Chanwit Kaewkasi <chanwit@gmail.com>

Makefile

@@ -17,6 +17,9 @@ vet:
 test: $(EMBEDDED_MANIFESTS_TARGET) tidy fmt vet
 	go test ./... -coverprofile cover.out
 
+e2e: $(EMBEDDED_MANIFESTS_TARGET) tidy fmt vet build
+	go test -v --tags=e2e ./test/e2e -coverprofile cover.out
+
 $(EMBEDDED_MANIFESTS_TARGET): $(call rwildcard,manifests/,*.yaml *.json)
 	./manifests/scripts/bundle.sh
 

go.mod

@@ -23,7 +23,11 @@ require (
 	github.com/olekukonko/tablewriter v0.0.4
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5
+	github.com/stretchr/testify v1.7.0
+	github.com/xlab/treeprint v1.1.0 // indirect
 	golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b
+	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
+	gotest.tools v2.2.0+incompatible
 	k8s.io/api v0.21.1
 	k8s.io/apiextensions-apiserver v0.21.1
 	k8s.io/apimachinery v0.21.1

go.sum

@@ -744,8 +744,9 @@ github.com/xanzy/go-gitlab v0.43.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfD
 github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI=
 github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
+github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk=
+github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -960,8 +961,9 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=

pkg/status/status.go

@@ -19,6 +19,7 @@ package status
 import (
 	"context"
 	"fmt"
+	"sort"
 	"strings"
 	"time"
 
@@ -74,7 +75,13 @@ func (sc *StatusChecker) Assess(identifiers ...object.ObjMetadata) error {
 
 	<-done
 
-	for _, rs := range coll.ResourceStatuses {
+	// we use sorted identifiers to loop over the resource statuses because a Go's map is unordered.
+	// sorting identifiers by object's name makes sure that the logs look stable for every run
+	sort.SliceStable(identifiers, func(i, j int) bool {
+		return strings.Compare(identifiers[i].Name, identifiers[j].Name) < 0
+	})
+	for _, id := range identifiers {
+		rs := coll.ResourceStatuses[id]
 		switch rs.Status {
 		case status.CurrentStatus:
 			sc.logger.Successf("%s: %s ready", rs.Identifier.Name, strings.ToLower(rs.Identifier.GroupKind.Kind))

test/e2e/e2e_init.go

@@ -0,0 +1,64 @@
+// +build e2e
+
+package e2e
+
+import (
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var fluxBinary string
+
+func init() {
+	fluxBinary = os.Getenv("FLUX_E2E_BINARY")
+	if fluxBinary == "" {
+		fluxBinary = "../../bin/flux"
+	}
+}
+
+func run(name string, args ...string) ([]string, error) {
+	cmd := exec.Command(name, args...)
+	out, err := cmd.CombinedOutput()
+	lines := strings.Split(string(out), "\n")
+	return lines, err
+}
+
+func runAsBytes(name string, args ...string) ([]byte, error) {
+	cmd := exec.Command(name, args...)
+	return cmd.CombinedOutput()
+}
+
+func getAndWaitUntilObjectNotFound(args ...string) {
+	for ; ; {
+		var lines []string
+		var err error
+
+		lines, err = run(fluxBinary, append([]string{"get"}, args...)...)
+		if err != nil {
+			return
+		}
+
+		if args[0] == "source" &&
+			args[1] == "git" &&
+			lines[0] == "✗ no GitRepository objects found in flux-system namespace" {
+			return
+		}
+
+		if args[0] == "kustomization" &&
+			lines[0] == "✗ no Kustomization objects found in flux-system namespace" {
+			return
+		}
+
+		time.Sleep(3 * time.Second)
+	}
+}
+
+func deleteObject(t *testing.T, args ...string) {
+	_, err := run(fluxBinary, append([]string{"delete"}, args...)...)
+	assert.NoError(t, err)
+}

test/e2e/e2e_test.go

@@ -0,0 +1,24 @@
+// +build e2e
+
+package e2e
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"gotest.tools/golden"
+)
+
+func TestFluxInstallManifests(t *testing.T) {
+	actual, err := runAsBytes(fluxBinary, "install")
+	assert.NoError(t, err)
+
+	golden.Assert(t, string(actual), "../../../testdata/flux_install_manifests.golden")
+}
+
+func TestFluxInstallManifestsExtra(t *testing.T) {
+	actual, err := runAsBytes(fluxBinary, "install", "--components-extra", "image-reflector-controller,image-automation-controller")
+	assert.NoError(t, err)
+
+	golden.Assert(t, string(actual), "../../../testdata/flux_install_manifests_extra.golden")
+}

testdata/flux_install_manifests.golden

@@ -0,0 +1,9 @@
+✚ generating manifests
+✔ manifests build completed
+► installing components in flux-system namespace
+◎ verifying installation
+✔ helm-controller: deployment ready
+✔ kustomize-controller: deployment ready
+✔ notification-controller: deployment ready
+✔ source-controller: deployment ready
+✔ install finished

testdata/flux_install_manifests_extra.golden

@@ -0,0 +1,11 @@
+✚ generating manifests
+✔ manifests build completed
+► installing components in flux-system namespace
+◎ verifying installation
+✔ helm-controller: deployment ready
+✔ image-automation-controller: deployment ready
+✔ image-reflector-controller: deployment ready
+✔ kustomize-controller: deployment ready
+✔ notification-controller: deployment ready
+✔ source-controller: deployment ready
+✔ install finished