Refactor components status check

- run install/bootstrap checks in parallel (1m timeout)
- list not found components

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>

cmd/flux/bootstrap.go

@@ -126,7 +126,7 @@ func generateInstallManifests(targetPath, namespace, tmpDir string, localManifes
 	opts := install.Options{
 		BaseURL:                localManifests,
 		Version:                bootstrapArgs.version,
-		Namespace:              rootArgs.namespace,
+		Namespace:              namespace,
 		Components:             bootstrapComponents(),
 		Registry:               bootstrapArgs.registry,
 		ImagePullSecret:        bootstrapArgs.imagePullSecret,
@@ -162,14 +162,14 @@ func applyInstallManifests(ctx context.Context, manifestPath string, components
 		return fmt.Errorf("install failed")
 	}
 
-	statusChecker := StatusChecker{}
+	statusChecker, err := NewStatusChecker(time.Second, time.Minute)
-	err := statusChecker.New(time.Second, rootArgs.timeout)
 	if err != nil {
-		return fmt.Errorf("install failed with: %v", err)
+		return fmt.Errorf("install failed: %w", err)
 	}
-	err = statusChecker.Assess(components...)
+
-	if err != nil {
+	logger.Waitingf("verifying installation")
-		return fmt.Errorf("install timed out waiting for rollout")
+	if err := statusChecker.Assess(components...); err != nil {
+		return fmt.Errorf("install failed")
 	}
 
 	return nil

cmd/flux/check.go

@@ -46,8 +46,9 @@ the local environment is configured correctly and if the installed components ar
 }
 
 type checkFlags struct {
-	pre        bool
+	pre             bool
-	components []string
+	components      []string
+	extraComponents []string
 }
 
 type kubectlVersion struct {
@@ -61,6 +62,8 @@ func init() {
 		"only run pre-installation checks")
 	checkCmd.Flags().StringSliceVar(&checkArgs.components, "components", rootArgs.defaults.Components,
 		"list of components, accepts comma-separated values")
+	checkCmd.Flags().StringSliceVar(&checkArgs.extraComponents, "components-extra", nil,
+		"list of components in addition to those supplied or defaulted, accepts comma-separated values")
 	rootCmd.AddCommand(checkCmd)
 }
 
@@ -173,21 +176,20 @@ func componentsCheck() bool {
 	ctx, cancel := context.WithTimeout(context.Background(), rootArgs.timeout)
 	defer cancel()
 
-	statusChecker := StatusChecker{}
+	statusChecker, err := NewStatusChecker(time.Second, 30*time.Second)
-	err := statusChecker.New(time.Second, rootArgs.timeout)
 	if err != nil {
 		return false
 	}
 
 	ok := true
-	for _, deployment := range checkArgs.components {
+	deployments := append(checkArgs.components, checkArgs.extraComponents...)
-		err = statusChecker.Assess(deployment)
+	for _, deployment := range deployments {
-		if err != nil {
+		if err := statusChecker.Assess(deployment); err != nil {
-			logger.Failuref("%s: timed out waiting for rollout", deployment)
 			ok = false
 		} else {
-			logger.Successf("%s: successfully rolled out", deployment)
+			logger.Successf("%s: healthy", deployment)
 		}
+
 		kubectlArgs := []string{"-n", rootArgs.namespace, "get", "deployment", deployment, "-o", "jsonpath=\"{..image}\""}
 		if output, err := utils.ExecKubectlCommand(ctx, utils.ModeCapture, rootArgs.kubeconfig, rootArgs.kubecontext, kubectlArgs...); err == nil {
 			logger.Actionf(strings.TrimPrefix(strings.TrimSuffix(output, "\""), "\""))

cmd/flux/install.go

@@ -175,23 +175,16 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
 	if installDryRun {
 		logger.Successf("install dry-run finished")
 		return nil
-	} else {
-		logger.Successf("install completed")
 	}
 
-	statusChecker := StatusChecker{}
+	statusChecker, err := NewStatusChecker(time.Second, time.Minute)
-	err = statusChecker.New(time.Second, rootArgs.timeout)
 	if err != nil {
-		return fmt.Errorf("install failed with: %v", err)
+		return fmt.Errorf("install failed: %w", err)
 	}
 
 	logger.Waitingf("verifying installation")
-	for _, deployment := range components {
+	if err := statusChecker.Assess(components...); err != nil {
-		err := statusChecker.Assess(deployment)
+		return fmt.Errorf("install failed")
-		if err != nil {
-			return fmt.Errorf("%s: install failed while rolling out deployment", deployment)
-		}
-		logger.Successf("%s ready", deployment)
 	}
 
 	logger.Successf("install finished")

cmd/flux/status.go

@@ -19,9 +19,9 @@ package main
 import (
 	"context"
 	"fmt"
-	"strings"
 	"time"
 
+	appsv1 "k8s.io/api/apps/v1"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -53,6 +53,7 @@ type statusable interface {
 type StatusChecker struct {
 	pollInterval time.Duration
 	timeout      time.Duration
+	client       client.Client
 	statusPoller *polling.StatusPoller
 }
 
@@ -81,24 +82,26 @@ func isReady(ctx context.Context, kubeClient client.Client,
 	}
 }
 
-func (sc *StatusChecker) New(pollInterval time.Duration, timeout time.Duration) error {
+func NewStatusChecker(pollInterval time.Duration, timeout time.Duration) (*StatusChecker, error) {
 	kubeConfig, err := utils.KubeConfig(rootArgs.kubeconfig, rootArgs.kubecontext)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	restMapper, err := apiutil.NewDynamicRESTMapper(kubeConfig)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	client, err := client.New(kubeConfig, client.Options{Mapper: restMapper})
 	if err != nil {
-		return err
+		return nil, err
 	}
-	statusPoller := polling.NewStatusPoller(client, restMapper)
+
-	sc.statusPoller = statusPoller
+	return &StatusChecker{
-	sc.pollInterval = pollInterval
+		pollInterval: pollInterval,
-	sc.timeout = timeout
+		timeout:      timeout,
-	return err
+		client:       client,
+		statusPoller: polling.NewStatusPoller(client, restMapper),
+	}, nil
 }
 
 func (sc *StatusChecker) Assess(components ...string) error {
@@ -130,20 +133,19 @@ func (sc *StatusChecker) Assess(components ...string) error {
 	)
 	<-done
 
-	if coll.Error != nil {
+	if coll.Error != nil || ctx.Err() == context.DeadlineExceeded {
-		return coll.Error
-	}
-
-	if ctx.Err() == context.DeadlineExceeded {
-		ids := []string{}
 		for _, rs := range coll.ResourceStatuses {
 			if rs.Status != status.CurrentStatus {
-				id := sc.objMetadataToString(rs.Identifier)
+				if !sc.deploymentExists(rs.Identifier) {
-				ids = append(ids, id)
+					logger.Failuref("%s: deployment not found", rs.Identifier.Name)
+				} else {
+					logger.Failuref("%s: unhealthy (timed out waiting for rollout)", rs.Identifier.Name)
+				}
 			}
 		}
-		return fmt.Errorf("Status check timed out for component(s): [%v]", strings.Join(ids, ", "))
+		return fmt.Errorf("timed out waiting for condition")
 	}
+
 	return nil
 }
 
@@ -162,3 +164,16 @@ func (sc *StatusChecker) getObjectRefs(components []string) ([]object.ObjMetadat
 func (sc *StatusChecker) objMetadataToString(om object.ObjMetadata) string {
 	return fmt.Sprintf("%s '%s/%s'", om.GroupKind.Kind, om.Namespace, om.Name)
 }
+
+func (sc *StatusChecker) deploymentExists(om object.ObjMetadata) bool {
+	ctx, cancel := context.WithTimeout(context.Background(), sc.timeout)
+	defer cancel()
+
+	namespacedName := types.NamespacedName{
+		Namespace: om.Namespace,
+		Name:      om.Name,
+	}
+	var existing appsv1.Deployment
+	err := sc.client.Get(ctx, namespacedName, &existing)
+	return err == nil
+}

docs/cmd/flux_check.md

@@ -25,9 +25,10 @@ flux check [flags]
 ### Options
 
 ```
-      --components strings   list of components, accepts comma-separated values (default [source-controller,kustomize-controller,helm-controller,notification-controller])
+      --components strings         list of components, accepts comma-separated values (default [source-controller,kustomize-controller,helm-controller,notification-controller])
-  -h, --help                 help for check
+      --components-extra strings   list of components in addition to those supplied or defaulted, accepts comma-separated values
-      --pre                  only run pre-installation checks
+  -h, --help                       help for check
+      --pre                        only run pre-installation checks
 ```
 
 ### Options inherited from parent commands