From ea451e7e498c9c3e052c24644f93a2bbdea9b955 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Fri, 9 Apr 2021 13:47:54 +0200 Subject: [PATCH] Always report components health in bootstrap This is useful in case the `Kustomization` does not reconcile successfully because for example the controller(s) are in a crash loop, which is not visible in the resource itself. Signed-off-by: Hidde Beydals --- internal/bootstrap/bootstrap.go | 29 ++++++++++++++++------ internal/bootstrap/bootstrap_plain_git.go | 30 ++++++++++++++++------- internal/bootstrap/bootstrap_provider.go | 4 +-- internal/bootstrap/git/git.go | 1 + internal/bootstrap/git/gogit/gogit.go | 11 +++++++++ 5 files changed, 57 insertions(+), 18 deletions(-) diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index 45fc0803..34d54840 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -58,11 +58,15 @@ type Reconciler interface { // ReconcileSyncConfig reconciles the sync configuration by generating // the sync manifests with the provided values, committing them to Git // and pushing to remote if there are any changes. - ReconcileSyncConfig(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error + ReconcileSyncConfig(ctx context.Context, options sync.Options) error - // ConfirmHealthy confirms that the components and extra components in - // install.Options are healthy. - ConfirmHealthy(ctx context.Context, options install.Options, timeout time.Duration) error + // ReportKustomizationHealth reports about the health of the + // Kustomization synchronizing the components. + ReportKustomizationHealth(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error + + // ReportComponentsHealth reports about the health for the components + // and extra components in install.Options. + ReportComponentsHealth(ctx context.Context, options install.Options, timeout time.Duration) error } type RepositoryReconciler interface { @@ -89,11 +93,22 @@ func Run(ctx context.Context, reconciler Reconciler, manifestsBase string, if err := reconciler.ReconcileSourceSecret(ctx, secretOpts); err != nil { return err } - if err := reconciler.ReconcileSyncConfig(ctx, syncOpts, pollInterval, timeout); err != nil { + if err := reconciler.ReconcileSyncConfig(ctx, syncOpts); err != nil { return err } - if err := reconciler.ConfirmHealthy(ctx, installOpts, timeout); err != nil { - return err + + var healthErrCount int + if err := reconciler.ReportKustomizationHealth(ctx, syncOpts, pollInterval, timeout); err != nil { + healthErrCount++ + } + if err := reconciler.ReportComponentsHealth(ctx, installOpts, timeout); err != nil { + healthErrCount++ + } + if healthErrCount > 0 { + // Composing a "smart" error message here from the returned + // errors does not result in any useful information for the + // user, as both methods log the failures they run into. + err = fmt.Errorf("bootstrap failed with %d health check failure(s)", healthErrCount) } return err diff --git a/internal/bootstrap/bootstrap_plain_git.go b/internal/bootstrap/bootstrap_plain_git.go index b3ee125e..19fbae18 100644 --- a/internal/bootstrap/bootstrap_plain_git.go +++ b/internal/bootstrap/bootstrap_plain_git.go @@ -34,7 +34,6 @@ import ( kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1" "github.com/fluxcd/flux2/internal/bootstrap/git" - "github.com/fluxcd/flux2/internal/utils" "github.com/fluxcd/flux2/pkg/log" "github.com/fluxcd/flux2/pkg/manifestgen/install" @@ -207,7 +206,7 @@ func (b *PlainGitBootstrapper) ReconcileSourceSecret(ctx context.Context, option return nil } -func (b *PlainGitBootstrapper) ReconcileSyncConfig(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error { +func (b *PlainGitBootstrapper) ReconcileSyncConfig(ctx context.Context, options sync.Options) error { // Confirm that sync configuration does not overwrite existing config if curPath, err := kustomizationPathDiffers(ctx, b.kube, client.ObjectKey{Name: options.Name, Namespace: options.Namespace}, options.TargetPath); err != nil { return fmt.Errorf("failed to determine if sync configuration would overwrite existing Kustomization: %w", err) @@ -283,22 +282,35 @@ func (b *PlainGitBootstrapper) ReconcileSyncConfig(ctx context.Context, options if _, err = utils.ExecKubectlCommand(ctx, utils.ModeStderrOS, b.kubeconfig, b.kubecontext, kubectlArgs...); err != nil { return err } - b.logger.Successf("applied sync manifests") + b.logger.Successf("reconciled sync configuration") - // Wait till Kustomization is reconciled + return nil +} + +func (b *PlainGitBootstrapper) ReportKustomizationHealth(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error { + head, err := b.git.Head() + if err != nil { + return err + } + + objKey := client.ObjectKey{Name: options.Name, Namespace: options.Namespace} + + b.logger.Waitingf("waiting for Kustomization %q to be reconciled", objKey.String()) + + expectRevision := fmt.Sprintf("%s/%s", options.Branch, head) var k kustomizev1.Kustomization - expectRevision := fmt.Sprintf("%s/%s", options.Branch, commit) if err := wait.PollImmediate(pollInterval, timeout, kustomizationReconciled( - ctx, b.kube, client.ObjectKey{Name: options.Name, Namespace: options.Namespace}, &k, expectRevision), + ctx, b.kube, objKey, &k, expectRevision), ); err != nil { - return fmt.Errorf("failed waiting for Kustomization: %w", err) + b.logger.Failuref(err.Error()) + return err } - b.logger.Successf("reconciled sync configuration") + b.logger.Successf("Kustomization reconciled successfully") return nil } -func (b *PlainGitBootstrapper) ConfirmHealthy(ctx context.Context, install install.Options, timeout time.Duration) error { +func (b *PlainGitBootstrapper) ReportComponentsHealth(ctx context.Context, install install.Options, timeout time.Duration) error { cfg, err := utils.KubeConfig(b.kubeconfig, b.kubecontext) if err != nil { return err diff --git a/internal/bootstrap/bootstrap_provider.go b/internal/bootstrap/bootstrap_provider.go index 5a0bdea3..367b2c72 100644 --- a/internal/bootstrap/bootstrap_provider.go +++ b/internal/bootstrap/bootstrap_provider.go @@ -168,7 +168,7 @@ func (o sshHostnameOption) applyGitProvider(b *GitProviderBootstrapper) { b.sshHostname = string(o) } -func (b *GitProviderBootstrapper) ReconcileSyncConfig(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error { +func (b *GitProviderBootstrapper) ReconcileSyncConfig(ctx context.Context, options sync.Options) error { repo, err := b.getRepository(ctx) if err != nil { return err @@ -187,7 +187,7 @@ func (b *GitProviderBootstrapper) ReconcileSyncConfig(ctx context.Context, optio } options.URL = syncURL } - return b.PlainGitBootstrapper.ReconcileSyncConfig(ctx, options, pollInterval, timeout) + return b.PlainGitBootstrapper.ReconcileSyncConfig(ctx, options) } // ReconcileRepository reconciles an organization or user repository with the diff --git a/internal/bootstrap/git/git.go b/internal/bootstrap/git/git.go index cbafadf9..86466e79 100644 --- a/internal/bootstrap/git/git.go +++ b/internal/bootstrap/git/git.go @@ -47,5 +47,6 @@ type Git interface { Commit(message Commit) (string, error) Push(ctx context.Context) error Status() (bool, error) + Head() (string, error) Path() string } diff --git a/internal/bootstrap/git/gogit/gogit.go b/internal/bootstrap/git/gogit/gogit.go index 619dd552..b11acd0c 100644 --- a/internal/bootstrap/git/gogit/gogit.go +++ b/internal/bootstrap/git/gogit/gogit.go @@ -212,6 +212,17 @@ func (g *GoGit) Status() (bool, error) { return status.IsClean(), nil } +func (g *GoGit) Head() (string, error) { + if g.repository == nil { + return "", git.ErrNoGitRepository + } + head, err := g.repository.Head() + if err != nil { + return "", err + } + return head.Hash().String(), nil +} + func (g *GoGit) Path() string { return g.path }