From 7756faec1f30ae721e2569d13d42daea72955a0a Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Tue, 22 Mar 2022 15:16:39 +0200 Subject: [PATCH] Retry bootstrap operations on Git conflict errors When running bootstrap in-parallel for many clusters that target the same repository, the 2nd commit with the sync files fails with ` non-fast-forward update`. We now detect the conflict, and we retry the operations by creating a fresh clone from upstream. Signed-off-by: Stefan Prodan --- internal/bootstrap/bootstrap_plain_git.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/internal/bootstrap/bootstrap_plain_git.go b/internal/bootstrap/bootstrap_plain_git.go index f661e936..a5979476 100644 --- a/internal/bootstrap/bootstrap_plain_git.go +++ b/internal/bootstrap/bootstrap_plain_git.go @@ -24,6 +24,7 @@ import ( "strings" "time" + gogit "github.com/go-git/go-git/v5" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" @@ -287,10 +288,28 @@ func (b *PlainGitBootstrapper) ReconcileSyncConfig(ctx context.Context, options if err != nil && err != git.ErrNoStagedFiles { return fmt.Errorf("failed to commit sync manifests: %w", err) } + if err == nil { b.logger.Successf("committed sync manifests to %q (%q)", b.branch, commit) b.logger.Actionf("pushing sync manifests to %q", b.url) - if err = b.git.Push(ctx, b.caBundle); err != nil { + err = b.git.Push(ctx, b.caBundle) + if err != nil { + if strings.HasPrefix(err.Error(), gogit.ErrNonFastForwardUpdate.Error()) { + b.logger.Waitingf("git conflict detected, retrying with a fresh clone") + if err := os.RemoveAll(b.git.Path()); err != nil { + return fmt.Errorf("failed to remove tmp dir: %w", err) + } + if err := os.Mkdir(b.git.Path(), 0o700); err != nil { + return fmt.Errorf("failed to recreate tmp dir: %w", err) + } + if err = retry(1, 2*time.Second, func() (err error) { + _, err = b.git.Clone(ctx, b.url, b.branch, b.caBundle) + return + }); err != nil { + return fmt.Errorf("failed to clone repository: %w", err) + } + return b.ReconcileSyncConfig(ctx, options) + } return fmt.Errorf("failed to push sync manifests: %w", err) } } else {