Use git package for bootstrap

cmd/tk/bootstrap.go

@@ -1,7 +1,25 @@
 package main
 
 import (
+	"context"
+	"fmt"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"sigs.k8s.io/yaml"
+	"strings"
+	"time"
+
 	"github.com/spf13/cobra"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1alpha1"
+	sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1"
 )
 
 var bootstrapCmd = &cobra.Command{
@@ -13,8 +31,203 @@ var (
 	bootstrapVersion string
 )
 
+const (
+	bootstrapBranch                = "master"
+	bootstrapInstallManifest       = "toolkit-components.yaml"
+	bootstrapSourceManifest        = "toolkit-source.yaml"
+	bootstrapKustomizationManifest = "toolkit-kustomization.yaml"
+)
+
 func init() {
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapVersion, "version", "master", "toolkit tag or branch")
 
 	rootCmd.AddCommand(bootstrapCmd)
 }
+
+func generateInstallManifests(targetPath, namespace, tmpDir string) (string, error) {
+	tkDir := path.Join(tmpDir, ".tk")
+	defer os.RemoveAll(tkDir)
+
+	if err := os.MkdirAll(tkDir, os.ModePerm); err != nil {
+		return "", fmt.Errorf("generating manifests failed: %w", err)
+	}
+
+	if err := genInstallManifests(bootstrapVersion, namespace, components, tkDir); err != nil {
+		return "", fmt.Errorf("generating manifests failed: %w", err)
+	}
+
+	manifestsDir := path.Join(tmpDir, targetPath, namespace)
+	if err := os.MkdirAll(manifestsDir, os.ModePerm); err != nil {
+		return "", fmt.Errorf("generating manifests failed: %w", err)
+	}
+
+	manifest := path.Join(manifestsDir, bootstrapInstallManifest)
+	if err := buildKustomization(tkDir, manifest); err != nil {
+		return "", fmt.Errorf("build kustomization failed: %w", err)
+	}
+
+	return manifest, nil
+}
+
+func applyInstallManifests(ctx context.Context, manifestPath string, components []string) error {
+	command := fmt.Sprintf("kubectl apply -f %s", manifestPath)
+	if _, err := utils.execCommand(ctx, ModeOS, command); err != nil {
+		return fmt.Errorf("install failed")
+	}
+
+	for _, deployment := range components {
+		command = fmt.Sprintf("kubectl -n %s rollout status deployment %s --timeout=%s",
+			namespace, deployment, timeout.String())
+		if _, err := utils.execCommand(ctx, ModeOS, command); err != nil {
+			return fmt.Errorf("install failed")
+		}
+	}
+	return nil
+}
+
+func generateSyncManifests(url, name, namespace, targetPath, tmpDir string, interval time.Duration) error {
+	gvk := sourcev1.GroupVersion.WithKind("GitRepository")
+	gitRepository := sourcev1.GitRepository{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       gvk.Kind,
+			APIVersion: gvk.GroupVersion().String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Spec: sourcev1.GitRepositorySpec{
+			URL: url,
+			Interval: metav1.Duration{
+				Duration: interval,
+			},
+			Reference: &sourcev1.GitRepositoryRef{
+				Branch: "master",
+			},
+			SecretRef: &corev1.LocalObjectReference{
+				Name: name,
+			},
+		},
+	}
+
+	gitData, err := yaml.Marshal(gitRepository)
+	if err != nil {
+		return err
+	}
+
+	if err := utils.writeFile(string(gitData), filepath.Join(tmpDir, targetPath, namespace, bootstrapSourceManifest)); err != nil {
+		return err
+	}
+
+	gvk = kustomizev1.GroupVersion.WithKind("Kustomization")
+	emptyAPIGroup := ""
+	kustomization := kustomizev1.Kustomization{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       gvk.Kind,
+			APIVersion: gvk.GroupVersion().String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Spec: kustomizev1.KustomizationSpec{
+			Interval: metav1.Duration{
+				Duration: 10 * time.Minute,
+			},
+			Path:  fmt.Sprintf("./%s", strings.TrimPrefix(targetPath, "./")),
+			Prune: true,
+			SourceRef: corev1.TypedLocalObjectReference{
+				APIGroup: &emptyAPIGroup,
+				Kind:     "GitRepository",
+				Name:     name,
+			},
+		},
+	}
+
+	ksData, err := yaml.Marshal(kustomization)
+	if err != nil {
+		return err
+	}
+
+	if err := utils.writeFile(string(ksData), filepath.Join(tmpDir, targetPath, namespace, bootstrapKustomizationManifest)); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func applySyncManifests(ctx context.Context, kubeClient client.Client, name, namespace, targetPath, tmpDir string) error {
+	command := fmt.Sprintf("kubectl apply -f %s", filepath.Join(tmpDir, targetPath, namespace))
+	if _, err := utils.execCommand(ctx, ModeStderrOS, command); err != nil {
+		return err
+	}
+
+	logWaiting("waiting for cluster sync")
+
+	if err := wait.PollImmediate(pollInterval, timeout,
+		isGitRepositoryReady(ctx, kubeClient, name, namespace)); err != nil {
+		return err
+	}
+
+	if err := wait.PollImmediate(pollInterval, timeout,
+		isKustomizationReady(ctx, kubeClient, name, namespace)); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func shouldInstallManifests(ctx context.Context, kubeClient client.Client, namespace string) bool {
+	namespacedName := types.NamespacedName{
+		Namespace: namespace,
+		Name:      namespace,
+	}
+	var kustomization kustomizev1.Kustomization
+	if err := kubeClient.Get(ctx, namespacedName, &kustomization); err != nil {
+		return true
+	}
+
+	return kustomization.Status.LastAppliedRevision == ""
+}
+
+func shouldCreateDeployKey(ctx context.Context, kubeClient client.Client, namespace string) bool {
+	namespacedName := types.NamespacedName{
+		Namespace: namespace,
+		Name:      namespace,
+	}
+
+	var existing corev1.Secret
+	if err := kubeClient.Get(ctx, namespacedName, &existing); err != nil {
+		return true
+	}
+	return false
+}
+
+func generateDeployKey(ctx context.Context, kubeClient client.Client, url *url.URL, namespace string) (string, error) {
+	pair, err := generateKeyPair(ctx)
+	if err != nil {
+		return "", err
+	}
+
+	hostKey, err := scanHostKey(ctx, url)
+	if err != nil {
+		return "", err
+	}
+
+	secret := corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      namespace,
+			Namespace: namespace,
+		},
+		StringData: map[string]string{
+			"identity":     string(pair.PrivateKey),
+			"identity.pub": string(pair.PublicKey),
+			"known_hosts":  string(hostKey),
+		},
+	}
+	if err := upsertSecret(ctx, kubeClient, secret); err != nil {
+		return "", err
+	}
+
+	return string(pair.PublicKey), nil
+}

cmd/tk/bootstrap_github.go

@@ -7,25 +7,11 @@ import (
 	"net/url"
 	"os"
 	"path"
-	"path/filepath"
-	"sigs.k8s.io/yaml"
-	"strings"
 	"time"
 
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/go-git/go-git/v5/plumbing/object"
-	"github.com/go-git/go-git/v5/plumbing/transport/http"
-	"github.com/google/go-github/v32/github"
 	"github.com/spf13/cobra"
-	corev1 "k8s.io/api/core/v1"
+
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"github.com/fluxcd/toolkit/pkg/git"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/apimachinery/pkg/util/wait"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1alpha1"
-	sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1"
 )
 
 var bootstrapGitHubCmd = &cobra.Command{
@@ -70,13 +56,7 @@ var (
 )
 
 const (
-	ghTokenName             = "GITHUB_TOKEN"
+	ghDefaultPermission = "maintain"
-	ghBranch                = "master"
-	ghInstallManifest       = "toolkit-components.yaml"
-	ghSourceManifest        = "toolkit-source.yaml"
-	ghKustomizationManifest = "toolkit-kustomization.yaml"
-	ghDefaultHostname       = "github.com"
-	ghDefaultPermission     = "maintain"
 )
 
 func init() {
@@ -86,22 +66,26 @@ func init() {
 	bootstrapGitHubCmd.Flags().BoolVar(&ghPersonal, "personal", false, "is personal repository")
 	bootstrapGitHubCmd.Flags().BoolVar(&ghPrivate, "private", true, "is private repository")
 	bootstrapGitHubCmd.Flags().DurationVar(&ghInterval, "interval", time.Minute, "sync interval")
-	bootstrapGitHubCmd.Flags().StringVar(&ghHostname, "hostname", ghDefaultHostname, "GitHub hostname")
+	bootstrapGitHubCmd.Flags().StringVar(&ghHostname, "hostname", git.GitHubDefaultHostname, "GitHub hostname")
 	bootstrapGitHubCmd.Flags().StringVar(&ghPath, "path", "", "repository path, when specified the cluster sync will be scoped to this path")
 
 	bootstrapCmd.AddCommand(bootstrapGitHubCmd)
 }
 
 func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error {
-	ghToken := os.Getenv(ghTokenName)
+	ghToken := os.Getenv(git.GitHubTokenName)
 	if ghToken == "" {
-		return fmt.Errorf("%s environment variable not found", ghTokenName)
+		return fmt.Errorf("%s environment variable not found", git.GitHubTokenName)
+	}
+
+	repository, err := git.NewRepository(ghRepository, ghOwner, ghHostname, ghToken, "tk", "tk@users.noreply.github.com")
+	if err != nil {
+		return err
 	}
 
-	ghURL := fmt.Sprintf("https://%s/%s/%s", ghHostname, ghOwner, ghRepository)
+	provider := &git.GithubProvider{
-	sshURL := fmt.Sprintf("ssh://git@%s/%s/%s", ghHostname, ghOwner, ghRepository)
+		IsPrivate:  ghPrivate,
-	if ghOwner == "" || ghRepository == "" {
+		IsPersonal: ghPersonal,
-		return fmt.Errorf("owner and repository are required")
 	}
 
 	kubeClient, err := utils.kubeClient(kubeconfig)
@@ -120,46 +104,49 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error {
 
 	// create GitHub repository if doesn't exists
 	logAction("connecting to %s", ghHostname)
-	if err := createGitHubRepository(ctx, ghHostname, ghOwner, ghRepository, ghToken, ghPrivate, ghPersonal); err != nil {
+	changed, err := provider.CreateRepository(ctx, repository)
+	if err != nil {
 		return err
 	}
+	if changed {
+		logSuccess("repository created")
+	}
 
 	withErrors := false
 	// add teams to org repository
 	if !ghPersonal {
 		for _, team := range ghTeams {
-			if err := addGitHubTeam(ctx, ghHostname, ghOwner, ghRepository, ghToken, team, ghDefaultPermission); err != nil {
+			if changed, err := provider.AddTeam(ctx, repository, team, ghDefaultPermission); err != nil {
 				logFailure(err.Error())
 				withErrors = true
-			} else {
+			} else if changed {
 				logSuccess("%s team access granted", team)
 			}
 		}
 	}
 
 	// clone repository and checkout the master branch
-	repo, err := checkoutGitHubRepository(ctx, ghURL, ghBranch, ghToken, tmpDir)
+	if err := repository.Checkout(ctx, bootstrapBranch, tmpDir); err != nil {
-	if err != nil {
 		return err
 	}
 	logSuccess("repository cloned")
 
 	// generate install manifests
 	logGenerate("generating manifests")
-	manifest, err := generateGitHubInstall(ghPath, namespace, tmpDir)
+	manifest, err := generateInstallManifests(ghPath, namespace, tmpDir)
 	if err != nil {
 		return err
 	}
 
 	// stage install manifests
-	changed, err := commitGitHubManifests(repo, ghPath, namespace)
+	changed, err = repository.Commit(ctx, path.Join(ghPath, namespace), "Add manifests")
 	if err != nil {
 		return err
 	}
 
 	// push install manifests
 	if changed {
-		if err := pushGitHubRepository(ctx, repo, ghToken); err != nil {
+		if err := repository.Push(ctx); err != nil {
 			return err
 		}
 		logSuccess("components manifests pushed")
@@ -168,74 +155,63 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error {
 	}
 
 	// determine if repo synchronization is working
-	isInstall := shouldInstallGitHub(ctx, kubeClient, namespace)
+	isInstall := shouldInstallManifests(ctx, kubeClient, namespace)
 
 	if isInstall {
 		// apply install manifests
 		logAction("installing components in %s namespace", namespace)
-		command := fmt.Sprintf("kubectl apply -f %s", manifest)
+		if err := applyInstallManifests(ctx, manifest, components); err != nil {
-		if _, err := utils.execCommand(ctx, ModeOS, command); err != nil {
+			return err
-			return fmt.Errorf("install failed")
 		}
 		logSuccess("install completed")
-
-		// check installation
-		logWaiting("verifying installation")
-		for _, deployment := range components {
-			command = fmt.Sprintf("kubectl -n %s rollout status deployment %s --timeout=%s",
-				namespace, deployment, timeout.String())
-			if _, err := utils.execCommand(ctx, ModeOS, command); err != nil {
-				return fmt.Errorf("install failed")
-			} else {
-				logSuccess("%s ready", deployment)
-			}
-		}
 	}
 
 	// setup SSH deploy key
-	if shouldCreateGitHubDeployKey(ctx, kubeClient, namespace) {
+	if shouldCreateDeployKey(ctx, kubeClient, namespace) {
 		logAction("configuring deploy key")
-		u, err := url.Parse(sshURL)
+		u, err := url.Parse(repository.GetSSH())
 		if err != nil {
 			return fmt.Errorf("git URL parse failed: %w", err)
 		}
 
-		key, err := generateGitHubDeployKey(ctx, kubeClient, u, namespace)
+		key, err := generateDeployKey(ctx, kubeClient, u, namespace)
 		if err != nil {
 			return fmt.Errorf("generating deploy key failed: %w", err)
 		}
 
-		if err := createGitHubDeployKey(ctx, key, ghHostname, ghOwner, ghRepository, ghPath, ghToken); err != nil {
+		keyName := "tk"
+		if ghPath != "" {
+			keyName = fmt.Sprintf("tk-%s", ghPath)
+		}
+
+		if changed, err := provider.AddDeployKey(ctx, repository, key, keyName); err != nil {
 			return err
+		} else if changed {
+			logSuccess("deploy key configured")
 		}
-		logSuccess("deploy key configured")
 	}
 
 	// configure repo synchronization
 	if isInstall {
 		// generate source and kustomization manifests
 		logAction("generating sync manifests")
-		if err := generateGitHubKustomization(sshURL, namespace, namespace, ghPath, tmpDir, ghInterval); err != nil {
+		if err := generateSyncManifests(repository.GetSSH(), namespace, namespace, ghPath, tmpDir, ghInterval); err != nil {
 			return err
 		}
 
-		// stage manifests
+		// commit and push manifests
-		changed, err = commitGitHubManifests(repo, ghPath, namespace)
+		if changed, err = repository.Commit(ctx, path.Join(ghPath, namespace), "Add manifests"); err != nil {
-		if err != nil {
 			return err
-		}
+		} else if changed {
-
+			if err := repository.Push(ctx); err != nil {
-		// push manifests
-		if changed {
-			if err := pushGitHubRepository(ctx, repo, ghToken); err != nil {
 				return err
 			}
+			logSuccess("sync manifests pushed")
 		}
-		logSuccess("sync manifests pushed")
 
 		// apply manifests and waiting for sync
 		logAction("applying sync manifests")
-		if err := applyGitHubKustomization(ctx, kubeClient, namespace, namespace, ghPath, tmpDir); err != nil {
+		if err := applySyncManifests(ctx, kubeClient, namespace, namespace, ghPath, tmpDir); err != nil {
 			return err
 		}
 	}
@@ -247,390 +223,3 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error {
 	logSuccess("bootstrap finished")
 	return nil
 }
-
-func makeGitHubClient(hostname, token string) (*github.Client, error) {
-	auth := github.BasicAuthTransport{
-		Username: "git",
-		Password: token,
-	}
-
-	gh := github.NewClient(auth.Client())
-	if hostname != ghDefaultHostname {
-		baseURL := fmt.Sprintf("https://%s/api/v3/", hostname)
-		uploadURL := fmt.Sprintf("https://%s/api/uploads/", hostname)
-		if g, err := github.NewEnterpriseClient(baseURL, uploadURL, auth.Client()); err == nil {
-			gh = g
-		} else {
-			return nil, fmt.Errorf("github client error: %w", err)
-		}
-	}
-
-	return gh, nil
-}
-
-func createGitHubRepository(ctx context.Context, hostname, owner, name, token string, isPrivate, isPersonal bool) error {
-	gh, err := makeGitHubClient(hostname, token)
-	if err != nil {
-		return err
-	}
-	org := ""
-	if !isPersonal {
-		org = owner
-	}
-
-	if _, _, err := gh.Repositories.Get(ctx, org, name); err == nil {
-		return nil
-	}
-
-	autoInit := true
-	_, _, err = gh.Repositories.Create(ctx, org, &github.Repository{
-		AutoInit: &autoInit,
-		Name:     &name,
-		Private:  &isPrivate,
-	})
-	if err != nil {
-		if !strings.Contains(err.Error(), "name already exists on this account") {
-			return fmt.Errorf("github create repository error: %w", err)
-		}
-	} else {
-		logSuccess("repository created")
-	}
-	return nil
-}
-
-func addGitHubTeam(ctx context.Context, hostname, owner, repository, token string, teamSlug, permission string) error {
-	gh, err := makeGitHubClient(hostname, token)
-	if err != nil {
-		return err
-	}
-
-	// check team exists
-	_, _, err = gh.Teams.GetTeamBySlug(ctx, owner, teamSlug)
-	if err != nil {
-		return fmt.Errorf("github get team %s error: %w", teamSlug, err)
-	}
-
-	// check if team is assigned to the repo
-	_, resp, err := gh.Teams.IsTeamRepoBySlug(ctx, owner, teamSlug, owner, repository)
-	if resp == nil && err != nil {
-		return fmt.Errorf("github is team %s error: %w", teamSlug, err)
-	}
-
-	// add team to the repo
-	if resp.StatusCode == 404 {
-		_, err = gh.Teams.AddTeamRepoBySlug(ctx, owner, teamSlug, owner, repository, &github.TeamAddTeamRepoOptions{
-			Permission: permission,
-		})
-		if err != nil {
-			return fmt.Errorf("github add team %s error: %w", teamSlug, err)
-		}
-	}
-
-	return nil
-}
-
-func checkoutGitHubRepository(ctx context.Context, url, branch, token, path string) (*git.Repository, error) {
-	auth := &http.BasicAuth{
-		Username: "git",
-		Password: token,
-	}
-	repo, err := git.PlainCloneContext(ctx, path, false, &git.CloneOptions{
-		URL:           url,
-		Auth:          auth,
-		RemoteName:    git.DefaultRemoteName,
-		ReferenceName: plumbing.NewBranchReferenceName(branch),
-		SingleBranch:  true,
-		NoCheckout:    false,
-		Progress:      nil,
-		Tags:          git.NoTags,
-	})
-	if err != nil {
-		return nil, fmt.Errorf("git clone error: %w", err)
-	}
-
-	_, err = repo.Head()
-	if err != nil {
-		return nil, fmt.Errorf("git resolve HEAD error: %w", err)
-	}
-
-	return repo, nil
-}
-
-func generateGitHubInstall(targetPath, namespace, tmpDir string) (string, error) {
-	tkDir := path.Join(tmpDir, ".tk")
-	defer os.RemoveAll(tkDir)
-
-	if err := os.MkdirAll(tkDir, os.ModePerm); err != nil {
-		return "", fmt.Errorf("generating manifests failed: %w", err)
-	}
-
-	if err := genInstallManifests(bootstrapVersion, namespace, components, tkDir); err != nil {
-		return "", fmt.Errorf("generating manifests failed: %w", err)
-	}
-
-	manifestsDir := path.Join(tmpDir, targetPath, namespace)
-	if err := os.MkdirAll(manifestsDir, os.ModePerm); err != nil {
-		return "", fmt.Errorf("generating manifests failed: %w", err)
-	}
-
-	manifest := path.Join(manifestsDir, ghInstallManifest)
-	if err := buildKustomization(tkDir, manifest); err != nil {
-		return "", fmt.Errorf("build kustomization failed: %w", err)
-	}
-
-	return manifest, nil
-}
-
-func commitGitHubManifests(repo *git.Repository, targetPath, namespace string) (bool, error) {
-	w, err := repo.Worktree()
-	if err != nil {
-		return false, err
-	}
-
-	_, err = w.Add(path.Join(targetPath, namespace))
-	if err != nil {
-		return false, err
-	}
-
-	status, err := w.Status()
-	if err != nil {
-		return false, err
-	}
-
-	if !status.IsClean() {
-		if _, err := w.Commit("Add manifests", &git.CommitOptions{
-			Author: &object.Signature{
-				Name:  "tk",
-				Email: "tk@users.noreply.github.com",
-				When:  time.Now(),
-			},
-		}); err != nil {
-			return false, err
-		}
-		return true, nil
-	}
-
-	return false, nil
-}
-
-func pushGitHubRepository(ctx context.Context, repo *git.Repository, token string) error {
-	auth := &http.BasicAuth{
-		Username: "git",
-		Password: token,
-	}
-	err := repo.PushContext(ctx, &git.PushOptions{
-		Auth:     auth,
-		Progress: nil,
-	})
-	if err != nil {
-		return fmt.Errorf("git push error: %w", err)
-	}
-	return nil
-}
-
-func generateGitHubKustomization(url, name, namespace, targetPath, tmpDir string, interval time.Duration) error {
-	gvk := sourcev1.GroupVersion.WithKind("GitRepository")
-	gitRepository := sourcev1.GitRepository{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       gvk.Kind,
-			APIVersion: gvk.GroupVersion().String(),
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      name,
-			Namespace: namespace,
-		},
-		Spec: sourcev1.GitRepositorySpec{
-			URL: url,
-			Interval: metav1.Duration{
-				Duration: interval,
-			},
-			Reference: &sourcev1.GitRepositoryRef{
-				Branch: "master",
-			},
-			SecretRef: &corev1.LocalObjectReference{
-				Name: name,
-			},
-		},
-	}
-
-	gitData, err := yaml.Marshal(gitRepository)
-	if err != nil {
-		return err
-	}
-
-	if err := utils.writeFile(string(gitData), filepath.Join(tmpDir, targetPath, namespace, ghSourceManifest)); err != nil {
-		return err
-	}
-
-	gvk = kustomizev1.GroupVersion.WithKind("Kustomization")
-	emptyAPIGroup := ""
-	kustomization := kustomizev1.Kustomization{
-		TypeMeta: metav1.TypeMeta{
-			Kind:       gvk.Kind,
-			APIVersion: gvk.GroupVersion().String(),
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      name,
-			Namespace: namespace,
-		},
-		Spec: kustomizev1.KustomizationSpec{
-			Interval: metav1.Duration{
-				Duration: 10 * time.Minute,
-			},
-			Path:  fmt.Sprintf("./%s", strings.TrimPrefix(targetPath, "./")),
-			Prune: true,
-			SourceRef: corev1.TypedLocalObjectReference{
-				APIGroup: &emptyAPIGroup,
-				Kind:     "GitRepository",
-				Name:     name,
-			},
-		},
-	}
-
-	ksData, err := yaml.Marshal(kustomization)
-	if err != nil {
-		return err
-	}
-
-	if err := utils.writeFile(string(ksData), filepath.Join(tmpDir, targetPath, namespace, ghKustomizationManifest)); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func applyGitHubKustomization(ctx context.Context, kubeClient client.Client, name, namespace, targetPath, tmpDir string) error {
-	command := fmt.Sprintf("kubectl apply -f %s", filepath.Join(tmpDir, targetPath, namespace))
-	if _, err := utils.execCommand(ctx, ModeStderrOS, command); err != nil {
-		return err
-	}
-
-	logWaiting("waiting for cluster sync")
-
-	if err := wait.PollImmediate(pollInterval, timeout,
-		isGitRepositoryReady(ctx, kubeClient, name, namespace)); err != nil {
-		return err
-	}
-
-	if err := wait.PollImmediate(pollInterval, timeout,
-		isKustomizationReady(ctx, kubeClient, name, namespace)); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func shouldInstallGitHub(ctx context.Context, kubeClient client.Client, namespace string) bool {
-	namespacedName := types.NamespacedName{
-		Namespace: namespace,
-		Name:      namespace,
-	}
-	var kustomization kustomizev1.Kustomization
-	if err := kubeClient.Get(ctx, namespacedName, &kustomization); err != nil {
-		return true
-	}
-
-	return kustomization.Status.LastAppliedRevision == ""
-}
-
-func shouldCreateGitHubDeployKey(ctx context.Context, kubeClient client.Client, namespace string) bool {
-	namespacedName := types.NamespacedName{
-		Namespace: namespace,
-		Name:      namespace,
-	}
-
-	var existing corev1.Secret
-	if err := kubeClient.Get(ctx, namespacedName, &existing); err != nil {
-		return true
-	}
-	return false
-}
-
-func generateGitHubDeployKey(ctx context.Context, kubeClient client.Client, url *url.URL, namespace string) (string, error) {
-	pair, err := generateKeyPair(ctx)
-	if err != nil {
-		return "", err
-	}
-
-	hostKey, err := scanHostKey(ctx, url)
-	if err != nil {
-		return "", err
-	}
-
-	secret := corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      namespace,
-			Namespace: namespace,
-		},
-		StringData: map[string]string{
-			"identity":     string(pair.PrivateKey),
-			"identity.pub": string(pair.PublicKey),
-			"known_hosts":  string(hostKey),
-		},
-	}
-	if err := upsertSecret(ctx, kubeClient, secret); err != nil {
-		return "", err
-	}
-
-	return string(pair.PublicKey), nil
-}
-
-func createGitHubDeployKey(ctx context.Context, key, hostname, owner, repository, targetPath, token string) error {
-	gh, err := makeGitHubClient(hostname, token)
-	if err != nil {
-		return err
-	}
-	keyName := "tk"
-	if targetPath != "" {
-		keyName = fmt.Sprintf("tk-%s", targetPath)
-	}
-
-	// list deploy keys
-	keys, resp, err := gh.Repositories.ListKeys(ctx, owner, repository, nil)
-	if err != nil {
-		return fmt.Errorf("github list deploy keys error: %w", err)
-	}
-	if resp.StatusCode >= 300 {
-		return fmt.Errorf("github list deploy keys failed with status code: %s", resp.Status)
-	}
-
-	// check if the key exists
-	shouldCreateKey := true
-	var existingKey *github.Key
-	for _, k := range keys {
-		if k.Title != nil && k.Key != nil && *k.Title == keyName {
-			if *k.Key != key {
-				existingKey = k
-			} else {
-				shouldCreateKey = false
-			}
-			break
-		}
-	}
-
-	// delete existing key if the value differs
-	if existingKey != nil {
-		resp, err := gh.Repositories.DeleteKey(ctx, owner, repository, *existingKey.ID)
-		if err != nil {
-			return fmt.Errorf("github delete deploy key error: %w", err)
-		}
-		if resp.StatusCode >= 300 {
-			return fmt.Errorf("github delete deploy key failed with status code: %s", resp.Status)
-		}
-	}
-
-	// create key
-	if shouldCreateKey {
-		isReadOnly := true
-		_, _, err = gh.Repositories.CreateKey(ctx, owner, repository, &github.Key{
-			Title:    &keyName,
-			Key:      &key,
-			ReadOnly: &isReadOnly,
-		})
-		if err != nil {
-			return fmt.Errorf("github create deploy key error: %w", err)
-		}
-	}
-
-	return nil
-}

cmd/tk/bootstrap_gitlab.go

@@ -6,10 +6,12 @@ import (
 	"io/ioutil"
 	"net/url"
 	"os"
+	"path"
 	"time"
 
 	"github.com/spf13/cobra"
-	"github.com/xanzy/go-gitlab"
+
+	"github.com/fluxcd/toolkit/pkg/git"
 )
 
 var bootstrapGitLabCmd = &cobra.Command{
@@ -21,14 +23,20 @@ commits the toolkit components manifests to the master branch.
 Then it configure the target cluster to synchronize with the repository.
 If the toolkit components are present on the cluster,
 the bootstrap command will perform an upgrade if needed.`,
-	Example: `  # Create a GitLab personal access token and export it as an env var
+	Example: `  # Create a GitLab API token and export it as an env var
   export GITLAB_TOKEN=<my-token>
 
-  # Run bootstrap for a private repo owned by a GitLab organization
+  # Run bootstrap for a private repo owned by a GitLab group
-  bootstrap gitlab --owner=<organization> --repository=<repo name>
+  bootstrap gitlab --owner=<group> --repository=<repo name>
+
+  # Run bootstrap for a repository path
+  bootstrap gitlab --owner=<group> --repository=<repo name> --path=dev-cluster
+
+  # Run bootstrap for a public repository on a personal account
+  bootstrap gitlab --owner=<user> --repository=<repo name> --private=false --personal=true 
 
   # Run bootstrap for a private repo hosted on GitLab server 
-  bootstrap gitlab --owner=<organization> --repository=<repo name> --hostname=<domain>
+  bootstrap gitlab --owner=<group> --repository=<repo name> --hostname=<domain>
 `,
 	RunE: bootstrapGitLabCmdRun,
 }
@@ -43,33 +51,32 @@ var (
 	glPath       string
 )
 
-const (
-	glTokenName       = "GITLAB_TOKEN"
-	glDefaultHostname = "gitlab.com"
-)
-
 func init() {
 	bootstrapGitLabCmd.Flags().StringVar(&glOwner, "owner", "", "GitLab user or organization name")
 	bootstrapGitLabCmd.Flags().StringVar(&glRepository, "repository", "", "GitLab repository name")
 	bootstrapGitLabCmd.Flags().BoolVar(&glPersonal, "personal", false, "is personal repository")
 	bootstrapGitLabCmd.Flags().BoolVar(&glPrivate, "private", true, "is private repository")
 	bootstrapGitLabCmd.Flags().DurationVar(&glInterval, "interval", time.Minute, "sync interval")
-	bootstrapGitLabCmd.Flags().StringVar(&glHostname, "hostname", glDefaultHostname, "GitLab hostname")
+	bootstrapGitLabCmd.Flags().StringVar(&glHostname, "hostname", git.GitLabDefaultHostname, "GitLab hostname")
 	bootstrapGitLabCmd.Flags().StringVar(&glPath, "path", "", "repository path, when specified the cluster sync will be scoped to this path")
 
 	bootstrapCmd.AddCommand(bootstrapGitLabCmd)
 }
 
 func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error {
-	glToken := os.Getenv(glTokenName)
+	glToken := os.Getenv(git.GitLabTokenName)
 	if glToken == "" {
-		return fmt.Errorf("%s environment variable not found", glTokenName)
+		return fmt.Errorf("%s environment variable not found", git.GitLabTokenName)
 	}
 
-	gitURL := fmt.Sprintf("https://%s/%s/%s", glHostname, glOwner, glRepository)
+	repository, err := git.NewRepository(glRepository, glOwner, glHostname, glToken, "tk", "tk@users.noreply.gitlab.com")
-	sshURL := fmt.Sprintf("ssh://git@%s/%s/%s", glHostname, glOwner, glRepository)
+	if err != nil {
-	if glOwner == "" || glRepository == "" {
+		return err
-		return fmt.Errorf("owner and repository are required")
+	}
+
+	provider := &git.GitLabProvider{
+		IsPrivate:  glPrivate,
+		IsPersonal: glPersonal,
 	}
 
 	kubeClient, err := utils.kubeClient(kubeconfig)
@@ -88,32 +95,36 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error {
 
 	// create GitLab project if doesn't exists
 	logAction("connecting to %s", glHostname)
-	if err := createGitLabRepository(ctx, glHostname, glOwner, glRepository, glToken, glPrivate, glPersonal); err != nil {
+	changed, err := provider.CreateRepository(ctx, repository)
+	if err != nil {
 		return err
 	}
+	if changed {
+		logSuccess("repository created")
+	}
 
 	// clone repository and checkout the master branch
-	repo, err := checkoutGitHubRepository(ctx, gitURL, ghBranch, glToken, tmpDir)
+	if err := repository.Checkout(ctx, bootstrapBranch, tmpDir); err != nil {
-	if err != nil {
 		return err
 	}
 	logSuccess("repository cloned")
 
 	// generate install manifests
 	logGenerate("generating manifests")
-	manifest, err := generateGitHubInstall(glPath, namespace, tmpDir)
+	manifest, err := generateInstallManifests(glPath, namespace, tmpDir)
 	if err != nil {
 		return err
 	}
 
 	// stage install manifests
-	changed, err := commitGitHubManifests(repo, glPath, namespace)
+	changed, err = repository.Commit(ctx, path.Join(glPath, namespace), "Add manifests")
 	if err != nil {
 		return err
 	}
 
+	// push install manifests
 	if changed {
-		if err := pushGitHubRepository(ctx, repo, glToken); err != nil {
+		if err := repository.Push(ctx); err != nil {
 			return err
 		}
 		logSuccess("components manifests pushed")
@@ -122,74 +133,63 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error {
 	}
 
 	// determine if repo synchronization is working
-	isInstall := shouldInstallGitHub(ctx, kubeClient, namespace)
+	isInstall := shouldInstallManifests(ctx, kubeClient, namespace)
 
 	if isInstall {
 		// apply install manifests
 		logAction("installing components in %s namespace", namespace)
-		command := fmt.Sprintf("kubectl apply -f %s", manifest)
+		if err := applyInstallManifests(ctx, manifest, components); err != nil {
-		if _, err := utils.execCommand(ctx, ModeOS, command); err != nil {
+			return err
-			return fmt.Errorf("install failed")
 		}
 		logSuccess("install completed")
-
-		// check installation
-		logWaiting("verifying installation")
-		for _, deployment := range components {
-			command = fmt.Sprintf("kubectl -n %s rollout status deployment %s --timeout=%s",
-				namespace, deployment, timeout.String())
-			if _, err := utils.execCommand(ctx, ModeOS, command); err != nil {
-				return fmt.Errorf("install failed")
-			} else {
-				logSuccess("%s ready", deployment)
-			}
-		}
 	}
 
 	// setup SSH deploy key
-	if shouldCreateGitHubDeployKey(ctx, kubeClient, namespace) {
+	if shouldCreateDeployKey(ctx, kubeClient, namespace) {
 		logAction("configuring deploy key")
-		u, err := url.Parse(sshURL)
+		u, err := url.Parse(repository.GetSSH())
 		if err != nil {
 			return fmt.Errorf("git URL parse failed: %w", err)
 		}
 
-		key, err := generateGitHubDeployKey(ctx, kubeClient, u, namespace)
+		key, err := generateDeployKey(ctx, kubeClient, u, namespace)
 		if err != nil {
 			return fmt.Errorf("generating deploy key failed: %w", err)
 		}
 
-		if err := createGitLabDeployKey(ctx, key, glHostname, glOwner, glRepository, glPath, glToken); err != nil {
+		keyName := "tk"
+		if glPath != "" {
+			keyName = fmt.Sprintf("tk-%s", glPath)
+		}
+
+		if changed, err := provider.AddDeployKey(ctx, repository, key, keyName); err != nil {
 			return err
+		} else if changed {
+			logSuccess("deploy key configured")
 		}
-		logSuccess("deploy key configured")
 	}
 
 	// configure repo synchronization
 	if isInstall {
 		// generate source and kustomization manifests
 		logAction("generating sync manifests")
-		if err := generateGitHubKustomization(sshURL, namespace, namespace, glPath, tmpDir, glInterval); err != nil {
+		if err := generateSyncManifests(repository.GetSSH(), namespace, namespace, glPath, tmpDir, glInterval); err != nil {
 			return err
 		}
 
-		// stage manifests
+		// commit and push manifests
-		changed, err = commitGitHubManifests(repo, glPath, namespace)
+		if changed, err = repository.Commit(ctx, path.Join(glPath, namespace), "Add manifests"); err != nil {
-		if err != nil {
 			return err
-		}
+		} else if changed {
-
+			if err := repository.Push(ctx); err != nil {
-		// push manifests
-		if changed {
-			if err := pushGitHubRepository(ctx, repo, glToken); err != nil {
 				return err
 			}
+			logSuccess("sync manifests pushed")
 		}
-		logSuccess("sync manifests pushed")
 
 		// apply manifests and waiting for sync
 		logAction("applying sync manifests")
-		if err := applyGitHubKustomization(ctx, kubeClient, namespace, namespace, glPath, tmpDir); err != nil {
+		if err := applySyncManifests(ctx, kubeClient, namespace, namespace, glPath, tmpDir); err != nil {
 			return err
 		}
 	}
@@ -197,128 +197,3 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error {
 	logSuccess("bootstrap finished")
 	return nil
 }
-
-func makeGitLabClient(hostname, token string) (*gitlab.Client, error) {
-	gl, err := gitlab.NewClient(token)
-	if err != nil {
-		return nil, err
-	}
-
-	if glHostname != glDefaultHostname {
-		gl, err = gitlab.NewClient(token, gitlab.WithBaseURL(fmt.Sprintf("https://%s/api/v4", hostname)))
-		if err != nil {
-			return nil, err
-		}
-	}
-	return gl, nil
-}
-
-func createGitLabRepository(ctx context.Context, hostname, owner, repository, token string, isPrivate, isPersonal bool) error {
-	gl, err := makeGitLabClient(hostname, token)
-	if err != nil {
-		return fmt.Errorf("client error: %w", err)
-	}
-
-	var id *int
-	if !isPersonal {
-		groups, _, err := gl.Groups.ListGroups(&gitlab.ListGroupsOptions{Search: gitlab.String(owner)}, gitlab.WithContext(ctx))
-		if err != nil {
-			return fmt.Errorf("list groups error: %w", err)
-		}
-
-		if len(groups) > 0 {
-			id = &groups[0].ID
-		}
-	}
-
-	visibility := gitlab.PublicVisibility
-	if isPrivate {
-		visibility = gitlab.PrivateVisibility
-	}
-
-	projects, _, err := gl.Projects.ListProjects(&gitlab.ListProjectsOptions{Search: gitlab.String(repository)}, gitlab.WithContext(ctx))
-	if err != nil {
-		return fmt.Errorf("list projects error: %w", err)
-	}
-
-	if len(projects) == 0 {
-		p := &gitlab.CreateProjectOptions{
-			Name:                 gitlab.String(repository),
-			NamespaceID:          id,
-			Visibility:           &visibility,
-			InitializeWithReadme: gitlab.Bool(true),
-		}
-
-		project, _, err := gl.Projects.CreateProject(p)
-		if err != nil {
-			return fmt.Errorf("create project error: %w", err)
-		}
-		logSuccess("project created id: %v", project.ID)
-	}
-
-	return nil
-}
-
-func createGitLabDeployKey(ctx context.Context, key, hostname, owner, repository, targetPath, token string) error {
-	gl, err := makeGitLabClient(hostname, token)
-	if err != nil {
-		return fmt.Errorf("client error: %w", err)
-	}
-
-	var projId int
-	projects, _, err := gl.Projects.ListProjects(&gitlab.ListProjectsOptions{Search: gitlab.String(repository)}, gitlab.WithContext(ctx))
-	if err != nil {
-		return fmt.Errorf("list projects error: %w", err)
-	}
-	if len(projects) > 0 {
-		projId = projects[0].ID
-	} else {
-		return fmt.Errorf("no project found")
-	}
-
-	keyName := "tk"
-	if targetPath != "" {
-		keyName = fmt.Sprintf("tk-%s", targetPath)
-	}
-
-	// check if the key exists
-	keys, _, err := gl.DeployKeys.ListProjectDeployKeys(projId, &gitlab.ListProjectDeployKeysOptions{})
-	if err != nil {
-		return fmt.Errorf("list keys error: %w", err)
-	}
-
-	shouldCreateKey := true
-	var existingKey *gitlab.DeployKey
-	for _, k := range keys {
-		if k.Title == keyName {
-			if k.Key != key {
-				existingKey = k
-			} else {
-				shouldCreateKey = false
-			}
-			break
-		}
-	}
-
-	// delete existing key if the value differs
-	if existingKey != nil {
-		_, err := gl.DeployKeys.DeleteDeployKey(projId, existingKey.ID, gitlab.WithContext(ctx))
-		if err != nil {
-			return fmt.Errorf("delete key error: %w", err)
-		}
-	}
-
-	// create key
-	if shouldCreateKey {
-		_, _, err := gl.DeployKeys.AddDeployKey(projId, &gitlab.AddDeployKeyOptions{
-			Title:   gitlab.String(keyName),
-			Key:     gitlab.String(key),
-			CanPush: gitlab.Bool(false),
-		}, gitlab.WithContext(ctx))
-		if err != nil {
-			return fmt.Errorf("add key error: %w", err)
-		}
-	}
-
-	return nil
-}

pkg/git/provider.go

@@ -2,6 +2,7 @@ package git
 
 import "context"
 
+// Provider is the interface that a git provider should implement
 type Provider interface {
 	CreateRepository(ctx context.Context, r *Repository) (bool, error)
 	AddTeam(ctx context.Context, r *Repository, name, permission string) (bool, error)

pkg/git/repository.go

@@ -17,14 +17,15 @@ type Repository struct {
 	Name        string
 	Owner       string
 	Host        string
-	Branch      string
 	Token       string
 	AuthorName  string
 	AuthorEmail string
+
+	repo *git.Repository
 }
 
 // NewRepository returns a git repository wrapper
-func NewRepository(name, owner, host, branch, token, authorName, authorEmail string) (*Repository, error) {
+func NewRepository(name, owner, host, token, authorName, authorEmail string) (*Repository, error) {
 	if name == "" {
 		return nil, fmt.Errorf("name required")
 	}
@@ -34,24 +35,20 @@ func NewRepository(name, owner, host, branch, token, authorName, authorEmail str
 	if host == "" {
 		return nil, fmt.Errorf("host required")
 	}
-	if branch == "" {
-		return nil, fmt.Errorf("branch required")
-	}
 	if token == "" {
 		return nil, fmt.Errorf("token required")
 	}
 	if authorName == "" {
-		authorName = "tk"
+		return nil, fmt.Errorf("author name required")
 	}
 	if authorEmail == "" {
-		authorEmail = "tk@users.noreply.git-scm.com"
+		return nil, fmt.Errorf("author email required")
 	}
 
 	return &Repository{
 		Name:        name,
 		Owner:       owner,
 		Host:        host,
-		Branch:      branch,
 		Token:       token,
 		AuthorName:  authorName,
 		AuthorEmail: authorEmail,
@@ -75,33 +72,38 @@ func (r *Repository) auth() transport.AuthMethod {
 	}
 }
 
-// Checkout repository at specified path
+// Checkout repository branch at specified path
-func (r *Repository) Checkout(ctx context.Context, path string) (*git.Repository, error) {
+func (r *Repository) Checkout(ctx context.Context, branch, path string) error {
 	repo, err := git.PlainCloneContext(ctx, path, false, &git.CloneOptions{
 		URL:           r.GetURL(),
 		Auth:          r.auth(),
 		RemoteName:    git.DefaultRemoteName,
-		ReferenceName: plumbing.NewBranchReferenceName(r.Branch),
+		ReferenceName: plumbing.NewBranchReferenceName(branch),
 		SingleBranch:  true,
 		NoCheckout:    false,
 		Progress:      nil,
 		Tags:          git.NoTags,
 	})
 	if err != nil {
-		return nil, fmt.Errorf("git clone error: %w", err)
+		return fmt.Errorf("git clone error: %w", err)
 	}
 
 	_, err = repo.Head()
 	if err != nil {
-		return nil, fmt.Errorf("git resolve HEAD error: %w", err)
+		return fmt.Errorf("git resolve HEAD error: %w", err)
 	}
 
-	return repo, nil
+	r.repo = repo
+	return nil
 }
 
-// Commit changes for the specified path, returns false if no changes are made
+// Commit changes for the specified path, returns false if no changes are detected
-func (r *Repository) Commit(ctx context.Context, repo *git.Repository, path, message string) (bool, error) {
+func (r *Repository) Commit(ctx context.Context, path, message string) (bool, error) {
-	w, err := repo.Worktree()
+	if r.repo == nil {
+		return false, fmt.Errorf("repository hasn't been cloned")
+	}
+
+	w, err := r.repo.Worktree()
 	if err != nil {
 		return false, err
 	}
@@ -133,8 +135,12 @@ func (r *Repository) Commit(ctx context.Context, repo *git.Repository, path, mes
 }
 
 // Push commits to origin
-func (r *Repository) Push(ctx context.Context, repo *git.Repository) error {
+func (r *Repository) Push(ctx context.Context) error {
-	err := repo.PushContext(ctx, &git.PushOptions{
+	if r.repo == nil {
+		return fmt.Errorf("repository hasn't been cloned")
+	}
+
+	err := r.repo.PushContext(ctx, &git.PushOptions{
 		Auth:     r.auth(),
 		Progress: nil,
 	})