From f3d50e158af1a2ce11719bf5928185070ef01c41 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 18 Jun 2020 00:32:32 +0300 Subject: [PATCH 1/4] Implement GitLab bootstrap --- cmd/tk/bootstrap_gitlab.go | 324 +++++++++++++++++++++++++++++++++++++ go.mod | 8 + go.sum | 47 ++++++ 3 files changed, 379 insertions(+) create mode 100644 cmd/tk/bootstrap_gitlab.go diff --git a/cmd/tk/bootstrap_gitlab.go b/cmd/tk/bootstrap_gitlab.go new file mode 100644 index 00000000..56efd6fd --- /dev/null +++ b/cmd/tk/bootstrap_gitlab.go @@ -0,0 +1,324 @@ +package main + +import ( + "context" + "fmt" + "io/ioutil" + "net/url" + "os" + "time" + + "github.com/spf13/cobra" + "github.com/xanzy/go-gitlab" +) + +var bootstrapGitLabCmd = &cobra.Command{ + Use: "gitlab", + Short: "Bootstrap GitLab repository", + Long: ` +The bootstrap command creates the GitHub repository if it doesn't exists and +commits the toolkit components manifests to the master branch. +Then it configure the target cluster to synchronize with the repository. +If the toolkit components are present on the cluster, +the bootstrap command will perform an upgrade if needed.`, + Example: ` # Create a GitLab personal access token and export it as an env var + export GITLAB_TOKEN= + + # Run bootstrap for a private repo owned by a GitLab organization + bootstrap gitlab --owner= --repository= + + # Run bootstrap for a private repo hosted on GitLab server + bootstrap gitlab --owner= --repository= --hostname= +`, + RunE: bootstrapGitLabCmdRun, +} + +var ( + glOwner string + glRepository string + glInterval time.Duration + glPersonal bool + glPrivate bool + glHostname string + glPath string +) + +const ( + glTokenName = "GITLAB_TOKEN" + glDefaultHostname = "gitlab.com" +) + +func init() { + bootstrapGitLabCmd.Flags().StringVar(&glOwner, "owner", "", "GitLab user or organization name") + bootstrapGitLabCmd.Flags().StringVar(&glRepository, "repository", "", "GitLab repository name") + bootstrapGitLabCmd.Flags().BoolVar(&glPersonal, "personal", false, "is personal repository") + bootstrapGitLabCmd.Flags().BoolVar(&glPrivate, "private", true, "is private repository") + bootstrapGitLabCmd.Flags().DurationVar(&glInterval, "interval", time.Minute, "sync interval") + bootstrapGitLabCmd.Flags().StringVar(&glHostname, "hostname", glDefaultHostname, "GitLab hostname") + bootstrapGitLabCmd.Flags().StringVar(&glPath, "path", "", "repository path, when specified the cluster sync will be scoped to this path") + + bootstrapCmd.AddCommand(bootstrapGitLabCmd) +} + +func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error { + glToken := os.Getenv(glTokenName) + if glToken == "" { + return fmt.Errorf("%s environment variable not found", glTokenName) + } + + gitURL := fmt.Sprintf("https://%s/%s/%s", glHostname, glOwner, glRepository) + sshURL := fmt.Sprintf("ssh://git@%s/%s/%s", glHostname, glOwner, glRepository) + if glOwner == "" || glRepository == "" { + return fmt.Errorf("owner and repository are required") + } + + kubeClient, err := utils.kubeClient(kubeconfig) + if err != nil { + return err + } + + tmpDir, err := ioutil.TempDir("", namespace) + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + // create GitLab project if doesn't exists + logAction("connecting to %s", glHostname) + if err := createGitLabRepository(ctx, glHostname, glOwner, glRepository, glToken, glPrivate, glPersonal); err != nil { + return err + } + + // clone repository and checkout the master branch + repo, err := checkoutGitHubRepository(ctx, gitURL, ghBranch, glToken, tmpDir) + if err != nil { + return err + } + logSuccess("repository cloned") + + // generate install manifests + logGenerate("generating manifests") + manifest, err := generateGitHubInstall(glPath, namespace, tmpDir) + if err != nil { + return err + } + + // stage install manifests + changed, err := commitGitHubManifests(repo, glPath, namespace) + if err != nil { + return err + } + + if changed { + if err := pushGitHubRepository(ctx, repo, glToken); err != nil { + return err + } + logSuccess("components manifests pushed") + } else { + logSuccess("components are up to date") + } + + // determine if repo synchronization is working + isInstall := shouldInstallGitHub(ctx, kubeClient, namespace) + + if isInstall { + // apply install manifests + logAction("installing components in %s namespace", namespace) + command := fmt.Sprintf("kubectl apply -f %s", manifest) + if _, err := utils.execCommand(ctx, ModeOS, command); err != nil { + return fmt.Errorf("install failed") + } + logSuccess("install completed") + + // check installation + logWaiting("verifying installation") + for _, deployment := range components { + command = fmt.Sprintf("kubectl -n %s rollout status deployment %s --timeout=%s", + namespace, deployment, timeout.String()) + if _, err := utils.execCommand(ctx, ModeOS, command); err != nil { + return fmt.Errorf("install failed") + } else { + logSuccess("%s ready", deployment) + } + } + } + + // setup SSH deploy key + if shouldCreateGitHubDeployKey(ctx, kubeClient, namespace) { + logAction("configuring deploy key") + u, err := url.Parse(sshURL) + if err != nil { + return fmt.Errorf("git URL parse failed: %w", err) + } + + key, err := generateGitHubDeployKey(ctx, kubeClient, u, namespace) + if err != nil { + return fmt.Errorf("generating deploy key failed: %w", err) + } + + if err := createGitLabDeployKey(ctx, key, glHostname, glOwner, glRepository, glPath, glToken); err != nil { + return err + } + logSuccess("deploy key configured") + } + + // configure repo synchronization + if isInstall { + // generate source and kustomization manifests + logAction("generating sync manifests") + if err := generateGitHubKustomization(sshURL, namespace, namespace, glPath, tmpDir, glInterval); err != nil { + return err + } + + // stage manifests + changed, err = commitGitHubManifests(repo, glPath, namespace) + if err != nil { + return err + } + + // push manifests + if changed { + if err := pushGitHubRepository(ctx, repo, glToken); err != nil { + return err + } + } + logSuccess("sync manifests pushed") + + // apply manifests and waiting for sync + logAction("applying sync manifests") + if err := applyGitHubKustomization(ctx, kubeClient, namespace, namespace, glPath, tmpDir); err != nil { + return err + } + } + + logSuccess("bootstrap finished") + return nil +} + +func makeGitLabClient(hostname, token string) (*gitlab.Client, error) { + gl, err := gitlab.NewClient(token) + if err != nil { + return nil, err + } + + if glHostname != glDefaultHostname { + gl, err = gitlab.NewClient(token, gitlab.WithBaseURL(fmt.Sprintf("https://%s/api/v4", hostname))) + if err != nil { + return nil, err + } + } + return gl, nil +} + +func createGitLabRepository(ctx context.Context, hostname, owner, repository, token string, isPrivate, isPersonal bool) error { + gl, err := makeGitLabClient(hostname, token) + if err != nil { + return fmt.Errorf("client error: %w", err) + } + + var id *int + if !isPersonal { + groups, _, err := gl.Groups.ListGroups(&gitlab.ListGroupsOptions{Search: gitlab.String(owner)}, gitlab.WithContext(ctx)) + if err != nil { + return fmt.Errorf("list groups error: %w", err) + } + + if len(groups) > 0 { + id = &groups[0].ID + } + } + + visibility := gitlab.PublicVisibility + if isPrivate { + visibility = gitlab.PrivateVisibility + } + + projects, _, err := gl.Projects.ListProjects(&gitlab.ListProjectsOptions{Search: gitlab.String(repository)}, gitlab.WithContext(ctx)) + if err != nil { + return fmt.Errorf("list projects error: %w", err) + } + + if len(projects) == 0 { + p := &gitlab.CreateProjectOptions{ + Name: gitlab.String(repository), + NamespaceID: id, + Visibility: &visibility, + InitializeWithReadme: gitlab.Bool(true), + } + + project, _, err := gl.Projects.CreateProject(p) + if err != nil { + return fmt.Errorf("create project error: %w", err) + } + logSuccess("project created id: %v", project.ID) + } + + return nil +} + +func createGitLabDeployKey(ctx context.Context, key, hostname, owner, repository, targetPath, token string) error { + gl, err := makeGitLabClient(hostname, token) + if err != nil { + return fmt.Errorf("client error: %w", err) + } + + var projId int + projects, _, err := gl.Projects.ListProjects(&gitlab.ListProjectsOptions{Search: gitlab.String(repository)}, gitlab.WithContext(ctx)) + if err != nil { + return fmt.Errorf("list projects error: %w", err) + } + if len(projects) > 0 { + projId = projects[0].ID + } else { + return fmt.Errorf("no project found") + } + + keyName := "tk" + if targetPath != "" { + keyName = fmt.Sprintf("tk-%s", targetPath) + } + + // check if the key exists + keys, _, err := gl.DeployKeys.ListProjectDeployKeys(projId, &gitlab.ListProjectDeployKeysOptions{}) + if err != nil { + return fmt.Errorf("list keys error: %w", err) + } + + shouldCreateKey := true + var existingKey *gitlab.DeployKey + for _, k := range keys { + if k.Title == keyName { + if k.Key != key { + existingKey = k + } else { + shouldCreateKey = false + } + break + } + } + + // delete existing key if the value differs + if existingKey != nil { + _, err := gl.DeployKeys.DeleteDeployKey(projId, existingKey.ID, gitlab.WithContext(ctx)) + if err != nil { + return fmt.Errorf("delete key error: %w", err) + } + } + + // create key + if shouldCreateKey { + _, _, err := gl.DeployKeys.AddDeployKey(projId, &gitlab.AddDeployKeyOptions{ + Title: gitlab.String(keyName), + Key: gitlab.String(key), + CanPush: gitlab.Bool(false), + }, gitlab.WithContext(ctx)) + if err != nil { + return fmt.Errorf("add key error: %w", err) + } + } + + return nil +} diff --git a/go.mod b/go.mod index d8b917b0..93697b8d 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,18 @@ require ( github.com/fluxcd/kustomize-controller v0.0.1-beta.2 github.com/fluxcd/source-controller v0.0.1-beta.2 github.com/go-git/go-git/v5 v5.0.0 + github.com/golang/protobuf v1.4.2 // indirect github.com/google/go-github/v32 v32.0.0 + github.com/hashicorp/go-retryablehttp v0.6.6 // indirect github.com/manifoldco/promptui v0.7.0 github.com/spf13/cobra v1.0.0 + github.com/xanzy/go-gitlab v0.32.1 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 + golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect + golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect + golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect + google.golang.org/appengine v1.6.6 // indirect + google.golang.org/protobuf v1.24.0 // indirect k8s.io/api v0.18.2 k8s.io/apimachinery v0.18.2 k8s.io/client-go v0.18.2 diff --git a/go.sum b/go.sum index b0d1eb80..4f610e47 100644 --- a/go.sum +++ b/go.sum @@ -293,6 +293,14 @@ github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= @@ -317,6 +325,7 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github/v32 v32.0.0 h1:q74KVb22spUq0U5HqZ9VCYqQz8YRuOtL/39ZnfwO+NM= @@ -360,7 +369,15 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= +github.com/hashicorp/go-retryablehttp v0.6.4 h1:BbgctKO892xEyOXnGiaAwIoSq1QZ/SS4AhjoAh9DnfY= +github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-retryablehttp v0.6.6 h1:HJunrbHTDDbBb/ay4kxa1n+dLmttUlnP3V9oNE4hmsM= +github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0= @@ -617,6 +634,8 @@ github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= +github.com/xanzy/go-gitlab v0.32.1 h1:eKGfAP2FWbqStD7DtGoRBb18IYwjuCxdtEVea2rNge4= +github.com/xanzy/go-gitlab v0.32.1/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -686,6 +705,7 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -696,6 +716,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -707,11 +728,16 @@ golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -748,6 +774,8 @@ golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hq golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -758,6 +786,10 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 h1:NusfzzA6yGQ+ua51ck7E3omNUX/JuqbFSaRGqU8CcLI= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -797,9 +829,12 @@ google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+ google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -808,6 +843,7 @@ google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -817,6 +853,17 @@ google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From bd781bbcfb8957b08c54f5e3a47b05a5c7c6e6cb Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 18 Jun 2020 01:55:29 +0300 Subject: [PATCH 2/4] Extract git operations --- pkg/git/provider.go | 9 +++ pkg/git/provider_github.go | 161 +++++++++++++++++++++++++++++++++++++ pkg/git/provider_gitlab.go | 147 +++++++++++++++++++++++++++++++++ pkg/git/repository.go | 145 +++++++++++++++++++++++++++++++++ 4 files changed, 462 insertions(+) create mode 100644 pkg/git/provider.go create mode 100644 pkg/git/provider_github.go create mode 100644 pkg/git/provider_gitlab.go create mode 100644 pkg/git/repository.go diff --git a/pkg/git/provider.go b/pkg/git/provider.go new file mode 100644 index 00000000..8ec35c8e --- /dev/null +++ b/pkg/git/provider.go @@ -0,0 +1,9 @@ +package git + +import "context" + +type Provider interface { + CreateRepository(ctx context.Context, r *Repository) (bool, error) + AddTeam(ctx context.Context, r *Repository, name, permission string) (bool, error) + AddDeployKey(ctx context.Context, r *Repository, key, keyName string) (bool, error) +} diff --git a/pkg/git/provider_github.go b/pkg/git/provider_github.go new file mode 100644 index 00000000..efb2edc1 --- /dev/null +++ b/pkg/git/provider_github.go @@ -0,0 +1,161 @@ +package git + +import ( + "context" + "fmt" + "github.com/google/go-github/v32/github" + "strings" +) + +// GithubProvider represents a GitHub API wrapper +type GithubProvider struct { + IsPrivate bool + IsPersonal bool +} + +const ( + GitHubTokenName = "GITHUB_TOKEN" + GitHubDefaultHostname = "github.com" +) + +func (p *GithubProvider) newClient(r *Repository) (*github.Client, error) { + auth := github.BasicAuthTransport{ + Username: "git", + Password: r.Token, + } + + gh := github.NewClient(auth.Client()) + if r.Host != GitHubDefaultHostname { + baseURL := fmt.Sprintf("https://%s/api/v3/", r.Host) + uploadURL := fmt.Sprintf("https://%s/api/uploads/", r.Host) + if g, err := github.NewEnterpriseClient(baseURL, uploadURL, auth.Client()); err == nil { + gh = g + } else { + return nil, err + } + } + + return gh, nil +} + +// CreateRepository returns false if the repository exists +func (p *GithubProvider) CreateRepository(ctx context.Context, r *Repository) (bool, error) { + gh, err := p.newClient(r) + if err != nil { + return false, fmt.Errorf("client error: %w", err) + } + org := "" + if !p.IsPersonal { + org = r.Owner + } + + if _, _, err := gh.Repositories.Get(ctx, org, r.Name); err == nil { + return false, nil + } + + autoInit := true + _, _, err = gh.Repositories.Create(ctx, org, &github.Repository{ + AutoInit: &autoInit, + Name: &r.Name, + Private: &p.IsPrivate, + }) + if err != nil { + if !strings.Contains(err.Error(), "name already exists on this account") { + return false, fmt.Errorf("create repository error: %w", err) + } + } else { + return true, nil + } + return false, nil +} + +// AddTeam returns false if the team is already assigned to the repository +func (p *GithubProvider) AddTeam(ctx context.Context, r *Repository, name, permission string) (bool, error) { + gh, err := p.newClient(r) + if err != nil { + return false, fmt.Errorf("client error: %w", err) + } + + // check team exists + _, _, err = gh.Teams.GetTeamBySlug(ctx, r.Owner, name) + if err != nil { + return false, fmt.Errorf("get team %s error: %w", name, err) + } + + // check if team is assigned to the repo + _, resp, err := gh.Teams.IsTeamRepoBySlug(ctx, r.Owner, name, r.Owner, r.Name) + if resp == nil && err != nil { + return false, fmt.Errorf("is team %s error: %w", name, err) + } + + // add team to the repo + if resp.StatusCode == 404 { + _, err = gh.Teams.AddTeamRepoBySlug(ctx, r.Owner, name, r.Owner, r.Name, &github.TeamAddTeamRepoOptions{ + Permission: permission, + }) + if err != nil { + return false, fmt.Errorf("add team %s error: %w", name, err) + } + return true, nil + } + + return false, nil +} + +// AddDeployKey returns false if the key exists and the content is the same +func (p *GithubProvider) AddDeployKey(ctx context.Context, r *Repository, key, keyName string) (bool, error) { + gh, err := p.newClient(r) + if err != nil { + return false, fmt.Errorf("client error: %w", err) + } + + // list deploy keys + keys, resp, err := gh.Repositories.ListKeys(ctx, r.Owner, r.Name, nil) + if err != nil { + return false, fmt.Errorf("list deploy keys error: %w", err) + } + if resp.StatusCode >= 300 { + return false, fmt.Errorf("list deploy keys failed with status code: %s", resp.Status) + } + + // check if the key exists + shouldCreateKey := true + var existingKey *github.Key + for _, k := range keys { + if k.Title != nil && k.Key != nil && *k.Title == keyName { + if *k.Key != key { + existingKey = k + } else { + shouldCreateKey = false + } + break + } + } + + // delete existing key if the value differs + if existingKey != nil { + resp, err := gh.Repositories.DeleteKey(ctx, r.Owner, r.Name, *existingKey.ID) + if err != nil { + return false, fmt.Errorf("delete deploy key error: %w", err) + } + if resp.StatusCode >= 300 { + return false, fmt.Errorf("delete deploy key failed with status code: %s", resp.Status) + } + } + + // create key + if shouldCreateKey { + isReadOnly := true + _, _, err = gh.Repositories.CreateKey(ctx, r.Owner, r.Name, &github.Key{ + Title: &keyName, + Key: &key, + ReadOnly: &isReadOnly, + }) + if err != nil { + return false, fmt.Errorf("create deploy key error: %w", err) + } + return true, nil + } + + return false, nil +} diff --git a/pkg/git/provider_gitlab.go b/pkg/git/provider_gitlab.go new file mode 100644 index 00000000..07d97649 --- /dev/null +++ b/pkg/git/provider_gitlab.go @@ -0,0 +1,147 @@ +package git + +import ( + "context" + "fmt" + "github.com/xanzy/go-gitlab" +) + +// GitLabProvider represents a GitLab API wrapper +type GitLabProvider struct { + IsPrivate bool + IsPersonal bool +} + +const ( + GitLabTokenName = "GITLAB_TOKEN" + GitLabDefaultHostname = "gitlab.com" +) + +func (p *GitLabProvider) newClient(r *Repository) (*gitlab.Client, error) { + gl, err := gitlab.NewClient(r.Token) + if err != nil { + return nil, err + } + + if r.Host != GitLabDefaultHostname { + gl, err = gitlab.NewClient(r.Token, gitlab.WithBaseURL(fmt.Sprintf("https://%s/api/v4", r.Host))) + if err != nil { + return nil, err + } + } + return gl, nil +} + +// CreateRepository returns false if the repository already exists +func (p *GitLabProvider) CreateRepository(ctx context.Context, r *Repository) (bool, error) { + gl, err := p.newClient(r) + if err != nil { + return false, fmt.Errorf("client error: %w", err) + } + + var id *int + if !p.IsPersonal { + groups, _, err := gl.Groups.ListGroups(&gitlab.ListGroupsOptions{Search: gitlab.String(r.Owner)}, gitlab.WithContext(ctx)) + if err != nil { + return false, fmt.Errorf("list groups error: %w", err) + } + + if len(groups) > 0 { + id = &groups[0].ID + } + } + + visibility := gitlab.PublicVisibility + if p.IsPrivate { + visibility = gitlab.PrivateVisibility + } + + projects, _, err := gl.Projects.ListProjects(&gitlab.ListProjectsOptions{Search: gitlab.String(r.Name)}, gitlab.WithContext(ctx)) + if err != nil { + return false, fmt.Errorf("list projects error: %w", err) + } + + if len(projects) == 0 { + p := &gitlab.CreateProjectOptions{ + Name: gitlab.String(r.Name), + NamespaceID: id, + Visibility: &visibility, + InitializeWithReadme: gitlab.Bool(true), + } + + _, _, err := gl.Projects.CreateProject(p) + if err != nil { + return false, fmt.Errorf("create project error: %w", err) + } + return true, nil + } + + return false, nil +} + +// AddTeam returns false if the team is already assigned to the repository +func (p *GitLabProvider) AddTeam(ctx context.Context, r *Repository, name, permission string) (bool, error) { + return false, nil +} + +// AddDeployKey returns false if the key exists and the content is the same +func (p *GitLabProvider) AddDeployKey(ctx context.Context, r *Repository, key, keyName string) (bool, error) { + gl, err := p.newClient(r) + if err != nil { + return false, fmt.Errorf("client error: %w", err) + } + + // list deploy keys + var projId int + projects, _, err := gl.Projects.ListProjects(&gitlab.ListProjectsOptions{Search: gitlab.String(r.Name)}, gitlab.WithContext(ctx)) + if err != nil { + return false, fmt.Errorf("list projects error: %w", err) + } + if len(projects) > 0 { + projId = projects[0].ID + } else { + return false, fmt.Errorf("no project found") + } + + // check if the key exists + keys, _, err := gl.DeployKeys.ListProjectDeployKeys(projId, &gitlab.ListProjectDeployKeysOptions{}) + if err != nil { + return false, fmt.Errorf("list keys error: %w", err) + } + + shouldCreateKey := true + var existingKey *gitlab.DeployKey + for _, k := range keys { + if k.Title == keyName { + if k.Key != key { + existingKey = k + } else { + shouldCreateKey = false + } + break + } + } + + // delete existing key if the value differs + if existingKey != nil { + _, err := gl.DeployKeys.DeleteDeployKey(projId, existingKey.ID, gitlab.WithContext(ctx)) + if err != nil { + return false, fmt.Errorf("delete key error: %w", err) + } + } + + // create key + if shouldCreateKey { + _, _, err := gl.DeployKeys.AddDeployKey(projId, &gitlab.AddDeployKeyOptions{ + Title: gitlab.String(keyName), + Key: gitlab.String(key), + CanPush: gitlab.Bool(false), + }, gitlab.WithContext(ctx)) + if err != nil { + return false, fmt.Errorf("add key error: %w", err) + } + return true, nil + } + + return false, nil +} diff --git a/pkg/git/repository.go b/pkg/git/repository.go new file mode 100644 index 00000000..76440e6b --- /dev/null +++ b/pkg/git/repository.go @@ -0,0 +1,145 @@ +package git + +import ( + "context" + "fmt" + "time" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http" +) + +// Repository represents a git repository wrapper +type Repository struct { + Name string + Owner string + Host string + Branch string + Token string + AuthorName string + AuthorEmail string +} + +// NewRepository returns a git repository wrapper +func NewRepository(name, owner, host, branch, token, authorName, authorEmail string) (*Repository, error) { + if name == "" { + return nil, fmt.Errorf("name required") + } + if owner == "" { + return nil, fmt.Errorf("owner required") + } + if host == "" { + return nil, fmt.Errorf("host required") + } + if branch == "" { + return nil, fmt.Errorf("branch required") + } + if token == "" { + return nil, fmt.Errorf("token required") + } + if authorName == "" { + authorName = "tk" + } + if authorEmail == "" { + authorEmail = "tk@users.noreply.git-scm.com" + } + + return &Repository{ + Name: name, + Owner: owner, + Host: host, + Branch: branch, + Token: token, + AuthorName: authorName, + AuthorEmail: authorEmail, + }, nil +} + +// GetURL returns the repository HTTPS address +func (r *Repository) GetURL() string { + return fmt.Sprintf("https://%s/%s/%s", r.Host, r.Owner, r.Name) +} + +// GetSSH returns the repository SSH address +func (r *Repository) GetSSH() string { + return fmt.Sprintf("ssh://git@%s/%s/%s", r.Host, r.Owner, r.Name) +} + +func (r *Repository) auth() transport.AuthMethod { + return &http.BasicAuth{ + Username: "git", + Password: r.Token, + } +} + +// Checkout repository at specified path +func (r *Repository) Checkout(ctx context.Context, path string) (*git.Repository, error) { + repo, err := git.PlainCloneContext(ctx, path, false, &git.CloneOptions{ + URL: r.GetURL(), + Auth: r.auth(), + RemoteName: git.DefaultRemoteName, + ReferenceName: plumbing.NewBranchReferenceName(r.Branch), + SingleBranch: true, + NoCheckout: false, + Progress: nil, + Tags: git.NoTags, + }) + if err != nil { + return nil, fmt.Errorf("git clone error: %w", err) + } + + _, err = repo.Head() + if err != nil { + return nil, fmt.Errorf("git resolve HEAD error: %w", err) + } + + return repo, nil +} + +// Commit changes for the specified path, returns false if no changes are made +func (r *Repository) Commit(ctx context.Context, repo *git.Repository, path, message string) (bool, error) { + w, err := repo.Worktree() + if err != nil { + return false, err + } + + _, err = w.Add(path) + if err != nil { + return false, err + } + + status, err := w.Status() + if err != nil { + return false, err + } + + if !status.IsClean() { + if _, err := w.Commit(message, &git.CommitOptions{ + Author: &object.Signature{ + Name: r.AuthorName, + Email: r.AuthorEmail, + When: time.Now(), + }, + }); err != nil { + return false, err + } + return true, nil + } + + return false, nil +} + +// Push commits to origin +func (r *Repository) Push(ctx context.Context, repo *git.Repository) error { + err := repo.PushContext(ctx, &git.PushOptions{ + Auth: r.auth(), + Progress: nil, + }) + if err != nil { + return fmt.Errorf("git push error: %w", err) + } + return nil +} From d0a79c2b4c25fee71eb25bb1b5ae461326189ff2 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 18 Jun 2020 09:59:39 +0300 Subject: [PATCH 3/4] Use git package for bootstrap --- cmd/tk/bootstrap.go | 213 ++++++++++++++++ cmd/tk/bootstrap_github.go | 503 ++++--------------------------------- cmd/tk/bootstrap_gitlab.go | 235 ++++------------- pkg/git/provider.go | 1 + pkg/git/repository.go | 44 ++-- 5 files changed, 340 insertions(+), 656 deletions(-) diff --git a/cmd/tk/bootstrap.go b/cmd/tk/bootstrap.go index 052186c6..33fb6dde 100644 --- a/cmd/tk/bootstrap.go +++ b/cmd/tk/bootstrap.go @@ -1,7 +1,25 @@ package main import ( + "context" + "fmt" + "net/url" + "os" + "path" + "path/filepath" + "sigs.k8s.io/yaml" + "strings" + "time" + "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + + kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1alpha1" + sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1" ) var bootstrapCmd = &cobra.Command{ @@ -13,8 +31,203 @@ var ( bootstrapVersion string ) +const ( + bootstrapBranch = "master" + bootstrapInstallManifest = "toolkit-components.yaml" + bootstrapSourceManifest = "toolkit-source.yaml" + bootstrapKustomizationManifest = "toolkit-kustomization.yaml" +) + func init() { bootstrapCmd.PersistentFlags().StringVar(&bootstrapVersion, "version", "master", "toolkit tag or branch") rootCmd.AddCommand(bootstrapCmd) } + +func generateInstallManifests(targetPath, namespace, tmpDir string) (string, error) { + tkDir := path.Join(tmpDir, ".tk") + defer os.RemoveAll(tkDir) + + if err := os.MkdirAll(tkDir, os.ModePerm); err != nil { + return "", fmt.Errorf("generating manifests failed: %w", err) + } + + if err := genInstallManifests(bootstrapVersion, namespace, components, tkDir); err != nil { + return "", fmt.Errorf("generating manifests failed: %w", err) + } + + manifestsDir := path.Join(tmpDir, targetPath, namespace) + if err := os.MkdirAll(manifestsDir, os.ModePerm); err != nil { + return "", fmt.Errorf("generating manifests failed: %w", err) + } + + manifest := path.Join(manifestsDir, bootstrapInstallManifest) + if err := buildKustomization(tkDir, manifest); err != nil { + return "", fmt.Errorf("build kustomization failed: %w", err) + } + + return manifest, nil +} + +func applyInstallManifests(ctx context.Context, manifestPath string, components []string) error { + command := fmt.Sprintf("kubectl apply -f %s", manifestPath) + if _, err := utils.execCommand(ctx, ModeOS, command); err != nil { + return fmt.Errorf("install failed") + } + + for _, deployment := range components { + command = fmt.Sprintf("kubectl -n %s rollout status deployment %s --timeout=%s", + namespace, deployment, timeout.String()) + if _, err := utils.execCommand(ctx, ModeOS, command); err != nil { + return fmt.Errorf("install failed") + } + } + return nil +} + +func generateSyncManifests(url, name, namespace, targetPath, tmpDir string, interval time.Duration) error { + gvk := sourcev1.GroupVersion.WithKind("GitRepository") + gitRepository := sourcev1.GitRepository{ + TypeMeta: metav1.TypeMeta{ + Kind: gvk.Kind, + APIVersion: gvk.GroupVersion().String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: sourcev1.GitRepositorySpec{ + URL: url, + Interval: metav1.Duration{ + Duration: interval, + }, + Reference: &sourcev1.GitRepositoryRef{ + Branch: "master", + }, + SecretRef: &corev1.LocalObjectReference{ + Name: name, + }, + }, + } + + gitData, err := yaml.Marshal(gitRepository) + if err != nil { + return err + } + + if err := utils.writeFile(string(gitData), filepath.Join(tmpDir, targetPath, namespace, bootstrapSourceManifest)); err != nil { + return err + } + + gvk = kustomizev1.GroupVersion.WithKind("Kustomization") + emptyAPIGroup := "" + kustomization := kustomizev1.Kustomization{ + TypeMeta: metav1.TypeMeta{ + Kind: gvk.Kind, + APIVersion: gvk.GroupVersion().String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: kustomizev1.KustomizationSpec{ + Interval: metav1.Duration{ + Duration: 10 * time.Minute, + }, + Path: fmt.Sprintf("./%s", strings.TrimPrefix(targetPath, "./")), + Prune: true, + SourceRef: corev1.TypedLocalObjectReference{ + APIGroup: &emptyAPIGroup, + Kind: "GitRepository", + Name: name, + }, + }, + } + + ksData, err := yaml.Marshal(kustomization) + if err != nil { + return err + } + + if err := utils.writeFile(string(ksData), filepath.Join(tmpDir, targetPath, namespace, bootstrapKustomizationManifest)); err != nil { + return err + } + + return nil +} + +func applySyncManifests(ctx context.Context, kubeClient client.Client, name, namespace, targetPath, tmpDir string) error { + command := fmt.Sprintf("kubectl apply -f %s", filepath.Join(tmpDir, targetPath, namespace)) + if _, err := utils.execCommand(ctx, ModeStderrOS, command); err != nil { + return err + } + + logWaiting("waiting for cluster sync") + + if err := wait.PollImmediate(pollInterval, timeout, + isGitRepositoryReady(ctx, kubeClient, name, namespace)); err != nil { + return err + } + + if err := wait.PollImmediate(pollInterval, timeout, + isKustomizationReady(ctx, kubeClient, name, namespace)); err != nil { + return err + } + + return nil +} + +func shouldInstallManifests(ctx context.Context, kubeClient client.Client, namespace string) bool { + namespacedName := types.NamespacedName{ + Namespace: namespace, + Name: namespace, + } + var kustomization kustomizev1.Kustomization + if err := kubeClient.Get(ctx, namespacedName, &kustomization); err != nil { + return true + } + + return kustomization.Status.LastAppliedRevision == "" +} + +func shouldCreateDeployKey(ctx context.Context, kubeClient client.Client, namespace string) bool { + namespacedName := types.NamespacedName{ + Namespace: namespace, + Name: namespace, + } + + var existing corev1.Secret + if err := kubeClient.Get(ctx, namespacedName, &existing); err != nil { + return true + } + return false +} + +func generateDeployKey(ctx context.Context, kubeClient client.Client, url *url.URL, namespace string) (string, error) { + pair, err := generateKeyPair(ctx) + if err != nil { + return "", err + } + + hostKey, err := scanHostKey(ctx, url) + if err != nil { + return "", err + } + + secret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Namespace: namespace, + }, + StringData: map[string]string{ + "identity": string(pair.PrivateKey), + "identity.pub": string(pair.PublicKey), + "known_hosts": string(hostKey), + }, + } + if err := upsertSecret(ctx, kubeClient, secret); err != nil { + return "", err + } + + return string(pair.PublicKey), nil +} diff --git a/cmd/tk/bootstrap_github.go b/cmd/tk/bootstrap_github.go index 0b145203..3133f891 100644 --- a/cmd/tk/bootstrap_github.go +++ b/cmd/tk/bootstrap_github.go @@ -7,25 +7,11 @@ import ( "net/url" "os" "path" - "path/filepath" - "sigs.k8s.io/yaml" - "strings" "time" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" - "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/google/go-github/v32/github" "github.com/spf13/cobra" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" - "sigs.k8s.io/controller-runtime/pkg/client" - - kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1alpha1" - sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1" + + "github.com/fluxcd/toolkit/pkg/git" ) var bootstrapGitHubCmd = &cobra.Command{ @@ -70,13 +56,7 @@ var ( ) const ( - ghTokenName = "GITHUB_TOKEN" - ghBranch = "master" - ghInstallManifest = "toolkit-components.yaml" - ghSourceManifest = "toolkit-source.yaml" - ghKustomizationManifest = "toolkit-kustomization.yaml" - ghDefaultHostname = "github.com" - ghDefaultPermission = "maintain" + ghDefaultPermission = "maintain" ) func init() { @@ -86,22 +66,26 @@ func init() { bootstrapGitHubCmd.Flags().BoolVar(&ghPersonal, "personal", false, "is personal repository") bootstrapGitHubCmd.Flags().BoolVar(&ghPrivate, "private", true, "is private repository") bootstrapGitHubCmd.Flags().DurationVar(&ghInterval, "interval", time.Minute, "sync interval") - bootstrapGitHubCmd.Flags().StringVar(&ghHostname, "hostname", ghDefaultHostname, "GitHub hostname") + bootstrapGitHubCmd.Flags().StringVar(&ghHostname, "hostname", git.GitHubDefaultHostname, "GitHub hostname") bootstrapGitHubCmd.Flags().StringVar(&ghPath, "path", "", "repository path, when specified the cluster sync will be scoped to this path") bootstrapCmd.AddCommand(bootstrapGitHubCmd) } func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { - ghToken := os.Getenv(ghTokenName) + ghToken := os.Getenv(git.GitHubTokenName) if ghToken == "" { - return fmt.Errorf("%s environment variable not found", ghTokenName) + return fmt.Errorf("%s environment variable not found", git.GitHubTokenName) + } + + repository, err := git.NewRepository(ghRepository, ghOwner, ghHostname, ghToken, "tk", "tk@users.noreply.github.com") + if err != nil { + return err } - ghURL := fmt.Sprintf("https://%s/%s/%s", ghHostname, ghOwner, ghRepository) - sshURL := fmt.Sprintf("ssh://git@%s/%s/%s", ghHostname, ghOwner, ghRepository) - if ghOwner == "" || ghRepository == "" { - return fmt.Errorf("owner and repository are required") + provider := &git.GithubProvider{ + IsPrivate: ghPrivate, + IsPersonal: ghPersonal, } kubeClient, err := utils.kubeClient(kubeconfig) @@ -120,46 +104,49 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { // create GitHub repository if doesn't exists logAction("connecting to %s", ghHostname) - if err := createGitHubRepository(ctx, ghHostname, ghOwner, ghRepository, ghToken, ghPrivate, ghPersonal); err != nil { + changed, err := provider.CreateRepository(ctx, repository) + if err != nil { return err } + if changed { + logSuccess("repository created") + } withErrors := false // add teams to org repository if !ghPersonal { for _, team := range ghTeams { - if err := addGitHubTeam(ctx, ghHostname, ghOwner, ghRepository, ghToken, team, ghDefaultPermission); err != nil { + if changed, err := provider.AddTeam(ctx, repository, team, ghDefaultPermission); err != nil { logFailure(err.Error()) withErrors = true - } else { + } else if changed { logSuccess("%s team access granted", team) } } } // clone repository and checkout the master branch - repo, err := checkoutGitHubRepository(ctx, ghURL, ghBranch, ghToken, tmpDir) - if err != nil { + if err := repository.Checkout(ctx, bootstrapBranch, tmpDir); err != nil { return err } logSuccess("repository cloned") // generate install manifests logGenerate("generating manifests") - manifest, err := generateGitHubInstall(ghPath, namespace, tmpDir) + manifest, err := generateInstallManifests(ghPath, namespace, tmpDir) if err != nil { return err } // stage install manifests - changed, err := commitGitHubManifests(repo, ghPath, namespace) + changed, err = repository.Commit(ctx, path.Join(ghPath, namespace), "Add manifests") if err != nil { return err } // push install manifests if changed { - if err := pushGitHubRepository(ctx, repo, ghToken); err != nil { + if err := repository.Push(ctx); err != nil { return err } logSuccess("components manifests pushed") @@ -168,74 +155,63 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { } // determine if repo synchronization is working - isInstall := shouldInstallGitHub(ctx, kubeClient, namespace) + isInstall := shouldInstallManifests(ctx, kubeClient, namespace) if isInstall { // apply install manifests logAction("installing components in %s namespace", namespace) - command := fmt.Sprintf("kubectl apply -f %s", manifest) - if _, err := utils.execCommand(ctx, ModeOS, command); err != nil { - return fmt.Errorf("install failed") + if err := applyInstallManifests(ctx, manifest, components); err != nil { + return err } logSuccess("install completed") - - // check installation - logWaiting("verifying installation") - for _, deployment := range components { - command = fmt.Sprintf("kubectl -n %s rollout status deployment %s --timeout=%s", - namespace, deployment, timeout.String()) - if _, err := utils.execCommand(ctx, ModeOS, command); err != nil { - return fmt.Errorf("install failed") - } else { - logSuccess("%s ready", deployment) - } - } } // setup SSH deploy key - if shouldCreateGitHubDeployKey(ctx, kubeClient, namespace) { + if shouldCreateDeployKey(ctx, kubeClient, namespace) { logAction("configuring deploy key") - u, err := url.Parse(sshURL) + u, err := url.Parse(repository.GetSSH()) if err != nil { return fmt.Errorf("git URL parse failed: %w", err) } - key, err := generateGitHubDeployKey(ctx, kubeClient, u, namespace) + key, err := generateDeployKey(ctx, kubeClient, u, namespace) if err != nil { return fmt.Errorf("generating deploy key failed: %w", err) } - if err := createGitHubDeployKey(ctx, key, ghHostname, ghOwner, ghRepository, ghPath, ghToken); err != nil { + keyName := "tk" + if ghPath != "" { + keyName = fmt.Sprintf("tk-%s", ghPath) + } + + if changed, err := provider.AddDeployKey(ctx, repository, key, keyName); err != nil { return err + } else if changed { + logSuccess("deploy key configured") } - logSuccess("deploy key configured") } // configure repo synchronization if isInstall { // generate source and kustomization manifests logAction("generating sync manifests") - if err := generateGitHubKustomization(sshURL, namespace, namespace, ghPath, tmpDir, ghInterval); err != nil { + if err := generateSyncManifests(repository.GetSSH(), namespace, namespace, ghPath, tmpDir, ghInterval); err != nil { return err } - // stage manifests - changed, err = commitGitHubManifests(repo, ghPath, namespace) - if err != nil { + // commit and push manifests + if changed, err = repository.Commit(ctx, path.Join(ghPath, namespace), "Add manifests"); err != nil { return err - } - - // push manifests - if changed { - if err := pushGitHubRepository(ctx, repo, ghToken); err != nil { + } else if changed { + if err := repository.Push(ctx); err != nil { return err } + logSuccess("sync manifests pushed") } - logSuccess("sync manifests pushed") // apply manifests and waiting for sync logAction("applying sync manifests") - if err := applyGitHubKustomization(ctx, kubeClient, namespace, namespace, ghPath, tmpDir); err != nil { + if err := applySyncManifests(ctx, kubeClient, namespace, namespace, ghPath, tmpDir); err != nil { return err } } @@ -247,390 +223,3 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { logSuccess("bootstrap finished") return nil } - -func makeGitHubClient(hostname, token string) (*github.Client, error) { - auth := github.BasicAuthTransport{ - Username: "git", - Password: token, - } - - gh := github.NewClient(auth.Client()) - if hostname != ghDefaultHostname { - baseURL := fmt.Sprintf("https://%s/api/v3/", hostname) - uploadURL := fmt.Sprintf("https://%s/api/uploads/", hostname) - if g, err := github.NewEnterpriseClient(baseURL, uploadURL, auth.Client()); err == nil { - gh = g - } else { - return nil, fmt.Errorf("github client error: %w", err) - } - } - - return gh, nil -} - -func createGitHubRepository(ctx context.Context, hostname, owner, name, token string, isPrivate, isPersonal bool) error { - gh, err := makeGitHubClient(hostname, token) - if err != nil { - return err - } - org := "" - if !isPersonal { - org = owner - } - - if _, _, err := gh.Repositories.Get(ctx, org, name); err == nil { - return nil - } - - autoInit := true - _, _, err = gh.Repositories.Create(ctx, org, &github.Repository{ - AutoInit: &autoInit, - Name: &name, - Private: &isPrivate, - }) - if err != nil { - if !strings.Contains(err.Error(), "name already exists on this account") { - return fmt.Errorf("github create repository error: %w", err) - } - } else { - logSuccess("repository created") - } - return nil -} - -func addGitHubTeam(ctx context.Context, hostname, owner, repository, token string, teamSlug, permission string) error { - gh, err := makeGitHubClient(hostname, token) - if err != nil { - return err - } - - // check team exists - _, _, err = gh.Teams.GetTeamBySlug(ctx, owner, teamSlug) - if err != nil { - return fmt.Errorf("github get team %s error: %w", teamSlug, err) - } - - // check if team is assigned to the repo - _, resp, err := gh.Teams.IsTeamRepoBySlug(ctx, owner, teamSlug, owner, repository) - if resp == nil && err != nil { - return fmt.Errorf("github is team %s error: %w", teamSlug, err) - } - - // add team to the repo - if resp.StatusCode == 404 { - _, err = gh.Teams.AddTeamRepoBySlug(ctx, owner, teamSlug, owner, repository, &github.TeamAddTeamRepoOptions{ - Permission: permission, - }) - if err != nil { - return fmt.Errorf("github add team %s error: %w", teamSlug, err) - } - } - - return nil -} - -func checkoutGitHubRepository(ctx context.Context, url, branch, token, path string) (*git.Repository, error) { - auth := &http.BasicAuth{ - Username: "git", - Password: token, - } - repo, err := git.PlainCloneContext(ctx, path, false, &git.CloneOptions{ - URL: url, - Auth: auth, - RemoteName: git.DefaultRemoteName, - ReferenceName: plumbing.NewBranchReferenceName(branch), - SingleBranch: true, - NoCheckout: false, - Progress: nil, - Tags: git.NoTags, - }) - if err != nil { - return nil, fmt.Errorf("git clone error: %w", err) - } - - _, err = repo.Head() - if err != nil { - return nil, fmt.Errorf("git resolve HEAD error: %w", err) - } - - return repo, nil -} - -func generateGitHubInstall(targetPath, namespace, tmpDir string) (string, error) { - tkDir := path.Join(tmpDir, ".tk") - defer os.RemoveAll(tkDir) - - if err := os.MkdirAll(tkDir, os.ModePerm); err != nil { - return "", fmt.Errorf("generating manifests failed: %w", err) - } - - if err := genInstallManifests(bootstrapVersion, namespace, components, tkDir); err != nil { - return "", fmt.Errorf("generating manifests failed: %w", err) - } - - manifestsDir := path.Join(tmpDir, targetPath, namespace) - if err := os.MkdirAll(manifestsDir, os.ModePerm); err != nil { - return "", fmt.Errorf("generating manifests failed: %w", err) - } - - manifest := path.Join(manifestsDir, ghInstallManifest) - if err := buildKustomization(tkDir, manifest); err != nil { - return "", fmt.Errorf("build kustomization failed: %w", err) - } - - return manifest, nil -} - -func commitGitHubManifests(repo *git.Repository, targetPath, namespace string) (bool, error) { - w, err := repo.Worktree() - if err != nil { - return false, err - } - - _, err = w.Add(path.Join(targetPath, namespace)) - if err != nil { - return false, err - } - - status, err := w.Status() - if err != nil { - return false, err - } - - if !status.IsClean() { - if _, err := w.Commit("Add manifests", &git.CommitOptions{ - Author: &object.Signature{ - Name: "tk", - Email: "tk@users.noreply.github.com", - When: time.Now(), - }, - }); err != nil { - return false, err - } - return true, nil - } - - return false, nil -} - -func pushGitHubRepository(ctx context.Context, repo *git.Repository, token string) error { - auth := &http.BasicAuth{ - Username: "git", - Password: token, - } - err := repo.PushContext(ctx, &git.PushOptions{ - Auth: auth, - Progress: nil, - }) - if err != nil { - return fmt.Errorf("git push error: %w", err) - } - return nil -} - -func generateGitHubKustomization(url, name, namespace, targetPath, tmpDir string, interval time.Duration) error { - gvk := sourcev1.GroupVersion.WithKind("GitRepository") - gitRepository := sourcev1.GitRepository{ - TypeMeta: metav1.TypeMeta{ - Kind: gvk.Kind, - APIVersion: gvk.GroupVersion().String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: sourcev1.GitRepositorySpec{ - URL: url, - Interval: metav1.Duration{ - Duration: interval, - }, - Reference: &sourcev1.GitRepositoryRef{ - Branch: "master", - }, - SecretRef: &corev1.LocalObjectReference{ - Name: name, - }, - }, - } - - gitData, err := yaml.Marshal(gitRepository) - if err != nil { - return err - } - - if err := utils.writeFile(string(gitData), filepath.Join(tmpDir, targetPath, namespace, ghSourceManifest)); err != nil { - return err - } - - gvk = kustomizev1.GroupVersion.WithKind("Kustomization") - emptyAPIGroup := "" - kustomization := kustomizev1.Kustomization{ - TypeMeta: metav1.TypeMeta{ - Kind: gvk.Kind, - APIVersion: gvk.GroupVersion().String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: kustomizev1.KustomizationSpec{ - Interval: metav1.Duration{ - Duration: 10 * time.Minute, - }, - Path: fmt.Sprintf("./%s", strings.TrimPrefix(targetPath, "./")), - Prune: true, - SourceRef: corev1.TypedLocalObjectReference{ - APIGroup: &emptyAPIGroup, - Kind: "GitRepository", - Name: name, - }, - }, - } - - ksData, err := yaml.Marshal(kustomization) - if err != nil { - return err - } - - if err := utils.writeFile(string(ksData), filepath.Join(tmpDir, targetPath, namespace, ghKustomizationManifest)); err != nil { - return err - } - - return nil -} - -func applyGitHubKustomization(ctx context.Context, kubeClient client.Client, name, namespace, targetPath, tmpDir string) error { - command := fmt.Sprintf("kubectl apply -f %s", filepath.Join(tmpDir, targetPath, namespace)) - if _, err := utils.execCommand(ctx, ModeStderrOS, command); err != nil { - return err - } - - logWaiting("waiting for cluster sync") - - if err := wait.PollImmediate(pollInterval, timeout, - isGitRepositoryReady(ctx, kubeClient, name, namespace)); err != nil { - return err - } - - if err := wait.PollImmediate(pollInterval, timeout, - isKustomizationReady(ctx, kubeClient, name, namespace)); err != nil { - return err - } - - return nil -} - -func shouldInstallGitHub(ctx context.Context, kubeClient client.Client, namespace string) bool { - namespacedName := types.NamespacedName{ - Namespace: namespace, - Name: namespace, - } - var kustomization kustomizev1.Kustomization - if err := kubeClient.Get(ctx, namespacedName, &kustomization); err != nil { - return true - } - - return kustomization.Status.LastAppliedRevision == "" -} - -func shouldCreateGitHubDeployKey(ctx context.Context, kubeClient client.Client, namespace string) bool { - namespacedName := types.NamespacedName{ - Namespace: namespace, - Name: namespace, - } - - var existing corev1.Secret - if err := kubeClient.Get(ctx, namespacedName, &existing); err != nil { - return true - } - return false -} - -func generateGitHubDeployKey(ctx context.Context, kubeClient client.Client, url *url.URL, namespace string) (string, error) { - pair, err := generateKeyPair(ctx) - if err != nil { - return "", err - } - - hostKey, err := scanHostKey(ctx, url) - if err != nil { - return "", err - } - - secret := corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - Namespace: namespace, - }, - StringData: map[string]string{ - "identity": string(pair.PrivateKey), - "identity.pub": string(pair.PublicKey), - "known_hosts": string(hostKey), - }, - } - if err := upsertSecret(ctx, kubeClient, secret); err != nil { - return "", err - } - - return string(pair.PublicKey), nil -} - -func createGitHubDeployKey(ctx context.Context, key, hostname, owner, repository, targetPath, token string) error { - gh, err := makeGitHubClient(hostname, token) - if err != nil { - return err - } - keyName := "tk" - if targetPath != "" { - keyName = fmt.Sprintf("tk-%s", targetPath) - } - - // list deploy keys - keys, resp, err := gh.Repositories.ListKeys(ctx, owner, repository, nil) - if err != nil { - return fmt.Errorf("github list deploy keys error: %w", err) - } - if resp.StatusCode >= 300 { - return fmt.Errorf("github list deploy keys failed with status code: %s", resp.Status) - } - - // check if the key exists - shouldCreateKey := true - var existingKey *github.Key - for _, k := range keys { - if k.Title != nil && k.Key != nil && *k.Title == keyName { - if *k.Key != key { - existingKey = k - } else { - shouldCreateKey = false - } - break - } - } - - // delete existing key if the value differs - if existingKey != nil { - resp, err := gh.Repositories.DeleteKey(ctx, owner, repository, *existingKey.ID) - if err != nil { - return fmt.Errorf("github delete deploy key error: %w", err) - } - if resp.StatusCode >= 300 { - return fmt.Errorf("github delete deploy key failed with status code: %s", resp.Status) - } - } - - // create key - if shouldCreateKey { - isReadOnly := true - _, _, err = gh.Repositories.CreateKey(ctx, owner, repository, &github.Key{ - Title: &keyName, - Key: &key, - ReadOnly: &isReadOnly, - }) - if err != nil { - return fmt.Errorf("github create deploy key error: %w", err) - } - } - - return nil -} diff --git a/cmd/tk/bootstrap_gitlab.go b/cmd/tk/bootstrap_gitlab.go index 56efd6fd..0ec0e5ac 100644 --- a/cmd/tk/bootstrap_gitlab.go +++ b/cmd/tk/bootstrap_gitlab.go @@ -6,10 +6,12 @@ import ( "io/ioutil" "net/url" "os" + "path" "time" "github.com/spf13/cobra" - "github.com/xanzy/go-gitlab" + + "github.com/fluxcd/toolkit/pkg/git" ) var bootstrapGitLabCmd = &cobra.Command{ @@ -21,14 +23,20 @@ commits the toolkit components manifests to the master branch. Then it configure the target cluster to synchronize with the repository. If the toolkit components are present on the cluster, the bootstrap command will perform an upgrade if needed.`, - Example: ` # Create a GitLab personal access token and export it as an env var + Example: ` # Create a GitLab API token and export it as an env var export GITLAB_TOKEN= - # Run bootstrap for a private repo owned by a GitLab organization - bootstrap gitlab --owner= --repository= + # Run bootstrap for a private repo owned by a GitLab group + bootstrap gitlab --owner= --repository= + + # Run bootstrap for a repository path + bootstrap gitlab --owner= --repository= --path=dev-cluster + + # Run bootstrap for a public repository on a personal account + bootstrap gitlab --owner= --repository= --private=false --personal=true # Run bootstrap for a private repo hosted on GitLab server - bootstrap gitlab --owner= --repository= --hostname= + bootstrap gitlab --owner= --repository= --hostname= `, RunE: bootstrapGitLabCmdRun, } @@ -43,33 +51,32 @@ var ( glPath string ) -const ( - glTokenName = "GITLAB_TOKEN" - glDefaultHostname = "gitlab.com" -) - func init() { bootstrapGitLabCmd.Flags().StringVar(&glOwner, "owner", "", "GitLab user or organization name") bootstrapGitLabCmd.Flags().StringVar(&glRepository, "repository", "", "GitLab repository name") bootstrapGitLabCmd.Flags().BoolVar(&glPersonal, "personal", false, "is personal repository") bootstrapGitLabCmd.Flags().BoolVar(&glPrivate, "private", true, "is private repository") bootstrapGitLabCmd.Flags().DurationVar(&glInterval, "interval", time.Minute, "sync interval") - bootstrapGitLabCmd.Flags().StringVar(&glHostname, "hostname", glDefaultHostname, "GitLab hostname") + bootstrapGitLabCmd.Flags().StringVar(&glHostname, "hostname", git.GitLabDefaultHostname, "GitLab hostname") bootstrapGitLabCmd.Flags().StringVar(&glPath, "path", "", "repository path, when specified the cluster sync will be scoped to this path") bootstrapCmd.AddCommand(bootstrapGitLabCmd) } func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error { - glToken := os.Getenv(glTokenName) + glToken := os.Getenv(git.GitLabTokenName) if glToken == "" { - return fmt.Errorf("%s environment variable not found", glTokenName) + return fmt.Errorf("%s environment variable not found", git.GitLabTokenName) } - gitURL := fmt.Sprintf("https://%s/%s/%s", glHostname, glOwner, glRepository) - sshURL := fmt.Sprintf("ssh://git@%s/%s/%s", glHostname, glOwner, glRepository) - if glOwner == "" || glRepository == "" { - return fmt.Errorf("owner and repository are required") + repository, err := git.NewRepository(glRepository, glOwner, glHostname, glToken, "tk", "tk@users.noreply.gitlab.com") + if err != nil { + return err + } + + provider := &git.GitLabProvider{ + IsPrivate: glPrivate, + IsPersonal: glPersonal, } kubeClient, err := utils.kubeClient(kubeconfig) @@ -88,32 +95,36 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error { // create GitLab project if doesn't exists logAction("connecting to %s", glHostname) - if err := createGitLabRepository(ctx, glHostname, glOwner, glRepository, glToken, glPrivate, glPersonal); err != nil { + changed, err := provider.CreateRepository(ctx, repository) + if err != nil { return err } + if changed { + logSuccess("repository created") + } // clone repository and checkout the master branch - repo, err := checkoutGitHubRepository(ctx, gitURL, ghBranch, glToken, tmpDir) - if err != nil { + if err := repository.Checkout(ctx, bootstrapBranch, tmpDir); err != nil { return err } logSuccess("repository cloned") // generate install manifests logGenerate("generating manifests") - manifest, err := generateGitHubInstall(glPath, namespace, tmpDir) + manifest, err := generateInstallManifests(glPath, namespace, tmpDir) if err != nil { return err } // stage install manifests - changed, err := commitGitHubManifests(repo, glPath, namespace) + changed, err = repository.Commit(ctx, path.Join(glPath, namespace), "Add manifests") if err != nil { return err } + // push install manifests if changed { - if err := pushGitHubRepository(ctx, repo, glToken); err != nil { + if err := repository.Push(ctx); err != nil { return err } logSuccess("components manifests pushed") @@ -122,74 +133,63 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error { } // determine if repo synchronization is working - isInstall := shouldInstallGitHub(ctx, kubeClient, namespace) + isInstall := shouldInstallManifests(ctx, kubeClient, namespace) if isInstall { // apply install manifests logAction("installing components in %s namespace", namespace) - command := fmt.Sprintf("kubectl apply -f %s", manifest) - if _, err := utils.execCommand(ctx, ModeOS, command); err != nil { - return fmt.Errorf("install failed") + if err := applyInstallManifests(ctx, manifest, components); err != nil { + return err } logSuccess("install completed") - - // check installation - logWaiting("verifying installation") - for _, deployment := range components { - command = fmt.Sprintf("kubectl -n %s rollout status deployment %s --timeout=%s", - namespace, deployment, timeout.String()) - if _, err := utils.execCommand(ctx, ModeOS, command); err != nil { - return fmt.Errorf("install failed") - } else { - logSuccess("%s ready", deployment) - } - } } // setup SSH deploy key - if shouldCreateGitHubDeployKey(ctx, kubeClient, namespace) { + if shouldCreateDeployKey(ctx, kubeClient, namespace) { logAction("configuring deploy key") - u, err := url.Parse(sshURL) + u, err := url.Parse(repository.GetSSH()) if err != nil { return fmt.Errorf("git URL parse failed: %w", err) } - key, err := generateGitHubDeployKey(ctx, kubeClient, u, namespace) + key, err := generateDeployKey(ctx, kubeClient, u, namespace) if err != nil { return fmt.Errorf("generating deploy key failed: %w", err) } - if err := createGitLabDeployKey(ctx, key, glHostname, glOwner, glRepository, glPath, glToken); err != nil { + keyName := "tk" + if glPath != "" { + keyName = fmt.Sprintf("tk-%s", glPath) + } + + if changed, err := provider.AddDeployKey(ctx, repository, key, keyName); err != nil { return err + } else if changed { + logSuccess("deploy key configured") } - logSuccess("deploy key configured") } // configure repo synchronization if isInstall { // generate source and kustomization manifests logAction("generating sync manifests") - if err := generateGitHubKustomization(sshURL, namespace, namespace, glPath, tmpDir, glInterval); err != nil { + if err := generateSyncManifests(repository.GetSSH(), namespace, namespace, glPath, tmpDir, glInterval); err != nil { return err } - // stage manifests - changed, err = commitGitHubManifests(repo, glPath, namespace) - if err != nil { + // commit and push manifests + if changed, err = repository.Commit(ctx, path.Join(glPath, namespace), "Add manifests"); err != nil { return err - } - - // push manifests - if changed { - if err := pushGitHubRepository(ctx, repo, glToken); err != nil { + } else if changed { + if err := repository.Push(ctx); err != nil { return err } + logSuccess("sync manifests pushed") } - logSuccess("sync manifests pushed") // apply manifests and waiting for sync logAction("applying sync manifests") - if err := applyGitHubKustomization(ctx, kubeClient, namespace, namespace, glPath, tmpDir); err != nil { + if err := applySyncManifests(ctx, kubeClient, namespace, namespace, glPath, tmpDir); err != nil { return err } } @@ -197,128 +197,3 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error { logSuccess("bootstrap finished") return nil } - -func makeGitLabClient(hostname, token string) (*gitlab.Client, error) { - gl, err := gitlab.NewClient(token) - if err != nil { - return nil, err - } - - if glHostname != glDefaultHostname { - gl, err = gitlab.NewClient(token, gitlab.WithBaseURL(fmt.Sprintf("https://%s/api/v4", hostname))) - if err != nil { - return nil, err - } - } - return gl, nil -} - -func createGitLabRepository(ctx context.Context, hostname, owner, repository, token string, isPrivate, isPersonal bool) error { - gl, err := makeGitLabClient(hostname, token) - if err != nil { - return fmt.Errorf("client error: %w", err) - } - - var id *int - if !isPersonal { - groups, _, err := gl.Groups.ListGroups(&gitlab.ListGroupsOptions{Search: gitlab.String(owner)}, gitlab.WithContext(ctx)) - if err != nil { - return fmt.Errorf("list groups error: %w", err) - } - - if len(groups) > 0 { - id = &groups[0].ID - } - } - - visibility := gitlab.PublicVisibility - if isPrivate { - visibility = gitlab.PrivateVisibility - } - - projects, _, err := gl.Projects.ListProjects(&gitlab.ListProjectsOptions{Search: gitlab.String(repository)}, gitlab.WithContext(ctx)) - if err != nil { - return fmt.Errorf("list projects error: %w", err) - } - - if len(projects) == 0 { - p := &gitlab.CreateProjectOptions{ - Name: gitlab.String(repository), - NamespaceID: id, - Visibility: &visibility, - InitializeWithReadme: gitlab.Bool(true), - } - - project, _, err := gl.Projects.CreateProject(p) - if err != nil { - return fmt.Errorf("create project error: %w", err) - } - logSuccess("project created id: %v", project.ID) - } - - return nil -} - -func createGitLabDeployKey(ctx context.Context, key, hostname, owner, repository, targetPath, token string) error { - gl, err := makeGitLabClient(hostname, token) - if err != nil { - return fmt.Errorf("client error: %w", err) - } - - var projId int - projects, _, err := gl.Projects.ListProjects(&gitlab.ListProjectsOptions{Search: gitlab.String(repository)}, gitlab.WithContext(ctx)) - if err != nil { - return fmt.Errorf("list projects error: %w", err) - } - if len(projects) > 0 { - projId = projects[0].ID - } else { - return fmt.Errorf("no project found") - } - - keyName := "tk" - if targetPath != "" { - keyName = fmt.Sprintf("tk-%s", targetPath) - } - - // check if the key exists - keys, _, err := gl.DeployKeys.ListProjectDeployKeys(projId, &gitlab.ListProjectDeployKeysOptions{}) - if err != nil { - return fmt.Errorf("list keys error: %w", err) - } - - shouldCreateKey := true - var existingKey *gitlab.DeployKey - for _, k := range keys { - if k.Title == keyName { - if k.Key != key { - existingKey = k - } else { - shouldCreateKey = false - } - break - } - } - - // delete existing key if the value differs - if existingKey != nil { - _, err := gl.DeployKeys.DeleteDeployKey(projId, existingKey.ID, gitlab.WithContext(ctx)) - if err != nil { - return fmt.Errorf("delete key error: %w", err) - } - } - - // create key - if shouldCreateKey { - _, _, err := gl.DeployKeys.AddDeployKey(projId, &gitlab.AddDeployKeyOptions{ - Title: gitlab.String(keyName), - Key: gitlab.String(key), - CanPush: gitlab.Bool(false), - }, gitlab.WithContext(ctx)) - if err != nil { - return fmt.Errorf("add key error: %w", err) - } - } - - return nil -} diff --git a/pkg/git/provider.go b/pkg/git/provider.go index 8ec35c8e..909f3967 100644 --- a/pkg/git/provider.go +++ b/pkg/git/provider.go @@ -2,6 +2,7 @@ package git import "context" +// Provider is the interface that a git provider should implement type Provider interface { CreateRepository(ctx context.Context, r *Repository) (bool, error) AddTeam(ctx context.Context, r *Repository, name, permission string) (bool, error) diff --git a/pkg/git/repository.go b/pkg/git/repository.go index 76440e6b..ca555859 100644 --- a/pkg/git/repository.go +++ b/pkg/git/repository.go @@ -17,14 +17,15 @@ type Repository struct { Name string Owner string Host string - Branch string Token string AuthorName string AuthorEmail string + + repo *git.Repository } // NewRepository returns a git repository wrapper -func NewRepository(name, owner, host, branch, token, authorName, authorEmail string) (*Repository, error) { +func NewRepository(name, owner, host, token, authorName, authorEmail string) (*Repository, error) { if name == "" { return nil, fmt.Errorf("name required") } @@ -34,24 +35,20 @@ func NewRepository(name, owner, host, branch, token, authorName, authorEmail str if host == "" { return nil, fmt.Errorf("host required") } - if branch == "" { - return nil, fmt.Errorf("branch required") - } if token == "" { return nil, fmt.Errorf("token required") } if authorName == "" { - authorName = "tk" + return nil, fmt.Errorf("author name required") } if authorEmail == "" { - authorEmail = "tk@users.noreply.git-scm.com" + return nil, fmt.Errorf("author email required") } return &Repository{ Name: name, Owner: owner, Host: host, - Branch: branch, Token: token, AuthorName: authorName, AuthorEmail: authorEmail, @@ -75,33 +72,38 @@ func (r *Repository) auth() transport.AuthMethod { } } -// Checkout repository at specified path -func (r *Repository) Checkout(ctx context.Context, path string) (*git.Repository, error) { +// Checkout repository branch at specified path +func (r *Repository) Checkout(ctx context.Context, branch, path string) error { repo, err := git.PlainCloneContext(ctx, path, false, &git.CloneOptions{ URL: r.GetURL(), Auth: r.auth(), RemoteName: git.DefaultRemoteName, - ReferenceName: plumbing.NewBranchReferenceName(r.Branch), + ReferenceName: plumbing.NewBranchReferenceName(branch), SingleBranch: true, NoCheckout: false, Progress: nil, Tags: git.NoTags, }) if err != nil { - return nil, fmt.Errorf("git clone error: %w", err) + return fmt.Errorf("git clone error: %w", err) } _, err = repo.Head() if err != nil { - return nil, fmt.Errorf("git resolve HEAD error: %w", err) + return fmt.Errorf("git resolve HEAD error: %w", err) } - return repo, nil + r.repo = repo + return nil } -// Commit changes for the specified path, returns false if no changes are made -func (r *Repository) Commit(ctx context.Context, repo *git.Repository, path, message string) (bool, error) { - w, err := repo.Worktree() +// Commit changes for the specified path, returns false if no changes are detected +func (r *Repository) Commit(ctx context.Context, path, message string) (bool, error) { + if r.repo == nil { + return false, fmt.Errorf("repository hasn't been cloned") + } + + w, err := r.repo.Worktree() if err != nil { return false, err } @@ -133,8 +135,12 @@ func (r *Repository) Commit(ctx context.Context, repo *git.Repository, path, mes } // Push commits to origin -func (r *Repository) Push(ctx context.Context, repo *git.Repository) error { - err := repo.PushContext(ctx, &git.PushOptions{ +func (r *Repository) Push(ctx context.Context) error { + if r.repo == nil { + return fmt.Errorf("repository hasn't been cloned") + } + + err := r.repo.PushContext(ctx, &git.PushOptions{ Auth: r.auth(), Progress: nil, }) From badd2a102f694f0ee65ad06af516a64689847c73 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 18 Jun 2020 12:03:49 +0300 Subject: [PATCH 4/4] Generate bootstrap docs --- docs/cmd/tk.md | 2 +- docs/cmd/tk_bootstrap.md | 3 +- docs/cmd/tk_bootstrap_github.md | 10 ++++- docs/cmd/tk_bootstrap_gitlab.md | 66 ++++++++++++++++++++++++++++ docs/cmd/tk_check.md | 2 +- docs/cmd/tk_completion.md | 2 +- docs/cmd/tk_create.md | 2 +- docs/cmd/tk_create_kustomization.md | 2 +- docs/cmd/tk_create_source.md | 2 +- docs/cmd/tk_create_source_git.md | 4 +- docs/cmd/tk_delete.md | 2 +- docs/cmd/tk_delete_kustomization.md | 2 +- docs/cmd/tk_delete_source.md | 2 +- docs/cmd/tk_delete_source_git.md | 2 +- docs/cmd/tk_export.md | 2 +- docs/cmd/tk_export_kustomization.md | 2 +- docs/cmd/tk_export_source.md | 2 +- docs/cmd/tk_export_source_git.md | 2 +- docs/cmd/tk_get.md | 2 +- docs/cmd/tk_get_kustomizations.md | 2 +- docs/cmd/tk_get_sources.md | 2 +- docs/cmd/tk_get_sources_git.md | 2 +- docs/cmd/tk_install.md | 2 +- docs/cmd/tk_resume.md | 2 +- docs/cmd/tk_resume_kustomization.md | 2 +- docs/cmd/tk_suspend.md | 2 +- docs/cmd/tk_suspend_kustomization.md | 2 +- docs/cmd/tk_sync.md | 2 +- docs/cmd/tk_sync_kustomization.md | 2 +- docs/cmd/tk_sync_source.md | 2 +- docs/cmd/tk_sync_source_git.md | 2 +- docs/cmd/tk_uninstall.md | 11 ++--- docs/internal/release.md | 7 +-- 33 files changed, 113 insertions(+), 42 deletions(-) create mode 100644 docs/cmd/tk_bootstrap_gitlab.md diff --git a/docs/cmd/tk.md b/docs/cmd/tk.md index a957dea0..4381649d 100644 --- a/docs/cmd/tk.md +++ b/docs/cmd/tk.md @@ -90,4 +90,4 @@ Command line utility for assembling Kubernetes CD pipelines the GitOps way. * [tk sync](tk_sync.md) - Synchronize commands * [tk uninstall](tk_uninstall.md) - Uninstall the toolkit components -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_bootstrap.md b/docs/cmd/tk_bootstrap.md index 7d0990d7..09227700 100644 --- a/docs/cmd/tk_bootstrap.md +++ b/docs/cmd/tk_bootstrap.md @@ -27,5 +27,6 @@ Bootstrap commands * [tk](tk.md) - Command line utility for assembling Kubernetes CD pipelines * [tk bootstrap github](tk_bootstrap_github.md) - Bootstrap GitHub repository +* [tk bootstrap gitlab](tk_bootstrap_gitlab.md) - Bootstrap GitLab repository -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_bootstrap_github.md b/docs/cmd/tk_bootstrap_github.md index 2210f60c..80c4bf65 100644 --- a/docs/cmd/tk_bootstrap_github.md +++ b/docs/cmd/tk_bootstrap_github.md @@ -24,6 +24,12 @@ tk bootstrap github [flags] # Run bootstrap for a private repo owned by a GitHub organization bootstrap github --owner= --repository= + # Run bootstrap for a private repo and assign organization teams to it + bootstrap github --owner= --repository= --team= --team= + + # Run bootstrap for a repository path + bootstrap github --owner= --repository= --path=dev-cluster + # Run bootstrap for a public repository on a personal account bootstrap github --owner= --repository= --private=false --personal=true @@ -39,9 +45,11 @@ tk bootstrap github [flags] --hostname string GitHub hostname (default "github.com") --interval duration sync interval (default 1m0s) --owner string GitHub user or organization name + --path string repository path, when specified the cluster sync will be scoped to this path --personal is personal repository --private is private repository (default true) --repository string GitHub repository name + --team stringArray GitHub team to be given maintainer access ``` ### Options inherited from parent commands @@ -59,4 +67,4 @@ tk bootstrap github [flags] * [tk bootstrap](tk_bootstrap.md) - Bootstrap commands -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_bootstrap_gitlab.md b/docs/cmd/tk_bootstrap_gitlab.md new file mode 100644 index 00000000..983ede4c --- /dev/null +++ b/docs/cmd/tk_bootstrap_gitlab.md @@ -0,0 +1,66 @@ +## tk bootstrap gitlab + +Bootstrap GitLab repository + +### Synopsis + + +The bootstrap command creates the GitHub repository if it doesn't exists and +commits the toolkit components manifests to the master branch. +Then it configure the target cluster to synchronize with the repository. +If the toolkit components are present on the cluster, +the bootstrap command will perform an upgrade if needed. + +``` +tk bootstrap gitlab [flags] +``` + +### Examples + +``` + # Create a GitLab API token and export it as an env var + export GITLAB_TOKEN= + + # Run bootstrap for a private repo owned by a GitLab group + bootstrap gitlab --owner= --repository= + + # Run bootstrap for a repository path + bootstrap gitlab --owner= --repository= --path=dev-cluster + + # Run bootstrap for a public repository on a personal account + bootstrap gitlab --owner= --repository= --private=false --personal=true + + # Run bootstrap for a private repo hosted on GitLab server + bootstrap gitlab --owner= --repository= --hostname= + +``` + +### Options + +``` + -h, --help help for gitlab + --hostname string GitLab hostname (default "gitlab.com") + --interval duration sync interval (default 1m0s) + --owner string GitLab user or organization name + --path string repository path, when specified the cluster sync will be scoped to this path + --personal is personal repository + --private is private repository (default true) + --repository string GitLab repository name +``` + +### Options inherited from parent commands + +``` + --components strings list of components, accepts comma-separated values (default [source-controller,kustomize-controller]) + --kubeconfig string path to the kubeconfig file (default "~/.kube/config") + --namespace string the namespace scope for this operation (default "gitops-system") + --timeout duration timeout for this operation (default 5m0s) + --verbose print generated objects + --version string toolkit tag or branch (default "master") +``` + +### SEE ALSO + +* [tk bootstrap](tk_bootstrap.md) - Bootstrap commands + +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_check.md b/docs/cmd/tk_check.md index 3f065592..de378dc8 100644 --- a/docs/cmd/tk_check.md +++ b/docs/cmd/tk_check.md @@ -44,4 +44,4 @@ tk check [flags] * [tk](tk.md) - Command line utility for assembling Kubernetes CD pipelines -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_completion.md b/docs/cmd/tk_completion.md index 14f7c146..904dbedf 100644 --- a/docs/cmd/tk_completion.md +++ b/docs/cmd/tk_completion.md @@ -44,4 +44,4 @@ To configure your bash shell to load completions for each session add to your ba * [tk](tk.md) - Command line utility for assembling Kubernetes CD pipelines -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_create.md b/docs/cmd/tk_create.md index 286a204c..4441d63d 100644 --- a/docs/cmd/tk_create.md +++ b/docs/cmd/tk_create.md @@ -30,4 +30,4 @@ Create commands * [tk create kustomization](tk_create_kustomization.md) - Create or update a kustomization resource * [tk create source](tk_create_source.md) - Create source commands -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_create_kustomization.md b/docs/cmd/tk_create_kustomization.md index 6cfb8f84..1242decb 100644 --- a/docs/cmd/tk_create_kustomization.md +++ b/docs/cmd/tk_create_kustomization.md @@ -78,4 +78,4 @@ tk create kustomization [name] [flags] * [tk create](tk_create.md) - Create commands -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_create_source.md b/docs/cmd/tk_create_source.md index 4d2559a6..e4566e02 100644 --- a/docs/cmd/tk_create_source.md +++ b/docs/cmd/tk_create_source.md @@ -29,4 +29,4 @@ Create source commands * [tk create](tk_create.md) - Create commands * [tk create source git](tk_create_source_git.md) - Create or update a git source -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_create_source_git.md b/docs/cmd/tk_create_source_git.md index b6d2687b..2ab78045 100644 --- a/docs/cmd/tk_create_source_git.md +++ b/docs/cmd/tk_create_source_git.md @@ -58,7 +58,7 @@ tk create source git [name] [flags] --branch string git branch (default "master") -h, --help help for git -p, --password string basic authentication password - --ssh-ecdsa-curve ecdsaCurve SSH ECDSA public key curve (p521, p256, p384) (default p384) + --ssh-ecdsa-curve ecdsaCurve SSH ECDSA public key curve (p256, p384, p521) (default p384) --ssh-key-algorithm publicKeyAlgorithm SSH public key algorithm (rsa, ecdsa, ed25519) (default rsa) --ssh-rsa-bits rsaKeyBits SSH RSA public key bit size (multiplies of 8) (default 2048) --tag string git tag @@ -83,4 +83,4 @@ tk create source git [name] [flags] * [tk create source](tk_create_source.md) - Create source commands -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_delete.md b/docs/cmd/tk_delete.md index 09d149fc..438aaf65 100644 --- a/docs/cmd/tk_delete.md +++ b/docs/cmd/tk_delete.md @@ -29,4 +29,4 @@ Delete commands * [tk delete kustomization](tk_delete_kustomization.md) - Delete kustomization * [tk delete source](tk_delete_source.md) - Delete sources commands -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_delete_kustomization.md b/docs/cmd/tk_delete_kustomization.md index be27ba4a..4cc8db77 100644 --- a/docs/cmd/tk_delete_kustomization.md +++ b/docs/cmd/tk_delete_kustomization.md @@ -31,4 +31,4 @@ tk delete kustomization [name] [flags] * [tk delete](tk_delete.md) - Delete commands -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_delete_source.md b/docs/cmd/tk_delete_source.md index 0aaac494..78184fac 100644 --- a/docs/cmd/tk_delete_source.md +++ b/docs/cmd/tk_delete_source.md @@ -28,4 +28,4 @@ Delete sources commands * [tk delete](tk_delete.md) - Delete commands * [tk delete source git](tk_delete_source_git.md) - Delete git source -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_delete_source_git.md b/docs/cmd/tk_delete_source_git.md index 45b5c40e..57b315fd 100644 --- a/docs/cmd/tk_delete_source_git.md +++ b/docs/cmd/tk_delete_source_git.md @@ -31,4 +31,4 @@ tk delete source git [name] [flags] * [tk delete source](tk_delete_source.md) - Delete sources commands -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_export.md b/docs/cmd/tk_export.md index b9675062..fe894afa 100644 --- a/docs/cmd/tk_export.md +++ b/docs/cmd/tk_export.md @@ -29,4 +29,4 @@ Export commands * [tk export kustomization](tk_export_kustomization.md) - Export kustomization in YAML format * [tk export source](tk_export_source.md) - Export source commands -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_export_kustomization.md b/docs/cmd/tk_export_kustomization.md index b1383578..f855247a 100644 --- a/docs/cmd/tk_export_kustomization.md +++ b/docs/cmd/tk_export_kustomization.md @@ -42,4 +42,4 @@ tk export kustomization [name] [flags] * [tk export](tk_export.md) - Export commands -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_export_source.md b/docs/cmd/tk_export_source.md index c9fed0ab..638dbea3 100644 --- a/docs/cmd/tk_export_source.md +++ b/docs/cmd/tk_export_source.md @@ -29,4 +29,4 @@ Export source commands * [tk export](tk_export.md) - Export commands * [tk export source git](tk_export_source_git.md) - Export git sources in YAML format -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_export_source_git.md b/docs/cmd/tk_export_source_git.md index 222aca9e..5d41f1bb 100644 --- a/docs/cmd/tk_export_source_git.md +++ b/docs/cmd/tk_export_source_git.md @@ -43,4 +43,4 @@ tk export source git [name] [flags] * [tk export source](tk_export_source.md) - Export source commands -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_get.md b/docs/cmd/tk_get.md index 4ed527cf..afcf9f31 100644 --- a/docs/cmd/tk_get.md +++ b/docs/cmd/tk_get.md @@ -28,4 +28,4 @@ Get commands * [tk get kustomizations](tk_get_kustomizations.md) - Get kustomizations status * [tk get sources](tk_get_sources.md) - Get sources commands -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_get_kustomizations.md b/docs/cmd/tk_get_kustomizations.md index e41c66e5..c7386fa0 100644 --- a/docs/cmd/tk_get_kustomizations.md +++ b/docs/cmd/tk_get_kustomizations.md @@ -31,4 +31,4 @@ tk get kustomizations [flags] * [tk get](tk_get.md) - Get commands -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_get_sources.md b/docs/cmd/tk_get_sources.md index c5a31c8c..b70d4224 100644 --- a/docs/cmd/tk_get_sources.md +++ b/docs/cmd/tk_get_sources.md @@ -27,4 +27,4 @@ Get sources commands * [tk get](tk_get.md) - Get commands * [tk get sources git](tk_get_sources_git.md) - Get git sources status -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_get_sources_git.md b/docs/cmd/tk_get_sources_git.md index 630ab8c6..6a26f86b 100644 --- a/docs/cmd/tk_get_sources_git.md +++ b/docs/cmd/tk_get_sources_git.md @@ -31,4 +31,4 @@ tk get sources git [flags] * [tk get sources](tk_get_sources.md) - Get sources commands -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_install.md b/docs/cmd/tk_install.md index 0da21720..7b077b31 100644 --- a/docs/cmd/tk_install.md +++ b/docs/cmd/tk_install.md @@ -49,4 +49,4 @@ tk install [flags] * [tk](tk.md) - Command line utility for assembling Kubernetes CD pipelines -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_resume.md b/docs/cmd/tk_resume.md index 5127bb75..81c23458 100644 --- a/docs/cmd/tk_resume.md +++ b/docs/cmd/tk_resume.md @@ -27,4 +27,4 @@ Resume commands * [tk](tk.md) - Command line utility for assembling Kubernetes CD pipelines * [tk resume kustomization](tk_resume_kustomization.md) - Resume kustomization -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_resume_kustomization.md b/docs/cmd/tk_resume_kustomization.md index a0de59ba..180bc8d1 100644 --- a/docs/cmd/tk_resume_kustomization.md +++ b/docs/cmd/tk_resume_kustomization.md @@ -30,4 +30,4 @@ tk resume kustomization [name] [flags] * [tk resume](tk_resume.md) - Resume commands -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_suspend.md b/docs/cmd/tk_suspend.md index b15f0e89..d99a4ddf 100644 --- a/docs/cmd/tk_suspend.md +++ b/docs/cmd/tk_suspend.md @@ -27,4 +27,4 @@ Suspend commands * [tk](tk.md) - Command line utility for assembling Kubernetes CD pipelines * [tk suspend kustomization](tk_suspend_kustomization.md) - Suspend kustomization -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_suspend_kustomization.md b/docs/cmd/tk_suspend_kustomization.md index 50844a5f..eaaf8c60 100644 --- a/docs/cmd/tk_suspend_kustomization.md +++ b/docs/cmd/tk_suspend_kustomization.md @@ -30,4 +30,4 @@ tk suspend kustomization [name] [flags] * [tk suspend](tk_suspend.md) - Suspend commands -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_sync.md b/docs/cmd/tk_sync.md index 25351c40..df50a411 100644 --- a/docs/cmd/tk_sync.md +++ b/docs/cmd/tk_sync.md @@ -28,4 +28,4 @@ Synchronize commands * [tk sync kustomization](tk_sync_kustomization.md) - Synchronize kustomization * [tk sync source](tk_sync_source.md) - Synchronize source commands -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_sync_kustomization.md b/docs/cmd/tk_sync_kustomization.md index 24b06fc8..47c2a65d 100644 --- a/docs/cmd/tk_sync_kustomization.md +++ b/docs/cmd/tk_sync_kustomization.md @@ -43,4 +43,4 @@ tk sync kustomization [name] [flags] * [tk sync](tk_sync.md) - Synchronize commands -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_sync_source.md b/docs/cmd/tk_sync_source.md index 35ba621f..7343d584 100644 --- a/docs/cmd/tk_sync_source.md +++ b/docs/cmd/tk_sync_source.md @@ -27,4 +27,4 @@ Synchronize source commands * [tk sync](tk_sync.md) - Synchronize commands * [tk sync source git](tk_sync_source_git.md) - Synchronize git source -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_sync_source_git.md b/docs/cmd/tk_sync_source_git.md index a08420c8..cf4cf226 100644 --- a/docs/cmd/tk_sync_source_git.md +++ b/docs/cmd/tk_sync_source_git.md @@ -39,4 +39,4 @@ tk sync source git [name] [flags] * [tk sync source](tk_sync_source.md) - Synchronize source commands -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/cmd/tk_uninstall.md b/docs/cmd/tk_uninstall.md index 467ddeb3..1e7e5f25 100644 --- a/docs/cmd/tk_uninstall.md +++ b/docs/cmd/tk_uninstall.md @@ -26,10 +26,11 @@ tk uninstall [flags] ### Options ``` - --crds removes all CRDs previously installed - --dry-run only print the object that would be deleted - -h, --help help for uninstall - -s, --silent delete components without asking for confirmation + --crds removes all CRDs previously installed + --dry-run only print the object that would be deleted + -h, --help help for uninstall + --kustomizations removes all kustomizations previously installed + -s, --silent delete components without asking for confirmation ``` ### Options inherited from parent commands @@ -46,4 +47,4 @@ tk uninstall [flags] * [tk](tk.md) - Command line utility for assembling Kubernetes CD pipelines -###### Auto generated by spf13/cobra on 9-Jun-2020 +###### Auto generated by spf13/cobra on 18-Jun-2020 diff --git a/docs/internal/release.md b/docs/internal/release.md index 987c2e3c..fde81cae 100644 --- a/docs/internal/release.md +++ b/docs/internal/release.md @@ -2,11 +2,6 @@ To release a new version the following steps should be followed: -1. Create a new branch from `master` i.e. `release-`. This - will function as your release preparation branch. -1. Change the `VERSION` value in `cmd/tk/main.go` to that of the - semver release you are going to make. Commit and push your changes. -1. Create a PR for your release branch and get it merged into `master`. -1. Create a `` tag for the merge commit in `master` and +1. Create a `` tag form `master` and push it to remote. 1. Confirm CI builds and releases the newly tagged version.