From 6390812cbb6b92ea630673eb6a639db30342b21a Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Sat, 20 Mar 2021 11:34:17 +0100 Subject: [PATCH 01/11] Factor bootstrap logic into `bootstrap` package This commit factors out the bootstrap logic into a new `bootstrap` package, while also moving to `go-git-providers` to handle things around Git providers (e.g. repository creation, deploy key upsertions). The `GitProviderBootstrapper` is a superset of the `PlainGitBootstrapper` that besides `Reconciler` also implements the `RepositoryReconciler`. The Git actions rely on an interface, making it easier to support other implementations than `go-git` at a later moment, to for example support bootstrapping to Git servers that only support the v2 protocol. Signed-off-by: Hidde Beydals --- go.mod | 2 + go.sum | 16 + internal/bootstrap/bootstrap.go | 184 ++++++++ internal/bootstrap/bootstrap_plain_git.go | 315 +++++++++++++ internal/bootstrap/bootstrap_provider.go | 530 ++++++++++++++++++++++ internal/bootstrap/git/git.go | 51 +++ internal/bootstrap/git/gogit/gogit.go | 198 ++++++++ internal/bootstrap/options.go | 100 ++++ internal/bootstrap/provider/factory.go | 58 +++ internal/bootstrap/provider/provider.go | 39 ++ 10 files changed, 1493 insertions(+) create mode 100644 internal/bootstrap/bootstrap.go create mode 100644 internal/bootstrap/bootstrap_plain_git.go create mode 100644 internal/bootstrap/bootstrap_provider.go create mode 100644 internal/bootstrap/git/git.go create mode 100644 internal/bootstrap/git/gogit/gogit.go create mode 100644 internal/bootstrap/options.go create mode 100644 internal/bootstrap/provider/factory.go create mode 100644 internal/bootstrap/provider/provider.go diff --git a/go.mod b/go.mod index fc5ab920..eab2d3b5 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.16 require ( github.com/Masterminds/semver/v3 v3.1.0 github.com/cyphar/filepath-securejoin v0.2.2 + github.com/fluxcd/go-git-providers v0.0.3 github.com/fluxcd/helm-controller/api v0.9.0 github.com/fluxcd/image-automation-controller/api v0.7.0 github.com/fluxcd/image-reflector-controller/api v0.7.1 @@ -17,6 +18,7 @@ require ( github.com/fluxcd/pkg/untar v0.0.5 github.com/fluxcd/pkg/version v0.0.1 github.com/fluxcd/source-controller/api v0.11.0 + github.com/go-git/go-git/v5 v5.1.0 github.com/google/go-containerregistry v0.2.0 github.com/manifoldco/promptui v0.7.0 github.com/olekukonko/tablewriter v0.0.4 diff --git a/go.sum b/go.sum index 507787ea..60fa4d6a 100644 --- a/go.sum +++ b/go.sum @@ -188,6 +188,8 @@ github.com/evanphx/json-patch/v5 v5.1.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2Vvl github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fluxcd/go-git-providers v0.0.3 h1:pquQvTpd1a4V1efPyZWuVPeIKrTgV8QRoDY0VGH+qiw= +github.com/fluxcd/go-git-providers v0.0.3/go.mod h1:iaXf3nEq8MB/LzxfbNcCl48sAtIReUU7jqjJ7CEnfFQ= github.com/fluxcd/helm-controller/api v0.9.0 h1:L60KmCblTQo3UimgCzVQGe330tC+b15CrLozvhPNmJU= github.com/fluxcd/helm-controller/api v0.9.0/go.mod h1:HIWSF3n1QU3hdqjQMFizFUZVr1uV+abmlGAEpB7vB9A= github.com/fluxcd/image-automation-controller/api v0.7.0 h1:mLaELYT52/FpZ93Mr+QMSK8UT0OBVQT4oA9kxO8NiEk= @@ -340,6 +342,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -384,6 +387,8 @@ github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-containerregistry v0.2.0 h1:cWFYx+kOkKdyOET0pcp7GMCmxj7da40StvluSuSXWCg= github.com/google/go-containerregistry v0.2.0/go.mod h1:Ts3Wioz1r5ayWx8sS6vLcWltWcM1aqFjd/eVrkFhrWM= +github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= +github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-github/v33 v33.0.0 h1:qAf9yP0qc54ufQxzwv+u9H0tiVOnPJxo0lI/JXqw3ZM= github.com/google/go-github/v33 v33.0.0/go.mod h1:GMdDnVZY/2TsWgp/lkYnpSAh6TrzhANBBwm6k6TTEXg= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= @@ -422,6 +427,8 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -491,6 +498,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a h1:FaWFmfWdAUKbSCtOU2QjDaorUexogfaMgbipgYATUMU= github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -511,6 +520,7 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/ktrysmt/go-bitbucket v0.6.2/go.mod h1:9u0v3hsd2rqCHRIpbir1oP7F58uo5dq19sBYvuMoyQ4= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= @@ -557,6 +567,7 @@ github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUb github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -591,6 +602,7 @@ github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M= github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= @@ -722,6 +734,7 @@ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV github.com/vdemeester/k8s-pkg-credentialprovider v1.18.1-0.20201019120933-f1d16962a4db/go.mod h1:grWy0bkr1XO6hqbaaCKaPXqkBVlMGHYG6PGykktwbJc= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= +github.com/xanzy/go-gitlab v0.33.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug= github.com/xanzy/go-gitlab v0.43.0 h1:rpOZQjxVJGW/ch+Jy4j7W4o7BB1mxkXJNVGuplZ7PUs= github.com/xanzy/go-gitlab v0.43.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= @@ -858,6 +871,7 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/oauth2 v0.0.0-20180227000427-d7d64896b5ff/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -874,6 +888,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180224232135-f6cff0780e54/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1021,6 +1036,7 @@ google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/ google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/appengine v1.0.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go new file mode 100644 index 00000000..a4025478 --- /dev/null +++ b/internal/bootstrap/bootstrap.go @@ -0,0 +1,184 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierr "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1" + "github.com/fluxcd/pkg/apis/meta" + + "github.com/fluxcd/flux2/pkg/manifestgen/install" + "github.com/fluxcd/flux2/pkg/manifestgen/sourcesecret" + "github.com/fluxcd/flux2/pkg/manifestgen/sync" +) + +var ( + ErrReconciledWithWarning = errors.New("reconciled with warning") +) + +type Reconciler interface { + // ReconcileComponents reconciles the components by generating the + // manifests with the provided values, committing them to Git and + // pushing to remote if there are any changes, and applying them + // to the cluster. + ReconcileComponents(ctx context.Context, manifestsBase string, options install.Options) error + + // ReconcileSourceSecret reconciles the source secret by generating + // a new secret with the provided values if the secret does not + // already exists on the cluster, or if any of the configuration + // options changed. + ReconcileSourceSecret(ctx context.Context, options sourcesecret.Options) error + + // ReconcileSyncConfig reconciles the sync configuration by generating + // the sync manifests with the provided values, committing them to Git + // and pushing to remote if there are any changes. + ReconcileSyncConfig(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error + + // ConfirmHealthy confirms that the components and extra components in + // install.Options are healthy. + ConfirmHealthy(ctx context.Context, options install.Options, timeout time.Duration) error +} + +type RepositoryReconciler interface { + // ReconcileRepository reconciles an external Git repository. + ReconcileRepository(ctx context.Context) error +} + +type PostGenerateSecretFunc func(ctx context.Context, secret corev1.Secret, options sourcesecret.Options) error + +func Run(ctx context.Context, reconciler Reconciler, manifestsBase string, + installOpts install.Options, secretOpts sourcesecret.Options, syncOpts sync.Options, + pollInterval, timeout time.Duration) error { + + var err error + if r, ok := reconciler.(RepositoryReconciler); ok { + if err = r.ReconcileRepository(ctx); err != nil && !errors.Is(err, ErrReconciledWithWarning) { + return err + } + } + + if err := reconciler.ReconcileComponents(ctx, manifestsBase, installOpts); err != nil { + return err + } + if err := reconciler.ReconcileSourceSecret(ctx, secretOpts); err != nil { + return err + } + if err := reconciler.ReconcileSyncConfig(ctx, syncOpts, pollInterval, timeout); err != nil { + return err + } + if err := reconciler.ConfirmHealthy(ctx, installOpts, timeout); err != nil { + return err + } + + return err +} + +func mustInstallManifests(ctx context.Context, kube client.Client, namespace string) bool { + namespacedName := types.NamespacedName{ + Namespace: namespace, + Name: namespace, + } + var k kustomizev1.Kustomization + if err := kube.Get(ctx, namespacedName, &k); err != nil { + return true + } + return k.Status.LastAppliedRevision == "" +} + +func secretExists(ctx context.Context, kube client.Client, objKey client.ObjectKey) (bool, error) { + if err := kube.Get(ctx, objKey, &corev1.Secret{}); err != nil { + if apierr.IsNotFound(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func reconcileSecret(ctx context.Context, kube client.Client, secret corev1.Secret) error { + objKey := client.ObjectKeyFromObject(&secret) + var existing corev1.Secret + err := kube.Get(ctx, objKey, &existing) + if err != nil { + if apierr.IsNotFound(err) { + return kube.Create(ctx, &secret) + } + return err + } + existing.StringData = secret.StringData + return kube.Update(ctx, &existing) +} + +func kustomizationPathDiffers(ctx context.Context, kube client.Client, objKey client.ObjectKey, path string) (string, error) { + var k kustomizev1.Kustomization + if err := kube.Get(ctx, objKey, &k); err != nil { + if apierr.IsNotFound(err) { + return "", nil + } + return "", err + } + normalizePath := func(p string) string { + return fmt.Sprintf("./%s", strings.TrimPrefix(p, "./")) + } + if normalizePath(path) == normalizePath(k.Spec.Path) { + return "", nil + } + return k.Spec.Path, nil +} + +func kustomizationReconciled(ctx context.Context, kube client.Client, objKey client.ObjectKey, + kustomization *kustomizev1.Kustomization, expectRevision string) func() (bool, error) { + + return func() (bool, error) { + if err := kube.Get(ctx, objKey, kustomization); err != nil { + return false, err + } + + // Confirm the state we are observing is for the current generation + if kustomization.Generation != kustomization.Status.ObservedGeneration { + return false, nil + } + + // Confirm the given revision has been attempted by the controller + if kustomization.Status.LastAttemptedRevision != expectRevision { + return false, nil + } + + // Confirm the resource is healthy + if c := apimeta.FindStatusCondition(kustomization.Status.Conditions, meta.ReadyCondition); c != nil { + switch c.Status { + case metav1.ConditionTrue: + return true, nil + case metav1.ConditionFalse: + return false, fmt.Errorf(c.Message) + } + } + return false, nil + } +} diff --git a/internal/bootstrap/bootstrap_plain_git.go b/internal/bootstrap/bootstrap_plain_git.go new file mode 100644 index 00000000..b293b78b --- /dev/null +++ b/internal/bootstrap/bootstrap_plain_git.go @@ -0,0 +1,315 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/cli-utils/pkg/object" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/kustomize/api/filesys" + "sigs.k8s.io/yaml" + + kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1" + + "github.com/fluxcd/flux2/internal/bootstrap/git" + + "github.com/fluxcd/flux2/internal/utils" + "github.com/fluxcd/flux2/pkg/log" + "github.com/fluxcd/flux2/pkg/manifestgen/install" + "github.com/fluxcd/flux2/pkg/manifestgen/kustomization" + "github.com/fluxcd/flux2/pkg/manifestgen/sourcesecret" + "github.com/fluxcd/flux2/pkg/manifestgen/sync" + "github.com/fluxcd/flux2/pkg/status" +) + +type PlainGitBootstrapper struct { + url string + branch string + + author git.Author + + kubeconfig string + kubecontext string + + postGenerateSecret []PostGenerateSecretFunc + + git git.Git + kube client.Client + logger log.Logger +} + +type GitOption interface { + applyGit(b *PlainGitBootstrapper) +} + +func WithRepositoryURL(url string) GitOption { + return repositoryURLOption(url) +} + +type repositoryURLOption string + +func (o repositoryURLOption) applyGit(b *PlainGitBootstrapper) { + b.url = string(o) +} + +func WithPostGenerateSecretFunc(callback PostGenerateSecretFunc) GitOption { + return postGenerateSecret(callback) +} + +type postGenerateSecret PostGenerateSecretFunc + +func (o postGenerateSecret) applyGit(b *PlainGitBootstrapper) { + b.postGenerateSecret = append(b.postGenerateSecret, PostGenerateSecretFunc(o)) +} + +func NewPlainGitProvider(git git.Git, kube client.Client, opts ...GitOption) (*PlainGitBootstrapper, error) { + b := &PlainGitBootstrapper{ + git: git, + kube: kube, + } + for _, opt := range opts { + opt.applyGit(b) + } + return b, nil +} + +func (b *PlainGitBootstrapper) ReconcileComponents(ctx context.Context, manifestsBase string, options install.Options) error { + // Clone if not already + if _, err := b.git.Status(); err != nil { + if err != git.ErrNoGitRepository { + return err + } + + b.logger.Actionf("cloning branch %q from Git repository %q", b.branch, b.url) + cloned, err := b.git.Clone(ctx, b.url, b.branch) + if err != nil { + return fmt.Errorf("failed to clone repository: %w", err) + } + if cloned { + b.logger.Successf("cloned repository") + } + } + + // Generate component manifests + b.logger.Actionf("generating component manifests") + manifests, err := install.Generate(options, manifestsBase) + if err != nil { + return fmt.Errorf("component manifest generation failed: %w", err) + } + b.logger.Successf("generated component manifests") + + // Write manifest to Git repository + if err = b.git.Write(manifests.Path, strings.NewReader(manifests.Content)); err != nil { + return fmt.Errorf("failed to write manifest %q: %w", manifests.Path, err) + } + + // Git commit generated + commit, err := b.git.Commit(git.Commit{ + Author: b.author, + Message: fmt.Sprintf("Add Flux %s component manifests", options.Version), + }) + if err != nil && err != git.ErrNoStagedFiles { + return fmt.Errorf("failed to commit sync manifests: %w", err) + } + if err == nil { + b.logger.Successf("committed sync manifests to %q (%q)", b.branch, commit) + b.logger.Actionf("pushing component manifests to %q", b.url) + if err = b.git.Push(ctx); err != nil { + return fmt.Errorf("failed to push manifests: %w", err) + } + } else { + b.logger.Successf("component manifests are up to date") + } + + // Conditionally install manifests + if mustInstallManifests(ctx, b.kube, options.Namespace) { + b.logger.Actionf("installing components in %q namespace", options.Namespace) + kubectlArgs := []string{"apply", "-f", filepath.Join(b.git.Path(), manifests.Path)} + if _, err = utils.ExecKubectlCommand(ctx, utils.ModeStderrOS, b.kubeconfig, b.kubecontext, kubectlArgs...); err != nil { + return err + } + b.logger.Successf("installed components") + } + + b.logger.Successf("reconciled components") + return nil +} + +func (b *PlainGitBootstrapper) ReconcileSourceSecret(ctx context.Context, options sourcesecret.Options) error { + // Determine if there is an existing secret + secretKey := client.ObjectKey{Name: options.Name, Namespace: options.Namespace} + b.logger.Actionf("determining if source secret %q exists", secretKey) + ok, err := secretExists(ctx, b.kube, secretKey) + if err != nil { + return fmt.Errorf("failed to determine if deploy key secret exists: %w", err) + } + + // Return early if exists and no custom config is passed + if ok && len(options.CAFilePath+options.PrivateKeyPath+options.Username+options.Password) == 0 { + b.logger.Successf("source secret up to date") + return nil + } + + // Generate source secret + b.logger.Actionf("generating source secret") + manifest, err := sourcesecret.Generate(options) + if err != nil { + return err + } + var secret corev1.Secret + if err := yaml.Unmarshal([]byte(manifest.Content), &secret); err != nil { + return fmt.Errorf("failed to unmarshal generated source secret manifest: %w", err) + } + + for _, callback := range b.postGenerateSecret { + if err = callback(ctx, secret, options); err != nil { + return err + } + } + + // Apply source secret + b.logger.Actionf("applying source secret %q", secretKey) + if err = reconcileSecret(ctx, b.kube, secret); err != nil { + return err + } + b.logger.Successf("reconciled source secret") + + return nil +} + +func (b *PlainGitBootstrapper) ReconcileSyncConfig(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error { + // Confirm that sync configuration does not overwrite existing config + if curPath, err := kustomizationPathDiffers(ctx, b.kube, client.ObjectKey{Name: options.Name, Namespace: options.Namespace}, options.TargetPath); err != nil { + return fmt.Errorf("failed to determine if sync configuration would overwrite existing Kustomization: %w", err) + } else if curPath != "" { + return fmt.Errorf("sync path configuration (%q) would overwrite path (%q) of existing Kustomization", options.TargetPath, curPath) + } + + // Clone if not already + if _, err := b.git.Status(); err != nil { + if err == git.ErrNoGitRepository { + b.logger.Actionf("cloning branch %q from Git repository %q", b.branch, b.url) + cloned, err := b.git.Clone(ctx, b.url, b.branch) + if err != nil { + return fmt.Errorf("failed to clone repository: %w", err) + } + if cloned { + b.logger.Successf("cloned repository", b.url) + } + } + return err + } + + // Generate sync manifests and write to Git repository + b.logger.Actionf("generating sync manifests") + manifests, err := sync.Generate(options) + if err != nil { + return fmt.Errorf("sync manifests generation failed: %w", err) + } + if err = b.git.Write(manifests.Path, strings.NewReader(manifests.Content)); err != nil { + return fmt.Errorf("failed to write manifest %q: %w", manifests.Path, err) + } + kusManifests, err := kustomization.Generate(kustomization.Options{ + FileSystem: filesys.MakeFsOnDisk(), + BaseDir: b.git.Path(), + TargetPath: filepath.Dir(manifests.Path), + }) + if err != nil { + return fmt.Errorf("kustomization.yaml generation failed: %w", err) + } + if err = b.git.Write(kusManifests.Path, strings.NewReader(kusManifests.Content)); err != nil { + return fmt.Errorf("failed to write manifest %q: %w", kusManifests.Path, err) + } + b.logger.Successf("generated sync manifests") + + // Git commit generated + commit, err := b.git.Commit(git.Commit{ + Author: b.author, + Message: fmt.Sprintf("Add Flux sync manifests"), + }) + if err != nil && err != git.ErrNoStagedFiles { + return fmt.Errorf("failed to commit sync manifests: %w", err) + } + if err == nil { + b.logger.Successf("committed sync manifests to %q (%q)", b.branch, commit) + b.logger.Actionf("pushing sync manifests to %q", b.url) + if err = b.git.Push(ctx); err != nil { + return fmt.Errorf("failed to push sync manifests: %w", err) + } + } else { + b.logger.Successf("sync manifests are up to date") + } + + // Apply to cluster + b.logger.Actionf("applying sync manifests") + kubectlArgs := []string{"apply", "-k", filepath.Join(b.git.Path(), filepath.Dir(kusManifests.Path))} + if _, err = utils.ExecKubectlCommand(ctx, utils.ModeStderrOS, b.kubeconfig, b.kubecontext, kubectlArgs...); err != nil { + return err + } + b.logger.Successf("applied sync manifests") + + // Wait till Kustomization is reconciled + var k kustomizev1.Kustomization + expectRevision := fmt.Sprintf("%s/%s", options.Branch, commit) + if err := wait.PollImmediate(pollInterval, timeout, kustomizationReconciled( + ctx, b.kube, client.ObjectKey{Name: options.Name, Namespace: options.Namespace}, &k, expectRevision), + ); err != nil { + return fmt.Errorf("failed waiting for Kustomization: %w", err) + } + + b.logger.Successf("reconciled sync configuration") + return nil +} + +func (b *PlainGitBootstrapper) ConfirmHealthy(ctx context.Context, install install.Options, timeout time.Duration) error { + cfg, err := utils.KubeConfig(b.kubeconfig, b.kubecontext) + if err != nil { + return err + } + + checker, err := status.NewStatusChecker(cfg, 2*time.Second, timeout, b.logger) + if err != nil { + return err + } + + var components = install.Components + components = append(components, install.ComponentsExtra...) + + var identifiers []object.ObjMetadata + for _, component := range components { + identifiers = append(identifiers, object.ObjMetadata{ + Namespace: install.Namespace, + Name: component, + GroupKind: schema.GroupKind{Group: "apps", Kind: "Deployment"}, + }) + } + + b.logger.Actionf("confirming components are healthy") + if err := checker.Assess(identifiers...); err != nil { + return err + } + b.logger.Successf("all components are healthy") + return nil +} diff --git a/internal/bootstrap/bootstrap_provider.go b/internal/bootstrap/bootstrap_provider.go new file mode 100644 index 00000000..a6a17d4e --- /dev/null +++ b/internal/bootstrap/bootstrap_provider.go @@ -0,0 +1,530 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "context" + "errors" + "fmt" + "net/url" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/fluxcd/go-git-providers/gitprovider" + + "github.com/fluxcd/flux2/internal/bootstrap/git" + "github.com/fluxcd/flux2/pkg/manifestgen/sourcesecret" + "github.com/fluxcd/flux2/pkg/manifestgen/sync" +) + +type GitProviderBootstrapper struct { + *PlainGitBootstrapper + + owner string + repository string + personal bool + + description string + defaultBranch string + visibility string + + teams map[string]string + + readWriteKey bool + + bootstrapTransportType string + syncTransportType string + + sshHostname string + + provider gitprovider.Client +} + +func NewGitProviderBootstrapper(git git.Git, provider gitprovider.Client, kube client.Client, opts ...GitProviderOption) (*GitProviderBootstrapper, error) { + b := &GitProviderBootstrapper{ + PlainGitBootstrapper: &PlainGitBootstrapper{ + git: git, + kube: kube, + }, + bootstrapTransportType: "https", + syncTransportType: "ssh", + provider: provider, + } + b.PlainGitBootstrapper.postGenerateSecret = append(b.PlainGitBootstrapper.postGenerateSecret, b.reconcileDeployKey) + for _, opt := range opts { + opt.applyGitProvider(b) + } + return b, nil +} + +type GitProviderOption interface { + applyGitProvider(b *GitProviderBootstrapper) +} + +func WithProviderRepository(owner, repository string, personal bool) GitProviderOption { + return providerRepositoryOption{ + owner: owner, + repository: repository, + personal: personal, + } +} + +type providerRepositoryOption struct { + owner string + repository string + personal bool +} + +func (o providerRepositoryOption) applyGitProvider(b *GitProviderBootstrapper) { + b.owner = o.owner + b.repository = o.repository + b.personal = o.personal +} + +func WithProviderRepositoryConfig(description, defaultBranch, visibility string) GitProviderOption { + return providerRepositoryConfigOption{ + description: description, + defaultBranch: defaultBranch, + visibility: visibility, + } +} + +type providerRepositoryConfigOption struct { + description string + defaultBranch string + visibility string +} + +func (o providerRepositoryConfigOption) applyGitProvider(b *GitProviderBootstrapper) { + b.description = o.description + b.defaultBranch = o.defaultBranch + b.visibility = o.visibility +} + +func WithProviderTeamPermissions(teams map[string]string) GitProviderOption { + return providerRepositoryTeamPermissionsOption(teams) +} + +type providerRepositoryTeamPermissionsOption map[string]string + +func (o providerRepositoryTeamPermissionsOption) applyGitProvider(b *GitProviderBootstrapper) { + b.teams = o +} + +func WithReadWriteKeyPermissions(b bool) GitProviderOption { + return withReadWriteKeyPermissionsOption(b) +} + +type withReadWriteKeyPermissionsOption bool + +func (o withReadWriteKeyPermissionsOption) applyGitProvider(b *GitProviderBootstrapper) { + b.readWriteKey = bool(o) +} + +func WithBootstrapTransportType(protocol string) GitProviderOption { + return bootstrapTransportTypeOption(protocol) +} + +type bootstrapTransportTypeOption string + +func (o bootstrapTransportTypeOption) applyGitProvider(b *GitProviderBootstrapper) { + b.bootstrapTransportType = string(o) +} + +func WithSyncTransportType(protocol string) GitProviderOption { + return syncProtocolOption(protocol) +} + +type syncProtocolOption string + +func (o syncProtocolOption) applyGitProvider(b *GitProviderBootstrapper) { + b.syncTransportType = string(o) +} + +func WithSSHHostname(hostname string) GitProviderOption { + return sshHostnameOption(hostname) +} + +type sshHostnameOption string + +func (o sshHostnameOption) applyGitProvider(b *GitProviderBootstrapper) { + b.sshHostname = string(o) +} + +func (b *GitProviderBootstrapper) ReconcileSyncConfig(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error { + repo, err := b.getRepository(ctx) + if err != nil { + return err + } + if b.url == "" { + bootstrapURL, err := b.getCloneURL(repo, gitprovider.TransportType(b.bootstrapTransportType)) + if err != nil { + return err + } + WithRepositoryURL(bootstrapURL).applyGit(b.PlainGitBootstrapper) + } + if options.URL == "" { + syncURL, err := b.getCloneURL(repo, gitprovider.TransportType(b.syncTransportType)) + if err != nil { + return err + } + options.URL = syncURL + } + return b.PlainGitBootstrapper.ReconcileSyncConfig(ctx, options, pollInterval, timeout) +} + +// ReconcileRepository reconciles an organization or user repository with the +// GitProviderBootstrapper configuration. On success, the URL in the embedded +// PlainGitBootstrapper is set to clone URL for the configured protocol. +// +// When part of the reconciliation fails with a warning without aborting, an +// ErrReconciledWithWarning error is returned. +func (b *GitProviderBootstrapper) ReconcileRepository(ctx context.Context) error { + var repo gitprovider.UserRepository + var err error + + if b.personal { + repo, err = b.reconcileUserRepository(ctx) + } else { + repo, err = b.reconcileOrgRepository(ctx) + } + if err != nil && !errors.Is(err, ErrReconciledWithWarning) { + return err + } + + cloneURL := repo.Repository().GetCloneURL(gitprovider.TransportType(b.bootstrapTransportType)) + WithRepositoryURL(cloneURL).applyGit(b.PlainGitBootstrapper) + + return err +} + +func (b *GitProviderBootstrapper) reconcileDeployKey(ctx context.Context, secret corev1.Secret, options sourcesecret.Options) error { + ppk, ok := secret.StringData[sourcesecret.PublicKeySecretKey] + if !ok { + return nil + } + b.logger.Successf("public key: %s", strings.TrimSpace(ppk)) + + repo, err := b.getRepository(ctx) + if err != nil { + return err + } + + name := deployKeyName(options.Namespace, b.branch, options.Name, options.TargetPath) + deployKeyInfo := newDeployKeyInfo(name, ppk, b.readWriteKey) + var changed bool + if _, changed, err = repo.DeployKeys().Reconcile(ctx, deployKeyInfo); err != nil { + return err + } + if changed { + b.logger.Successf("configured deploy key %q for %q", deployKeyInfo.Name, repo.Repository().String()) + } + return nil +} + +// reconcileOrgRepository reconciles a gitprovider.OrgRepository +// with the GitProviderBootstrapper values, including any +// gitprovider.TeamAccessInfo configurations. +// +// If one of the gitprovider.TeamAccessInfo does not reconcile +// successfully, the gitprovider.UserRepository and an +// ErrReconciledWithWarning error are returned. +func (b *GitProviderBootstrapper) reconcileOrgRepository(ctx context.Context) (gitprovider.UserRepository, error) { + b.logger.Actionf("connecting to %s", b.provider.SupportedDomain()) + + // Construct the repository and other configuration objects + // go-git-provider likes to work with + subOrgs, repoName := splitSubOrganizationsFromRepositoryName(b.repository) + orgRef := newOrganizationRef(b.provider.SupportedDomain(), b.owner, subOrgs) + repoRef := newOrgRepositoryRef(orgRef, repoName) + repoInfo := newRepositoryInfo(b.description, b.defaultBranch, b.visibility) + + // Reconcile the organization repository + repo, err := b.provider.OrgRepositories().Get(ctx, repoRef) + if err != nil { + if !errors.Is(err, gitprovider.ErrNotFound) { + return nil, fmt.Errorf("failed to get Git repository %q: %w", repoRef.String(), err) + } + // go-git-providers has at present some issues with the idempotency + // of the available Reconcile methods, and setting e.g. the default + // branch correctly. Resort to Create with AutoInit until this has + // been resolved. + repo, err = b.provider.OrgRepositories().Create(ctx, repoRef, repoInfo, &gitprovider.RepositoryCreateOptions{ + AutoInit: gitprovider.BoolVar(true), + }) + if err != nil { + return nil, fmt.Errorf("failed to create new Git repository %q: %w", repoRef.String(), err) + } + b.logger.Successf("repository %q created", repoRef.String()) + } + // Set default branch before calling Reconcile due to bug described + // above. + repoInfo.DefaultBranch = repo.Get().DefaultBranch + var changed bool + if repo, changed, err = b.provider.OrgRepositories().Reconcile(ctx, repoRef, repoInfo); err != nil { + return nil, fmt.Errorf("failed to reconcile Git repository %q: %w", repoRef.String(), err) + } + if changed { + b.logger.Successf("repository %q reconciled", repoRef.String()) + } + + // Build the team access config + teamAccessInfo, err := buildTeamAccessInfo(b.teams, gitprovider.RepositoryPermissionVar(gitprovider.RepositoryPermissionMaintain)) + if err != nil { + return nil, fmt.Errorf("failed to reconcile repository team access: %w", err) + } + + // Reconcile the team access config on best effort (that being: + // record the error as a warning, but continue with the + // reconciliation of the others) + var warning error + if count := len(teamAccessInfo); count > 0 { + b.logger.Actionf("reconciling repository permissions") + for _, i := range teamAccessInfo { + var err error + _, changed, err = repo.TeamAccess().Reconcile(ctx, i) + if err != nil { + warning = fmt.Errorf("failed to grant permissions to team: %w", ErrReconciledWithWarning) + b.logger.Failuref("failed to grant %q permissions to %q: %w", *i.Permission, i.Name, err) + } + if changed { + b.logger.Successf("granted %q permissions to %q", *i.Permission, i.Name) + } + } + b.logger.Successf("reconciled repository permissions") + } + return repo, warning +} + +// reconcileUserRepository reconciles a gitprovider.UserRepository +// with the GitProviderBootstrapper values. It returns the reconciled +// gitprovider.UserRepository, or an error. +func (b *GitProviderBootstrapper) reconcileUserRepository(ctx context.Context) (gitprovider.UserRepository, error) { + b.logger.Actionf("connecting to %s", b.provider.SupportedDomain()) + + // Construct the repository and other metadata objects + // go-git-provider likes to work with. + _, repoName := splitSubOrganizationsFromRepositoryName(b.repository) + userRef := newUserRef(b.provider.SupportedDomain(), b.owner) + repoRef := newUserRepositoryRef(userRef, repoName) + repoInfo := newRepositoryInfo(b.description, b.defaultBranch, b.visibility) + + repo, err := b.provider.UserRepositories().Get(ctx, repoRef) + if err != nil { + if !errors.Is(err, gitprovider.ErrNotFound) { + return nil, fmt.Errorf("failed to get Git repository %q: %w", repoRef.String(), err) + } + // go-git-providers has at present some issues with the idempotency + // of the available Reconcile methods, and setting e.g. the default + // branch correctly. Resort to Create with AutoInit until this has + // been resolved. + repo, err = b.provider.UserRepositories().Create(ctx, repoRef, repoInfo, &gitprovider.RepositoryCreateOptions{ + AutoInit: gitprovider.BoolVar(true), + }) + if err != nil { + return nil, fmt.Errorf("failed to create new Git repository %q: %w", repoRef.String(), err) + } + b.logger.Successf("repository %q created", repoRef.String()) + } + + // Set default branch before calling Reconcile due to bug described + // above. + repoInfo.DefaultBranch = repo.Get().DefaultBranch + var changed bool + if repo, changed, err = b.provider.UserRepositories().Reconcile(ctx, repoRef, repoInfo); err != nil { + return nil, err + } + if changed { + b.logger.Successf("repository %q reconciled", repoRef.String()) + } + return repo, nil +} + +// getRepository retrieves and returns the gitprovider.UserRepository +// for organization and user repositories using the +// GitProviderBootstrapper values. +// As gitprovider.OrgRepository is a superset of gitprovider.UserRepository, this +// type is returned. +func (b *GitProviderBootstrapper) getRepository(ctx context.Context) (gitprovider.UserRepository, error) { + subOrgs, repoName := splitSubOrganizationsFromRepositoryName(b.repository) + + if b.personal { + userRef := newUserRef(b.provider.SupportedDomain(), b.owner) + return b.provider.UserRepositories().Get(ctx, newUserRepositoryRef(userRef, repoName)) + } + + orgRef := newOrganizationRef(b.provider.SupportedDomain(), b.owner, subOrgs) + return b.provider.OrgRepositories().Get(ctx, newOrgRepositoryRef(orgRef, repoName)) +} + +// getCloneURL returns the Git clone URL for the given +// gitprovider.UserRepository. If the given transport type is +// gitprovider.TransportTypeSSH and a custom SSH hostname is configured, +// the hostname of the URL will be modified to this hostname. +func (b *GitProviderBootstrapper) getCloneURL(repository gitprovider.UserRepository, transport gitprovider.TransportType) (string, error) { + u := repository.Repository().GetCloneURL(transport) + var err error + if transport == gitprovider.TransportTypeSSH && b.sshHostname != "" { + if u, err = setHostname(u, b.sshHostname); err != nil { + err = fmt.Errorf("failed to set SSH hostname for URL %q: %w", u, err) + } + } + return u, err +} + +// splitSubOrganizationsFromRepositoryName removes any prefixed sub +// organizations from the given repository name by splitting the +// string into a slice by '/'. +// The last (or only) item of the slice result is assumed to be the +// repository name, other items (nested) sub organizations. +func splitSubOrganizationsFromRepositoryName(name string) ([]string, string) { + elements := strings.Split(name, "/") + i := len(elements) + switch i { + case 1: + return nil, name + default: + return elements[:i-1], elements[i-1] + } +} + +// buildTeamAccessInfo constructs a gitprovider.TeamAccessInfo slice +// from the given string map of team names to permissions. +// +// Providing a default gitprovider.RepositoryPermission is optional, +// and omitting it will make it default to the go-git-provider default. +// +// An error is returned if any of the given permissions is invalid. +func buildTeamAccessInfo(m map[string]string, defaultPermissions *gitprovider.RepositoryPermission) ([]gitprovider.TeamAccessInfo, error) { + var infos []gitprovider.TeamAccessInfo + if defaultPermissions != nil { + if err := gitprovider.ValidateRepositoryPermission(*defaultPermissions); err != nil { + return nil, fmt.Errorf("invalid default team permission %q", *defaultPermissions) + } + } + for n, p := range m { + permission := defaultPermissions + if p != "" { + p := gitprovider.RepositoryPermission(p) + if err := gitprovider.ValidateRepositoryPermission(p); err != nil { + return nil, fmt.Errorf("invalid permission %q for team %q", p, n) + } + permission = &p + } + i := gitprovider.TeamAccessInfo{ + Name: n, + Permission: permission, + } + infos = append(infos, i) + } + return infos, nil +} + +// newOrganizationRef constructs a gitprovider.OrganizationRef with the +// given values and returns the result. +func newOrganizationRef(domain, organization string, subOrganizations []string) gitprovider.OrganizationRef { + return gitprovider.OrganizationRef{ + Domain: domain, + Organization: organization, + SubOrganizations: subOrganizations, + } +} + +// newOrgRepositoryRef constructs a gitprovider.OrgRepositoryRef with +// the given values and returns the result. +func newOrgRepositoryRef(organizationRef gitprovider.OrganizationRef, name string) gitprovider.OrgRepositoryRef { + return gitprovider.OrgRepositoryRef{ + OrganizationRef: organizationRef, + RepositoryName: name, + } +} + +// newUserRef constructs a gitprovider.UserRef with the given values +// and returns the result. +func newUserRef(domain, login string) gitprovider.UserRef { + return gitprovider.UserRef{ + Domain: domain, + UserLogin: login, + } +} + +// newUserRepositoryRef constructs a gitprovider.UserRepositoryRef with +// the given values and returns the result. +func newUserRepositoryRef(userRef gitprovider.UserRef, name string) gitprovider.UserRepositoryRef { + return gitprovider.UserRepositoryRef{ + UserRef: userRef, + RepositoryName: name, + } +} + +// newRepositoryInfo constructs a gitprovider.RepositoryInfo with the +// given values and returns the result. +func newRepositoryInfo(description, defaultBranch, visibility string) gitprovider.RepositoryInfo { + var i gitprovider.RepositoryInfo + if description != "" { + i.Description = gitprovider.StringVar(description) + } + if defaultBranch != "" { + i.DefaultBranch = gitprovider.StringVar(defaultBranch) + } + if visibility != "" { + i.Visibility = gitprovider.RepositoryVisibilityVar(gitprovider.RepositoryVisibility(visibility)) + } + return i +} + +// newDeployKeyInfo constructs a gitprovider.DeployKeyInfo with the +// given values and returns the result. +func newDeployKeyInfo(name, publicKey string, readWrite bool) gitprovider.DeployKeyInfo { + keyInfo := gitprovider.DeployKeyInfo{ + Name: name, + Key: []byte(publicKey), + } + if readWrite { + keyInfo.ReadOnly = gitprovider.BoolVar(false) + } + return keyInfo +} + +func deployKeyName(namespace, secretName, branch, path string) string { + var name string + for _, v := range []string{namespace, secretName, branch, path} { + if v == "" { + continue + } + if name == "" { + name = v + } else { + name = name + "-" + v + } + } + return name +} + +// setHostname is a helper to replace the hostname of the given URL. +// TODO(hidde): support for this should be added in go-git-providers. +func setHostname(URL, hostname string) (string, error) { + u, err := url.Parse(URL) + if err != nil { + return URL, err + } + u.Host = hostname + return u.String(), nil +} diff --git a/internal/bootstrap/git/git.go b/internal/bootstrap/git/git.go new file mode 100644 index 00000000..cbafadf9 --- /dev/null +++ b/internal/bootstrap/git/git.go @@ -0,0 +1,51 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "context" + "errors" + "io" +) + +var ( + ErrNoGitRepository = errors.New("no git repository") + ErrNoStagedFiles = errors.New("no staged files") +) + +type Author struct { + Name string + Email string +} + +type Commit struct { + Author + Hash string + Message string +} + +// Git is an interface for basic Git operations on a single branch of a +// remote repository. +type Git interface { + Init(url, branch string) (bool, error) + Clone(ctx context.Context, url, branch string) (bool, error) + Write(path string, reader io.Reader) error + Commit(message Commit) (string, error) + Push(ctx context.Context) error + Status() (bool, error) + Path() string +} diff --git a/internal/bootstrap/git/gogit/gogit.go b/internal/bootstrap/git/gogit/gogit.go new file mode 100644 index 00000000..2060f2f9 --- /dev/null +++ b/internal/bootstrap/git/gogit/gogit.go @@ -0,0 +1,198 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gogit + +import ( + "context" + "fmt" + "io" + "strings" + "time" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/transport" + + "github.com/fluxcd/flux2/internal/bootstrap/git" +) + +type GoGit struct { + path string + auth transport.AuthMethod + repository *gogit.Repository +} + +func New(path string, auth transport.AuthMethod) *GoGit { + return &GoGit{ + path: path, + auth: auth, + } +} + +func (g *GoGit) Init(url, branch string) (bool, error) { + if g.repository != nil { + return false, nil + } + + r, err := gogit.PlainInit(g.path, false) + if err != nil { + return false, err + } + if _, err = r.CreateRemote(&config.RemoteConfig{ + Name: gogit.DefaultRemoteName, + URLs: []string{url}, + }); err != nil { + return false, err + } + branchRef := plumbing.NewBranchReferenceName(branch) + if err = r.CreateBranch(&config.Branch{ + Name: branch, + Remote: gogit.DefaultRemoteName, + Merge: branchRef, + }); err != nil { + return false, err + } + // PlainInit assumes the initial branch to always be master, we can + // overwrite this by setting the reference of the Storer to a new + // symbolic reference (as there are no commits yet) that points + // the HEAD to our new branch. + if err = r.Storer.SetReference(plumbing.NewSymbolicReference(plumbing.HEAD, branchRef)); err != nil { + return false, err + } + + g.repository = r + return true, nil +} + +func (g *GoGit) Clone(ctx context.Context, url, branch string) (bool, error) { + branchRef := plumbing.NewBranchReferenceName(branch) + r, err := gogit.PlainCloneContext(ctx, g.path, false, &gogit.CloneOptions{ + URL: url, + Auth: g.auth, + RemoteName: gogit.DefaultRemoteName, + ReferenceName: branchRef, + SingleBranch: true, + + NoCheckout: false, + Progress: nil, + Tags: gogit.NoTags, + }) + if err != nil { + if err == transport.ErrEmptyRemoteRepository || isRemoteBranchNotFoundErr(err, branchRef.String()) { + return g.Init(url, branch) + } + return false, err + } + + g.repository = r + return true, nil +} + +func (g *GoGit) Write(path string, reader io.Reader) error { + if g.repository == nil { + return git.ErrNoGitRepository + } + + wt, err := g.repository.Worktree() + if err != nil { + return err + } + + f, err := wt.Filesystem.Create(path) + if err != nil { + return err + } + defer f.Close() + + _, err = io.Copy(f, reader) + return err +} + +func (g *GoGit) Commit(message git.Commit) (string, error) { + if g.repository == nil { + return "", git.ErrNoGitRepository + } + + wt, err := g.repository.Worktree() + if err != nil { + return "", err + } + + status, err := wt.Status() + if err != nil { + return "", err + } + if status.IsClean() { + head, err := g.repository.Head() + if err != nil { + return "", err + } + return head.Hash().String(), git.ErrNoStagedFiles + } + if _, err = wt.Add("."); err != nil { + return "", err + } + + commit, err := wt.Commit(message.Message, &gogit.CommitOptions{ + Author: &object.Signature{ + Name: message.Name, + Email: message.Email, + When: time.Now(), + }, + }) + if err != nil { + return "", err + } + return commit.String(), nil +} + +func (g *GoGit) Push(ctx context.Context) error { + if g.repository == nil { + return git.ErrNoGitRepository + } + + return g.repository.PushContext(ctx, &gogit.PushOptions{ + RemoteName: gogit.DefaultRemoteName, + Auth: g.auth, + Progress: nil, + }) +} + +func (g *GoGit) Status() (bool, error) { + if g.repository == nil { + return false, git.ErrNoGitRepository + } + wt, err := g.repository.Worktree() + if err != nil { + return false, err + } + status, err := wt.Status() + if err != nil { + return false, err + } + return status.IsClean(), nil +} + +func (g *GoGit) Path() string { + return g.path +} + +func isRemoteBranchNotFoundErr(err error, ref string) bool { + return strings.Contains(err.Error(), fmt.Sprintf("couldn't find remote ref %q", ref)) +} diff --git a/internal/bootstrap/options.go b/internal/bootstrap/options.go new file mode 100644 index 00000000..9f737771 --- /dev/null +++ b/internal/bootstrap/options.go @@ -0,0 +1,100 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "github.com/fluxcd/flux2/internal/bootstrap/git" + "github.com/fluxcd/flux2/pkg/log" +) + +type Option interface { + GitOption + GitProviderOption +} + +func WithBranch(branch string) Option { + return branchOption(branch) +} + +type branchOption string + +func (o branchOption) applyGit(b *PlainGitBootstrapper) { + b.branch = string(o) +} + +func (o branchOption) applyGitProvider(b *GitProviderBootstrapper) { + o.applyGit(b.PlainGitBootstrapper) +} + +func WithAuthor(name, email string) Option { + return authorOption{ + Name: name, + Email: email, + } +} + +type authorOption git.Author + +func (o authorOption) applyGit(b *PlainGitBootstrapper) { + if o.Name != "" { + b.author.Name = o.Name + } + if o.Email != "" { + b.author.Email = o.Email + } +} + +func (o authorOption) applyGitProvider(b *GitProviderBootstrapper) { + o.applyGit(b.PlainGitBootstrapper) +} + +func WithKubeconfig(kubeconfig, kubecontext string) Option { + return kubeconfigOption{ + kubeconfig: kubeconfig, + kubecontext: kubecontext, + } +} + +type kubeconfigOption struct { + kubeconfig string + kubecontext string +} + +func (o kubeconfigOption) applyGit(b *PlainGitBootstrapper) { + b.kubeconfig = o.kubeconfig + b.kubecontext = o.kubecontext +} + +func (o kubeconfigOption) applyGitProvider(b *GitProviderBootstrapper) { + o.applyGit(b.PlainGitBootstrapper) +} + +func WithLogger(logger log.Logger) Option { + return loggerOption{logger} +} + +type loggerOption struct { + logger log.Logger +} + +func (o loggerOption) applyGit(b *PlainGitBootstrapper) { + b.logger = o.logger +} + +func (o loggerOption) applyGitProvider(b *GitProviderBootstrapper) { + b.logger = o.logger +} diff --git a/internal/bootstrap/provider/factory.go b/internal/bootstrap/provider/factory.go new file mode 100644 index 00000000..7f2d56bb --- /dev/null +++ b/internal/bootstrap/provider/factory.go @@ -0,0 +1,58 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provider + +import ( + "fmt" + + "github.com/fluxcd/go-git-providers/github" + "github.com/fluxcd/go-git-providers/gitlab" + "github.com/fluxcd/go-git-providers/gitprovider" +) + +// BuildGitProvider builds a gitprovider.Client for the provided +// Config. It returns an error if the Config.Provider +// is not supported, or if the construction of the client fails. +func BuildGitProvider(config Config) (gitprovider.Client, error) { + var client gitprovider.Client + var err error + switch config.Provider { + case GitProviderGitHub: + opts := []github.ClientOption{ + github.WithOAuth2Token(config.Token), + } + if config.Hostname != "" { + opts = append(opts, github.WithDomain(config.Hostname)) + } + if client, err = github.NewClient(opts...); err != nil { + return nil, err + } + case GitProviderGitLab: + opts := []gitlab.ClientOption{ + gitlab.WithConditionalRequests(true), + } + if config.Hostname != "" { + opts = append(opts, gitlab.WithDomain(config.Hostname)) + } + if client, err = gitlab.NewClient(config.Token, "", opts...); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unsupported Git provider '%s'", config.Provider) + } + return client, err +} diff --git a/internal/bootstrap/provider/provider.go b/internal/bootstrap/provider/provider.go new file mode 100644 index 00000000..1755e029 --- /dev/null +++ b/internal/bootstrap/provider/provider.go @@ -0,0 +1,39 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provider + +// GitProvider holds a Git provider definition. +type GitProvider string + +const ( + GitProviderGitHub GitProvider = "github" + GitProviderGitLab GitProvider = "gitlab" +) + +// Config defines the configuration for connecting to a GitProvider. +type Config struct { + // Provider defines the GitProvider. + Provider GitProvider + + // Hostname is the HTTP/S hostname of the Provider, + // e.g. github.example.com. + Hostname string + + // Token contains the token used to authenticate with the + // Provider. + Token string +} From 9055e753a9f5cdb0fa3e7fc440fc8bfa63277dab Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Fri, 5 Mar 2021 12:29:19 +0100 Subject: [PATCH 02/11] Add `app.kubernetes.io/part-of: flux` label To be used in a future version of Flux to better select Flux components in a namespace, as the namespace value for the `app.kubernetes.io/instance` could be used by non Flux related workloads. Signed-off-by: Hidde Beydals --- pkg/manifestgen/install/templates.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/manifestgen/install/templates.go b/pkg/manifestgen/install/templates.go index 1dcec5c9..8cdce319 100644 --- a/pkg/manifestgen/install/templates.go +++ b/pkg/manifestgen/install/templates.go @@ -154,6 +154,7 @@ metadata: labels: app.kubernetes.io/instance: {{.Namespace}} app.kubernetes.io/version: "{{.Version}}" + app.kubernetes.io/part-of: flux fieldSpecs: - path: metadata/labels create: true From 1d3a381389efd6a3573d313690463c0048dfb20d Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Tue, 16 Mar 2021 11:32:06 +0100 Subject: [PATCH 03/11] Test giving access to team in bootstrap e2e Signed-off-by: Hidde Beydals --- .github/workflows/bootstrap.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/bootstrap.yaml index 8c832a2b..84265a71 100644 --- a/.github/workflows/bootstrap.yaml +++ b/.github/workflows/bootstrap.yaml @@ -47,7 +47,8 @@ jobs: --owner=fluxcd-testing \ --repository=${{ steps.vars.outputs.test_repo_name }} \ --branch=main \ - --path=test-cluster + --path=test-cluster \ + --team=team-z env: GITHUB_TOKEN: ${{ secrets.GITPROVIDER_BOT_TOKEN }} - name: bootstrap no-op @@ -56,7 +57,8 @@ jobs: --owner=fluxcd-testing \ --repository=${{ steps.vars.outputs.test_repo_name }} \ --branch=main \ - --path=test-cluster + --path=test-cluster \ + --team=team-z env: GITHUB_TOKEN: ${{ secrets.GITPROVIDER_BOT_TOKEN }} - name: uninstall @@ -69,7 +71,8 @@ jobs: --owner=fluxcd-testing \ --repository=${{ steps.vars.outputs.test_repo_name }} \ --branch=main \ - --path=test-cluster + --path=test-cluster \ + --team=team-z env: GITHUB_TOKEN: ${{ secrets.GITPROVIDER_BOT_TOKEN }} - name: delete repository From f57ce147548768cfb1640fa53f5e7d6492d7ca19 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Sat, 20 Mar 2021 11:49:29 +0100 Subject: [PATCH 04/11] Implement `bootstrap` package in commands This includes making a lot of things configurable (e.g. SSH key algorithm, RSA bit size, etc.) that used to be static. Signed-off-by: Hidde Beydals --- cmd/flux/bootstrap.go | 267 +++++++------------------- cmd/flux/bootstrap_github.go | 303 +++++++++++------------------- cmd/flux/bootstrap_gitlab.go | 297 ++++++++++++----------------- docs/cmd/flux_bootstrap.md | 35 ++-- docs/cmd/flux_bootstrap_github.md | 45 +++-- docs/cmd/flux_bootstrap_gitlab.md | 46 +++-- go.mod | 2 +- go.sum | 4 - 8 files changed, 380 insertions(+), 619 deletions(-) diff --git a/cmd/flux/bootstrap.go b/cmd/flux/bootstrap.go index ba84a91d..46d7fe38 100644 --- a/cmd/flux/bootstrap.go +++ b/cmd/flux/bootstrap.go @@ -17,26 +17,15 @@ limitations under the License. package main import ( - "context" + "crypto/elliptic" "fmt" - "path/filepath" - "time" + "io/ioutil" "github.com/spf13/cobra" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" - "sigs.k8s.io/controller-runtime/pkg/client" - - kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1" - sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" "github.com/fluxcd/flux2/internal/flags" "github.com/fluxcd/flux2/internal/utils" - "github.com/fluxcd/flux2/pkg/manifestgen/install" - kus "github.com/fluxcd/flux2/pkg/manifestgen/kustomization" - "github.com/fluxcd/flux2/pkg/manifestgen/sync" - "github.com/fluxcd/flux2/pkg/status" + "github.com/fluxcd/flux2/pkg/manifestgen/sourcesecret" ) var bootstrapCmd = &cobra.Command{ @@ -46,21 +35,36 @@ var bootstrapCmd = &cobra.Command{ } type bootstrapFlags struct { - version string + version string + arch flags.Arch + logLevel flags.LogLevel + + branch string + manifestsPath string + defaultComponents []string extraComponents []string - registry string - imagePullSecret string - branch string + requiredComponents []string + + registry string + imagePullSecret string + + secretName string + tokenAuth bool + keyAlgorithm flags.PublicKeyAlgorithm + keyRSABits flags.RSAKeyBits + keyECDSACurve flags.ECDSACurve + sshHostname string + caFile string + privateKeyFile string + watchAllNamespaces bool networkPolicy bool - manifestsPath string - arch flags.Arch - logLevel flags.LogLevel - requiredComponents []string - tokenAuth bool clusterDomain string tolerationKeys []string + + authorName string + authorEmail string } const ( @@ -72,17 +76,21 @@ var bootstrapArgs = NewBootstrapFlags() func init() { bootstrapCmd.PersistentFlags().StringVarP(&bootstrapArgs.version, "version", "v", "", "toolkit version, when specified the manifests are downloaded from https://github.com/fluxcd/flux2/releases") + bootstrapCmd.PersistentFlags().StringSliceVar(&bootstrapArgs.defaultComponents, "components", rootArgs.defaults.Components, "list of components, accepts comma-separated values") bootstrapCmd.PersistentFlags().StringSliceVar(&bootstrapArgs.extraComponents, "components-extra", nil, "list of components in addition to those supplied or defaulted, accepts comma-separated values") + bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.registry, "registry", "ghcr.io/fluxcd", "container registry where the toolkit images are published") bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.imagePullSecret, "image-pull-secret", "", "Kubernetes secret name used for pulling the toolkit images from a private registry") - bootstrapCmd.PersistentFlags().Var(&bootstrapArgs.arch, "arch", bootstrapArgs.arch.Description()) + bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.branch, "branch", bootstrapDefaultBranch, "default branch (for GitHub this must match the default branch setting for the organization)") + bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.manifestsPath, "manifests", "", "path to the manifest directory") + bootstrapCmd.PersistentFlags().BoolVar(&bootstrapArgs.watchAllNamespaces, "watch-all-namespaces", true, "watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed") bootstrapCmd.PersistentFlags().BoolVar(&bootstrapArgs.networkPolicy, "network-policy", true, @@ -90,12 +98,25 @@ func init() { bootstrapCmd.PersistentFlags().BoolVar(&bootstrapArgs.tokenAuth, "token-auth", false, "when enabled, the personal access token will be used instead of SSH deploy key") bootstrapCmd.PersistentFlags().Var(&bootstrapArgs.logLevel, "log-level", bootstrapArgs.logLevel.Description()) - bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.manifestsPath, "manifests", "", "path to the manifest directory") bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.clusterDomain, "cluster-domain", rootArgs.defaults.ClusterDomain, "internal cluster domain") bootstrapCmd.PersistentFlags().StringSliceVar(&bootstrapArgs.tolerationKeys, "toleration-keys", nil, "list of toleration keys used to schedule the components pods onto nodes with matching taints") - bootstrapCmd.PersistentFlags().MarkHidden("manifests") + + bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.secretName, "secret-name", rootArgs.defaults.Namespace, "name of the secret the sync credentials can be found in or stored to") + bootstrapCmd.PersistentFlags().Var(&bootstrapArgs.keyAlgorithm, "ssh-key-algorithm", bootstrapArgs.keyAlgorithm.Description()) + bootstrapCmd.PersistentFlags().Var(&bootstrapArgs.keyRSABits, "ssh-rsa-bits", bootstrapArgs.keyRSABits.Description()) + bootstrapCmd.PersistentFlags().Var(&bootstrapArgs.keyECDSACurve, "ssh-ecdsa-curve", bootstrapArgs.keyECDSACurve.Description()) + bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.sshHostname, "ssh-hostname", "", "SSH hostname, to be used when the SSH host differs from the HTTPS one") + bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.caFile, "ca-file", "", "path to TLS CA file used for validating self-signed certificates") + bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.privateKeyFile, "private-key-file", "", "path to a private key file used for authenticating to the Git SSH server") + + bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.authorName, "author-name", "Flux", "author name for Git commits") + bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.authorEmail, "author-email", "", "author email for Git commits") + + bootstrapCmd.PersistentFlags().Var(&bootstrapArgs.arch, "arch", bootstrapArgs.arch.Description()) bootstrapCmd.PersistentFlags().MarkDeprecated("arch", "multi-arch container image is now available for AMD64, ARMv7 and ARM64") + bootstrapCmd.PersistentFlags().MarkHidden("manifests") + rootCmd.AddCommand(bootstrapCmd) } @@ -103,6 +124,9 @@ func NewBootstrapFlags() bootstrapFlags { return bootstrapFlags{ logLevel: flags.LogLevel(rootArgs.defaults.LogLevel), requiredComponents: []string{"source-controller", "kustomize-controller"}, + keyAlgorithm: flags.PublicKeyAlgorithm(sourcesecret.RSAPrivateKeyAlgorithm), + keyRSABits: 2048, + keyECDSACurve: flags.ECDSACurve{Curve: elliptic.P384()}, } } @@ -110,194 +134,39 @@ func bootstrapComponents() []string { return append(bootstrapArgs.defaultComponents, bootstrapArgs.extraComponents...) } -func bootstrapValidate() error { - components := bootstrapComponents() - for _, component := range bootstrapArgs.requiredComponents { - if !utils.ContainsItemString(components, component) { - return fmt.Errorf("component %s is required", component) - } - } - - if err := utils.ValidateComponents(components); err != nil { - return err - } - - return nil -} - -func generateInstallManifests(targetPath, namespace, tmpDir string, localManifests string) (string, error) { - if ver, err := getVersion(bootstrapArgs.version); err != nil { - return "", err - } else { - bootstrapArgs.version = ver - } - - manifestsBase := "" - if isEmbeddedVersion(bootstrapArgs.version) { - if err := writeEmbeddedManifests(tmpDir); err != nil { - return "", err - } - manifestsBase = tmpDir - } - - opts := install.Options{ - BaseURL: localManifests, - Version: bootstrapArgs.version, - Namespace: namespace, - Components: bootstrapComponents(), - Registry: bootstrapArgs.registry, - ImagePullSecret: bootstrapArgs.imagePullSecret, - WatchAllNamespaces: bootstrapArgs.watchAllNamespaces, - NetworkPolicy: bootstrapArgs.networkPolicy, - LogLevel: bootstrapArgs.logLevel.String(), - NotificationController: rootArgs.defaults.NotificationController, - ManifestFile: rootArgs.defaults.ManifestFile, - Timeout: rootArgs.timeout, - TargetPath: targetPath, - ClusterDomain: bootstrapArgs.clusterDomain, - TolerationKeys: bootstrapArgs.tolerationKeys, - } - - if localManifests == "" { - opts.BaseURL = rootArgs.defaults.BaseURL - } - - output, err := install.Generate(opts, manifestsBase) - if err != nil { - return "", fmt.Errorf("generating install manifests failed: %w", err) - } - - filePath, err := output.WriteFile(tmpDir) - if err != nil { - return "", fmt.Errorf("generating install manifests failed: %w", err) - } - return filePath, nil -} - -func applyInstallManifests(ctx context.Context, manifestPath string, components []string) error { - kubectlArgs := []string{"apply", "-f", manifestPath} - if _, err := utils.ExecKubectlCommand(ctx, utils.ModeOS, rootArgs.kubeconfig, rootArgs.kubecontext, kubectlArgs...); err != nil { - return fmt.Errorf("install failed: %w", err) - } - kubeConfig, err := utils.KubeConfig(rootArgs.kubeconfig, rootArgs.kubecontext) - if err != nil { - return fmt.Errorf("install failed: %w", err) - } - statusChecker, err := status.NewStatusChecker(kubeConfig, time.Second, rootArgs.timeout, logger) - if err != nil { - return fmt.Errorf("install failed: %w", err) - } - componentRefs, err := buildComponentObjectRefs(components...) - if err != nil { - return fmt.Errorf("install failed: %w", err) - } - logger.Waitingf("verifying installation") - if err := statusChecker.Assess(componentRefs...); err != nil { - return fmt.Errorf("install failed") - } - return nil -} - -func generateSyncManifests(url, branch, name, namespace, targetPath, tmpDir string, interval time.Duration) (string, error) { - opts := sync.Options{ - Name: name, - Namespace: namespace, - URL: url, - Branch: branch, - Interval: interval, - Secret: namespace, - TargetPath: targetPath, - ManifestFile: sync.MakeDefaultOptions().ManifestFile, +func buildEmbeddedManifestBase() (string, error) { + if !isEmbeddedVersion(bootstrapArgs.version) { + return "", nil } - - manifest, err := sync.Generate(opts) - if err != nil { - return "", fmt.Errorf("generating install manifests failed: %w", err) - } - - output, err := manifest.WriteFile(tmpDir) + tmpBaseDir, err := ioutil.TempDir("", "flux-manifests-") if err != nil { return "", err } - outputDir := filepath.Dir(output) - - kusOpts := kus.MakeDefaultOptions() - kusOpts.BaseDir = tmpDir - kusOpts.TargetPath = filepath.Dir(manifest.Path) - - kustomization, err := kus.Generate(kusOpts) - if err != nil { - return "", err - } - if _, err = kustomization.WriteFile(tmpDir); err != nil { + if err := writeEmbeddedManifests(tmpBaseDir); err != nil { return "", err } - - return outputDir, nil + return tmpBaseDir, nil } -func applySyncManifests(ctx context.Context, kubeClient client.Client, name, namespace, manifestsPath string) error { - kubectlArgs := []string{"apply", "-k", manifestsPath} - if _, err := utils.ExecKubectlCommand(ctx, utils.ModeStderrOS, rootArgs.kubeconfig, rootArgs.kubecontext, kubectlArgs...); err != nil { - return err - } - - logger.Waitingf("waiting for cluster sync") - - var gitRepository sourcev1.GitRepository - if err := wait.PollImmediate(rootArgs.pollInterval, rootArgs.timeout, - isGitRepositoryReady(ctx, kubeClient, types.NamespacedName{Name: name, Namespace: namespace}, &gitRepository)); err != nil { - return err +func bootstrapValidate() error { + components := bootstrapComponents() + for _, component := range bootstrapArgs.requiredComponents { + if !utils.ContainsItemString(components, component) { + return fmt.Errorf("component %s is required", component) + } } - var kustomization kustomizev1.Kustomization - if err := wait.PollImmediate(rootArgs.pollInterval, rootArgs.timeout, - isKustomizationReady(ctx, kubeClient, types.NamespacedName{Name: name, Namespace: namespace}, &kustomization)); err != nil { + if err := utils.ValidateComponents(components); err != nil { return err } return nil } -func shouldInstallManifests(ctx context.Context, kubeClient client.Client, namespace string) bool { - namespacedName := types.NamespacedName{ - Namespace: namespace, - Name: namespace, - } - var kustomization kustomizev1.Kustomization - if err := kubeClient.Get(ctx, namespacedName, &kustomization); err != nil { - return true +func mapTeamSlice(s []string, defaultPermission string) map[string]string { + m := make(map[string]string, len(s)) + for _, v := range s { + m[v] = defaultPermission } - - return kustomization.Status.LastAppliedRevision == "" -} - -func shouldCreateDeployKey(ctx context.Context, kubeClient client.Client, namespace string) bool { - namespacedName := types.NamespacedName{ - Namespace: namespace, - Name: namespace, - } - - var existing corev1.Secret - if err := kubeClient.Get(ctx, namespacedName, &existing); err != nil { - return true - } - return false -} - -func checkIfBootstrapPathDiffers(ctx context.Context, kubeClient client.Client, namespace string, path string) (string, bool) { - namespacedName := types.NamespacedName{ - Name: namespace, - Namespace: namespace, - } - var fluxSystemKustomization kustomizev1.Kustomization - err := kubeClient.Get(ctx, namespacedName, &fluxSystemKustomization) - if err != nil { - return "", false - } - if fluxSystemKustomization.Spec.Path == path { - return "", false - } - - return fluxSystemKustomization.Spec.Path, true + return m } diff --git a/cmd/flux/bootstrap_github.go b/cmd/flux/bootstrap_github.go index 7ccd4907..747d9829 100644 --- a/cmd/flux/bootstrap_github.go +++ b/cmd/flux/bootstrap_github.go @@ -20,20 +20,20 @@ import ( "context" "fmt" "io/ioutil" - "net/url" "os" - "path" - "path/filepath" "time" - "github.com/fluxcd/pkg/git" + "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/spf13/cobra" - corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/yaml" + "github.com/fluxcd/flux2/internal/bootstrap" + "github.com/fluxcd/flux2/internal/bootstrap/git/gogit" + "github.com/fluxcd/flux2/internal/bootstrap/provider" "github.com/fluxcd/flux2/internal/flags" "github.com/fluxcd/flux2/internal/utils" + "github.com/fluxcd/flux2/pkg/manifestgen/install" "github.com/fluxcd/flux2/pkg/manifestgen/sourcesecret" + "github.com/fluxcd/flux2/pkg/manifestgen/sync" ) var bootstrapGitHubCmd = &cobra.Command{ @@ -71,19 +71,21 @@ the bootstrap command will perform an upgrade if needed.`, } type githubFlags struct { - owner string - repository string - interval time.Duration - personal bool - private bool - hostname string - path flags.SafeRelativePath - teams []string - sshHostname string + owner string + repository string + interval time.Duration + personal bool + private bool + hostname string + path flags.SafeRelativePath + teams []string + readWriteKey bool } const ( ghDefaultPermission = "maintain" + ghDefaultDomain = "github.com" + ghTokenEnvVar = "GITHUB_TOKEN" ) var githubArgs githubFlags @@ -95,17 +97,17 @@ func init() { bootstrapGitHubCmd.Flags().BoolVar(&githubArgs.personal, "personal", false, "if true, the owner is assumed to be a GitHub user; otherwise an org") bootstrapGitHubCmd.Flags().BoolVar(&githubArgs.private, "private", true, "if true, the repository is assumed to be private") bootstrapGitHubCmd.Flags().DurationVar(&githubArgs.interval, "interval", time.Minute, "sync interval") - bootstrapGitHubCmd.Flags().StringVar(&githubArgs.hostname, "hostname", git.GitHubDefaultHostname, "GitHub hostname") - bootstrapGitHubCmd.Flags().StringVar(&githubArgs.sshHostname, "ssh-hostname", "", "GitHub SSH hostname, to be used when the SSH host differs from the HTTPS one") + bootstrapGitHubCmd.Flags().StringVar(&githubArgs.hostname, "hostname", ghDefaultDomain, "GitHub hostname") bootstrapGitHubCmd.Flags().Var(&githubArgs.path, "path", "path relative to the repository root, when specified the cluster sync will be scoped to this path") + bootstrapGitHubCmd.Flags().BoolVar(&githubArgs.readWriteKey, "read-write-key", false, "if true, the deploy key is configured with read/write permissions") bootstrapCmd.AddCommand(bootstrapGitHubCmd) } func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { - ghToken := os.Getenv(git.GitHubTokenName) + ghToken := os.Getenv(ghTokenEnvVar) if ghToken == "" { - return fmt.Errorf("%s environment variable not found", git.GitHubTokenName) + return fmt.Errorf("%s environment variable not found", ghTokenEnvVar) } if err := bootstrapValidate(); err != nil { @@ -120,205 +122,124 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { return err } - usedPath, bootstrapPathDiffers := checkIfBootstrapPathDiffers( - ctx, - kubeClient, - rootArgs.namespace, - filepath.ToSlash(githubArgs.path.String()), - ) - - if bootstrapPathDiffers { - return fmt.Errorf("cluster already bootstrapped to %v path", usedPath) + // Manifest base + if ver, err := getVersion(bootstrapArgs.version); err == nil { + bootstrapArgs.version = ver } - - repository, err := git.NewRepository( - githubArgs.repository, - githubArgs.owner, - githubArgs.hostname, - ghToken, - "flux", - githubArgs.owner+"@users.noreply.github.com", - ) + manifestsBase, err := buildEmbeddedManifestBase() if err != nil { return err } + defer os.RemoveAll(manifestsBase) - if githubArgs.sshHostname != "" { - repository.SSHHost = githubArgs.sshHostname - } - - provider := &git.GithubProvider{ - IsPrivate: githubArgs.private, - IsPersonal: githubArgs.personal, + // Build GitHub provider + providerCfg := provider.Config{ + Provider: provider.GitProviderGitHub, + Hostname: githubArgs.hostname, + Token: ghToken, } - - tmpDir, err := ioutil.TempDir("", rootArgs.namespace) + providerClient, err := provider.BuildGitProvider(providerCfg) if err != nil { return err } - defer os.RemoveAll(tmpDir) - // create GitHub repository if doesn't exists - logger.Actionf("connecting to %s", githubArgs.hostname) - changed, err := provider.CreateRepository(ctx, repository) + // Lazy go-git repository + tmpDir, err := ioutil.TempDir("", "flux-bootstrap-") if err != nil { - return err - } - if changed { - logger.Successf("repository created") - } - - withErrors := false - // add teams to org repository - if !githubArgs.personal { - for _, team := range githubArgs.teams { - if changed, err := provider.AddTeam(ctx, repository, team, ghDefaultPermission); err != nil { - logger.Failuref(err.Error()) - withErrors = true - } else if changed { - logger.Successf("%s team access granted", team) - } - } - } - - // clone repository and checkout the main branch - if err := repository.Checkout(ctx, bootstrapArgs.branch, tmpDir); err != nil { - return err - } - logger.Successf("repository cloned") - - // generate install manifests - logger.Generatef("generating manifests") - installManifest, err := generateInstallManifests( - githubArgs.path.String(), - rootArgs.namespace, - tmpDir, - bootstrapArgs.manifestsPath, - ) - if err != nil { - return err - } - - // stage install manifests - changed, err = repository.Commit( - ctx, - path.Join(githubArgs.path.String(), rootArgs.namespace), - fmt.Sprintf("Add flux %s components manifests", bootstrapArgs.version), - ) - if err != nil { - return err - } - - // push install manifests - if changed { - if err := repository.Push(ctx); err != nil { - return err - } - logger.Successf("components manifests pushed") - } else { - logger.Successf("components are up to date") + return fmt.Errorf("failed to create temporary working dir: %w", err) } - - // determine if repository synchronization is working - isInstall := shouldInstallManifests(ctx, kubeClient, rootArgs.namespace) - - if isInstall { - // apply install manifests - logger.Actionf("installing components in %s namespace", rootArgs.namespace) - if err := applyInstallManifests(ctx, installManifest, bootstrapComponents()); err != nil { - return err - } - logger.Successf("install completed") - } - - repoURL := repository.GetSSH() + defer os.RemoveAll(tmpDir) + gitClient := gogit.New(tmpDir, &http.BasicAuth{ + Username: githubArgs.owner, + Password: ghToken, + }) + + // Install manifest config + installOptions := install.Options{ + BaseURL: rootArgs.defaults.BaseURL, + Version: bootstrapArgs.version, + Namespace: rootArgs.namespace, + Components: bootstrapComponents(), + Registry: bootstrapArgs.registry, + ImagePullSecret: bootstrapArgs.imagePullSecret, + WatchAllNamespaces: bootstrapArgs.watchAllNamespaces, + NetworkPolicy: bootstrapArgs.networkPolicy, + LogLevel: bootstrapArgs.logLevel.String(), + NotificationController: rootArgs.defaults.NotificationController, + ManifestFile: rootArgs.defaults.ManifestFile, + Timeout: rootArgs.timeout, + TargetPath: githubArgs.path.String(), + ClusterDomain: bootstrapArgs.clusterDomain, + TolerationKeys: bootstrapArgs.tolerationKeys, + } + if customBaseURL := bootstrapArgs.manifestsPath; customBaseURL != "" { + installOptions.BaseURL = customBaseURL + } + + // Source generation and secret config secretOpts := sourcesecret.Options{ - Name: rootArgs.namespace, - Namespace: rootArgs.namespace, + Name: bootstrapArgs.secretName, + Namespace: rootArgs.namespace, + TargetPath: githubArgs.path.String(), + ManifestFile: sourcesecret.MakeDefaultOptions().ManifestFile, } if bootstrapArgs.tokenAuth { - // Setup HTTPS token auth - repoURL = repository.GetURL() secretOpts.Username = "git" secretOpts.Password = ghToken - } else if shouldCreateDeployKey(ctx, kubeClient, rootArgs.namespace) { - // Setup SSH auth - u, err := url.Parse(repoURL) - if err != nil { - return fmt.Errorf("git URL parse failed: %w", err) - } - secretOpts.SSHHostname = u.Host - secretOpts.PrivateKeyAlgorithm = sourcesecret.RSAPrivateKeyAlgorithm - secretOpts.RSAKeyBits = 2048 - } - secret, err := sourcesecret.Generate(secretOpts) - if err != nil { - return err - } - var s corev1.Secret - if err := yaml.Unmarshal([]byte(secret.Content), &s); err != nil { - return err - } - if len(s.StringData) > 0 { - logger.Actionf("configuring deploy key") - if err := upsertSecret(ctx, kubeClient, s); err != nil { - return err + if bootstrapArgs.caFile != "" { + secretOpts.CAFilePath = bootstrapArgs.caFile } + } else { + secretOpts.PrivateKeyAlgorithm = sourcesecret.PrivateKeyAlgorithm(bootstrapArgs.keyAlgorithm) + secretOpts.RSAKeyBits = int(bootstrapArgs.keyRSABits) + secretOpts.ECDSACurve = bootstrapArgs.keyECDSACurve.Curve + secretOpts.SSHHostname = githubArgs.hostname - if ppk, ok := s.StringData[sourcesecret.PublicKeySecretKey]; ok { - keyName := "flux" - if githubArgs.path != "" { - keyName = fmt.Sprintf("flux-%s", githubArgs.path) - } - - if changed, err := provider.AddDeployKey(ctx, repository, ppk, keyName); err != nil { - return err - } else if changed { - logger.Successf("deploy key configured") - } + if bootstrapArgs.sshHostname != "" { + secretOpts.SSHHostname = bootstrapArgs.sshHostname } } - // configure repository synchronization - logger.Actionf("generating sync manifests") - syncManifests, err := generateSyncManifests( - repoURL, - bootstrapArgs.branch, - rootArgs.namespace, - rootArgs.namespace, - filepath.ToSlash(githubArgs.path.String()), - tmpDir, - githubArgs.interval, - ) - if err != nil { - return err + // Sync manifest config + syncOpts := sync.Options{ + Interval: githubArgs.interval, + Name: rootArgs.namespace, + Namespace: rootArgs.namespace, + Branch: bootstrapArgs.branch, + Secret: bootstrapArgs.secretName, + TargetPath: githubArgs.path.String(), + ManifestFile: sync.MakeDefaultOptions().ManifestFile, + GitImplementation: sourceGitArgs.gitImplementation.String(), + } + + // Bootstrap config + bootstrapOpts := []bootstrap.GitProviderOption{ + bootstrap.WithProviderRepository(githubArgs.owner, githubArgs.repository, githubArgs.personal), + bootstrap.WithBranch(bootstrapArgs.branch), + bootstrap.WithBootstrapTransportType("https"), + bootstrap.WithAuthor(bootstrapArgs.authorName, bootstrapArgs.authorEmail), + bootstrap.WithProviderTeamPermissions(mapTeamSlice(githubArgs.teams, ghDefaultPermission)), + bootstrap.WithReadWriteKeyPermissions(githubArgs.readWriteKey), + bootstrap.WithKubeconfig(rootArgs.kubeconfig, rootArgs.kubecontext), + bootstrap.WithLogger(logger), + } + if bootstrapArgs.sshHostname != "" { + bootstrapOpts = append(bootstrapOpts, bootstrap.WithSSHHostname(bootstrapArgs.sshHostname)) } - - // commit and push manifests - if changed, err = repository.Commit( - ctx, - path.Join(githubArgs.path.String(), rootArgs.namespace), - fmt.Sprintf("Add flux %s sync manifests", bootstrapArgs.version), - ); err != nil { - return err - } else if changed { - if err := repository.Push(ctx); err != nil { - return err - } - logger.Successf("sync manifests pushed") + if bootstrapArgs.tokenAuth { + bootstrapOpts = append(bootstrapOpts, bootstrap.WithSyncTransportType("https")) } - - // apply manifests and waiting for sync - logger.Actionf("applying sync manifests") - if err := applySyncManifests(ctx, kubeClient, rootArgs.namespace, rootArgs.namespace, syncManifests); err != nil { - return err + if !githubArgs.private { + bootstrapOpts = append(bootstrapOpts, bootstrap.WithProviderRepositoryConfig("", "", "public")) } - if withErrors { - return fmt.Errorf("bootstrap completed with errors") + // Setup bootstrapper with constructed configs + b, err := bootstrap.NewGitProviderBootstrapper(gitClient, providerClient, kubeClient, bootstrapOpts...) + if err != nil { + return err } - logger.Successf("bootstrap finished") - return nil + // Run + return bootstrap.Run(ctx, b, manifestsBase, installOptions, secretOpts, syncOpts, rootArgs.pollInterval, rootArgs.timeout) } diff --git a/cmd/flux/bootstrap_gitlab.go b/cmd/flux/bootstrap_gitlab.go index f1f36554..48768c80 100644 --- a/cmd/flux/bootstrap_gitlab.go +++ b/cmd/flux/bootstrap_gitlab.go @@ -20,22 +20,21 @@ import ( "context" "fmt" "io/ioutil" - "net/url" "os" - "path" - "path/filepath" "regexp" "time" + "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/spf13/cobra" - corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/yaml" - - "github.com/fluxcd/pkg/git" + "github.com/fluxcd/flux2/internal/bootstrap" + "github.com/fluxcd/flux2/internal/bootstrap/git/gogit" + "github.com/fluxcd/flux2/internal/bootstrap/provider" "github.com/fluxcd/flux2/internal/flags" "github.com/fluxcd/flux2/internal/utils" + "github.com/fluxcd/flux2/pkg/manifestgen/install" "github.com/fluxcd/flux2/pkg/manifestgen/sourcesecret" + "github.com/fluxcd/flux2/pkg/manifestgen/sync" ) var bootstrapGitLabCmd = &cobra.Command{ @@ -70,18 +69,22 @@ the bootstrap command will perform an upgrade if needed.`, } const ( - gitlabProjectRegex = `\A[[:alnum:]\x{00A9}-\x{1f9ff}_][[:alnum:]\p{Pd}\x{00A9}-\x{1f9ff}_\.]*\z` + glDefaultPermission = "maintain" + glDefaultDomain = "gitlab.com" + glTokenEnvVar = "GITLAB_TOKEN" + gitlabProjectRegex = `\A[[:alnum:]\x{00A9}-\x{1f9ff}_][[:alnum:]\p{Pd}\x{00A9}-\x{1f9ff}_\.]*\z` ) type gitlabFlags struct { - owner string - repository string - interval time.Duration - personal bool - private bool - hostname string - sshHostname string - path flags.SafeRelativePath + owner string + repository string + interval time.Duration + personal bool + private bool + hostname string + path flags.SafeRelativePath + teams []string + readWriteKey bool } var gitlabArgs gitlabFlags @@ -89,29 +92,29 @@ var gitlabArgs gitlabFlags func init() { bootstrapGitLabCmd.Flags().StringVar(&gitlabArgs.owner, "owner", "", "GitLab user or group name") bootstrapGitLabCmd.Flags().StringVar(&gitlabArgs.repository, "repository", "", "GitLab repository name") + bootstrapGitLabCmd.Flags().StringArrayVar(&gitlabArgs.teams, "team", []string{}, "GitLab teams to be given maintainer access") bootstrapGitLabCmd.Flags().BoolVar(&gitlabArgs.personal, "personal", false, "if true, the owner is assumed to be a GitLab user; otherwise a group") bootstrapGitLabCmd.Flags().BoolVar(&gitlabArgs.private, "private", true, "if true, the repository is assumed to be private") bootstrapGitLabCmd.Flags().DurationVar(&gitlabArgs.interval, "interval", time.Minute, "sync interval") - bootstrapGitLabCmd.Flags().StringVar(&gitlabArgs.hostname, "hostname", git.GitLabDefaultHostname, "GitLab hostname") - bootstrapGitLabCmd.Flags().StringVar(&gitlabArgs.sshHostname, "ssh-hostname", "", "GitLab SSH hostname, to be used when the SSH host differs from the HTTPS one") + bootstrapGitLabCmd.Flags().StringVar(&gitlabArgs.hostname, "hostname", glDefaultDomain, "GitLab hostname") bootstrapGitLabCmd.Flags().Var(&gitlabArgs.path, "path", "path relative to the repository root, when specified the cluster sync will be scoped to this path") + bootstrapGitLabCmd.Flags().BoolVar(&gitlabArgs.readWriteKey, "read-write-key", false, "if true, the deploy key is configured with read/write permissions") bootstrapCmd.AddCommand(bootstrapGitLabCmd) } func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error { - glToken := os.Getenv(git.GitLabTokenName) + glToken := os.Getenv(glTokenEnvVar) if glToken == "" { - return fmt.Errorf("%s environment variable not found", git.GitLabTokenName) + return fmt.Errorf("%s environment variable not found", glTokenEnvVar) } - projectNameIsValid, err := regexp.MatchString(gitlabProjectRegex, gitlabArgs.repository) - if err != nil { + if projectNameIsValid, err := regexp.MatchString(gitlabProjectRegex, gitlabArgs.repository); err != nil || !projectNameIsValid { + if err == nil { + err = fmt.Errorf("%s is an invalid project name for gitlab.\nIt can contain only letters, digits, emojis, '_', '.', dash, space. It must start with letter, digit, emoji or '_'.", gitlabArgs.repository) + } return err } - if !projectNameIsValid { - return fmt.Errorf("%s is an invalid project name for gitlab.\nIt can contain only letters, digits, emojis, '_', '.', dash, space. It must start with letter, digit, emoji or '_'.", gitlabArgs.repository) - } if err := bootstrapValidate(); err != nil { return err @@ -125,183 +128,127 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error { return err } - usedPath, bootstrapPathDiffers := checkIfBootstrapPathDiffers(ctx, kubeClient, rootArgs.namespace, filepath.ToSlash(gitlabArgs.path.String())) - - if bootstrapPathDiffers { - return fmt.Errorf("cluster already bootstrapped to %v path", usedPath) + // Manifest base + if ver, err := getVersion(bootstrapArgs.version); err == nil { + bootstrapArgs.version = ver } - - repository, err := git.NewRepository( - gitlabArgs.repository, - gitlabArgs.owner, - gitlabArgs.hostname, - glToken, - "flux", - gitlabArgs.owner+"@users.noreply.gitlab.com", - ) + manifestsBase, err := buildEmbeddedManifestBase() if err != nil { return err } + defer os.RemoveAll(manifestsBase) - if gitlabArgs.sshHostname != "" { - repository.SSHHost = gitlabArgs.sshHostname + // Build GitLab provider + providerCfg := provider.Config{ + Provider: provider.GitProviderGitLab, + Hostname: gitlabArgs.hostname, + Token: glToken, } - - tmpDir, err := ioutil.TempDir("", rootArgs.namespace) + providerClient, err := provider.BuildGitProvider(providerCfg) if err != nil { return err } - defer os.RemoveAll(tmpDir) - - provider := &git.GitLabProvider{ - IsPrivate: gitlabArgs.private, - IsPersonal: gitlabArgs.personal, - } - // create GitLab project if doesn't exists - logger.Actionf("connecting to %s", gitlabArgs.hostname) - changed, err := provider.CreateRepository(ctx, repository) + // Lazy go-git repository + tmpDir, err := ioutil.TempDir("", "flux-bootstrap-") if err != nil { - return err - } - if changed { - logger.Successf("repository created") + return fmt.Errorf("failed to create temporary working dir: %w", err) } - - // clone repository and checkout the master branch - if err := repository.Checkout(ctx, bootstrapArgs.branch, tmpDir); err != nil { - return err - } - logger.Successf("repository cloned") - - // generate install manifests - logger.Generatef("generating manifests") - installManifest, err := generateInstallManifests( - gitlabArgs.path.String(), - rootArgs.namespace, - tmpDir, - bootstrapArgs.manifestsPath, - ) - if err != nil { - return err - } - - // stage install manifests - changed, err = repository.Commit( - ctx, - path.Join(gitlabArgs.path.String(), rootArgs.namespace), - fmt.Sprintf("Add flux %s components manifests", bootstrapArgs.version), - ) - if err != nil { - return err - } - - // push install manifests - if changed { - if err := repository.Push(ctx); err != nil { - return err - } - logger.Successf("components manifests pushed") - } else { - logger.Successf("components are up to date") - } - - // determine if repository synchronization is working - isInstall := shouldInstallManifests(ctx, kubeClient, rootArgs.namespace) - - if isInstall { - // apply install manifests - logger.Actionf("installing components in %s namespace", rootArgs.namespace) - if err := applyInstallManifests(ctx, installManifest, bootstrapComponents()); err != nil { - return err - } - logger.Successf("install completed") - } - - repoURL := repository.GetSSH() + defer os.RemoveAll(tmpDir) + gitClient := gogit.New(tmpDir, &http.BasicAuth{ + Username: gitlabArgs.owner, + Password: glToken, + }) + + // Install manifest config + installOptions := install.Options{ + BaseURL: rootArgs.defaults.BaseURL, + Version: bootstrapArgs.version, + Namespace: rootArgs.namespace, + Components: bootstrapComponents(), + Registry: bootstrapArgs.registry, + ImagePullSecret: bootstrapArgs.imagePullSecret, + WatchAllNamespaces: bootstrapArgs.watchAllNamespaces, + NetworkPolicy: bootstrapArgs.networkPolicy, + LogLevel: bootstrapArgs.logLevel.String(), + NotificationController: rootArgs.defaults.NotificationController, + ManifestFile: rootArgs.defaults.ManifestFile, + Timeout: rootArgs.timeout, + TargetPath: gitlabArgs.path.String(), + ClusterDomain: bootstrapArgs.clusterDomain, + TolerationKeys: bootstrapArgs.tolerationKeys, + } + if customBaseURL := bootstrapArgs.manifestsPath; customBaseURL != "" { + installOptions.BaseURL = customBaseURL + } + + // Source generation and secret config secretOpts := sourcesecret.Options{ - Name: rootArgs.namespace, - Namespace: rootArgs.namespace, + Name: bootstrapArgs.secretName, + Namespace: rootArgs.namespace, + TargetPath: gitlabArgs.path.String(), + ManifestFile: sourcesecret.MakeDefaultOptions().ManifestFile, } if bootstrapArgs.tokenAuth { - // Setup HTTPS token auth - repoURL = repository.GetURL() secretOpts.Username = "git" secretOpts.Password = glToken - } else if shouldCreateDeployKey(ctx, kubeClient, rootArgs.namespace) { - // Setup SSH auth - u, err := url.Parse(repoURL) - if err != nil { - return fmt.Errorf("git URL parse failed: %w", err) - } - secretOpts.SSHHostname = u.Host - secretOpts.PrivateKeyAlgorithm = sourcesecret.RSAPrivateKeyAlgorithm - secretOpts.RSAKeyBits = 2048 - } - secret, err := sourcesecret.Generate(secretOpts) - if err != nil { - return err - } - var s corev1.Secret - if err := yaml.Unmarshal([]byte(secret.Content), &s); err != nil { - return err - } - if len(s.StringData) > 0 { - logger.Actionf("configuring deploy key") - if err := upsertSecret(ctx, kubeClient, s); err != nil { - return err + if bootstrapArgs.caFile != "" { + secretOpts.CAFilePath = bootstrapArgs.caFile } + } else { + secretOpts.PrivateKeyAlgorithm = sourcesecret.PrivateKeyAlgorithm(bootstrapArgs.keyAlgorithm) + secretOpts.RSAKeyBits = int(bootstrapArgs.keyRSABits) + secretOpts.ECDSACurve = bootstrapArgs.keyECDSACurve.Curve + secretOpts.SSHHostname = githubArgs.hostname - if ppk, ok := s.StringData[sourcesecret.PublicKeySecretKey]; ok { - keyName := "flux" - if gitlabArgs.path != "" { - keyName = fmt.Sprintf("flux-%s", gitlabArgs.path) - } - - if changed, err := provider.AddDeployKey(ctx, repository, ppk, keyName); err != nil { - return err - } else if changed { - logger.Successf("deploy key configured") - } + if bootstrapArgs.privateKeyFile != "" { + secretOpts.PrivateKeyPath = bootstrapArgs.privateKeyFile + } + if bootstrapArgs.sshHostname != "" { + secretOpts.SSHHostname = bootstrapArgs.sshHostname } } - // configure repository synchronization - logger.Actionf("generating sync manifests") - syncManifests, err := generateSyncManifests( - repoURL, - bootstrapArgs.branch, - rootArgs.namespace, - rootArgs.namespace, - filepath.ToSlash(gitlabArgs.path.String()), - tmpDir, - gitlabArgs.interval, - ) - if err != nil { - return err + // Sync manifest config + syncOpts := sync.Options{ + Interval: gitlabArgs.interval, + Name: rootArgs.namespace, + Namespace: rootArgs.namespace, + Branch: bootstrapArgs.branch, + Secret: bootstrapArgs.secretName, + TargetPath: gitlabArgs.path.String(), + ManifestFile: sync.MakeDefaultOptions().ManifestFile, + GitImplementation: sourceGitArgs.gitImplementation.String(), + } + + // Bootstrap config + bootstrapOpts := []bootstrap.GitProviderOption{ + bootstrap.WithProviderRepository(gitlabArgs.owner, gitlabArgs.repository, gitlabArgs.personal), + bootstrap.WithBranch(bootstrapArgs.branch), + bootstrap.WithBootstrapTransportType("https"), + bootstrap.WithAuthor(bootstrapArgs.authorName, bootstrapArgs.authorEmail), + bootstrap.WithProviderTeamPermissions(mapTeamSlice(gitlabArgs.teams, glDefaultPermission)), + bootstrap.WithReadWriteKeyPermissions(gitlabArgs.readWriteKey), + bootstrap.WithKubeconfig(rootArgs.kubeconfig, rootArgs.kubecontext), + bootstrap.WithLogger(logger), + } + if bootstrapArgs.sshHostname != "" { + bootstrapOpts = append(bootstrapOpts, bootstrap.WithSSHHostname(bootstrapArgs.sshHostname)) } - - // commit and push manifests - if changed, err = repository.Commit( - ctx, - path.Join(gitlabArgs.path.String(), rootArgs.namespace), - fmt.Sprintf("Add flux %s sync manifests", bootstrapArgs.version), - ); err != nil { - return err - } else if changed { - if err := repository.Push(ctx); err != nil { - return err - } - logger.Successf("sync manifests pushed") + if bootstrapArgs.tokenAuth { + bootstrapOpts = append(bootstrapOpts, bootstrap.WithSyncTransportType("https")) + } + if !gitlabArgs.private { + bootstrapOpts = append(bootstrapOpts, bootstrap.WithProviderRepositoryConfig("", "", "public")) } - // apply manifests and waiting for sync - logger.Actionf("applying sync manifests") - if err := applySyncManifests(ctx, kubeClient, rootArgs.namespace, rootArgs.namespace, syncManifests); err != nil { + // Setup bootstrapper with constructed configs + b, err := bootstrap.NewGitProviderBootstrapper(gitClient, providerClient, kubeClient, bootstrapOpts...) + if err != nil { return err } - logger.Successf("bootstrap finished") - return nil + // Run + return bootstrap.Run(ctx, b, manifestsBase, installOptions, secretOpts, syncOpts, rootArgs.pollInterval, rootArgs.timeout) } diff --git a/docs/cmd/flux_bootstrap.md b/docs/cmd/flux_bootstrap.md index 26dbec70..78760620 100644 --- a/docs/cmd/flux_bootstrap.md +++ b/docs/cmd/flux_bootstrap.md @@ -12,19 +12,28 @@ The bootstrap sub-commands bootstrap the toolkit components on the targeted Git ### Options ``` - --branch string default branch (for GitHub this must match the default branch setting for the organization) (default "main") - --cluster-domain string internal cluster domain (default "cluster.local") - --components strings list of components, accepts comma-separated values (default [source-controller,kustomize-controller,helm-controller,notification-controller]) - --components-extra strings list of components in addition to those supplied or defaulted, accepts comma-separated values - -h, --help help for bootstrap - --image-pull-secret string Kubernetes secret name used for pulling the toolkit images from a private registry - --log-level logLevel log level, available options are: (debug, info, error) (default info) - --network-policy deny ingress access to the toolkit controllers from other namespaces using network policies (default true) - --registry string container registry where the toolkit images are published (default "ghcr.io/fluxcd") - --token-auth when enabled, the personal access token will be used instead of SSH deploy key - --toleration-keys strings list of toleration keys used to schedule the components pods onto nodes with matching taints - -v, --version string toolkit version, when specified the manifests are downloaded from https://github.com/fluxcd/flux2/releases - --watch-all-namespaces watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true) + --author-email string author email for Git commits + --author-name string author name for Git commits (default "Flux") + --branch string default branch (for GitHub this must match the default branch setting for the organization) (default "main") + --ca-file string path to TLS CA file used for validating self-signed certificates + --cluster-domain string internal cluster domain (default "cluster.local") + --components strings list of components, accepts comma-separated values (default [source-controller,kustomize-controller,helm-controller,notification-controller]) + --components-extra strings list of components in addition to those supplied or defaulted, accepts comma-separated values + -h, --help help for bootstrap + --image-pull-secret string Kubernetes secret name used for pulling the toolkit images from a private registry + --log-level logLevel log level, available options are: (debug, info, error) (default info) + --network-policy deny ingress access to the toolkit controllers from other namespaces using network policies (default true) + --private-key-file string path to a private key file used for authenticating to the Git SSH server + --registry string container registry where the toolkit images are published (default "ghcr.io/fluxcd") + --secret-name string name of the secret the sync credentials can be found in or stored to (default "flux-system") + --ssh-ecdsa-curve ecdsaCurve SSH ECDSA public key curve (p256, p384, p521) (default p384) + --ssh-hostname string SSH hostname, to be used when the SSH host differs from the HTTPS one + --ssh-key-algorithm publicKeyAlgorithm SSH public key algorithm (rsa, ecdsa, ed25519) (default rsa) + --ssh-rsa-bits rsaKeyBits SSH RSA public key bit size (multiplies of 8) (default 2048) + --token-auth when enabled, the personal access token will be used instead of SSH deploy key + --toleration-keys strings list of toleration keys used to schedule the components pods onto nodes with matching taints + -v, --version string toolkit version, when specified the manifests are downloaded from https://github.com/fluxcd/flux2/releases + --watch-all-namespaces watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true) ``` ### Options inherited from parent commands diff --git a/docs/cmd/flux_bootstrap_github.md b/docs/cmd/flux_bootstrap_github.md index 80ed707e..cde0605d 100644 --- a/docs/cmd/flux_bootstrap_github.md +++ b/docs/cmd/flux_bootstrap_github.md @@ -55,31 +55,40 @@ flux bootstrap github [flags] --path safeRelativePath path relative to the repository root, when specified the cluster sync will be scoped to this path --personal if true, the owner is assumed to be a GitHub user; otherwise an org --private if true, the repository is assumed to be private (default true) + --read-write-key if true, the deploy key is configured with read/write permissions --repository string GitHub repository name - --ssh-hostname string GitHub SSH hostname, to be used when the SSH host differs from the HTTPS one --team stringArray GitHub team to be given maintainer access ``` ### Options inherited from parent commands ``` - --branch string default branch (for GitHub this must match the default branch setting for the organization) (default "main") - --cluster-domain string internal cluster domain (default "cluster.local") - --components strings list of components, accepts comma-separated values (default [source-controller,kustomize-controller,helm-controller,notification-controller]) - --components-extra strings list of components in addition to those supplied or defaulted, accepts comma-separated values - --context string kubernetes context to use - --image-pull-secret string Kubernetes secret name used for pulling the toolkit images from a private registry - --kubeconfig string absolute path to the kubeconfig file - --log-level logLevel log level, available options are: (debug, info, error) (default info) - -n, --namespace string the namespace scope for this operation (default "flux-system") - --network-policy deny ingress access to the toolkit controllers from other namespaces using network policies (default true) - --registry string container registry where the toolkit images are published (default "ghcr.io/fluxcd") - --timeout duration timeout for this operation (default 5m0s) - --token-auth when enabled, the personal access token will be used instead of SSH deploy key - --toleration-keys strings list of toleration keys used to schedule the components pods onto nodes with matching taints - --verbose print generated objects - -v, --version string toolkit version, when specified the manifests are downloaded from https://github.com/fluxcd/flux2/releases - --watch-all-namespaces watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true) + --author-email string author email for Git commits + --author-name string author name for Git commits (default "Flux") + --branch string default branch (for GitHub this must match the default branch setting for the organization) (default "main") + --ca-file string path to TLS CA file used for validating self-signed certificates + --cluster-domain string internal cluster domain (default "cluster.local") + --components strings list of components, accepts comma-separated values (default [source-controller,kustomize-controller,helm-controller,notification-controller]) + --components-extra strings list of components in addition to those supplied or defaulted, accepts comma-separated values + --context string kubernetes context to use + --image-pull-secret string Kubernetes secret name used for pulling the toolkit images from a private registry + --kubeconfig string absolute path to the kubeconfig file + --log-level logLevel log level, available options are: (debug, info, error) (default info) + -n, --namespace string the namespace scope for this operation (default "flux-system") + --network-policy deny ingress access to the toolkit controllers from other namespaces using network policies (default true) + --private-key-file string path to a private key file used for authenticating to the Git SSH server + --registry string container registry where the toolkit images are published (default "ghcr.io/fluxcd") + --secret-name string name of the secret the sync credentials can be found in or stored to (default "flux-system") + --ssh-ecdsa-curve ecdsaCurve SSH ECDSA public key curve (p256, p384, p521) (default p384) + --ssh-hostname string SSH hostname, to be used when the SSH host differs from the HTTPS one + --ssh-key-algorithm publicKeyAlgorithm SSH public key algorithm (rsa, ecdsa, ed25519) (default rsa) + --ssh-rsa-bits rsaKeyBits SSH RSA public key bit size (multiplies of 8) (default 2048) + --timeout duration timeout for this operation (default 5m0s) + --token-auth when enabled, the personal access token will be used instead of SSH deploy key + --toleration-keys strings list of toleration keys used to schedule the components pods onto nodes with matching taints + --verbose print generated objects + -v, --version string toolkit version, when specified the manifests are downloaded from https://github.com/fluxcd/flux2/releases + --watch-all-namespaces watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true) ``` ### SEE ALSO diff --git a/docs/cmd/flux_bootstrap_gitlab.md b/docs/cmd/flux_bootstrap_gitlab.md index 16acf645..3d0b9c43 100644 --- a/docs/cmd/flux_bootstrap_gitlab.md +++ b/docs/cmd/flux_bootstrap_gitlab.md @@ -52,30 +52,40 @@ flux bootstrap gitlab [flags] --path safeRelativePath path relative to the repository root, when specified the cluster sync will be scoped to this path --personal if true, the owner is assumed to be a GitLab user; otherwise a group --private if true, the repository is assumed to be private (default true) + --read-write-key if true, the deploy key is configured with read/write permissions --repository string GitLab repository name - --ssh-hostname string GitLab SSH hostname, to be used when the SSH host differs from the HTTPS one + --team stringArray GitLab teams to be given maintainer access ``` ### Options inherited from parent commands ``` - --branch string default branch (for GitHub this must match the default branch setting for the organization) (default "main") - --cluster-domain string internal cluster domain (default "cluster.local") - --components strings list of components, accepts comma-separated values (default [source-controller,kustomize-controller,helm-controller,notification-controller]) - --components-extra strings list of components in addition to those supplied or defaulted, accepts comma-separated values - --context string kubernetes context to use - --image-pull-secret string Kubernetes secret name used for pulling the toolkit images from a private registry - --kubeconfig string absolute path to the kubeconfig file - --log-level logLevel log level, available options are: (debug, info, error) (default info) - -n, --namespace string the namespace scope for this operation (default "flux-system") - --network-policy deny ingress access to the toolkit controllers from other namespaces using network policies (default true) - --registry string container registry where the toolkit images are published (default "ghcr.io/fluxcd") - --timeout duration timeout for this operation (default 5m0s) - --token-auth when enabled, the personal access token will be used instead of SSH deploy key - --toleration-keys strings list of toleration keys used to schedule the components pods onto nodes with matching taints - --verbose print generated objects - -v, --version string toolkit version, when specified the manifests are downloaded from https://github.com/fluxcd/flux2/releases - --watch-all-namespaces watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true) + --author-email string author email for Git commits + --author-name string author name for Git commits (default "Flux") + --branch string default branch (for GitHub this must match the default branch setting for the organization) (default "main") + --ca-file string path to TLS CA file used for validating self-signed certificates + --cluster-domain string internal cluster domain (default "cluster.local") + --components strings list of components, accepts comma-separated values (default [source-controller,kustomize-controller,helm-controller,notification-controller]) + --components-extra strings list of components in addition to those supplied or defaulted, accepts comma-separated values + --context string kubernetes context to use + --image-pull-secret string Kubernetes secret name used for pulling the toolkit images from a private registry + --kubeconfig string absolute path to the kubeconfig file + --log-level logLevel log level, available options are: (debug, info, error) (default info) + -n, --namespace string the namespace scope for this operation (default "flux-system") + --network-policy deny ingress access to the toolkit controllers from other namespaces using network policies (default true) + --private-key-file string path to a private key file used for authenticating to the Git SSH server + --registry string container registry where the toolkit images are published (default "ghcr.io/fluxcd") + --secret-name string name of the secret the sync credentials can be found in or stored to (default "flux-system") + --ssh-ecdsa-curve ecdsaCurve SSH ECDSA public key curve (p256, p384, p521) (default p384) + --ssh-hostname string SSH hostname, to be used when the SSH host differs from the HTTPS one + --ssh-key-algorithm publicKeyAlgorithm SSH public key algorithm (rsa, ecdsa, ed25519) (default rsa) + --ssh-rsa-bits rsaKeyBits SSH RSA public key bit size (multiplies of 8) (default 2048) + --timeout duration timeout for this operation (default 5m0s) + --token-auth when enabled, the personal access token will be used instead of SSH deploy key + --toleration-keys strings list of toleration keys used to schedule the components pods onto nodes with matching taints + --verbose print generated objects + -v, --version string toolkit version, when specified the manifests are downloaded from https://github.com/fluxcd/flux2/releases + --watch-all-namespaces watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true) ``` ### SEE ALSO diff --git a/go.mod b/go.mod index eab2d3b5..ffdbc1a7 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/fluxcd/kustomize-controller/api v0.10.0 github.com/fluxcd/notification-controller/api v0.11.0 github.com/fluxcd/pkg/apis/meta v0.8.0 - github.com/fluxcd/pkg/git v0.3.0 github.com/fluxcd/pkg/runtime v0.10.1 github.com/fluxcd/pkg/ssh v0.0.5 github.com/fluxcd/pkg/untar v0.0.5 @@ -24,6 +23,7 @@ require ( github.com/olekukonko/tablewriter v0.0.4 github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 + github.com/xanzy/go-gitlab v0.43.0 // indirect golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 k8s.io/api v0.20.2 k8s.io/apiextensions-apiserver v0.20.2 diff --git a/go.sum b/go.sum index 60fa4d6a..aa8adafd 100644 --- a/go.sum +++ b/go.sum @@ -204,8 +204,6 @@ github.com/fluxcd/pkg/apis/kustomize v0.0.1 h1:TkA80R0GopRY27VJqzKyS6ifiKIAfwBd7 github.com/fluxcd/pkg/apis/kustomize v0.0.1/go.mod h1:JAFPfnRmcrAoG1gNiA8kmEXsnOBuDyZ/F5X4DAQcVV0= github.com/fluxcd/pkg/apis/meta v0.8.0 h1:wqWpUsxhKHB1ZztcvOz+vnyhdKW9cWmjFp8Vci/XOdk= github.com/fluxcd/pkg/apis/meta v0.8.0/go.mod h1:yHuY8kyGHYz22I0jQzqMMGCcHViuzC/WPdo9Gisk8Po= -github.com/fluxcd/pkg/git v0.3.0 h1:nrKZWZ/ymDevud3Wf1LEieO/QcNPnqz1/MrkZBFcg9o= -github.com/fluxcd/pkg/git v0.3.0/go.mod h1:ZwG0iLOqNSyNw6lsPIAO+v6+BqqCXyV+r1Oq6Lm+slg= github.com/fluxcd/pkg/runtime v0.10.1 h1:NV0pe6lFzodKBIz0dT3xkoR0wJnTCicXwM/v/d5T0+Y= github.com/fluxcd/pkg/runtime v0.10.1/go.mod h1:JD0eZIn5xkTeHHQUWXSqJPIh/ecO0d0qrUKbSVHnpnw= github.com/fluxcd/pkg/ssh v0.0.5 h1:rnbFZ7voy2JBlUfMbfyqArX2FYaLNpDhccGFC3qW83A= @@ -389,8 +387,6 @@ github.com/google/go-containerregistry v0.2.0 h1:cWFYx+kOkKdyOET0pcp7GMCmxj7da40 github.com/google/go-containerregistry v0.2.0/go.mod h1:Ts3Wioz1r5ayWx8sS6vLcWltWcM1aqFjd/eVrkFhrWM= github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= -github.com/google/go-github/v33 v33.0.0 h1:qAf9yP0qc54ufQxzwv+u9H0tiVOnPJxo0lI/JXqw3ZM= -github.com/google/go-github/v33 v33.0.0/go.mod h1:GMdDnVZY/2TsWgp/lkYnpSAh6TrzhANBBwm6k6TTEXg= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= From 22648cae3ba730d484dbc6ba4cb936b2020348d0 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Sat, 20 Mar 2021 11:51:24 +0100 Subject: [PATCH 05/11] Add command to bootstrap to generic Git server This command makes it possible to bootstrap to a generic Git server using the local SSH agent, or a given password or private key file. If a private key is generated, the user is prompted to give the generated key access to the repository. Signed-off-by: Hidde Beydals --- cmd/flux/bootstrap_git.go | 240 +++++++++++++++++++++++++++++++++ docs/cmd/flux_bootstrap.md | 1 + docs/cmd/flux_bootstrap_git.md | 78 +++++++++++ 3 files changed, 319 insertions(+) create mode 100644 cmd/flux/bootstrap_git.go create mode 100644 docs/cmd/flux_bootstrap_git.md diff --git a/cmd/flux/bootstrap_git.go b/cmd/flux/bootstrap_git.go new file mode 100644 index 00000000..cd01bc08 --- /dev/null +++ b/cmd/flux/bootstrap_git.go @@ -0,0 +1,240 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + "io/ioutil" + "net/url" + "os" + "strings" + "time" + + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/plumbing/transport/ssh" + "github.com/manifoldco/promptui" + "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + + "github.com/fluxcd/flux2/internal/bootstrap" + "github.com/fluxcd/flux2/internal/bootstrap/git/gogit" + "github.com/fluxcd/flux2/internal/flags" + "github.com/fluxcd/flux2/internal/utils" + "github.com/fluxcd/flux2/pkg/manifestgen/install" + "github.com/fluxcd/flux2/pkg/manifestgen/sourcesecret" + "github.com/fluxcd/flux2/pkg/manifestgen/sync" +) + +var bootstrapGitCmd = &cobra.Command{ + Use: "git", + Short: "Bootstrap toolkit components in a Git repository", + Long: `The bootstrap git command commits the toolkit components manifests to the +branch of a Git repository. It then configures the target cluster to synchronize with +the repository. If the toolkit components are present on the cluster, the bootstrap +command will perform an upgrade if needed.`, + Example: ` # Run bootstrap for a Git repository and authenticate with your SSH agent + flux bootstrap git --url=ssh://git@example.com/repository.git + + # Run bootstrap for a Git repository and authenticate using a password + flux bootstrap git --url=https://example.com/repository.git --password= + + # Run bootstrap for a Git repository with a passwordless private key + flux bootstrap git --url=ssh://git@example.com/repository.git --private-key-file= +`, + RunE: bootstrapGitCmdRun, +} + +type gitFlags struct { + url string + interval time.Duration + path flags.SafeRelativePath + username string + password string +} + +var gitArgs gitFlags + +func init() { + bootstrapGitCmd.Flags().StringVar(&gitArgs.url, "url", "", "Git repository URL") + bootstrapGitCmd.Flags().DurationVar(&gitArgs.interval, "interval", time.Minute, "sync interval") + bootstrapGitCmd.Flags().Var(&gitArgs.path, "path", "path relative to the repository root, when specified the cluster sync will be scoped to this path") + bootstrapGitCmd.Flags().StringVarP(&gitArgs.username, "username", "u", "git", "basic authentication username") + bootstrapGitCmd.Flags().StringVarP(&gitArgs.password, "password", "p", "", "basic authentication password") + + bootstrapCmd.AddCommand(bootstrapGitCmd) +} + +func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error { + if err := bootstrapValidate(); err != nil { + return err + } + + repositoryURL, err := url.Parse(gitArgs.url) + if err != nil { + return err + } + gitAuth, err := transportForURL(repositoryURL) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), rootArgs.timeout) + defer cancel() + + kubeClient, err := utils.KubeClient(rootArgs.kubeconfig, rootArgs.kubecontext) + if err != nil { + return err + } + + // Manifest base + if ver, err := getVersion(bootstrapArgs.version); err == nil { + bootstrapArgs.version = ver + } + manifestsBase, err := buildEmbeddedManifestBase() + if err != nil { + return err + } + defer os.RemoveAll(manifestsBase) + + // Lazy go-git repository + tmpDir, err := ioutil.TempDir("", "flux-bootstrap-") + if err != nil { + return fmt.Errorf("failed to create temporary working dir: %w", err) + } + defer os.RemoveAll(tmpDir) + gitClient := gogit.New(tmpDir, gitAuth) + + // Install manifest config + installOptions := install.Options{ + BaseURL: rootArgs.defaults.BaseURL, + Version: bootstrapArgs.version, + Namespace: rootArgs.namespace, + Components: bootstrapComponents(), + Registry: bootstrapArgs.registry, + ImagePullSecret: bootstrapArgs.imagePullSecret, + WatchAllNamespaces: bootstrapArgs.watchAllNamespaces, + NetworkPolicy: bootstrapArgs.networkPolicy, + LogLevel: bootstrapArgs.logLevel.String(), + NotificationController: rootArgs.defaults.NotificationController, + ManifestFile: rootArgs.defaults.ManifestFile, + Timeout: rootArgs.timeout, + TargetPath: gitArgs.path.String(), + ClusterDomain: bootstrapArgs.clusterDomain, + TolerationKeys: bootstrapArgs.tolerationKeys, + } + if customBaseURL := bootstrapArgs.manifestsPath; customBaseURL != "" { + installOptions.BaseURL = customBaseURL + } + + // Source generation and secret config + secretOpts := sourcesecret.Options{ + Name: bootstrapArgs.secretName, + Namespace: rootArgs.namespace, + TargetPath: gitArgs.path.String(), + ManifestFile: sourcesecret.MakeDefaultOptions().ManifestFile, + } + if bootstrapArgs.tokenAuth { + secretOpts.Username = gitArgs.username + secretOpts.Password = gitArgs.password + + if bootstrapArgs.caFile != "" { + secretOpts.CAFilePath = bootstrapArgs.caFile + } + } else { + secretOpts.PrivateKeyAlgorithm = sourcesecret.PrivateKeyAlgorithm(bootstrapArgs.keyAlgorithm) + secretOpts.RSAKeyBits = int(bootstrapArgs.keyRSABits) + secretOpts.ECDSACurve = bootstrapArgs.keyECDSACurve.Curve + secretOpts.SSHHostname = repositoryURL.Host + + if bootstrapArgs.sshHostname != "" { + secretOpts.SSHHostname = bootstrapArgs.sshHostname + } + } + + // Sync manifest config + syncOpts := sync.Options{ + Interval: gitArgs.interval, + Name: rootArgs.namespace, + Namespace: rootArgs.namespace, + URL: gitArgs.url, + Branch: bootstrapArgs.branch, + Secret: bootstrapArgs.secretName, + TargetPath: gitArgs.path.String(), + ManifestFile: sync.MakeDefaultOptions().ManifestFile, + GitImplementation: sourceGitArgs.gitImplementation.String(), + } + + // Bootstrap config + bootstrapOpts := []bootstrap.GitOption{ + bootstrap.WithRepositoryURL(gitArgs.url), + bootstrap.WithBranch(bootstrapArgs.branch), + bootstrap.WithAuthor(bootstrapArgs.authorName, bootstrapArgs.authorEmail), + bootstrap.WithKubeconfig(rootArgs.kubeconfig, rootArgs.kubecontext), + bootstrap.WithPostGenerateSecretFunc(promptPublicKey), + bootstrap.WithLogger(logger), + } + + // Setup bootstrapper with constructed configs + b, err := bootstrap.NewPlainGitProvider(gitClient, kubeClient, bootstrapOpts...) + if err != nil { + return err + } + + // Run + return bootstrap.Run(ctx, b, manifestsBase, installOptions, secretOpts, syncOpts, rootArgs.pollInterval, rootArgs.timeout) +} + +// transportForURL constructs a transport.AuthMethod based on the scheme +// of the given URL and the configured flags. If the protocol equals +// "ssh" but no private key is configured, authentication using the local +// SSH-agent is attempted. +func transportForURL(u *url.URL) (transport.AuthMethod, error) { + switch u.Scheme { + case "https": + return &http.BasicAuth{ + Username: gitArgs.username, + Password: gitArgs.password, + }, nil + case "ssh": + if bootstrapArgs.privateKeyFile != "" { + return ssh.NewPublicKeysFromFile(u.User.Username(), bootstrapArgs.privateKeyFile, "") + } + return nil, nil + default: + return nil, fmt.Errorf("scheme %q is not supported", u.Scheme) + } +} + +func promptPublicKey(ctx context.Context, secret corev1.Secret, _ sourcesecret.Options) error { + ppk, ok := secret.StringData[sourcesecret.PublicKeySecretKey] + if !ok { + return nil + } + + logger.Successf("public key: %s", strings.TrimSpace(ppk)) + prompt := promptui.Prompt{ + Label: "Please give the key access to your repository", + IsConfirm: true, + } + _, err := prompt.Run() + if err != nil { + return fmt.Errorf("aborting") + } + return nil +} diff --git a/docs/cmd/flux_bootstrap.md b/docs/cmd/flux_bootstrap.md index 78760620..d6ca280a 100644 --- a/docs/cmd/flux_bootstrap.md +++ b/docs/cmd/flux_bootstrap.md @@ -49,6 +49,7 @@ The bootstrap sub-commands bootstrap the toolkit components on the targeted Git ### SEE ALSO * [flux](../flux/) - Command line utility for assembling Kubernetes CD pipelines +* [flux bootstrap git](../flux_bootstrap_git/) - Bootstrap toolkit components in a Git repository * [flux bootstrap github](../flux_bootstrap_github/) - Bootstrap toolkit components in a GitHub repository * [flux bootstrap gitlab](../flux_bootstrap_gitlab/) - Bootstrap toolkit components in a GitLab repository diff --git a/docs/cmd/flux_bootstrap_git.md b/docs/cmd/flux_bootstrap_git.md new file mode 100644 index 00000000..63819939 --- /dev/null +++ b/docs/cmd/flux_bootstrap_git.md @@ -0,0 +1,78 @@ +--- +title: "flux bootstrap git command" +--- +## flux bootstrap git + +Bootstrap toolkit components in a Git repository + +### Synopsis + +The bootstrap git command commits the toolkit components manifests to the +branch of a Git repository. It then configures the target cluster to synchronize with +the repository. If the toolkit components are present on the cluster, the bootstrap +command will perform an upgrade if needed. + +``` +flux bootstrap git [flags] +``` + +### Examples + +``` + # Run bootstrap for a Git repository and authenticate with your SSH agent + flux bootstrap git --url=ssh://git@example.com/repository.git + + # Run bootstrap for a Git repository and authenticate using a password + flux bootstrap git --url=https://example.com/repository.git --password= + + # Run bootstrap for a Git repository with a passwordless private key + flux bootstrap git --url=ssh://git@example.com/repository.git --private-key-file= + +``` + +### Options + +``` + -h, --help help for git + --interval duration sync interval (default 1m0s) + -p, --password string basic authentication password + --path safeRelativePath path relative to the repository root, when specified the cluster sync will be scoped to this path + --url string Git repository URL + -u, --username string basic authentication username (default "git") +``` + +### Options inherited from parent commands + +``` + --author-email string author email for Git commits + --author-name string author name for Git commits (default "Flux") + --branch string default branch (for GitHub this must match the default branch setting for the organization) (default "main") + --ca-file string path to TLS CA file used for validating self-signed certificates + --cluster-domain string internal cluster domain (default "cluster.local") + --components strings list of components, accepts comma-separated values (default [source-controller,kustomize-controller,helm-controller,notification-controller]) + --components-extra strings list of components in addition to those supplied or defaulted, accepts comma-separated values + --context string kubernetes context to use + --image-pull-secret string Kubernetes secret name used for pulling the toolkit images from a private registry + --kubeconfig string absolute path to the kubeconfig file + --log-level logLevel log level, available options are: (debug, info, error) (default info) + -n, --namespace string the namespace scope for this operation (default "flux-system") + --network-policy deny ingress access to the toolkit controllers from other namespaces using network policies (default true) + --private-key-file string path to a private key file used for authenticating to the Git SSH server + --registry string container registry where the toolkit images are published (default "ghcr.io/fluxcd") + --secret-name string name of the secret the sync credentials can be found in or stored to (default "flux-system") + --ssh-ecdsa-curve ecdsaCurve SSH ECDSA public key curve (p256, p384, p521) (default p384) + --ssh-hostname string SSH hostname, to be used when the SSH host differs from the HTTPS one + --ssh-key-algorithm publicKeyAlgorithm SSH public key algorithm (rsa, ecdsa, ed25519) (default rsa) + --ssh-rsa-bits rsaKeyBits SSH RSA public key bit size (multiplies of 8) (default 2048) + --timeout duration timeout for this operation (default 5m0s) + --token-auth when enabled, the personal access token will be used instead of SSH deploy key + --toleration-keys strings list of toleration keys used to schedule the components pods onto nodes with matching taints + --verbose print generated objects + -v, --version string toolkit version, when specified the manifests are downloaded from https://github.com/fluxcd/flux2/releases + --watch-all-namespaces watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true) +``` + +### SEE ALSO + +* [flux bootstrap](../flux_bootstrap/) - Bootstrap toolkit components + From 96c373d0459a2bb32e9a98bc6f07376203427ce1 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Mon, 22 Mar 2021 21:22:50 +0100 Subject: [PATCH 06/11] Properly configure sync URL based on auth settings Signed-off-by: Hidde Beydals --- cmd/flux/bootstrap_git.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/cmd/flux/bootstrap_git.go b/cmd/flux/bootstrap_git.go index cd01bc08..0774d1d9 100644 --- a/cmd/flux/bootstrap_git.go +++ b/cmd/flux/bootstrap_git.go @@ -156,15 +156,26 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error { if bootstrapArgs.caFile != "" { secretOpts.CAFilePath = bootstrapArgs.caFile } + + // Configure repository URL to match auth config for sync. + repositoryURL.User = nil + repositoryURL.Scheme = "https" + repositoryURL.Host = repositoryURL.Hostname() } else { secretOpts.PrivateKeyAlgorithm = sourcesecret.PrivateKeyAlgorithm(bootstrapArgs.keyAlgorithm) secretOpts.RSAKeyBits = int(bootstrapArgs.keyRSABits) secretOpts.ECDSACurve = bootstrapArgs.keyECDSACurve.Curve - secretOpts.SSHHostname = repositoryURL.Host + // Configure repository URL to match auth config for sync. + repositoryURL.User = url.User(gitArgs.username) + repositoryURL.Scheme = "ssh" + repositoryURL.Host = repositoryURL.Hostname() if bootstrapArgs.sshHostname != "" { - secretOpts.SSHHostname = bootstrapArgs.sshHostname + repositoryURL.Host = bootstrapArgs.sshHostname } + + // Configure last as it depends on the config above. + secretOpts.SSHHostname = repositoryURL.Host } // Sync manifest config @@ -172,7 +183,7 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error { Interval: gitArgs.interval, Name: rootArgs.namespace, Namespace: rootArgs.namespace, - URL: gitArgs.url, + URL: repositoryURL.String(), Branch: bootstrapArgs.branch, Secret: bootstrapArgs.secretName, TargetPath: gitArgs.path.String(), From 7f0bc2ada24753f54eda098b3a83d2e8264cbee0 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Mon, 29 Mar 2021 14:21:23 +0200 Subject: [PATCH 07/11] Provide option to add appendix to commit messages Using the `--commit-message-appendix` flag a string can be added to the commit messages made by the bootstrapper process to for example skip CI actions from executing using e.g. `[skip ci]`. Signed-off-by: Hidde Beydals --- cmd/flux/bootstrap.go | 4 ++++ cmd/flux/bootstrap_git.go | 1 + cmd/flux/bootstrap_github.go | 1 + cmd/flux/bootstrap_gitlab.go | 1 + docs/cmd/flux_bootstrap.md | 1 + docs/cmd/flux_bootstrap_git.md | 1 + docs/cmd/flux_bootstrap_github.md | 1 + docs/cmd/flux_bootstrap_gitlab.md | 1 + internal/bootstrap/bootstrap_plain_git.go | 15 ++++++++++++--- internal/bootstrap/options.go | 14 ++++++++++++++ 10 files changed, 37 insertions(+), 3 deletions(-) diff --git a/cmd/flux/bootstrap.go b/cmd/flux/bootstrap.go index 46d7fe38..05139ab4 100644 --- a/cmd/flux/bootstrap.go +++ b/cmd/flux/bootstrap.go @@ -65,6 +65,8 @@ type bootstrapFlags struct { authorName string authorEmail string + + commitMessageAppendix string } const ( @@ -113,6 +115,8 @@ func init() { bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.authorName, "author-name", "Flux", "author name for Git commits") bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.authorEmail, "author-email", "", "author email for Git commits") + bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.commitMessageAppendix, "commit-message-appendix", "", "string to add to the commit messages, e.g. '[ci skip]'") + bootstrapCmd.PersistentFlags().Var(&bootstrapArgs.arch, "arch", bootstrapArgs.arch.Description()) bootstrapCmd.PersistentFlags().MarkDeprecated("arch", "multi-arch container image is now available for AMD64, ARMv7 and ARM64") bootstrapCmd.PersistentFlags().MarkHidden("manifests") diff --git a/cmd/flux/bootstrap_git.go b/cmd/flux/bootstrap_git.go index 0774d1d9..fa8eb584 100644 --- a/cmd/flux/bootstrap_git.go +++ b/cmd/flux/bootstrap_git.go @@ -196,6 +196,7 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error { bootstrap.WithRepositoryURL(gitArgs.url), bootstrap.WithBranch(bootstrapArgs.branch), bootstrap.WithAuthor(bootstrapArgs.authorName, bootstrapArgs.authorEmail), + bootstrap.WithCommitMessageAppendix(bootstrapArgs.commitMessageAppendix), bootstrap.WithKubeconfig(rootArgs.kubeconfig, rootArgs.kubecontext), bootstrap.WithPostGenerateSecretFunc(promptPublicKey), bootstrap.WithLogger(logger), diff --git a/cmd/flux/bootstrap_github.go b/cmd/flux/bootstrap_github.go index 747d9829..81238331 100644 --- a/cmd/flux/bootstrap_github.go +++ b/cmd/flux/bootstrap_github.go @@ -219,6 +219,7 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { bootstrap.WithBranch(bootstrapArgs.branch), bootstrap.WithBootstrapTransportType("https"), bootstrap.WithAuthor(bootstrapArgs.authorName, bootstrapArgs.authorEmail), + bootstrap.WithCommitMessageAppendix(bootstrapArgs.commitMessageAppendix), bootstrap.WithProviderTeamPermissions(mapTeamSlice(githubArgs.teams, ghDefaultPermission)), bootstrap.WithReadWriteKeyPermissions(githubArgs.readWriteKey), bootstrap.WithKubeconfig(rootArgs.kubeconfig, rootArgs.kubecontext), diff --git a/cmd/flux/bootstrap_gitlab.go b/cmd/flux/bootstrap_gitlab.go index 48768c80..6be6c6f5 100644 --- a/cmd/flux/bootstrap_gitlab.go +++ b/cmd/flux/bootstrap_gitlab.go @@ -228,6 +228,7 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error { bootstrap.WithBranch(bootstrapArgs.branch), bootstrap.WithBootstrapTransportType("https"), bootstrap.WithAuthor(bootstrapArgs.authorName, bootstrapArgs.authorEmail), + bootstrap.WithCommitMessageAppendix(bootstrapArgs.commitMessageAppendix), bootstrap.WithProviderTeamPermissions(mapTeamSlice(gitlabArgs.teams, glDefaultPermission)), bootstrap.WithReadWriteKeyPermissions(gitlabArgs.readWriteKey), bootstrap.WithKubeconfig(rootArgs.kubeconfig, rootArgs.kubecontext), diff --git a/docs/cmd/flux_bootstrap.md b/docs/cmd/flux_bootstrap.md index d6ca280a..7d944dbd 100644 --- a/docs/cmd/flux_bootstrap.md +++ b/docs/cmd/flux_bootstrap.md @@ -17,6 +17,7 @@ The bootstrap sub-commands bootstrap the toolkit components on the targeted Git --branch string default branch (for GitHub this must match the default branch setting for the organization) (default "main") --ca-file string path to TLS CA file used for validating self-signed certificates --cluster-domain string internal cluster domain (default "cluster.local") + --commit-message-appendix string string to add to the commit messages, e.g. '[ci skip]' --components strings list of components, accepts comma-separated values (default [source-controller,kustomize-controller,helm-controller,notification-controller]) --components-extra strings list of components in addition to those supplied or defaulted, accepts comma-separated values -h, --help help for bootstrap diff --git a/docs/cmd/flux_bootstrap_git.md b/docs/cmd/flux_bootstrap_git.md index 63819939..0587e536 100644 --- a/docs/cmd/flux_bootstrap_git.md +++ b/docs/cmd/flux_bootstrap_git.md @@ -49,6 +49,7 @@ flux bootstrap git [flags] --branch string default branch (for GitHub this must match the default branch setting for the organization) (default "main") --ca-file string path to TLS CA file used for validating self-signed certificates --cluster-domain string internal cluster domain (default "cluster.local") + --commit-message-appendix string string to add to the commit messages, e.g. '[ci skip]' --components strings list of components, accepts comma-separated values (default [source-controller,kustomize-controller,helm-controller,notification-controller]) --components-extra strings list of components in addition to those supplied or defaulted, accepts comma-separated values --context string kubernetes context to use diff --git a/docs/cmd/flux_bootstrap_github.md b/docs/cmd/flux_bootstrap_github.md index cde0605d..05dbe90e 100644 --- a/docs/cmd/flux_bootstrap_github.md +++ b/docs/cmd/flux_bootstrap_github.md @@ -68,6 +68,7 @@ flux bootstrap github [flags] --branch string default branch (for GitHub this must match the default branch setting for the organization) (default "main") --ca-file string path to TLS CA file used for validating self-signed certificates --cluster-domain string internal cluster domain (default "cluster.local") + --commit-message-appendix string string to add to the commit messages, e.g. '[ci skip]' --components strings list of components, accepts comma-separated values (default [source-controller,kustomize-controller,helm-controller,notification-controller]) --components-extra strings list of components in addition to those supplied or defaulted, accepts comma-separated values --context string kubernetes context to use diff --git a/docs/cmd/flux_bootstrap_gitlab.md b/docs/cmd/flux_bootstrap_gitlab.md index 3d0b9c43..a3a6823f 100644 --- a/docs/cmd/flux_bootstrap_gitlab.md +++ b/docs/cmd/flux_bootstrap_gitlab.md @@ -65,6 +65,7 @@ flux bootstrap gitlab [flags] --branch string default branch (for GitHub this must match the default branch setting for the organization) (default "main") --ca-file string path to TLS CA file used for validating self-signed certificates --cluster-domain string internal cluster domain (default "cluster.local") + --commit-message-appendix string string to add to the commit messages, e.g. '[ci skip]' --components strings list of components, accepts comma-separated values (default [source-controller,kustomize-controller,helm-controller,notification-controller]) --components-extra strings list of components in addition to those supplied or defaulted, accepts comma-separated values --context string kubernetes context to use diff --git a/internal/bootstrap/bootstrap_plain_git.go b/internal/bootstrap/bootstrap_plain_git.go index b293b78b..8119e79a 100644 --- a/internal/bootstrap/bootstrap_plain_git.go +++ b/internal/bootstrap/bootstrap_plain_git.go @@ -48,7 +48,8 @@ type PlainGitBootstrapper struct { url string branch string - author git.Author + author git.Author + commitMessageAppendix string kubeconfig string kubecontext string @@ -126,9 +127,13 @@ func (b *PlainGitBootstrapper) ReconcileComponents(ctx context.Context, manifest } // Git commit generated + commitMsg := fmt.Sprintf("Add Flux %s component manifests", options.Version) + if b.commitMessageAppendix != "" { + commitMsg = commitMsg + "\n\n" + b.commitMessageAppendix + } commit, err := b.git.Commit(git.Commit{ Author: b.author, - Message: fmt.Sprintf("Add Flux %s component manifests", options.Version), + Message: commitMsg, }) if err != nil && err != git.ErrNoStagedFiles { return fmt.Errorf("failed to commit sync manifests: %w", err) @@ -245,9 +250,13 @@ func (b *PlainGitBootstrapper) ReconcileSyncConfig(ctx context.Context, options b.logger.Successf("generated sync manifests") // Git commit generated + commitMsg := fmt.Sprintf("Add Flux sync manifests") + if b.commitMessageAppendix != "" { + commitMsg = commitMsg + "\n\n" + b.commitMessageAppendix + } commit, err := b.git.Commit(git.Commit{ Author: b.author, - Message: fmt.Sprintf("Add Flux sync manifests"), + Message: commitMsg, }) if err != nil && err != git.ErrNoStagedFiles { return fmt.Errorf("failed to commit sync manifests: %w", err) diff --git a/internal/bootstrap/options.go b/internal/bootstrap/options.go index 9f737771..88972c1e 100644 --- a/internal/bootstrap/options.go +++ b/internal/bootstrap/options.go @@ -62,6 +62,20 @@ func (o authorOption) applyGitProvider(b *GitProviderBootstrapper) { o.applyGit(b.PlainGitBootstrapper) } +func WithCommitMessageAppendix(appendix string) Option { + return commitMessageAppendixOption(appendix) +} + +type commitMessageAppendixOption string + +func (o commitMessageAppendixOption) applyGit(b *PlainGitBootstrapper) { + b.commitMessageAppendix = string(o) +} + +func (o commitMessageAppendixOption) applyGitProvider(b *GitProviderBootstrapper) { + o.applyGit(b.PlainGitBootstrapper) +} + func WithKubeconfig(kubeconfig, kubecontext string) Option { return kubeconfigOption{ kubeconfig: kubeconfig, From ef576128e3869baa7aa0c61e7fb32133a15e68e3 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Mon, 29 Mar 2021 15:39:19 +0200 Subject: [PATCH 08/11] Use correct hostname argument for secret gen Signed-off-by: Hidde Beydals --- cmd/flux/bootstrap_gitlab.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/flux/bootstrap_gitlab.go b/cmd/flux/bootstrap_gitlab.go index 6be6c6f5..fb22972c 100644 --- a/cmd/flux/bootstrap_gitlab.go +++ b/cmd/flux/bootstrap_gitlab.go @@ -200,7 +200,7 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error { secretOpts.PrivateKeyAlgorithm = sourcesecret.PrivateKeyAlgorithm(bootstrapArgs.keyAlgorithm) secretOpts.RSAKeyBits = int(bootstrapArgs.keyRSABits) secretOpts.ECDSACurve = bootstrapArgs.keyECDSACurve.Curve - secretOpts.SSHHostname = githubArgs.hostname + secretOpts.SSHHostname = gitlabArgs.hostname if bootstrapArgs.privateKeyFile != "" { secretOpts.PrivateKeyPath = bootstrapArgs.privateKeyFile From e65a5beaaecee6881602125ae7ac5ec14fb2e487 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Tue, 30 Mar 2021 16:58:10 +0200 Subject: [PATCH 09/11] Work around custom client domain issue With this commit comes a lot of evil. Signed-off-by: Hidde Beydals --- cmd/flux/bootstrap_gitlab.go | 7 +++++++ internal/bootstrap/bootstrap_provider.go | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/cmd/flux/bootstrap_gitlab.go b/cmd/flux/bootstrap_gitlab.go index fb22972c..a57b9aa5 100644 --- a/cmd/flux/bootstrap_gitlab.go +++ b/cmd/flux/bootstrap_gitlab.go @@ -22,6 +22,7 @@ import ( "io/ioutil" "os" "regexp" + "strings" "time" "github.com/go-git/go-git/v5/plumbing/transport/http" @@ -144,6 +145,12 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error { Hostname: gitlabArgs.hostname, Token: glToken, } + // Workaround for: https://github.com/fluxcd/go-git-providers/issues/55 + if hostname := providerCfg.Hostname; hostname != glDefaultDomain && + !strings.HasPrefix(hostname, "https://") && + !strings.HasPrefix(hostname, "http://") { + providerCfg.Hostname = "https://" + providerCfg.Hostname + } providerClient, err := provider.BuildGitProvider(providerCfg) if err != nil { return err diff --git a/internal/bootstrap/bootstrap_provider.go b/internal/bootstrap/bootstrap_provider.go index a6a17d4e..34c94410 100644 --- a/internal/bootstrap/bootstrap_provider.go +++ b/internal/bootstrap/bootstrap_provider.go @@ -210,6 +210,10 @@ func (b *GitProviderBootstrapper) ReconcileRepository(ctx context.Context) error } cloneURL := repo.Repository().GetCloneURL(gitprovider.TransportType(b.bootstrapTransportType)) + // TODO(hidde): https://github.com/fluxcd/go-git-providers/issues/55 + if strings.HasPrefix(cloneURL, "https://https://") { + cloneURL = strings.TrimPrefix(cloneURL, "https://") + } WithRepositoryURL(cloneURL).applyGit(b.PlainGitBootstrapper) return err @@ -380,6 +384,10 @@ func (b *GitProviderBootstrapper) getRepository(ctx context.Context) (gitprovide // the hostname of the URL will be modified to this hostname. func (b *GitProviderBootstrapper) getCloneURL(repository gitprovider.UserRepository, transport gitprovider.TransportType) (string, error) { u := repository.Repository().GetCloneURL(transport) + // TODO(hidde): https://github.com/fluxcd/go-git-providers/issues/55 + if strings.HasPrefix(u, "https://https://") { + u = strings.TrimPrefix(u, "https://") + } var err error if transport == gitprovider.TransportTypeSSH && b.sshHostname != "" { if u, err = setHostname(u, b.sshHostname); err != nil { From 4ece12348bd9032f27f2169ec10537c644dbc020 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Tue, 6 Apr 2021 13:11:52 +0200 Subject: [PATCH 10/11] Ignore broken symlinks and outside path, in commit Signed-off-by: Hidde Beydals --- internal/bootstrap/git/gogit/gogit.go | 31 +++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/internal/bootstrap/git/gogit/gogit.go b/internal/bootstrap/git/gogit/gogit.go index 2060f2f9..619dd552 100644 --- a/internal/bootstrap/git/gogit/gogit.go +++ b/internal/bootstrap/git/gogit/gogit.go @@ -20,6 +20,8 @@ import ( "context" "fmt" "io" + "os" + "path/filepath" "strings" "time" @@ -138,16 +140,37 @@ func (g *GoGit) Commit(message git.Commit) (string, error) { if err != nil { return "", err } - if status.IsClean() { + + // go-git has [a bug](https://github.com/go-git/go-git/issues/253) + // whereby it thinks broken symlinks to absolute paths are + // modified. There's no circumstance in which we want to commit a + // change to a broken symlink: so, detect and skip those. + var changed bool + for file, _ := range status { + abspath := filepath.Join(g.path, file) + info, err := os.Lstat(abspath) + if err != nil { + return "", fmt.Errorf("checking if %s is a symlink: %w", file, err) + } + if info.Mode()&os.ModeSymlink > 0 { + // symlinks are OK; broken symlinks are probably a result + // of the bug mentioned above, but not of interest in any + // case. + if _, err := os.Stat(abspath); os.IsNotExist(err) { + continue + } + } + _, _ = wt.Add(file) + changed = true + } + + if !changed { head, err := g.repository.Head() if err != nil { return "", err } return head.Hash().String(), git.ErrNoStagedFiles } - if _, err = wt.Add("."); err != nil { - return "", err - } commit, err := wt.Commit(message.Message, &gogit.CommitOptions{ Author: &object.Signature{ From 7481c6beb03897a2c6518a616ae33d6d5948ebb1 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Tue, 6 Apr 2021 15:35:09 +0200 Subject: [PATCH 11/11] Retry reconcile and clone actions once We have observed that the code at times outperforms GitHub mechanics, resulting in not found errors that are only true for a millisecond. Retrying those actions once with a 2 second delay should be more friendly to users. Signed-off-by: Hidde Beydals --- internal/bootstrap/bootstrap.go | 14 ++++++++++++++ internal/bootstrap/bootstrap_plain_git.go | 16 +++++++++++----- internal/bootstrap/bootstrap_provider.go | 14 +++++++++++--- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index a4025478..bb265a57 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -182,3 +182,17 @@ func kustomizationReconciled(ctx context.Context, kube client.Client, objKey cli return false, nil } } + +func retry(retries int, wait time.Duration, fn func() error) (err error) { + for i := 0; ; i++ { + err = fn() + if err == nil { + return + } + if i >= retries { + break + } + time.Sleep(wait) + } + return err +} diff --git a/internal/bootstrap/bootstrap_plain_git.go b/internal/bootstrap/bootstrap_plain_git.go index 8119e79a..b3ee125e 100644 --- a/internal/bootstrap/bootstrap_plain_git.go +++ b/internal/bootstrap/bootstrap_plain_git.go @@ -104,8 +104,11 @@ func (b *PlainGitBootstrapper) ReconcileComponents(ctx context.Context, manifest } b.logger.Actionf("cloning branch %q from Git repository %q", b.branch, b.url) - cloned, err := b.git.Clone(ctx, b.url, b.branch) - if err != nil { + var cloned bool + if err = retry(1, 2*time.Second, func() (err error) { + cloned, err = b.git.Clone(ctx, b.url, b.branch) + return + }); err != nil { return fmt.Errorf("failed to clone repository: %w", err) } if cloned { @@ -216,12 +219,15 @@ func (b *PlainGitBootstrapper) ReconcileSyncConfig(ctx context.Context, options if _, err := b.git.Status(); err != nil { if err == git.ErrNoGitRepository { b.logger.Actionf("cloning branch %q from Git repository %q", b.branch, b.url) - cloned, err := b.git.Clone(ctx, b.url, b.branch) - if err != nil { + var cloned bool + if err = retry(1, 2*time.Second, func() (err error) { + cloned, err = b.git.Clone(ctx, b.url, b.branch) + return + }); err != nil { return fmt.Errorf("failed to clone repository: %w", err) } if cloned { - b.logger.Successf("cloned repository", b.url) + b.logger.Successf("cloned repository") } } return err diff --git a/internal/bootstrap/bootstrap_provider.go b/internal/bootstrap/bootstrap_provider.go index 34c94410..5a0bdea3 100644 --- a/internal/bootstrap/bootstrap_provider.go +++ b/internal/bootstrap/bootstrap_provider.go @@ -278,11 +278,15 @@ func (b *GitProviderBootstrapper) reconcileOrgRepository(ctx context.Context) (g } b.logger.Successf("repository %q created", repoRef.String()) } + // Set default branch before calling Reconcile due to bug described // above. repoInfo.DefaultBranch = repo.Get().DefaultBranch var changed bool - if repo, changed, err = b.provider.OrgRepositories().Reconcile(ctx, repoRef, repoInfo); err != nil { + if err = retry(1, 2*time.Second, func() (err error) { + repo, changed, err = b.provider.OrgRepositories().Reconcile(ctx, repoRef, repoInfo) + return + }); err != nil { return nil, fmt.Errorf("failed to reconcile Git repository %q: %w", repoRef.String(), err) } if changed { @@ -352,12 +356,16 @@ func (b *GitProviderBootstrapper) reconcileUserRepository(ctx context.Context) ( // above. repoInfo.DefaultBranch = repo.Get().DefaultBranch var changed bool - if repo, changed, err = b.provider.UserRepositories().Reconcile(ctx, repoRef, repoInfo); err != nil { - return nil, err + if err = retry(1, 2*time.Second, func() (err error) { + repo, changed, err = b.provider.UserRepositories().Reconcile(ctx, repoRef, repoInfo) + return + }); err != nil { + return nil, fmt.Errorf("failed to reconcile Git repository %q: %w", repoRef.String(), err) } if changed { b.logger.Successf("repository %q reconciled", repoRef.String()) } + return repo, nil }