diff --git a/go.mod b/go.mod index fc5ab920..eab2d3b5 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.16 require ( github.com/Masterminds/semver/v3 v3.1.0 github.com/cyphar/filepath-securejoin v0.2.2 + github.com/fluxcd/go-git-providers v0.0.3 github.com/fluxcd/helm-controller/api v0.9.0 github.com/fluxcd/image-automation-controller/api v0.7.0 github.com/fluxcd/image-reflector-controller/api v0.7.1 @@ -17,6 +18,7 @@ require ( github.com/fluxcd/pkg/untar v0.0.5 github.com/fluxcd/pkg/version v0.0.1 github.com/fluxcd/source-controller/api v0.11.0 + github.com/go-git/go-git/v5 v5.1.0 github.com/google/go-containerregistry v0.2.0 github.com/manifoldco/promptui v0.7.0 github.com/olekukonko/tablewriter v0.0.4 diff --git a/go.sum b/go.sum index 507787ea..60fa4d6a 100644 --- a/go.sum +++ b/go.sum @@ -188,6 +188,8 @@ github.com/evanphx/json-patch/v5 v5.1.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2Vvl github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fluxcd/go-git-providers v0.0.3 h1:pquQvTpd1a4V1efPyZWuVPeIKrTgV8QRoDY0VGH+qiw= +github.com/fluxcd/go-git-providers v0.0.3/go.mod h1:iaXf3nEq8MB/LzxfbNcCl48sAtIReUU7jqjJ7CEnfFQ= github.com/fluxcd/helm-controller/api v0.9.0 h1:L60KmCblTQo3UimgCzVQGe330tC+b15CrLozvhPNmJU= github.com/fluxcd/helm-controller/api v0.9.0/go.mod h1:HIWSF3n1QU3hdqjQMFizFUZVr1uV+abmlGAEpB7vB9A= github.com/fluxcd/image-automation-controller/api v0.7.0 h1:mLaELYT52/FpZ93Mr+QMSK8UT0OBVQT4oA9kxO8NiEk= @@ -340,6 +342,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -384,6 +387,8 @@ github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-containerregistry v0.2.0 h1:cWFYx+kOkKdyOET0pcp7GMCmxj7da40StvluSuSXWCg= github.com/google/go-containerregistry v0.2.0/go.mod h1:Ts3Wioz1r5ayWx8sS6vLcWltWcM1aqFjd/eVrkFhrWM= +github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= +github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-github/v33 v33.0.0 h1:qAf9yP0qc54ufQxzwv+u9H0tiVOnPJxo0lI/JXqw3ZM= github.com/google/go-github/v33 v33.0.0/go.mod h1:GMdDnVZY/2TsWgp/lkYnpSAh6TrzhANBBwm6k6TTEXg= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= @@ -422,6 +427,8 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= @@ -491,6 +498,8 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a h1:FaWFmfWdAUKbSCtOU2QjDaorUexogfaMgbipgYATUMU= github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -511,6 +520,7 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/ktrysmt/go-bitbucket v0.6.2/go.mod h1:9u0v3hsd2rqCHRIpbir1oP7F58uo5dq19sBYvuMoyQ4= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= @@ -557,6 +567,7 @@ github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUb github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -591,6 +602,7 @@ github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M= github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= @@ -722,6 +734,7 @@ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV github.com/vdemeester/k8s-pkg-credentialprovider v1.18.1-0.20201019120933-f1d16962a4db/go.mod h1:grWy0bkr1XO6hqbaaCKaPXqkBVlMGHYG6PGykktwbJc= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= +github.com/xanzy/go-gitlab v0.33.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug= github.com/xanzy/go-gitlab v0.43.0 h1:rpOZQjxVJGW/ch+Jy4j7W4o7BB1mxkXJNVGuplZ7PUs= github.com/xanzy/go-gitlab v0.43.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= @@ -858,6 +871,7 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/oauth2 v0.0.0-20180227000427-d7d64896b5ff/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -874,6 +888,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180224232135-f6cff0780e54/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1021,6 +1036,7 @@ google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/ google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/appengine v1.0.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go new file mode 100644 index 00000000..a4025478 --- /dev/null +++ b/internal/bootstrap/bootstrap.go @@ -0,0 +1,184 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierr "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1" + "github.com/fluxcd/pkg/apis/meta" + + "github.com/fluxcd/flux2/pkg/manifestgen/install" + "github.com/fluxcd/flux2/pkg/manifestgen/sourcesecret" + "github.com/fluxcd/flux2/pkg/manifestgen/sync" +) + +var ( + ErrReconciledWithWarning = errors.New("reconciled with warning") +) + +type Reconciler interface { + // ReconcileComponents reconciles the components by generating the + // manifests with the provided values, committing them to Git and + // pushing to remote if there are any changes, and applying them + // to the cluster. + ReconcileComponents(ctx context.Context, manifestsBase string, options install.Options) error + + // ReconcileSourceSecret reconciles the source secret by generating + // a new secret with the provided values if the secret does not + // already exists on the cluster, or if any of the configuration + // options changed. + ReconcileSourceSecret(ctx context.Context, options sourcesecret.Options) error + + // ReconcileSyncConfig reconciles the sync configuration by generating + // the sync manifests with the provided values, committing them to Git + // and pushing to remote if there are any changes. + ReconcileSyncConfig(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error + + // ConfirmHealthy confirms that the components and extra components in + // install.Options are healthy. + ConfirmHealthy(ctx context.Context, options install.Options, timeout time.Duration) error +} + +type RepositoryReconciler interface { + // ReconcileRepository reconciles an external Git repository. + ReconcileRepository(ctx context.Context) error +} + +type PostGenerateSecretFunc func(ctx context.Context, secret corev1.Secret, options sourcesecret.Options) error + +func Run(ctx context.Context, reconciler Reconciler, manifestsBase string, + installOpts install.Options, secretOpts sourcesecret.Options, syncOpts sync.Options, + pollInterval, timeout time.Duration) error { + + var err error + if r, ok := reconciler.(RepositoryReconciler); ok { + if err = r.ReconcileRepository(ctx); err != nil && !errors.Is(err, ErrReconciledWithWarning) { + return err + } + } + + if err := reconciler.ReconcileComponents(ctx, manifestsBase, installOpts); err != nil { + return err + } + if err := reconciler.ReconcileSourceSecret(ctx, secretOpts); err != nil { + return err + } + if err := reconciler.ReconcileSyncConfig(ctx, syncOpts, pollInterval, timeout); err != nil { + return err + } + if err := reconciler.ConfirmHealthy(ctx, installOpts, timeout); err != nil { + return err + } + + return err +} + +func mustInstallManifests(ctx context.Context, kube client.Client, namespace string) bool { + namespacedName := types.NamespacedName{ + Namespace: namespace, + Name: namespace, + } + var k kustomizev1.Kustomization + if err := kube.Get(ctx, namespacedName, &k); err != nil { + return true + } + return k.Status.LastAppliedRevision == "" +} + +func secretExists(ctx context.Context, kube client.Client, objKey client.ObjectKey) (bool, error) { + if err := kube.Get(ctx, objKey, &corev1.Secret{}); err != nil { + if apierr.IsNotFound(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func reconcileSecret(ctx context.Context, kube client.Client, secret corev1.Secret) error { + objKey := client.ObjectKeyFromObject(&secret) + var existing corev1.Secret + err := kube.Get(ctx, objKey, &existing) + if err != nil { + if apierr.IsNotFound(err) { + return kube.Create(ctx, &secret) + } + return err + } + existing.StringData = secret.StringData + return kube.Update(ctx, &existing) +} + +func kustomizationPathDiffers(ctx context.Context, kube client.Client, objKey client.ObjectKey, path string) (string, error) { + var k kustomizev1.Kustomization + if err := kube.Get(ctx, objKey, &k); err != nil { + if apierr.IsNotFound(err) { + return "", nil + } + return "", err + } + normalizePath := func(p string) string { + return fmt.Sprintf("./%s", strings.TrimPrefix(p, "./")) + } + if normalizePath(path) == normalizePath(k.Spec.Path) { + return "", nil + } + return k.Spec.Path, nil +} + +func kustomizationReconciled(ctx context.Context, kube client.Client, objKey client.ObjectKey, + kustomization *kustomizev1.Kustomization, expectRevision string) func() (bool, error) { + + return func() (bool, error) { + if err := kube.Get(ctx, objKey, kustomization); err != nil { + return false, err + } + + // Confirm the state we are observing is for the current generation + if kustomization.Generation != kustomization.Status.ObservedGeneration { + return false, nil + } + + // Confirm the given revision has been attempted by the controller + if kustomization.Status.LastAttemptedRevision != expectRevision { + return false, nil + } + + // Confirm the resource is healthy + if c := apimeta.FindStatusCondition(kustomization.Status.Conditions, meta.ReadyCondition); c != nil { + switch c.Status { + case metav1.ConditionTrue: + return true, nil + case metav1.ConditionFalse: + return false, fmt.Errorf(c.Message) + } + } + return false, nil + } +} diff --git a/internal/bootstrap/bootstrap_plain_git.go b/internal/bootstrap/bootstrap_plain_git.go new file mode 100644 index 00000000..b293b78b --- /dev/null +++ b/internal/bootstrap/bootstrap_plain_git.go @@ -0,0 +1,315 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/cli-utils/pkg/object" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/kustomize/api/filesys" + "sigs.k8s.io/yaml" + + kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1" + + "github.com/fluxcd/flux2/internal/bootstrap/git" + + "github.com/fluxcd/flux2/internal/utils" + "github.com/fluxcd/flux2/pkg/log" + "github.com/fluxcd/flux2/pkg/manifestgen/install" + "github.com/fluxcd/flux2/pkg/manifestgen/kustomization" + "github.com/fluxcd/flux2/pkg/manifestgen/sourcesecret" + "github.com/fluxcd/flux2/pkg/manifestgen/sync" + "github.com/fluxcd/flux2/pkg/status" +) + +type PlainGitBootstrapper struct { + url string + branch string + + author git.Author + + kubeconfig string + kubecontext string + + postGenerateSecret []PostGenerateSecretFunc + + git git.Git + kube client.Client + logger log.Logger +} + +type GitOption interface { + applyGit(b *PlainGitBootstrapper) +} + +func WithRepositoryURL(url string) GitOption { + return repositoryURLOption(url) +} + +type repositoryURLOption string + +func (o repositoryURLOption) applyGit(b *PlainGitBootstrapper) { + b.url = string(o) +} + +func WithPostGenerateSecretFunc(callback PostGenerateSecretFunc) GitOption { + return postGenerateSecret(callback) +} + +type postGenerateSecret PostGenerateSecretFunc + +func (o postGenerateSecret) applyGit(b *PlainGitBootstrapper) { + b.postGenerateSecret = append(b.postGenerateSecret, PostGenerateSecretFunc(o)) +} + +func NewPlainGitProvider(git git.Git, kube client.Client, opts ...GitOption) (*PlainGitBootstrapper, error) { + b := &PlainGitBootstrapper{ + git: git, + kube: kube, + } + for _, opt := range opts { + opt.applyGit(b) + } + return b, nil +} + +func (b *PlainGitBootstrapper) ReconcileComponents(ctx context.Context, manifestsBase string, options install.Options) error { + // Clone if not already + if _, err := b.git.Status(); err != nil { + if err != git.ErrNoGitRepository { + return err + } + + b.logger.Actionf("cloning branch %q from Git repository %q", b.branch, b.url) + cloned, err := b.git.Clone(ctx, b.url, b.branch) + if err != nil { + return fmt.Errorf("failed to clone repository: %w", err) + } + if cloned { + b.logger.Successf("cloned repository") + } + } + + // Generate component manifests + b.logger.Actionf("generating component manifests") + manifests, err := install.Generate(options, manifestsBase) + if err != nil { + return fmt.Errorf("component manifest generation failed: %w", err) + } + b.logger.Successf("generated component manifests") + + // Write manifest to Git repository + if err = b.git.Write(manifests.Path, strings.NewReader(manifests.Content)); err != nil { + return fmt.Errorf("failed to write manifest %q: %w", manifests.Path, err) + } + + // Git commit generated + commit, err := b.git.Commit(git.Commit{ + Author: b.author, + Message: fmt.Sprintf("Add Flux %s component manifests", options.Version), + }) + if err != nil && err != git.ErrNoStagedFiles { + return fmt.Errorf("failed to commit sync manifests: %w", err) + } + if err == nil { + b.logger.Successf("committed sync manifests to %q (%q)", b.branch, commit) + b.logger.Actionf("pushing component manifests to %q", b.url) + if err = b.git.Push(ctx); err != nil { + return fmt.Errorf("failed to push manifests: %w", err) + } + } else { + b.logger.Successf("component manifests are up to date") + } + + // Conditionally install manifests + if mustInstallManifests(ctx, b.kube, options.Namespace) { + b.logger.Actionf("installing components in %q namespace", options.Namespace) + kubectlArgs := []string{"apply", "-f", filepath.Join(b.git.Path(), manifests.Path)} + if _, err = utils.ExecKubectlCommand(ctx, utils.ModeStderrOS, b.kubeconfig, b.kubecontext, kubectlArgs...); err != nil { + return err + } + b.logger.Successf("installed components") + } + + b.logger.Successf("reconciled components") + return nil +} + +func (b *PlainGitBootstrapper) ReconcileSourceSecret(ctx context.Context, options sourcesecret.Options) error { + // Determine if there is an existing secret + secretKey := client.ObjectKey{Name: options.Name, Namespace: options.Namespace} + b.logger.Actionf("determining if source secret %q exists", secretKey) + ok, err := secretExists(ctx, b.kube, secretKey) + if err != nil { + return fmt.Errorf("failed to determine if deploy key secret exists: %w", err) + } + + // Return early if exists and no custom config is passed + if ok && len(options.CAFilePath+options.PrivateKeyPath+options.Username+options.Password) == 0 { + b.logger.Successf("source secret up to date") + return nil + } + + // Generate source secret + b.logger.Actionf("generating source secret") + manifest, err := sourcesecret.Generate(options) + if err != nil { + return err + } + var secret corev1.Secret + if err := yaml.Unmarshal([]byte(manifest.Content), &secret); err != nil { + return fmt.Errorf("failed to unmarshal generated source secret manifest: %w", err) + } + + for _, callback := range b.postGenerateSecret { + if err = callback(ctx, secret, options); err != nil { + return err + } + } + + // Apply source secret + b.logger.Actionf("applying source secret %q", secretKey) + if err = reconcileSecret(ctx, b.kube, secret); err != nil { + return err + } + b.logger.Successf("reconciled source secret") + + return nil +} + +func (b *PlainGitBootstrapper) ReconcileSyncConfig(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error { + // Confirm that sync configuration does not overwrite existing config + if curPath, err := kustomizationPathDiffers(ctx, b.kube, client.ObjectKey{Name: options.Name, Namespace: options.Namespace}, options.TargetPath); err != nil { + return fmt.Errorf("failed to determine if sync configuration would overwrite existing Kustomization: %w", err) + } else if curPath != "" { + return fmt.Errorf("sync path configuration (%q) would overwrite path (%q) of existing Kustomization", options.TargetPath, curPath) + } + + // Clone if not already + if _, err := b.git.Status(); err != nil { + if err == git.ErrNoGitRepository { + b.logger.Actionf("cloning branch %q from Git repository %q", b.branch, b.url) + cloned, err := b.git.Clone(ctx, b.url, b.branch) + if err != nil { + return fmt.Errorf("failed to clone repository: %w", err) + } + if cloned { + b.logger.Successf("cloned repository", b.url) + } + } + return err + } + + // Generate sync manifests and write to Git repository + b.logger.Actionf("generating sync manifests") + manifests, err := sync.Generate(options) + if err != nil { + return fmt.Errorf("sync manifests generation failed: %w", err) + } + if err = b.git.Write(manifests.Path, strings.NewReader(manifests.Content)); err != nil { + return fmt.Errorf("failed to write manifest %q: %w", manifests.Path, err) + } + kusManifests, err := kustomization.Generate(kustomization.Options{ + FileSystem: filesys.MakeFsOnDisk(), + BaseDir: b.git.Path(), + TargetPath: filepath.Dir(manifests.Path), + }) + if err != nil { + return fmt.Errorf("kustomization.yaml generation failed: %w", err) + } + if err = b.git.Write(kusManifests.Path, strings.NewReader(kusManifests.Content)); err != nil { + return fmt.Errorf("failed to write manifest %q: %w", kusManifests.Path, err) + } + b.logger.Successf("generated sync manifests") + + // Git commit generated + commit, err := b.git.Commit(git.Commit{ + Author: b.author, + Message: fmt.Sprintf("Add Flux sync manifests"), + }) + if err != nil && err != git.ErrNoStagedFiles { + return fmt.Errorf("failed to commit sync manifests: %w", err) + } + if err == nil { + b.logger.Successf("committed sync manifests to %q (%q)", b.branch, commit) + b.logger.Actionf("pushing sync manifests to %q", b.url) + if err = b.git.Push(ctx); err != nil { + return fmt.Errorf("failed to push sync manifests: %w", err) + } + } else { + b.logger.Successf("sync manifests are up to date") + } + + // Apply to cluster + b.logger.Actionf("applying sync manifests") + kubectlArgs := []string{"apply", "-k", filepath.Join(b.git.Path(), filepath.Dir(kusManifests.Path))} + if _, err = utils.ExecKubectlCommand(ctx, utils.ModeStderrOS, b.kubeconfig, b.kubecontext, kubectlArgs...); err != nil { + return err + } + b.logger.Successf("applied sync manifests") + + // Wait till Kustomization is reconciled + var k kustomizev1.Kustomization + expectRevision := fmt.Sprintf("%s/%s", options.Branch, commit) + if err := wait.PollImmediate(pollInterval, timeout, kustomizationReconciled( + ctx, b.kube, client.ObjectKey{Name: options.Name, Namespace: options.Namespace}, &k, expectRevision), + ); err != nil { + return fmt.Errorf("failed waiting for Kustomization: %w", err) + } + + b.logger.Successf("reconciled sync configuration") + return nil +} + +func (b *PlainGitBootstrapper) ConfirmHealthy(ctx context.Context, install install.Options, timeout time.Duration) error { + cfg, err := utils.KubeConfig(b.kubeconfig, b.kubecontext) + if err != nil { + return err + } + + checker, err := status.NewStatusChecker(cfg, 2*time.Second, timeout, b.logger) + if err != nil { + return err + } + + var components = install.Components + components = append(components, install.ComponentsExtra...) + + var identifiers []object.ObjMetadata + for _, component := range components { + identifiers = append(identifiers, object.ObjMetadata{ + Namespace: install.Namespace, + Name: component, + GroupKind: schema.GroupKind{Group: "apps", Kind: "Deployment"}, + }) + } + + b.logger.Actionf("confirming components are healthy") + if err := checker.Assess(identifiers...); err != nil { + return err + } + b.logger.Successf("all components are healthy") + return nil +} diff --git a/internal/bootstrap/bootstrap_provider.go b/internal/bootstrap/bootstrap_provider.go new file mode 100644 index 00000000..a6a17d4e --- /dev/null +++ b/internal/bootstrap/bootstrap_provider.go @@ -0,0 +1,530 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "context" + "errors" + "fmt" + "net/url" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/fluxcd/go-git-providers/gitprovider" + + "github.com/fluxcd/flux2/internal/bootstrap/git" + "github.com/fluxcd/flux2/pkg/manifestgen/sourcesecret" + "github.com/fluxcd/flux2/pkg/manifestgen/sync" +) + +type GitProviderBootstrapper struct { + *PlainGitBootstrapper + + owner string + repository string + personal bool + + description string + defaultBranch string + visibility string + + teams map[string]string + + readWriteKey bool + + bootstrapTransportType string + syncTransportType string + + sshHostname string + + provider gitprovider.Client +} + +func NewGitProviderBootstrapper(git git.Git, provider gitprovider.Client, kube client.Client, opts ...GitProviderOption) (*GitProviderBootstrapper, error) { + b := &GitProviderBootstrapper{ + PlainGitBootstrapper: &PlainGitBootstrapper{ + git: git, + kube: kube, + }, + bootstrapTransportType: "https", + syncTransportType: "ssh", + provider: provider, + } + b.PlainGitBootstrapper.postGenerateSecret = append(b.PlainGitBootstrapper.postGenerateSecret, b.reconcileDeployKey) + for _, opt := range opts { + opt.applyGitProvider(b) + } + return b, nil +} + +type GitProviderOption interface { + applyGitProvider(b *GitProviderBootstrapper) +} + +func WithProviderRepository(owner, repository string, personal bool) GitProviderOption { + return providerRepositoryOption{ + owner: owner, + repository: repository, + personal: personal, + } +} + +type providerRepositoryOption struct { + owner string + repository string + personal bool +} + +func (o providerRepositoryOption) applyGitProvider(b *GitProviderBootstrapper) { + b.owner = o.owner + b.repository = o.repository + b.personal = o.personal +} + +func WithProviderRepositoryConfig(description, defaultBranch, visibility string) GitProviderOption { + return providerRepositoryConfigOption{ + description: description, + defaultBranch: defaultBranch, + visibility: visibility, + } +} + +type providerRepositoryConfigOption struct { + description string + defaultBranch string + visibility string +} + +func (o providerRepositoryConfigOption) applyGitProvider(b *GitProviderBootstrapper) { + b.description = o.description + b.defaultBranch = o.defaultBranch + b.visibility = o.visibility +} + +func WithProviderTeamPermissions(teams map[string]string) GitProviderOption { + return providerRepositoryTeamPermissionsOption(teams) +} + +type providerRepositoryTeamPermissionsOption map[string]string + +func (o providerRepositoryTeamPermissionsOption) applyGitProvider(b *GitProviderBootstrapper) { + b.teams = o +} + +func WithReadWriteKeyPermissions(b bool) GitProviderOption { + return withReadWriteKeyPermissionsOption(b) +} + +type withReadWriteKeyPermissionsOption bool + +func (o withReadWriteKeyPermissionsOption) applyGitProvider(b *GitProviderBootstrapper) { + b.readWriteKey = bool(o) +} + +func WithBootstrapTransportType(protocol string) GitProviderOption { + return bootstrapTransportTypeOption(protocol) +} + +type bootstrapTransportTypeOption string + +func (o bootstrapTransportTypeOption) applyGitProvider(b *GitProviderBootstrapper) { + b.bootstrapTransportType = string(o) +} + +func WithSyncTransportType(protocol string) GitProviderOption { + return syncProtocolOption(protocol) +} + +type syncProtocolOption string + +func (o syncProtocolOption) applyGitProvider(b *GitProviderBootstrapper) { + b.syncTransportType = string(o) +} + +func WithSSHHostname(hostname string) GitProviderOption { + return sshHostnameOption(hostname) +} + +type sshHostnameOption string + +func (o sshHostnameOption) applyGitProvider(b *GitProviderBootstrapper) { + b.sshHostname = string(o) +} + +func (b *GitProviderBootstrapper) ReconcileSyncConfig(ctx context.Context, options sync.Options, pollInterval, timeout time.Duration) error { + repo, err := b.getRepository(ctx) + if err != nil { + return err + } + if b.url == "" { + bootstrapURL, err := b.getCloneURL(repo, gitprovider.TransportType(b.bootstrapTransportType)) + if err != nil { + return err + } + WithRepositoryURL(bootstrapURL).applyGit(b.PlainGitBootstrapper) + } + if options.URL == "" { + syncURL, err := b.getCloneURL(repo, gitprovider.TransportType(b.syncTransportType)) + if err != nil { + return err + } + options.URL = syncURL + } + return b.PlainGitBootstrapper.ReconcileSyncConfig(ctx, options, pollInterval, timeout) +} + +// ReconcileRepository reconciles an organization or user repository with the +// GitProviderBootstrapper configuration. On success, the URL in the embedded +// PlainGitBootstrapper is set to clone URL for the configured protocol. +// +// When part of the reconciliation fails with a warning without aborting, an +// ErrReconciledWithWarning error is returned. +func (b *GitProviderBootstrapper) ReconcileRepository(ctx context.Context) error { + var repo gitprovider.UserRepository + var err error + + if b.personal { + repo, err = b.reconcileUserRepository(ctx) + } else { + repo, err = b.reconcileOrgRepository(ctx) + } + if err != nil && !errors.Is(err, ErrReconciledWithWarning) { + return err + } + + cloneURL := repo.Repository().GetCloneURL(gitprovider.TransportType(b.bootstrapTransportType)) + WithRepositoryURL(cloneURL).applyGit(b.PlainGitBootstrapper) + + return err +} + +func (b *GitProviderBootstrapper) reconcileDeployKey(ctx context.Context, secret corev1.Secret, options sourcesecret.Options) error { + ppk, ok := secret.StringData[sourcesecret.PublicKeySecretKey] + if !ok { + return nil + } + b.logger.Successf("public key: %s", strings.TrimSpace(ppk)) + + repo, err := b.getRepository(ctx) + if err != nil { + return err + } + + name := deployKeyName(options.Namespace, b.branch, options.Name, options.TargetPath) + deployKeyInfo := newDeployKeyInfo(name, ppk, b.readWriteKey) + var changed bool + if _, changed, err = repo.DeployKeys().Reconcile(ctx, deployKeyInfo); err != nil { + return err + } + if changed { + b.logger.Successf("configured deploy key %q for %q", deployKeyInfo.Name, repo.Repository().String()) + } + return nil +} + +// reconcileOrgRepository reconciles a gitprovider.OrgRepository +// with the GitProviderBootstrapper values, including any +// gitprovider.TeamAccessInfo configurations. +// +// If one of the gitprovider.TeamAccessInfo does not reconcile +// successfully, the gitprovider.UserRepository and an +// ErrReconciledWithWarning error are returned. +func (b *GitProviderBootstrapper) reconcileOrgRepository(ctx context.Context) (gitprovider.UserRepository, error) { + b.logger.Actionf("connecting to %s", b.provider.SupportedDomain()) + + // Construct the repository and other configuration objects + // go-git-provider likes to work with + subOrgs, repoName := splitSubOrganizationsFromRepositoryName(b.repository) + orgRef := newOrganizationRef(b.provider.SupportedDomain(), b.owner, subOrgs) + repoRef := newOrgRepositoryRef(orgRef, repoName) + repoInfo := newRepositoryInfo(b.description, b.defaultBranch, b.visibility) + + // Reconcile the organization repository + repo, err := b.provider.OrgRepositories().Get(ctx, repoRef) + if err != nil { + if !errors.Is(err, gitprovider.ErrNotFound) { + return nil, fmt.Errorf("failed to get Git repository %q: %w", repoRef.String(), err) + } + // go-git-providers has at present some issues with the idempotency + // of the available Reconcile methods, and setting e.g. the default + // branch correctly. Resort to Create with AutoInit until this has + // been resolved. + repo, err = b.provider.OrgRepositories().Create(ctx, repoRef, repoInfo, &gitprovider.RepositoryCreateOptions{ + AutoInit: gitprovider.BoolVar(true), + }) + if err != nil { + return nil, fmt.Errorf("failed to create new Git repository %q: %w", repoRef.String(), err) + } + b.logger.Successf("repository %q created", repoRef.String()) + } + // Set default branch before calling Reconcile due to bug described + // above. + repoInfo.DefaultBranch = repo.Get().DefaultBranch + var changed bool + if repo, changed, err = b.provider.OrgRepositories().Reconcile(ctx, repoRef, repoInfo); err != nil { + return nil, fmt.Errorf("failed to reconcile Git repository %q: %w", repoRef.String(), err) + } + if changed { + b.logger.Successf("repository %q reconciled", repoRef.String()) + } + + // Build the team access config + teamAccessInfo, err := buildTeamAccessInfo(b.teams, gitprovider.RepositoryPermissionVar(gitprovider.RepositoryPermissionMaintain)) + if err != nil { + return nil, fmt.Errorf("failed to reconcile repository team access: %w", err) + } + + // Reconcile the team access config on best effort (that being: + // record the error as a warning, but continue with the + // reconciliation of the others) + var warning error + if count := len(teamAccessInfo); count > 0 { + b.logger.Actionf("reconciling repository permissions") + for _, i := range teamAccessInfo { + var err error + _, changed, err = repo.TeamAccess().Reconcile(ctx, i) + if err != nil { + warning = fmt.Errorf("failed to grant permissions to team: %w", ErrReconciledWithWarning) + b.logger.Failuref("failed to grant %q permissions to %q: %w", *i.Permission, i.Name, err) + } + if changed { + b.logger.Successf("granted %q permissions to %q", *i.Permission, i.Name) + } + } + b.logger.Successf("reconciled repository permissions") + } + return repo, warning +} + +// reconcileUserRepository reconciles a gitprovider.UserRepository +// with the GitProviderBootstrapper values. It returns the reconciled +// gitprovider.UserRepository, or an error. +func (b *GitProviderBootstrapper) reconcileUserRepository(ctx context.Context) (gitprovider.UserRepository, error) { + b.logger.Actionf("connecting to %s", b.provider.SupportedDomain()) + + // Construct the repository and other metadata objects + // go-git-provider likes to work with. + _, repoName := splitSubOrganizationsFromRepositoryName(b.repository) + userRef := newUserRef(b.provider.SupportedDomain(), b.owner) + repoRef := newUserRepositoryRef(userRef, repoName) + repoInfo := newRepositoryInfo(b.description, b.defaultBranch, b.visibility) + + repo, err := b.provider.UserRepositories().Get(ctx, repoRef) + if err != nil { + if !errors.Is(err, gitprovider.ErrNotFound) { + return nil, fmt.Errorf("failed to get Git repository %q: %w", repoRef.String(), err) + } + // go-git-providers has at present some issues with the idempotency + // of the available Reconcile methods, and setting e.g. the default + // branch correctly. Resort to Create with AutoInit until this has + // been resolved. + repo, err = b.provider.UserRepositories().Create(ctx, repoRef, repoInfo, &gitprovider.RepositoryCreateOptions{ + AutoInit: gitprovider.BoolVar(true), + }) + if err != nil { + return nil, fmt.Errorf("failed to create new Git repository %q: %w", repoRef.String(), err) + } + b.logger.Successf("repository %q created", repoRef.String()) + } + + // Set default branch before calling Reconcile due to bug described + // above. + repoInfo.DefaultBranch = repo.Get().DefaultBranch + var changed bool + if repo, changed, err = b.provider.UserRepositories().Reconcile(ctx, repoRef, repoInfo); err != nil { + return nil, err + } + if changed { + b.logger.Successf("repository %q reconciled", repoRef.String()) + } + return repo, nil +} + +// getRepository retrieves and returns the gitprovider.UserRepository +// for organization and user repositories using the +// GitProviderBootstrapper values. +// As gitprovider.OrgRepository is a superset of gitprovider.UserRepository, this +// type is returned. +func (b *GitProviderBootstrapper) getRepository(ctx context.Context) (gitprovider.UserRepository, error) { + subOrgs, repoName := splitSubOrganizationsFromRepositoryName(b.repository) + + if b.personal { + userRef := newUserRef(b.provider.SupportedDomain(), b.owner) + return b.provider.UserRepositories().Get(ctx, newUserRepositoryRef(userRef, repoName)) + } + + orgRef := newOrganizationRef(b.provider.SupportedDomain(), b.owner, subOrgs) + return b.provider.OrgRepositories().Get(ctx, newOrgRepositoryRef(orgRef, repoName)) +} + +// getCloneURL returns the Git clone URL for the given +// gitprovider.UserRepository. If the given transport type is +// gitprovider.TransportTypeSSH and a custom SSH hostname is configured, +// the hostname of the URL will be modified to this hostname. +func (b *GitProviderBootstrapper) getCloneURL(repository gitprovider.UserRepository, transport gitprovider.TransportType) (string, error) { + u := repository.Repository().GetCloneURL(transport) + var err error + if transport == gitprovider.TransportTypeSSH && b.sshHostname != "" { + if u, err = setHostname(u, b.sshHostname); err != nil { + err = fmt.Errorf("failed to set SSH hostname for URL %q: %w", u, err) + } + } + return u, err +} + +// splitSubOrganizationsFromRepositoryName removes any prefixed sub +// organizations from the given repository name by splitting the +// string into a slice by '/'. +// The last (or only) item of the slice result is assumed to be the +// repository name, other items (nested) sub organizations. +func splitSubOrganizationsFromRepositoryName(name string) ([]string, string) { + elements := strings.Split(name, "/") + i := len(elements) + switch i { + case 1: + return nil, name + default: + return elements[:i-1], elements[i-1] + } +} + +// buildTeamAccessInfo constructs a gitprovider.TeamAccessInfo slice +// from the given string map of team names to permissions. +// +// Providing a default gitprovider.RepositoryPermission is optional, +// and omitting it will make it default to the go-git-provider default. +// +// An error is returned if any of the given permissions is invalid. +func buildTeamAccessInfo(m map[string]string, defaultPermissions *gitprovider.RepositoryPermission) ([]gitprovider.TeamAccessInfo, error) { + var infos []gitprovider.TeamAccessInfo + if defaultPermissions != nil { + if err := gitprovider.ValidateRepositoryPermission(*defaultPermissions); err != nil { + return nil, fmt.Errorf("invalid default team permission %q", *defaultPermissions) + } + } + for n, p := range m { + permission := defaultPermissions + if p != "" { + p := gitprovider.RepositoryPermission(p) + if err := gitprovider.ValidateRepositoryPermission(p); err != nil { + return nil, fmt.Errorf("invalid permission %q for team %q", p, n) + } + permission = &p + } + i := gitprovider.TeamAccessInfo{ + Name: n, + Permission: permission, + } + infos = append(infos, i) + } + return infos, nil +} + +// newOrganizationRef constructs a gitprovider.OrganizationRef with the +// given values and returns the result. +func newOrganizationRef(domain, organization string, subOrganizations []string) gitprovider.OrganizationRef { + return gitprovider.OrganizationRef{ + Domain: domain, + Organization: organization, + SubOrganizations: subOrganizations, + } +} + +// newOrgRepositoryRef constructs a gitprovider.OrgRepositoryRef with +// the given values and returns the result. +func newOrgRepositoryRef(organizationRef gitprovider.OrganizationRef, name string) gitprovider.OrgRepositoryRef { + return gitprovider.OrgRepositoryRef{ + OrganizationRef: organizationRef, + RepositoryName: name, + } +} + +// newUserRef constructs a gitprovider.UserRef with the given values +// and returns the result. +func newUserRef(domain, login string) gitprovider.UserRef { + return gitprovider.UserRef{ + Domain: domain, + UserLogin: login, + } +} + +// newUserRepositoryRef constructs a gitprovider.UserRepositoryRef with +// the given values and returns the result. +func newUserRepositoryRef(userRef gitprovider.UserRef, name string) gitprovider.UserRepositoryRef { + return gitprovider.UserRepositoryRef{ + UserRef: userRef, + RepositoryName: name, + } +} + +// newRepositoryInfo constructs a gitprovider.RepositoryInfo with the +// given values and returns the result. +func newRepositoryInfo(description, defaultBranch, visibility string) gitprovider.RepositoryInfo { + var i gitprovider.RepositoryInfo + if description != "" { + i.Description = gitprovider.StringVar(description) + } + if defaultBranch != "" { + i.DefaultBranch = gitprovider.StringVar(defaultBranch) + } + if visibility != "" { + i.Visibility = gitprovider.RepositoryVisibilityVar(gitprovider.RepositoryVisibility(visibility)) + } + return i +} + +// newDeployKeyInfo constructs a gitprovider.DeployKeyInfo with the +// given values and returns the result. +func newDeployKeyInfo(name, publicKey string, readWrite bool) gitprovider.DeployKeyInfo { + keyInfo := gitprovider.DeployKeyInfo{ + Name: name, + Key: []byte(publicKey), + } + if readWrite { + keyInfo.ReadOnly = gitprovider.BoolVar(false) + } + return keyInfo +} + +func deployKeyName(namespace, secretName, branch, path string) string { + var name string + for _, v := range []string{namespace, secretName, branch, path} { + if v == "" { + continue + } + if name == "" { + name = v + } else { + name = name + "-" + v + } + } + return name +} + +// setHostname is a helper to replace the hostname of the given URL. +// TODO(hidde): support for this should be added in go-git-providers. +func setHostname(URL, hostname string) (string, error) { + u, err := url.Parse(URL) + if err != nil { + return URL, err + } + u.Host = hostname + return u.String(), nil +} diff --git a/internal/bootstrap/git/git.go b/internal/bootstrap/git/git.go new file mode 100644 index 00000000..cbafadf9 --- /dev/null +++ b/internal/bootstrap/git/git.go @@ -0,0 +1,51 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package git + +import ( + "context" + "errors" + "io" +) + +var ( + ErrNoGitRepository = errors.New("no git repository") + ErrNoStagedFiles = errors.New("no staged files") +) + +type Author struct { + Name string + Email string +} + +type Commit struct { + Author + Hash string + Message string +} + +// Git is an interface for basic Git operations on a single branch of a +// remote repository. +type Git interface { + Init(url, branch string) (bool, error) + Clone(ctx context.Context, url, branch string) (bool, error) + Write(path string, reader io.Reader) error + Commit(message Commit) (string, error) + Push(ctx context.Context) error + Status() (bool, error) + Path() string +} diff --git a/internal/bootstrap/git/gogit/gogit.go b/internal/bootstrap/git/gogit/gogit.go new file mode 100644 index 00000000..2060f2f9 --- /dev/null +++ b/internal/bootstrap/git/gogit/gogit.go @@ -0,0 +1,198 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gogit + +import ( + "context" + "fmt" + "io" + "strings" + "time" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/transport" + + "github.com/fluxcd/flux2/internal/bootstrap/git" +) + +type GoGit struct { + path string + auth transport.AuthMethod + repository *gogit.Repository +} + +func New(path string, auth transport.AuthMethod) *GoGit { + return &GoGit{ + path: path, + auth: auth, + } +} + +func (g *GoGit) Init(url, branch string) (bool, error) { + if g.repository != nil { + return false, nil + } + + r, err := gogit.PlainInit(g.path, false) + if err != nil { + return false, err + } + if _, err = r.CreateRemote(&config.RemoteConfig{ + Name: gogit.DefaultRemoteName, + URLs: []string{url}, + }); err != nil { + return false, err + } + branchRef := plumbing.NewBranchReferenceName(branch) + if err = r.CreateBranch(&config.Branch{ + Name: branch, + Remote: gogit.DefaultRemoteName, + Merge: branchRef, + }); err != nil { + return false, err + } + // PlainInit assumes the initial branch to always be master, we can + // overwrite this by setting the reference of the Storer to a new + // symbolic reference (as there are no commits yet) that points + // the HEAD to our new branch. + if err = r.Storer.SetReference(plumbing.NewSymbolicReference(plumbing.HEAD, branchRef)); err != nil { + return false, err + } + + g.repository = r + return true, nil +} + +func (g *GoGit) Clone(ctx context.Context, url, branch string) (bool, error) { + branchRef := plumbing.NewBranchReferenceName(branch) + r, err := gogit.PlainCloneContext(ctx, g.path, false, &gogit.CloneOptions{ + URL: url, + Auth: g.auth, + RemoteName: gogit.DefaultRemoteName, + ReferenceName: branchRef, + SingleBranch: true, + + NoCheckout: false, + Progress: nil, + Tags: gogit.NoTags, + }) + if err != nil { + if err == transport.ErrEmptyRemoteRepository || isRemoteBranchNotFoundErr(err, branchRef.String()) { + return g.Init(url, branch) + } + return false, err + } + + g.repository = r + return true, nil +} + +func (g *GoGit) Write(path string, reader io.Reader) error { + if g.repository == nil { + return git.ErrNoGitRepository + } + + wt, err := g.repository.Worktree() + if err != nil { + return err + } + + f, err := wt.Filesystem.Create(path) + if err != nil { + return err + } + defer f.Close() + + _, err = io.Copy(f, reader) + return err +} + +func (g *GoGit) Commit(message git.Commit) (string, error) { + if g.repository == nil { + return "", git.ErrNoGitRepository + } + + wt, err := g.repository.Worktree() + if err != nil { + return "", err + } + + status, err := wt.Status() + if err != nil { + return "", err + } + if status.IsClean() { + head, err := g.repository.Head() + if err != nil { + return "", err + } + return head.Hash().String(), git.ErrNoStagedFiles + } + if _, err = wt.Add("."); err != nil { + return "", err + } + + commit, err := wt.Commit(message.Message, &gogit.CommitOptions{ + Author: &object.Signature{ + Name: message.Name, + Email: message.Email, + When: time.Now(), + }, + }) + if err != nil { + return "", err + } + return commit.String(), nil +} + +func (g *GoGit) Push(ctx context.Context) error { + if g.repository == nil { + return git.ErrNoGitRepository + } + + return g.repository.PushContext(ctx, &gogit.PushOptions{ + RemoteName: gogit.DefaultRemoteName, + Auth: g.auth, + Progress: nil, + }) +} + +func (g *GoGit) Status() (bool, error) { + if g.repository == nil { + return false, git.ErrNoGitRepository + } + wt, err := g.repository.Worktree() + if err != nil { + return false, err + } + status, err := wt.Status() + if err != nil { + return false, err + } + return status.IsClean(), nil +} + +func (g *GoGit) Path() string { + return g.path +} + +func isRemoteBranchNotFoundErr(err error, ref string) bool { + return strings.Contains(err.Error(), fmt.Sprintf("couldn't find remote ref %q", ref)) +} diff --git a/internal/bootstrap/options.go b/internal/bootstrap/options.go new file mode 100644 index 00000000..9f737771 --- /dev/null +++ b/internal/bootstrap/options.go @@ -0,0 +1,100 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "github.com/fluxcd/flux2/internal/bootstrap/git" + "github.com/fluxcd/flux2/pkg/log" +) + +type Option interface { + GitOption + GitProviderOption +} + +func WithBranch(branch string) Option { + return branchOption(branch) +} + +type branchOption string + +func (o branchOption) applyGit(b *PlainGitBootstrapper) { + b.branch = string(o) +} + +func (o branchOption) applyGitProvider(b *GitProviderBootstrapper) { + o.applyGit(b.PlainGitBootstrapper) +} + +func WithAuthor(name, email string) Option { + return authorOption{ + Name: name, + Email: email, + } +} + +type authorOption git.Author + +func (o authorOption) applyGit(b *PlainGitBootstrapper) { + if o.Name != "" { + b.author.Name = o.Name + } + if o.Email != "" { + b.author.Email = o.Email + } +} + +func (o authorOption) applyGitProvider(b *GitProviderBootstrapper) { + o.applyGit(b.PlainGitBootstrapper) +} + +func WithKubeconfig(kubeconfig, kubecontext string) Option { + return kubeconfigOption{ + kubeconfig: kubeconfig, + kubecontext: kubecontext, + } +} + +type kubeconfigOption struct { + kubeconfig string + kubecontext string +} + +func (o kubeconfigOption) applyGit(b *PlainGitBootstrapper) { + b.kubeconfig = o.kubeconfig + b.kubecontext = o.kubecontext +} + +func (o kubeconfigOption) applyGitProvider(b *GitProviderBootstrapper) { + o.applyGit(b.PlainGitBootstrapper) +} + +func WithLogger(logger log.Logger) Option { + return loggerOption{logger} +} + +type loggerOption struct { + logger log.Logger +} + +func (o loggerOption) applyGit(b *PlainGitBootstrapper) { + b.logger = o.logger +} + +func (o loggerOption) applyGitProvider(b *GitProviderBootstrapper) { + b.logger = o.logger +} diff --git a/internal/bootstrap/provider/factory.go b/internal/bootstrap/provider/factory.go new file mode 100644 index 00000000..7f2d56bb --- /dev/null +++ b/internal/bootstrap/provider/factory.go @@ -0,0 +1,58 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provider + +import ( + "fmt" + + "github.com/fluxcd/go-git-providers/github" + "github.com/fluxcd/go-git-providers/gitlab" + "github.com/fluxcd/go-git-providers/gitprovider" +) + +// BuildGitProvider builds a gitprovider.Client for the provided +// Config. It returns an error if the Config.Provider +// is not supported, or if the construction of the client fails. +func BuildGitProvider(config Config) (gitprovider.Client, error) { + var client gitprovider.Client + var err error + switch config.Provider { + case GitProviderGitHub: + opts := []github.ClientOption{ + github.WithOAuth2Token(config.Token), + } + if config.Hostname != "" { + opts = append(opts, github.WithDomain(config.Hostname)) + } + if client, err = github.NewClient(opts...); err != nil { + return nil, err + } + case GitProviderGitLab: + opts := []gitlab.ClientOption{ + gitlab.WithConditionalRequests(true), + } + if config.Hostname != "" { + opts = append(opts, gitlab.WithDomain(config.Hostname)) + } + if client, err = gitlab.NewClient(config.Token, "", opts...); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unsupported Git provider '%s'", config.Provider) + } + return client, err +} diff --git a/internal/bootstrap/provider/provider.go b/internal/bootstrap/provider/provider.go new file mode 100644 index 00000000..1755e029 --- /dev/null +++ b/internal/bootstrap/provider/provider.go @@ -0,0 +1,39 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provider + +// GitProvider holds a Git provider definition. +type GitProvider string + +const ( + GitProviderGitHub GitProvider = "github" + GitProviderGitLab GitProvider = "gitlab" +) + +// Config defines the configuration for connecting to a GitProvider. +type Config struct { + // Provider defines the GitProvider. + Provider GitProvider + + // Hostname is the HTTP/S hostname of the Provider, + // e.g. github.example.com. + Hostname string + + // Token contains the token used to authenticate with the + // Provider. + Token string +}