From 4a4af94d6c4682900d2d089e23a87472ed53476f Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Tue, 20 Apr 2021 13:00:48 +0300 Subject: [PATCH 1/3] Allow pre-bootstrap customisation of Flux components Signed-off-by: Stefan Prodan --- internal/bootstrap/bootstrap_plain_git.go | 49 +++++++++++++++++++++-- internal/utils/utils.go | 49 +++++++++++++++++++++++ 2 files changed, 94 insertions(+), 4 deletions(-) diff --git a/internal/bootstrap/bootstrap_plain_git.go b/internal/bootstrap/bootstrap_plain_git.go index 19fbae18..c932375d 100644 --- a/internal/bootstrap/bootstrap_plain_git.go +++ b/internal/bootstrap/bootstrap_plain_git.go @@ -19,6 +19,8 @@ package bootstrap import ( "context" "fmt" + "io/ioutil" + "os" "path/filepath" "strings" "time" @@ -29,6 +31,7 @@ import ( "sigs.k8s.io/cli-utils/pkg/object" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/kustomize/api/filesys" + "sigs.k8s.io/kustomize/api/konfig" "sigs.k8s.io/yaml" kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1" @@ -152,10 +155,48 @@ func (b *PlainGitBootstrapper) ReconcileComponents(ctx context.Context, manifest // Conditionally install manifests if mustInstallManifests(ctx, b.kube, options.Namespace) { - b.logger.Actionf("installing components in %q namespace", options.Namespace) - kubectlArgs := []string{"apply", "-f", filepath.Join(b.git.Path(), manifests.Path)} - if _, err = utils.ExecKubectlCommand(ctx, utils.ModeStderrOS, b.kubeconfig, b.kubecontext, kubectlArgs...); err != nil { - return err + componentsYAML := filepath.Join(b.git.Path(), manifests.Path) + + // Apply components using any existing customisations + kfile := filepath.Join(filepath.Dir(componentsYAML), konfig.DefaultKustomizationFileName()) + if _, err := os.Stat(kfile); err == nil { + tmpDir, err := ioutil.TempDir("", "gotk-crds") + defer os.RemoveAll(tmpDir) + + // Extract the CRDs from the components manifest + crdsYAML := filepath.Join(tmpDir, "gotk-crds.yaml") + if err := utils.ExtractCRDs(componentsYAML, crdsYAML); err != nil { + return err + } + + // Apply the CRDs + b.logger.Actionf("installing toolkit.fluxcd.io CRDs") + kubectlArgs := []string{"apply", "-f", crdsYAML} + if _, err = utils.ExecKubectlCommand(ctx, utils.ModeStderrOS, b.kubeconfig, b.kubecontext, kubectlArgs...); err != nil { + return err + } + + // Wait for CRDs to be established + b.logger.Waitingf("waiting for CRDs to be reconciled") + kubectlArgs = []string{"wait", "--for", "condition=established", "-f", crdsYAML} + if _, err = utils.ExecKubectlCommand(ctx, utils.ModeStderrOS, b.kubeconfig, b.kubecontext, kubectlArgs...); err != nil { + return err + } + b.logger.Successf("CRDs reconciled successfully") + + // Apply the components and their patches + b.logger.Actionf("installing components in %q namespace", options.Namespace) + kubectlArgs = []string{"apply", "-k", filepath.Dir(componentsYAML)} + if _, err = utils.ExecKubectlCommand(ctx, utils.ModeStderrOS, b.kubeconfig, b.kubecontext, kubectlArgs...); err != nil { + return err + } + } else { + // Apply the CRDs and controllers + b.logger.Actionf("installing components in %q namespace", options.Namespace) + kubectlArgs := []string{"apply", "-f", componentsYAML} + if _, err = utils.ExecKubectlCommand(ctx, utils.ModeStderrOS, b.kubeconfig, b.kubecontext, kubectlArgs...); err != nil { + return err + } } b.logger.Successf("installed components") } diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 5cefc4c4..1229d792 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -22,6 +22,7 @@ import ( "context" "fmt" "io" + "io/ioutil" "os" "os/exec" "path/filepath" @@ -47,6 +48,16 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + helmv2 "github.com/fluxcd/helm-controller/api/v2beta1" + imageautov1 "github.com/fluxcd/image-automation-controller/api/v1alpha1" + imagereflectv1 "github.com/fluxcd/image-reflector-controller/api/v1alpha1" + kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1" + notificationv1 "github.com/fluxcd/notification-controller/api/v1beta1" + "github.com/fluxcd/pkg/runtime/dependency" + "github.com/fluxcd/pkg/version" + sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" "github.com/fluxcd/flux2/pkg/manifestgen/install" ) @@ -313,3 +324,41 @@ func CompatibleVersion(binary, target string) bool { } return binSv.Major() == targetSv.Major() && binSv.Minor() == targetSv.Minor() } + +func ExtractCRDs(inManifestPath, outManifestPath string) error { + manifests, err := ioutil.ReadFile(inManifestPath) + if err != nil { + return err + } + + crds := "" + reader := sigyaml.NewYAMLOrJSONDecoder(bytes.NewReader(manifests), 2048) + + for { + var obj unstructured.Unstructured + err := reader.Decode(&obj) + if err == io.EOF { + break + } else if err != nil { + return err + } + + if obj.GetKind() == "CustomResourceDefinition" { + b, err := obj.MarshalJSON() + if err != nil { + return err + } + y, err := yaml.JSONToYAML(b) + if err != nil { + return err + } + crds += "---\n" + string(y) + } + } + + if crds == "" { + return fmt.Errorf("no CRDs found in %s", inManifestPath) + } + + return ioutil.WriteFile(outManifestPath, []byte(crds), os.ModePerm) +} From 5df9118365eb86c3d9dea610086381fc789802e5 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Tue, 20 Apr 2021 13:40:46 +0300 Subject: [PATCH 2/3] Add pre-bootstrap customisation to install docs Signed-off-by: Stefan Prodan --- docs/guides/installation.md | 42 ++++++++++++++++++++++++++----------- internal/utils/utils.go | 10 ++------- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/docs/guides/installation.md b/docs/guides/installation.md index 086764ea..b528b9e6 100644 --- a/docs/guides/installation.md +++ b/docs/guides/installation.md @@ -337,16 +337,23 @@ please see [fluxcd/terraform-provider-flux](https://github.com/fluxcd/terraform- ## Customize Flux manifests -You can customize the Flux components in the Git repository where you've run bootstrap with Kustomize patches. +You can customize the Flux components before or after running bootstrap. -First clone the repository locally and generate a `kustomization.yaml` file with: +Assuming you want to customise the Flux controllers before they get deployed on the cluster, +first you'll need to create a Git repository and clone it locally. + +Create the file structure required by bootstrap with: ```sh -cd ./clusters/production && kustomize create --autodetect +mkdir -p clusters/my-cluster/flux-system +touch clusters/my-cluster/flux-system/gotk-components.yaml \ + clusters/my-cluster/flux-system/gotk-patches.yaml \ + clusters/my-cluster/flux-system/gotk-sync.yaml \ + clusters/my-cluster/flux-system/kustomization.yaml ``` -Assuming you want to add custom annotations and labels to the Flux controllers in `clusters/production`. -Create a Kustomize patch and set the metadata for source-controller and kustomize-controller pods: +Assuming you want to add custom annotations and labels to the Flux controllers, +edit `clusters/my-cluster/gotk-patches.yaml` and set the metadata for source-controller and kustomize-controller pods: ```yaml apiVersion: apps/v1 @@ -376,26 +383,37 @@ spec: custom: label ``` -Save the above file as `flux-system-patch.yaml` inside the `clusters/production` dir. - -Edit `clusters/production/kustomization.yaml` and add the patch: +Edit `clusters/my-cluster/kustomization.yaml` and set the resources and patches: ```yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - flux-system + - gotk-components.yaml + - gotk-sync.yaml patchesStrategicMerge: - - flux-system-patch.yaml + - gotk-patches.yaml ``` Push the changes to main branch: ```sh -git add -A && git commit -m "add production metadata" && git push +git add -A && git commit -m "add flux customisations" && git push ``` -Flux will detect the change and will update itself on the production cluster. +Now run the bootstrap for `clusters/my-cluster`: + +```sh +flux bootstrap git \ + --url=ssh://git@// \ + --branch=main \ + --path=clusters/my-cluster +``` + +When the controllers are deployed for the first time on your cluster, they will contain all +the customisations from `gotk-patches.yaml`. + +You can make changes to the patches after bootstrap and Flux will apply them in-cluster on its own. ## Dev install diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 1229d792..4a2fc517 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -30,21 +30,15 @@ import ( "strings" "text/template" - helmv2 "github.com/fluxcd/helm-controller/api/v2beta1" - imageautov1 "github.com/fluxcd/image-automation-controller/api/v1alpha2" - imagereflectv1 "github.com/fluxcd/image-reflector-controller/api/v1alpha2" - kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1" - notificationv1 "github.com/fluxcd/notification-controller/api/v1beta1" - "github.com/fluxcd/pkg/runtime/dependency" - "github.com/fluxcd/pkg/version" - sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" "github.com/olekukonko/tablewriter" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" apiruntime "k8s.io/apimachinery/pkg/runtime" + sigyaml "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" From 6aed4631e7f6a551d66fd625d59ff510e093f09f Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 22 Apr 2021 15:26:30 +0200 Subject: [PATCH 3/3] Register v1alpha2 APIs in runtime Scheme Signed-off-by: Hidde Beydals --- internal/utils/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 4a2fc517..c8bdb657 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -45,8 +45,8 @@ import ( "sigs.k8s.io/yaml" helmv2 "github.com/fluxcd/helm-controller/api/v2beta1" - imageautov1 "github.com/fluxcd/image-automation-controller/api/v1alpha1" - imagereflectv1 "github.com/fluxcd/image-reflector-controller/api/v1alpha1" + imageautov1 "github.com/fluxcd/image-automation-controller/api/v1alpha2" + imagereflectv1 "github.com/fluxcd/image-reflector-controller/api/v1alpha2" kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta1" notificationv1 "github.com/fluxcd/notification-controller/api/v1beta1" "github.com/fluxcd/pkg/runtime/dependency"