From 6d590fe4d2f7bc55cc19043fa21342a3c0cd86e7 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Wed, 29 Apr 2020 23:13:56 +0300 Subject: [PATCH 1/3] Implement delete for git sources - add delete source git command - add delete source git e2e test --- .github/workflows/e2e.yaml | 3 ++ cmd/tk/delete_source.go | 14 ++++++++ cmd/tk/delete_source_git.go | 68 +++++++++++++++++++++++++++++++++++++ cmd/tk/main.go | 29 +++++++++++++--- 4 files changed, 110 insertions(+), 4 deletions(-) create mode 100644 cmd/tk/delete_source.go create mode 100644 cmd/tk/delete_source_git.go diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 99fd2160..004f731f 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -83,6 +83,9 @@ jobs: - name: tk delete kustomization run: | ./bin/tk delete kustomization podinfo --silent + - name: tk delete source git + run: | + ./bin/tk delete source git podinfo --silent - name: tk check run: | ./bin/tk check diff --git a/cmd/tk/delete_source.go b/cmd/tk/delete_source.go new file mode 100644 index 00000000..31e95266 --- /dev/null +++ b/cmd/tk/delete_source.go @@ -0,0 +1,14 @@ +package main + +import ( + "github.com/spf13/cobra" +) + +var deleteSourceCmd = &cobra.Command{ + Use: "source", + Short: "Delete sources commands", +} + +func init() { + deleteCmd.AddCommand(deleteSourceCmd) +} diff --git a/cmd/tk/delete_source_git.go b/cmd/tk/delete_source_git.go new file mode 100644 index 00000000..ea9fb5a1 --- /dev/null +++ b/cmd/tk/delete_source_git.go @@ -0,0 +1,68 @@ +package main + +import ( + "context" + "fmt" + sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1" + + "github.com/manifoldco/promptui" + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/types" +) + +var deleteSourceGitCmd = &cobra.Command{ + Use: "git [name]", + Short: "Delete git source", + RunE: deleteSourceGitCmdRun, +} + +func init() { + deleteSourceCmd.AddCommand(deleteSourceGitCmd) +} + +func deleteSourceGitCmdRun(cmd *cobra.Command, args []string) error { + if len(args) < 1 { + return fmt.Errorf("git name is required") + } + name := args[0] + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + kubeClient, err := utils.kubeClient(kubeconfig) + if err != nil { + return err + } + + namespacedName := types.NamespacedName{ + Namespace: namespace, + Name: name, + } + + var git sourcev1.GitRepository + err = kubeClient.Get(ctx, namespacedName, &git) + if err != nil { + return err + } + + if !deleteSilent { + prompt := promptui.Prompt{ + Label: fmt.Sprintf( + "Are you sure you want to delete the %s source from the %s namespace", name, namespace, + ), + IsConfirm: true, + } + if _, err := prompt.Run(); err != nil { + return fmt.Errorf("aborting") + } + } + + logAction("deleting source %s in %s namespace", name, namespace) + err = kubeClient.Delete(ctx, &git) + if err != nil { + return err + } + logSuccess("source deleted") + + return nil +} diff --git a/cmd/tk/main.go b/cmd/tk/main.go index afd97f65..5ea1f48b 100644 --- a/cmd/tk/main.go +++ b/cmd/tk/main.go @@ -27,14 +27,20 @@ var rootCmd = &cobra.Command{ tk install --version=master # Create a source from a public Git repository - tk create source git webapp \ + tk create source git webapp-latest \ --url=https://github.com/stefanprodan/podinfo \ --branch=master \ --interval=3m + # List git sources and their status + tk get sources git + + # Trigger a git sync + tk sync source git webapp-latest + # Create a kustomization for deploying a series of microservices - tk create kustomization webapp \ - --source=webapp \ + tk create kustomization webapp-dev \ + --source=webapp-latest \ --path="./deploy/webapp/" \ --prune="instance=webapp" \ --generate=true \ @@ -45,7 +51,22 @@ var rootCmd = &cobra.Command{ --health-check-timeout=2m # Trigger a git sync and apply changes if any - sync kustomization webapp --with-source + tk sync kustomization webapp-dev --with-source + + # Suspend a kustomization reconciliation + tk suspend kustomization webapp-dev + + # Resume a kustomization reconciliation + tk resume kustomization webapp-dev + + # Delete a kustomization + tk delete kustomization webapp-dev + + # Delete a git source + tk delete source git webapp-latest + + # Uninstall the toolkit and delete CRDs + tk uninstall --crds `, } From f127adc8ea49f47d4e60a4fc4824c0f8c69d81d2 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 30 Apr 2020 00:12:16 +0300 Subject: [PATCH 2/3] Implement export to YAML - add export commands for git sources and kustomizations - add export e2e tests --- .github/workflows/e2e.yaml | 4 ++ cmd/tk/delete_source_git.go | 2 +- cmd/tk/export.go | 20 ++++++++ cmd/tk/export_kustomization.go | 94 ++++++++++++++++++++++++++++++++++ cmd/tk/export_source.go | 14 +++++ cmd/tk/export_source_git.go | 93 +++++++++++++++++++++++++++++++++ cmd/tk/main.go | 6 +++ go.mod | 1 + 8 files changed, 233 insertions(+), 1 deletion(-) create mode 100644 cmd/tk/export.go create mode 100644 cmd/tk/export_kustomization.go create mode 100644 cmd/tk/export_source.go create mode 100644 cmd/tk/export_source_git.go diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 004f731f..42e9e6eb 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -80,6 +80,10 @@ jobs: - name: tk resume kustomization run: | ./bin/tk resume kustomization podinfo + - name: tk export + run: | + ./bin/tk export source git --all + ./bin/tk export kustomization --all - name: tk delete kustomization run: | ./bin/tk delete kustomization podinfo --silent diff --git a/cmd/tk/delete_source_git.go b/cmd/tk/delete_source_git.go index ea9fb5a1..9ff9bb39 100644 --- a/cmd/tk/delete_source_git.go +++ b/cmd/tk/delete_source_git.go @@ -3,8 +3,8 @@ package main import ( "context" "fmt" - sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1" + sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1" "github.com/manifoldco/promptui" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/types" diff --git a/cmd/tk/export.go b/cmd/tk/export.go new file mode 100644 index 00000000..55aebbec --- /dev/null +++ b/cmd/tk/export.go @@ -0,0 +1,20 @@ +package main + +import ( + "github.com/spf13/cobra" +) + +var exportCmd = &cobra.Command{ + Use: "export", + Short: "Export commands", +} + +var ( + exportAll bool +) + +func init() { + exportCmd.PersistentFlags().BoolVar(&exportAll, "all", false, "select all resources") + + rootCmd.AddCommand(exportCmd) +} diff --git a/cmd/tk/export_kustomization.go b/cmd/tk/export_kustomization.go new file mode 100644 index 00000000..b100297d --- /dev/null +++ b/cmd/tk/export_kustomization.go @@ -0,0 +1,94 @@ +package main + +import ( + "context" + "fmt" + + kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1alpha1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" +) + +var exportKsCmd = &cobra.Command{ + Use: "kustomization [name]", + Aliases: []string{"ks"}, + Short: "Export kustomization in YAML format", + RunE: exportKsCmdRun, +} + +func init() { + exportCmd.AddCommand(exportKsCmd) +} + +func exportKsCmdRun(cmd *cobra.Command, args []string) error { + if !exportAll && len(args) < 1 { + return fmt.Errorf("kustomization name is required") + } + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + kubeClient, err := utils.kubeClient(kubeconfig) + if err != nil { + return err + } + + if exportAll { + var list kustomizev1.KustomizationList + err = kubeClient.List(ctx, &list, client.InNamespace(namespace)) + if err != nil { + return err + } + + if len(list.Items) == 0 { + logFailure("no kustomizations found in %s namespace", namespace) + return nil + } + + for _, kustomization := range list.Items { + if err := exportKs(kustomization); err != nil { + return err + } + } + } else { + name := args[0] + namespacedName := types.NamespacedName{ + Namespace: namespace, + Name: name, + } + var kustomization kustomizev1.Kustomization + err = kubeClient.Get(ctx, namespacedName, &kustomization) + if err != nil { + return err + } + return exportKs(kustomization) + } + return nil +} + +func exportKs(kustomization kustomizev1.Kustomization) error { + gvk := kustomizev1.GroupVersion.WithKind("Kustomization") + export := kustomizev1.Kustomization{ + TypeMeta: metav1.TypeMeta{ + Kind: gvk.Kind, + APIVersion: gvk.GroupVersion().String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: kustomization.Name, + Namespace: kustomization.Namespace, + }, + Spec: kustomization.Spec, + } + + data, err := yaml.Marshal(export) + if err != nil { + return err + } + + fmt.Println("---") + fmt.Println(string(data)) + return nil +} diff --git a/cmd/tk/export_source.go b/cmd/tk/export_source.go new file mode 100644 index 00000000..9412d16b --- /dev/null +++ b/cmd/tk/export_source.go @@ -0,0 +1,14 @@ +package main + +import ( + "github.com/spf13/cobra" +) + +var exportSourceCmd = &cobra.Command{ + Use: "source", + Short: "Export source commands", +} + +func init() { + exportCmd.AddCommand(exportSourceCmd) +} diff --git a/cmd/tk/export_source_git.go b/cmd/tk/export_source_git.go new file mode 100644 index 00000000..18facc57 --- /dev/null +++ b/cmd/tk/export_source_git.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "fmt" + + sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" +) + +var exportSourceGitCmd = &cobra.Command{ + Use: "git [name]", + Short: "Export git source in YAML format", + RunE: exportSourceGitCmdRun, +} + +func init() { + exportSourceCmd.AddCommand(exportSourceGitCmd) +} + +func exportSourceGitCmdRun(cmd *cobra.Command, args []string) error { + if !exportAll && len(args) < 1 { + return fmt.Errorf("kustomization name is required") + } + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + kubeClient, err := utils.kubeClient(kubeconfig) + if err != nil { + return err + } + + if exportAll { + var list sourcev1.GitRepositoryList + err = kubeClient.List(ctx, &list, client.InNamespace(namespace)) + if err != nil { + return err + } + + if len(list.Items) == 0 { + logFailure("no source found in %s namespace", namespace) + return nil + } + + for _, repository := range list.Items { + if err := exportGit(repository); err != nil { + return err + } + } + } else { + name := args[0] + namespacedName := types.NamespacedName{ + Namespace: namespace, + Name: name, + } + var repository sourcev1.GitRepository + err = kubeClient.Get(ctx, namespacedName, &repository) + if err != nil { + return err + } + return exportGit(repository) + } + return nil +} + +func exportGit(source sourcev1.GitRepository) error { + gvk := sourcev1.GroupVersion.WithKind("GitRepository") + export := sourcev1.GitRepository{ + TypeMeta: metav1.TypeMeta{ + Kind: gvk.Kind, + APIVersion: gvk.GroupVersion().String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: source.Name, + Namespace: source.Namespace, + }, + Spec: source.Spec, + } + + data, err := yaml.Marshal(export) + if err != nil { + return err + } + + fmt.Println("---") + fmt.Println(string(data)) + return nil +} diff --git a/cmd/tk/main.go b/cmd/tk/main.go index 5ea1f48b..cedadbd1 100644 --- a/cmd/tk/main.go +++ b/cmd/tk/main.go @@ -38,6 +38,9 @@ var rootCmd = &cobra.Command{ # Trigger a git sync tk sync source git webapp-latest + # Export git sources in YAML format + tk export source git --all > sources.yaml + # Create a kustomization for deploying a series of microservices tk create kustomization webapp-dev \ --source=webapp-latest \ @@ -56,6 +59,9 @@ var rootCmd = &cobra.Command{ # Suspend a kustomization reconciliation tk suspend kustomization webapp-dev + # Export kustomizations in YAML format + tk export kustomization --all > kustomizations.yaml + # Resume a kustomization reconciliation tk resume kustomization webapp-dev diff --git a/go.mod b/go.mod index e5d8063a..867428c3 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( k8s.io/apimachinery v0.18.2 k8s.io/client-go v0.18.2 sigs.k8s.io/controller-runtime v0.6.0 + sigs.k8s.io/yaml v1.2.0 ) // fix AKS auth From 55303625682a826e04a879f28a30611404ca0bb6 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 30 Apr 2020 10:59:23 +0300 Subject: [PATCH 3/3] Add credentials export option for git sources --- cmd/tk/export_kustomization.go | 8 ++++- cmd/tk/export_source.go | 6 ++++ cmd/tk/export_source_git.go | 59 ++++++++++++++++++++++++++++++++-- cmd/tk/utils.go | 2 ++ 4 files changed, 71 insertions(+), 4 deletions(-) diff --git a/cmd/tk/export_kustomization.go b/cmd/tk/export_kustomization.go index b100297d..dbb59b4b 100644 --- a/cmd/tk/export_kustomization.go +++ b/cmd/tk/export_kustomization.go @@ -16,7 +16,13 @@ var exportKsCmd = &cobra.Command{ Use: "kustomization [name]", Aliases: []string{"ks"}, Short: "Export kustomization in YAML format", - RunE: exportKsCmdRun, + Example: ` # Export all kustomizations + export kustomization --all > kustomizations.yaml + + # Export a kustomization + export kustomization my-app > kustomization.yaml +`, + RunE: exportKsCmdRun, } func init() { diff --git a/cmd/tk/export_source.go b/cmd/tk/export_source.go index 9412d16b..78524a98 100644 --- a/cmd/tk/export_source.go +++ b/cmd/tk/export_source.go @@ -9,6 +9,12 @@ var exportSourceCmd = &cobra.Command{ Short: "Export source commands", } +var ( + exportSourceWithCred bool +) + func init() { + exportSourceCmd.PersistentFlags().BoolVar(&exportSourceWithCred, "with-credentials", false, "include credential secrets") + exportCmd.AddCommand(exportSourceCmd) } diff --git a/cmd/tk/export_source_git.go b/cmd/tk/export_source_git.go index 18facc57..bd8535d6 100644 --- a/cmd/tk/export_source_git.go +++ b/cmd/tk/export_source_git.go @@ -6,6 +6,7 @@ import ( sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1" "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -14,8 +15,14 @@ import ( var exportSourceGitCmd = &cobra.Command{ Use: "git [name]", - Short: "Export git source in YAML format", - RunE: exportSourceGitCmdRun, + Short: "Export git sources in YAML format", + Example: ` # Export all git sources + export source git --all > sources.yaml + + # Export a git source including the SSH keys or basic auth credentials + export source git my-private-repo --with-credentials > source.yaml +`, + RunE: exportSourceGitCmdRun, } func init() { @@ -51,6 +58,11 @@ func exportSourceGitCmdRun(cmd *cobra.Command, args []string) error { if err := exportGit(repository); err != nil { return err } + if exportSourceWithCred { + if err := exportGitCredentials(ctx, kubeClient, repository); err != nil { + return err + } + } } } else { name := args[0] @@ -63,7 +75,12 @@ func exportSourceGitCmdRun(cmd *cobra.Command, args []string) error { if err != nil { return err } - return exportGit(repository) + if err := exportGit(repository); err != nil { + return err + } + if exportSourceWithCred { + return exportGitCredentials(ctx, kubeClient, repository) + } } return nil } @@ -91,3 +108,39 @@ func exportGit(source sourcev1.GitRepository) error { fmt.Println(string(data)) return nil } + +func exportGitCredentials(ctx context.Context, kubeClinet client.Client, source sourcev1.GitRepository) error { + if source.Spec.SecretRef != nil { + namespacedName := types.NamespacedName{ + Namespace: source.Namespace, + Name: source.Spec.SecretRef.Name, + } + var cred corev1.Secret + err := kubeClinet.Get(ctx, namespacedName, &cred) + if err != nil { + return fmt.Errorf("get secret failed: %w", err) + } + + exported := corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: namespacedName.Name, + Namespace: namespacedName.Namespace, + }, + Data: cred.Data, + Type: cred.Type, + } + + data, err := yaml.Marshal(exported) + if err != nil { + return err + } + + fmt.Println("---") + fmt.Println(string(data)) + } + return nil +} diff --git a/cmd/tk/utils.go b/cmd/tk/utils.go index c5aaad86..41ae679f 100644 --- a/cmd/tk/utils.go +++ b/cmd/tk/utils.go @@ -12,6 +12,7 @@ import ( kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1alpha1" sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" @@ -98,6 +99,7 @@ func (*Utils) kubeClient(config string) (client.Client, error) { } scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) _ = sourcev1.AddToScheme(scheme) _ = kustomizev1.AddToScheme(scheme)