Implement bucket CRUD commands

2026-02-06 19:05:55 +00:00 · 2020-09-23 12:04:28 +03:00
parent a8f72564f4
commit a02452ccb9
16 changed files with 929 additions and 0 deletions
--- a/cmd/gotk/create_source_bucket.go
+++ b/cmd/gotk/create_source_bucket.go
@@ -0,0 +1,229 @@
+/*
+Copyright 2020 The Flux CD contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/spf13/cobra"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1"
+)
+
+var createSourceBucketCmd = &cobra.Command{
+	Use:   "bucket [name]",
+	Short: "Create or update a Bucket source",
+	Long: `
+The create source bucket command generates a Bucket resource and waits for it to be downloaded.
+For Buckets with static authentication, the credentials are stored in a Kubernetes secret.`,
+	Example: `  # Create a source from a Buckets using static authentication
+  gotk create source bucket podinfo \
+	--bucket-name=podinfo \
+    --endpoint=minio.minio.svc.cluster.local:9000 \
+	--insecure=true \
+	--access-key=myaccesskey \
+	--secret-key=mysecretkey \
+    --interval=10m
+
+  # Create a source from an Amazon S3 Bucket using IAM authentication
+  gotk create source bucket podinfo \
+	--bucket-name=podinfo \
+	--provider=aws \
+    --endpoint=s3.amazonaws.com \
+	--region=us-east-1 \
+    --interval=10m
+`,
+	RunE: createSourceBucketCmdRun,
+}
+
+var (
+	sourceBucketName      string
+	sourceBucketProvider  string
+	sourceBucketEndpoint  string
+	sourceBucketAccessKey string
+	sourceBucketSecretKey string
+	sourceBucketRegion    string
+	sourceBucketInsecure  bool
+)
+
+func init() {
+	createSourceBucketCmd.Flags().StringVar(&sourceBucketProvider, "provider", "generic", "the S3 compatible storage provider name, can be 'generic' or 'aws'")
+	createSourceBucketCmd.Flags().StringVar(&sourceBucketName, "bucket-name", "", "the bucket name")
+	createSourceBucketCmd.Flags().StringVar(&sourceBucketEndpoint, "endpoint", "", "the bucket endpoint address")
+	createSourceBucketCmd.Flags().StringVar(&sourceBucketAccessKey, "access-key", "", "the bucket access key")
+	createSourceBucketCmd.Flags().StringVar(&sourceBucketSecretKey, "secret-key", "", "the bucket secret key")
+	createSourceBucketCmd.Flags().StringVar(&sourceBucketRegion, "region", "", "the bucket region")
+	createSourceBucketCmd.Flags().BoolVar(&sourceBucketInsecure, "insecure", false, "for when connecting to a non-TLS S3 HTTP endpoint")
+
+	createSourceCmd.AddCommand(createSourceBucketCmd)
+}
+
+func createSourceBucketCmdRun(cmd *cobra.Command, args []string) error {
+	if len(args) < 1 {
+		return fmt.Errorf("source name is required")
+	}
+	name := args[0]
+	secretName := fmt.Sprintf("bucket-%s", name)
+
+	if sourceBucketProvider == "" {
+		return fmt.Errorf("provider is required")
+	}
+
+	if sourceBucketName == "" {
+		return fmt.Errorf("bucket-name is required")
+	}
+
+	if sourceBucketEndpoint == "" {
+		return fmt.Errorf("endpoint is required")
+	}
+
+	sourceLabels, err := parseLabels()
+	if err != nil {
+		return err
+	}
+
+	tmpDir, err := ioutil.TempDir("", name)
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpDir)
+
+	bucket := sourcev1.Bucket{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+			Labels:    sourceLabels,
+		},
+		Spec: sourcev1.BucketSpec{
+			BucketName: sourceBucketName,
+			Provider:   sourceBucketProvider,
+			Insecure:   sourceBucketInsecure,
+			Endpoint:   sourceBucketEndpoint,
+			Region:     sourceBucketRegion,
+			Interval: metav1.Duration{
+				Duration: interval,
+			},
+		},
+	}
+
+	if export {
+		return exportBucket(bucket)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	kubeClient, err := utils.kubeClient(kubeconfig)
+	if err != nil {
+		return err
+	}
+
+	logger.Generatef("generating source")
+
+	secret := corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secretName,
+			Namespace: namespace,
+		},
+		StringData: map[string]string{},
+	}
+
+	if sourceBucketAccessKey != "" && sourceBucketSecretKey != "" {
+		secret.StringData["accesskey"] = sourceBucketAccessKey
+		secret.StringData["secretkey"] = sourceBucketSecretKey
+	}
+
+	if len(secret.StringData) > 0 {
+		logger.Actionf("applying secret with the bucket credentials")
+		if err := upsertSecret(ctx, kubeClient, secret); err != nil {
+			return err
+		}
+		bucket.Spec.SecretRef = &corev1.LocalObjectReference{
+			Name: secretName,
+		}
+		logger.Successf("authentication configured")
+	}
+
+	logger.Actionf("applying source")
+	if err := upsertBucket(ctx, kubeClient, bucket); err != nil {
+		return err
+	}
+
+	logger.Waitingf("waiting for download")
+	if err := wait.PollImmediate(pollInterval, timeout,
+		isBucketReady(ctx, kubeClient, name, namespace)); err != nil {
+		return err
+	}
+
+	logger.Successf("download completed")
+
+	namespacedName := types.NamespacedName{
+		Namespace: namespace,
+		Name:      name,
+	}
+	err = kubeClient.Get(ctx, namespacedName, &bucket)
+	if err != nil {
+		return fmt.Errorf("helm index failed: %w", err)
+	}
+
+	if bucket.Status.Artifact != nil {
+		logger.Successf("fetched revision: %s", bucket.Status.Artifact.Revision)
+	} else {
+		return fmt.Errorf("index download failed, artifact not found")
+	}
+
+	return nil
+}
+
+func upsertBucket(ctx context.Context, kubeClient client.Client, bucket sourcev1.Bucket) error {
+	namespacedName := types.NamespacedName{
+		Namespace: bucket.GetNamespace(),
+		Name:      bucket.GetName(),
+	}
+
+	var existing sourcev1.Bucket
+	err := kubeClient.Get(ctx, namespacedName, &existing)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			if err := kubeClient.Create(ctx, &bucket); err != nil {
+				return err
+			} else {
+				logger.Successf("source created")
+				return nil
+			}
+		}
+		return err
+	}
+
+	existing.Labels = bucket.Labels
+	existing.Spec = bucket.Spec
+	if err := kubeClient.Update(ctx, &existing); err != nil {
+		return err
+	}
+
+	logger.Successf("source updated")
+	return nil
+}
--- a/cmd/gotk/delete_source_bucket.go
+++ b/cmd/gotk/delete_source_bucket.go
@@ -0,0 +1,86 @@
+/*
+Copyright 2020 The Flux CD contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"fmt"
+
+	sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1"
+	"github.com/manifoldco/promptui"
+	"github.com/spf13/cobra"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+var deleteSourceBucketCmd = &cobra.Command{
+	Use:   "bucket [name]",
+	Short: "Delete a Bucket source",
+	Long:  "The delete source bucket command deletes the given Bucket from the cluster.",
+	Example: `  # Delete a Bucket source
+  gotk delete source bucket podinfo
+`,
+	RunE: deleteSourceBucketCmdRun,
+}
+
+func init() {
+	deleteSourceCmd.AddCommand(deleteSourceBucketCmd)
+}
+
+func deleteSourceBucketCmdRun(cmd *cobra.Command, args []string) error {
+	if len(args) < 1 {
+		return fmt.Errorf("name is required")
+	}
+	name := args[0]
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	kubeClient, err := utils.kubeClient(kubeconfig)
+	if err != nil {
+		return err
+	}
+
+	namespacedName := types.NamespacedName{
+		Namespace: namespace,
+		Name:      name,
+	}
+
+	var bucket sourcev1.Bucket
+	err = kubeClient.Get(ctx, namespacedName, &bucket)
+	if err != nil {
+		return err
+	}
+
+	if !deleteSilent {
+		prompt := promptui.Prompt{
+			Label:     "Are you sure you want to delete this source",
+			IsConfirm: true,
+		}
+		if _, err := prompt.Run(); err != nil {
+			return fmt.Errorf("aborting")
+		}
+	}
+
+	logger.Actionf("deleting source %s in %s namespace", name, namespace)
+	err = kubeClient.Delete(ctx, &bucket)
+	if err != nil {
+		return err
+	}
+	logger.Successf("source deleted")
+
+	return nil
+}
--- a/cmd/gotk/export_source_bucket.go
+++ b/cmd/gotk/export_source_bucket.go
@@ -0,0 +1,166 @@
+/*
+Copyright 2020 The Flux CD contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/spf13/cobra"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/yaml"
+
+	sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1"
+)
+
+var exportSourceBucketCmd = &cobra.Command{
+	Use:   "bucket [name]",
+	Short: "Export Bucket sources in YAML format",
+	Long:  "The export source git command exports on or all Bucket sources in YAML format.",
+	Example: `  # Export all Bucket sources
+  gotk export source bucket --all > sources.yaml
+
+  # Export a Bucket source including the static credentials
+  gotk export source bucket my-bucket --with-credentials > source.yaml
+`,
+	RunE: exportSourceBucketCmdRun,
+}
+
+func init() {
+	exportSourceCmd.AddCommand(exportSourceBucketCmd)
+}
+
+func exportSourceBucketCmdRun(cmd *cobra.Command, args []string) error {
+	if !exportAll && len(args) < 1 {
+		return fmt.Errorf("name is required")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	kubeClient, err := utils.kubeClient(kubeconfig)
+	if err != nil {
+		return err
+	}
+
+	if exportAll {
+		var list sourcev1.BucketList
+		err = kubeClient.List(ctx, &list, client.InNamespace(namespace))
+		if err != nil {
+			return err
+		}
+
+		if len(list.Items) == 0 {
+			logger.Failuref("no source found in %s namespace", namespace)
+			return nil
+		}
+
+		for _, bucket := range list.Items {
+			if err := exportBucket(bucket); err != nil {
+				return err
+			}
+			if exportSourceWithCred {
+				if err := exportBucketCredentials(ctx, kubeClient, bucket); err != nil {
+					return err
+				}
+			}
+		}
+	} else {
+		name := args[0]
+		namespacedName := types.NamespacedName{
+			Namespace: namespace,
+			Name:      name,
+		}
+		var bucket sourcev1.Bucket
+		err = kubeClient.Get(ctx, namespacedName, &bucket)
+		if err != nil {
+			return err
+		}
+		if err := exportBucket(bucket); err != nil {
+			return err
+		}
+		if exportSourceWithCred {
+			return exportBucketCredentials(ctx, kubeClient, bucket)
+		}
+	}
+	return nil
+}
+
+func exportBucket(source sourcev1.Bucket) error {
+	gvk := sourcev1.GroupVersion.WithKind(sourcev1.BucketKind)
+	export := sourcev1.Bucket{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       gvk.Kind,
+			APIVersion: gvk.GroupVersion().String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        source.Name,
+			Namespace:   source.Namespace,
+			Labels:      source.Labels,
+			Annotations: source.Annotations,
+		},
+		Spec: source.Spec,
+	}
+
+	data, err := yaml.Marshal(export)
+	if err != nil {
+		return err
+	}
+
+	fmt.Println("---")
+	fmt.Println(resourceToString(data))
+	return nil
+}
+
+func exportBucketCredentials(ctx context.Context, kubeClient client.Client, source sourcev1.Bucket) error {
+	if source.Spec.SecretRef != nil {
+		namespacedName := types.NamespacedName{
+			Namespace: source.Namespace,
+			Name:      source.Spec.SecretRef.Name,
+		}
+		var cred corev1.Secret
+		err := kubeClient.Get(ctx, namespacedName, &cred)
+		if err != nil {
+			return fmt.Errorf("failed to retrieve secret %s, error: %w", namespacedName.Name, err)
+		}
+
+		exported := corev1.Secret{
+			TypeMeta: metav1.TypeMeta{
+				APIVersion: "v1",
+				Kind:       "Secret",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      namespacedName.Name,
+				Namespace: namespacedName.Namespace,
+			},
+			Data: cred.Data,
+			Type: cred.Type,
+		}
+
+		data, err := yaml.Marshal(exported)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println("---")
+		fmt.Println(resourceToString(data))
+	}
+	return nil
+}
--- a/cmd/gotk/get_source_bucket.go
+++ b/cmd/gotk/get_source_bucket.go
@@ -0,0 +1,80 @@
+/*
+Copyright 2020 The Flux CD contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+
+	sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1"
+	"github.com/spf13/cobra"
+	corev1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var getSourceBucketCmd = &cobra.Command{
+	Use:   "bucket",
+	Short: "Get Bucket source statuses",
+	Long:  "The get sources bucket command prints the status of the Bucket sources.",
+	Example: `  # List all Buckets and their status
+  gotk get sources bucket
+`,
+	RunE: getSourceBucketCmdRun,
+}
+
+func init() {
+	getSourceCmd.AddCommand(getSourceBucketCmd)
+}
+
+func getSourceBucketCmdRun(cmd *cobra.Command, args []string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	kubeClient, err := utils.kubeClient(kubeconfig)
+	if err != nil {
+		return err
+	}
+
+	var list sourcev1.BucketList
+	err = kubeClient.List(ctx, &list, client.InNamespace(namespace))
+	if err != nil {
+		return err
+	}
+
+	if len(list.Items) == 0 {
+		logger.Failuref("no sources found in %s namespace", namespace)
+		return nil
+	}
+
+	for _, source := range list.Items {
+		isInitialized := false
+		for _, condition := range source.Status.Conditions {
+			if condition.Type == sourcev1.ReadyCondition {
+				if condition.Status != corev1.ConditionFalse {
+					logger.Successf("%s last fetched revision: %s", source.GetName(), source.Status.Artifact.Revision)
+				} else {
+					logger.Failuref("%s %s", source.GetName(), condition.Message)
+				}
+				isInitialized = true
+				break
+			}
+		}
+		if !isInitialized {
+			logger.Failuref("%s is not ready", source.GetName())
+		}
+	}
+	return nil
+}
--- a/cmd/gotk/reconcile_source_bucket.go
+++ b/cmd/gotk/reconcile_source_bucket.go
@@ -0,0 +1,131 @@
+/*
+Copyright 2020 The Flux CD contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/spf13/cobra"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	consts "github.com/fluxcd/pkg/runtime"
+	sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1"
+)
+
+var reconcileSourceBucketCmd = &cobra.Command{
+	Use:   "bucket [name]",
+	Short: "Reconcile a Bucket source",
+	Long:  `The reconcile source command triggers a reconciliation of a Bucket resource and waits for it to finish.`,
+	Example: `  # Trigger a reconciliation for an existing source
+  gotk reconcile source bucket podinfo
+`,
+	RunE: syncSourceBucketCmdRun,
+}
+
+func init() {
+	reconcileSourceCmd.AddCommand(reconcileSourceBucketCmd)
+}
+
+func syncSourceBucketCmdRun(cmd *cobra.Command, args []string) error {
+	if len(args) < 1 {
+		return fmt.Errorf("source name is required")
+	}
+	name := args[0]
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	kubeClient, err := utils.kubeClient(kubeconfig)
+	if err != nil {
+		return err
+	}
+
+	namespacedName := types.NamespacedName{
+		Namespace: namespace,
+		Name:      name,
+	}
+
+	logger.Actionf("annotating source %s in %s namespace", name, namespace)
+	var bucket sourcev1.Bucket
+	err = kubeClient.Get(ctx, namespacedName, &bucket)
+	if err != nil {
+		return err
+	}
+
+	if bucket.Annotations == nil {
+		bucket.Annotations = map[string]string{
+			consts.ReconcileAtAnnotation: time.Now().Format(time.RFC3339Nano),
+		}
+	} else {
+		bucket.Annotations[consts.ReconcileAtAnnotation] = time.Now().Format(time.RFC3339Nano)
+	}
+	if err := kubeClient.Update(ctx, &bucket); err != nil {
+		return err
+	}
+	logger.Successf("source annotated")
+
+	logger.Waitingf("waiting for reconciliation")
+	if err := wait.PollImmediate(pollInterval, timeout,
+		isBucketReady(ctx, kubeClient, name, namespace)); err != nil {
+		return err
+	}
+
+	logger.Successf("bucket reconciliation completed")
+
+	err = kubeClient.Get(ctx, namespacedName, &bucket)
+	if err != nil {
+		return err
+	}
+
+	if bucket.Status.Artifact != nil {
+		logger.Successf("fetched revision %s", bucket.Status.Artifact.Revision)
+	} else {
+		return fmt.Errorf("bucket reconciliation failed, artifact not found")
+	}
+	return nil
+}
+
+func isBucketReady(ctx context.Context, kubeClient client.Client, name, namespace string) wait.ConditionFunc {
+	return func() (bool, error) {
+		var bucket sourcev1.Bucket
+		namespacedName := types.NamespacedName{
+			Namespace: namespace,
+			Name:      name,
+		}
+
+		err := kubeClient.Get(ctx, namespacedName, &bucket)
+		if err != nil {
+			return false, err
+		}
+
+		for _, condition := range bucket.Status.Conditions {
+			if condition.Type == sourcev1.ReadyCondition {
+				if condition.Status == corev1.ConditionTrue {
+					return true, nil
+				} else if condition.Status == corev1.ConditionFalse {
+					return false, fmt.Errorf(condition.Message)
+				}
+			}
+		}
+		return false, nil
+	}
+}