Merge pull request #1191 from fluxcd/recurse-submodules

Add recurse submodules arg to create source git and bootstrap cmd

.github/workflows/e2e.yaml

@@ -188,7 +188,8 @@ jobs:
         run: |
           /tmp/flux create source git flux-system \
           --url=https://github.com/fluxcd/flux2-kustomize-helm-example \
-          --branch=main
+          --branch=main \
+          --recurse-submodules
           /tmp/flux create kustomization flux-system \
           --source=flux-system \
           --path=./clusters/staging

cmd/flux/bootstrap.go

@@ -39,8 +39,9 @@ type bootstrapFlags struct {
 	arch     flags.Arch
 	logLevel flags.LogLevel
 
-	branch        string
-	manifestsPath string
+	branch            string
+	recurseSubmodules bool
+	manifestsPath     string
 
 	defaultComponents  []string
 	extraComponents    []string
@@ -89,8 +90,10 @@ func init() {
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.imagePullSecret, "image-pull-secret", "",
 		"Kubernetes secret name used for pulling the toolkit images from a private registry")
 
-	bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.branch, "branch", bootstrapDefaultBranch,
-		"default branch (for GitHub this must match the default branch setting for the organization)")
+	bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.branch, "branch", bootstrapDefaultBranch, "Git branch")
+	bootstrapCmd.PersistentFlags().BoolVar(&bootstrapArgs.recurseSubmodules, "recurse-submodules", false,
+		"when enabled, configures the GitRepository source to initialize and include Git submodules in the artifact it produces")
+
 	bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.manifestsPath, "manifests", "", "path to the manifest directory")
 
 	bootstrapCmd.PersistentFlags().BoolVar(&bootstrapArgs.watchAllNamespaces, "watch-all-namespaces", true,

cmd/flux/bootstrap_git.go

@@ -189,6 +189,7 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error {
 		TargetPath:        gitArgs.path.String(),
 		ManifestFile:      sync.MakeDefaultOptions().ManifestFile,
 		GitImplementation: sourceGitArgs.gitImplementation.String(),
+		RecurseSubmodules: bootstrapArgs.recurseSubmodules,
 	}
 
 	// Bootstrap config

cmd/flux/bootstrap_github.go

@@ -211,6 +211,7 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error {
 		TargetPath:        githubArgs.path.String(),
 		ManifestFile:      sync.MakeDefaultOptions().ManifestFile,
 		GitImplementation: sourceGitArgs.gitImplementation.String(),
+		RecurseSubmodules: bootstrapArgs.recurseSubmodules,
 	}
 
 	// Bootstrap config

cmd/flux/bootstrap_gitlab.go

@@ -227,6 +227,7 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error {
 		TargetPath:        gitlabArgs.path.String(),
 		ManifestFile:      sync.MakeDefaultOptions().ManifestFile,
 		GitImplementation: sourceGitArgs.gitImplementation.String(),
+		RecurseSubmodules: bootstrapArgs.recurseSubmodules,
 	}
 
 	// Bootstrap config

cmd/flux/create_source_git.go

@@ -56,6 +56,7 @@ type sourceGitFlags struct {
 	gitImplementation flags.GitImplementation
 	caFile            string
 	privateKeyFile    string
+	recurseSubmodules bool
 }
 
 var createSourceGitCmd = &cobra.Command{
@@ -122,8 +123,10 @@ func init() {
 	createSourceGitCmd.Flags().Var(&sourceGitArgs.keyECDSACurve, "ssh-ecdsa-curve", sourceGitArgs.keyECDSACurve.Description())
 	createSourceGitCmd.Flags().StringVar(&sourceGitArgs.secretRef, "secret-ref", "", "the name of an existing secret containing SSH or basic credentials")
 	createSourceGitCmd.Flags().Var(&sourceGitArgs.gitImplementation, "git-implementation", sourceGitArgs.gitImplementation.Description())
-	createSourceGitCmd.Flags().StringVar(&sourceGitArgs.caFile, "ca-file", "", "path to TLS CA file used for validating self-signed certificates, requires libgit2")
+	createSourceGitCmd.Flags().StringVar(&sourceGitArgs.caFile, "ca-file", "", "path to TLS CA file used for validating self-signed certificates")
 	createSourceGitCmd.Flags().StringVar(&sourceGitArgs.privateKeyFile, "private-key-file", "", "path to a passwordless private key file used for authenticating to the Git SSH server")
+	createSourceGitCmd.Flags().BoolVar(&sourceGitArgs.recurseSubmodules, "recurse-submodules", false,
+		"when enabled, configures the GitRepository source to initialize and include Git submodules in the artifact it produces")
 
 	createSourceCmd.AddCommand(createSourceGitCmd)
 }
@@ -146,16 +149,6 @@ func createSourceGitCmdRun(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("url is required")
 	}
 
-	if sourceGitArgs.gitImplementation.String() != sourcev1.LibGit2Implementation && sourceGitArgs.caFile != "" {
-		return fmt.Errorf("specifing a CA file requires --git-implementation=%s", sourcev1.LibGit2Implementation)
-	}
-
-	tmpDir, err := ioutil.TempDir("", name)
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(tmpDir)
-
 	u, err := url.Parse(sourceGitArgs.url)
 	if err != nil {
 		return fmt.Errorf("git URL parse failed: %w", err)
@@ -164,6 +157,20 @@ func createSourceGitCmdRun(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("git URL scheme '%s' not supported, can be: ssh, http and https", u.Scheme)
 	}
 
+	if sourceGitArgs.caFile != "" && u.Scheme == "ssh" {
+		return fmt.Errorf("specifing a CA file is not supported for Git over SSH")
+	}
+
+	if sourceGitArgs.recurseSubmodules && sourceGitArgs.gitImplementation == sourcev1.LibGit2Implementation {
+		return fmt.Errorf("recurse submodules requires --git-implementation=%s", sourcev1.GoGitImplementation)
+	}
+
+	tmpDir, err := ioutil.TempDir("", name)
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpDir)
+
 	sourceLabels, err := parseLabels()
 	if err != nil {
 		return err
@@ -180,7 +187,8 @@ func createSourceGitCmdRun(cmd *cobra.Command, args []string) error {
 			Interval: metav1.Duration{
 				Duration: createArgs.interval,
 			},
-			Reference: &sourcev1.GitRepositoryRef{},
+			RecurseSubmodules: sourceGitArgs.recurseSubmodules,
+			Reference:         &sourcev1.GitRepositoryRef{},
 		},
 	}
 

docs/cmd/flux_bootstrap.md

@@ -14,7 +14,7 @@ The bootstrap sub-commands bootstrap the toolkit components on the targeted Git
 ```
       --author-email string                    author email for Git commits
       --author-name string                     author name for Git commits (default "Flux")
-      --branch string                          default branch (for GitHub this must match the default branch setting for the organization) (default "main")
+      --branch string                          Git branch (default "main")
       --ca-file string                         path to TLS CA file used for validating self-signed certificates
       --cluster-domain string                  internal cluster domain (default "cluster.local")
       --commit-message-appendix string         string to add to the commit messages, e.g. '[ci skip]'
@@ -25,6 +25,7 @@ The bootstrap sub-commands bootstrap the toolkit components on the targeted Git
       --log-level logLevel                     log level, available options are: (debug, info, error) (default info)
       --network-policy                         deny ingress access to the toolkit controllers from other namespaces using network policies (default true)
       --private-key-file string                path to a private key file used for authenticating to the Git SSH server
+      --recurse-submodules                     when enabled, configures the GitRepository source to initialize and include Git submodules in the artifact it produces
       --registry string                        container registry where the toolkit images are published (default "ghcr.io/fluxcd")
       --secret-name string                     name of the secret the sync credentials can be found in or stored to (default "flux-system")
       --ssh-ecdsa-curve ecdsaCurve             SSH ECDSA public key curve (p256, p384, p521) (default p384)

docs/cmd/flux_bootstrap_git.md

@@ -46,7 +46,7 @@ flux bootstrap git [flags]
 ```
       --author-email string                    author email for Git commits
       --author-name string                     author name for Git commits (default "Flux")
-      --branch string                          default branch (for GitHub this must match the default branch setting for the organization) (default "main")
+      --branch string                          Git branch (default "main")
       --ca-file string                         path to TLS CA file used for validating self-signed certificates
       --cluster-domain string                  internal cluster domain (default "cluster.local")
       --commit-message-appendix string         string to add to the commit messages, e.g. '[ci skip]'
@@ -59,6 +59,7 @@ flux bootstrap git [flags]
   -n, --namespace string                       the namespace scope for this operation (default "flux-system")
       --network-policy                         deny ingress access to the toolkit controllers from other namespaces using network policies (default true)
       --private-key-file string                path to a private key file used for authenticating to the Git SSH server
+      --recurse-submodules                     when enabled, configures the GitRepository source to initialize and include Git submodules in the artifact it produces
       --registry string                        container registry where the toolkit images are published (default "ghcr.io/fluxcd")
       --secret-name string                     name of the secret the sync credentials can be found in or stored to (default "flux-system")
       --ssh-ecdsa-curve ecdsaCurve             SSH ECDSA public key curve (p256, p384, p521) (default p384)

docs/cmd/flux_bootstrap_github.md

@@ -65,7 +65,7 @@ flux bootstrap github [flags]
 ```
       --author-email string                    author email for Git commits
       --author-name string                     author name for Git commits (default "Flux")
-      --branch string                          default branch (for GitHub this must match the default branch setting for the organization) (default "main")
+      --branch string                          Git branch (default "main")
       --ca-file string                         path to TLS CA file used for validating self-signed certificates
       --cluster-domain string                  internal cluster domain (default "cluster.local")
       --commit-message-appendix string         string to add to the commit messages, e.g. '[ci skip]'
@@ -78,6 +78,7 @@ flux bootstrap github [flags]
   -n, --namespace string                       the namespace scope for this operation (default "flux-system")
       --network-policy                         deny ingress access to the toolkit controllers from other namespaces using network policies (default true)
       --private-key-file string                path to a private key file used for authenticating to the Git SSH server
+      --recurse-submodules                     when enabled, configures the GitRepository source to initialize and include Git submodules in the artifact it produces
       --registry string                        container registry where the toolkit images are published (default "ghcr.io/fluxcd")
       --secret-name string                     name of the secret the sync credentials can be found in or stored to (default "flux-system")
       --ssh-ecdsa-curve ecdsaCurve             SSH ECDSA public key curve (p256, p384, p521) (default p384)

docs/cmd/flux_bootstrap_gitlab.md

@@ -62,7 +62,7 @@ flux bootstrap gitlab [flags]
 ```
       --author-email string                    author email for Git commits
       --author-name string                     author name for Git commits (default "Flux")
-      --branch string                          default branch (for GitHub this must match the default branch setting for the organization) (default "main")
+      --branch string                          Git branch (default "main")
       --ca-file string                         path to TLS CA file used for validating self-signed certificates
       --cluster-domain string                  internal cluster domain (default "cluster.local")
       --commit-message-appendix string         string to add to the commit messages, e.g. '[ci skip]'
@@ -75,6 +75,7 @@ flux bootstrap gitlab [flags]
   -n, --namespace string                       the namespace scope for this operation (default "flux-system")
       --network-policy                         deny ingress access to the toolkit controllers from other namespaces using network policies (default true)
       --private-key-file string                path to a private key file used for authenticating to the Git SSH server
+      --recurse-submodules                     when enabled, configures the GitRepository source to initialize and include Git submodules in the artifact it produces
       --registry string                        container registry where the toolkit images are published (default "ghcr.io/fluxcd")
       --secret-name string                     name of the secret the sync credentials can be found in or stored to (default "flux-system")
       --ssh-ecdsa-curve ecdsaCurve             SSH ECDSA public key curve (p256, p384, p521) (default p384)

docs/cmd/flux_create_source_git.md

@@ -65,11 +65,12 @@ flux create source git [name] [flags]
 
 ```
       --branch string                          git branch (default "master")
-      --ca-file string                         path to TLS CA file used for validating self-signed certificates, requires libgit2
+      --ca-file string                         path to TLS CA file used for validating self-signed certificates
       --git-implementation gitImplementation   the Git implementation to use, available options are: (go-git, libgit2)
   -h, --help                                   help for git
   -p, --password string                        basic authentication password
       --private-key-file string                path to a passwordless private key file used for authenticating to the Git SSH server
+      --recurse-submodules                     when enabled, configures the GitRepository source to initialize and include Git submodules in the artifact it produces
       --secret-ref string                      the name of an existing secret containing SSH or basic credentials
       --ssh-ecdsa-curve ecdsaCurve             SSH ECDSA public key curve (p256, p384, p521) (default p384)
       --ssh-key-algorithm publicKeyAlgorithm   SSH public key algorithm (rsa, ecdsa, ed25519) (default rsa)

pkg/manifestgen/sync/options.go

@@ -30,6 +30,7 @@ type Options struct {
 	TargetPath        string
 	ManifestFile      string
 	GitImplementation string
+	RecurseSubmodules bool
 }
 
 func MakeDefaultOptions() Options {

pkg/manifestgen/sync/sync.go

@@ -56,6 +56,7 @@ func Generate(options Options) (*manifestgen.Manifest, error) {
 				Name: options.Secret,
 			},
 			GitImplementation: options.GitImplementation,
+			RecurseSubmodules: options.RecurseSubmodules,
 		},
 	}