diff --git a/.github/runners/README.md b/.github/runners/README.md index a7964234..440c6865 100644 --- a/.github/runners/README.md +++ b/.github/runners/README.md @@ -1,24 +1,32 @@ # Flux ARM64 GitHub runners -The Flux ARM64 end-to-end tests run on Equinix instances provisioned with Docker and GitHub self-hosted runners. +The Flux ARM64 end-to-end tests run on Equinix Metal instances provisioned with Docker and GitHub self-hosted runners. ## Current instances -| Runner | Instance | Region | -|---------------|---------------------|--------| -| equinix-arm-1 | flux-equinix-arm-01 | AMS1 | -| equinix-arm-2 | flux-equinix-arm-01 | AMS1 | -| equinix-arm-3 | flux-equinix-arm-01 | AMS1 | -| equinix-arm-4 | flux-equinix-arm-02 | DFW2 | -| equinix-arm-5 | flux-equinix-arm-02 | DFW2 | -| equinix-arm-6 | flux-equinix-arm-02 | DFW2 | +| Repository | Runner | Instance | Location | +|-----------------------------|------------------|------------------------|---------------| +| flux2 | equinix-arm-dc-1 | flux-equinix-arm-dc-01 | Washington DC | +| flux2 | equinix-arm-dc-2 | flux-equinix-arm-dc-01 | Washington DC | +| flux2 | equinix-arm-da-1 | flux-equinix-arm-da-01 | Dallas | +| flux2 | equinix-arm-da-2 | flux-equinix-arm-da-01 | Dallas | +| source-controller | equinix-arm-dc-1 | flux-equinix-arm-dc-01 | Washington DC | +| source-controller | equinix-arm-da-1 | flux-equinix-arm-da-01 | Dallas | +| image-automation-controller | equinix-arm-dc-1 | flux-equinix-arm-dc-01 | Washington DC | +| image-automation-controller | equinix-arm-da-1 | flux-equinix-arm-da-01 | Dallas | + +Instance spec: +- Ampere Altra Q80-30 80-core processor @ 2.8GHz +- 2 x 960GB NVME +- 256GB RAM +- 2 x 25Gbps ## Instance setup In order to add a new runner to the GitHub Actions pool, first create a server on Equinix with the following configuration: -- Type: c2.large.arm -- OS: Ubuntu 20.04 +- Type: `c3.large.arm64` +- OS: `Ubuntu 22.04 LTS` ### Install prerequisites @@ -54,14 +62,14 @@ sudo ./prereq.sh - Retrieve the GitHub runner token from the repository [settings page](https://github.com/fluxcd/flux2/settings/actions/runners/new?arch=arm64&os=linux) -- Create 3 directories `runner1`, `runner2`, `runner3` +- Create two directories `flux2-01`, `flux2-02` - In each dir run: ```shell curl -sL https://raw.githubusercontent.com/fluxcd/flux2/main/.github/runners/runner-setup.sh > runner-setup.sh \ && chmod +x ./runner-setup.sh -./runner-setup.sh equinix-arm- +./runner-setup.sh equinix-arm- ``` - Reboot the instance diff --git a/.github/runners/prereq.sh b/.github/runners/prereq.sh index 186adb8e..043876eb 100755 --- a/.github/runners/prereq.sh +++ b/.github/runners/prereq.sh @@ -18,11 +18,11 @@ set -eu -KIND_VERSION=0.14.0 +KIND_VERSION=0.17.0 KUBECTL_VERSION=1.24.0 -KUSTOMIZE_VERSION=4.5.4 -HELM_VERSION=3.8.2 -GITHUB_RUNNER_VERSION=2.291.1 +KUSTOMIZE_VERSION=4.5.7 +HELM_VERSION=3.10.1 +GITHUB_RUNNER_VERSION=2.298.2 PACKAGES="apt-transport-https ca-certificates software-properties-common build-essential libssl-dev gnupg lsb-release jq pkg-config" # install prerequisites @@ -31,6 +31,10 @@ apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# fix Kubernetes DNS resolution +rm /etc/resolv.conf +cat "/run/systemd/resolve/stub-resolv.conf" | sed '/search/d' > /etc/resolv.conf + # install docker curl -fsSL https://get.docker.com -o get-docker.sh \ && chmod +x get-docker.sh diff --git a/.github/runners/runner-setup.sh b/.github/runners/runner-setup.sh index cef53faf..b13d87e2 100755 --- a/.github/runners/runner-setup.sh +++ b/.github/runners/runner-setup.sh @@ -22,7 +22,7 @@ RUNNER_NAME=$1 REPOSITORY_TOKEN=$2 REPOSITORY_URL=${3:-https://github.com/fluxcd/flux2} -GITHUB_RUNNER_VERSION=2.285.1 +GITHUB_RUNNER_VERSION=2.298.2 # download runner curl -o actions-runner-linux-arm64.tar.gz -L https://github.com/actions/runner/releases/download/v${GITHUB_RUNNER_VERSION}/actions-runner-linux-arm64-${GITHUB_RUNNER_VERSION}.tar.gz \ diff --git a/.github/workflows/e2e-arm64.yaml b/.github/workflows/e2e-arm64.yaml index bafea2fd..c39e45fe 100644 --- a/.github/workflows/e2e-arm64.yaml +++ b/.github/workflows/e2e-arm64.yaml @@ -3,7 +3,7 @@ name: e2e-arm64 on: workflow_dispatch: push: - branches: [ main, update-components ] + branches: [ main, update-components, e2e-arm64* ] permissions: contents: read @@ -13,6 +13,10 @@ jobs: # Hosted on Equinix # Docs: https://github.com/fluxcd/flux2/tree/main/.github/runners runs-on: [self-hosted, Linux, ARM64, equinix] + strategy: + matrix: + # Keep this list up-to-date with https://endoflife.date/kubernetes + KUBERNETES_VERSION: [ 1.23.13, 1.24.7, 1.25.3 ] steps: - name: Checkout uses: actions/checkout@v3 @@ -23,16 +27,73 @@ jobs: - name: Prepare id: prep run: | - echo ::set-output name=CLUSTER::arm64-${GITHUB_SHA:0:7}-$(date +%s) - echo ::set-output name=CONTEXT::kind-arm64-${GITHUB_SHA:0:7}-$(date +%s) + ID=${GITHUB_SHA:0:7}-${{ matrix.KUBERNETES_VERSION }}-$(date +%s) + echo "CLUSTER=arm64-${ID}" >> $GITHUB_OUTPUT - name: Build run: | make build - name: Setup Kubernetes Kind run: | - kind create cluster --name ${{ steps.prep.outputs.CLUSTER }} --kubeconfig=/tmp/${{ steps.prep.outputs.CLUSTER }} + kind create cluster \ + --wait 5m \ + --name ${{ steps.prep.outputs.CLUSTER }} \ + --kubeconfig=/tmp/${{ steps.prep.outputs.CLUSTER }} \ + --image=kindest/node:v${{ matrix.KUBERNETES_VERSION }} - name: Run e2e tests run: TEST_KUBECONFIG=/tmp/${{ steps.prep.outputs.CLUSTER }} make e2e + - name: Run multi-tenancy tests + env: + KUBECONFIG: /tmp/${{ steps.prep.outputs.CLUSTER }} + run: | + ./bin/flux install + ./bin/flux create source git flux-system \ + --interval=15m \ + --url=https://github.com/fluxcd/flux2-multi-tenancy \ + --branch=main \ + --ignore-paths="./clusters/**/flux-system/" + ./bin/flux create kustomization flux-system \ + --interval=15m \ + --source=flux-system \ + --path=./clusters/staging + kubectl -n flux-system wait kustomization/tenants --for=condition=ready --timeout=5m + kubectl -n apps wait kustomization/dev-team --for=condition=ready --timeout=1m + kubectl -n apps wait helmrelease/podinfo --for=condition=ready --timeout=1m + - name: Run monitoring tests + # Keep this test in sync with https://fluxcd.io/flux/guides/monitoring/ + env: + KUBECONFIG: /tmp/${{ steps.prep.outputs.CLUSTER }} + run: | + ./bin/flux create source git flux-monitoring \ + --interval=30m \ + --url=https://github.com/fluxcd/flux2 \ + --branch=${GITHUB_REF#refs/heads/} + ./bin/flux create kustomization kube-prometheus-stack \ + --interval=1h \ + --prune \ + --source=flux-monitoring \ + --path="./manifests/monitoring/kube-prometheus-stack" \ + --health-check-timeout=5m \ + --wait + ./bin/flux create kustomization monitoring-config \ + --depends-on=kube-prometheus-stack \ + --interval=1h \ + --prune=true \ + --source=flux-monitoring \ + --path="./manifests/monitoring/monitoring-config" \ + --health-check-timeout=1m \ + --wait + kubectl -n flux-system wait kustomization/kube-prometheus-stack --for=condition=ready --timeout=5m + kubectl -n flux-system wait kustomization/monitoring-config --for=condition=ready --timeout=5m + kubectl -n monitoring wait helmrelease/kube-prometheus-stack --for=condition=ready --timeout=1m + - name: Debug failure + if: failure() + env: + KUBECONFIG: /tmp/${{ steps.prep.outputs.CLUSTER }} + run: | + kubectl -n flux-system get all + kubectl -n flux-system describe po + kubectl -n flux-system logs deploy/source-controller + kubectl -n flux-system logs deploy/kustomize-controller - name: Cleanup if: always() run: | diff --git a/cmd/flux/bootstrap_bitbucket_server.go b/cmd/flux/bootstrap_bitbucket_server.go index be8e0045..af93f8f0 100644 --- a/cmd/flux/bootstrap_bitbucket_server.go +++ b/cmd/flux/bootstrap_bitbucket_server.go @@ -212,19 +212,18 @@ func bootstrapBServerCmdRun(cmd *cobra.Command, args []string) error { secretOpts.Username = bServerArgs.username } secretOpts.Password = bitbucketToken - - if bootstrapArgs.caFile != "" { - secretOpts.CAFilePath = bootstrapArgs.caFile - } + secretOpts.CAFile = caBundle } else { + keypair, err := sourcesecret.LoadKeyPairFromPath(bootstrapArgs.privateKeyFile, gitArgs.password) + if err != nil { + return err + } + secretOpts.Keypair = keypair secretOpts.PrivateKeyAlgorithm = sourcesecret.PrivateKeyAlgorithm(bootstrapArgs.keyAlgorithm) secretOpts.RSAKeyBits = int(bootstrapArgs.keyRSABits) secretOpts.ECDSACurve = bootstrapArgs.keyECDSACurve.Curve - secretOpts.SSHHostname = bServerArgs.hostname - if bootstrapArgs.privateKeyFile != "" { - secretOpts.PrivateKeyPath = bootstrapArgs.privateKeyFile - } + secretOpts.SSHHostname = bServerArgs.hostname if bootstrapArgs.sshHostname != "" { secretOpts.SSHHostname = bootstrapArgs.sshHostname } @@ -243,7 +242,13 @@ func bootstrapBServerCmdRun(cmd *cobra.Command, args []string) error { RecurseSubmodules: bootstrapArgs.recurseSubmodules, } + entityList, err := bootstrap.LoadEntityListFromPath(bootstrapArgs.gpgKeyRingPath) + if err != nil { + return err + } + // Bootstrap config + bootstrapOpts := []bootstrap.GitProviderOption{ bootstrap.WithProviderRepository(bServerArgs.owner, bServerArgs.repository, bServerArgs.personal), bootstrap.WithBranch(bootstrapArgs.branch), @@ -255,7 +260,7 @@ func bootstrapBServerCmdRun(cmd *cobra.Command, args []string) error { bootstrap.WithKubeconfig(kubeconfigArgs, kubeclientOptions), bootstrap.WithLogger(logger), bootstrap.WithCABundle(caBundle), - bootstrap.WithGitCommitSigning(bootstrapArgs.gpgKeyRingPath, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID), + bootstrap.WithGitCommitSigning(entityList, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID), } if bootstrapArgs.sshHostname != "" { bootstrapOpts = append(bootstrapOpts, bootstrap.WithSSHHostname(bootstrapArgs.sshHostname)) diff --git a/cmd/flux/bootstrap_git.go b/cmd/flux/bootstrap_git.go index e0da5bb4..8d975749 100644 --- a/cmd/flux/bootstrap_git.go +++ b/cmd/flux/bootstrap_git.go @@ -169,6 +169,15 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error { installOptions.BaseURL = customBaseURL } + var caBundle []byte + if bootstrapArgs.caFile != "" { + var err error + caBundle, err = os.ReadFile(bootstrapArgs.caFile) + if err != nil { + return fmt.Errorf("unable to read TLS CA file: %w", err) + } + } + // Source generation and secret config secretOpts := sourcesecret.Options{ Name: bootstrapArgs.secretName, @@ -179,10 +188,7 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error { if bootstrapArgs.tokenAuth { secretOpts.Username = gitArgs.username secretOpts.Password = gitArgs.password - - if bootstrapArgs.caFile != "" { - secretOpts.CAFilePath = bootstrapArgs.caFile - } + secretOpts.CAFile = caBundle // Remove port of the given host when not syncing over HTTP/S to not assume port for protocol // This _might_ be overwritten later on by e.g. --ssh-hostname @@ -213,9 +219,12 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error { if bootstrapArgs.sshHostname != "" { repositoryURL.Host = bootstrapArgs.sshHostname } - if bootstrapArgs.privateKeyFile != "" { - secretOpts.PrivateKeyPath = bootstrapArgs.privateKeyFile + + keypair, err := sourcesecret.LoadKeyPairFromPath(bootstrapArgs.privateKeyFile, gitArgs.password) + if err != nil { + return err } + secretOpts.Keypair = keypair // Configure last as it depends on the config above. secretOpts.SSHHostname = repositoryURL.Host @@ -235,13 +244,9 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error { RecurseSubmodules: bootstrapArgs.recurseSubmodules, } - var caBundle []byte - if bootstrapArgs.caFile != "" { - var err error - caBundle, err = os.ReadFile(bootstrapArgs.caFile) - if err != nil { - return fmt.Errorf("unable to read TLS CA file: %w", err) - } + entityList, err := bootstrap.LoadEntityListFromPath(bootstrapArgs.gpgKeyRingPath) + if err != nil { + return err } // Bootstrap config @@ -254,7 +259,7 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error { bootstrap.WithPostGenerateSecretFunc(promptPublicKey), bootstrap.WithLogger(logger), bootstrap.WithCABundle(caBundle), - bootstrap.WithGitCommitSigning(bootstrapArgs.gpgKeyRingPath, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID), + bootstrap.WithGitCommitSigning(entityList, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID), } // Setup bootstrapper with constructed configs diff --git a/cmd/flux/bootstrap_github.go b/cmd/flux/bootstrap_github.go index c49bd483..34ad8e2f 100644 --- a/cmd/flux/bootstrap_github.go +++ b/cmd/flux/bootstrap_github.go @@ -204,16 +204,13 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { if bootstrapArgs.tokenAuth { secretOpts.Username = "git" secretOpts.Password = ghToken - - if bootstrapArgs.caFile != "" { - secretOpts.CAFilePath = bootstrapArgs.caFile - } + secretOpts.CAFile = caBundle } else { secretOpts.PrivateKeyAlgorithm = sourcesecret.PrivateKeyAlgorithm(bootstrapArgs.keyAlgorithm) secretOpts.RSAKeyBits = int(bootstrapArgs.keyRSABits) secretOpts.ECDSACurve = bootstrapArgs.keyECDSACurve.Curve - secretOpts.SSHHostname = githubArgs.hostname + secretOpts.SSHHostname = githubArgs.hostname if bootstrapArgs.sshHostname != "" { secretOpts.SSHHostname = bootstrapArgs.sshHostname } @@ -232,6 +229,11 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { RecurseSubmodules: bootstrapArgs.recurseSubmodules, } + entityList, err := bootstrap.LoadEntityListFromPath(bootstrapArgs.gpgKeyRingPath) + if err != nil { + return err + } + // Bootstrap config bootstrapOpts := []bootstrap.GitProviderOption{ bootstrap.WithProviderRepository(githubArgs.owner, githubArgs.repository, githubArgs.personal), @@ -244,7 +246,7 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error { bootstrap.WithKubeconfig(kubeconfigArgs, kubeclientOptions), bootstrap.WithLogger(logger), bootstrap.WithCABundle(caBundle), - bootstrap.WithGitCommitSigning(bootstrapArgs.gpgKeyRingPath, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID), + bootstrap.WithGitCommitSigning(entityList, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID), } if bootstrapArgs.sshHostname != "" { bootstrapOpts = append(bootstrapOpts, bootstrap.WithSSHHostname(bootstrapArgs.sshHostname)) diff --git a/cmd/flux/bootstrap_gitlab.go b/cmd/flux/bootstrap_gitlab.go index 86f498c6..cb38cdae 100644 --- a/cmd/flux/bootstrap_gitlab.go +++ b/cmd/flux/bootstrap_gitlab.go @@ -215,19 +215,18 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error { if bootstrapArgs.tokenAuth { secretOpts.Username = "git" secretOpts.Password = glToken - - if bootstrapArgs.caFile != "" { - secretOpts.CAFilePath = bootstrapArgs.caFile - } + secretOpts.CAFile = caBundle } else { + keypair, err := sourcesecret.LoadKeyPairFromPath(bootstrapArgs.privateKeyFile, gitArgs.password) + if err != nil { + return err + } + secretOpts.Keypair = keypair secretOpts.PrivateKeyAlgorithm = sourcesecret.PrivateKeyAlgorithm(bootstrapArgs.keyAlgorithm) secretOpts.RSAKeyBits = int(bootstrapArgs.keyRSABits) secretOpts.ECDSACurve = bootstrapArgs.keyECDSACurve.Curve - secretOpts.SSHHostname = gitlabArgs.hostname - if bootstrapArgs.privateKeyFile != "" { - secretOpts.PrivateKeyPath = bootstrapArgs.privateKeyFile - } + secretOpts.SSHHostname = gitlabArgs.hostname if bootstrapArgs.sshHostname != "" { secretOpts.SSHHostname = bootstrapArgs.sshHostname } @@ -246,6 +245,11 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error { RecurseSubmodules: bootstrapArgs.recurseSubmodules, } + entityList, err := bootstrap.LoadEntityListFromPath(bootstrapArgs.gpgKeyRingPath) + if err != nil { + return err + } + // Bootstrap config bootstrapOpts := []bootstrap.GitProviderOption{ bootstrap.WithProviderRepository(gitlabArgs.owner, gitlabArgs.repository, gitlabArgs.personal), @@ -258,7 +262,7 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error { bootstrap.WithKubeconfig(kubeconfigArgs, kubeclientOptions), bootstrap.WithLogger(logger), bootstrap.WithCABundle(caBundle), - bootstrap.WithGitCommitSigning(bootstrapArgs.gpgKeyRingPath, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID), + bootstrap.WithGitCommitSigning(entityList, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID), } if bootstrapArgs.sshHostname != "" { bootstrapOpts = append(bootstrapOpts, bootstrap.WithSSHHostname(bootstrapArgs.sshHostname)) diff --git a/cmd/flux/create_secret_git.go b/cmd/flux/create_secret_git.go index 054c8a98..2948a2e0 100644 --- a/cmd/flux/create_secret_git.go +++ b/cmd/flux/create_secret_git.go @@ -21,6 +21,7 @@ import ( "crypto/elliptic" "fmt" "net/url" + "os" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" @@ -135,8 +136,12 @@ func createSecretGitCmdRun(cmd *cobra.Command, args []string) error { } switch u.Scheme { case "ssh": + keypair, err := sourcesecret.LoadKeyPairFromPath(secretGitArgs.privateKeyFile, secretGitArgs.password) + if err != nil { + return err + } + opts.Keypair = keypair opts.SSHHostname = u.Host - opts.PrivateKeyPath = secretGitArgs.privateKeyFile opts.PrivateKeyAlgorithm = sourcesecret.PrivateKeyAlgorithm(secretGitArgs.keyAlgorithm) opts.RSAKeyBits = int(secretGitArgs.rsaBits) opts.ECDSACurve = secretGitArgs.ecdsaCurve.Curve @@ -147,7 +152,13 @@ func createSecretGitCmdRun(cmd *cobra.Command, args []string) error { } opts.Username = secretGitArgs.username opts.Password = secretGitArgs.password - opts.CAFilePath = secretGitArgs.caFile + if secretGitArgs.caFile != "" { + caBundle, err := os.ReadFile(secretGitArgs.caFile) + if err != nil { + return fmt.Errorf("unable to read TLS CA file: %w", err) + } + opts.CAFile = caBundle + } default: return fmt.Errorf("git URL scheme '%s' not supported, can be: ssh, http and https", u.Scheme) } diff --git a/cmd/flux/create_secret_helm.go b/cmd/flux/create_secret_helm.go index 1928c109..aba9e734 100644 --- a/cmd/flux/create_secret_helm.go +++ b/cmd/flux/create_secret_helm.go @@ -18,6 +18,8 @@ package main import ( "context" + "fmt" + "os" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" @@ -74,15 +76,34 @@ func createSecretHelmCmdRun(cmd *cobra.Command, args []string) error { return err } + caBundle := []byte{} + if secretHelmArgs.caFile != "" { + var err error + caBundle, err = os.ReadFile(secretHelmArgs.caFile) + if err != nil { + return fmt.Errorf("unable to read TLS CA file: %w", err) + } + } + + var certFile, keyFile []byte + if secretHelmArgs.certFile != "" && secretHelmArgs.keyFile != "" { + if certFile, err = os.ReadFile(secretHelmArgs.certFile); err != nil { + return fmt.Errorf("failed to read cert file: %w", err) + } + if keyFile, err = os.ReadFile(secretHelmArgs.keyFile); err != nil { + return fmt.Errorf("failed to read key file: %w", err) + } + } + opts := sourcesecret.Options{ - Name: name, - Namespace: *kubeconfigArgs.Namespace, - Labels: labels, - Username: secretHelmArgs.username, - Password: secretHelmArgs.password, - CAFilePath: secretHelmArgs.caFile, - CertFilePath: secretHelmArgs.certFile, - KeyFilePath: secretHelmArgs.keyFile, + Name: name, + Namespace: *kubeconfigArgs.Namespace, + Labels: labels, + Username: secretHelmArgs.username, + Password: secretHelmArgs.password, + CAFile: caBundle, + CertFile: certFile, + KeyFile: keyFile, } secret, err := sourcesecret.Generate(opts) if err != nil { diff --git a/cmd/flux/create_secret_tls.go b/cmd/flux/create_secret_tls.go index e3afd380..640fef5d 100644 --- a/cmd/flux/create_secret_tls.go +++ b/cmd/flux/create_secret_tls.go @@ -18,6 +18,8 @@ package main import ( "context" + "fmt" + "os" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -73,13 +75,32 @@ func createSecretTLSCmdRun(cmd *cobra.Command, args []string) error { return err } + caBundle := []byte{} + if secretTLSArgs.caFile != "" { + var err error + caBundle, err = os.ReadFile(secretTLSArgs.caFile) + if err != nil { + return fmt.Errorf("unable to read TLS CA file: %w", err) + } + } + + var certFile, keyFile []byte + if secretTLSArgs.certFile != "" && secretTLSArgs.keyFile != "" { + if certFile, err = os.ReadFile(secretTLSArgs.certFile); err != nil { + return fmt.Errorf("failed to read cert file: %w", err) + } + if keyFile, err = os.ReadFile(secretTLSArgs.keyFile); err != nil { + return fmt.Errorf("failed to read key file: %w", err) + } + } + opts := sourcesecret.Options{ - Name: name, - Namespace: *kubeconfigArgs.Namespace, - Labels: labels, - CAFilePath: secretTLSArgs.caFile, - CertFilePath: secretTLSArgs.certFile, - KeyFilePath: secretTLSArgs.keyFile, + Name: name, + Namespace: *kubeconfigArgs.Namespace, + Labels: labels, + CAFile: caBundle, + CertFile: certFile, + KeyFile: keyFile, } secret, err := sourcesecret.Generate(opts) if err != nil { diff --git a/cmd/flux/create_source_git.go b/cmd/flux/create_source_git.go index c5cb9e1a..22306dc5 100644 --- a/cmd/flux/create_source_git.go +++ b/cmd/flux/create_source_git.go @@ -259,16 +259,26 @@ func createSourceGitCmdRun(cmd *cobra.Command, args []string) error { } switch u.Scheme { case "ssh": + keypair, err := sourcesecret.LoadKeyPairFromPath(sourceGitArgs.privateKeyFile, sourceGitArgs.password) + if err != nil { + return err + } + secretOpts.Keypair = keypair secretOpts.SSHHostname = u.Host - secretOpts.PrivateKeyPath = sourceGitArgs.privateKeyFile secretOpts.PrivateKeyAlgorithm = sourcesecret.PrivateKeyAlgorithm(sourceGitArgs.keyAlgorithm) secretOpts.RSAKeyBits = int(sourceGitArgs.keyRSABits) secretOpts.ECDSACurve = sourceGitArgs.keyECDSACurve.Curve secretOpts.Password = sourceGitArgs.password case "https": + if sourceGitArgs.caFile != "" { + caBundle, err := os.ReadFile(sourceGitArgs.caFile) + if err != nil { + return fmt.Errorf("unable to read TLS CA file: %w", err) + } + secretOpts.CAFile = caBundle + } secretOpts.Username = sourceGitArgs.username secretOpts.Password = sourceGitArgs.password - secretOpts.CAFilePath = sourceGitArgs.caFile case "http": logger.Warningf("insecure configuration: credentials configured for an HTTP URL") secretOpts.Username = sourceGitArgs.username diff --git a/cmd/flux/create_source_helm.go b/cmd/flux/create_source_helm.go index 4b56f37a..49a4b026 100644 --- a/cmd/flux/create_source_helm.go +++ b/cmd/flux/create_source_helm.go @@ -168,6 +168,25 @@ func createSourceHelmCmdRun(cmd *cobra.Command, args []string) error { return err } + caBundle := []byte{} + if sourceHelmArgs.caFile != "" { + var err error + caBundle, err = os.ReadFile(sourceHelmArgs.caFile) + if err != nil { + return fmt.Errorf("unable to read TLS CA file: %w", err) + } + } + + var certFile, keyFile []byte + if sourceHelmArgs.certFile != "" && sourceHelmArgs.keyFile != "" { + if certFile, err = os.ReadFile(sourceHelmArgs.certFile); err != nil { + return fmt.Errorf("failed to read cert file: %w", err) + } + if keyFile, err = os.ReadFile(sourceHelmArgs.keyFile); err != nil { + return fmt.Errorf("failed to read key file: %w", err) + } + } + logger.Generatef("generating HelmRepository source") if sourceHelmArgs.secretRef == "" { secretName := fmt.Sprintf("helm-%s", name) @@ -176,9 +195,9 @@ func createSourceHelmCmdRun(cmd *cobra.Command, args []string) error { Namespace: *kubeconfigArgs.Namespace, Username: sourceHelmArgs.username, Password: sourceHelmArgs.password, - CertFilePath: sourceHelmArgs.certFile, - KeyFilePath: sourceHelmArgs.keyFile, - CAFilePath: sourceHelmArgs.caFile, + CAFile: caBundle, + CertFile: certFile, + KeyFile: keyFile, ManifestFile: sourcesecret.MakeDefaultOptions().ManifestFile, } secret, err := sourcesecret.Generate(secretOpts) diff --git a/manifests/monitoring/kube-prometheus-stack/release.yaml b/manifests/monitoring/kube-prometheus-stack/release.yaml index 13aa4952..30b87357 100644 --- a/manifests/monitoring/kube-prometheus-stack/release.yaml +++ b/manifests/monitoring/kube-prometheus-stack/release.yaml @@ -6,11 +6,13 @@ spec: interval: 5m chart: spec: - version: "35.x" + version: "41.x" chart: kube-prometheus-stack sourceRef: kind: HelmRepository name: prometheus-community + verify: + provider: cosign interval: 60m install: crds: Create diff --git a/manifests/monitoring/kube-prometheus-stack/repository.yaml b/manifests/monitoring/kube-prometheus-stack/repository.yaml index 49355b53..d2beb6b4 100644 --- a/manifests/monitoring/kube-prometheus-stack/repository.yaml +++ b/manifests/monitoring/kube-prometheus-stack/repository.yaml @@ -4,4 +4,5 @@ metadata: name: prometheus-community spec: interval: 120m - url: https://prometheus-community.github.io/helm-charts + type: oci + url: oci://ghcr.io/prometheus-community/charts diff --git a/pkg/bootstrap/bootstrap_plain_git.go b/pkg/bootstrap/bootstrap_plain_git.go index 909eb3b2..4f82a72e 100644 --- a/pkg/bootstrap/bootstrap_plain_git.go +++ b/pkg/bootstrap/bootstrap_plain_git.go @@ -24,6 +24,7 @@ import ( "strings" "time" + "github.com/ProtonMail/go-crypto/openpgp" gogit "github.com/go-git/go-git/v5" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -56,9 +57,9 @@ type PlainGitBootstrapper struct { author git.Author commitMessageAppendix string - gpgKeyRingPath string - gpgPassphrase string - gpgKeyID string + gpgKeyRing openpgp.EntityList + gpgPassphrase string + gpgKeyID string restClientGetter genericclioptions.RESTClientGetter restClientOptions *runclient.Options @@ -139,7 +140,7 @@ func (b *PlainGitBootstrapper) ReconcileComponents(ctx context.Context, manifest } // Git commit generated - gpgOpts := git.WithGpgSigningOption(b.gpgKeyRingPath, b.gpgPassphrase, b.gpgKeyID) + gpgOpts := git.WithGpgSigningOption(b.gpgKeyRing, b.gpgPassphrase, b.gpgKeyID) commitMsg := fmt.Sprintf("Add Flux %s component manifests", options.Version) if b.commitMessageAppendix != "" { commitMsg = commitMsg + "\n\n" + b.commitMessageAppendix @@ -195,7 +196,7 @@ func (b *PlainGitBootstrapper) ReconcileSourceSecret(ctx context.Context, option } // Return early if exists and no custom config is passed - if ok && len(options.CAFilePath+options.PrivateKeyPath+options.Username+options.Password) == 0 { + if ok && options.Keypair == nil && len(options.CAFile) == 0 && len(options.Username+options.Password) == 0 { b.logger.Successf("source secret up to date") return nil } @@ -284,7 +285,7 @@ func (b *PlainGitBootstrapper) ReconcileSyncConfig(ctx context.Context, options b.logger.Successf("generated sync manifests") // Git commit generated - gpgOpts := git.WithGpgSigningOption(b.gpgKeyRingPath, b.gpgPassphrase, b.gpgKeyID) + gpgOpts := git.WithGpgSigningOption(b.gpgKeyRing, b.gpgPassphrase, b.gpgKeyID) commitMsg := fmt.Sprintf("Add Flux sync manifests") if b.commitMessageAppendix != "" { commitMsg = commitMsg + "\n\n" + b.commitMessageAppendix diff --git a/pkg/bootstrap/git/commit_options.go b/pkg/bootstrap/git/commit_options.go index e39614d2..b30491f6 100644 --- a/pkg/bootstrap/git/commit_options.go +++ b/pkg/bootstrap/git/commit_options.go @@ -1,5 +1,9 @@ package git +import ( + "github.com/ProtonMail/go-crypto/openpgp" +) + // Option is a some configuration that modifies options for a commit. type Option interface { // ApplyToCommit applies this configuration to a given commit option. @@ -13,9 +17,9 @@ type CommitOptions struct { // GPGSigningInfo contains information for signing a commit. type GPGSigningInfo struct { - KeyRingPath string - Passphrase string - KeyID string + KeyRing openpgp.EntityList + Passphrase string + KeyID string } type GpgSigningOption struct { @@ -26,17 +30,17 @@ func (w GpgSigningOption) ApplyToCommit(in *CommitOptions) { in.GPGSigningInfo = w.GPGSigningInfo } -func WithGpgSigningOption(path, passphrase, keyID string) Option { +func WithGpgSigningOption(keyRing openpgp.EntityList, passphrase, keyID string) Option { // Return nil if no path is set, even if other options are configured. - if path == "" { + if len(keyRing) == 0 { return GpgSigningOption{} } return GpgSigningOption{ GPGSigningInfo: &GPGSigningInfo{ - KeyRingPath: path, - Passphrase: passphrase, - KeyID: keyID, + KeyRing: keyRing, + Passphrase: passphrase, + KeyID: keyID, }, } } diff --git a/pkg/bootstrap/git/gogit/gogit.go b/pkg/bootstrap/git/gogit/gogit.go index 559633b5..c1c6200a 100644 --- a/pkg/bootstrap/git/gogit/gogit.go +++ b/pkg/bootstrap/git/gogit/gogit.go @@ -258,23 +258,13 @@ func isRemoteBranchNotFoundErr(err error, ref string) bool { } func getOpenPgpEntity(info git.GPGSigningInfo) (*openpgp.Entity, error) { - r, err := os.Open(info.KeyRingPath) - if err != nil { - return nil, fmt.Errorf("unable to open GPG key ring: %w", err) - } - - entityList, err := openpgp.ReadKeyRing(r) - if err != nil { - return nil, err - } - - if len(entityList) == 0 { + if len(info.KeyRing) == 0 { return nil, fmt.Errorf("empty GPG key ring") } var entity *openpgp.Entity if info.KeyID != "" { - for _, ent := range entityList { + for _, ent := range info.KeyRing { if ent.PrimaryKey.KeyIdString() == info.KeyID { entity = ent } @@ -284,10 +274,10 @@ func getOpenPgpEntity(info git.GPGSigningInfo) (*openpgp.Entity, error) { return nil, fmt.Errorf("no GPG private key matching key id '%s' found", info.KeyID) } } else { - entity = entityList[0] + entity = info.KeyRing[0] } - err = entity.PrivateKey.Decrypt([]byte(info.Passphrase)) + err := entity.PrivateKey.Decrypt([]byte(info.Passphrase)) if err != nil { return nil, fmt.Errorf("unable to decrypt GPG private key: %w", err) } diff --git a/pkg/bootstrap/git/gogit/gogit_test.go b/pkg/bootstrap/git/gogit/gogit_test.go index 02c5ea58..03e4545b 100644 --- a/pkg/bootstrap/git/gogit/gogit_test.go +++ b/pkg/bootstrap/git/gogit/gogit_test.go @@ -4,8 +4,10 @@ package gogit import ( + "os" "testing" + "github.com/ProtonMail/go-crypto/openpgp" "github.com/fluxcd/flux2/pkg/bootstrap/git" ) @@ -49,10 +51,21 @@ func TestGetOpenPgpEntity(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + var entityList openpgp.EntityList + if tt.keyPath != "" { + r, err := os.Open(tt.keyPath) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + entityList, err = openpgp.ReadKeyRing(r) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + } gpgInfo := git.GPGSigningInfo{ - KeyRingPath: tt.keyPath, - Passphrase: tt.passphrase, - KeyID: tt.id, + KeyRing: entityList, + Passphrase: tt.passphrase, + KeyID: tt.id, } _, err := getOpenPgpEntity(gpgInfo) diff --git a/pkg/bootstrap/options.go b/pkg/bootstrap/options.go index 9dada56f..be6ce3e8 100644 --- a/pkg/bootstrap/options.go +++ b/pkg/bootstrap/options.go @@ -17,8 +17,12 @@ limitations under the License. package bootstrap import ( + "fmt" + "os" + "k8s.io/cli-runtime/pkg/genericclioptions" + "github.com/ProtonMail/go-crypto/openpgp" runclient "github.com/fluxcd/pkg/runtime/client" "github.com/fluxcd/flux2/pkg/bootstrap/git" @@ -131,22 +135,22 @@ func (o loggerOption) applyGitProvider(b *GitProviderBootstrapper) { b.logger = o.logger } -func WithGitCommitSigning(path, passphrase, keyID string) Option { +func WithGitCommitSigning(gpgKeyRing openpgp.EntityList, passphrase, keyID string) Option { return gitCommitSigningOption{ - gpgKeyRingPath: path, - gpgPassphrase: passphrase, - gpgKeyID: keyID, + gpgKeyRing: gpgKeyRing, + gpgPassphrase: passphrase, + gpgKeyID: keyID, } } type gitCommitSigningOption struct { - gpgKeyRingPath string - gpgPassphrase string - gpgKeyID string + gpgKeyRing openpgp.EntityList + gpgPassphrase string + gpgKeyID string } func (o gitCommitSigningOption) applyGit(b *PlainGitBootstrapper) { - b.gpgKeyRingPath = o.gpgKeyRingPath + b.gpgKeyRing = o.gpgKeyRing b.gpgPassphrase = o.gpgPassphrase b.gpgKeyID = o.gpgKeyID } @@ -154,3 +158,18 @@ func (o gitCommitSigningOption) applyGit(b *PlainGitBootstrapper) { func (o gitCommitSigningOption) applyGitProvider(b *GitProviderBootstrapper) { o.applyGit(b.PlainGitBootstrapper) } + +func LoadEntityListFromPath(path string) (openpgp.EntityList, error) { + if path == "" { + return nil, nil + } + r, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("unable to open GPG key ring: %w", err) + } + entityList, err := openpgp.ReadKeyRing(r) + if err != nil { + return nil, err + } + return entityList, nil +} diff --git a/pkg/manifestgen/sourcesecret/options.go b/pkg/manifestgen/sourcesecret/options.go index 371e6f9b..d9cec044 100644 --- a/pkg/manifestgen/sourcesecret/options.go +++ b/pkg/manifestgen/sourcesecret/options.go @@ -18,6 +18,8 @@ package sourcesecret import ( "crypto/elliptic" + + "github.com/fluxcd/pkg/ssh" ) type PrivateKeyAlgorithm string @@ -48,12 +50,12 @@ type Options struct { PrivateKeyAlgorithm PrivateKeyAlgorithm RSAKeyBits int ECDSACurve elliptic.Curve - PrivateKeyPath string + Keypair *ssh.KeyPair Username string Password string - CAFilePath string - CertFilePath string - KeyFilePath string + CAFile []byte + CertFile []byte + KeyFile []byte TargetPath string ManifestFile string } @@ -64,12 +66,11 @@ func MakeDefaultOptions() Options { Namespace: "flux-system", Labels: map[string]string{}, PrivateKeyAlgorithm: RSAPrivateKeyAlgorithm, - PrivateKeyPath: "", Username: "", Password: "", - CAFilePath: "", - CertFilePath: "", - KeyFilePath: "", + CAFile: []byte{}, + CertFile: []byte{}, + KeyFile: []byte{}, ManifestFile: "secret.yaml", } } diff --git a/pkg/manifestgen/sourcesecret/sourcesecret.go b/pkg/manifestgen/sourcesecret/sourcesecret.go index 87567168..575af3df 100644 --- a/pkg/manifestgen/sourcesecret/sourcesecret.go +++ b/pkg/manifestgen/sourcesecret/sourcesecret.go @@ -66,10 +66,8 @@ func Generate(options Options) (*manifestgen.Manifest, error) { switch { case options.Username != "" && options.Password != "": // noop - case len(options.PrivateKeyPath) > 0: - if keypair, err = loadKeyPair(options.PrivateKeyPath, options.Password); err != nil { - return nil, err - } + case options.Keypair != nil: + keypair = options.Keypair case len(options.PrivateKeyAlgorithm) > 0: if keypair, err = generateKeyPair(options); err != nil { return nil, err @@ -83,23 +81,6 @@ func Generate(options Options) (*manifestgen.Manifest, error) { } } - var caFile []byte - if options.CAFilePath != "" { - if caFile, err = os.ReadFile(options.CAFilePath); err != nil { - return nil, fmt.Errorf("failed to read CA file: %w", err) - } - } - - var certFile, keyFile []byte - if options.CertFilePath != "" && options.KeyFilePath != "" { - if certFile, err = os.ReadFile(options.CertFilePath); err != nil { - return nil, fmt.Errorf("failed to read cert file: %w", err) - } - if keyFile, err = os.ReadFile(options.KeyFilePath); err != nil { - return nil, fmt.Errorf("failed to read key file: %w", err) - } - } - var dockerCfgJson []byte if options.Registry != "" { dockerCfgJson, err = generateDockerConfigJson(options.Registry, options.Username, options.Password) @@ -108,7 +89,7 @@ func Generate(options Options) (*manifestgen.Manifest, error) { } } - secret := buildSecret(keypair, hostKey, caFile, certFile, keyFile, dockerCfgJson, options) + secret := buildSecret(keypair, hostKey, options.CAFile, options.CertFile, options.KeyFile, dockerCfgJson, options) b, err := yaml.Marshal(secret) if err != nil { return nil, err @@ -120,6 +101,35 @@ func Generate(options Options) (*manifestgen.Manifest, error) { }, nil } +func LoadKeyPairFromPath(path, password string) (*ssh.KeyPair, error) { + if path == "" { + return nil, nil + } + + b, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to open private key file: %w", err) + } + return LoadKeyPair(b, password) +} + +func LoadKeyPair(privateKey []byte, password string) (*ssh.KeyPair, error) { + var ppk cryptssh.Signer + var err error + if password != "" { + ppk, err = cryptssh.ParsePrivateKeyWithPassphrase(privateKey, []byte(password)) + } else { + ppk, err = cryptssh.ParsePrivateKey(privateKey) + } + if err != nil { + return nil, err + } + return &ssh.KeyPair{ + PublicKey: cryptssh.MarshalAuthorizedKey(ppk.PublicKey()), + PrivateKey: privateKey, + }, nil +} + func buildSecret(keypair *ssh.KeyPair, hostKey, caFile, certFile, keyFile, dockerCfg []byte, options Options) (secret corev1.Secret) { secret.TypeMeta = metav1.TypeMeta{ APIVersion: "v1", @@ -143,16 +153,16 @@ func buildSecret(keypair *ssh.KeyPair, hostKey, caFile, certFile, keyFile, docke secret.StringData[PasswordSecretKey] = options.Password } - if caFile != nil { + if len(caFile) != 0 { secret.StringData[CAFileSecretKey] = string(caFile) } - if certFile != nil && keyFile != nil { + if len(certFile) != 0 && len(keyFile) != 0 { secret.StringData[CertFileSecretKey] = string(certFile) secret.StringData[KeyFileSecretKey] = string(keyFile) } - if keypair != nil && hostKey != nil { + if keypair != nil && len(hostKey) != 0 { secret.StringData[PrivateKeySecretKey] = string(keypair.PrivateKey) secret.StringData[PublicKeySecretKey] = string(keypair.PublicKey) secret.StringData[KnownHostsSecretKey] = string(hostKey) @@ -165,29 +175,6 @@ func buildSecret(keypair *ssh.KeyPair, hostKey, caFile, certFile, keyFile, docke return } -func loadKeyPair(path string, password string) (*ssh.KeyPair, error) { - b, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("failed to open private key file: %w", err) - } - - var ppk cryptssh.Signer - if password != "" { - ppk, err = cryptssh.ParsePrivateKeyWithPassphrase(b, []byte(password)) - } else { - ppk, err = cryptssh.ParsePrivateKey(b) - } - - if err != nil { - return nil, err - } - - return &ssh.KeyPair{ - PublicKey: cryptssh.MarshalAuthorizedKey(ppk.PublicKey()), - PrivateKey: b, - }, nil -} - func generateKeyPair(options Options) (*ssh.KeyPair, error) { var keyGen ssh.KeyPairGenerator switch options.PrivateKeyAlgorithm { diff --git a/pkg/manifestgen/sourcesecret/sourcesecret_test.go b/pkg/manifestgen/sourcesecret/sourcesecret_test.go index e225bbd8..8eb619d4 100644 --- a/pkg/manifestgen/sourcesecret/sourcesecret_test.go +++ b/pkg/manifestgen/sourcesecret/sourcesecret_test.go @@ -48,7 +48,7 @@ func Test_passwordLoadKeyPair(t *testing.T) { pk, _ := os.ReadFile(tt.privateKeyPath) ppk, _ := os.ReadFile(tt.publicKeyPath) - got, err := loadKeyPair(tt.privateKeyPath, tt.password) + got, err := LoadKeyPair(pk, tt.password) if err != nil { t.Errorf("loadKeyPair() error = %v", err) return @@ -67,24 +67,13 @@ func Test_passwordLoadKeyPair(t *testing.T) { func Test_PasswordlessLoadKeyPair(t *testing.T) { for algo, privateKey := range testdata.PEMBytes { t.Run(algo, func(t *testing.T) { - f, err := os.CreateTemp("", "test-private-key-") - if err != nil { - t.Fatalf("unable to create temporary file. err: %s", err) - } - defer os.Remove(f.Name()) - - if _, err = f.Write(privateKey); err != nil { - t.Fatalf("unable to write private key to file. err: %s", err) - } - - got, err := loadKeyPair(f.Name(), "") + got, err := LoadKeyPair(privateKey, "") if err != nil { t.Errorf("loadKeyPair() error = %v", err) return } - pk, _ := os.ReadFile(f.Name()) - if !reflect.DeepEqual(got.PrivateKey, pk) { + if !reflect.DeepEqual(got.PrivateKey, privateKey) { t.Errorf("PrivateKey %s != %s", got.PrivateKey, string(privateKey)) }