Add terraform files and config for GCP

Signed-off-by: Somtochi Onyekwere <somtochionyekwere@gmail.com>

tests/integration/.env.sample

@@ -13,5 +13,14 @@ export AZUREDEVOPS_SSH_PUB=
 # export ARM_SUBSCRIPTION_ID=
 # export ARM_TENANT_ID=
 
+## GCP
+export TF_VAR_gcp_project_id=
+export TF_VAR_gcp_zone=
+export TF_VAR_gcp_keyring=
+export TF_VAR_gcp_crypto_key=
+## Set the following only when using service account.
+## Provide absolute path to the service account JSON key file.
+# export GOOGLE_APPLICATION_CREDENTIALS=
+
 ## Common variables
 # export TF_VAR_tags='{"environment"="dev", "createdat"='"\"$(date -u +x%Y-%m-%d_%Hh%Mm%Ss)\""'}'

tests/integration/README.md

@@ -55,6 +55,52 @@ the tests:
 - `Microsoft.KeyVault/*`
 - `Microsoft.EventHub/*`
 
+## GCP
+
+### Architecture
+
+The [gcp](./terraform/gcp) terraform files create the GKE cluster and related resources to run the tests. It creates:
+- An Google Container Registry and Artifact Registry
+- An Google Kubernetes Cluster
+- Two Google Cloud Source Repositories
+
+Note: It doesn't create Google KMS keyrings and crypto keys because these cannot be destroyed. Instead, you have
+to pass in the crypto key and keyring that would be used to test the sops encryption in Flux. Please see `.env.sample`
+for the terraform variables
+
+### Requirements
+
+- GCP account with an active project to be able to create GKE and GCR, and permission to assign roles.
+- Existing GCP KMS keyring and crypto key.
+- gcloud CLI, need to be logged in using `gcloud auth login` as a User (not a
+  Service Account), configure application default credentials with `gcloud auth
+  application-default login` and docker credential helper with `gcloud auth configure-docker`.
+
+  **NOTE:** To use Service Account (for example in CI environment), set
+  `GOOGLE_APPLICATION_CREDENTIALS` variable in `.env` with the path to the JSON
+  key file, source it and authenticate gcloud CLI with:
+  ```console
+  $ gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS
+  ```
+  Depending on the Container/Artifact Registry host used in the test, authenticate
+  docker accordingly
+  ```console
+  $ gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://us-central1-docker.pkg.dev
+  $ gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://gcr.io
+  ```
+  In this case, the GCP client in terraform uses the Service Account to
+  authenticate and the gcloud CLI is used only to authenticate with Google
+  Container Registry and Google Artifact Registry.
+
+  **NOTE FOR CI USAGE:** When saving the JSON key file as a CI secret, compress
+  the file content with
+  ```console
+  $ cat key.json | jq -r tostring
+  ```
+  to prevent aggressive masking in the logs. Refer
+  [aggressive replacement in logs](https://github.com/google-github-actions/auth/blob/v1.1.0/docs/TROUBLESHOOTING.md#aggressive--replacement-in-logs)
+  for more details.
+- Register SSH Keys with Google Cloud. # add docs
 
 ## Tests
 

tests/integration/gcp_test.go

@@ -0,0 +1,129 @@
+/*
+Copyright 2023 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/fluxcd/pkg/git"
+	"github.com/fluxcd/test-infra/tftestenv"
+	tfjson "github.com/hashicorp/terraform-json"
+)
+
+const (
+	gkeDevOpsKnownHosts = "[source.developers.google.com]:2022 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBB5Iy4/cq/gt/fPqe3uyMy4jwv1Alc94yVPxmnwNhBzJqEV5gRPiRk5u4/JJMbbu9QUVAguBABxL7sBZa5PH/xY="
+)
+
+// createKubeConfigGKE constructs kubeconfig for a GKE cluster from the
+// terraform state output at the given kubeconfig path.
+func createKubeConfigGKE(ctx context.Context, state map[string]*tfjson.StateOutput, kcPath string) error {
+	kubeconfigYaml, ok := state["kubeconfig"].Value.(string)
+	if !ok || kubeconfigYaml == "" {
+		return fmt.Errorf("failed to obtain kubeconfig from tf output")
+	}
+	return tftestenv.CreateKubeconfigGKE(ctx, kubeconfigYaml, kcPath)
+}
+
+// registryLoginGCR logs into the container/artifact registries using the
+// provider's CLI tools and returns a list of test repositories.
+func registryLoginGCR(ctx context.Context, output map[string]*tfjson.StateOutput) (string, error) {
+	// NOTE: ACR registry accept dynamic repository creation by just pushing a
+	// new image with a new repository name.
+	project := output["gcp_project"].Value.(string)
+	region := output["gcp_region"].Value.(string)
+	repositoryID := output["artifact_registry_id"].Value.(string)
+	artifactRegistryURL, artifactRepoURL := tftestenv.GetGoogleArtifactRegistryAndRepository(project, region, repositoryID)
+	if err := tftestenv.RegistryLoginGCR(ctx, artifactRegistryURL); err != nil {
+		return "", err
+	}
+
+	return artifactRepoURL, nil
+}
+
+func getTestConfigGKE(ctx context.Context, outputs map[string]*tfjson.StateOutput) (*testConfig, error) {
+	sharedSopsId := outputs["sops_id"].Value.(string)
+
+	privateKeyFile, ok := os.LookupEnv("GCP_SOURCEREPO_SSH")
+	if !ok {
+		return nil, fmt.Errorf("GCP_SOURCEREPO_SSH env variable isn't set")
+	}
+	privateKeyData, err := os.ReadFile(privateKeyFile)
+	if err != nil {
+		return nil, fmt.Errorf("error getting gcp source repositories private key, '%s': %w", privateKeyFile, err)
+	}
+
+	pubKeyFile, ok := os.LookupEnv("GCP_SOURCEREPO_SSH_PUB")
+	if !ok {
+		return nil, fmt.Errorf("GCP_SOURCEREPO_SSH_PUB env variable isn't set")
+	}
+	pubKeyData, err := os.ReadFile(pubKeyFile)
+	if err != nil {
+		return nil, fmt.Errorf("error getting ssh pubkey '%s', %w", pubKeyFile, err)
+	}
+
+	config := &testConfig{
+		defaultGitTransport: git.SSH,
+		gitUsername:         "git",
+		gitPrivateKey:       string(privateKeyData),
+		gitPublicKey:        string(pubKeyData),
+		knownHosts:          gkeDevOpsKnownHosts,
+		fleetInfraRepository: repoConfig{
+			ssh: outputs["fleet_infra_url"].Value.(string),
+		},
+		applicationRepository: repoConfig{
+			ssh: outputs["application_url"].Value.(string),
+		},
+		sopsArgs: fmt.Sprintf("--gcp-kms %s", sharedSopsId),
+	}
+
+	opts, err := authOpts(config.fleetInfraRepository.ssh, map[string][]byte{
+		"identity":    []byte(config.gitPrivateKey),
+		"known_hosts": []byte(config.knownHosts),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	config.defaultAuthOpts = opts
+
+	// In Azure, the repository is initialized with a default branch through
+	// terraform. We have to do it manually here for GCP to prevent errors
+	// when trying to clone later. We only need to do it for the application repository
+	// since flux bootstrap pushes to the main branch.
+	files := make(map[string]io.Reader)
+	files["README.md"] = strings.NewReader("# Flux test repo")
+	tmpDir, err := os.MkdirTemp("", "*-flux-test")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(tmpDir)
+
+	client, err := getRepository(context.Background(), tmpDir, config.applicationRepository.ssh, defaultBranch, config.defaultAuthOpts)
+	if err != nil {
+		return nil, err
+	}
+	err = commitAndPushAll(context.Background(), client, files, defaultBranch)
+	if err != nil {
+		return nil, err
+	}
+
+	return config, nil
+}

tests/integration/suite_test.go

@@ -45,6 +45,9 @@ const (
 	// azureTerraformPath is the path to the folder containing the
 	// terraform files for azure infra
 	azureTerraformPath = "./terraform/azure"
+	// gcpTerraformPath is the path to the folder containing the
+	// terraform files for gcp infra
+	gcpTerraformPath = "./terraform/gcp"
 
 	// kubeconfigPath is the path of the file containing the kubeconfig
 	kubeconfigPath = "./build/kubeconfig"
@@ -55,7 +58,7 @@ const (
 
 var (
 	// supportedProviders are the providers supported by the test.
-	supportedProviders = []string{"azure"}
+	supportedProviders = []string{"azure", "gcp"}
 
 	// cfg is a struct containing different variables needed for the test.
 	cfg *testConfig
@@ -269,8 +272,14 @@ func getProviderConfig(provider string) *providerConfig {
 			getTestConfig:    getTestConfigAKS,
 			registryLogin:    registryLoginACR,
 		}
+	case "gcp":
+		return &providerConfig{
+			terraformPath:    gcpTerraformPath,
+			createKubeconfig: createKubeConfigGKE,
+			getTestConfig:    getTestConfigGKE,
+			registryLogin:    registryLoginGCR,
+		}
 	}
-
 	return nil
 }
 

tests/integration/terraform/gcp/main.tf

@@ -0,0 +1,49 @@
+provider "google" {
+  project = var.gcp_project_id
+  region  = var.gcp_region
+  zone    = var.gcp_zone
+}
+
+resource "random_pet" "suffix" {}
+
+data "google_kms_key_ring" "keyring" {
+  name     = var.gcp_keyring
+  location = "global"
+}
+
+data "google_kms_crypto_key" "my_crypto_key" {
+  name     = var.gcp_crypto_key
+  key_ring = data.google_kms_key_ring.keyring.id
+}
+
+data "google_project" "project" {
+}
+
+module "gke" {
+  source = "git::https://github.com/fluxcd/test-infra.git//tf-modules/gcp/gke"
+  name   = "flux-e2e-${random_pet.suffix.id}"
+  tags   = var.tags
+}
+
+module "gcr" {
+  source = "git::https://github.com/fluxcd/test-infra.git//tf-modules/gcp/gcr"
+  name   = "flux-e2e-${random_pet.suffix.id}"
+  tags   = var.tags
+}
+
+resource "google_sourcerepo_repository" "fleet-infra" {
+  name = "fleet-infra-${random_pet.suffix.id}"
+}
+
+resource "google_sourcerepo_repository" "application" {
+  name = "application-${random_pet.suffix.id}"
+}
+
+resource "google_kms_key_ring_iam_binding" "key_ring" {
+  key_ring_id = data.google_kms_key_ring.keyring.id
+  role        = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
+
+  members = [
+    "serviceAccount:${data.google_project.project.number}-compute@developer.gserviceaccount.com",
+  ]
+}

tests/integration/terraform/gcp/outputs.tf

@@ -0,0 +1,28 @@
+output "kubeconfig" {
+  value     = module.gke.kubeconfig
+  sensitive = true
+}
+
+output "gcp_project" {
+  value = var.gcp_project_id
+}
+
+output "gcp_region" {
+  value = var.gcp_region
+}
+
+output "artifact_registry_id" {
+  value = module.gcr.artifact_repository_id
+}
+
+output "sops_id" {
+  value = data.google_kms_crypto_key.my_crypto_key.id
+}
+
+output "fleet_infra_url" {
+  value = "ssh://${var.gcp_email}@source.developers.google.com:2022/p/${var.gcp_project_id}/r/${google_sourcerepo_repository.fleet-infra.name}"
+}
+
+output "application_url" {
+  value = "ssh://${var.gcp_email}@source.developers.google.com:2022/p/${var.gcp_project_id}/r/${google_sourcerepo_repository.application.name}"
+}

tests/integration/terraform/gcp/variables.tf

@@ -0,0 +1,37 @@
+variable "gcp_project_id" {
+  type        = string
+  description = "GCP project to create the resources in"
+}
+
+variable "gcp_email" {
+  type        = string
+  description = "GCP email"
+}
+
+variable "gcp_region" {
+  type        = string
+  default     = "us-central1"
+  description = "GCP region"
+}
+
+variable "gcp_zone" {
+  type        = string
+  default     = "us-central1"
+  description = "GCP region"
+}
+
+
+variable "gcp_keyring" {
+  type        = string
+  description = "GCP keyring that contains crypto key for encrypting secrets"
+}
+
+variable "gcp_crypto_key" {
+  type        = string
+  description = "GCP crypto key for encrypting secrets"
+}
+
+variable "tags" {
+  type    = map(string)
+  default = {}
+}