diff --git a/tests/.gitignore b/tests/.gitignore index 0acf4705..fda47f9d 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -13,6 +13,7 @@ build/ # .tfstate files *.tfstate *.tfstate.* +*.terraform.lock.hcl # Crash log files crash.log diff --git a/tests/integration/.env.sample b/tests/integration/.env.sample index c77e3ea4..767e9b7e 100644 --- a/tests/integration/.env.sample +++ b/tests/integration/.env.sample @@ -13,5 +13,14 @@ export AZUREDEVOPS_SSH_PUB= # export ARM_SUBSCRIPTION_ID= # export ARM_TENANT_ID= +## GCP +export TF_VAR_gcp_project_id= +export TF_VAR_gcp_zone= +export TF_VAR_gcp_keyring= +export TF_VAR_gcp_crypto_key= +## Set the following only when using service account. +## Provide absolute path to the service account JSON key file. +# export GOOGLE_APPLICATION_CREDENTIALS= + ## Common variables # export TF_VAR_tags='{"environment"="dev", "createdat"='"\"$(date -u +x%Y-%m-%d_%Hh%Mm%Ss)\""'}' diff --git a/tests/integration/README.md b/tests/integration/README.md index 35484802..f4232f1d 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -55,6 +55,52 @@ the tests: - `Microsoft.KeyVault/*` - `Microsoft.EventHub/*` +## GCP + +### Architecture + +The [gcp](./terraform/gcp) terraform files create the GKE cluster and related resources to run the tests. It creates: +- An Google Container Registry and Artifact Registry +- An Google Kubernetes Cluster +- Two Google Cloud Source Repositories + +Note: It doesn't create Google KMS keyrings and crypto keys because these cannot be destroyed. Instead, you have +to pass in the crypto key and keyring that would be used to test the sops encryption in Flux. Please see `.env.sample` +for the terraform variables + +### Requirements + +- GCP account with an active project to be able to create GKE and GCR, and permission to assign roles. +- Existing GCP KMS keyring and crypto key. +- gcloud CLI, need to be logged in using `gcloud auth login` as a User (not a + Service Account), configure application default credentials with `gcloud auth + application-default login` and docker credential helper with `gcloud auth configure-docker`. + + **NOTE:** To use Service Account (for example in CI environment), set + `GOOGLE_APPLICATION_CREDENTIALS` variable in `.env` with the path to the JSON + key file, source it and authenticate gcloud CLI with: + ```console + $ gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS + ``` + Depending on the Container/Artifact Registry host used in the test, authenticate + docker accordingly + ```console + $ gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://us-central1-docker.pkg.dev + $ gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://gcr.io + ``` + In this case, the GCP client in terraform uses the Service Account to + authenticate and the gcloud CLI is used only to authenticate with Google + Container Registry and Google Artifact Registry. + + **NOTE FOR CI USAGE:** When saving the JSON key file as a CI secret, compress + the file content with + ```console + $ cat key.json | jq -r tostring + ``` + to prevent aggressive masking in the logs. Refer + [aggressive replacement in logs](https://github.com/google-github-actions/auth/blob/v1.1.0/docs/TROUBLESHOOTING.md#aggressive--replacement-in-logs) + for more details. +- Register SSH Keys with Google Cloud. # add docs ## Tests diff --git a/tests/integration/gcp_test.go b/tests/integration/gcp_test.go new file mode 100644 index 00000000..02bc6ab4 --- /dev/null +++ b/tests/integration/gcp_test.go @@ -0,0 +1,129 @@ +/* +Copyright 2023 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "fmt" + "io" + "os" + "strings" + + "github.com/fluxcd/pkg/git" + "github.com/fluxcd/test-infra/tftestenv" + tfjson "github.com/hashicorp/terraform-json" +) + +const ( + gkeDevOpsKnownHosts = "[source.developers.google.com]:2022 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBB5Iy4/cq/gt/fPqe3uyMy4jwv1Alc94yVPxmnwNhBzJqEV5gRPiRk5u4/JJMbbu9QUVAguBABxL7sBZa5PH/xY=" +) + +// createKubeConfigGKE constructs kubeconfig for a GKE cluster from the +// terraform state output at the given kubeconfig path. +func createKubeConfigGKE(ctx context.Context, state map[string]*tfjson.StateOutput, kcPath string) error { + kubeconfigYaml, ok := state["kubeconfig"].Value.(string) + if !ok || kubeconfigYaml == "" { + return fmt.Errorf("failed to obtain kubeconfig from tf output") + } + return tftestenv.CreateKubeconfigGKE(ctx, kubeconfigYaml, kcPath) +} + +// registryLoginGCR logs into the container/artifact registries using the +// provider's CLI tools and returns a list of test repositories. +func registryLoginGCR(ctx context.Context, output map[string]*tfjson.StateOutput) (string, error) { + // NOTE: ACR registry accept dynamic repository creation by just pushing a + // new image with a new repository name. + project := output["gcp_project"].Value.(string) + region := output["gcp_region"].Value.(string) + repositoryID := output["artifact_registry_id"].Value.(string) + artifactRegistryURL, artifactRepoURL := tftestenv.GetGoogleArtifactRegistryAndRepository(project, region, repositoryID) + if err := tftestenv.RegistryLoginGCR(ctx, artifactRegistryURL); err != nil { + return "", err + } + + return artifactRepoURL, nil +} + +func getTestConfigGKE(ctx context.Context, outputs map[string]*tfjson.StateOutput) (*testConfig, error) { + sharedSopsId := outputs["sops_id"].Value.(string) + + privateKeyFile, ok := os.LookupEnv("GCP_SOURCEREPO_SSH") + if !ok { + return nil, fmt.Errorf("GCP_SOURCEREPO_SSH env variable isn't set") + } + privateKeyData, err := os.ReadFile(privateKeyFile) + if err != nil { + return nil, fmt.Errorf("error getting gcp source repositories private key, '%s': %w", privateKeyFile, err) + } + + pubKeyFile, ok := os.LookupEnv("GCP_SOURCEREPO_SSH_PUB") + if !ok { + return nil, fmt.Errorf("GCP_SOURCEREPO_SSH_PUB env variable isn't set") + } + pubKeyData, err := os.ReadFile(pubKeyFile) + if err != nil { + return nil, fmt.Errorf("error getting ssh pubkey '%s', %w", pubKeyFile, err) + } + + config := &testConfig{ + defaultGitTransport: git.SSH, + gitUsername: "git", + gitPrivateKey: string(privateKeyData), + gitPublicKey: string(pubKeyData), + knownHosts: gkeDevOpsKnownHosts, + fleetInfraRepository: repoConfig{ + ssh: outputs["fleet_infra_url"].Value.(string), + }, + applicationRepository: repoConfig{ + ssh: outputs["application_url"].Value.(string), + }, + sopsArgs: fmt.Sprintf("--gcp-kms %s", sharedSopsId), + } + + opts, err := authOpts(config.fleetInfraRepository.ssh, map[string][]byte{ + "identity": []byte(config.gitPrivateKey), + "known_hosts": []byte(config.knownHosts), + }) + if err != nil { + return nil, err + } + + config.defaultAuthOpts = opts + + // In Azure, the repository is initialized with a default branch through + // terraform. We have to do it manually here for GCP to prevent errors + // when trying to clone later. We only need to do it for the application repository + // since flux bootstrap pushes to the main branch. + files := make(map[string]io.Reader) + files["README.md"] = strings.NewReader("# Flux test repo") + tmpDir, err := os.MkdirTemp("", "*-flux-test") + if err != nil { + return nil, err + } + defer os.RemoveAll(tmpDir) + + client, err := getRepository(context.Background(), tmpDir, config.applicationRepository.ssh, defaultBranch, config.defaultAuthOpts) + if err != nil { + return nil, err + } + err = commitAndPushAll(context.Background(), client, files, defaultBranch) + if err != nil { + return nil, err + } + + return config, nil +} diff --git a/tests/integration/suite_test.go b/tests/integration/suite_test.go index f8696caa..9272c84f 100644 --- a/tests/integration/suite_test.go +++ b/tests/integration/suite_test.go @@ -45,6 +45,9 @@ const ( // azureTerraformPath is the path to the folder containing the // terraform files for azure infra azureTerraformPath = "./terraform/azure" + // gcpTerraformPath is the path to the folder containing the + // terraform files for gcp infra + gcpTerraformPath = "./terraform/gcp" // kubeconfigPath is the path of the file containing the kubeconfig kubeconfigPath = "./build/kubeconfig" @@ -55,7 +58,7 @@ const ( var ( // supportedProviders are the providers supported by the test. - supportedProviders = []string{"azure"} + supportedProviders = []string{"azure", "gcp"} // cfg is a struct containing different variables needed for the test. cfg *testConfig @@ -269,8 +272,14 @@ func getProviderConfig(provider string) *providerConfig { getTestConfig: getTestConfigAKS, registryLogin: registryLoginACR, } + case "gcp": + return &providerConfig{ + terraformPath: gcpTerraformPath, + createKubeconfig: createKubeConfigGKE, + getTestConfig: getTestConfigGKE, + registryLogin: registryLoginGCR, + } } - return nil } diff --git a/tests/integration/terraform/gcp/main.tf b/tests/integration/terraform/gcp/main.tf new file mode 100644 index 00000000..e8d1691c --- /dev/null +++ b/tests/integration/terraform/gcp/main.tf @@ -0,0 +1,49 @@ +provider "google" { + project = var.gcp_project_id + region = var.gcp_region + zone = var.gcp_zone +} + +resource "random_pet" "suffix" {} + +data "google_kms_key_ring" "keyring" { + name = var.gcp_keyring + location = "global" +} + +data "google_kms_crypto_key" "my_crypto_key" { + name = var.gcp_crypto_key + key_ring = data.google_kms_key_ring.keyring.id +} + +data "google_project" "project" { +} + +module "gke" { + source = "git::https://github.com/fluxcd/test-infra.git//tf-modules/gcp/gke" + name = "flux-e2e-${random_pet.suffix.id}" + tags = var.tags +} + +module "gcr" { + source = "git::https://github.com/fluxcd/test-infra.git//tf-modules/gcp/gcr" + name = "flux-e2e-${random_pet.suffix.id}" + tags = var.tags +} + +resource "google_sourcerepo_repository" "fleet-infra" { + name = "fleet-infra-${random_pet.suffix.id}" +} + +resource "google_sourcerepo_repository" "application" { + name = "application-${random_pet.suffix.id}" +} + +resource "google_kms_key_ring_iam_binding" "key_ring" { + key_ring_id = data.google_kms_key_ring.keyring.id + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + + members = [ + "serviceAccount:${data.google_project.project.number}-compute@developer.gserviceaccount.com", + ] +} diff --git a/tests/integration/terraform/gcp/outputs.tf b/tests/integration/terraform/gcp/outputs.tf new file mode 100644 index 00000000..cdf6fbd3 --- /dev/null +++ b/tests/integration/terraform/gcp/outputs.tf @@ -0,0 +1,28 @@ +output "kubeconfig" { + value = module.gke.kubeconfig + sensitive = true +} + +output "gcp_project" { + value = var.gcp_project_id +} + +output "gcp_region" { + value = var.gcp_region +} + +output "artifact_registry_id" { + value = module.gcr.artifact_repository_id +} + +output "sops_id" { + value = data.google_kms_crypto_key.my_crypto_key.id +} + +output "fleet_infra_url" { + value = "ssh://${var.gcp_email}@source.developers.google.com:2022/p/${var.gcp_project_id}/r/${google_sourcerepo_repository.fleet-infra.name}" +} + +output "application_url" { + value = "ssh://${var.gcp_email}@source.developers.google.com:2022/p/${var.gcp_project_id}/r/${google_sourcerepo_repository.application.name}" +} diff --git a/tests/integration/terraform/gcp/variables.tf b/tests/integration/terraform/gcp/variables.tf new file mode 100644 index 00000000..5996ddcc --- /dev/null +++ b/tests/integration/terraform/gcp/variables.tf @@ -0,0 +1,37 @@ +variable "gcp_project_id" { + type = string + description = "GCP project to create the resources in" +} + +variable "gcp_email" { + type = string + description = "GCP email" +} + +variable "gcp_region" { + type = string + default = "us-central1" + description = "GCP region" +} + +variable "gcp_zone" { + type = string + default = "us-central1" + description = "GCP region" +} + + +variable "gcp_keyring" { + type = string + description = "GCP keyring that contains crypto key for encrypting secrets" +} + +variable "gcp_crypto_key" { + type = string + description = "GCP crypto key for encrypting secrets" +} + +variable "tags" { + type = map(string) + default = {} +}