From ab94c8064c63135935a1f8d518e8325d53c7bcff Mon Sep 17 00:00:00 2001 From: Sunny Date: Wed, 19 Jul 2023 17:40:01 +0000 Subject: [PATCH] tests/int: Add IAM setup automation docs and misc fixes Add instructions about how to create service accounts with IAM permissions and populate the secrets and variables required in the CI. Update the panic recovery code to ensure that the exit status surfaces on panic along with a log message. Signed-off-by: Sunny --- .github/workflows/e2e-azure.yaml | 33 +++--- tests/integration/README.md | 138 +++++++++++++++++++++-- tests/integration/suite_test.go | 9 +- tests/integration/terraform/azure/aks.tf | 1 + 4 files changed, 148 insertions(+), 33 deletions(-) diff --git a/.github/workflows/e2e-azure.yaml b/.github/workflows/e2e-azure.yaml index 3236c73a..def40dab 100644 --- a/.github/workflows/e2e-azure.yaml +++ b/.github/workflows/e2e-azure.yaml @@ -3,7 +3,7 @@ name: e2e-azure on: workflow_dispatch: schedule: - - cron: '0 6 * * *' + - cron: '0 6 * * *' push: branches: - main @@ -48,7 +48,7 @@ jobs: wget https://github.com/mozilla/sops/releases/download/v3.7.1/sops-v3.7.1.linux -O $HOME/.local/bin/sops chmod +x $HOME/.local/bin/sops - name: Setup Terraform - uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2 + uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2 with: terraform_version: 1.2.8 terraform_wrapper: false @@ -71,6 +71,7 @@ jobs: defaults: run: working-directory: ./tests/integration + if: (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && github.actor != 'dependabot[bot]' steps: - name: CheckoutD uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 @@ -92,7 +93,7 @@ jobs: - name: Authenticate to Azure uses: Azure/login@92a5484dfaf04ca78a94597f4f19fea633851fa2 # v1.4.6 with: - creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}' + creds: '{"clientId":"${{ secrets.AZ_ARM_CLIENT_ID }}","clientSecret":"${{ secrets.AZ_ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.AZ_ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.AZ_ARM_TENANT_ID }}"}' - name: Set dynamic variables in .env run: | cat > .env < build/ssh/key - export AZUREDEVOPS_SSH=build/ssh/key + echo $GITREPO_SSH_CONTENTS | base64 -d > build/ssh/key + export GITREPO_SSH_PATH=build/ssh/key touch ./build/ssh/key.pub - echo $AZUREDEVOPS_SSH_PUB_CONTENTS | base64 -d > ./build/ssh/key.pub - export AZUREDEVOPS_SSH_PUB=build/ssh/key.pub + echo $GITREPO_SSH_PUB_CONTENTS | base64 -d > ./build/ssh/key.pub + export GITREPO_SSH_PUB_PATH=build/ssh/key.pub make test-azure diff --git a/tests/integration/README.md b/tests/integration/README.md index 5477628d..aadfe49b 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -55,6 +55,63 @@ the tests: - `Microsoft.KeyVault/*` - `Microsoft.EventHub/*` +### IAM and CI setup + +To create the necessary IAM role with all the permissions, set up CI secrets and +variables using +[azure-gh-actions](https://github.com/fluxcd/test-infra/tree/main/tf-modules/azure/github-actions) +use: + +```hcl +resource "tls_private_key" "privatekey" { + algorithm = "RSA" + rsa_bits = 4096 +} + +module "azure_gh_actions" { + source = "git::https://github.com/fluxcd/test-infra.git//tf-modules/azure/github-actions" + + azure_owners = ["owner-id-1", "owner-id-2"] + azure_app_name = "flux2-e2e" + azure_app_description = "flux2 e2e" + azure_app_secret_name = "flux2-e2e" + azure_permissions = [ + "Microsoft.Kubernetes/*", + "Microsoft.Resources/*", + "Microsoft.Authorization/roleAssignments/Read", + "Microsoft.Authorization/roleAssignments/Write", + "Microsoft.Authorization/roleAssignments/Delete", + "Microsoft.ContainerRegistry/*", + "Microsoft.ContainerService/*", + "Microsoft.KeyVault/*", + "Microsoft.EventHub/*" + ] + azure_location = "eastus" + + github_project = "flux2" + + github_secret_client_id_name = "AZ_ARM_CLIENT_ID" + github_secret_client_secret_name = "AZ_ARM_CLIENT_SECRET" + github_secret_subscription_id_name = "AZ_ARM_SUBSCRIPTION_ID" + github_secret_tenant_id_name = "AZ_ARM_TENANT_ID" + + github_secret_custom = { + "TF_VAR_azuredevops_org" = "", + "TF_VAR_azuredevops_pat" = "", + "GITREPO_SSH_CONTENTS" = base64encode(tls_private_key.privatekey.private_key_openssh), + "GITREPO_SSH_PUB_CONTENTS" = base64encode(tls_private_key.privatekey.public_key_openssh) + } +} + +output "publickey" { + value = tls_private_key.privatekey.public_key_openssh +} +``` + +Copy the `publickey` output printed after applying, or run `terraform output` to +print it again, and add it in the Azure DevOps SSH public keys under the user +account that'll be used by flux in the tests. + ## GCP ### Architecture @@ -73,8 +130,11 @@ for the terraform variables - GCP account with an active project to be able to create GKE and GCR, and permission to assign roles. - Existing GCP KMS keyring and crypto key. - - [Create a Keyring](https://cloud.google.com/kms/docs/create-key-ring) - - [Create a Crypto Key](https://cloud.google.com/kms/docs/create-key) + - [Create a Keyring](https://cloud.google.com/kms/docs/create-key-ring) in + `global` location. + - [Create a Crypto Key](https://cloud.google.com/kms/docs/create-key) with + symmetric algorithm for encryption and decryption, and software based + protection level. - gcloud CLI, need to be logged in using `gcloud auth login` as a User (not a Service Account), configure application default credentials with `gcloud auth application-default login` and docker credential helper with `gcloud auth configure-docker`. @@ -112,15 +172,71 @@ for the terraform variables Following roles are needed for provisioning the infrastructure and running the tests: -- Compute Instance Admin (v1) -- Kubernetes Engine Admin -- Service Account User -- Artifact Registry Administrator -- Artifact Registry Repository Administrator -- Cloud KMS Admin -- Cloud KMS CryptoKey Encrypter -- Source Repository Administrator -- Pub/Sub Admin +- Compute Instance Admin (v1) - `roles/compute.instanceAdmin.v1` +- Kubernetes Engine Admin - `roles/container.admin` +- Service Account User - `roles/iam.serviceAccountUser` +- Service Account Token Creator - `roles/iam.serviceAccountTokenCreator` +- Artifact Registry Administrator - `roles/artifactregistry.admin` +- Artifact Registry Repository Administrator - `roles/artifactregistry.repoAdmin` +- Cloud KMS Admin - `roles/cloudkms.admin` +- Cloud KMS CryptoKey Encrypter - `roles/cloudkms.cryptoKeyEncrypt` +- Source Repository Administrator - `roles/source.admin` +- Pub/Sub Admin - `roles/pubsub.admin` + +### IAM and CI setup + +To create the necessary IAM role with all the permissions, set up CI secrets and +variables using +[gcp-gh-actions](https://github.com/fluxcd/test-infra/tree/main/tf-modules/gcp/github-actions) +use: + +```hcl +provider "google" {} + +resource "tls_private_key" "privatekey" { + algorithm = "RSA" + rsa_bits = 4096 +} + +module "gcp_gh_actions" { + source = "git::https://github.com/fluxcd/test-infra.git//tf-modules/gcp/github-actions" + + gcp_service_account_id = "flux2-e2e-test" + gcp_service_account_name = "flux2-e2e-test" + gcp_roles = [ + "roles/compute.instanceAdmin.v1", + "roles/container.admin", + "roles/iam.serviceAccountUser", + "roles/iam.serviceAccountTokenCreator", + "roles/artifactregistry.admin", + "roles/artifactregistry.repoAdmin", + "roles/cloudkms.admin", + "roles/cloudkms.cryptoKeyEncrypter", + "roles/source.admin", + "roles/pubsub.admin" + ] + + github_project = "flux2" + + github_secret_credentials_name = "FLUX2_E2E_GOOGLE_CREDENTIALS" + + github_secret_custom = { + "TF_VAR_gcp_keyring" = "", + "TF_VAR_gcp_crypto_key" = "", + "TF_VAR_gcp_email" = "", + "GITREPO_SSH_CONTENTS" = base64encode(tls_private_key.privatekey.private_key_openssh), + "GITREPO_SSH_PUB_CONTENTS" = base64encode(tls_private_key.privatekey.public_key_openssh) + } +} + +output "publickey" { + value = tls_private_key.privatekey.public_key_openssh +} +``` + +Copy the `publickey` output printed after applying, or run `terraform output` to +print it again, and add it in the Google Source Repository SSH public keys under +the user account with email address referred in `TF_VAR_gcp_email` above. ## Tests diff --git a/tests/integration/suite_test.go b/tests/integration/suite_test.go index be2ca0cf..b6ee3589 100644 --- a/tests/integration/suite_test.go +++ b/tests/integration/suite_test.go @@ -222,14 +222,11 @@ func TestMain(m *testing.M) { log.Printf("Failed to stop environment: %v", err) } - // Calling exit on panic prevents logging of panic error. - // Exit only on normal return. Explicitly detect panic and log the error - // on panic. - if err := recover(); err == nil { - os.Exit(exitCode) - } else { + // Log the panic error before exit to surface the cause of panic. + if err := recover(); err != nil { log.Printf("panic: %v", err) } + os.Exit(exitCode) }() // get terrraform infrastructure diff --git a/tests/integration/terraform/azure/aks.tf b/tests/integration/terraform/azure/aks.tf index 122cafa3..8ffadfce 100644 --- a/tests/integration/terraform/azure/aks.tf +++ b/tests/integration/terraform/azure/aks.tf @@ -13,6 +13,7 @@ module "acr" { location = var.azure_location aks_principal_id = [module.aks.principal_id] resource_group = module.aks.resource_group + admin_enabled = true tags = var.tags depends_on = [module.aks]