diff --git a/.github/workflows/e2e-azure.yaml b/.github/workflows/e2e-azure.yaml index 3236c73a..def40dab 100644 --- a/.github/workflows/e2e-azure.yaml +++ b/.github/workflows/e2e-azure.yaml @@ -3,7 +3,7 @@ name: e2e-azure on: workflow_dispatch: schedule: - - cron: '0 6 * * *' + - cron: '0 6 * * *' push: branches: - main @@ -48,7 +48,7 @@ jobs: wget https://github.com/mozilla/sops/releases/download/v3.7.1/sops-v3.7.1.linux -O $HOME/.local/bin/sops chmod +x $HOME/.local/bin/sops - name: Setup Terraform - uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2 + uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2 with: terraform_version: 1.2.8 terraform_wrapper: false @@ -71,6 +71,7 @@ jobs: defaults: run: working-directory: ./tests/integration + if: (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && github.actor != 'dependabot[bot]' steps: - name: CheckoutD uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 @@ -92,7 +93,7 @@ jobs: - name: Authenticate to Azure uses: Azure/login@92a5484dfaf04ca78a94597f4f19fea633851fa2 # v1.4.6 with: - creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}' + creds: '{"clientId":"${{ secrets.AZ_ARM_CLIENT_ID }}","clientSecret":"${{ secrets.AZ_ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.AZ_ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.AZ_ARM_TENANT_ID }}"}' - name: Set dynamic variables in .env run: | cat > .env < build/ssh/key - export AZUREDEVOPS_SSH=build/ssh/key + echo $GITREPO_SSH_CONTENTS | base64 -d > build/ssh/key + export GITREPO_SSH_PATH=build/ssh/key touch ./build/ssh/key.pub - echo $AZUREDEVOPS_SSH_PUB_CONTENTS | base64 -d > ./build/ssh/key.pub - export AZUREDEVOPS_SSH_PUB=build/ssh/key.pub + echo $GITREPO_SSH_PUB_CONTENTS | base64 -d > ./build/ssh/key.pub + export GITREPO_SSH_PUB_PATH=build/ssh/key.pub make test-azure diff --git a/tests/integration/README.md b/tests/integration/README.md index 5477628d..aadfe49b 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -55,6 +55,63 @@ the tests: - `Microsoft.KeyVault/*` - `Microsoft.EventHub/*` +### IAM and CI setup + +To create the necessary IAM role with all the permissions, set up CI secrets and +variables using +[azure-gh-actions](https://github.com/fluxcd/test-infra/tree/main/tf-modules/azure/github-actions) +use: + +```hcl +resource "tls_private_key" "privatekey" { + algorithm = "RSA" + rsa_bits = 4096 +} + +module "azure_gh_actions" { + source = "git::https://github.com/fluxcd/test-infra.git//tf-modules/azure/github-actions" + + azure_owners = ["owner-id-1", "owner-id-2"] + azure_app_name = "flux2-e2e" + azure_app_description = "flux2 e2e" + azure_app_secret_name = "flux2-e2e" + azure_permissions = [ + "Microsoft.Kubernetes/*", + "Microsoft.Resources/*", + "Microsoft.Authorization/roleAssignments/Read", + "Microsoft.Authorization/roleAssignments/Write", + "Microsoft.Authorization/roleAssignments/Delete", + "Microsoft.ContainerRegistry/*", + "Microsoft.ContainerService/*", + "Microsoft.KeyVault/*", + "Microsoft.EventHub/*" + ] + azure_location = "eastus" + + github_project = "flux2" + + github_secret_client_id_name = "AZ_ARM_CLIENT_ID" + github_secret_client_secret_name = "AZ_ARM_CLIENT_SECRET" + github_secret_subscription_id_name = "AZ_ARM_SUBSCRIPTION_ID" + github_secret_tenant_id_name = "AZ_ARM_TENANT_ID" + + github_secret_custom = { + "TF_VAR_azuredevops_org" = "", + "TF_VAR_azuredevops_pat" = "", + "GITREPO_SSH_CONTENTS" = base64encode(tls_private_key.privatekey.private_key_openssh), + "GITREPO_SSH_PUB_CONTENTS" = base64encode(tls_private_key.privatekey.public_key_openssh) + } +} + +output "publickey" { + value = tls_private_key.privatekey.public_key_openssh +} +``` + +Copy the `publickey` output printed after applying, or run `terraform output` to +print it again, and add it in the Azure DevOps SSH public keys under the user +account that'll be used by flux in the tests. + ## GCP ### Architecture @@ -73,8 +130,11 @@ for the terraform variables - GCP account with an active project to be able to create GKE and GCR, and permission to assign roles. - Existing GCP KMS keyring and crypto key. - - [Create a Keyring](https://cloud.google.com/kms/docs/create-key-ring) - - [Create a Crypto Key](https://cloud.google.com/kms/docs/create-key) + - [Create a Keyring](https://cloud.google.com/kms/docs/create-key-ring) in + `global` location. + - [Create a Crypto Key](https://cloud.google.com/kms/docs/create-key) with + symmetric algorithm for encryption and decryption, and software based + protection level. - gcloud CLI, need to be logged in using `gcloud auth login` as a User (not a Service Account), configure application default credentials with `gcloud auth application-default login` and docker credential helper with `gcloud auth configure-docker`. @@ -112,15 +172,71 @@ for the terraform variables Following roles are needed for provisioning the infrastructure and running the tests: -- Compute Instance Admin (v1) -- Kubernetes Engine Admin -- Service Account User -- Artifact Registry Administrator -- Artifact Registry Repository Administrator -- Cloud KMS Admin -- Cloud KMS CryptoKey Encrypter -- Source Repository Administrator -- Pub/Sub Admin +- Compute Instance Admin (v1) - `roles/compute.instanceAdmin.v1` +- Kubernetes Engine Admin - `roles/container.admin` +- Service Account User - `roles/iam.serviceAccountUser` +- Service Account Token Creator - `roles/iam.serviceAccountTokenCreator` +- Artifact Registry Administrator - `roles/artifactregistry.admin` +- Artifact Registry Repository Administrator - `roles/artifactregistry.repoAdmin` +- Cloud KMS Admin - `roles/cloudkms.admin` +- Cloud KMS CryptoKey Encrypter - `roles/cloudkms.cryptoKeyEncrypt` +- Source Repository Administrator - `roles/source.admin` +- Pub/Sub Admin - `roles/pubsub.admin` + +### IAM and CI setup + +To create the necessary IAM role with all the permissions, set up CI secrets and +variables using +[gcp-gh-actions](https://github.com/fluxcd/test-infra/tree/main/tf-modules/gcp/github-actions) +use: + +```hcl +provider "google" {} + +resource "tls_private_key" "privatekey" { + algorithm = "RSA" + rsa_bits = 4096 +} + +module "gcp_gh_actions" { + source = "git::https://github.com/fluxcd/test-infra.git//tf-modules/gcp/github-actions" + + gcp_service_account_id = "flux2-e2e-test" + gcp_service_account_name = "flux2-e2e-test" + gcp_roles = [ + "roles/compute.instanceAdmin.v1", + "roles/container.admin", + "roles/iam.serviceAccountUser", + "roles/iam.serviceAccountTokenCreator", + "roles/artifactregistry.admin", + "roles/artifactregistry.repoAdmin", + "roles/cloudkms.admin", + "roles/cloudkms.cryptoKeyEncrypter", + "roles/source.admin", + "roles/pubsub.admin" + ] + + github_project = "flux2" + + github_secret_credentials_name = "FLUX2_E2E_GOOGLE_CREDENTIALS" + + github_secret_custom = { + "TF_VAR_gcp_keyring" = "", + "TF_VAR_gcp_crypto_key" = "", + "TF_VAR_gcp_email" = "", + "GITREPO_SSH_CONTENTS" = base64encode(tls_private_key.privatekey.private_key_openssh), + "GITREPO_SSH_PUB_CONTENTS" = base64encode(tls_private_key.privatekey.public_key_openssh) + } +} + +output "publickey" { + value = tls_private_key.privatekey.public_key_openssh +} +``` + +Copy the `publickey` output printed after applying, or run `terraform output` to +print it again, and add it in the Google Source Repository SSH public keys under +the user account with email address referred in `TF_VAR_gcp_email` above. ## Tests diff --git a/tests/integration/suite_test.go b/tests/integration/suite_test.go index be2ca0cf..b6ee3589 100644 --- a/tests/integration/suite_test.go +++ b/tests/integration/suite_test.go @@ -222,14 +222,11 @@ func TestMain(m *testing.M) { log.Printf("Failed to stop environment: %v", err) } - // Calling exit on panic prevents logging of panic error. - // Exit only on normal return. Explicitly detect panic and log the error - // on panic. - if err := recover(); err == nil { - os.Exit(exitCode) - } else { + // Log the panic error before exit to surface the cause of panic. + if err := recover(); err != nil { log.Printf("panic: %v", err) } + os.Exit(exitCode) }() // get terrraform infrastructure diff --git a/tests/integration/terraform/azure/aks.tf b/tests/integration/terraform/azure/aks.tf index 122cafa3..8ffadfce 100644 --- a/tests/integration/terraform/azure/aks.tf +++ b/tests/integration/terraform/azure/aks.tf @@ -13,6 +13,7 @@ module "acr" { location = var.azure_location aks_principal_id = [module.aks.principal_id] resource_group = module.aks.resource_group + admin_enabled = true tags = var.tags depends_on = [module.aks]