tests/int: Add IAM setup automation docs and misc fixes

Add instructions about how to create service accounts with IAM
permissions and populate the secrets and variables required in the CI.

Update the panic recovery code to ensure that the exit status surfaces
on panic along with a log message.

Signed-off-by: Sunny <darkowlzz@protonmail.com>

.github/workflows/e2e-azure.yaml

@@ -3,7 +3,7 @@ name: e2e-azure
 on:
   workflow_dispatch:
   schedule:
-    - cron:  '0 6 * * *'
+    - cron: '0 6 * * *'
   push:
     branches:
       - main
@@ -48,7 +48,7 @@ jobs:
           wget https://github.com/mozilla/sops/releases/download/v3.7.1/sops-v3.7.1.linux -O $HOME/.local/bin/sops
           chmod +x $HOME/.local/bin/sops
       - name: Setup Terraform
-        uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1  # v2
+        uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2
         with:
           terraform_version: 1.2.8
           terraform_wrapper: false
@@ -71,6 +71,7 @@ jobs:
     defaults:
       run:
         working-directory: ./tests/integration
+    if: (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && github.actor != 'dependabot[bot]'
     steps:
       - name: CheckoutD
         uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
@@ -92,7 +93,7 @@ jobs:
       - name: Authenticate to Azure
         uses: Azure/login@92a5484dfaf04ca78a94597f4f19fea633851fa2 # v1.4.6
         with:
-          creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
+          creds: '{"clientId":"${{ secrets.AZ_ARM_CLIENT_ID }}","clientSecret":"${{ secrets.AZ_ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.AZ_ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.AZ_ARM_TENANT_ID }}"}'
       - name: Set dynamic variables in .env
         run: |
           cat > .env <<EOF
@@ -102,22 +103,22 @@ jobs:
         run: cat .env
       - name: Run Azure e2e tests
         env:
-          ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
+          ARM_CLIENT_ID: ${{ secrets.AZ_ARM_CLIENT_ID }}
-          ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
+          ARM_CLIENT_SECRET: ${{ secrets.AZ_ARM_CLIENT_SECRET }}
-          ARM_SUBSCRIPTION_ID: ${{ secrets.ARM_SUBSCRIPTION_ID }}
+          ARM_SUBSCRIPTION_ID: ${{ secrets.AZ_ARM_SUBSCRIPTION_ID }}
-          ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
+          ARM_TENANT_ID: ${{ secrets.AZ_ARM_TENANT_ID }}
-          TF_VAR_azuredevops_org: ${{ secrets.AZUREDEVOPS_ORG }}
+          TF_VAR_azuredevops_org: ${{ secrets.TF_VAR_azuredevops_org }}
-          TF_VAR_azuredevops_pat: ${{ secrets.AZUREDEVOPS_PAT }}
+          TF_VAR_azuredevops_pat: ${{ secrets.TF_VAR_azuredevops_pat }}
-          TF_VAR_location: "southcentralus"
+          TF_VAR_location: ${{ vars.TF_VAR_azure_location }}
-          AZUREDEVOPS_SSH_CONTENTS: ${{ secrets.AZUREDEVOPS_ID_RSA }}
+          GITREPO_SSH_CONTENTS: ${{ secrets.GITREPO_SSH_CONTENTS }}
-          AZUREDEVOPS_SSH_PUB_CONTENTS: ${{ secrets.AZUREDEVOPS_ID_RSA_PUB }}
+          GITREPO_SSH_PUB_CONTENTS: ${{ secrets.GITREPO_SSH_PUB_CONTENTS }}
         run: |
           source .env
           mkdir -p ./build/ssh
           touch ./build/ssh/key
-          echo $AZUREDEVOPS_SSH_CONTENTS | base64 -d > build/ssh/key
+          echo $GITREPO_SSH_CONTENTS | base64 -d > build/ssh/key
-          export AZUREDEVOPS_SSH=build/ssh/key
+          export GITREPO_SSH_PATH=build/ssh/key
           touch ./build/ssh/key.pub
-          echo $AZUREDEVOPS_SSH_PUB_CONTENTS | base64 -d > ./build/ssh/key.pub
+          echo $GITREPO_SSH_PUB_CONTENTS | base64 -d > ./build/ssh/key.pub
-          export AZUREDEVOPS_SSH_PUB=build/ssh/key.pub
+          export GITREPO_SSH_PUB_PATH=build/ssh/key.pub
           make test-azure

tests/integration/README.md

@@ -55,6 +55,63 @@ the tests:
 - `Microsoft.KeyVault/*`
 - `Microsoft.EventHub/*`
 
+### IAM and CI setup
+
+To create the necessary IAM role with all the permissions, set up CI secrets and
+variables using
+[azure-gh-actions](https://github.com/fluxcd/test-infra/tree/main/tf-modules/azure/github-actions)
+use:
+
+```hcl
+resource "tls_private_key" "privatekey" {
+  algorithm = "RSA"
+  rsa_bits  = 4096
+}
+
+module "azure_gh_actions" {
+  source = "git::https://github.com/fluxcd/test-infra.git//tf-modules/azure/github-actions"
+
+  azure_owners          = ["owner-id-1", "owner-id-2"]
+  azure_app_name        = "flux2-e2e"
+  azure_app_description = "flux2 e2e"
+  azure_app_secret_name = "flux2-e2e"
+  azure_permissions = [
+    "Microsoft.Kubernetes/*",
+    "Microsoft.Resources/*",
+    "Microsoft.Authorization/roleAssignments/Read",
+    "Microsoft.Authorization/roleAssignments/Write",
+    "Microsoft.Authorization/roleAssignments/Delete",
+    "Microsoft.ContainerRegistry/*",
+    "Microsoft.ContainerService/*",
+    "Microsoft.KeyVault/*",
+    "Microsoft.EventHub/*"
+  ]
+  azure_location = "eastus"
+
+  github_project = "flux2"
+
+  github_secret_client_id_name       = "AZ_ARM_CLIENT_ID"
+  github_secret_client_secret_name   = "AZ_ARM_CLIENT_SECRET"
+  github_secret_subscription_id_name = "AZ_ARM_SUBSCRIPTION_ID"
+  github_secret_tenant_id_name       = "AZ_ARM_TENANT_ID"
+
+  github_secret_custom = {
+    "TF_VAR_azuredevops_org"   = "<org-name>",
+    "TF_VAR_azuredevops_pat"   = "<pat>",
+    "GITREPO_SSH_CONTENTS"     = base64encode(tls_private_key.privatekey.private_key_openssh),
+    "GITREPO_SSH_PUB_CONTENTS" = base64encode(tls_private_key.privatekey.public_key_openssh)
+  }
+}
+
+output "publickey" {
+  value = tls_private_key.privatekey.public_key_openssh
+}
+```
+
+Copy the `publickey` output printed after applying, or run `terraform output` to
+print it again, and add it in the Azure DevOps SSH public keys under the user
+account that'll be used by flux in the tests.
+
 ## GCP
 
 ### Architecture
@@ -73,8 +130,11 @@ for the terraform variables
 
 - GCP account with an active project to be able to create GKE and GCR, and permission to assign roles.
 - Existing GCP KMS keyring and crypto key.
-  - [Create a Keyring](https://cloud.google.com/kms/docs/create-key-ring)
+  - [Create a Keyring](https://cloud.google.com/kms/docs/create-key-ring) in
-  - [Create a Crypto Key](https://cloud.google.com/kms/docs/create-key)
+    `global` location.
+  - [Create a Crypto Key](https://cloud.google.com/kms/docs/create-key) with
+    symmetric algorithm for encryption and decryption, and software based
+    protection level.
 - gcloud CLI, need to be logged in using `gcloud auth login` as a User (not a
   Service Account), configure application default credentials with `gcloud auth
   application-default login` and docker credential helper with `gcloud auth configure-docker`.
@@ -112,15 +172,71 @@ for the terraform variables
 
 Following roles are needed for provisioning the infrastructure and running the tests:
 
-- Compute Instance Admin (v1)
+- Compute Instance Admin (v1) - `roles/compute.instanceAdmin.v1`
-- Kubernetes Engine Admin
+- Kubernetes Engine Admin - `roles/container.admin`
-- Service Account User
+- Service Account User - `roles/iam.serviceAccountUser`
-- Artifact Registry Administrator
+- Service Account Token Creator - `roles/iam.serviceAccountTokenCreator`
-- Artifact Registry Repository Administrator
+- Artifact Registry Administrator - `roles/artifactregistry.admin`
-- Cloud KMS Admin
+- Artifact Registry Repository Administrator - `roles/artifactregistry.repoAdmin`
-- Cloud KMS CryptoKey Encrypter
+- Cloud KMS Admin - `roles/cloudkms.admin`
-- Source Repository Administrator
+- Cloud KMS CryptoKey Encrypter - `roles/cloudkms.cryptoKeyEncrypt`
-- Pub/Sub Admin
+- Source Repository Administrator - `roles/source.admin`
+- Pub/Sub Admin - `roles/pubsub.admin`
+
+### IAM and CI setup
+
+To create the necessary IAM role with all the permissions, set up CI secrets and
+variables using
+[gcp-gh-actions](https://github.com/fluxcd/test-infra/tree/main/tf-modules/gcp/github-actions)
+use:
+
+```hcl
+provider "google" {}
+
+resource "tls_private_key" "privatekey" {
+  algorithm = "RSA"
+  rsa_bits  = 4096
+}
+
+module "gcp_gh_actions" {
+  source = "git::https://github.com/fluxcd/test-infra.git//tf-modules/gcp/github-actions"
+
+  gcp_service_account_id   = "flux2-e2e-test"
+  gcp_service_account_name = "flux2-e2e-test"
+  gcp_roles = [
+    "roles/compute.instanceAdmin.v1",
+    "roles/container.admin",
+    "roles/iam.serviceAccountUser",
+    "roles/iam.serviceAccountTokenCreator",
+    "roles/artifactregistry.admin",
+    "roles/artifactregistry.repoAdmin",
+    "roles/cloudkms.admin",
+    "roles/cloudkms.cryptoKeyEncrypter",
+    "roles/source.admin",
+    "roles/pubsub.admin"
+  ]
+
+  github_project = "flux2"
+
+  github_secret_credentials_name = "FLUX2_E2E_GOOGLE_CREDENTIALS"
+
+  github_secret_custom = {
+    "TF_VAR_gcp_keyring"       = "<keyring-name>",
+    "TF_VAR_gcp_crypto_key"    = "<key-name>",
+    "TF_VAR_gcp_email"         = "<email>",
+    "GITREPO_SSH_CONTENTS"     = base64encode(tls_private_key.privatekey.private_key_openssh),
+    "GITREPO_SSH_PUB_CONTENTS" = base64encode(tls_private_key.privatekey.public_key_openssh)
+  }
+}
+
+output "publickey" {
+  value = tls_private_key.privatekey.public_key_openssh
+}
+```
+
+Copy the `publickey` output printed after applying, or run `terraform output` to
+print it again, and add it in the Google Source Repository SSH public keys under
+the user account with email address referred in `TF_VAR_gcp_email` above.
 
 ## Tests
 

tests/integration/suite_test.go

@@ -222,14 +222,11 @@ func TestMain(m *testing.M) {
 			log.Printf("Failed to stop environment: %v", err)
 		}
 
-		// Calling exit on panic prevents logging of panic error.
+		// Log the panic error before exit to surface the cause of panic.
-		// Exit only on normal return. Explicitly detect panic and log the error
+		if err := recover(); err != nil {
-		// on panic.
-		if err := recover(); err == nil {
-			os.Exit(exitCode)
-		} else {
 			log.Printf("panic: %v", err)
 		}
+		os.Exit(exitCode)
 	}()
 
 	// get terrraform infrastructure

tests/integration/terraform/azure/aks.tf

@@ -13,6 +13,7 @@ module "acr" {
   location         = var.azure_location
   aks_principal_id = [module.aks.principal_id]
   resource_group   = module.aks.resource_group
+  admin_enabled    = true
   tags             = var.tags
 
   depends_on = [module.aks]