From 2dfb5366001bcd16a62980d98e512ca94e17644b Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Wed, 17 Apr 2024 03:04:19 +0300 Subject: [PATCH 1/4] e2e: Run OpenShift from Replicated Signed-off-by: Stefan Prodan --- .github/workflows/e2e-openshift.yaml | 55 +++++++++++++++++++++ manifests/openshift/kustomization.yaml | 4 ++ manifests/openshift/rbac.yaml | 68 ++++++++++++++++++++++++++ 3 files changed, 127 insertions(+) create mode 100644 .github/workflows/e2e-openshift.yaml create mode 100644 manifests/openshift/kustomization.yaml create mode 100644 manifests/openshift/rbac.yaml diff --git a/.github/workflows/e2e-openshift.yaml b/.github/workflows/e2e-openshift.yaml new file mode 100644 index 00000000..f40739c2 --- /dev/null +++ b/.github/workflows/e2e-openshift.yaml @@ -0,0 +1,55 @@ +name: e2e-openshift + +on: + workflow_dispatch: + push: + branches: [ 'main', 'update-components', 'openshift-*', 'release/**' ] + +permissions: + contents: read + +jobs: + e2e-openshift: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - name: Setup Go + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + with: + go-version-file: 'go.mod' + cache-dependency-path: | + **/go.sum + **/go.mod + - name: Prepare + id: prep + run: | + ID=${GITHUB_SHA:0:7}-$(date +%s) + echo "cluster=fluxcd-openshift-${ID}" >> $GITHUB_OUTPUT + - name: Setup Kustomize + uses: fluxcd/pkg/actions/kustomize@main + - name: Build + run: make build-dev + - name: Create cluster + id: create-cluster + uses: replicatedhq/compatibility-actions/create-cluster@v1 + with: + api-token: ${{ secrets.REPLICATED_API_TOKEN }} + kubernetes-distribution: "openshift" + kubernetes-version: "4.15.0-okd" + ttl: 20m + cluster-name: "${{ steps.prep.outputs.cluster }}" + - name: Run flux check + run: | + echo "${{ steps.create-cluster.outputs.cluster-kubeconfig }}" > kubeconfig.yaml + ./bin/flux check --kubeconfig=kubeconfig.yaml + - name: Apply openshift prerequisites + run: | + kubectl apply -k ./manifests/openshift --kubeconfig=kubeconfig.yaml + - name: Remove cluster + if: ${{ always() }} + uses: replicatedhq/replicated-actions/remove-cluster@v1 + continue-on-error: true + with: + api-token: ${{ secrets.REPLICATED_API_TOKEN }} + cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} diff --git a/manifests/openshift/kustomization.yaml b/manifests/openshift/kustomization.yaml new file mode 100644 index 00000000..b5633ece --- /dev/null +++ b/manifests/openshift/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - rbac.yaml diff --git a/manifests/openshift/rbac.yaml b/manifests/openshift/rbac.yaml new file mode 100644 index 00000000..fefb26fe --- /dev/null +++ b/manifests/openshift/rbac.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: List +items: + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: flux-scc + rules: + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - nonroot + verbs: + - use + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: flux-scc-source-controller + namespace: flux-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: flux-scc + subjects: + - kind: ServiceAccount + name: source-controller + namespace: flux-system + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: flux-scc-kustomize-controller + namespace: flux-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: flux-scc + subjects: + - kind: ServiceAccount + name: kustomize-controller + namespace: flux-system + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: flux-scc-helm-controller + namespace: flux-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: flux-scc + subjects: + - kind: ServiceAccount + name: helm-controller + namespace: flux-system + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: flux-scc-notification-controller + namespace: flux-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: flux-scc + subjects: + - kind: ServiceAccount + name: notification-controller + namespace: flux-system From b3a29b56bbc2684d84f366570c28ea5cb9e11e5c Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Wed, 17 Apr 2024 03:21:42 +0300 Subject: [PATCH 2/4] e2e: Install Flux on OpenShift Signed-off-by: Stefan Prodan --- .github/workflows/e2e-openshift.yaml | 13 +++-- manifests/openshift/kustomization.yaml | 46 ++++++++++++++++- manifests/openshift/labels.yaml | 10 ++++ manifests/openshift/namespace.yaml | 4 ++ manifests/openshift/rbac.yaml | 68 -------------------------- manifests/openshift/scc.yaml | 42 ++++++++++++++++ 6 files changed, 110 insertions(+), 73 deletions(-) create mode 100644 manifests/openshift/labels.yaml create mode 100644 manifests/openshift/namespace.yaml delete mode 100644 manifests/openshift/rbac.yaml create mode 100644 manifests/openshift/scc.yaml diff --git a/.github/workflows/e2e-openshift.yaml b/.github/workflows/e2e-openshift.yaml index f40739c2..c0c4e1fe 100644 --- a/.github/workflows/e2e-openshift.yaml +++ b/.github/workflows/e2e-openshift.yaml @@ -39,13 +39,18 @@ jobs: kubernetes-version: "4.15.0-okd" ttl: 20m cluster-name: "${{ steps.prep.outputs.cluster }}" - - name: Run flux check + - name: Setup kubeconfig run: | echo "${{ steps.create-cluster.outputs.cluster-kubeconfig }}" > kubeconfig.yaml - ./bin/flux check --kubeconfig=kubeconfig.yaml - - name: Apply openshift prerequisites + - name: Install Flux run: | - kubectl apply -k ./manifests/openshift --kubeconfig=kubeconfig.yaml + ./bin/flux install --manifests ./manifests/openshift/ --kubeconfig=kubeconfig.yaml + - name: Debug + run: | + kubectl -n flux-system get all --kubeconfig=kubeconfig.yaml + kubectl -n flux-system describe pods --kubeconfig=kubeconfig.yaml + kubectl -n flux-system logs deploy/source-controller --kubeconfig=kubeconfig.yaml + kubectl -n flux-system logs deploy/kustomize-controller --kubeconfig=kubeconfig.yaml - name: Remove cluster if: ${{ always() }} uses: replicatedhq/replicated-actions/remove-cluster@v1 diff --git a/manifests/openshift/kustomization.yaml b/manifests/openshift/kustomization.yaml index b5633ece..a161f2b1 100644 --- a/manifests/openshift/kustomization.yaml +++ b/manifests/openshift/kustomization.yaml @@ -1,4 +1,48 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: flux-system resources: - - rbac.yaml + - namespace.yaml + - scc.yaml + - ../bases/source-controller + - ../bases/kustomize-controller + - ../bases/notification-controller + - ../bases/helm-controller + - ../bases/image-reflector-controller + - ../bases/image-automation-controller + - ../rbac + - ../policies +transformers: + - labels.yaml +images: + - name: fluxcd/source-controller + newName: ghcr.io/fluxcd/source-controller + - name: fluxcd/kustomize-controller + newName: ghcr.io/fluxcd/kustomize-controller + - name: fluxcd/helm-controller + newName: ghcr.io/fluxcd/helm-controller + - name: fluxcd/notification-controller + newName: ghcr.io/fluxcd/notification-controller + - name: fluxcd/image-reflector-controller + newName: ghcr.io/fluxcd/image-reflector-controller + - name: fluxcd/image-automation-controller + newName: ghcr.io/fluxcd/image-automation-controller +patches: + - patch: | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: all + spec: + template: + spec: + securityContext: + $patch: delete + containers: + - name: manager + securityContext: + runAsUser: 65534 + seccompProfile: + $patch: delete + target: + kind: Deployment diff --git a/manifests/openshift/labels.yaml b/manifests/openshift/labels.yaml new file mode 100644 index 00000000..5a5d78b4 --- /dev/null +++ b/manifests/openshift/labels.yaml @@ -0,0 +1,10 @@ +apiVersion: builtin +kind: LabelTransformer +metadata: + name: labels +labels: + app.kubernetes.io/part-of: flux + app.kubernetes.io/instance: flux-system +fieldSpecs: + - path: metadata/labels + create: true diff --git a/manifests/openshift/namespace.yaml b/manifests/openshift/namespace.yaml new file mode 100644 index 00000000..c00a4321 --- /dev/null +++ b/manifests/openshift/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: flux-system diff --git a/manifests/openshift/rbac.yaml b/manifests/openshift/rbac.yaml deleted file mode 100644 index fefb26fe..00000000 --- a/manifests/openshift/rbac.yaml +++ /dev/null @@ -1,68 +0,0 @@ -apiVersion: v1 -kind: List -items: - - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: flux-scc - rules: - - apiGroups: - - security.openshift.io - resources: - - securitycontextconstraints - resourceNames: - - nonroot - verbs: - - use - - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - name: flux-scc-source-controller - namespace: flux-system - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: flux-scc - subjects: - - kind: ServiceAccount - name: source-controller - namespace: flux-system - - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - name: flux-scc-kustomize-controller - namespace: flux-system - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: flux-scc - subjects: - - kind: ServiceAccount - name: kustomize-controller - namespace: flux-system - - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - name: flux-scc-helm-controller - namespace: flux-system - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: flux-scc - subjects: - - kind: ServiceAccount - name: helm-controller - namespace: flux-system - - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - name: flux-scc-notification-controller - namespace: flux-system - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: flux-scc - subjects: - - kind: ServiceAccount - name: notification-controller - namespace: flux-system diff --git a/manifests/openshift/scc.yaml b/manifests/openshift/scc.yaml new file mode 100644 index 00000000..da8bcd99 --- /dev/null +++ b/manifests/openshift/scc.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flux-scc +rules: + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - nonroot + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: flux-scc +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flux-scc +subjects: + - kind: ServiceAccount + name: source-controller + namespace: flux-system + - kind: ServiceAccount + name: kustomize-controller + namespace: flux-system + - kind: ServiceAccount + name: helm-controller + namespace: flux-system + - kind: ServiceAccount + name: notification-controller + namespace: flux-system + - kind: ServiceAccount + name: image-reflector-controller + namespace: flux-system + - kind: ServiceAccount + name: image-automation-controller + namespace: flux-system From cbe41a6bf9cfef730d198a4859eaba7ba10346b2 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Wed, 17 Apr 2024 04:00:30 +0300 Subject: [PATCH 3/4] e2e: Run integration test suite on OpenShift Signed-off-by: Stefan Prodan --- .github/workflows/e2e-openshift.yaml | 57 +++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/.github/workflows/e2e-openshift.yaml b/.github/workflows/e2e-openshift.yaml index c0c4e1fe..3c191284 100644 --- a/.github/workflows/e2e-openshift.yaml +++ b/.github/workflows/e2e-openshift.yaml @@ -30,6 +30,11 @@ jobs: uses: fluxcd/pkg/actions/kustomize@main - name: Build run: make build-dev + - name: Create repository + run: | + gh repo create --private --add-readme fluxcd-testing/${{ steps.prep.outputs.cluster }} + env: + GITHUB_TOKEN: ${{ secrets.GITPROVIDER_BOT_TOKEN }} - name: Create cluster id: create-cluster uses: replicatedhq/compatibility-actions/create-cluster@v1 @@ -39,22 +44,54 @@ jobs: kubernetes-version: "4.15.0-okd" ttl: 20m cluster-name: "${{ steps.prep.outputs.cluster }}" - - name: Setup kubeconfig + - name: Create kubeconfig + id: kubeconfig + run: | + KPATH="$(git rev-parse --show-toplevel)/bin/kubeconfig.yaml" + echo "::add-mask::${{ steps.create-cluster.outputs.cluster-kubeconfig }}" > $KPATH + echo "KUBECONFIG=$KPATH" >> $GITHUB_ENV + - name: Run flux bootstrap run: | - echo "${{ steps.create-cluster.outputs.cluster-kubeconfig }}" > kubeconfig.yaml - - name: Install Flux + ./bin/flux bootstrap git --manifests ./manifests/openshift/ \ + --components-extra=image-reflector-controller,image-automation-controller \ + --url=https://github.com/fluxcd-testing/${{ steps.prep.outputs.cluster }} \ + --branch=main \ + --path=clusters/openshift \ + --token-auth + env: + GIT_PASSWORD: ${{ secrets.GITPROVIDER_BOT_TOKEN }} + - name: Run flux check run: | - ./bin/flux install --manifests ./manifests/openshift/ --kubeconfig=kubeconfig.yaml - - name: Debug + ./bin/flux check + - name: Run flux reconcile run: | - kubectl -n flux-system get all --kubeconfig=kubeconfig.yaml - kubectl -n flux-system describe pods --kubeconfig=kubeconfig.yaml - kubectl -n flux-system logs deploy/source-controller --kubeconfig=kubeconfig.yaml - kubectl -n flux-system logs deploy/kustomize-controller --kubeconfig=kubeconfig.yaml - - name: Remove cluster + ./bin/flux reconcile ks flux-system --with-source + ./bin/flux get all + ./bin/flux events + - name: Collect reconcile logs + if: ${{ always() }} + continue-on-error: true + run: | + kubectl -n flux-system get all + kubectl -n flux-system describe pods + kubectl -n flux-system logs deploy/source-controller + kubectl -n flux-system logs deploy/kustomize-controller + kubectl -n flux-system logs deploy/notification-controller + - name: Delete flux + run: | + ./bin/flux uninstall -s --keep-namespace + kubectl delete ns flux-system --wait + - name: Delete cluster if: ${{ always() }} uses: replicatedhq/replicated-actions/remove-cluster@v1 continue-on-error: true with: api-token: ${{ secrets.REPLICATED_API_TOKEN }} cluster-id: ${{ steps.create-cluster.outputs.cluster-id }} + - name: Delete repository + if: ${{ always() }} + continue-on-error: true + run: | + gh repo delete fluxcd-testing/${{ steps.prep.outputs.cluster }} --yes + env: + GITHUB_TOKEN: ${{ secrets.GITPROVIDER_BOT_TOKEN }} From 12993874088b8f7d8fb11049dd05e4e416c655bf Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Wed, 17 Apr 2024 12:05:34 +0300 Subject: [PATCH 4/4] e2e: Run tests for OpenShift v4.14 and v4.15 Signed-off-by: Stefan Prodan --- .github/workflows/e2e-openshift.yaml | 22 +++++++++++++--------- manifests/openshift/scc.yaml | 3 ++- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/e2e-openshift.yaml b/.github/workflows/e2e-openshift.yaml index 3c191284..607477e5 100644 --- a/.github/workflows/e2e-openshift.yaml +++ b/.github/workflows/e2e-openshift.yaml @@ -11,6 +11,11 @@ permissions: jobs: e2e-openshift: runs-on: ubuntu-latest + strategy: + matrix: + # Keep this list up-to-date with https://endoflife.date/red-hat-openshift + OPENSHIFT_VERSION: [ 4.14.0-okd, 4.15.0-okd ] + fail-fast: false steps: - name: Checkout uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 @@ -24,8 +29,11 @@ jobs: - name: Prepare id: prep run: | - ID=${GITHUB_SHA:0:7}-$(date +%s) - echo "cluster=fluxcd-openshift-${ID}" >> $GITHUB_OUTPUT + ID=${GITHUB_SHA:0:7}-${{ matrix.OPENSHIFT_VERSION }}-$(date +%s) + PSEUDO_RAND_SUFFIX=$(echo "${ID}" | shasum | awk '{print $1}') + echo "cluster=flux2-openshift-${PSEUDO_RAND_SUFFIX}" >> $GITHUB_OUTPUT + KUBECONFIG_PATH="$(git rev-parse --show-toplevel)/bin/kubeconfig.yaml" + echo "kubeconfig-path=${KUBECONFIG_PATH}" >> $GITHUB_OUTPUT - name: Setup Kustomize uses: fluxcd/pkg/actions/kustomize@main - name: Build @@ -41,15 +49,11 @@ jobs: with: api-token: ${{ secrets.REPLICATED_API_TOKEN }} kubernetes-distribution: "openshift" - kubernetes-version: "4.15.0-okd" + kubernetes-version: ${{ matrix.OPENSHIFT_VERSION }} ttl: 20m cluster-name: "${{ steps.prep.outputs.cluster }}" - - name: Create kubeconfig - id: kubeconfig - run: | - KPATH="$(git rev-parse --show-toplevel)/bin/kubeconfig.yaml" - echo "::add-mask::${{ steps.create-cluster.outputs.cluster-kubeconfig }}" > $KPATH - echo "KUBECONFIG=$KPATH" >> $GITHUB_ENV + kubeconfig-path: ${{ steps.prep.outputs.kubeconfig-path }} + export-kubeconfig: true - name: Run flux bootstrap run: | ./bin/flux bootstrap git --manifests ./manifests/openshift/ \ diff --git a/manifests/openshift/scc.yaml b/manifests/openshift/scc.yaml index da8bcd99..6a25f70b 100644 --- a/manifests/openshift/scc.yaml +++ b/manifests/openshift/scc.yaml @@ -1,4 +1,5 @@ ---- +# Allow Flux controllers to run as non-root on OpenShift +# Docs: https://fluxcd.io/flux/installation/configuration/openshift/ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: