diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
index 83e6eb56..866af2c6 100644
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -29,10 +29,6 @@ jobs:
           version: v0.11.1
           image: kindest/node:v1.19.11@sha256:07db187ae84b4b7de440a73886f008cf903fcf5764ba8106a9fd5243d6f32729
           config: .github/kind/config.yaml # disable KIND-net
-      - name: Setup envtest
-        uses: fluxcd/pkg/actions/envtest@main
-        with:
-          version: "1.21.x"
       - name: Setup Calico for network policy
         run: |
           kubectl apply -f https://docs.projectcalico.org/v3.20/manifests/calico.yaml
diff --git a/.gitignore b/.gitignore
index c431bf85..69729b00 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,6 +20,7 @@ bin/
 output/
 cmd/flux/manifests/
 cmd/flux/.manifests.done
+testbin/
 
 # Docs
 site/
diff --git a/Makefile b/Makefile
index 4a58a157..29ca788a 100644
--- a/Makefile
+++ b/Makefile
@@ -1,8 +1,8 @@
 VERSION?=$(shell grep 'VERSION' cmd/flux/main.go | awk '{ print $$4 }' | head -n 1 | tr -d '"')
 EMBEDDED_MANIFESTS_TARGET=cmd/flux/.manifests.done
 TEST_KUBECONFIG?=/tmp/flux-e2e-test-kubeconfig
-ENVTEST_BIN_VERSION?=latest
-KUBEBUILDER_ASSETS?=$(shell $(SETUP_ENVTEST) use -i $(ENVTEST_BIN_VERSION) -p path)
+# Architecture to use envtest with
+ENVTEST_ARCH ?= amd64
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -34,6 +34,7 @@ cleanup-kind:
 	kind delete cluster --name=flux-e2e-test
 	rm $(TEST_KUBECONFIG)
 
+KUBEBUILDER_ASSETS?="$(shell $(ENVTEST) --arch=$(ENVTEST_ARCH) use -i $(ENVTEST_KUBERNETES_VERSION) --bin-dir=$(ENVTEST_ASSETS_DIR) -p path)"
 test: $(EMBEDDED_MANIFESTS_TARGET) tidy fmt vet install-envtest
 	KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test ./... -coverprofile cover.out --tags=unit
 
@@ -59,27 +60,33 @@ install:
 install-dev:
 	CGO_ENABLED=0 go build -o /usr/local/bin ./cmd/flux
 
-install-envtest:  setup-envtest
-	 $(SETUP_ENVTEST) use $(ENVTEST_BIN_VERSION)
-
 setup-bootstrap-patch:
 	go run ./tests/bootstrap/main.go
 
 setup-image-automation:
 	cd tests/image-automation && go run main.go
 
-# Find or download setup-envtest
-setup-envtest:
-ifeq (, $(shell which setup-envtest))
-	@{ \
-	set -e ;\
-	SETUP_ENVTEST_TMP_DIR=$$(mktemp -d) ;\
-	cd $$SETUP_ENVTEST_TMP_DIR ;\
-	go mod init tmp ;\
-	go get sigs.k8s.io/controller-runtime/tools/setup-envtest@latest ;\
-	rm -rf $$SETUP_ENVTEST_TMP_DIR ;\
-	}
-SETUP_ENVTEST=$(GOBIN)/setup-envtest
-else
-SETUP_ENVTEST=$(shell which setup-envtest)
-endif
+ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
+ENVTEST_KUBERNETES_VERSION?=latest
+install-envtest: setup-envtest
+	mkdir -p ${ENVTEST_ASSETS_DIR}
+	$(ENVTEST) use $(ENVTEST_KUBERNETES_VERSION) --arch=$(ENVTEST_ARCH) --bin-dir=$(ENVTEST_ASSETS_DIR)
+
+ENVTEST = $(shell pwd)/bin/setup-envtest
+.PHONY: envtest
+setup-envtest: ## Download envtest-setup locally if necessary.
+	$(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest)
+
+# go-install-tool will 'go install' any package $2 and install it to $1.
+PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
+define go-install-tool
+@[ -f $(1) ] || { \
+set -e ;\
+TMP_DIR=$$(mktemp -d) ;\
+cd $$TMP_DIR ;\
+go mod init tmp ;\
+echo "Downloading $(2)" ;\
+GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
+rm -rf $$TMP_DIR ;\
+}
+endef