diff --git a/Makefile b/Makefile
index 35d88c479e..14a13ea566 100644
--- a/Makefile
+++ b/Makefile
@@ -674,7 +674,7 @@ check-dependency-conflicts:
 	@echo
 	# Verify there are no conflicting dependencies
 	cat st2*/requirements.txt contrib/runners/*/requirements.txt | sort -u > req.txt && \
-	$(VIRTUALENV_DIR)/bin/pip-compile req.txt; \
+	$(VIRTUALENV_DIR)/bin/pip-compile req.txt || exit 1; \
 	if [[ -e req.txt ]]; then rm req.txt; fi
 
 .PHONY: virtualenv
diff --git a/test-requirements.txt b/test-requirements.txt
index 23738101bc..1c45f904b0 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -31,8 +31,10 @@ rstcheck>=3.3.1,<3.4
 tox==3.14.1
 pyrabbit
 # pip-tools provides pip-compile: to check for version conflicts
-# pip-tools 5.4 needs pip>=20.1, but we use 20.0.2
-pip-tools<5.4
+# pip-tools 5.3 needs pip<20.3
+# pip-tools 5.4 needs pip>=20.1
+# pip-tools 6.0 needs pip>=20.3
+pip-tools>=5.4,<6.1
 pytest==5.4.3
 pytest-benchmark==3.2.3
 # zstandard is used for micro benchmarks