diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3571e33e..f38a8f89 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -272,6 +272,13 @@ if(PYBIND11_INSTALL)
   endif()
 endif()
 
-if(PYBIND11_TEST OR (BUILD_TESTING AND PYBIND11_MASTER_PROJECT))
-  add_subdirectory(tests)
+# BUILD_TESTING takes priority, but only if this is the master project
+if(PYBIND11_MASTER_PROJECT AND DEFINED BUILD_TESTING)
+  if(BUILD_TESTING)
+    add_subdirectory(tests)
+  endif()
+else()
+  if(PYBIND11_TEST)
+    add_subdirectory(tests)
+  endif()
 endif()