diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 24da69780d..855295c152 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -239,7 +239,6 @@ if (NOT LLAMA_SANITIZE_ADDRESS AND NOT GGML_SCHED_NO_REALLOC)
   # TODO: repair known memory leaks
   llama_build_and_test(test-opt.cpp)
 endif()
-llama_build_and_test(test-gguf.cpp)
 llama_build_and_test(test-backend-ops.cpp)
 
 llama_build_and_test(test-model-load-cancel.cpp LABEL "model")
@@ -299,11 +298,15 @@ get_filename_component(TEST_TARGET test-c.c NAME_WE)
 add_executable(${TEST_TARGET} test-c.c)
 target_link_libraries(${TEST_TARGET} PRIVATE llama)
 
-llama_build_and_test(test-alloc.cpp)
-target_include_directories(test-alloc PRIVATE ${PROJECT_SOURCE_DIR}/ggml/src)
+if (NOT LLAMA_USE_SYSTEM_GGML)
+    # Needs non-public ggml-impl.h
+    llama_build_and_test(test-gguf.cpp)
+
+    # Needs non-public ggml{,-backend}-impl.h
+    llama_build_and_test(test-alloc.cpp)
+endif()
 
 llama_build(test-export-graph-ops.cpp)
-target_include_directories(test-export-graph-ops PRIVATE ${PROJECT_SOURCE_DIR}/ggml/src)
 if (TARGET gguf-model-data)
     target_link_libraries(test-export-graph-ops PRIVATE gguf-model-data)
     target_compile_definitions(test-export-graph-ops PRIVATE LLAMA_HF_FETCH)
diff --git a/tests/test-alloc.cpp b/tests/test-alloc.cpp
index 7ae739ad2e..6d5428493e 100644
--- a/tests/test-alloc.cpp
+++ b/tests/test-alloc.cpp
@@ -1,8 +1,8 @@
-#include <ggml-alloc.h>
-#include <ggml-backend-impl.h>
-#include <ggml-cpp.h>
-#include <ggml-impl.h>
-#include <ggml.h>
+#include "ggml-alloc.h"
+#include "../ggml/src/ggml-backend-impl.h"
+#include "ggml-cpp.h"
+#include "../ggml/src/ggml-impl.h"
+#include "ggml.h"
 
 #include <algorithm>
 #include <exception>
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 084344fb25..86d17b3f60 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -15,10 +15,10 @@
 // ##############################
 
 
-#include <ggml.h>
-#include <ggml-alloc.h>
-#include <ggml-backend.h>
-#include <ggml-cpp.h>
+#include "ggml.h"
+#include "ggml-alloc.h"
+#include "ggml-backend.h"
+#include "ggml-cpp.h"
 
 #include <algorithm>
 #include <atomic>