ClaudeBot/llama.cpp
0
0
Fork 1
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-17 20:31:47 +02:00
Code Issues Packages Projects Releases Wiki Activity

CI/tests: refactor argument handling, add --errors-only to emit only WARN/ERR messages

Browse Source
This commit is contained in:
Piotr Wilkin
2026-09-17 18:47:59 +02:00
parent 2f3fd02526
commit 10532516c3
64 changed files with 2352 additions and 948 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/fusion.yml
+2 -2
View File
@@ -63,9 +63,9 @@ jobs:
id: generate_models
run: |
rm -rf build-ci-models && mkdir -p build-ci-models
./build/bin/test-llama-archs -o build-ci-models
./build/bin/test-llama-archs -o build-ci-models --errors-only
- name: Test fusion
id: test_fusion
run: |
./build/bin/test-fusion --models build-ci-models --device MTL0 --check tests/fusion/MTL.csv
./build/bin/test-fusion --models build-ci-models --device MTL0 --check tests/fusion/MTL.csv --errors-only
ci/run.sh
+15 -15
View File
@@ -278,17 +278,17 @@ function gg_run_test_llama_archs_tensor_split {
set -e
if [ ! -z ${GG_BUILD_CUDA} ]; then
GGML_CUDA_DEVICES=1 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
GGML_CUDA_DEVICES=2 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
GGML_CUDA_DEVICES=3 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
GGML_CUDA_DEVICES=4 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
GGML_CUDA_DEVICES=1 ./build-ci-release/bin/test-llama-archs -s 1 --errors-only 2>&1
GGML_CUDA_DEVICES=2 ./build-ci-release/bin/test-llama-archs -s 1 --errors-only 2>&1
GGML_CUDA_DEVICES=3 ./build-ci-release/bin/test-llama-archs -s 1 --errors-only 2>&1
GGML_CUDA_DEVICES=4 ./build-ci-release/bin/test-llama-archs -s 1 --errors-only 2>&1
fi
if [ ! -z ${GG_BUILD_METAL} ]; then
GGML_METAL_DEVICES=1 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
GGML_METAL_DEVICES=2 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
GGML_METAL_DEVICES=3 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
GGML_METAL_DEVICES=4 ./build-ci-release/bin/test-llama-archs -s 1 2>&1
GGML_METAL_DEVICES=1 ./build-ci-release/bin/test-llama-archs -s 1 --errors-only 2>&1
GGML_METAL_DEVICES=2 ./build-ci-release/bin/test-llama-archs -s 1 --errors-only 2>&1
GGML_METAL_DEVICES=3 ./build-ci-release/bin/test-llama-archs -s 1 --errors-only 2>&1
GGML_METAL_DEVICES=4 ./build-ci-release/bin/test-llama-archs -s 1 --errors-only 2>&1
fi
set +e
@@ -307,7 +307,7 @@ function gg_run_test_llama_archs_models {
rm -rf build-ci-models && mkdir -p build-ci-models
# generate the dummy models used by the model-dependent tests
./build-ci-release/bin/test-llama-archs -o build-ci-models 2>&1
./build-ci-release/bin/test-llama-archs -o build-ci-models --errors-only 2>&1
fi
set +e
@@ -455,10 +455,10 @@ function gg_run_qwen3_0_6b {
(time ./bin/llama-imatrix --model ${model_f16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log
(time ./bin/test-save-load-state --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
(time ./bin/test-save-load-state --model ${model_q4_0} -ngl 10 -c 1024 -fa on --no-op-offload) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
(time ./bin/test-save-load-state --model ${model_q4_0} -ngl 99 -c 1024 -fa off ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
(time ./bin/test-save-load-state --model ${model_q4_0} -ngl 99 -c 1024 -fa on ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
(time ./bin/test-save-load-state --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload --errors-only) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
(time ./bin/test-save-load-state --model ${model_q4_0} -ngl 10 -c 1024 -fa on --no-op-offload --errors-only) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
(time ./bin/test-save-load-state --model ${model_q4_0} -ngl 99 -c 1024 -fa off --errors-only) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
(time ./bin/test-save-load-state --model ${model_q4_0} -ngl 99 -c 1024 -fa on --errors-only) 2>&1 | tee -a $OUT/${ci}-save-load-state.log
function check_ppl {
qnt="$1"
@@ -676,9 +676,9 @@ function gg_run_test_backend_ops {
# TODO: reduce the test-backend-ops timeout to 1800s
if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then
(time timeout 3600 ./bin/test-backend-ops ${args_extra} -b CPU) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
(time timeout 3600 ./bin/test-backend-ops ${args_extra} -b CPU --errors-only) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
else
(time timeout 3600 ./bin/test-backend-ops ${args_extra} ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
(time timeout 3600 ./bin/test-backend-ops ${args_extra} --errors-only) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log
fi
set +e
common/arg.cpp
+8
View File
@@ -3937,6 +3937,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
common_log_set_verbosity_thold(value);
}
).set_env("LLAMA_ARG_LOG_VERBOSITY"));
add_opt(common_arg(
{"--errors-only"},
"Set verbosity threshold to WARN (only print warnings and errors)",
[](common_params & params) {
params.verbosity = LOG_LEVEL_WARN;
common_log_set_verbosity_thold(LOG_LEVEL_WARN);
}
));
add_opt(common_arg(
{"--log-prefix"},
{"--no-log-prefix"},
common/common.cpp
+3 -1
View File
@@ -1573,7 +1573,9 @@ char * common_get_model_or_exit(int argc, char * argv[]) {
char * path = getenv("LLAMACPP_TEST_MODELFILE");
if (!path || strlen(path) == 0) {
fprintf(stderr, "\033[33mWARNING: No model file provided. Skipping this test. Set LLAMACPP_TEST_MODELFILE=<gguf_model_path> to silence this warning and run this test.\n\033[0m");
const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN;
fprintf(stderr, "%sWARNING: No model file provided. Skipping this test. Set LLAMACPP_TEST_MODELFILE=<gguf_model_path> to silence this warning and run this test.\n%s",
use_color ? LOG_COL_YELLOW : "", use_color ? LOG_COL_DEFAULT : "");
exit(EXIT_SUCCESS);
}
scripts/snapdragon/qdc/tests/linux/run_linux.sh
+1 -1
View File
@@ -197,7 +197,7 @@ run_backend_ops_case() {
esac
echo "=== [backend-ops:$dtype] test-backend-ops -b HTP0 -o MUL_MAT ==="
timeout 600 env GGML_HEXAGON_NDEV=1 GGML_HEXAGON_HOSTBUF=0 ./bin/test-backend-ops \
-b HTP0 -o MUL_MAT -p "$pattern" \
-b HTP0 -o MUL_MAT -p "$pattern" --errors-only \
> "$log" 2>&1
local rc=$?
note_timeout_if_triggered "$rc" 600 "$log"
scripts/snapdragon/qdc/tests/run_backend_ops_posix.py
+1 -1
View File
@@ -32,7 +32,7 @@ def test_backend_ops_htp0(type_a):
pattern = f"type_a={type_a}"
result = run_snapdragon(
["test-backend-ops", "-b", "HTP0", "-o", "MUL_MAT", "-p", pattern],
["test-backend-ops", "-b", "HTP0", "-o", "MUL_MAT", "-p", pattern, "--errors-only"],
)
write_qdc_log(f"backend_ops_{type_a}.log", result.stdout or "")
assert result.returncode == 0, (
src/llama-vocab.cpp
+1 -1
View File
@@ -1955,7 +1955,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
// read vocab size from metadata
uint32_t n_tokens = 0;
if (ml.get_key(LLM_KV_VOCAB_SIZE, n_tokens, false)) {
LLAMA_LOG_WARN("%s: adding %u dummy tokens\n", __func__, n_tokens);
LLAMA_LOG_INFO("%s: adding %u dummy tokens\n", __func__, n_tokens);
id_to_token.resize(n_tokens);
}
tests/CMakeLists.txt
+2 -2
View File
@@ -40,7 +40,7 @@ function(llama_test target)
add_test(
NAME ${TEST_NAME}
WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY}
COMMAND $<TARGET_FILE:${TEST_TARGET}>
COMMAND $<TARGET_FILE:${TEST_TARGET}> --errors-only
${LLAMA_TEST_ARGS})
set_property(TEST ${TEST_NAME} PROPERTY LABELS ${LLAMA_TEST_LABEL})
@@ -110,7 +110,7 @@ function(llama_build_and_test source)
add_test(
NAME ${TEST_TARGET}
WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY}
COMMAND $<TARGET_FILE:${TEST_TARGET}>
COMMAND $<TARGET_FILE:${TEST_TARGET}> --errors-only
${LLAMA_TEST_ARGS})
set_property(TEST ${TEST_TARGET} PROPERTY LABELS ${LLAMA_TEST_LABEL})
tests/test-alloc.cpp
+23 -6
View File
@@ -4,6 +4,10 @@
#include "../ggml/src/ggml-impl.h"
#include "ggml.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#include <algorithm>
#include <exception>
#include <memory>
@@ -189,7 +193,7 @@ static int get_leaf_id(ggml_cgraph * graph, const char * tensor_name) {
return i;
}
}
fprintf(stderr, "leaf not found: %s\n", tensor_name);
LOG_ERR("leaf not found: %s\n", tensor_name);
return -1;
}
@@ -199,7 +203,7 @@ static int get_node_id(ggml_cgraph * graph, const char * tensor_name) {
return i;
}
}
fprintf(stderr, "node not found: %s", tensor_name);
LOG_ERR("node not found: %s", tensor_name);
return -1;
}
@@ -651,13 +655,23 @@ static void test_graph_optimize_alloc_dep() {
}
static void run(const char * name, void (*f)()) {
printf("%s ", name);
fflush(stdout);
LOG_INF(" running %s\n", name);
// drain the queue, so the name is on disk if f() aborts
common_log_flush(common_log_main());
f();
printf("PASSED\n");
LOG_CNT("PASSED\n");
}
int main() {
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-alloc");
run("test_max_size_too_many_tensors", test_max_size_too_many_tensors);
run("test_max_size_tensor_too_large", test_max_size_tensor_too_large);
run("test_tensor_larger_than_max_size", test_tensor_larger_than_max_size);
@@ -672,5 +686,8 @@ int main() {
run("test_buffer_size_zero", test_buffer_size_zero);
run("test_reallocation", test_reallocation);
run("test_graph_optimize_alloc_dep", test_graph_optimize_alloc_dep);
LOG("%s: %s\n", "test-alloc", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-arg-parser.cpp
+35 -20
View File
@@ -2,6 +2,7 @@
#include "common.h"
#include "download.h"
#include "llama.h"
#include "log.h"
#include "speculative.h"
#include <cmath>
@@ -101,7 +102,7 @@ static void test(void) {
assert(draft.n_outputs_max_per_seq == 1);
}
printf("test-arg-parser: make sure there is no duplicated arguments in any examples\n\n");
LOG_INF("test-arg-parser: make sure there is no duplicated arguments in any examples\n\n");
for (int ex = 0; ex < LLAMA_EXAMPLE_COUNT; ex++) {
try {
auto ctx_arg = common_params_parser_init(params, (enum llama_example)ex);
@@ -114,7 +115,7 @@ static void test(void) {
if (seen_args.find(arg) == seen_args.end()) {
seen_args.insert(arg);
} else {
fprintf(stderr, "test-arg-parser: found different handlers for the same argument: %s", arg.c_str());
LOG_ERR("test-arg-parser: found different handlers for the same argument: %s", arg.c_str());
exit(1);
}
}
@@ -123,7 +124,7 @@ static void test(void) {
if (seen_env_vars.find(env) == seen_env_vars.end()) {
seen_env_vars.insert(env);
} else {
fprintf(stderr, "test-arg-parser: found different handlers for the same env var: %s", env.c_str());
LOG_ERR("test-arg-parser: found different handlers for the same env var: %s", env.c_str());
exit(1);
}
}
@@ -138,7 +139,7 @@ static void test(void) {
const std::string last(opt.args.back());
if (first.length() > last.length()) {
fprintf(stderr, "test-arg-parser: shorter argument should come before longer one: %s, %s\n",
LOG_ERR("test-arg-parser: shorter argument should come before longer one: %s, %s\n",
first.c_str(), last.c_str());
assert(false);
}
@@ -150,14 +151,15 @@ static void test(void) {
const std::string last(opt.args_neg.back());
if (first.length() > last.length()) {
fprintf(stderr, "test-arg-parser: shorter negated argument should come before longer one: %s, %s\n",
LOG_ERR("test-arg-parser: shorter negated argument should come before longer one: %s, %s\n",
first.c_str(), last.c_str());
assert(false);
}
}
}
} catch (std::exception & e) {
printf("%s\n", e.what());
LOG_ERR("%s\n", e.what());
common_log_flush(common_log_main());
assert(false);
}
}
@@ -172,7 +174,7 @@ static void test(void) {
std::vector<std::string> argv;
printf("test-arg-parser: test invalid usage\n\n");
LOG_INF("test-arg-parser: test invalid usage\n\n");
// missing value
argv = {"binary_name", "-m"};
@@ -229,7 +231,7 @@ static void test(void) {
argv = {"binary_name", "-lm", "hello"};
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
printf("test-arg-parser: test valid usage\n\n");
LOG_INF("test-arg-parser: test valid usage\n\n");
argv = {"binary_name", "-m", "model_file.gguf"};
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
@@ -305,9 +307,9 @@ static void test(void) {
// skip this part on windows, because setenv is not supported
#ifdef _WIN32
printf("test-arg-parser: skip on windows build\n");
LOG_INF("test-arg-parser: skip on windows build\n");
#else
printf("test-arg-parser: test environment variables (valid + invalid usages)\n\n");
LOG_INF("test-arg-parser: test environment variables (valid + invalid usages)\n\n");
setenv("LLAMA_ARG_THREADS", "blah", true);
argv = {"binary_name"};
@@ -344,7 +346,7 @@ static void test(void) {
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
assert(params.load_mode == LLAMA_LOAD_MODE_DIRECT_IO);
printf("test-arg-parser: test negated environment variables\n\n");
LOG_INF("test-arg-parser: test negated environment variables\n\n");
setenv("LLAMA_ARG_LOAD_MODE", "none", true);
setenv("LLAMA_ARG_NO_PERF", "1", true); // legacy format
@@ -353,7 +355,7 @@ static void test(void) {
assert(params.load_mode == LLAMA_LOAD_MODE_NONE);
assert(params.no_perf == true);
printf("test-arg-parser: test environment variables being overwritten\n\n");
LOG_INF("test-arg-parser: test environment variables being overwritten\n\n");
setenv("LLAMA_ARG_MODEL", "blah.gguf", true);
setenv("LLAMA_ARG_THREADS", "1010", true);
@@ -363,12 +365,12 @@ static void test(void) {
assert(params.cpuparams.n_threads == 1010);
#endif // _WIN32
printf("test-arg-parser: test download functions\n\n");
LOG_INF("test-arg-parser: test download functions\n\n");
const char * GOOD_URL = "http://ggml.ai/";
const char * BAD_URL = "http://ggml.ai/404";
{
printf("test-arg-parser: test good URL\n\n");
LOG_INF("test-arg-parser: test good URL\n\n");
auto res = common_remote_get_content(GOOD_URL, {});
assert(res.first == 200);
assert(res.second.size() > 0);
@@ -377,32 +379,45 @@ static void test(void) {
}
{
printf("test-arg-parser: test bad URL\n\n");
LOG_INF("test-arg-parser: test bad URL\n\n");
auto res = common_remote_get_content(BAD_URL, {});
assert(res.first == 404);
}
{
printf("test-arg-parser: test max size error\n");
LOG_INF("test-arg-parser: test max size error\n");
common_remote_params params;
params.max_size = 1;
try {
common_remote_get_content(GOOD_URL, params);
assert(false && "it should throw an error");
} catch (std::exception & e) {
printf(" expected error: %s\n\n", e.what());
LOG_INF(" expected error: %s\n\n", e.what());
}
}
printf("test-arg-parser: all tests OK\n\n");
LOG_INF("test-arg-parser: all tests OK\n\n");
}
int main(void) {
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-arg-parser");
try {
test();
} catch (std::exception & e) {
fprintf(stderr, "test-arg-parser: exception: %s\n", e.what());
LOG_ERR("test-arg-parser: exception: %s\n", e.what());
LOG("%s: %s\n", "test-arg-parser", "FAILED");
common_log_flush(common_log_main());
return 1;
}
LOG("%s: %s\n", "test-arg-parser", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-autorelease.cpp
+33 -1
View File
@@ -1,13 +1,43 @@
// ref: https://github.com/ggml-org/llama.cpp/issues/4952#issuecomment-1892864763
#include <thread>
#include <vector>
#include "llama.h"
#include "common.h"
#include "arg.h"
#include "log.h"
// This creates a new context inside a pthread and then tries to exit cleanly.
int main(int argc, char ** argv) {
auto * model_path = common_get_model_or_exit(argc, argv);
// the model path is this test's only positional argument
char * model_argv[2] = { argv[0], nullptr };
{
common_params params;
params.model.path = "."; // this test takes no model
common_init();
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else if (model_argv[1] == nullptr) {
model_argv[1] = argv[i];
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
}
// falls back to LLAMACPP_TEST_MODELFILE, or warns and exits if no model is given
auto * model_path = common_get_model_or_exit(model_argv[1] == nullptr ? 1 : 2, model_argv);
// that call exits when no model is given, so the verdict below is only reached with a model
LOG("%s: running\n", "test-autorelease");
std::thread([&model_path]() {
llama_backend_init();
@@ -18,5 +48,7 @@ int main(int argc, char ** argv) {
llama_backend_free();
}).join();
LOG("%s: %s\n", "test-autorelease", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-backend-ops.cpp
+84 -49
View File
@@ -20,6 +20,10 @@
#include "ggml-backend.h"
#include "ggml-cpp.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#include <algorithm>
#include <atomic>
#include <array>
@@ -833,6 +837,14 @@ struct printer {
};
struct console_printer : public printer {
// the printer is created after the CLI has been parsed, so the threshold is final here:
// no ANSI escapes at WARN or lower (--errors-only)
const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN;
const char * const col_ok = use_color ? "\033[1;32m" : "";
const char * const col_fail = use_color ? "\033[1;31m" : "";
const char * const col_blue = use_color ? "\033[1;34m" : "";
const char * const col_end = use_color ? "\033[0m" : "";
void print_test_result(const test_result & result) override {
if (result.test_mode == "test") {
print_test_console(result);
@@ -844,21 +856,21 @@ struct console_printer : public printer {
}
void print_operation(const test_operation_info & info) override {
printf(" %s(%s): ", info.op_name.c_str(), info.op_params.c_str());
LOG_CNT(" %s(%s): ", info.op_name.c_str(), info.op_params.c_str());
fflush(stdout);
// Handle large tensor skip first
if (info.is_large_tensor_skip) {
printf("skipping large tensors for speed \n");
LOG_CNT("skipping large tensors for speed \n");
return;
}
// Handle not supported status
if (info.status == test_status_t::NOT_SUPPORTED) {
if (!info.failure_reason.empty()) {
printf("not supported [%s]\n", info.failure_reason.c_str());
LOG_CNT("not supported [%s]\n", info.failure_reason.c_str());
} else {
printf("not supported [%s]\n", info.backend_name.c_str());
LOG_CNT("not supported [%s]\n", info.backend_name.c_str());
}
return;
}
@@ -866,52 +878,52 @@ struct console_printer : public printer {
// Handle errors and additional information
if (info.has_error) {
if (info.error_component == "allocation") {
fprintf(stderr, "failed to allocate tensors [%s] ", info.backend_name.c_str());
LOG_ERR("failed to allocate tensors [%s] ", info.backend_name.c_str());
} else if (info.error_component == "backend") {
fprintf(stderr, " Failed to initialize %s backend\n", info.backend_name.c_str());
LOG_ERR(" Failed to initialize %s backend\n", info.backend_name.c_str());
} else {
fprintf(stderr, "Error in %s: %s\n", info.error_component.c_str(), info.error_details.c_str());
LOG_ERR("Error in %s: %s\n", info.error_component.c_str(), info.error_details.c_str());
}
}
// Handle gradient info
if (info.has_gradient_info) {
printf("[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", info.op_name.c_str(), info.gradient_index,
info.gradient_param_name.c_str(), info.gradient_value);
LOG_CNT("[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", info.op_name.c_str(), info.gradient_index,
info.gradient_param_name.c_str(), info.gradient_value);
}
// Handle MAA error
if (info.has_maa_error) {
printf("[%s] MAA = %.9f > %.9f ", info.op_name.c_str(), info.maa_error, info.maa_threshold);
LOG_CNT("[%s] MAA = %.9f > %.9f ", info.op_name.c_str(), info.maa_error, info.maa_threshold);
}
// Handle compare failure
if (info.is_compare_failure) {
printf("compare failed ");
LOG_CNT("compare failed ");
}
// Print final status
if (info.status == test_status_t::OK) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
}
void print_summary(const test_summary_info & info) override {
if (info.is_backend_summary) {
printf("%zu/%zu backends passed\n", info.tests_passed, info.tests_total);
LOG_CNT("%zu/%zu backends passed\n", info.tests_passed, info.tests_total);
} else {
printf(" %zu/%zu tests passed\n", info.tests_passed, info.tests_total);
LOG_CNT(" %zu/%zu tests passed\n", info.tests_passed, info.tests_total);
}
}
void print_backend_status(const backend_status_info & info) override {
printf(" Backend %s: ", info.backend_name.c_str());
LOG_CNT(" Backend %s: ", info.backend_name.c_str());
if (info.status == test_status_t::OK) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
}
@@ -920,30 +932,30 @@ struct console_printer : public printer {
}
void print_backend_init(const backend_init_info & info) override {
printf("Backend %zu/%zu: %s\n", info.device_index + 1, info.total_devices, info.device_name.c_str());
LOG_INF("Backend %zu/%zu: %s\n", info.device_index + 1, info.total_devices, info.device_name.c_str());
if (info.skipped) {
printf(" %s\n", info.skip_reason.c_str());
LOG_INF(" %s\n", info.skip_reason.c_str());
return;
}
if (!info.description.empty()) {
printf(" Device description: %s\n", info.description.c_str());
LOG_INF(" Device description: %s\n", info.description.c_str());
}
if (info.has_memory_info) {
printf(" Device memory: %zu MB (%zu MB free)\n", info.memory_total_mb, info.memory_free_mb);
LOG_INF(" Device memory: %zu MB (%zu MB free)\n", info.memory_total_mb, info.memory_free_mb);
}
printf("\n");
LOG_CNT("\n");
}
void print_overall_summary(const overall_summary_info & info) override {
printf("%zu/%zu backends passed\n", info.backends_passed, info.backends_total);
if (info.all_passed) {
printf("\033[1;32mOK\033[0m\n");
printf("%sOK%s\n", col_ok, col_end);
} else {
printf("\033[1;31mFAIL\033[0m\n");
printf("%sFAIL%s\n", col_fail, col_end);
}
}
@@ -960,28 +972,30 @@ struct console_printer : public printer {
private:
void print_test_console(const test_result & result) {
printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str());
LOG_CNT(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str());
fflush(stdout);
if (!result.supported) {
printf("not supported [%s] ", result.backend_name.c_str());
printf("\n");
LOG_CNT("not supported [%s] ", result.backend_name.c_str());
LOG_CNT("\n");
return;
}
if (result.passed) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
}
void print_perf_console(const test_result & result) {
int len = printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str());
const std::string label = " " + result.op_name + "(" + result.op_params + "): ";
int len = (int) label.size();
LOG_CNT("%s", label.c_str());
fflush(stdout);
if (!result.supported) {
printf("not supported\n");
LOG_CNT("not supported\n");
return;
}
@@ -991,9 +1005,9 @@ struct console_printer : public printer {
if (last - len < 5) {
last += align;
}
printf("%*s", last - len, "");
LOG_CNT("%*s", last - len, "");
printf(" %8d runs - %8.2f us/run - ", result.n_runs, result.time_us);
LOG_CNT(" %8d runs - %8.2f us/run - ", result.n_runs, result.time_us);
if (result.flops > 0) {
auto format_flops = [](double flops) -> std::string {
@@ -1010,22 +1024,22 @@ struct console_printer : public printer {
return buf;
};
uint64_t op_flops_per_run = result.flops * result.time_us / 1e6;
printf("%s/run - \033[1;34m%sS\033[0m", format_flops(op_flops_per_run).c_str(),
format_flops(result.flops).c_str());
LOG_CNT("%s/run - %s%sS%s", format_flops(op_flops_per_run).c_str(), col_blue,
format_flops(result.flops).c_str(), col_end);
} else {
printf("%8zu kB/run - \033[1;34m%7.2f GB/s\033[0m", result.memory_kb, result.bandwidth_gb_s);
LOG_CNT("%8zu kB/run - %s%7.2f GB/s%s", result.memory_kb, col_blue, result.bandwidth_gb_s, col_end);
}
printf("\n");
LOG_CNT("\n");
}
void print_support_console(const test_result & result) {
printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str());
LOG_CNT(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str());
fflush(stdout);
if (result.supported) {
printf("\033[1;32mSUPPORTED\033[0m\n");
LOG_CNT("%sSUPPORTED%s\n", col_ok, col_end);
} else {
printf("\033[1;31mNOT SUPPORTED\033[0m\n");
LOG_CNT("%sNOT SUPPORTED%s\n", col_fail, col_end);
}
}
};
@@ -1595,7 +1609,7 @@ struct test_case {
// warmup run
ggml_status status = ggml_backend_graph_compute(backend, gf);
if (status != GGML_STATUS_SUCCESS) {
fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
LOG_ERR("%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
return false;
}
@@ -1650,7 +1664,7 @@ struct test_case {
int64_t start_time = ggml_time_us();
ggml_status status = ggml_backend_graph_compute(backend, gf);
if (status != GGML_STATUS_SUCCESS) {
fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
LOG_ERR("%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
return false;
}
int64_t end_time = ggml_time_us();
@@ -1841,12 +1855,12 @@ struct test_case {
ggml_status status = ggml_backend_graph_compute(backend, gf);
if (status != GGML_STATUS_SUCCESS) {
fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
LOG_ERR("%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
return false;
}
status = ggml_backend_graph_compute(backend, gb);
if (status != GGML_STATUS_SUCCESS) {
fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
LOG_ERR("%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
return false;
}
@@ -1900,7 +1914,7 @@ struct test_case {
ggml_backend_tensor_set(t, &xiu, i*sizeof(float), sizeof(float));
status = ggml_backend_graph_compute(backend, gf);
if (status != GGML_STATUS_SUCCESS) {
fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
LOG_ERR("%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
return false;
}
ggml_backend_tensor_get(out, &fu, 0, ggml_nbytes(out));
@@ -1908,7 +1922,7 @@ struct test_case {
ggml_backend_tensor_set(t, &xid, i*sizeof(float), sizeof(float));
status = ggml_backend_graph_compute(backend, gf);
if (status != GGML_STATUS_SUCCESS) {
fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
LOG_ERR("%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
return false;
}
ggml_backend_tensor_get(out, &fd, 0, ggml_nbytes(out));
@@ -1917,7 +1931,7 @@ struct test_case {
ggml_backend_tensor_set(t, &xiuh, i*sizeof(float), sizeof(float));
status = ggml_backend_graph_compute(backend, gf);
if (status != GGML_STATUS_SUCCESS) {
fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
LOG_ERR("%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
return false;
}
ggml_backend_tensor_get(out, &fuh, 0, ggml_nbytes(out));
@@ -1925,7 +1939,7 @@ struct test_case {
ggml_backend_tensor_set(t, &xidh, i*sizeof(float), sizeof(float));
status = ggml_backend_graph_compute(backend, gf);
if (status != GGML_STATUS_SUCCESS) {
fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
LOG_ERR("%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status));
return false;
}
ggml_backend_tensor_get(out, &fdh, 0, ggml_nbytes(out));
@@ -11918,6 +11932,10 @@ static void usage(char ** argv) {
}
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
test_mode mode = MODE_TEST;
output_formats output_format = CONSOLE;
const char * op_names_filter = nullptr;
@@ -11926,6 +11944,9 @@ int main(int argc, char ** argv) {
const char * test_file_path = nullptr;
int parallel_workers = 1;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "test") == 0) {
mode = MODE_TEST;
@@ -11990,12 +12011,21 @@ int main(int argc, char ** argv) {
usage(argv);
return 1;
}
} else if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else {
usage(argv);
return 1;
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-backend-ops");
// load and enumerate backends
ggml_backend_load_all();
@@ -12056,12 +12086,17 @@ int main(int argc, char ** argv) {
output_printer->print_footer();
}
common_log_flush(common_log_main());
output_printer->print_overall_summary(
overall_summary_info(n_ok, ggml_backend_dev_count(), n_ok == ggml_backend_dev_count()));
if (n_ok != ggml_backend_dev_count()) {
LOG("%s: %s\n", "test-backend-ops", "FAILED");
common_log_flush(common_log_main());
return 1;
}
LOG("%s: %s\n", "test-backend-ops", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-backend-sampler.cpp
+156 -99
View File
@@ -3,6 +3,9 @@
#include "llama-cpp.h"
#include "common.h"
#include "arg.h"
#include "log.h"
#ifdef NDEBUG
#undef NDEBUG
#endif
@@ -40,7 +43,7 @@ static llama_model_ptr load_model(const test_args & args) {
devs[0] = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU);
if (devs[0] == nullptr) {
fprintf(stderr, "Error: GPU requested but not available\n");
LOG_ERR("Error: GPU requested but not available\n");
return nullptr;
}
@@ -50,13 +53,13 @@ static llama_model_ptr load_model(const test_args & args) {
mparams.n_gpu_layers = 0;
} else {
fprintf(stderr, "Error: invalid device '%s'\n", args.device.c_str());
LOG_ERR("Error: invalid device '%s'\n", args.device.c_str());
return nullptr;
}
mparams.devices = devs;
fprintf(stderr, "Using device: %s\n", ggml_backend_dev_name(devs[0]));
LOG_INF("Using device: %s\n", ggml_backend_dev_name(devs[0]));
}
llama_model_ptr res;
@@ -64,7 +67,7 @@ static llama_model_ptr load_model(const test_args & args) {
res.reset(llama_model_load_from_file(args.model.c_str(), mparams));
if (!res) {
fprintf(stderr, "Warning: failed to load model '%s', skipping test\n", args.model.c_str());
LOG_WRN("Warning: failed to load model '%s', skipping test\n", args.model.c_str());
return nullptr;
}
@@ -140,7 +143,7 @@ struct test_context {
prompt_tokens.data(), prompt_tokens.size(),
false, false);
if (n_tokens < 0) {
fprintf(stderr, "Warning: tokenization failed for seq_id %d\n", seq_id);
LOG_WRN("Warning: tokenization failed for seq_id %d\n", seq_id);
llama_batch_free(batch);
return false;
}
@@ -162,19 +165,19 @@ struct test_context {
}
printf("Batch contents:\n");
printf("n_tokens: %d\n", batch.n_tokens);
LOG_CNT("Batch contents:\n");
LOG_CNT("n_tokens: %d\n", batch.n_tokens);
for (int i = 0; i < batch.n_tokens; i++) {
printf("token[%d]: tok=%-5d, pos=%d, n_seq_id=%d, seq_ids=[", i, batch.token[i], batch.pos[i], batch.n_seq_id[i]);
LOG_CNT("token[%d]: tok=%-5d, pos=%d, n_seq_id=%d, seq_ids=[", i, batch.token[i], batch.pos[i], batch.n_seq_id[i]);
for (int j = 0; j < batch.n_seq_id[i]; j++) {
printf("%d%s", batch.seq_id[i][j], j < batch.n_seq_id[i]-1 ? ", " : "");
LOG_CNT("%d%s", batch.seq_id[i][j], j < batch.n_seq_id[i]-1 ? ", " : "");
}
printf("], logits=%d\n", batch.logits[i]);
LOG_CNT("], logits=%d\n", batch.logits[i]);
}
if (llama_decode(ctx.get(), batch) != 0) {
fprintf(stderr, "Warning: llama_decode failed\n");
LOG_WRN("Warning: llama_decode failed\n");
llama_batch_free(batch);
return false;
}
@@ -194,7 +197,7 @@ struct test_context {
int32_t idx_for_seq(llama_seq_id seq_id) {
auto it = last_batch_info.find(seq_id);
if (it == last_batch_info.end()) {
fprintf(stderr, "Error: no batch index found for seq_id %d\n", seq_id);
LOG_ERR("Error: no batch index found for seq_id %d\n", seq_id);
return -1;
}
return it->second;
@@ -218,7 +221,7 @@ struct test_context {
common_batch_add(batch, token, pos, { seq_id }, true);
if (llama_decode(ctx.get(), batch) != 0) {
fprintf(stderr, "Warning: llama_decode failed for token %d in seq %d\n", token, seq_id);
LOG_WRN("Warning: llama_decode failed for token %d in seq %d\n", token, seq_id);
llama_batch_free(batch);
return false;
}
@@ -242,7 +245,7 @@ struct test_context {
}
if (llama_decode(ctx.get(), batch) != 0) {
fprintf(stderr, "Warning: llama_decode failed for batch tokens\n");
LOG_WRN("Warning: llama_decode failed for batch tokens\n");
llama_batch_free(batch);
return false;
}
@@ -352,17 +355,17 @@ static void test_backend_greedy_sampling(const test_params & params) {
int32_t batch_idx = test_ctx.idx_for_seq(seq_id);
llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx);
printf("greedy sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str());
LOG_CNT("greedy sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str());
GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);
token = llama_get_sampled_token_ith(test_ctx.ctx.get(), -1);
printf("greedy sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str());
LOG_CNT("greedy sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str());
GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);
for (int i = 0; i < 10; i++) {
int32_t loop_idx = test_ctx.idx_for_seq(seq_id);
llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), loop_idx);
printf("Generation step %d: token id:%d, string: %s\n", i, token, test_ctx.token_to_piece(token, false).c_str());
LOG_CNT("Generation step %d: token id:%d, string: %s\n", i, token, test_ctx.token_to_piece(token, false).c_str());
if (!test_ctx.decode_token(token, 0)) {
GGML_ASSERT(false && "Failed to decode token");
}
@@ -388,14 +391,14 @@ static void test_backend_top_k_sampling(const test_params & params) {
float * logits = llama_get_sampled_logits_ith(test_ctx.ctx.get(), batch_idx);
uint32_t n_logits = llama_get_sampled_logits_count_ith(test_ctx.ctx.get(), batch_idx);
for (size_t i = 0; i < n_logits; ++i) {
printf("top_k logit[%zu] = %.6f\n", i, logits[i]);
LOG_CNT("top_k logit[%zu] = %.6f\n", i, logits[i]);
}
llama_token * candidates = llama_get_sampled_candidates_ith(test_ctx.ctx.get(), batch_idx);
uint32_t n_candidates = llama_get_sampled_candidates_count_ith(test_ctx.ctx.get(), batch_idx);
for (size_t i = 0; i < n_candidates; ++i) {
printf("top_k candidate[%zu] = %d : %s\n", i, candidates[i],
test_ctx.token_to_piece(candidates[i], false).c_str());
LOG_CNT("top_k candidate[%zu] = %d : %s\n", i, candidates[i],
test_ctx.token_to_piece(candidates[i], false).c_str());
}
// Sample using CPU sampler for verification that it is possible to do hybrid
@@ -408,7 +411,7 @@ static void test_backend_top_k_sampling(const test_params & params) {
llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx);
GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);
printf("backend top-k hybrid sampling test PASSED\n");
LOG_CNT("backend top-k hybrid sampling test PASSED\n");
}
static void test_backend_temp_sampling(const test_params & params) {
@@ -447,7 +450,7 @@ static void test_backend_temp_sampling(const test_params & params) {
llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx);
const std::string token_str = test_ctx.token_to_piece(token, false);
printf("Sequence 0 sampled token id:%d, string: '%s'\n", token, token_str.c_str());
LOG_CNT("Sequence 0 sampled token id:%d, string: '%s'\n", token, token_str.c_str());
GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);
}
@@ -463,14 +466,14 @@ static void test_backend_temp_sampling(const test_params & params) {
llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx);
const std::string token_str = test_ctx.token_to_piece(token, false);
printf("Sequence 1 sampled token id:%d, string: '%s'\n", token, token_str.c_str());
LOG_CNT("Sequence 1 sampled token id:%d, string: '%s'\n", token, token_str.c_str());
GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);
}
}
// lambda for testing non-positive temperature values.
auto test_argmax_temp = [&](float temp) {
printf("\nTesting temperature = %.1f\n", temp);
LOG_CNT("\nTesting temperature = %.1f\n", temp);
int seq_id = 0;
struct llama_sampler_chain_params backend_chain_params = llama_sampler_chain_default_params();
@@ -496,7 +499,7 @@ static void test_backend_temp_sampling(const test_params & params) {
test_argmax_temp(0.0f);
test_argmax_temp(-1.0f);
printf("backend temp sampling test PASSED\n");
LOG_CNT("backend temp sampling test PASSED\n");
}
static void test_backend_temp_ext_sampling(const test_params & params) {
@@ -529,7 +532,7 @@ static void test_backend_temp_ext_sampling(const test_params & params) {
// lambda for testing non-positive temp/delta/exponent values.
auto test_argmax_temp = [&](float temp, float delta, float exponent) {
printf("\nTesting temperature = %.1f, delta = %1.f, exponent = %1.f\n", temp, delta, exponent);
LOG_CNT("\nTesting temperature = %.1f, delta = %1.f, exponent = %1.f\n", temp, delta, exponent);
int seq_id = 0;
struct llama_sampler_chain_params backend_chain_params = llama_sampler_chain_default_params();
@@ -561,7 +564,7 @@ static void test_backend_temp_ext_sampling(const test_params & params) {
test_argmax_temp(-1.0f, 0.3f, 2.0f); // Greedy (temp<0)
test_argmax_temp(0.8f, 0.0f, 2.0f); // Temperature scaling
printf("backend temp_ext sampling test PASSED\n");
LOG_CNT("backend temp_ext sampling test PASSED\n");
}
static void test_backend_min_p_sampling(const test_params & params) {
@@ -600,20 +603,20 @@ static void test_backend_min_p_sampling(const test_params & params) {
llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx);
const std::string token_str = test_ctx.token_to_piece(token, false);
printf("min-p cpu sampled token id:%d, string: '%s'\n", token, token_str.c_str());
LOG_CNT("min-p cpu sampled token id:%d, string: '%s'\n", token, token_str.c_str());
GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);
// Decode and sample 10 more tokens
for (int i = 0; i < 10; i++) {
int32_t loop_idx = test_ctx.idx_for_seq(seq_id);
llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), loop_idx);
printf("min-p gen step %d: token id :%5.d, string: %s\n", i, token, test_ctx.token_to_piece(token, false).c_str());
LOG_CNT("min-p gen step %d: token id :%5.d, string: %s\n", i, token, test_ctx.token_to_piece(token, false).c_str());
if (!test_ctx.decode_token(token, 0)) {
GGML_ASSERT(false && "Failed to decode token");
}
}
printf("min-p sampling test PASSED\n");
LOG_CNT("min-p sampling test PASSED\n");
}
static void test_backend_top_p_sampling(const test_params & params) {
@@ -652,18 +655,18 @@ static void test_backend_top_p_sampling(const test_params & params) {
llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx);
const std::string token_str = test_ctx.token_to_piece(token, false);
printf("top-p cpu sampled token id:%d, string: '%s'\n", token, token_str.c_str());
LOG_CNT("top-p cpu sampled token id:%d, string: '%s'\n", token, token_str.c_str());
GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);
// Decode and sample 10 more tokens
for (int i = 0; i < 10; i++) {
int32_t loop_idx = test_ctx.idx_for_seq(seq_id);
llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), loop_idx);
printf("top-p gen step %d: token id :%5.d, string: %s\n", i, token, test_ctx.token_to_piece(token, false).c_str());
LOG_CNT("top-p gen step %d: token id :%5.d, string: %s\n", i, token, test_ctx.token_to_piece(token, false).c_str());
test_ctx.decode_token(token, 0);
}
printf("top-p sampling test PASSED\n");
LOG_CNT("top-p sampling test PASSED\n");
}
static void test_backend_multi_sequence_sampling(const test_params & params) {
@@ -697,7 +700,7 @@ static void test_backend_multi_sequence_sampling(const test_params & params) {
int32_t batch_idx = test_ctx.idx_for_seq(0);
llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx);
const std::string token_str = test_ctx.token_to_piece(token, false);
printf("Seq 0 sampled token id=%d, string='%s'\n", token, token_str.c_str());
LOG_CNT("Seq 0 sampled token id=%d, string='%s'\n", token, token_str.c_str());
GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);
}
@@ -706,12 +709,12 @@ static void test_backend_multi_sequence_sampling(const test_params & params) {
int32_t batch_idx= test_ctx.idx_for_seq(1);
llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx);
const std::string token_str = test_ctx.token_to_piece(token, false);
printf("Seq 1 sampled token id=%d, string='%s'\n", token, token_str.c_str());
LOG_CNT("Seq 1 sampled token id=%d, string='%s'\n", token, token_str.c_str());
GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);
}
// Generate tokens for each sequence
printf("\nMulti-sequence generation:\n");
LOG_CNT("\nMulti-sequence generation:\n");
for (int step = 0; step < 4; step++) {
std::map<llama_seq_id, llama_token> tokens;
@@ -719,7 +722,7 @@ static void test_backend_multi_sequence_sampling(const test_params & params) {
int32_t idx = test_ctx.idx_for_seq(seq_id);
llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), idx);
const std::string token_str = test_ctx.token_to_piece(token, false);
printf(" Seq %d, step %d: token id=%d, string='%s'\n", seq_id, step, token, token_str.c_str());
LOG_CNT(" Seq %d, step %d: token id=%d, string='%s'\n", seq_id, step, token, token_str.c_str());
tokens[seq_id] = token;
}
@@ -729,7 +732,7 @@ static void test_backend_multi_sequence_sampling(const test_params & params) {
}
}
printf("backend multi-sequence sampling test PASSED\n");
LOG_CNT("backend multi-sequence sampling test PASSED\n");
}
static void test_backend_dist_sampling(const test_params & params) {
@@ -749,15 +752,15 @@ static void test_backend_dist_sampling(const test_params & params) {
int32_t batch_idx = test_ctx.idx_for_seq(seq_id);
llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx);
printf("dist sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str());
LOG_CNT("dist sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str());
GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);
//GGML_ASSERT(llama_get_sampled_logits_ith(test_ctx.ctx.get(), batch_idx) == nullptr);
token = llama_get_sampled_token_ith(test_ctx.ctx.get(), -1);
printf("dist sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str());
LOG_CNT("dist sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str());
GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);
printf("backend dist sampling test PASSED\n");
LOG_CNT("backend dist sampling test PASSED\n");
}
static void test_backend_dist_sampling_and_cpu(const test_params & params) {
@@ -784,10 +787,10 @@ static void test_backend_dist_sampling_and_cpu(const test_params & params) {
llama_token backend_token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx);
llama_token cpu_token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx);
printf("dist & cpu sampled id:%d, string:'%s'\n", cpu_token, test_ctx.token_to_piece(cpu_token, false).c_str());
LOG_CNT("dist & cpu sampled id:%d, string:'%s'\n", cpu_token, test_ctx.token_to_piece(cpu_token, false).c_str());
GGML_ASSERT(backend_token == cpu_token);
printf("backend dist & cpu sampling test PASSED\n");
LOG_CNT("backend dist & cpu sampling test PASSED\n");
}
static void test_backend_logit_bias_sampling(const test_params & params) {
@@ -809,7 +812,7 @@ static void test_backend_logit_bias_sampling(const test_params & params) {
//logit_bias.push_back({ bias_token, +100.0f });
logit_bias.push_back({ bias_token, +10.0f });
printf("biasing token piece '%s' -> token id %d\n", piece.c_str(), bias_token);
LOG_CNT("biasing token piece '%s' -> token id %d\n", piece.c_str(), bias_token);
struct llama_sampler_chain_params backend_chain_params = llama_sampler_chain_default_params();
llama_sampler_ptr backend_sampler_chain(llama_sampler_chain_init(backend_chain_params));
@@ -830,10 +833,11 @@ static void test_backend_logit_bias_sampling(const test_params & params) {
}
llama_token backend_token = llama_get_sampled_token_ith(test_ctx.ctx.get(), test_ctx.idx_for_seq(seq_id));
printf("sampled token = %d, expected = %d\n", backend_token, bias_token);
LOG_CNT("sampled token = %d, expected = %d\n", backend_token, bias_token);
common_log_flush(common_log_main()); // the assert below aborts
GGML_ASSERT(backend_token == bias_token);
printf("backend logit bias sampling test PASSED\n");
LOG_CNT("backend logit bias sampling test PASSED\n");
}
static void accept_prompt(llama_sampler * smpl, const llama_vocab * vocab, const std::string & prompt) {
@@ -1014,7 +1018,7 @@ static sampler_comparison_stats compare_sampler_outputs(
const float logit = actual.logits[i];
if (!seen.insert(token).second || std::isnan(logit)) {
if (result.n_mismatch < 5) {
printf("%s token %d has invalid backend output\n", name, token);
LOG_CNT("%s token %d has invalid backend output\n", name, token);
}
++result.n_mismatch;
continue;
@@ -1026,7 +1030,7 @@ static sampler_comparison_stats compare_sampler_outputs(
++result.n_masked;
} else if (!allow_extra_candidates) {
if (result.n_mismatch < 5) {
printf("%s token %d was not masked\n", name, token);
LOG_CNT("%s token %d was not masked\n", name, token);
}
++result.n_mismatch;
}
@@ -1037,8 +1041,8 @@ static sampler_comparison_stats compare_sampler_outputs(
result.max_diff = std::max(result.max_diff, diff);
if (!std::isfinite(logit) || diff > 1e-3f) {
if (result.n_mismatch < 5) {
printf("%s mismatch token %d: cpu=%.6f backend=%.6f diff=%.6f\n",
name, token, it->second, logit, diff);
LOG_CNT("%s mismatch token %d: cpu=%.6f backend=%.6f diff=%.6f\n",
name, token, it->second, logit, diff);
}
++result.n_mismatch;
}
@@ -1047,14 +1051,15 @@ static sampler_comparison_stats compare_sampler_outputs(
for (const auto & item : expected) {
if (seen.find(item.first) == seen.end()) {
if (result.n_mismatch < 5) {
printf("%s missing backend token %d\n", name, item.first);
LOG_CNT("%s missing backend token %d\n", name, item.first);
}
++result.n_mismatch;
}
}
printf("%s logits: max_diff=%.6f n_masked=%d n_mismatch=%d\n",
name, result.max_diff, result.n_masked, result.n_mismatch);
LOG_CNT("%s logits: max_diff=%.6f n_masked=%d n_mismatch=%d\n",
name, result.max_diff, result.n_masked, result.n_mismatch);
common_log_flush(common_log_main()); // the callers abort right after this on a mismatch
return result;
}
@@ -1311,10 +1316,10 @@ static void compare_masking_penalties_logits(
}
static void test_backend_penalties_sampling(const test_params & params) {
printf("Testing backend penalties (repeat + freq + presence)\n");
LOG_CNT("Testing backend penalties (repeat + freq + presence)\n");
compare_penalties_logits(params, 64, 1.1f, 0.5f, 0.25f, "Hello Hello world");
printf("Testing backend penalties with penalty_last_n > 64\n");
LOG_CNT("Testing backend penalties with penalty_last_n > 64\n");
const auto * vocab = llama_model_get_vocab(params.model.get());
std::vector<llama_token> tokens(8);
int32_t n_tok = llama_tokenize(vocab, "a", 1, tokens.data(), (int32_t) tokens.size(), false, false);
@@ -1332,52 +1337,52 @@ static void test_backend_penalties_sampling(const test_params & params) {
}
});
printf("Testing backend penalties without filler entries\n");
LOG_CNT("Testing backend penalties without filler entries\n");
compare_penalties_logits(params, 64, 1.1f, 0.5f, 0.25f, "Hello", [](llama_sampler * smpl) {
for (llama_token token = 0; token < 64; ++token) {
llama_sampler_accept(smpl, token);
}
});
printf("Testing backend top-k followed by penalties\n");
LOG_CNT("Testing backend top-k followed by penalties\n");
compare_top_k_penalties_logits(params, 8, 64, 1.1f, 0.5f, 0.25f, "Hello",
penalties_position::after_filter);
printf("Testing backend penalties followed by top-k\n");
LOG_CNT("Testing backend penalties followed by top-k\n");
compare_top_k_penalties_logits(params, 8, 64, 1.1f, 0.5f, 0.25f, "Hello",
penalties_position::before_filter);
printf("Testing backend top-p followed by penalties\n");
LOG_CNT("Testing backend top-p followed by penalties\n");
compare_masking_penalties_logits(params, "top-p", []() {
return llama_sampler_init_top_p(0.9f, 0);
}, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, true);
printf("Testing backend top-p followed by penalties with a large history window\n");
LOG_CNT("Testing backend top-p followed by penalties with a large history window\n");
compare_masking_penalties_logits(params, "top-p large-window", []() {
return llama_sampler_init_top_p(0.9f, 0);
}, 4096, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, true);
printf("Testing backend penalties followed by top-p\n");
LOG_CNT("Testing backend penalties followed by top-p\n");
compare_masking_penalties_logits(params, "top-p", []() {
return llama_sampler_init_top_p(0.9f, 0);
}, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::before_filter, true);
printf("Testing backend min-p followed by penalties\n");
LOG_CNT("Testing backend min-p followed by penalties\n");
compare_masking_penalties_logits(params, "min-p", []() {
return llama_sampler_init_min_p(0.1f, 0);
}, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, false);
printf("Testing backend penalties followed by min-p\n");
LOG_CNT("Testing backend penalties followed by min-p\n");
compare_masking_penalties_logits(params, "min-p", []() {
return llama_sampler_init_min_p(0.1f, 0);
}, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::before_filter, false);
printf("Testing backend top-p followed by penalties with empty history\n");
LOG_CNT("Testing backend top-p followed by penalties with empty history\n");
compare_masking_penalties_logits(params, "top-p empty", []() {
return llama_sampler_init_top_p(0.9f, 0);
}, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, true, false);
printf("Testing backend top-p followed by individual penalties\n");
LOG_CNT("Testing backend top-p followed by individual penalties\n");
compare_masking_penalties_logits(params, "top-p repeat", []() {
return llama_sampler_init_top_p(0.9f, 0);
}, 64, 1.1f, 0.0f, 0.0f, "Hello", penalties_position::after_filter, true);
@@ -1388,10 +1393,10 @@ static void test_backend_penalties_sampling(const test_params & params) {
return llama_sampler_init_top_p(0.9f, 0);
}, 64, 1.0f, 0.0f, 0.25f, "Hello", penalties_position::after_filter, true);
printf("Testing backend penalty parameter values\n");
LOG_CNT("Testing backend penalty parameter values\n");
test_penalty_parameter_values(params);
printf("backend penalties sampling test PASSED\n");
LOG_CNT("backend penalties sampling test PASSED\n");
}
// This test verifies that it is possible to have two different backend samplers,
@@ -1427,7 +1432,7 @@ static void test_backend_mixed_sampling(const test_params & params) {
int32_t batch_idx = test_ctx.idx_for_seq(0);
llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx);
const std::string token_str = test_ctx.token_to_piece(token, false);
printf("sampled token id=%d, string='%s'\n", token, token_str.c_str());
LOG_CNT("sampled token id=%d, string='%s'\n", token, token_str.c_str());
GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);
//GGML_ASSERT(llama_get_sampled_logits_ith(test_ctx.ctx.get(), batch_idx) == nullptr);
//GGML_ASSERT(llama_get_sampled_logits_count_ith(test_ctx.ctx.get(), batch_idx) == 0);
@@ -1443,7 +1448,7 @@ static void test_backend_mixed_sampling(const test_params & params) {
GGML_ASSERT(llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx) == LLAMA_TOKEN_NULL);
}
printf("backend mixed sampling test PASSED\n");
LOG_CNT("backend mixed sampling test PASSED\n");
}
static void test_backend_set_sampler(const test_params & params) {
@@ -1466,11 +1471,11 @@ static void test_backend_set_sampler(const test_params & params) {
// Sample using backend sampler configured above
llama_token backend_token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx);
const std::string backend_token_str = test_ctx.token_to_piece(backend_token, false);
printf("dist sampled token = %d, string='%s'\n", backend_token, backend_token_str.c_str());
LOG_CNT("dist sampled token = %d, string='%s'\n", backend_token, backend_token_str.c_str());
// Now clear the backend sampler for this sequence.
llama_set_sampler(test_ctx.ctx.get(), seq_id, nullptr);
printf("Cleared backend sampler for seq_id %d\n", seq_id);
LOG_CNT("Cleared backend sampler for seq_id %d\n", seq_id);
// Sample using CPU sampler
struct llama_sampler_chain_params chain_params = llama_sampler_chain_default_params();
@@ -1490,7 +1495,7 @@ static void test_backend_set_sampler(const test_params & params) {
// Sample the token using the CPU sampler chain.
llama_token token2 = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), seq_id);
const std::string token2_str = test_ctx.token_to_piece(token2, false);
printf("CPU sampled token after clearing backend sampler: id=%d, string='%s'\n", token2, token2_str.c_str());
LOG_CNT("CPU sampled token after clearing backend sampler: id=%d, string='%s'\n", token2, token2_str.c_str());
std::map<llama_seq_id, llama_token> tokens2 = { { seq_id, token2}, };
// Set a new backend sampler for the sequence.
@@ -1506,9 +1511,9 @@ static void test_backend_set_sampler(const test_params & params) {
llama_token new_backend_token = llama_get_sampled_token_ith(test_ctx.ctx.get(), test_ctx.idx_for_seq(seq_id));
const std::string new_backend_token_str = test_ctx.token_to_piece(new_backend_token, false);
printf("dist sampled token = %d, string='%s'\n", new_backend_token, new_backend_token_str.c_str());
LOG_CNT("dist sampled token = %d, string='%s'\n", new_backend_token, new_backend_token_str.c_str());
printf("backend set sampler test PASSED\n");
LOG_CNT("backend set sampler test PASSED\n");
}
static void test_backend_cpu_mixed_batch(const test_params & params) {
@@ -1538,7 +1543,7 @@ static void test_backend_cpu_mixed_batch(const test_params & params) {
int32_t batch_idx = test_ctx.idx_for_seq(0);
llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx);
const std::string token_str = test_ctx.token_to_piece(token, false);
printf("Seq 0 (backend) sampled token id=%d, string='%s'\n", token, token_str.c_str());
LOG_CNT("Seq 0 (backend) sampled token id=%d, string='%s'\n", token, token_str.c_str());
GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);
}
@@ -1555,7 +1560,7 @@ static void test_backend_cpu_mixed_batch(const test_params & params) {
llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx);
const std::string token_str = test_ctx.token_to_piece(token, false);
printf("Seq 1 (CPU) sampled token id=%d, string='%s'\n", token, token_str.c_str());
LOG_CNT("Seq 1 (CPU) sampled token id=%d, string='%s'\n", token, token_str.c_str());
GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);
}
@@ -1592,11 +1597,11 @@ static void test_backend_cpu_mixed_batch(const test_params & params) {
int32_t batch_idx = test_ctx.idx_for_seq(0);
llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx);
const std::string token_str = test_ctx.token_to_piece(token, false);
printf("re-added backend sampled token id=%d, string='%s'\n", token, token_str.c_str());
LOG_CNT("re-added backend sampled token id=%d, string='%s'\n", token, token_str.c_str());
GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);
}
printf("backend-cpu mixed batch test PASSED\n");
LOG_CNT("backend-cpu mixed batch test PASSED\n");
}
static void test_backend_multi_output_limit(const test_params & params) {
@@ -1612,14 +1617,14 @@ static void test_backend_multi_output_limit(const test_params & params) {
common_batch_add(batch, llama_vocab_bos(test_ctx.vocab), i, { seq_id }, true);
}
printf(">>> test_backend_multi_output_limit expected error start:\n");
LOG_CNT(">>> test_backend_multi_output_limit expected error start:\n");
const int ret = llama_decode(test_ctx.ctx.get(), batch);
GGML_ASSERT(ret != 0 && "llama_decode should reject outputs above the per-sequence limit");
printf("<<< test_backend_multi_output_limit expected error end.\n");
LOG_CNT("<<< test_backend_multi_output_limit expected error end.\n");
llama_batch_free(batch);
printf("backend multi-output limit test PASSED\n");
LOG_CNT("backend multi-output limit test PASSED\n");
}
static void test_backend_multi_sequence_multi_output_dist(const test_params & params) {
@@ -1708,7 +1713,7 @@ static void test_backend_multi_sequence_multi_output_dist(const test_params & pa
llama_batch_free(batch);
printf("backend multi-sequence multi-output dist test PASSED\n");
LOG_CNT("backend multi-sequence multi-output dist test PASSED\n");
}
static void test_backend_multi_output_dist_transaction(const test_params & params) {
@@ -1778,7 +1783,7 @@ static void test_backend_multi_output_dist_transaction(const test_params & param
verify_random(0, randoms[2]);
llama_batch_free(batch);
printf("backend multi-output dist transaction test PASSED\n");
LOG_CNT("backend multi-output dist transaction test PASSED\n");
}
static void test_backend_multi_output_sampling_chain(const test_params & params) {
@@ -1928,7 +1933,7 @@ static void test_backend_multi_output_sampling_chain(const test_params & params)
GGML_ASSERT(llama_decode(test_ctx.ctx.get(), batch) == 0);
llama_batch_free(batch);
printf("backend multi-output sampling chain test PASSED\n");
LOG_CNT("backend multi-output sampling chain test PASSED\n");
}
static void test_backend_multi_output_cpu_suffix(const test_params & params) {
@@ -1990,7 +1995,7 @@ static void test_backend_multi_output_cpu_suffix(const test_params & params) {
llama_batch_free(batch);
}
printf("backend multi-output CPU suffix test PASSED\n");
LOG_CNT("backend multi-output CPU suffix test PASSED\n");
}
struct backend_test_case {
@@ -2029,7 +2034,8 @@ static test_args parse_cli(int argc, char ** argv) {
if (std::strcmp(arg, "--test") == 0) {
if (i + 1 >= argc) {
fprintf(stderr, "--test expects a value\n");
LOG_ERR("--test expects a value\n");
common_log_flush(common_log_main());
exit(EXIT_FAILURE);
}
out.test = argv[++i];
@@ -2041,7 +2047,8 @@ static test_args parse_cli(int argc, char ** argv) {
}
if (std::strcmp(arg, "--model") == 0) {
if (i + 1 >= argc) {
fprintf(stderr, "--model expects a value\n");
LOG_ERR("--model expects a value\n");
common_log_flush(common_log_main());
exit(EXIT_FAILURE);
}
out.model = argv[++i];
@@ -2053,7 +2060,8 @@ static test_args parse_cli(int argc, char ** argv) {
}
if (std::strcmp(arg, "--device") == 0) {
if (i + 1 >= argc) {
fprintf(stderr, "--device expects a value (cpu or gpu)\n");
LOG_ERR("--device expects a value (cpu or gpu)\n");
common_log_flush(common_log_main());
exit(EXIT_FAILURE);
}
out.device = argv[++i];
@@ -2068,12 +2076,14 @@ static test_args parse_cli(int argc, char ** argv) {
continue;
}
fprintf(stderr, "Unexpected argument: %s\n", arg);
LOG_ERR("Unexpected argument: %s\n", arg);
common_log_flush(common_log_main());
exit(EXIT_FAILURE);
}
if (out.device != "cpu" && out.device != "gpu" && out.device != "auto") {
fprintf(stderr, "Invalid device '%s'. Must be 'cpu', 'gpu' or 'auto'\n", out.device.c_str());
LOG_ERR("Invalid device '%s'. Must be 'cpu', 'gpu' or 'auto'\n", out.device.c_str());
common_log_flush(common_log_main());
exit(EXIT_FAILURE);
}
@@ -2091,10 +2101,12 @@ static std::vector<const backend_test_case *> collect_tests_to_run(const std::st
}
}
if (selected.empty()) {
fprintf(stderr, "Unknown test '%s'. Available tests:\n", requested.c_str());
LOG_ERR("Unknown test '%s'. Available tests:\n", requested.c_str());
for (const auto & test : BACKEND_TESTS) {
fprintf(stderr, " %s\n", test.name.c_str());
LOG_ERR(" %s\n", test.name.c_str());
}
LOG("%s: %s\n", "test-backend-sampler", "FAILED");
common_log_flush(common_log_main());
exit(EXIT_FAILURE);
}
} else {
@@ -2108,7 +2120,7 @@ static std::vector<const backend_test_case *> collect_tests_to_run(const std::st
test.name == "mixed" || test.name == "top_p" ||
test.name == "multi_output_sampling_chain" ||
test.name == "multi_output_cpu") {
fprintf(stderr, "Skipping test '%s' on HIP backend (no backend TOP_K support)\n", test.name.c_str());
LOG_WRN("Skipping test '%s' on HIP backend (no backend TOP_K support)\n", test.name.c_str());
continue;
}
#endif // GGML_USE_HIP
@@ -2117,7 +2129,7 @@ static std::vector<const backend_test_case *> collect_tests_to_run(const std::st
}
if (selected.empty()) {
fprintf(stderr, "No backend sampling tests selected. Use --test=<name> to pick one.\n");
LOG_WRN("No backend sampling tests selected. Use --test=<name> to pick one.\n");
}
return selected;
@@ -2125,32 +2137,75 @@ static std::vector<const backend_test_case *> collect_tests_to_run(const std::st
static void run_tests(const std::vector<const backend_test_case *> & tests, const test_params & args) {
for (const auto & test : tests) {
fprintf(stderr, "\n=== %s ===\n", test->name.c_str());
LOG_INF("\n=== %s ===\n", test->name.c_str());
try {
test->fn(args);
} catch (const std::exception & e) {
fprintf(stderr, "Error running test '%s': %s\n", test->name.c_str(), e.what());
LOG_ERR("Error running test '%s': %s\n", test->name.c_str(), e.what());
LOG("%s: %s\n", "test-backend-sampler", "FAILED");
common_log_flush(common_log_main());
exit(EXIT_FAILURE);
}
}
}
int main(int argc, char ** argv) {
test_args args = parse_cli(argc, argv);
test_args args;
{
common_params params;
params.model.path = "."; // this test takes no model
common_init();
// the test's own options and the model path are handled here, everything else goes to common_params_parse
std::vector<char *> test_argv;
std::vector<char *> common_argv;
test_argv.push_back(argv[0]);
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "--test") == 0 || strcmp(argv[i], "--model") == 0 || strcmp(argv[i], "--device") == 0) {
test_argv.push_back(argv[i]);
if (i + 1 < argc) {
test_argv.push_back(argv[++i]); // the option value
}
continue;
}
if (strncmp(argv[i], "--test=", 7) == 0 || strncmp(argv[i], "--model=", 8) == 0 || strncmp(argv[i], "--device=", 9) == 0) {
test_argv.push_back(argv[i]);
continue;
}
if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else {
test_argv.push_back(argv[i]); // the model path
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
test_argv.push_back(nullptr);
args = parse_cli((int) test_argv.size() - 1, test_argv.data());
}
if (args.model.empty()) {
args.model = common_get_model_or_exit(1, argv);
}
LOG("%s: running\n", "test-backend-sampler");
{
std::ifstream file(args.model);
if (!file.is_open()) {
fprintf(stderr, "no model '%s' found\n", args.model.c_str());
LOG_ERR("no model '%s' found\n", args.model.c_str());
LOG("%s: %s\n", "test-backend-sampler", "FAILED");
common_log_flush(common_log_main());
return EXIT_FAILURE;
}
}
fprintf(stderr, "using '%s'\n", args.model.c_str());
LOG_INF("using '%s'\n", args.model.c_str());
llama_backend_init();
@@ -2163,5 +2218,7 @@ int main(int argc, char ** argv) {
run_tests(tests, params);
}
LOG("%s: %s\n", "test-backend-sampler", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-barrier.cpp
+56 -28
View File
@@ -1,11 +1,15 @@
#include "ggml.h"
#include "ggml-cpu.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#include <chrono>
#include <iostream>
#include <cstdio>
#include <cstdlib>
#include <cassert>
#include <string>
#include <vector>
#include <thread>
@@ -40,7 +44,8 @@ static void test_barrier(int n_threads, int n_rounds) {
struct ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads);
struct ggml_threadpool* threadpool = ggml_threadpool_new(&tpp);
if (!threadpool) {
fprintf(stderr, "threadpool create failed : n_threads %d\n", n_threads);
LOG_ERR("threadpool create failed : n_threads %d\n", n_threads);
common_log_flush(common_log_main());
exit(1);
}
@@ -50,11 +55,10 @@ static void test_barrier(int n_threads, int n_rounds) {
std::vector<uint8_t> work_data(cplan.work_size);
cplan.work_data = work_data.data();
std::cerr << "graph-compute with"
<< "\n n_threads: " << n_threads
<< "\n n_nodes: " << n_nodes
<< "\n n_rounds: " << n_rounds
<< "\n";
LOG_INF("graph-compute with"
"\n n_threads: %d"
"\n n_nodes: %d"
"\n n_rounds: %d\n", n_threads, n_nodes, n_rounds);
// ggml_graph_print(gf);
// Warmup
@@ -70,10 +74,10 @@ static void test_barrier(int n_threads, int n_rounds) {
auto usec = std::chrono::duration_cast<std::chrono::microseconds>(t1-t0).count();
auto nsec = std::chrono::duration_cast<std::chrono::nanoseconds>(t1-t0).count();
std::cerr << "graph-compute took " << usec << " usec "
<< "\n " << (float) usec / n_rounds << " usec per-iter"
<< "\n " << (float) nsec / (n_rounds * n_nodes) << " nsec per-node"
<< "\n";
LOG_INF("graph-compute took %lld usec "
"\n %g usec per-iter"
"\n %g nsec per-node\n",
(long long) usec, (float) usec / n_rounds, (float) nsec / (n_rounds * n_nodes));
ggml_threadpool_free(threadpool);
ggml_free(ctx);
@@ -108,15 +112,15 @@ static void test_active(int n_threads, int n_rounds) {
struct ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads);
struct ggml_threadpool* threadpool = ggml_threadpool_new(&tpp);
if (!threadpool) {
fprintf(stderr, "threadpool create failed : n_threads %d\n", n_threads);
LOG_ERR("threadpool create failed : n_threads %d\n", n_threads);
common_log_flush(common_log_main());
exit(1);
}
std::cerr << "graph-compute with"
<< "\n n_threads: " << n_threads
<< "\n n_nodes: " << n_nodes
<< "\n n_rounds: " << n_rounds
<< "\n";
LOG_INF("graph-compute with"
"\n n_threads: %d"
"\n n_nodes: %d"
"\n n_rounds: %d\n", n_threads, n_nodes, n_rounds);
// ggml_graph_print(gf);
// In this test we keep changing the number of threads every 4th iteration
@@ -181,16 +185,17 @@ static void test_multi_graph(int n_threads, int n_rounds) {
struct ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads);
struct ggml_threadpool* threadpool = ggml_threadpool_new(&tpp);
if (!threadpool) {
fprintf(stderr, "threadpool create failed : n_threads %d\n", n_threads);
LOG_ERR("threadpool create failed : n_threads %d\n", n_threads);
common_log_flush(common_log_main());
exit(1);
}
std::cerr << "graph-compute with"
<< "\n gf0 n_nodes: " << ggml_graph_n_nodes(gf0)
<< "\n gf1 n_nodes: " << ggml_graph_n_nodes(gf1)
<< "\n n_threads: " << n_threads
<< "\n n_rounds: " << n_rounds
<< "\n";
LOG_INF("graph-compute with"
"\n gf0 n_nodes: %d"
"\n gf1 n_nodes: %d"
"\n n_threads: %d"
"\n n_rounds: %d\n",
ggml_graph_n_nodes(gf0), ggml_graph_n_nodes(gf1), n_threads, n_rounds);
// In this test we keep changing the number of threads every 4th iteration
// and we compute two graphs back to back to test graph frequent graph switching
@@ -214,23 +219,46 @@ static void test_multi_graph(int n_threads, int n_rounds) {
int main(int argc, char *argv[]) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
// this test takes n_threads and n_rounds as positional arguments
std::vector<std::string> positional;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else {
positional.push_back(argv[i]);
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
int n_threads = std::max(1, std::min(4, (int) std::thread::hardware_concurrency()));
int n_rounds = 100;
if (argc > 1) {
n_threads = std::atoi(argv[1]);
if (positional.size() > 0) {
n_threads = std::atoi(positional[0].c_str());
}
if (argc > 2) {
n_rounds = std::atoi(argv[2]);
if (positional.size() > 1) {
n_rounds = std::atoi(positional[1].c_str());
}
LOG("%s: running\n", "test-barrier");
test_barrier(n_threads, n_rounds);
test_active(n_threads, n_rounds * 100);
test_multi_graph(n_threads, n_rounds * 10);
LOG("%s: %s\n", "test-barrier", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-batch-alloc.cpp
+30 -3
View File
@@ -1,6 +1,8 @@
#include "arg.h"
#include "testing.h"
#include "llama.h"
#include "log.h"
#include "../src/llama-batch.h"
#include "../src/llama-memory.h"
@@ -651,6 +653,28 @@ static void test_mrope(testing & t) {
}
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
// this test takes an optional filter as its only positional argument
std::string filter;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else if (filter.empty()) {
filter = argv[i];
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-batch-alloc");
testing t;
const char * verbose = getenv("LLAMA_TEST_VERBOSE");
@@ -661,8 +685,8 @@ int main(int argc, char ** argv) {
llama_log_set([](ggml_log_level, const char *, void *) {}, nullptr);
}
if (argc > 1) {
t.set_filter(argv[1]);
if (!filter.empty()) {
t.set_filter(filter);
}
t.test("init", test_init);
@@ -670,5 +694,8 @@ int main(int argc, char ** argv) {
t.test("keep_tail", test_keep_tail);
t.test("mrope", test_mrope);
return t.summary();
const int rc = t.summary();
LOG("%s: %s\n", "test-batch-alloc", rc == 0 ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return rc;
}
tests/test-c.c
+5 -1
View File
@@ -1,3 +1,7 @@
#include "llama.h"
int main(void) {}
int main(void) {
printf("test-c: running\n");
printf("test-c: PASSED\n");
return 0;
}
tests/test-chat-analysis.cpp
+87 -49
View File
@@ -1,6 +1,8 @@
#include "chat-auto-parser.h"
#include "chat-auto-parser-helpers.h"
#include "chat.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#include "jinja/caps.h"
#include "jinja/runtime.h"
@@ -84,16 +86,16 @@ static std::string read_file(const std::string & path) {
}
static void print_usage(const char * program_name) {
LOG_ERR("Debug the auto-parser's differential analysis: render a template with/without tools, reasoning, etc. and show the diffs.\n");
LOG_ERR("\nUsage: %s [options]\n", program_name);
LOG_ERR("\nOptions:\n");
LOG_ERR(" --template <name> Analyze specific template from test suite (e.g., 'deepseek' or 'DeepSeek-V3.1')\n");
LOG_ERR(" --template-file <path> Analyze custom template file\n");
LOG_ERR(" --all Analyze all templates from test suite (default when no arguments are given)\n");
LOG_ERR("\nExamples:\n");
LOG_ERR(" %s --all\n", program_name);
LOG_ERR(" %s --template deepseek\n", program_name);
LOG_ERR(" %s --template-file my-template.jinja\n", program_name);
LOG_INF("Debug the auto-parser's differential analysis: render a template with/without tools, reasoning, etc. and show the diffs.\n");
LOG_INF("\nUsage: %s [options]\n", program_name);
LOG_INF("\nOptions:\n");
LOG_INF(" --template <name> Analyze specific template from test suite (e.g., 'deepseek' or 'DeepSeek-V3.1')\n");
LOG_INF(" --template-file <path> Analyze custom template file\n");
LOG_INF(" --all Analyze all templates from test suite (default when no arguments are given)\n");
LOG_INF("\nExamples:\n");
LOG_INF(" %s --all\n", program_name);
LOG_INF(" %s --template deepseek\n", program_name);
LOG_INF(" %s --template-file my-template.jinja\n", program_name);
}
static bool parse_options(int argc, char ** argv, analysis_options & opts) {
@@ -270,15 +272,15 @@ static json make_assistant_one_tool_with_reasoning() {
}
static void print_diff_split(const std::string & title, const diff_split & diff) {
LOG_ERR("\n%s=== %s ===%s\n", ANSI_CYAN, title.c_str(), ANSI_RESET);
LOG_ERR("%sCommon Prefix:%s '%s'\n", ANSI_PREFIX, ANSI_RESET, diff.prefix.c_str());
LOG_ERR("%sCommon Suffix:%s '%s'\n", ANSI_SUFFIX, ANSI_RESET, diff.suffix.c_str());
LOG_ERR("%sLeft (difference):%s '%s'\n", ANSI_GREEN, ANSI_RESET, diff.left.c_str());
LOG_ERR("%sRight (difference):%s '%s'\n", ANSI_ORANGE, ANSI_RESET, diff.right.c_str());
LOG_INF("\n%s=== %s ===%s\n", ANSI_CYAN, title.c_str(), ANSI_RESET);
LOG_INF("%sCommon Prefix:%s '%s'\n", ANSI_PREFIX, ANSI_RESET, diff.prefix.c_str());
LOG_INF("%sCommon Suffix:%s '%s'\n", ANSI_SUFFIX, ANSI_RESET, diff.suffix.c_str());
LOG_INF("%sLeft (difference):%s '%s'\n", ANSI_GREEN, ANSI_RESET, diff.left.c_str());
LOG_INF("%sRight (difference):%s '%s'\n", ANSI_ORANGE, ANSI_RESET, diff.right.c_str());
}
static void check_reasoning_variables(const common_chat_template & tmpl) {
LOG_ERR("\n%s=== Checking Reasoning Variables ===%s\n", ANSI_CYAN, ANSI_RESET);
LOG_INF("\n%s=== Checking Reasoning Variables ===%s\n", ANSI_CYAN, ANSI_RESET);
try {
// Create a list of candidate reasoning/thinking variable names to probe
@@ -355,11 +357,11 @@ static void check_reasoning_variables(const common_chat_template & tmpl) {
}
if (accessed_vars.empty()) {
LOG_ERR("%sNo reasoning/thinking-related variables were queried by the template%s\n", ANSI_GRAY, ANSI_RESET);
LOG_INF("%sNo reasoning/thinking-related variables were queried by the template%s\n", ANSI_GRAY, ANSI_RESET);
} else {
LOG_ERR("Template queries the following reasoning/thinking-related variables:\n");
LOG_INF("Template queries the following reasoning/thinking-related variables:\n");
for (const auto & var : accessed_vars) {
LOG_ERR(" %s- %s%s\n", ANSI_ORANGE, var.c_str(), ANSI_RESET);
LOG_INF(" %s- %s%s\n", ANSI_ORANGE, var.c_str(), ANSI_RESET);
}
}
@@ -369,12 +371,12 @@ static void check_reasoning_variables(const common_chat_template & tmpl) {
}
static void analyze_template(const std::string & template_path) {
LOG_ERR("\n");
LOG_ERR("%s", ANSI_PURPLE);
LOG_ERR("================================================================================\n");
LOG_ERR(" ANALYZING TEMPLATE: %s\n", template_path.c_str());
LOG_ERR("================================================================================\n");
LOG_ERR("%s", ANSI_RESET);
LOG_INF("\n");
LOG_INF("%s", ANSI_PURPLE);
LOG_INF("================================================================================\n");
LOG_INF(" ANALYZING TEMPLATE: %s\n", template_path.c_str());
LOG_INF("================================================================================\n");
LOG_INF("%s", ANSI_RESET);
std::string template_source;
try {
@@ -389,14 +391,14 @@ static void analyze_template(const std::string & template_path) {
json tools = build_tools_definition();
// ===== CAPABILITIES ANALYSIS =====
LOG_ERR("\n%s=== Template Capabilities (from jinja::caps) ===%s\n", ANSI_CYAN, ANSI_RESET);
LOG_INF("\n%s=== Template Capabilities (from jinja::caps) ===%s\n", ANSI_CYAN, ANSI_RESET);
auto caps = chat_template.original_caps();
LOG_ERR("%ssupports_tools:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_tools ? "true" : "false");
LOG_ERR("%ssupports_tool_calls:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_tool_calls ? "true" : "false");
LOG_ERR("%ssupports_system_role:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_system_role ? "true" : "false");
LOG_ERR("%ssupports_parallel_tool_calls:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_parallel_tool_calls ? "true" : "false");
LOG_ERR("%ssupports_typed_content:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_typed_content ? "true" : "false");
LOG_ERR("%ssupports_string_content:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_string_content ? "true" : "false");
LOG_INF("%ssupports_tools:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_tools ? "true" : "false");
LOG_INF("%ssupports_tool_calls:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_tool_calls ? "true" : "false");
LOG_INF("%ssupports_system_role:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_system_role ? "true" : "false");
LOG_INF("%ssupports_parallel_tool_calls:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_parallel_tool_calls ? "true" : "false");
LOG_INF("%ssupports_typed_content:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_typed_content ? "true" : "false");
LOG_INF("%ssupports_string_content:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_string_content ? "true" : "false");
// ===== DIFFERENTIAL ANALYSIS =====
@@ -584,32 +586,68 @@ static void analyze_template(const std::string & template_path) {
}
int main(int argc, char ** argv) {
// Set log level to capture all output
common_log_set_verbosity_thold(99);
common_params params;
params.model.path = "."; // placeholder so common_params_parse does not require --model
common_init();
analysis_options opts;
if (!parse_options(argc, argv, opts)) {
// this tool handles --all/--template/--template-file and -h itself; every other option goes to the common parser
std::vector<char *> common_argv;
std::vector<char *> own_argv;
common_argv.push_back(argv[0]);
own_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
const std::string arg = argv[i];
if (arg == "--all" || arg == "-h" || arg == "--help") {
own_argv.push_back(argv[i]);
} else if (arg == "--template" || arg == "--template-file") {
own_argv.push_back(argv[i]);
if (i + 1 < argc) {
own_argv.push_back(argv[++i]);
}
} else if (arg[0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else {
own_argv.push_back(argv[i]); // unrecognised token: parse_options reports it
}
}
common_argv.push_back(nullptr);
own_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG_ERR("\n");
LOG_ERR("%s", ANSI_PURPLE);
LOG_ERR("================================================================================\n");
LOG_ERR(" TEMPLATE ANALYSIS TOOL\n");
LOG_ERR("================================================================================\n");
LOG_ERR("%s", ANSI_RESET);
LOG_ERR("Analyzing %s%zu%s template(s)\n", ANSI_CYAN, opts.template_paths.size(), ANSI_RESET);
// capture all output unless the user asked for a lower verbosity threshold
common_log_set_verbosity_thold(params.verbosity < LOG_DEFAULT_LLAMA ? params.verbosity : 99);
analysis_options opts;
if (!parse_options((int) own_argv.size() - 1, own_argv.data(), opts)) {
common_log_flush(common_log_main());
return 1;
}
LOG("%s: running\n", "test-chat-analysis");
LOG_INF("\n");
LOG_INF("%s", ANSI_PURPLE);
LOG_INF("================================================================================\n");
LOG_INF(" TEMPLATE ANALYSIS TOOL\n");
LOG_INF("================================================================================\n");
LOG_INF("%s", ANSI_RESET);
LOG_INF("Analyzing %s%zu%s template(s)\n", ANSI_CYAN, opts.template_paths.size(), ANSI_RESET);
for (const auto & path : opts.template_paths) {
LOG_INF(" running %s\n", path.c_str());
analyze_template(path);
}
LOG_ERR("\n");
LOG_ERR("%s", ANSI_GREEN);
LOG_ERR("================================================================================\n");
LOG_ERR(" ANALYSIS COMPLETE\n");
LOG_ERR("================================================================================\n");
LOG_ERR("%s", ANSI_RESET);
LOG_INF("\n");
LOG_INF("%s", ANSI_GREEN);
LOG_INF("================================================================================\n");
LOG_INF(" ANALYSIS COMPLETE\n");
LOG_INF("================================================================================\n");
LOG_INF("%s", ANSI_RESET);
LOG("%s: %s\n", "test-chat-analysis", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-chat-auto-parser.cpp
+106 -57
View File
@@ -4,6 +4,8 @@
#include "chat.h"
#include "gguf.h"
#include "jinja/runtime.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#include "peg-parser.h"
#include "testing.h"
@@ -16,6 +18,7 @@
#include <optional>
#include <sstream>
#include <string>
#include <vector>
using namespace autoparser;
@@ -175,24 +178,24 @@ static std::string read_gguf_chat_template(const std::string & path) {
}
static void print_usage(const char * program_name) {
LOG_ERR("Test the chat template auto-parser; also usable as a debug tool that shows the generated PEG parser, GBNF grammar and triggers for a given template.\n");
LOG_ERR("\nUsage: %s [filter_regex] run the automated tests (default)\n", program_name);
LOG_ERR(" %s <template_or_gguf_path> [options] debug a single template\n", program_name);
LOG_ERR("\nDebug mode options:\n");
LOG_ERR(" --no-tools Disable tool definitions\n");
LOG_ERR(" --force-tool-call Set tool calls to forced\n");
LOG_ERR(" --parallel-tool-calls=0|1 Set parallel_tool_calls (default: 1)\n");
LOG_ERR(" --generation-prompt=0|1 Set add_generation_prompt (default: 1)\n");
LOG_ERR(" --enable-reasoning=0|1 Enable reasoning parsing (default: 1)\n");
LOG_ERR(" --output=MODE Output mode: analysis, template, both (default: both)\n");
LOG_ERR(" --debug-jinja Enable Jinja fine-grained debug\n");
LOG_ERR(" --input-message=TYPE Message type to render:\n");
LOG_ERR(" content_only, reasoning_content, tool_call_only,\n");
LOG_ERR(" content_tool_call, reasoning_tool_call,\n");
LOG_ERR(" content_fake_tool_call, all\n");
LOG_ERR("\nExamples:\n");
LOG_ERR(" %s template.jinja --input-message=all --generation-prompt=1\n", program_name);
LOG_ERR(" %s template.jinja --output=template --input-message=tool_call_only\n", program_name);
LOG_INF("Test the chat template auto-parser; also usable as a debug tool that shows the generated PEG parser, GBNF grammar and triggers for a given template.\n");
LOG_INF("\nUsage: %s [filter_regex] run the automated tests (default)\n", program_name);
LOG_INF(" %s <template_or_gguf_path> [options] debug a single template\n", program_name);
LOG_INF("\nDebug mode options:\n");
LOG_INF(" --no-tools Disable tool definitions\n");
LOG_INF(" --force-tool-call Set tool calls to forced\n");
LOG_INF(" --parallel-tool-calls=0|1 Set parallel_tool_calls (default: 1)\n");
LOG_INF(" --generation-prompt=0|1 Set add_generation_prompt (default: 1)\n");
LOG_INF(" --enable-reasoning=0|1 Enable reasoning parsing (default: 1)\n");
LOG_INF(" --output=MODE Output mode: analysis, template, both (default: both)\n");
LOG_INF(" --debug-jinja Enable Jinja fine-grained debug\n");
LOG_INF(" --input-message=TYPE Message type to render:\n");
LOG_INF(" content_only, reasoning_content, tool_call_only,\n");
LOG_INF(" content_tool_call, reasoning_tool_call,\n");
LOG_INF(" content_fake_tool_call, all\n");
LOG_INF("\nExamples:\n");
LOG_INF(" %s template.jinja --input-message=all --generation-prompt=1\n", program_name);
LOG_INF(" %s template.jinja --output=template --input-message=tool_call_only\n", program_name);
}
static bool parse_bool_option(const std::string & value) {
@@ -336,8 +339,8 @@ static void render_scenario(const common_chat_template & tmpl,
const json & tools,
bool add_generation_prompt,
bool enable_thinking) {
LOG_ERR("\n=== Scenario: %s ===\n", scenario_name.c_str());
LOG_ERR("add_generation_prompt: %s, enable_thinking: %s\n", add_generation_prompt ? "true" : "false",
LOG_INF("\n=== Scenario: %s ===\n", scenario_name.c_str());
LOG_INF("add_generation_prompt: %s, enable_thinking: %s\n", add_generation_prompt ? "true" : "false",
enable_thinking ? "true" : "false");
// When add_generation_prompt is true, add a trailing user message to trigger the prompt
@@ -349,7 +352,7 @@ static void render_scenario(const common_chat_template & tmpl,
});
}
LOG_ERR("Messages:\n%s\n", final_messages.dump(2).c_str());
LOG_TRC("Messages:\n%s\n", final_messages.dump(2).c_str());
try {
generation_params inputs;
@@ -363,9 +366,9 @@ static void render_scenario(const common_chat_template & tmpl,
std::string output = common_chat_template_direct_apply(tmpl, inputs);
LOG_ERR("\n--- Rendered Output ---\n");
LOG_ERR("%s\n", output.c_str());
LOG_ERR("--- End Output (length: %zu) ---\n", output.length());
LOG_INF("\n--- Rendered Output ---\n");
LOG_INF("%s\n", output.c_str());
LOG_INF("--- End Output (length: %zu) ---\n", output.length());
} catch (const std::exception & e) {
LOG_ERR("Rendering failed: %s\n", e.what());
}
@@ -395,7 +398,7 @@ static void render_all_scenarios(const common_chat_template & tmpl,
// Also render with add_generation_prompt=true to show the prompt ending
if (message_type == input_message_type::ALL) {
LOG_ERR("\n\n=== Generation Prompt Scenarios (add_generation_prompt=true) ===\n");
LOG_INF("\n\n=== Generation Prompt Scenarios (add_generation_prompt=true) ===\n");
json prompt_messages = json::array({ user_msg });
render_scenario(tmpl, "generation_prompt_only", prompt_messages, tools, true, enable_thinking);
@@ -438,8 +441,8 @@ static int debug_single_template(const debug_options & opts) {
return 1;
}
LOG_ERR("Analyzing template: %s\n", opts.template_path.c_str());
LOG_ERR("Options: with_tools=%s, generation_prompt=%s, enable_reasoning=%s\n", opts.with_tools ? "true" : "false",
LOG_INF("Analyzing template: %s\n", opts.template_path.c_str());
LOG_INF("Options: with_tools=%s, generation_prompt=%s, enable_reasoning=%s\n", opts.with_tools ? "true" : "false",
opts.generation_prompt ? "true" : "false", opts.enable_reasoning ? "true" : "false");
try {
@@ -451,17 +454,17 @@ static int debug_single_template(const debug_options & opts) {
common_chat_params parser_data;
if (std::optional<common_chat_params> spec_tmpl =
common_chat_try_specialized_template(chat_template, template_source, params)) {
LOG_ERR("\n");
LOG_ERR("This template uses a specialized parser, analysis results will not be available.\n");
LOG_INF("\n");
LOG_INF("This template uses a specialized parser, analysis results will not be available.\n");
parser_data = *spec_tmpl;
} else {
// Render template scenarios if requested
if (opts.input_message != input_message_type::NONE &&
(opts.mode == output_mode::TEMPLATE || opts.mode == output_mode::BOTH)) {
LOG_ERR("\n");
LOG_ERR("================================================================================\n");
LOG_ERR(" TEMPLATE RENDERING OUTPUT\n");
LOG_ERR("================================================================================\n");
LOG_INF("\n");
LOG_INF("================================================================================\n");
LOG_INF(" TEMPLATE RENDERING OUTPUT\n");
LOG_INF("================================================================================\n");
render_all_scenarios(chat_template, tools, opts.generation_prompt, opts.enable_reasoning,
opts.input_message);
@@ -469,10 +472,10 @@ static int debug_single_template(const debug_options & opts) {
// Output analysis if requested
if (opts.mode == output_mode::ANALYSIS || opts.mode == output_mode::BOTH) {
LOG_ERR("\n");
LOG_ERR("================================================================================\n");
LOG_ERR(" TEMPLATE ANALYSIS\n");
LOG_ERR("================================================================================\n");
LOG_INF("\n");
LOG_INF("================================================================================\n");
LOG_INF(" TEMPLATE ANALYSIS\n");
LOG_INF("================================================================================\n");
struct autoparser analysis;
analysis.analyze_template(chat_template);
@@ -483,25 +486,25 @@ static int debug_single_template(const debug_options & opts) {
}
if (!std::empty(parser_data.parser)) {
LOG_ERR("\n=== Generated Parser ===\n");
LOG_INF("\n=== Generated Parser ===\n");
common_peg_arena arena;
arena.load(parser_data.parser);
LOG_ERR("%s\n", arena.dump(arena.root()).c_str());
LOG_INF("%s\n", arena.dump(arena.root()).c_str());
LOG_ERR("\n=== Generated Grammar ===\n");
LOG_ERR("%s\n", parser_data.grammar.c_str());
LOG_INF("\n=== Generated Grammar ===\n");
LOG_INF("%s\n", parser_data.grammar.c_str());
LOG_ERR("\n=== Generated Lazy Grammar ===\n");
LOG_ERR("%d\n", parser_data.grammar_lazy);
LOG_INF("\n=== Generated Lazy Grammar ===\n");
LOG_INF("%d\n", parser_data.grammar_lazy);
LOG_ERR("\n=== Generated Grammar Triggers ===\n");
LOG_INF("\n=== Generated Grammar Triggers ===\n");
for (const common_grammar_trigger & cgt : parser_data.grammar_triggers) {
LOG_ERR("Token: %d | Type: %d | Value: %s\n", cgt.token, cgt.type, cgt.value.c_str());
LOG_INF("Token: %d | Type: %d | Value: %s\n", cgt.token, cgt.type, cgt.value.c_str());
}
LOG_ERR("\n=== Preserved Tokens ===\n");
LOG_INF("\n=== Preserved Tokens ===\n");
for (const std::string & token : parser_data.preserved_tokens) {
LOG_ERR(" '%s'\n", token.c_str());
LOG_INF(" '%s'\n", token.c_str());
}
}
} catch (const std::exception & e) {
@@ -513,20 +516,59 @@ static int debug_single_template(const debug_options & opts) {
}
int main(int argc, char * argv[]) {
if (argc > 1) {
std::string arg = argv[1];
common_params params;
params.model.path = "."; // placeholder so common_params_parse does not require --model
common_init();
// this tool handles -h/--help, its debug options and one positional argument
// (template path or test filter) itself; every other option goes to the common parser
std::vector<char *> common_argv;
std::vector<char *> own_argv;
common_argv.push_back(argv[0]);
own_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
const std::string arg = argv[i];
if (arg == "-h" || arg == "--help" ||
arg == "--force-tool-call" || arg == "--debug-jinja" || arg == "--no-tools" ||
arg.rfind("--parallel-tool-calls=", 0) == 0 ||
arg.rfind("--generation-prompt=", 0) == 0 ||
arg.rfind("--enable-reasoning=", 0) == 0 ||
arg.rfind("--output=", 0) == 0 ||
arg.rfind("--input-message=", 0) == 0) {
own_argv.push_back(argv[i]);
} else if (arg[0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else {
own_argv.push_back(argv[i]); // positional: template path or test filter
}
}
common_argv.push_back(nullptr);
own_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
// capture all output unless the user asked for a lower verbosity threshold
const int own_verbosity = params.verbosity < LOG_DEFAULT_LLAMA ? params.verbosity : 99;
const int own_argc = (int) own_argv.size() - 1;
char ** own_argv_data = own_argv.data();
if (own_argc > 1) {
std::string arg = own_argv_data[1];
if (arg == "-h" || arg == "--help") {
common_log_set_verbosity_thold(99);
print_usage(argv[0]);
common_log_set_verbosity_thold(own_verbosity);
print_usage(own_argv_data[0]);
common_log_flush(common_log_main());
return 0;
}
// debug mode: if the first argument is an existing file, analyze that template instead of running the automated tests
if (std::filesystem::is_regular_file(arg)) {
common_log_set_verbosity_thold(99);
common_log_set_verbosity_thold(own_verbosity);
debug_options opts;
if (!parse_debug_options(argc, argv, opts)) {
if (!parse_debug_options(own_argc, own_argv_data, opts)) {
common_log_flush(common_log_main());
return 1;
}
@@ -534,17 +576,21 @@ int main(int argc, char * argv[]) {
jinja::enable_debug(true);
}
return debug_single_template(opts);
const int rc = debug_single_template(opts);
common_log_flush(common_log_main());
return rc;
}
}
LOG("%s: running\n", "test-chat-auto-parser");
testing t(std::cout);
t.verbose = true;
// usage: test-chat-auto-parser [filter_regex]
if (argc > 1) {
t.set_filter(argv[1]);
if (own_argc > 1) {
t.set_filter(own_argv_data[1]);
}
t.test("diff_split", test_calculate_diff_split);
@@ -565,7 +611,10 @@ int main(int argc, char * argv[]) {
t.test("bailing_v3", test_bailing_v3_tool_format);
t.test("role_markers_all_templates", test_role_markers_all_templates);
return t.summary();
const int rc = t.summary();
LOG("%s: %s\n", "test-chat-auto-parser", rc == 0 ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return rc;
}
static void test_marker_separation(testing & t) {
tests/test-chat-peg-parser.cpp
+31 -3
View File
@@ -1,7 +1,9 @@
#include "arg.h"
#include "chat-peg-parser.h"
#include "chat.h"
#include "common.h"
#include "json-schema-to-grammar.h"
#include "log.h"
#include "peg-parser.h"
#include "testing.h"
#include "peg-parser/simple-tokenize.h"
@@ -10,6 +12,7 @@
#include <numeric>
#include <regex>
#include <string>
#include <vector>
#include "json.h"
@@ -25,9 +28,31 @@ static void test_tagged_peg_parser(testing & t);
static void test_permute(testing & t);
int main(int argc, char * argv[]) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
// this test takes an optional filter as its only positional argument
std::string filter;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else if (filter.empty()) {
filter = argv[i];
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-chat-peg-parser");
testing t(std::cout);
if (argc >= 2) {
t.set_filter(argv[1]);
if (!filter.empty()) {
t.set_filter(filter);
}
const char * verbose = getenv("LLAMA_TEST_VERBOSE");
@@ -43,7 +68,10 @@ int main(int argc, char * argv[]) {
t.test("tagged peg parser", test_tagged_peg_parser);
t.test("permute", test_permute);
return t.summary();
const int rc = t.summary();
LOG("%s: %s\n", "test-chat-peg-parser", rc == 0 ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return rc;
}
static json create_tools() {
tests/test-chat-template.cpp
+33 -8
View File
@@ -13,7 +13,9 @@
#include <cassert>
#include "llama.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#include "chat.h"
#include "jinja/runtime.h"
#include "jinja/parser.h"
@@ -113,6 +115,10 @@ static std::string DEFAULT_JSON_WITH_TOOLS = R"({
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
std::vector<std::string> args(argv, argv + argc);
std::string tmpl_path;
@@ -123,6 +129,8 @@ int main(int argc, char ** argv) {
bool use_common = true;
bool dump_prog = false;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (size_t i = 1; i < args.size(); i++) {
if (args[i] == "--help" || args[i] == "-h") {
std::cout << HELP << "\n";
@@ -142,6 +150,8 @@ int main(int argc, char ** argv) {
use_common = false;
} else if (args[i] == "--dump-prog") {
dump_prog = true;
} else if (args[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else if (tmpl_path.empty()) {
tmpl_path = args[i];
} else {
@@ -150,16 +160,27 @@ int main(int argc, char ** argv) {
return 1;
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-chat-template");
if (tmpl_path.empty()) {
return main_automated_tests();
const int rc = main_automated_tests();
LOG("%s: %s\n", "test-chat-template", rc == 0 ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return rc;
}
json input_json;
if (!json_path.empty()) {
std::ifstream json_file(json_path);
if (!json_file) {
std::cerr << "Error: Could not open JSON file: " << json_path << "\n";
LOG_ERR("Error: Could not open JSON file: %s\n", json_path.c_str());
LOG("%s: %s\n", "test-chat-template", "FAILED");
common_log_flush(common_log_main());
return 1;
}
std::string content = std::string(
@@ -180,10 +201,14 @@ int main(int argc, char ** argv) {
std::istreambuf_iterator<char>());
run_single(contents, input_json, use_common, dump_prog, output_path);
} else {
std::cerr << "Error: PATH_TO_TEMPLATE is not a valid file or directory: " << tmpl_path << "\n";
LOG_ERR("Error: PATH_TO_TEMPLATE is not a valid file or directory: %s\n", tmpl_path.c_str());
LOG("%s: %s\n", "test-chat-template", "FAILED");
common_log_flush(common_log_main());
return 1;
}
LOG("%s: %s\n", "test-chat-template", "PASSED");
common_log_flush(common_log_main());
return 0;
}
@@ -197,7 +222,7 @@ void run_multiple(const std::string& dir_path, bool stop_on_first_fail, const js
// only process .jinja files
if (entry.path().extension() == ".jinja" && entry.is_regular_file()) {
test_count++;
std::cout << "\n\n=== RUNNING TEMPLATE FILE: " << entry.path().string() << " ===\n";
LOG_INF("=== RUNNING TEMPLATE FILE: %s ===\n", entry.path().string().c_str());
std::ifstream infile(entry.path());
std::string contents((std::istreambuf_iterator<char>(infile)), std::istreambuf_iterator<char>());
try {
@@ -662,9 +687,9 @@ int main_automated_tests(void) {
assert(res > 0);
supported_tmpl.resize(res);
res = llama_chat_builtin_templates(supported_tmpl.data(), supported_tmpl.size());
std::cout << "Built-in chat templates:\n";
LOG_INF("Built-in chat templates:\n");
for (const auto *tmpl : supported_tmpl) {
std::cout << " " << tmpl << "\n";
LOG_CNT(" %s\n", tmpl);
}
// test invalid chat template
@@ -673,7 +698,7 @@ int main_automated_tests(void) {
const auto add_generation_prompt = true;
for (const auto & test_case : test_cases) {
std::cout << "\n\n=== " << test_case.name << " ===\n\n";
LOG_INF("=== %s ===\n", test_case.name.c_str());
auto conv = conversation;
conv.insert(conv.end(), test_case.extra_conversation.begin(), test_case.extra_conversation.end());
formatted_chat.resize(2048);
@@ -705,7 +730,7 @@ int main_automated_tests(void) {
if (!test_case.supported_with_jinja) {
continue;
}
std::cout << "\n\n=== " << test_case.name << " (jinja) ===\n\n";
LOG_INF("=== %s (jinja) ===\n", test_case.name.c_str());
try {
auto msgs = messages;
for (const auto & msg : test_case.extra_conversation) {
tests/test-chat.cpp
+65 -33
View File
@@ -10,6 +10,7 @@
#include "../tools/server/server-chat.h"
#include "chat-auto-parser.h"
#include "chat.h"
#include "arg.h"
#include "common.h"
#include "ggml.h"
#include "log.h"
@@ -1118,6 +1119,8 @@ static bool g_force_reconstruction_test = false;
static void test_peg_parser(common_chat_templates * tmpls,
const std::function<void(peg_test_case &)> & init,
bool detailed_debug) {
const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN;
// UTF-8-safe truncation helper (same as in test_parser_with_streaming)
constexpr auto utf8_truncate_safe_len = [](const std::string_view s) -> size_t {
auto len = s.size();
@@ -1442,10 +1445,11 @@ static void test_peg_parser(common_chat_templates * tmpls,
size_t ctx_start = diff_pos > 60 ? diff_pos - 60 : 0;
size_t ctx_end_e = std::min(expected_text.size(), diff_pos + 40);
size_t ctx_end_r = std::min(reconstruction_params.prompt.size(), diff_pos + 40);
LOG_ERR("\x1b[31m[RECONSTRUCTION FAIL]\x1b[0m "
LOG_ERR("%s[RECONSTRUCTION FAIL]%s "
"first diff at byte %zu (expected len=%zu, reconstructed len=%zu)\n"
" expected: ...%s...\n"
" reconstructed: ...%s...\n",
use_color ? "\x1b[31m" : "", use_color ? "\x1b[0m" : "",
diff_pos, expected_text.size(), reconstruction_params.prompt.size(),
expected_text.substr(ctx_start, ctx_end_e - ctx_start).c_str(),
reconstruction_params.prompt.substr(ctx_start, ctx_end_r - ctx_start).c_str());
@@ -1461,6 +1465,18 @@ static void test_peg_parser(common_chat_templates * tmpls,
}
}
// Case-insensitive substring match against the --template filter
static bool template_matches_filter(const std::string & template_path) {
if (g_template_filter.empty()) {
return true;
}
std::string template_path_lower = template_path;
std::string filter_lower = g_template_filter;
std::transform(template_path_lower.begin(), template_path_lower.end(), template_path_lower.begin(), ::tolower);
std::transform(filter_lower.begin(), filter_lower.end(), filter_lower.begin(), ::tolower);
return template_path_lower.find(filter_lower) != std::string::npos;
}
// Fluent builder for PEG parser tests
class peg_test_builder;
@@ -1474,7 +1490,11 @@ class peg_tester {
explicit peg_tester(const std::string & template_path, const bool detailed_debug = false) :
tmpls_(read_templates(template_path)),
template_path_(template_path),
detailed_debug_(detailed_debug) {}
detailed_debug_(detailed_debug) {
if (template_matches_filter(template_path_)) {
LOG_INF(" running %s\n", template_path_.c_str());
}
}
const std::string & template_path() const { return template_path_; }
@@ -1571,19 +1591,11 @@ class peg_test_builder {
// Execute the test
void run() {
// Check template filter
if (!g_template_filter.empty()) {
// Case-insensitive substring match
std::string template_path_lower = tester_.template_path();
std::string filter_lower = g_template_filter;
std::transform(template_path_lower.begin(), template_path_lower.end(), template_path_lower.begin(),
::tolower);
std::transform(filter_lower.begin(), filter_lower.end(), filter_lower.begin(), ::tolower);
if (template_path_lower.find(filter_lower) == std::string::npos) {
// Skip this test
return;
}
if (!template_matches_filter(tester_.template_path())) {
// Skip this test
return;
}
LOG_INF("\n\x1b[38;5;126m[%s]\x1b[0m\n%s\n\n", tester_.template_path().c_str(), tc_.input.c_str());
LOG_TRC("\n\x1b[38;5;126m[%s]\x1b[0m\n%s\n\n", tester_.template_path().c_str(), tc_.input.c_str());
test_peg_parser(tester_.tmpls_.get(), [this](peg_test_case & t) { t = tc_; }, tester_.detailed_debug_);
}
};
@@ -7017,7 +7029,7 @@ static void test_developer_role_to_system_workaround() {
if (params.prompt.find("<|system|>You are a helpful developer assistant.<|end|>") == std::string::npos) {
throw std::runtime_error("Test failed: system message not found in output");
}
LOG_ERR("Test 1 passed: developer role changed to system\n");
LOG_INF("Test 1 passed: developer role changed to system\n");
}
}
@@ -7321,42 +7333,61 @@ static void test_msg_diffs_compute() {
}
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
bool detailed_debug = false;
bool only_run_filtered = false;
// Check for --template and --detailed flags
std::vector<std::string> positional;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
std::string arg = argv[i];
if (arg == "--template" && i + 1 < argc) {
g_template_filter = argv[++i];
// Only run PEG parser tests with the filter
only_run_filtered = true;
}
if (arg == "--detailed") {
if (arg == "--template") {
if (i + 1 < argc) {
g_template_filter = argv[++i];
// Only run PEG parser tests with the filter
only_run_filtered = true;
}
} else if (arg == "--detailed") {
detailed_debug = true;
common_log_set_verbosity_thold(999);
}
if (arg == "--force-reconstruction-test") {
} else if (arg == "--force-reconstruction-test") {
g_force_reconstruction_test = true;
only_run_filtered = true;
} else if (arg[0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else {
positional.push_back(arg); // .jinja template path
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
// keep the detailed output unless the user asked for a lower verbosity threshold
if (detailed_debug) {
common_log_set_verbosity_thold(params.verbosity < LOG_DEFAULT_LLAMA ? params.verbosity : 999);
}
LOG("%s: running\n", "test-chat");
if (only_run_filtered) {
test_template_output_peg_parsers(detailed_debug);
std::cout << "\n[chat] All template tests passed!" << '\n';
LOG("%s: %s\n", "test-chat", "PASSED");
common_log_flush(common_log_main());
return 0;
}
#ifndef _WIN32
// Check if any argument is a .jinja file (for template format detection mode)
bool has_jinja_files = false;
for (int i = 1; i < argc; i++) {
std::string arg = argv[i];
if (arg == "--detailed") {
continue;
}
if (arg.size() >= 6 && arg.rfind(".jinja") == arg.size() - 6) {
for (const auto & path : positional) {
if (path.size() >= 6 && path.rfind(".jinja") == path.size() - 6) {
has_jinja_files = true;
break;
}
@@ -7373,9 +7404,8 @@ int main(int argc, char ** argv) {
std::cout << "| Template | Format |\n";
std::cout << "|----------|--------|\n";
for (int i = 1; i < argc; i++) {
for (const auto & path : positional) {
try {
std::string path = argv[i];
if (path.rfind(".jinja") != path.size() - 6) {
std::cerr << "Skipping non-jinja file: " << path << '\n';
continue;
@@ -7386,7 +7416,7 @@ int main(int argc, char ** argv) {
const auto * format = common_chat_format_name(common_chat_templates_apply(tmpls.get(), inputs).format);
std::cout << "| " << name << " | " << format << " |\n";
} catch (const std::exception & e) {
std::cerr << "Failed to process " << argv[i] << ": " << e.what() << '\n';
std::cerr << "Failed to process " << path << ": " << e.what() << '\n';
}
}
} else
@@ -7407,5 +7437,7 @@ int main(int argc, char ** argv) {
test_template_output_peg_parsers(detailed_debug);
std::cout << "\n[chat] All tests passed!" << '\n';
}
LOG("%s: %s\n", "test-chat", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-col2im-1d.cpp
+21 -4
View File
@@ -13,6 +13,10 @@
#include "ggml.h"
#include "ggml-cpu.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#include <cmath>
#include <cstdint>
#include <cstdio>
@@ -86,7 +90,16 @@ static double nmse_cropped(const float * y, const float * ref, int64_t T_out, in
return num / (den + 1e-30);
}
int main(void) {
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-col2im-1d");
int fails = 0;
for (const col2im_case & c : CASES) {
@@ -148,12 +161,16 @@ int main(void) {
if (!ok) {
fails++;
}
printf("col2im_1d K=%2d OC=%2d T_in=%3d s0=%d p0=%d: nmse f32=%.2e f16=%.2e bf16=%.2e %s\n",
LOG_INF("col2im_1d K=%2d OC=%2d T_in=%3d s0=%d p0=%d: nmse f32=%.2e f16=%.2e bf16=%.2e %s\n",
(int) c.K, (int) c.OC, (int) c.T_in, c.s0, c.p0, e32, e16, ebf, ok ? "OK" : "FAIL");
ggml_free(ctx);
}
printf(fails == 0 ? "all col2im_1d checks passed\n" : "%d col2im_1d checks FAILED\n", fails);
return fails == 0 ? 0 : 1;
const bool ok = fails == 0;
printf(ok ? "all col2im_1d checks passed\n" : "%d col2im_1d checks FAILED\n", fails);
LOG("%s: %s\n", "test-col2im-1d", ok ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return ok ? 0 : 1;
}
tests/test-export-graph-ops.cpp
+22
View File
@@ -135,6 +135,8 @@ int main(int argc, char ** argv) {
return 1;
}
LOG("%s: running\n", "test-export-graph-ops");
// Load CPU-only
ggml_backend_dev_t cpu_device = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
params.devices = { cpu_device, nullptr };
@@ -154,6 +156,8 @@ int main(int argc, char ** argv) {
ctx = init_result->context();
if (!ctx) {
LOG_ERR("failed to initialize params\n");
LOG("%s: %s\n", "test-export-graph-ops", "FAILED");
common_log_flush(common_log_main());
return 1;
}
} else {
@@ -166,6 +170,8 @@ int main(int argc, char ** argv) {
gguf_context_ptr gguf_ctx = gguf_fetch_gguf_ctx(hf_repo, hf_quant);
if (!gguf_ctx) {
LOG_ERR("failed to fetch GGUF metadata from %s\n", hf_repo.c_str());
LOG("%s: %s\n", "test-export-graph-ops", "FAILED");
common_log_flush(common_log_main());
return 1;
}
@@ -177,6 +183,8 @@ int main(int argc, char ** argv) {
if (!model) {
LOG_ERR("failed to create llama_model from %s\n", hf_repo.c_str());
LOG("%s: %s\n", "test-export-graph-ops", "FAILED");
common_log_flush(common_log_main());
return 1;
}
@@ -186,10 +194,14 @@ int main(int argc, char ** argv) {
if (!ctx) {
LOG_ERR("failed to create llama_context\n");
LOG("%s: %s\n", "test-export-graph-ops", "FAILED");
common_log_flush(common_log_main());
return 1;
}
#else
LOG_ERR("test-export-graph-ops compiled without HF fetch support\n");
LOG("%s: %s\n", "test-export-graph-ops", "FAILED");
common_log_flush(common_log_main());
return 1;
#endif
}
@@ -197,11 +209,15 @@ int main(int argc, char ** argv) {
const uint32_t n_seqs = llama_n_seq_max(ctx);
const uint32_t n_tokens = std::min(llama_n_ctx(ctx), llama_n_ubatch(ctx));
LOG_INF(" running %s\n", params.model.get_name().c_str());
std::set<test_object> tests;
auto * gf_pp = llama_graph_reserve(ctx, n_tokens, n_seqs, n_tokens);
if (!gf_pp) {
LOG_ERR("failed to reserve prompt processing graph\n");
LOG("%s: %s\n", "test-export-graph-ops", "FAILED");
common_log_flush(common_log_main());
return 1;
}
extract_graph_ops(gf_pp, "pp", tests);
@@ -209,6 +225,8 @@ int main(int argc, char ** argv) {
auto * gf_tg = llama_graph_reserve(ctx, n_seqs, n_seqs, n_seqs);
if (!gf_tg) {
LOG_ERR("failed to reserve token generation graph\n");
LOG("%s: %s\n", "test-export-graph-ops", "FAILED");
common_log_flush(common_log_main());
return 1;
}
extract_graph_ops(gf_tg, "tg", tests);
@@ -219,6 +237,8 @@ int main(int argc, char ** argv) {
if (!f.is_open()) {
LOG_ERR("unable to open output file: %s\n", params.out_file.c_str());
LOG("%s: %s\n", "test-export-graph-ops", "FAILED");
common_log_flush(common_log_main());
return 1;
}
@@ -226,5 +246,7 @@ int main(int argc, char ** argv) {
test.serialize(f);
}
LOG("%s: %s\n", "test-export-graph-ops", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-fusion.cpp
+38 -2
View File
@@ -16,6 +16,7 @@
// test-fusion --models DIR --device MTL0 --check baseline.csv # validate against it
// test-fusion --model FILE --device MTL0 --check baseline.csv # validate a single model
#include "arg.h"
#include "common.h"
#include "log.h"
#include "llama-cpp.h"
@@ -224,17 +225,24 @@ static void usage(const char * argv0) {
}
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes its models from --models/--model
common_init();
std::string models_dir;
std::string model_file;
std::string device_name;
std::string record_path;
std::string check_path;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
const std::string arg = argv[i];
const auto next = [&](const char * name) -> std::string {
if (i + 1 >= argc) {
LOG_ERR("%s: %s requires an argument\n", __func__, name);
common_log_flush(common_log_main());
exit(1);
}
return argv[++i];
@@ -248,39 +256,56 @@ int main(int argc, char ** argv) {
else if (arg == "--device"){ device_name = next("--device"); }
else if (arg == "--record"){ record_path = next("--record"); }
else if (arg == "--check") { check_path = next("--check"); }
else {
else if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else {
LOG_ERR("%s: unknown argument: %s\n", __func__, arg.c_str());
common_log_flush(common_log_main());
return 1;
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
if (device_name.empty()) {
LOG_ERR("%s: --device NAME is required\n", __func__);
common_log_flush(common_log_main());
return 1;
}
if (models_dir.empty() && model_file.empty()) {
LOG_ERR("%s: --models DIR or --model FILE is required\n", __func__);
common_log_flush(common_log_main());
return 1;
}
if (!models_dir.empty() && !model_file.empty()) {
LOG_ERR("%s: --models DIR and --model FILE are mutually exclusive\n", __func__);
common_log_flush(common_log_main());
return 1;
}
if (!record_path.empty() && !check_path.empty()) {
LOG_ERR("%s: --record and --check are mutually exclusive\n", __func__);
common_log_flush(common_log_main());
return 1;
}
LOG("%s: running\n", "test-fusion");
std::vector<std::string> models;
if (!model_file.empty()) {
if (!std::filesystem::is_regular_file(model_file)) {
LOG_ERR("%s: model file '%s' does not exist\n", __func__, model_file.c_str());
LOG("%s: %s\n", "test-fusion", "FAILED");
common_log_flush(common_log_main());
return 1;
}
models.push_back(model_file);
} else {
if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) {
LOG_ERR("%s: models directory '%s' does not exist\n", __func__, models_dir.c_str());
LOG("%s: %s\n", "test-fusion", "FAILED");
common_log_flush(common_log_main());
return 1;
}
for (const auto & entry : std::filesystem::directory_iterator(models_dir)) {
@@ -292,17 +317,20 @@ int main(int argc, char ** argv) {
if (models.empty()) {
LOG_ERR("%s: no .gguf models found in '%s'\n", __func__, models_dir.c_str());
LOG("%s: %s\n", "test-fusion", "FAILED");
common_log_flush(common_log_main());
return 1;
}
}
common_init();
ggml_backend_load_all();
ggml_backend_dev_t dev = ggml_backend_dev_by_name(device_name.c_str());
if (!dev) {
LOG_WRN("%s: device '%s' not found - skipping (baseline is device-specific)\n",
__func__, device_name.c_str());
LOG("%s: %s\n", "test-fusion", "PASSED");
common_log_flush(common_log_main());
return 0;
}
@@ -324,6 +352,8 @@ int main(int argc, char ** argv) {
LOG_ERR("%s: device '%s' does not export the generic fusion debugging API "
"(ggml_backend_fusion_*) - cannot run the fusion regression test\n",
__func__, device_name.c_str());
LOG("%s: %s\n", "test-fusion", "FAILED");
common_log_flush(common_log_main());
return 1;
}
@@ -340,6 +370,8 @@ int main(int argc, char ** argv) {
std::ifstream in(check_path);
if (!in) {
LOG_ERR("%s: cannot open baseline '%s'\n", __func__, check_path.c_str());
LOG("%s: %s\n", "test-fusion", "FAILED");
common_log_flush(common_log_main());
return 1;
}
std::string line;
@@ -368,6 +400,8 @@ int main(int argc, char ** argv) {
const size_t seed = 1;
for (const auto & model_path : models) {
LOG_INF(" running %s\n", model_path.c_str());
const std::string arch = get_arch(model_path);
const bool moe = arch.find("moe") != std::string::npos;
@@ -560,6 +594,8 @@ int main(int argc, char ** argv) {
__func__, models_dir.c_str(), argv[0], device_name.c_str(), models_dir.c_str(), check_path.c_str());
}
LOG("%s: %s\n", "test-fusion", n_bad == 0 ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return n_bad;
}
}
tests/test-gbnf-validator.cpp
+42 -6
View File
@@ -1,6 +1,9 @@
#include "../src/unicode.h"
#include "../src/llama-grammar.h"
#include "arg.h"
#include "log.h"
#include <cstdio>
#include <cstdlib>
#include <sstream>
@@ -37,33 +40,61 @@ static bool llama_grammar_validate(struct llama_grammar * grammar, const std::st
}
static void print_error_message(const std::string & input_str, size_t error_pos, const std::string & error_msg) {
// no ANSI escapes at WARN or lower (--errors-only)
const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN;
const char * const col_pos = use_color ? "\033[1;31m" : "";
const char * const col_rest = use_color ? "\033[0;31m" : "";
const char * const col_end = use_color ? "\033[0m" : "";
fprintf(stdout, "Input string is invalid according to the grammar.\n");
fprintf(stdout, "Error: %s at position %zu\n", error_msg.c_str(), error_pos);
fprintf(stdout, "\n");
fprintf(stdout, "Input string:\n");
fprintf(stdout, "%s", input_str.substr(0, error_pos).c_str());
if (error_pos < input_str.size()) {
fprintf(stdout, "\033[1;31m%c", input_str[error_pos]);
fprintf(stdout, "%s%c", col_pos, input_str[error_pos]);
if (error_pos+1 < input_str.size()) {
fprintf(stdout, "\033[0;31m%s", input_str.substr(error_pos+1).c_str());
fprintf(stdout, "%s%s", col_rest, input_str.substr(error_pos+1).c_str());
}
fprintf(stdout, "\033[0m\n");
fprintf(stdout, "%s\n", col_end);
}
}
int main(int argc, char** argv) {
if (argc != 3) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
std::vector<std::string> positional;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else {
positional.push_back(argv[i]);
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
if (positional.size() != 2) {
fprintf(stdout, "Usage: %s <grammar_filename> <input_filename>\n", argv[0]);
return 1;
}
const std::string grammar_filename = argv[1];
const std::string input_filename = argv[2];
LOG("%s: running\n", "test-gbnf-validator");
const std::string grammar_filename = positional[0];
const std::string input_filename = positional[1];
// Read the GBNF grammar file
FILE* grammar_file = fopen(grammar_filename.c_str(), "r");
if (!grammar_file) {
fprintf(stdout, "Failed to open grammar file: %s\n", grammar_filename.c_str());
LOG("%s: %s\n", "test-gbnf-validator", "FAILED");
common_log_flush(common_log_main());
return 1;
}
@@ -79,6 +110,8 @@ int main(int argc, char** argv) {
llama_grammar * grammar = llama_grammar_init_impl(nullptr, grammar_str.c_str(), "root", false, nullptr, 0, nullptr, 0);
if (grammar == nullptr) {
fprintf(stdout, "Failed to initialize llama_grammar\n");
LOG("%s: %s\n", "test-gbnf-validator", "FAILED");
common_log_flush(common_log_main());
return 1;
}
// Read the input file
@@ -105,5 +138,8 @@ int main(int argc, char** argv) {
// Clean up
llama_grammar_free_impl(grammar);
LOG("%s: %s\n", "test-gbnf-validator", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-gguf-model-data.cpp
+65 -40
View File
@@ -1,39 +1,56 @@
#include "gguf-model-data.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#include <cstdio>
#define TEST_ASSERT(cond, msg) \
do { \
if (!(cond)) { \
fprintf(stderr, "FAIL: %s (line %d): %s\n", #cond, __LINE__, msg); \
LOG_ERR("FAIL: %s (line %d): %s\n", #cond, __LINE__, msg); \
LOG("%s: %s\n", "test-gguf-model-data", "FAILED"); \
common_log_flush(common_log_main()); \
return 1; \
} \
} while (0)
int main() {
fprintf(stderr, "=== test-gguf-model-data ===\n");
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-gguf-model-data");
LOG_INF("=== test-gguf-model-data ===\n");
// Fetch Qwen3-0.6B Q8_0 metadata
LOG_INF(" running %s\n", "ggml-org/Qwen3-0.6B-GGUF (Q8_0)");
auto result = gguf_fetch_model_meta("ggml-org/Qwen3-0.6B-GGUF", "Q8_0");
if (!result.has_value()) {
fprintf(stderr, "SKIP: could not fetch model metadata (no network or HTTP disabled)\n");
LOG_WRN("SKIP: could not fetch model metadata (no network or HTTP disabled)\n");
LOG("%s: %s\n", "test-gguf-model-data", "PASSED");
common_log_flush(common_log_main());
return 0;
}
const auto & model = result.value();
fprintf(stderr, "Architecture: %s\n", model.architecture.c_str());
fprintf(stderr, "n_embd: %u\n", model.n_embd);
fprintf(stderr, "n_ff: %u\n", model.n_ff);
fprintf(stderr, "n_vocab: %u\n", model.n_vocab);
fprintf(stderr, "n_layer: %u\n", model.n_layer);
fprintf(stderr, "n_head: %u\n", model.n_head);
fprintf(stderr, "n_head_kv: %u\n", model.n_head_kv);
fprintf(stderr, "n_expert: %u\n", model.n_expert);
fprintf(stderr, "n_embd_head_k: %u\n", model.n_embd_head_k);
fprintf(stderr, "n_embd_head_v: %u\n", model.n_embd_head_v);
fprintf(stderr, "tensors: %zu\n", model.tensors.size());
LOG_INF("Architecture: %s\n", model.architecture.c_str());
LOG_INF("n_embd: %u\n", model.n_embd);
LOG_INF("n_ff: %u\n", model.n_ff);
LOG_INF("n_vocab: %u\n", model.n_vocab);
LOG_INF("n_layer: %u\n", model.n_layer);
LOG_INF("n_head: %u\n", model.n_head);
LOG_INF("n_head_kv: %u\n", model.n_head_kv);
LOG_INF("n_expert: %u\n", model.n_expert);
LOG_INF("n_embd_head_k: %u\n", model.n_embd_head_k);
LOG_INF("n_embd_head_v: %u\n", model.n_embd_head_v);
LOG_INF("tensors: %zu\n", model.tensors.size());
// Verify architecture
TEST_ASSERT(model.architecture == "qwen3", "expected architecture 'qwen3'");
@@ -83,24 +100,27 @@ int main() {
TEST_ASSERT(result2->tensors.size() == model.tensors.size(), "cached result should match");
// Test a split MoE model without specifying quant (should default to Q8_0)
LOG_INF(" running %s\n", "ggml-org/GLM-4.6V-GGUF");
auto result3 = gguf_fetch_model_meta("ggml-org/GLM-4.6V-GGUF");
if (!result3.has_value()) {
fprintf(stderr, "SKIP: could not fetch GLM-4.6V metadata (no network?)\n");
LOG_WRN("SKIP: could not fetch GLM-4.6V metadata (no network?)\n");
LOG("%s: %s\n", "test-gguf-model-data", "PASSED");
common_log_flush(common_log_main());
return 0;
}
const auto & model3 = result3.value();
fprintf(stderr, "Architecture: %s\n", model3.architecture.c_str());
fprintf(stderr, "n_embd: %u\n", model3.n_embd);
fprintf(stderr, "n_ff: %u\n", model3.n_ff);
fprintf(stderr, "n_vocab: %u\n", model3.n_vocab);
fprintf(stderr, "n_layer: %u\n", model3.n_layer);
fprintf(stderr, "n_head: %u\n", model3.n_head);
fprintf(stderr, "n_head_kv: %u\n", model3.n_head_kv);
fprintf(stderr, "n_expert: %u\n", model3.n_expert);
fprintf(stderr, "n_embd_head_k: %u\n", model3.n_embd_head_k);
fprintf(stderr, "n_embd_head_v: %u\n", model3.n_embd_head_v);
fprintf(stderr, "tensors: %zu\n", model3.tensors.size());
LOG_INF("Architecture: %s\n", model3.architecture.c_str());
LOG_INF("n_embd: %u\n", model3.n_embd);
LOG_INF("n_ff: %u\n", model3.n_ff);
LOG_INF("n_vocab: %u\n", model3.n_vocab);
LOG_INF("n_layer: %u\n", model3.n_layer);
LOG_INF("n_head: %u\n", model3.n_head);
LOG_INF("n_head_kv: %u\n", model3.n_head_kv);
LOG_INF("n_expert: %u\n", model3.n_expert);
LOG_INF("n_embd_head_k: %u\n", model3.n_embd_head_k);
LOG_INF("n_embd_head_v: %u\n", model3.n_embd_head_v);
LOG_INF("tensors: %zu\n", model3.tensors.size());
// Verify architecture
TEST_ASSERT(model3.architecture == "glm4moe", "expected architecture 'glm4moe'");
@@ -117,24 +137,27 @@ int main() {
TEST_ASSERT(model3.tensors.size() == 780, "expected tensor count == 780");
// Test a hybrid-attention model with array-valued head counts
LOG_INF(" running %s\n", "ggml-org/Step-3.5-Flash-GGUF (Q4_K)");
auto result4 = gguf_fetch_model_meta("ggml-org/Step-3.5-Flash-GGUF", "Q4_K");
if (!result4.has_value()) {
fprintf(stderr, "FAIL: could not fetch Step-3.5-Flash metadata\n");
LOG_ERR("FAIL: could not fetch Step-3.5-Flash metadata\n");
LOG("%s: %s\n", "test-gguf-model-data", "FAILED");
common_log_flush(common_log_main());
return 1;
}
const auto & model4 = result4.value();
fprintf(stderr, "Architecture: %s\n", model4.architecture.c_str());
fprintf(stderr, "n_embd: %u\n", model4.n_embd);
fprintf(stderr, "n_ff: %u\n", model4.n_ff);
fprintf(stderr, "n_vocab: %u\n", model4.n_vocab);
fprintf(stderr, "n_layer: %u\n", model4.n_layer);
fprintf(stderr, "n_head: %u\n", model4.n_head);
fprintf(stderr, "n_head_kv: %u\n", model4.n_head_kv);
fprintf(stderr, "n_expert: %u\n", model4.n_expert);
fprintf(stderr, "n_embd_head_k: %u\n", model4.n_embd_head_k);
fprintf(stderr, "n_embd_head_v: %u\n", model4.n_embd_head_v);
fprintf(stderr, "tensors: %zu\n", model4.tensors.size());
LOG_INF("Architecture: %s\n", model4.architecture.c_str());
LOG_INF("n_embd: %u\n", model4.n_embd);
LOG_INF("n_ff: %u\n", model4.n_ff);
LOG_INF("n_vocab: %u\n", model4.n_vocab);
LOG_INF("n_layer: %u\n", model4.n_layer);
LOG_INF("n_head: %u\n", model4.n_head);
LOG_INF("n_head_kv: %u\n", model4.n_head_kv);
LOG_INF("n_expert: %u\n", model4.n_expert);
LOG_INF("n_embd_head_k: %u\n", model4.n_embd_head_k);
LOG_INF("n_embd_head_v: %u\n", model4.n_embd_head_v);
LOG_INF("tensors: %zu\n", model4.tensors.size());
TEST_ASSERT(model4.architecture == "step35", "expected architecture 'step35'");
@@ -149,6 +172,8 @@ int main() {
TEST_ASSERT(model4.n_vocab == 128896, "expected n_vocab == 128896");
TEST_ASSERT(model4.tensors.size() == 754, "expected tensor count == 754");
fprintf(stderr, "=== ALL TESTS PASSED ===\n");
LOG_INF("=== ALL TESTS PASSED ===\n");
LOG("%s: %s\n", "test-gguf-model-data", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-gguf.cpp
+130 -72
View File
@@ -3,6 +3,10 @@
#include "../ggml/src/ggml-impl.h"
#include "gguf.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#include <algorithm>
#include <array>
#include <cmath>
@@ -748,6 +752,14 @@ static bool handcrafted_check_tensor_data(const gguf_context * gguf_ctx, const u
}
static std::pair<int, int> test_handcrafted_file(const unsigned int seed) {
LOG_INF(" running %s\n", __func__);
// no ANSI escapes at WARN or lower (--errors-only)
const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN;
const char * const col_ok = use_color ? "\033[1;32m" : "";
const char * const col_fail = use_color ? "\033[1;31m" : "";
const char * const col_end = use_color ? "\033[0m" : "";
int npass = 0;
int ntest = 0;
@@ -791,7 +803,7 @@ static std::pair<int, int> test_handcrafted_file(const unsigned int seed) {
};
for (enum handcrafted_file_type hft : hfts) {
printf("%s: handcrafted_file_type=%s\n", __func__, handcrafted_file_type_name(hft).c_str());
LOG_CNT("%s: handcrafted_file_type=%s\n", __func__, handcrafted_file_type_name(hft).c_str());
FILE * file = get_handcrafted_file(seed, hft);
#ifdef _WIN32
@@ -813,24 +825,24 @@ static std::pair<int, int> test_handcrafted_file(const unsigned int seed) {
struct gguf_context * gguf_ctx = gguf_init_from_file_ptr(file, gguf_params);
if (expect_context_not_null(hft)) {
printf("%s: - context_not_null: ", __func__);
LOG_CNT("%s: - context_not_null: ", __func__);
} else {
printf("%s: - context_null: ", __func__);
LOG_CNT("%s: - context_null: ", __func__);
}
if (bool(gguf_ctx) == expect_context_not_null(hft)) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
ntest++;
if (hft >= offset_has_data && !expect_context_not_null(hft)) {
printf("%s: - no_dangling_ggml_context_pointer: ", __func__);
LOG_CNT("%s: - no_dangling_ggml_context_pointer: ", __func__);
if (ctx) {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
} else {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
}
ntest++;
@@ -839,23 +851,23 @@ static std::pair<int, int> test_handcrafted_file(const unsigned int seed) {
const bool alignment_defined = hft == HANDCRAFTED_TENSORS_CUSTOM_ALIGN || hft == HANDCRAFTED_DATA_CUSTOM_ALIGN;
if (expect_context_not_null(hft)) {
printf("%s: - check_header: ", __func__);
LOG_CNT("%s: - check_header: ", __func__);
if (handcrafted_check_header(gguf_ctx, seed, hft >= offset_has_kv, hft >= offset_has_tensors, alignment_defined)) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
ntest++;
}
if (expect_context_not_null(hft) && hft >= offset_has_kv) {
printf("%s: - check_kv: ", __func__);
LOG_CNT("%s: - check_kv: ", __func__);
if (handcrafted_check_kv(gguf_ctx, seed, hft >= offset_has_tensors, alignment_defined)) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
ntest++;
}
@@ -863,23 +875,23 @@ static std::pair<int, int> test_handcrafted_file(const unsigned int seed) {
// HANDCRAFTED_TENSORS_ZERO_DIM deliberately mangles the tensor shapes to 0 elements,
// so only assert that it loads without crashing; skip the exact-geometry comparison.
if (expect_context_not_null(hft) && hft >= offset_has_tensors && hft != HANDCRAFTED_TENSORS_ZERO_DIM) {
printf("%s: - check_tensors: ", __func__);
LOG_CNT("%s: - check_tensors: ", __func__);
if (handcrafted_check_tensors(gguf_ctx, seed)) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
ntest++;
}
if (expect_context_not_null(hft) && hft >= offset_has_data) {
printf("%s: - check_tensor_data: ", __func__);
LOG_CNT("%s: - check_tensor_data: ", __func__);
if (handcrafted_check_tensor_data(gguf_ctx, seed, file)) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
ntest++;
}
@@ -889,7 +901,7 @@ static std::pair<int, int> test_handcrafted_file(const unsigned int seed) {
ggml_free(ctx);
gguf_free(gguf_ctx);
}
printf("\n");
LOG_CNT("\n");
}
@@ -1185,10 +1197,16 @@ static std::pair<int, int> test_roundtrip(
ggml_backend_dev_t dev, const unsigned int seed, const bool only_meta,
const roundtrip_read_mode read_mode) {
ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
printf("%s: device=%s, backend=%s, only_meta=%s, read_mode=%s\n",
LOG_INF(" running %s: device=%s, backend=%s, only_meta=%s, read_mode=%s\n",
__func__, ggml_backend_dev_description(dev), ggml_backend_name(backend),
only_meta ? "yes" : "no", roundtrip_read_mode_name(read_mode));
// no ANSI escapes at WARN or lower (--errors-only)
const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN;
const char * const col_ok = use_color ? "\033[1;32m" : "";
const char * const col_fail = use_color ? "\033[1;31m" : "";
const char * const col_end = use_color ? "\033[0m" : "";
int npass = 0;
int ntest = 0;
@@ -1239,76 +1257,76 @@ static std::pair<int, int> test_roundtrip(
gguf_ctx_1 = gguf_init_from_file_ptr(file, gguf_params);
}
printf("%s: same_version: ", __func__);
LOG_CNT("%s: same_version: ", __func__);
if (gguf_get_version(gguf_ctx_0) == gguf_get_version(gguf_ctx_1)) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
ntest++;
printf("%s: same_n_kv: ", __func__);
LOG_CNT("%s: same_n_kv: ", __func__);
if (gguf_get_n_kv(gguf_ctx_0) == gguf_get_n_kv(gguf_ctx_1)) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
ntest++;
printf("%s: same_n_tensors: ", __func__);
LOG_CNT("%s: same_n_tensors: ", __func__);
if (gguf_get_n_tensors(gguf_ctx_0) == gguf_get_n_tensors(gguf_ctx_1)) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
ntest++;
printf("%s: all_orig_kv_in_read: ", __func__);
LOG_CNT("%s: all_orig_kv_in_read: ", __func__);
if (all_kv_in_other(gguf_ctx_0, gguf_ctx_1)) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
ntest++;
printf("%s: all_read_kv_in_orig: ", __func__);
LOG_CNT("%s: all_read_kv_in_orig: ", __func__);
if (all_kv_in_other(gguf_ctx_1, gguf_ctx_0)) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
ntest++;
printf("%s: all_orig_tensors_in_read: ", __func__);
LOG_CNT("%s: all_orig_tensors_in_read: ", __func__);
if (all_tensors_in_other(gguf_ctx_0, gguf_ctx_1)) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
ntest++;
printf("%s: all_read_tensors_in_orig: ", __func__);
LOG_CNT("%s: all_read_tensors_in_orig: ", __func__);
if (all_tensors_in_other(gguf_ctx_1, gguf_ctx_0)) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
ntest++;
if (!only_meta) {
printf("%s: same_tensor_data: ", __func__);
LOG_CNT("%s: same_tensor_data: ", __func__);
if (same_tensor_data(ctx_0, ctx_1)) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
ntest++;
}
@@ -1321,13 +1339,20 @@ static std::pair<int, int> test_roundtrip(
ggml_backend_free(backend);
fclose(file);
printf("\n");
LOG_CNT("\n");
return std::make_pair(npass, ntest);
}
static std::pair<int, int> test_gguf_set_kv(ggml_backend_dev_t dev, const unsigned int seed) {
ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
printf("%s: device=%s, backend=%s\n", __func__, ggml_backend_dev_description(dev), ggml_backend_name(backend));
LOG_INF(" running %s: device=%s, backend=%s\n",
__func__, ggml_backend_dev_description(dev), ggml_backend_name(backend));
// no ANSI escapes at WARN or lower (--errors-only)
const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN;
const char * const col_ok = use_color ? "\033[1;32m" : "";
const char * const col_fail = use_color ? "\033[1;31m" : "";
const char * const col_end = use_color ? "\033[0m" : "";
int npass = 0;
int ntest = 0;
@@ -1357,50 +1382,50 @@ static std::pair<int, int> test_gguf_set_kv(ggml_backend_dev_t dev, const unsign
gguf_set_kv(gguf_ctx_1, gguf_ctx_0);
gguf_set_kv(gguf_ctx_2, gguf_ctx_0);
printf("%s: same_n_kv: ", __func__);
LOG_CNT("%s: same_n_kv: ", __func__);
if (gguf_get_n_kv(gguf_ctx_0) == gguf_get_n_kv(gguf_ctx_2)) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
ntest++;
printf("%s: all_kv_0_in_1: ", __func__);
LOG_CNT("%s: all_kv_0_in_1: ", __func__);
if (all_kv_in_other(gguf_ctx_0, gguf_ctx_1)) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
ntest++;
printf("%s: all_kv_0_in_2: ", __func__);
LOG_CNT("%s: all_kv_0_in_2: ", __func__);
if (all_kv_in_other(gguf_ctx_0, gguf_ctx_2)) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
ntest++;
gguf_set_kv(gguf_ctx_0, gguf_ctx_1);
printf("%s: same_n_kv_after_double_copy: ", __func__);
LOG_CNT("%s: same_n_kv_after_double_copy: ", __func__);
if (gguf_get_n_kv(gguf_ctx_0) == gguf_get_n_kv(gguf_ctx_1)) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
ntest++;
printf("%s: all_kv_1_in_0_after_double_copy: ", __func__);
LOG_CNT("%s: all_kv_1_in_0_after_double_copy: ", __func__);
if (all_kv_in_other(gguf_ctx_1, gguf_ctx_0)) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
ntest++;
@@ -1413,7 +1438,7 @@ static std::pair<int, int> test_gguf_set_kv(ggml_backend_dev_t dev, const unsign
gguf_free(gguf_ctx_2);
ggml_backend_free(backend);
printf("\n");
LOG_CNT("\n");
return std::make_pair(npass, ntest);
}
@@ -1423,17 +1448,39 @@ static void print_usage() {
}
int main(int argc, char ** argv) {
if (argc > 2) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
// this test takes an optional seed as its only positional argument
std::vector<std::string> positional;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else {
positional.push_back(argv[i]);
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
if (positional.size() > 1) {
print_usage();
return 1;
}
std::random_device rd;
const unsigned int seed = argc < 2 ? rd() : std::stoi(argv[1]);
const unsigned int seed = positional.empty() ? rd() : std::stoi(positional[0]);
LOG("%s: running\n", "test-gguf");
// Initialize ggml backends early so the prints aren't interleaved with the test results:
ggml_backend_dev_count();
fprintf(stderr, "\n");
LOG_CNT("\n"); // bare separator, LOG_CNT adds no prefix
int npass = 0;
int ntest = 0;
@@ -1469,11 +1516,22 @@ int main(int argc, char ** argv) {
}
}
const bool ok = npass == ntest;
// no ANSI escapes at WARN or lower (--errors-only)
const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN;
const char * const col_ok = use_color ? "\033[1;32m" : "";
const char * const col_fail = use_color ? "\033[1;31m" : "";
const char * const col_end = use_color ? "\033[0m" : "";
common_log_flush(common_log_main());
printf("%d/%d tests passed\n", npass, ntest);
if (npass != ntest) {
printf("\033[1;31mFAIL\033[0m\n");
return 1;
if (!ok) {
printf("%sFAIL%s\n", col_fail, col_end);
} else {
printf("%sOK%s\n", col_ok, col_end);
}
printf("\033[1;32mOK\033[0m\n");
return 0;
LOG("%s: %s\n", "test-gguf", ok ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return ok ? 0 : 1;
}
tests/test-grammar-integration.cpp
+57 -29
View File
@@ -9,6 +9,10 @@
#include "json.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#include <cassert>
#include <string>
#include <vector>
@@ -24,14 +28,14 @@ static llama_grammar * build_grammar(const std::string & grammar_str) {
}
static bool test_build_grammar_fails(const std::string & grammar_str) {
fprintf(stderr, "⚫ Testing failure for grammar: %s\n", grammar_str.c_str());
LOG_INF("⚫ Testing failure for grammar: %s\n", grammar_str.c_str());
bool grammar_fails = false;
llama_grammar * grammar = build_grammar(grammar_str);
if (grammar != nullptr) {
fprintf(stderr, " ❌ Expected build failure, but succeeded\n");
LOG_ERR(" ❌ Expected build failure, but succeeded\n");
} else {
grammar_fails = true;
fprintf(stdout, " ✅︎\n");
LOG_CNT(" ✅︎\n");
}
return grammar_fails;
}
@@ -114,8 +118,8 @@ static bool match_string(const std::string & input, llama_grammar * grammar) {
}
static void test(const std::string & test_desc, const std::string & grammar_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {
fprintf(stderr, "⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str());
fflush(stderr);
LOG_INF("⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str());
common_log_flush(common_log_main());
auto * grammar = build_grammar(grammar_str);
@@ -124,17 +128,18 @@ static void test(const std::string & test_desc, const std::string & grammar_str,
llama_grammar_stacks & stacks_cur = llama_grammar_get_stacks(grammar);
fprintf(stderr, " 🔵 Valid strings:\n");
LOG_INF(" 🔵 Valid strings:\n");
// Passing strings
for (const auto & test_string : passing_strings) {
fprintf(stderr, " \"%s\" ", test_string.c_str());
fflush(stderr);
// partial line, completed by the next message: LOG_CNT adds no prefix
LOG_CNT(" \"%s\" ", test_string.c_str());
common_log_flush(common_log_main());
bool matched = match_string(test_string, grammar);
if (!matched) {
fprintf(stderr, "❌ (failed to match)\n");
LOG_ERR("❌ (failed to match)\n");
// DEBUG: Write strings to files so that we can analyze more easily with gbnf-validator program to see exactly where things failed.
// DEBUG: Write the grammar_str to test-grammar-integration.grammar.gbnf
@@ -151,9 +156,9 @@ static void test(const std::string & test_desc, const std::string & grammar_str,
fclose(string_file);
}
fprintf(stderr, "\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following command: ./llama-gbnf-validator test-grammar-integration.grammar.gbnf test-grammar-integration.string.txt\n\n");
LOG_ERR("\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following command: ./llama-gbnf-validator test-grammar-integration.grammar.gbnf test-grammar-integration.string.txt\n\n");
} else {
fprintf(stdout, "✅︎\n");
LOG_CNT("✅︎\n");
}
assert(matched);
@@ -162,19 +167,19 @@ static void test(const std::string & test_desc, const std::string & grammar_str,
stacks_cur = stacks_org;
}
fprintf(stderr, " 🟠 Invalid strings:\n");
LOG_INF(" 🟠 Invalid strings:\n");
// Failing strings
for (const auto & test_string : failing_strings) {
fprintf(stderr, " \"%s\" ", test_string.c_str());
fflush(stderr);
LOG_CNT(" \"%s\" ", test_string.c_str());
common_log_flush(common_log_main());
bool matched = match_string(test_string, grammar);
if (matched) {
fprintf(stderr, "❌ (incorrectly matched)\n");
LOG_ERR("❌ (incorrectly matched)\n");
} else {
fprintf(stdout, "✅︎\n");
LOG_CNT("✅︎\n");
}
assert(!matched);
@@ -193,6 +198,7 @@ static void test_schema(const std::string & test_desc, const std::string & schem
}
static void test_simple_grammar() {
LOG_INF(" running %s\n", __func__);
test_schema(
"min 0",
R"""({
@@ -510,6 +516,7 @@ static void test_simple_grammar() {
}
static void test_complex_grammar() {
LOG_INF(" running %s\n", __func__);
// Test case for a more complex grammar, with both failure strings and success strings
test_grammar(
"medium complexity grammar",
@@ -599,6 +606,7 @@ static void test_complex_grammar() {
}
static void test_special_chars() {
LOG_INF(" running %s\n", __func__);
// A collection of tests to exercise special characters such as "."
test_grammar(
"special characters",
@@ -626,6 +634,7 @@ static void test_special_chars() {
}
static void test_quantifiers() {
LOG_INF(" running %s\n", __func__);
// A collection of tests to exercise * + and ? quantifiers
test_grammar(
@@ -809,7 +818,8 @@ static void test_quantifiers() {
}
static void test_failure_missing_root() {
fprintf(stderr, "⚫ Testing missing root node:\n");
LOG_INF(" running %s\n", __func__);
LOG_INF("⚫ Testing missing root node:\n");
// Test case for a grammar that is missing a root rule
const std::string grammar_str = R"""(
rot ::= expr
@@ -825,11 +835,12 @@ static void test_failure_missing_root() {
// Ensure we do NOT have a root node
assert(parsed_grammar.symbol_ids.find("root") == parsed_grammar.symbol_ids.end());
fprintf(stderr, " ✅︎ Passed\n");
LOG_INF(" ✅︎ Passed\n");
}
static void test_failure_missing_reference() {
fprintf(stderr, "⚫ Testing missing reference node:\n");
LOG_INF(" running %s\n", __func__);
LOG_INF("⚫ Testing missing reference node:\n");
// Test case for a grammar that is missing a referenced rule
const std::string grammar_str =
@@ -838,6 +849,8 @@ static void test_failure_missing_reference() {
term ::= numero
number ::= [0-9]+)""";
// the parser below writes to stderr directly, so drain the log first
common_log_flush(common_log_main());
fprintf(stderr, " Expected error: ");
llama_grammar_parser parsed_grammar;
@@ -847,11 +860,12 @@ static void test_failure_missing_reference() {
assert(parsed_grammar.rules.empty());
fprintf(stderr, " End of expected error.\n");
fprintf(stderr, " ✅︎ Passed\n");
LOG_INF(" ✅︎ Passed\n");
}
static void test_failure_left_recursion() {
fprintf(stderr, "⚫ Testing left recursion detection:\n");
LOG_INF(" running %s\n", __func__);
LOG_INF("⚫ Testing left recursion detection:\n");
// Test simple left recursion detection
const std::string simple_str = R"""(root ::= "a" | root "a")""";
@@ -879,11 +893,12 @@ static void test_failure_left_recursion() {
empty ::= "blah" | )""";
assert(test_build_grammar_fails(hardest_str));
fprintf(stderr, " ✅︎ Passed\n");
LOG_INF(" ✅︎ Passed\n");
}
static void test_failure_missing_root_symbol() {
fprintf(stderr, "⚫ Testing missing root symbol:\n");
LOG_INF(" running %s\n", __func__);
LOG_INF("⚫ Testing missing root symbol:\n");
const std::string grammar_str = R"""(
root ::= "foobar"
@@ -892,11 +907,12 @@ static void test_failure_missing_root_symbol() {
llama_grammar * failure_result = build_grammar_with_root(grammar_str, "nonexistent");
assert(failure_result == nullptr);
fprintf(stderr, " ✅︎ Passed\n");
LOG_INF(" ✅︎ Passed\n");
}
static void test_custom_root_symbol_check() {
fprintf(stderr, "⚫ Testing custom root symbol check:\n");
LOG_INF(" running %s\n", __func__);
LOG_INF("⚫ Testing custom root symbol check:\n");
const std::string custom_root_grammar_str = R"""(
foobar ::= "foobar"
@@ -909,10 +925,11 @@ static void test_custom_root_symbol_check() {
assert(success_result != nullptr);
llama_grammar_free_impl(success_result);
fprintf(stderr, " ✅︎ Passed\n");
LOG_INF(" ✅︎ Passed\n");
}
static void test_json_schema() {
LOG_INF(" running %s\n", __func__);
// Note that this is similar to the regular grammar tests,
// but we convert each json schema to a grammar before parsing.
// Otherwise, this test structure is the same.
@@ -1478,8 +1495,16 @@ static void test_json_schema() {
);
}
int main() {
fprintf(stdout, "Running grammar integration tests...\n");
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-grammar-integration");
test_simple_grammar();
test_complex_grammar();
test_special_chars();
@@ -1490,6 +1515,9 @@ int main() {
test_failure_missing_root_symbol();
test_custom_root_symbol_check();
test_json_schema();
fprintf(stdout, "All tests passed.\n");
common_log_flush(common_log_main());
// the tests abort on failure, so reaching this point means they all passed
LOG("%s: %s\n", "test-grammar-integration", "PASSED");
return 0;
}
tests/test-grammar-llguidance.cpp
+48 -23
View File
@@ -4,6 +4,9 @@
#include "sampling.h"
#include "arg.h"
#include "log.h"
#include <cassert>
#include <string>
#include <vector>
@@ -49,22 +52,23 @@ static bool match_string(const std::string & input, llama_sampler * grammar) {
static void test(const std::string & test_desc, const std::string & grammar_str,
const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {
fprintf(stderr, "⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str());
fflush(stderr);
LOG_INF("⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str());
common_log_flush(common_log_main());
auto * grammar = llama_sampler_init_llg(vocab, "lark", grammar_str.c_str());
fprintf(stderr, " 🔵 Valid strings:\n");
LOG_INF(" 🔵 Valid strings:\n");
// Passing strings
for (const auto & test_string : passing_strings) {
fprintf(stderr, " \"%s\" ", test_string.c_str());
fflush(stderr);
// partial line, completed by the next message: LOG_CNT adds no prefix
LOG_CNT(" \"%s\" ", test_string.c_str());
common_log_flush(common_log_main());
bool matched = match_string(test_string, grammar);
if (!matched) {
fprintf(stderr, "❌ (failed to match)\n");
LOG_ERR("❌ (failed to match)\n");
// DEBUG: Write strings to files so that we can analyze more easily with gbnf-validator program to see exactly where things failed.
// DEBUG: Write the grammar_str to test-grammar-integration.grammar.gbnf
@@ -81,8 +85,7 @@ static void test(const std::string & test_desc, const std::string & grammar_str,
fclose(string_file);
}
fprintf(stderr,
"\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following "
LOG_ERR("\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following "
"command: ./test-gbnf-validator test-grammar-integration.grammar.gbnf "
"test-grammar-integration.string.txt\n\n");
} else {
@@ -92,17 +95,18 @@ static void test(const std::string & test_desc, const std::string & grammar_str,
assert(matched);
}
fprintf(stderr, " 🟠 Invalid strings:\n");
LOG_INF(" 🟠 Invalid strings:\n");
// Failing strings
for (const auto & test_string : failing_strings) {
fprintf(stderr, " \"%s\" ", test_string.c_str());
fflush(stderr);
// partial line, completed by the next message: LOG_CNT adds no prefix
LOG_CNT(" \"%s\" ", test_string.c_str());
common_log_flush(common_log_main());
bool matched = match_string(test_string, grammar);
if (matched) {
fprintf(stderr, "❌ (incorrectly matched)\n");
LOG_ERR("❌ (incorrectly matched)\n");
} else {
fprintf(stdout, "✅︎\n");
}
@@ -1124,11 +1128,11 @@ start: /[A-Z ]*/)";
for (const auto token : tokens) {
one_hot(tok_arr, token);
fprintf(stderr, "applying token: %d\n", token);
LOG_DBG("applying token: %d\n", token);
llama_sampler_apply(sampler, &tok_arr);
auto idx = tok_arr.selected;
fprintf(stderr, " -> %d %f\n", cur[idx].id, cur[idx].logit);
LOG_DBG(" -> %d %f\n", cur[idx].id, cur[idx].logit);
assert(cur[tok_arr.selected].id == token);
llama_sampler_accept(sampler, token);
}
@@ -1144,17 +1148,35 @@ start: /[A-Z ]*/)";
assert(cur[tok_arr.selected].id == tok_eos);
}
int main(int argc, const char ** argv) {
fprintf(stdout, "Running llguidance integration tests...\n");
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
if (argc != 2) {
fprintf(stderr, "Usage: %s <vocab-file>\n", argv[0]);
std::string vocab_file;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else if (vocab_file.empty()) {
vocab_file = argv[i];
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
const char * vocab_file = argv[1];
fprintf(stdout, "Running llguidance integration tests...\n");
fprintf(stderr, "reading vocab from: '%s'\n", vocab_file);
if (vocab_file.empty()) {
LOG_ERR("Usage: %s <vocab-file>\n", argv[0]);
common_log_flush(common_log_main());
return 1;
}
LOG_INF("reading vocab from: '%s'\n", vocab_file.c_str());
llama_model * model;
llama_context * ctx;
@@ -1167,10 +1189,11 @@ int main(int argc, const char ** argv) {
mparams.vocab_only = true;
model = llama_model_load_from_file(vocab_file, mparams);
model = llama_model_load_from_file(vocab_file.c_str(), mparams);
if (model == NULL) {
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, vocab_file);
LOG_ERR("%s: error: failed to load vocab '%s'\n", __func__, vocab_file.c_str());
common_log_flush(common_log_main());
return 1;
}
@@ -1180,8 +1203,9 @@ int main(int argc, const char ** argv) {
ctx = llama_init_from_model(model, cparams);
if (ctx == NULL) {
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, vocab_file);
LOG_ERR("%s: error: failed to load vocab '%s'\n", __func__, vocab_file.c_str());
llama_model_free(model);
common_log_flush(common_log_main());
return 1;
}
}
@@ -1199,6 +1223,7 @@ int main(int argc, const char ** argv) {
llama_free(ctx);
llama_model_free(model);
common_log_flush(common_log_main());
fprintf(stdout, "All tests passed.\n");
return 0;
}
tests/test-grammar-parser.cpp
+47 -30
View File
@@ -7,6 +7,10 @@
// TODO: shold not include libllama sources
#include "../src/llama-grammar.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#include <cassert>
static const char * type_str(llama_gretype type) {
@@ -33,52 +37,53 @@ static void verify_parsing(const char *grammar_bytes, const std::vector<std::pai
}
auto print_all = [&]() {
fprintf(stderr, " verify_parsing(R\"\"\"(%s)\"\"\", {\n", grammar_bytes);
// dump is emitted in partial lines: LOG_CNT adds no prefix
LOG_CNT(" verify_parsing(R\"\"\"(%s)\"\"\", {\n", grammar_bytes);
for (auto it = parsed_grammar.symbol_ids.begin(); it != parsed_grammar.symbol_ids.end(); ++it) {
fprintf(stderr, " {\"%s\", %u},\n", it->first.c_str(), it->second);
LOG_CNT(" {\"%s\", %u},\n", it->first.c_str(), it->second);
}
fprintf(stderr, " }, {\n");
LOG_CNT(" }, {\n");
for (size_t i_rule = 0; i_rule < parsed_grammar.rules.size(); i_rule++) {
fprintf(stderr, " // %s (index %zu)\n", symbol_names[i_rule].c_str(), i_rule);
LOG_CNT(" // %s (index %zu)\n", symbol_names[i_rule].c_str(), i_rule);
auto & rule = parsed_grammar.rules[i_rule];
for (uint32_t i = 0; i < rule.size(); i++) {
std::string rule_str;
fprintf(stderr, " {%s, ", type_str(rule[i].type));
LOG_CNT(" {%s, ", type_str(rule[i].type));
if (rule[i].type == LLAMA_GRETYPE_CHAR || rule[i].type == LLAMA_GRETYPE_CHAR_ALT ||
rule[i].type == LLAMA_GRETYPE_CHAR_NOT || rule[i].type == LLAMA_GRETYPE_CHAR_RNG_UPPER) {
char c = rule[i].value;
if (c == '\n') {
fprintf(stderr, "'\\n'");
LOG_CNT("'\\n'");
} else if (c == '\t') {
fprintf(stderr, "'\\t'");
LOG_CNT("'\\t'");
} else if (c == '\r') {
fprintf(stderr, "'\\r'");
LOG_CNT("'\\r'");
} else if (c == '\0') {
fprintf(stderr, "'\\0'");
LOG_CNT("'\\0'");
} else {
fprintf(stderr, "'%c'", c);
LOG_CNT("'%c'", c);
}
} else if (rule[i].type == LLAMA_GRETYPE_RULE_REF) {
fprintf(stderr, "/* %s */ %u", symbol_names[rule[i].value].c_str(), rule[i].value);
LOG_CNT("/* %s */ %u", symbol_names[rule[i].value].c_str(), rule[i].value);
} else {
fprintf(stderr, "%u", rule[i].value);
LOG_CNT("%u", rule[i].value);
}
fprintf(stderr, "},\n");
LOG_CNT("},\n");
}
}
fprintf(stderr, " });\n");
LOG_CNT(" });\n");
};
if (getenv("TEST_GRAMMAR_PARSER_PRINT_ALL")) {
print_all();
fprintf(stderr, "\n");
LOG_CNT("\n");
return;
}
fprintf(stderr, "Testing grammar:%s\n", grammar_bytes);
LOG_INF("Testing grammar:%s\n", grammar_bytes);
if (parsed_grammar.symbol_ids.size() != expected.size()) {
fprintf(stderr, "Code to update expectation (set TEST_GRAMMAR_PARSER_PRINT_ALL=1 to print all):\n");
LOG_ERR("Code to update expectation (set TEST_GRAMMAR_PARSER_PRINT_ALL=1 to print all):\n");
print_all();
assert(parsed_grammar.symbol_ids.size() == expected.size());
}
@@ -92,11 +97,11 @@ static void verify_parsing(const char *grammar_bytes, const std::vector<std::pai
// pretty print error message before asserting
if (expected_pair.first != key || expected_pair.second != value)
{
fprintf(stderr, "index: %u\n", index);
fprintf(stderr, "expected_pair: %s, %u\n", expected_pair.first.c_str(), expected_pair.second);
fprintf(stderr, "actual_pair: %s, %u\n", key.c_str(), value);
fprintf(stderr, "expected_pair != actual_pair\n");
fprintf(stderr, "Code to update expectation (set TEST_GRAMMAR_PARSER_PRINT_ALL=1 to print all):\n");
LOG_ERR("index: %u\n", index);
LOG_ERR("expected_pair: %s, %u\n", expected_pair.first.c_str(), expected_pair.second);
LOG_ERR("actual_pair: %s, %u\n", key.c_str(), value);
LOG_ERR("expected_pair != actual_pair\n");
LOG_ERR("Code to update expectation (set TEST_GRAMMAR_PARSER_PRINT_ALL=1 to print all):\n");
print_all();
}
@@ -117,12 +122,12 @@ static void verify_parsing(const char *grammar_bytes, const std::vector<std::pai
// pretty print error message before asserting
if (expected_element.type != element.type || expected_element.value != element.value)
{
fprintf(stderr, "index: %u\n", index);
fprintf(stderr, "expected_element: %s, %u\n", type_str(expected_element.type), expected_element.value);
fprintf(stderr, "actual_element: %s, %u\n", type_str(element.type), element.value);
fprintf(stderr, "expected_element != actual_element\n");
fprintf(stderr, "all elements:\n");
fprintf(stderr, "Code to update expectation (set TEST_GRAMMAR_PARSER_PRINT_ALL=1 to print all):\n");
LOG_ERR("index: %u\n", index);
LOG_ERR("expected_element: %s, %u\n", type_str(expected_element.type), expected_element.value);
LOG_ERR("actual_element: %s, %u\n", type_str(element.type), element.value);
LOG_ERR("expected_element != actual_element\n");
LOG_ERR("all elements:\n");
LOG_ERR("Code to update expectation (set TEST_GRAMMAR_PARSER_PRINT_ALL=1 to print all):\n");
print_all();
}
@@ -133,14 +138,23 @@ static void verify_parsing(const char *grammar_bytes, const std::vector<std::pai
}
static void verify_failure(const char * grammar_bytes) {
fprintf(stderr, "Testing expected failure:%s\n", grammar_bytes);
LOG_INF("Testing expected failure:%s\n", grammar_bytes);
llama_grammar_parser result;
result.parse(grammar_bytes);
assert(result.rules.empty() && "should have failed");
}
int main()
int main(int argc, char ** argv)
{
common_params params;
params.model.path = "."; // this test takes no model
common_init();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-grammar-parser");
verify_failure(R"""(
root ::= "a"{,}"
)""");
@@ -580,5 +594,8 @@ int main()
{LLAMA_GRETYPE_END, 0},
});
// the test aborts on a mismatch, so reaching this point means it passed
LOG("%s: %s\n", "test-grammar-parser", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-jinja.cpp
+21 -1
View File
@@ -2,7 +2,9 @@
#include <iostream>
#include <random>
#include <cstdlib>
#include <vector>
#include "arg.h"
#include "json.h"
#include "subproc.h"
@@ -12,6 +14,7 @@
#include "jinja/utils.h"
#include "jinja/caps.h"
#include "log.h"
#include "testing.h"
using json = common_json;
@@ -41,6 +44,10 @@ static void test_fuzzing(testing & t);
static bool g_python_mode = false;
int main(int argc, char *argv[]) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
testing t(std::cout);
t.verbose = true;
@@ -49,14 +56,24 @@ int main(int argc, char *argv[]) {
// only use this for cross-checking, not for correctness
// note: the implementation of this flag is basic, only intented to be used by maintainers
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
std::string arg = argv[i];
if (arg == "-py") {
g_python_mode = true;
} else if (arg[0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else {
t.set_filter(arg);
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-jinja");
t.test("whitespace control", test_whitespace_control);
t.test("conditionals", test_conditionals);
@@ -80,7 +97,10 @@ int main(int argc, char *argv[]) {
t.test("fuzzing", test_fuzzing);
}
return t.summary();
const int rc = t.summary();
LOG("%s: %s\n", "test-jinja", rc == 0 ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return rc;
}
static void test_whitespace_control(testing & t) {
tests/test-json-schema-to-grammar.cpp
+28 -11
View File
@@ -8,6 +8,10 @@
#include "json.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#include <cassert>
#include <regex>
@@ -30,12 +34,12 @@ struct TestCase {
std::string expected_grammar;
void _print_failure_header() const {
fprintf(stderr, "#\n# Test '%s' failed.\n#\n%s\n", name.c_str(), schema.c_str());
LOG_ERR("#\n# Test '%s' failed.\n#\n%s\n", name.c_str(), schema.c_str());
}
void verify(const std::string & actual_grammar) const {
if (trim(actual_grammar) != trim(expected_grammar)) {
_print_failure_header();
fprintf(stderr, "# EXPECTED:\n%s\n# ACTUAL:\n%s\n", expected_grammar.c_str(), actual_grammar.c_str());
LOG_ERR("# EXPECTED:\n%s\n# ACTUAL:\n%s\n", expected_grammar.c_str(), actual_grammar.c_str());
assert(false);
}
}
@@ -48,24 +52,24 @@ struct TestCase {
}
} catch (const std::runtime_error & ex) {
_print_failure_header();
fprintf(stderr, "# GRAMMAR ERROR: %s\n", ex.what());
LOG_ERR("# GRAMMAR ERROR: %s\n", ex.what());
assert(false);
}
}
void verify_status(TestCaseStatus status) const {
if (status != expected_status) {
_print_failure_header();
fprintf(stderr, "# EXPECTED STATUS: %s\n", expected_status == SUCCESS ? "SUCCESS" : "FAILURE");
fprintf(stderr, "# ACTUAL STATUS: %s\n", status == SUCCESS ? "SUCCESS" : "FAILURE");
LOG_ERR("# EXPECTED STATUS: %s\n", expected_status == SUCCESS ? "SUCCESS" : "FAILURE");
LOG_ERR("# ACTUAL STATUS: %s\n", status == SUCCESS ? "SUCCESS" : "FAILURE");
assert(false);
}
}
};
static void test_all(const std::string & title, std::function<void(const TestCase &)> runner) {
fprintf(stderr, "#\n# %s\n#\n", title.c_str());
LOG_INF("#\n# %s\n#\n", title.c_str());
auto test = [&](const TestCase & tc) {
fprintf(stderr, "- %s%s\n", tc.name.c_str(), tc.expected_status == FAILURE ? " (failure expected)" : "");
LOG_INF("- %s%s\n", tc.name.c_str(), tc.expected_status == FAILURE ? " (failure expected)" : "");
runner(tc);
};
@@ -1510,20 +1514,29 @@ static void test_all(const std::string & title, std::function<void(const TestCas
});
}
int main() {
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-json-schema-to-grammar");
test_all("JSON schema conversion", [](const TestCase & tc) {
try {
tc.verify(json_schema_to_grammar(common_json::parse(tc.schema), true));
tc.verify_status(SUCCESS);
} catch (const std::invalid_argument & ex) {
fprintf(stderr, "Error: %s\n", ex.what());
LOG_ERR("Error: %s\n", ex.what());
tc.verify_status(FAILURE);
}
});
// a document parsed up front gives the same grammar as the JSON, recursion included
{
fprintf(stderr, "- parsed document\n");
LOG_INF("- parsed document\n");
auto schema = common_json::parse(R"""({
"$ref": "#/$defs/node",
"$defs": {
@@ -1539,7 +1552,7 @@ int main() {
// a property node carries its $ref target, so its grammar names the ref rule
{
fprintf(stderr, "- sub-schema $ref\n");
LOG_INF("- sub-schema $ref\n");
auto parameters = common_json::parse(R"""({
"type": "object",
"properties": {"item": {"$ref": "#/$defs/item"}},
@@ -1577,4 +1590,8 @@ int main() {
tc.verify_expectation_parseable();
}
});
// the test aborts on a failure, so reaching this point means it passed
LOG("%s: %s\n", "test-json-schema-to-grammar", "PASSED");
common_log_flush(common_log_main());
}
tests/test-json-schema.cpp
+32 -3
View File
@@ -1,5 +1,8 @@
#include "arg.h"
#include "common.h"
#include "json-schema.h"
#include "json.h"
#include "log.h"
#include "testing.h"
#include <cstdlib>
@@ -7,6 +10,7 @@
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
static common_chat_schema_document parse(const std::string & schema) {
return common_chat_schema_from_json(common_json::parse(schema));
@@ -484,9 +488,31 @@ static void test_errors(testing & t) {
}
int main(int argc, char * argv[]) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
// this test takes an optional filter as its only positional argument
std::string filter;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else if (filter.empty()) {
filter = argv[i];
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-json-schema");
testing t(std::cout);
if (argc >= 2) {
t.set_filter(argv[1]);
if (!filter.empty()) {
t.set_filter(filter);
}
const char * verbose = getenv("LLAMA_TEST_VERBOSE");
@@ -509,5 +535,8 @@ int main(int argc, char * argv[]) {
t.test("value_types", test_value_types);
t.test("errors", test_errors);
return t.summary();
const int rc = t.summary();
LOG("%s: %s\n", "test-json-schema", rc == 0 ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return rc;
}
tests/test-llama-archs.cpp
+56 -28
View File
@@ -1,3 +1,4 @@
#include "arg.h"
#include "common.h"
#include "log.h"
#include "ggml-backend.h"
@@ -718,18 +719,24 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
const std::string template_row_cfg = std::string("|%" + std::to_string(max_arch_name_length) + "s|%") + std::to_string(max_device_label_length) + "s|%6s|";
const std::string template_row_res = "%15s %10s|%20s|\n";
// no ANSI escapes at WARN or lower (--errors-only)
const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN;
const std::string status_skip = use_color ? "\033[1;33mSKIP\033[0m" : "SKIP";
const std::string status_ok = use_color ? "\033[1;32mOK\033[0m" : "OK";
const std::string status_fail = use_color ? "\033[1;31mFAIL\033[0m" : "FAIL";
bool all_ok = true;
common_log_flush(common_log_main());
printf(template_header.c_str(), "Model arch.", "Device", "Config", "NMSE vs. CPU", "Roundtrip");
printf("|");
LOG_CNT(template_header.c_str(), "Model arch.", "Device", "Config", "NMSE vs. CPU", "Roundtrip");
LOG_CNT("|");
for (size_t i = 0; i < max_arch_name_length; i++) {
printf("-");
LOG_CNT("-");
}
printf("|");
LOG_CNT("|");
for (size_t i = 0; i < max_device_label_length; i++) {
printf("-");
LOG_CNT("-");
}
printf("|------|---------------|---------|\n");
LOG_CNT("|------|---------------|---------|\n");
for (const llm_arch & arch : llm_arch_all()) {
if (arch == LLM_ARCH_UNKNOWN) {
continue;
@@ -761,14 +768,14 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
std::vector<float> logits_cpu;
for (device_config & dc : dev_configs) {
// print test config first; should anything fail during model loading or inference, at least we know which test case caused it
printf(template_row_cfg.c_str(),
LOG_CNT(template_row_cfg.c_str(),
llm_arch_name(arch), dc.label.c_str(), config_name.c_str());
fflush(stdout);
common_log_flush(common_log_main());
std::pair<llama_model_ptr, llama_context_ptr> model_and_ctx_dev;
std::vector<float> logits_dev;
std::string status_nmse = "\033[1;33mSKIP\033[0m";
std::string status_roundtrip = "\033[1;33mSKIP\033[0m";
std::string status_nmse = status_skip;
std::string status_roundtrip = status_skip;
char nmse_str[12] = {0};
bool skip = !arch_supported(arch) || (dc.split_mode == LLAMA_SPLIT_MODE_TENSOR && dc.devs.empty());
@@ -782,10 +789,10 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
logits_dev = get_logits(model_and_ctx_dev.first.get(), model_and_ctx_dev.second.get(), tokens, encode);
const double nmse_val = nmse(logits_cpu, logits_dev);
snprintf(nmse_str, sizeof(nmse_str), "(%.2e)", nmse_val);
status_nmse = "\033[1;32mOK\033[0m";
status_nmse = status_ok;
if (nmse_val > 1e-4) {
all_ok = false;
status_nmse = "\033[1;31mFAIL\033[0m";
status_nmse = status_fail;
}
}
@@ -803,12 +810,12 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
auto model_and_ctx_roundtrip = get_model_and_ctx(nullptr, file, seed, dc.devs, dc.split_mode, encode);
const std::vector<float> logits_roundtrip = get_logits(
model_and_ctx_roundtrip.first.get(), model_and_ctx_roundtrip.second.get(), tokens, encode);
status_roundtrip = "\033[1;32mOK\033[0m";
status_roundtrip = status_ok;
GGML_ASSERT(logits_roundtrip.size() == logits_dev.size());
for (size_t i = 0; i < logits_roundtrip.size(); i++) {
if (logits_roundtrip[i] != logits_dev[i]) {
all_ok = false;
status_roundtrip = "\033[1;31mFAIL\033[0m";
status_roundtrip = status_fail;
break;
}
}
@@ -816,7 +823,7 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
}
// log the results for this test case
printf(template_row_res.c_str(),
LOG_CNT(template_row_res.c_str(),
status_nmse.c_str(), nmse_str, status_roundtrip.c_str());
}
}
@@ -826,8 +833,8 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in
}
int main(int argc, char ** argv) {
// init the logger at max verbosity. filter with a custom callback respecting the user-configure verbosity
common_log_set_verbosity_thold(LOG_LEVEL_DEBUG);
common_params params;
params.model.path = "."; // this test takes no model
common_init();
std::random_device rd;
@@ -838,58 +845,79 @@ int main(int argc, char ** argv) {
int verbosity = LOG_LEVEL_ERROR;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
usage(argv);
return 0;
}
if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--arch") == 0) {
} else if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--arch") == 0) {
if (i + 1 < argc) {
const std::string arch_name = argv[++i];
arch = llm_arch_from_string(arch_name);
if (arch == LLM_ARCH_UNKNOWN) {
LOG_ERR("%s: unkown LLM architecture: %s\n", __func__, arch_name.c_str());
common_log_flush(common_log_main());
return 1;
}
} else {
usage(argv);
return 1;
}
}
if (strcmp(argv[i], "-s") == 0 || strcmp(argv[i], "--seed") == 0) {
} else if (strcmp(argv[i], "-s") == 0 || strcmp(argv[i], "--seed") == 0) {
if (i + 1 < argc) {
seed = std::stoull(argv[++i]);
} else {
usage(argv);
return 1;
}
}
if (strcmp(argv[i], "-v") == 0) {
} else if (strcmp(argv[i], "-v") == 0) {
if (i + 1 < argc) {
verbosity = std::stoull(argv[++i]);
} else {
usage(argv);
return 1;
}
}
if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--out") == 0) {
} else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--out") == 0) {
if (i + 1 < argc) {
out = argv[++i];
} else {
usage(argv);
return 1;
}
} else if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
// the parser sets the threshold from the parsed verbosity; keep the max verbosity so that the
// test's own -v N stays the effective filter, unless the user asked for a lower threshold (--errors-only)
common_log_set_verbosity_thold(params.verbosity < LOG_DEFAULT_LLAMA ? params.verbosity : LOG_LEVEL_DEBUG);
LOG("%s: running\n", "test-llama-archs");
common_log_flush(common_log_main()); // the seed line below goes to stdout directly, so drain the start line first
printf("%s: using seed %zu\n", __func__, seed);
try {
if (!out.empty()) {
return save_models(arch, seed, verbosity, out);
const int rc = save_models(arch, seed, verbosity, out);
LOG("%s: %s\n", "test-llama-archs", rc == 0 ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return rc;
}
return test_backends(arch, seed, verbosity);
const int rc = test_backends(arch, seed, verbosity);
LOG("%s: %s\n", "test-llama-archs", rc == 0 ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return rc;
} catch (const std::exception & err) {
fprintf(stderr, "encountered runtime error: %s\n", err.what());
LOG_ERR("encountered runtime error: %s\n", err.what());
LOG("%s: %s\n", "test-llama-archs", "FAILED");
common_log_flush(common_log_main());
return -1;
}
}
tests/test-llama-grammar.cpp
+21 -5
View File
@@ -6,11 +6,24 @@
#include "../src/llama-grammar.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#include <cassert>
#include <stdexcept>
int main()
int main(int argc, char ** argv)
{
common_params params;
params.model.path = "."; // this test takes no model
common_init();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-llama-grammar");
llama_grammar_parser parsed_grammar;
std::vector<std::pair<std::string, uint32_t>> expected = {
@@ -183,10 +196,10 @@ int main()
// pretty print error message before asserting
if (expected_element.type != element->type || expected_element.value != element->value)
{
fprintf(stderr, "index: %d\n", index);
fprintf(stderr, "expected_element: %d, %u\n", expected_element.type, expected_element.value);
fprintf(stderr, "actual_element: %d, %u\n", element->type, element->value);
fprintf(stderr, "expected_element != actual_element\n");
LOG_ERR("index: %d\n", index);
LOG_ERR("expected_element: %d, %u\n", expected_element.type, expected_element.value);
LOG_ERR("actual_element: %d, %u\n", element->type, element->value);
LOG_ERR("expected_element != actual_element\n");
}
assert(expected_element.type == element->type && expected_element.value == element->value);
@@ -402,5 +415,8 @@ int main()
llama_grammar_free_impl(grammar);
// the test aborts on a mismatch, so reaching this point means it passed
LOG("%s: %s\n", "test-llama-grammar", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-log.cpp
+17 -5
View File
@@ -1,9 +1,20 @@
#include "arg.h"
#include "common.h"
#include "log.h"
#include <cstdlib>
#include <thread>
int main() {
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-log");
const int n_thread = 8;
std::thread threads[n_thread];
@@ -12,13 +23,12 @@ int main() {
const int n_msg = 1000;
for (int j = 0; j < n_msg; j++) {
const int log_type = std::rand() % 4;
const int log_type = std::rand() % 3;
switch (log_type) {
case 0: LOG_INF("Thread %d: %d\n", i, j); break;
case 1: LOG_WRN("Thread %d: %d\n", i, j); break;
case 2: LOG_ERR("Thread %d: %d\n", i, j); break;
case 3: LOG_DBG("Thread %d: %d\n", i, j); break;
case 1: LOG_TRC("Thread %d: %d\n", i, j); break;
case 2: LOG_DBG("Thread %d: %d\n", i, j); break;
default:
break;
}
@@ -35,6 +45,8 @@ int main() {
threads[i].join();
}
LOG("%s: %s\n", "test-log", "PASSED");
common_log_flush(common_log_main());
// We explicitly free the logger singleton to avoid hanging on Windows
// related to timing issues of thread startup and DLL teardown
tests/test-model-load-cancel.cpp
+39 -4
View File
@@ -1,17 +1,50 @@
#include "llama.h"
#include "common.h"
#include "arg.h"
#include "log.h"
#include <cstdlib>
#include <vector>
int main(int argc, char ** argv) {
// the model path is this test's only positional argument
char * model_argv[2] = { argv[0], nullptr };
{
common_params params;
params.model.path = "."; // this test takes no model
common_init();
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else if (model_argv[1] == nullptr) {
model_argv[1] = argv[i];
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
}
// falls back to LLAMACPP_TEST_MODELFILE, or warns and exits if no model is given
auto * model_path = common_get_model_or_exit(model_argv[1] == nullptr ? 1 : 2, model_argv);
// that call exits when no model is given, so the verdict below is only reached with a model
LOG("%s: running\n", "test-model-load-cancel");
int main(int argc, char *argv[] ) {
auto * model_path = common_get_model_or_exit(argc, argv);
auto * file = fopen(model_path, "r");
if (file == nullptr) {
fprintf(stderr, "no model at '%s' found\n", model_path);
LOG_ERR("no model at '%s' found\n", model_path);
LOG("%s: %s\n", "test-model-load-cancel", "FAILED");
common_log_flush(common_log_main());
return EXIT_FAILURE;
}
fprintf(stderr, "using '%s'\n", model_path);
LOG_INF("using '%s'\n", model_path);
fclose(file);
llama_backend_init();
@@ -23,5 +56,7 @@ int main(int argc, char *argv[] ) {
};
auto * model = llama_model_load_from_file(model_path, params);
llama_backend_free();
LOG("%s: %s\n", "test-model-load-cancel", model == nullptr ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return model == nullptr ? EXIT_SUCCESS : EXIT_FAILURE;
}
tests/test-model-resolution.cpp
+15 -1
View File
@@ -468,7 +468,16 @@ static void test_task_assembly() {
g_repos.clear();
}
int main(void) {
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-model-resolution");
// unbuffered, so a crash cannot swallow the reports already printed
setvbuf(stdout, nullptr, _IONBF, 0);
setvbuf(stderr, nullptr, _IONBF, 0);
@@ -502,5 +511,10 @@ int main(void) {
std::filesystem::remove_all(cache_dir);
printf("test-model-resolution: all tests OK\n");
// the log worker was paused above, restart it for the verdict
common_log_resume(common_log_main());
LOG("%s: %s\n", "test-model-resolution", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-mtmd-c-api.c
+6
View File
@@ -6,7 +6,10 @@
#include "mtmd.h"
#include "mtmd-helper.h"
// this test is C, it cannot use the common LOG_* macros
int main(void) {
printf("%s: running\n", "test-mtmd-c-api");
printf("\n\nTesting libmtmd C API...\n");
printf("--------\n\n");
@@ -17,6 +20,7 @@ int main(void) {
if (!chunks) {
fprintf(stderr, "Failed to create input chunks\n");
printf("%s: %s\n", "test-mtmd-c-api", "FAILED");
return 1;
}
@@ -168,5 +172,7 @@ int main(void) {
printf("\n\nDONE: test libmtmd C API...\n");
printf("%s: %s\n", "test-mtmd-c-api", "PASSED");
return 0;
}
tests/test-mtmd-impl.cpp
+23 -2
View File
@@ -1,3 +1,4 @@
#include "arg.h"
#include "testing.h"
#include "mtmd-image.h"
@@ -142,17 +143,37 @@ MAKE_TEST(test_temporal_merge_grouping) {
//
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
testing t(std::cout);
t.verbose = true;
// usage: test-mtmd-impl [filter_regex]
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
t.set_filter(argv[i]);
if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else {
t.set_filter(argv[i]);
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
// log.h cannot be included here (clip-impl.h redefines LOG_*), so print the start line directly
printf("%s: running\n", "test-mtmd-impl");
for (const auto & e : test_registry::all()) {
t.test(e.name, e.fn);
}
return t.summary();
const int rc = t.summary();
// log.h cannot be included here (clip-impl.h redefines LOG_*), so print the result line directly
printf("%s: %s\n", "test-mtmd-impl", rc == 0 ? "PASSED" : "FAILED");
return rc;
}
tests/test-opt.cpp
+68 -24
View File
@@ -5,6 +5,10 @@
#include "ggml-backend.h"
#include "ggml-opt.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#include <cmath>
#include <cinttypes>
#include <cstring>
@@ -171,25 +175,47 @@ static void helper_free_ctx_data(struct helper_ctx_data ctx_data) {
ggml_opt_dataset_free(ctx_data.dataset_unsupervised);
}
// verdict of a single sub-test; INFO, so hidden with --errors-only
static void print_ok_detail(bool subtest_ok) {
const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN;
const char * const col_ok = use_color ? "\033[1;32m" : "";
const char * const col_fail = use_color ? "\033[1;31m" : "";
const char * const col_end = use_color ? "\033[0m" : "";
if (subtest_ok) {
LOG_CNT("%sOK%s\n", col_ok, col_end);
} else {
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
}
// verdict of the whole test; always visible
static void print_ok(bool subtest_ok) {
printf(subtest_ok ? "\033[1;32mOK\033[0m\n" : "\033[1;31mFAIL\033[0m\n");
const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN;
const char * const col_ok = use_color ? "\033[1;32m" : "";
const char * const col_fail = use_color ? "\033[1;31m" : "";
const char * const col_end = use_color ? "\033[0m" : "";
if (subtest_ok) {
printf("%sOK%s\n", col_ok, col_end);
} else {
printf("%sFAIL%s\n", col_fail, col_end);
}
}
static void helper_after_test(
enum ggml_opt_optimizer_type optim,
const char * func, const bool high_level, const std::string options,
const std::string subtest, const bool subtest_ok, int & ntest, int & npass) {
printf(" %s(high_level=%s%s, subtest=%s, optimizer=%s): ",
func, high_level ? "yes" : "no", options.c_str(), subtest.c_str(), ggml_opt_optimizer_name(optim));
print_ok(subtest_ok);
LOG_CNT(" %s(high_level=%s%s, subtest=%s, optimizer=%s): ",
func, high_level ? "yes" : "no", options.c_str(), subtest.c_str(), ggml_opt_optimizer_name(optim));
print_ok_detail(subtest_ok);
if (subtest_ok)
npass++;
ntest++;
}
static void print_ok(const char * func, bool subtest_ok, int & npass, int & ntest, const char * args = "") {
printf(" %s(%s): ", func, args);
print_ok(subtest_ok);
LOG_CNT(" %s(%s): ", func, args);
print_ok_detail(subtest_ok);
if (subtest_ok)
npass++;
++ntest;
@@ -263,13 +289,13 @@ static std::pair<int, int> test_dataset(
}
}
printf(" %s(shuffle=%s, ndata_shard=%" PRId64 ", ndata_batch=%" PRId64 "): ",
__func__, shuffle ? "yes" : "no", ndata_shard, ndata_batch);
LOG_CNT(" %s(shuffle=%s, ndata_shard=%" PRId64 ", ndata_batch=%" PRId64 "): ",
__func__, shuffle ? "yes" : "no", ndata_shard, ndata_batch);
if (subtest_ok) {
printf("\033[1;32mOK\033[0m\n");
print_ok_detail(true);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
print_ok_detail(false);
}
ntest++;
}
@@ -310,12 +336,12 @@ static std::pair<int, int> test_grad(
subtest_ok = false;
}
}
printf(" %s(): ", __func__);
LOG_CNT(" %s(): ", __func__);
if (subtest_ok) {
printf("\033[1;32mOK\033[0m\n");
print_ok_detail(true);
npass++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
print_ok_detail(false);
}
ntest++;
}
@@ -896,7 +922,16 @@ static std::pair<int, int> test_backend(
}
int main(void) {
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-opt");
ggml_log_set(nullptr, nullptr);
ggml_backend_load_all();
const size_t dev_count = ggml_backend_dev_count();
@@ -931,12 +966,12 @@ int main(void) {
backends_modded.data(), nullptr, backends_modded.size(), GGML_DEFAULT_GRAPH_SIZE, false, true);
char const* devname = ggml_backend_dev_name(devs[i]);
printf("Backend %zu/%zu: %s\n", i + 1, dev_count, devname);
printf(" Device description: %s\n", ggml_backend_dev_description(devs[i]));
LOG_INF("Backend %zu/%zu: %s\n", i + 1, dev_count, devname);
LOG_INF(" Device description: %s\n", ggml_backend_dev_description(devs[i]));
size_t free, total; // NOLINT
ggml_backend_dev_memory(devs[i], &free, &total);
printf(" Device memory: %zu MB (%zu MB free)\n", total / 1024 / 1024, free / 1024 / 1024);
printf("\n");
LOG_INF(" Device memory: %zu MB (%zu MB free)\n", total / 1024 / 1024, free / 1024 / 1024);
LOG_CNT("\n");
bool skip;
{
@@ -973,21 +1008,27 @@ int main(void) {
std::pair<int, int> result;
if (!skip) {
result = test_backend(backend_sched, backends[i], optim);
printf(" %d/%d tests passed\n", result.first, result.second);
LOG_CNT(" %d/%d tests passed\n", result.first, result.second);
}
printf(" Backend %s %s: ", ggml_backend_name(backends[i]), ggml_opt_optimizer_name(optim));
const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN;
const char * const col_ok = use_color ? "\033[1;32m" : "";
const char * const col_fail = use_color ? "\033[1;31m" : "";
const char * const col_skip = use_color ? "\033[0;33m" : "";
const char * const col_end = use_color ? "\033[0m" : "";
LOG_CNT(" Backend %s %s: ", ggml_backend_name(backends[i]), ggml_opt_optimizer_name(optim));
if (skip) {
printf("\033[0;33mSKIPPED\033[0m\n");
LOG_CNT("%sSKIPPED%s\n", col_skip, col_end);
n_ok++;
} else if (result.first == result.second) {
printf("\033[1;32mOK\033[0m\n");
LOG_CNT("%sOK%s\n", col_ok, col_end);
n_ok++;
} else {
printf("\033[1;31mFAIL\033[0m\n");
LOG_CNT("%sFAIL%s\n", col_fail, col_end);
}
++n_total;
printf("\n");
LOG_CNT("\n");
ggml_backend_sched_free(backend_sched);
}
}
@@ -996,8 +1037,11 @@ int main(void) {
ggml_backend_free(backend);
}
common_log_flush(common_log_main());
printf("%zu/%zu backend*optimizer passed\n", n_ok, n_total);
bool ok = n_ok == n_total;
print_ok(ok);
LOG("%s: %s\n", "test-opt", ok ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return ok ? 0 : 1;
}
tests/test-peg-parser.cpp
+33 -3
View File
@@ -1,13 +1,40 @@
#include "arg.h"
#include "common.h"
#include "log.h"
#include <cstdlib>
#include <string>
#include <iostream>
#include <vector>
#include "peg-parser/tests.h"
int main(int argc, char *argv[]) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
// this test takes an optional filter as its only positional argument
std::string filter;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else if (filter.empty()) {
filter = argv[i];
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-peg-parser");
testing t(std::cout);
if (argc >= 2) {
t.set_filter(argv[1]);
if (!filter.empty()) {
t.set_filter(filter);
}
const char * verbose = getenv("LLAMA_TEST_VERBOSE");
@@ -22,5 +49,8 @@ int main(int argc, char *argv[]) {
t.test("serialization", test_json_serialization);
t.test("python-dict", test_python_dict_parser);
return t.summary();
const int rc = t.summary();
LOG("%s: %s\n", "test-peg-parser", rc == 0 ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return rc;
}
tests/test-quant-type-selection.cpp
+56 -29
View File
@@ -2,6 +2,8 @@
#include "ggml-cpp.h"
#include "gguf-model-data.h"
#include "llama.h"
#include "arg.h"
#include "log.h"
#include <algorithm>
#include <cstdio>
@@ -121,7 +123,7 @@ static bool parse_snapshot_file(const std::string & path, std::vector<snapshot_s
if (line[0] == '[') {
auto close = line.find(']');
if (close == std::string::npos) {
fprintf(stderr, "parse error: missing ] in '%s'\n", line.c_str());
LOG_ERR("parse error: missing ] in '%s'\n", line.c_str());
return false;
}
std::string ftype_str = line.substr(1, close - 1);
@@ -134,13 +136,13 @@ static bool parse_snapshot_file(const std::string & path, std::vector<snapshot_s
llama_ftype ftype = llama_ftype_from_name(ftype_str.c_str());
if ((int) ftype < 0) {
fprintf(stderr, "parse error: unknown ftype '%s'\n", ftype_str.c_str());
LOG_ERR("parse error: unknown ftype '%s'\n", ftype_str.c_str());
return false;
}
ggml_type dtype = ggml_type_from_name(default_str);
if (dtype == GGML_TYPE_COUNT) {
fprintf(stderr, "parse error: unknown default type '%s'\n", default_str.c_str());
LOG_ERR("parse error: unknown default type '%s'\n", default_str.c_str());
return false;
}
@@ -150,13 +152,13 @@ static bool parse_snapshot_file(const std::string & path, std::vector<snapshot_s
}
if (!cur) {
fprintf(stderr, "parse error: tensor line before any section: '%s'\n", line.c_str());
LOG_ERR("parse error: tensor line before any section: '%s'\n", line.c_str());
return false;
}
auto sp = line.rfind(' ');
if (sp == std::string::npos) {
fprintf(stderr, "parse error: no space in tensor line: '%s'\n", line.c_str());
LOG_ERR("parse error: no space in tensor line: '%s'\n", line.c_str());
return false;
}
@@ -165,7 +167,7 @@ static bool parse_snapshot_file(const std::string & path, std::vector<snapshot_s
ggml_type gt = ggml_type_from_name(ttype);
if (gt == GGML_TYPE_COUNT) {
fprintf(stderr, "parse error: unknown type '%s' for tensor '%s'\n", ttype.c_str(), tname.c_str());
LOG_ERR("parse error: unknown type '%s' for tensor '%s'\n", ttype.c_str(), tname.c_str());
return false;
}
@@ -326,15 +328,18 @@ static std::string generate_snapshot(const std::string & name,
}
static int run_generate(const std::string & snapshot_dir) {
fprintf(stderr, "This will overwrite all snapshot files in:\n %s\n", snapshot_dir.c_str());
fprintf(stderr, "Continue? [y/N] ");
// the confirmation prompt must stay visible under --errors-only
LOG_WRN("This will overwrite all snapshot files in:\n %s\n", snapshot_dir.c_str());
LOG_WRN("Continue? [y/N] ");
common_log_flush(common_log_main()); // the log is async, drain it before blocking on stdin
int ch = fgetc(stdin);
if (ch != 'y' && ch != 'Y') {
fprintf(stderr, "Aborted.\n");
LOG_WRN("Aborted.\n");
common_log_flush(common_log_main());
return 1;
}
fprintf(stderr, "\n");
LOG_WRN("\n");
int n_written = 0;
@@ -342,10 +347,11 @@ static int run_generate(const std::string & snapshot_dir) {
const auto & spec = model_specs[m];
std::string name = model_name_from_repo(spec.repo);
fprintf(stderr, "Fetching model metadata for %s from %s...\n", name.c_str(), spec.repo);
LOG_INF("Fetching model metadata for %s from %s...\n", name.c_str(), spec.repo);
auto result = gguf_fetch_model_meta(spec.repo, spec.quant);
if (!result.has_value()) {
fprintf(stderr, "ERROR: could not fetch model metadata for %s\n", name.c_str());
LOG_ERR("ERROR: could not fetch model metadata for %s\n", name.c_str());
common_log_flush(common_log_main());
return 1;
}
@@ -360,19 +366,20 @@ static int run_generate(const std::string & snapshot_dir) {
std::ofstream f(path);
if (!f.good()) {
fprintf(stderr, "ERROR: could not write %s\n", path.c_str());
LOG_ERR("ERROR: could not write %s\n", path.c_str());
llama_quant_free(qs);
llama_model_free(model);
common_log_flush(common_log_main());
return 1;
}
f << content;
n_written++;
fprintf(stderr, " wrote %s\n", path.c_str());
LOG_INF(" wrote %s\n", path.c_str());
llama_quant_free(qs);
llama_model_free(model);
}
fprintf(stderr, "%d files written\n", n_written);
LOG_INF("%d files written\n", n_written);
return 0;
}
@@ -384,8 +391,8 @@ static bool run_test_section(quantize_state_impl * qs, mock_tensors & mt, const
// verify default_type matches what llama_ftype_get_default_type returns
ggml_type computed_default = llama_ftype_get_default_type(section.ftype);
if (computed_default != section.default_type) {
printf(" FAIL [%s] default type mismatch: file says %s, code says %s\n", llama_ftype_to_name(section.ftype),
ggml_type_name(section.default_type), ggml_type_name(computed_default));
LOG_INF(" FAIL [%s] default type mismatch: file says %s, code says %s\n", llama_ftype_to_name(section.ftype),
ggml_type_name(section.default_type), ggml_type_name(computed_default));
return false;
}
@@ -409,15 +416,15 @@ static bool run_test_section(quantize_state_impl * qs, mock_tensors & mt, const
}
if (got != expected) {
printf(" FAIL %-50s %-10s expected %s, got %s\n", name, llama_ftype_to_name(section.ftype),
ggml_type_name(expected), ggml_type_name(got));
LOG_INF(" FAIL %-50s %-10s expected %s, got %s\n", name, llama_ftype_to_name(section.ftype),
ggml_type_name(expected), ggml_type_name(got));
all_pass = false;
}
}
if (n_override_found != (int) section.overrides.size()) {
printf(" FAIL [%s] override count mismatch: listed %d, matched %d\n", llama_ftype_to_name(section.ftype),
(int) section.overrides.size(), n_override_found);
LOG_INF(" FAIL [%s] override count mismatch: listed %d, matched %d\n", llama_ftype_to_name(section.ftype),
(int) section.overrides.size(), n_override_found);
all_pass = false;
}
@@ -432,11 +439,11 @@ static int run_remote_tests(const std::string & snapshot_dir, const char * argv0
for (int m = 0; m < n_model_specs; m++) {
const auto & spec = model_specs[m];
std::string name = model_name_from_repo(spec.repo);
printf("=== %s ===\n", name.c_str());
LOG_INF("=== %s ===\n", name.c_str());
auto result = gguf_fetch_model_meta(spec.repo, spec.quant, "", false);
if (!result.has_value()) {
printf(" SKIP (could not fetch model metadata)\n\n");
LOG_INF(" SKIP (could not fetch model metadata)\n\n");
total_skip++;
continue;
}
@@ -450,7 +457,7 @@ static int run_remote_tests(const std::string & snapshot_dir, const char * argv0
std::string snapshot_path = snapshot_dir + "/" + snapshot_file_from_name(name) + ".schema";
std::vector<snapshot_section> sections;
if (!parse_snapshot_file(snapshot_path, sections)) {
printf(" SKIP (could not read snapshot file: %s)\n\n", snapshot_path.c_str());
LOG_INF(" SKIP (could not read snapshot file: %s)\n\n", snapshot_path.c_str());
llama_quant_free(qs);
llama_model_free(model);
total_skip++;
@@ -469,9 +476,9 @@ static int run_remote_tests(const std::string & snapshot_dir, const char * argv0
}
}
printf(" %s %s: %d/%d ftype sections passed (%d tensors)\n", model_fail == 0 ? "PASS" : "FAIL", name.c_str(),
model_pass, model_pass + model_fail, (int) mt.tensors.size());
printf("\n");
LOG_INF(" %s %s: %d/%d ftype sections passed (%d tensors)\n", model_fail == 0 ? "PASS" : "FAIL", name.c_str(),
model_pass, model_pass + model_fail, (int) mt.tensors.size());
LOG_CNT("\n");
if (model_fail == 0) {
total_pass++;
@@ -498,23 +505,43 @@ static int run_remote_tests(const std::string & snapshot_dir, const char * argv0
}
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
std::string snapshot_dir = SNAPSHOT_DIR;
bool generate = false;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "--generate") == 0) {
generate = true;
} else if (strcmp(argv[i], "--snapshot-dir") == 0 && i + 1 < argc) {
snapshot_dir = argv[++i];
} else if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-quant-type-selection");
if (generate) {
return run_generate(snapshot_dir);
const int rc = run_generate(snapshot_dir);
LOG("%s: %s\n", "test-quant-type-selection", rc == 0 ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return rc;
}
// suppress llama log warnings during test (e.g. tensor type fallback messages)
llama_log_set([](enum ggml_log_level, const char *, void *) {}, nullptr);
return run_remote_tests(snapshot_dir, argv[0]);
const int rc = run_remote_tests(snapshot_dir, argv[0]);
LOG("%s: %s\n", "test-quant-type-selection", rc == 0 ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return rc;
}
tests/test-quantize-fns.cpp
+31 -10
View File
@@ -2,6 +2,8 @@
#include "ggml.h"
#include "ggml-cpu.h"
#include "arg.h"
#include "log.h"
#undef NDEBUG
#include <assert.h>
@@ -148,8 +150,8 @@ static int test_vec_dot_f32(bool verbose) {
const bool failed = !(error < MAX_QUANTIZATION_REFERENCE_ERROR);
num_failed += failed;
if (failed || verbose) {
printf(" f32 vec_dot n=%4d: %s (ref=%f got=%f err=%f)\n",
n, RESULT_STR[failed], ref, result, error);
LOG_INF(" f32 vec_dot n=%4d: %s (ref=%f got=%f err=%f)\n",
n, RESULT_STR[failed], ref, result, error);
}
}
return num_failed;
@@ -182,7 +184,7 @@ static int test_vec_dot_q(bool verbose) {
const ggml_type ei = (ggml_type)i;
printf("Testing %s\n", ggml_type_name((ggml_type) i));
LOG_INF("Testing %s\n", ggml_type_name((ggml_type) i));
ggml_quantize_init(ei);
if (qfns_cpu->from_float && qfns->to_float) {
@@ -201,14 +203,14 @@ static int test_vec_dot_q(bool verbose) {
bool failed = !(total_error < max_quantization_error);
num_failed += failed;
if (failed || verbose) {
printf("%5s absolute quantization error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], total_error);
LOG_INF("%5s absolute quantization error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], total_error);
}
const float reference_error = reference_quantization_error(qfns, qfns_cpu, test_size, test_data.data());
failed = !(reference_error < MAX_QUANTIZATION_REFERENCE_ERROR);
num_failed += failed;
if (failed || verbose) {
printf("%5s reference implementation error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], reference_error);
LOG_INF("%5s reference implementation error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], reference_error);
}
const float vec_dot_error = dot_product_error(qfns_cpu, type, test_size, test_data.data(), test_data2.data(), nullptr, nullptr, 1);
@@ -225,7 +227,7 @@ static int test_vec_dot_q(bool verbose) {
failed = !(vec_dot_error < max_allowed_error);
num_failed += failed;
if (failed || verbose) {
printf("%5s dot product error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error);
LOG_INF("%5s dot product error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error);
}
// Test nrc=2 path for types that support it
@@ -234,7 +236,7 @@ static int test_vec_dot_q(bool verbose) {
failed = !(vec_dot_error_nrc2 < max_allowed_error);
num_failed += failed;
if (failed || verbose) {
printf("%5s dot product error (nrc=2): %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error_nrc2);
LOG_INF("%5s dot product error (nrc=2): %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error_nrc2);
}
}
}
@@ -244,19 +246,34 @@ static int test_vec_dot_q(bool verbose) {
}
int main(int argc, char * argv[]) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
bool verbose = false;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
std::string arg;
for (int i = 1; i < argc; i++) {
arg = argv[i];
if (arg == "-v") {
verbose = true;
verbose = true; // this test's own -v, not the common --verbose
} else if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else {
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
LOG_ERR("error: unknown argument: %s\n", arg.c_str());
common_log_flush(common_log_main());
return 1;
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-quantize-fns");
ggml_cpu_init();
@@ -265,9 +282,13 @@ int main(int argc, char * argv[]) {
num_failed += test_vec_dot_f32(verbose);
num_failed += test_vec_dot_q(verbose);
const bool ok = num_failed == 0;
if (num_failed || verbose) {
printf("%d tests failed\n", num_failed);
}
return num_failed > 0;
LOG("%s: %s\n", "test-quantize-fns", ok ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return ok ? 0 : 1;
}
tests/test-quantize-perf.cpp
+50 -25
View File
@@ -2,6 +2,8 @@
#include "ggml.h"
#include "ggml-cpu.h"
#include "arg.h"
#include "log.h"
#undef NDEBUG
#include <algorithm>
@@ -101,10 +103,10 @@ static void benchmark_function(size_t size, size_t q_size, int64_t iterations, c
min_time_us = std::min(min_time_us, end_time - start_time);
}
printf(" min cycles/%d vals : %9.2f\n", QK, QK * min_time_cycles / (float) size);
printf(" avg cycles/%d vals : %9.2f\n", QK, QK * total_time_cycles / (float) (size * iterations));
printf(" float32 throughput : %9.2f GB/s\n", gigabytes_per_second(4 * size * iterations, total_time_us));
printf(" quantized throughput : %9.2f GB/s\n", gigabytes_per_second(q_size * iterations, total_time_us));
LOG_CNT(" min cycles/%d vals : %9.2f\n", QK, QK * min_time_cycles / (float) size);
LOG_CNT(" avg cycles/%d vals : %9.2f\n", QK, QK * total_time_cycles / (float) (size * iterations));
LOG_CNT(" float32 throughput : %9.2f GB/s\n", gigabytes_per_second(4 * size * iterations, total_time_us));
LOG_CNT(" quantized throughput : %9.2f GB/s\n", gigabytes_per_second(q_size * iterations, total_time_us));
}
static void usage(char * argv[]) {
@@ -138,12 +140,16 @@ static void usage(char * argv[]) {
}
int main(int argc, char * argv[]) {
common_init();
quantize_perf_params params {};
// read command line
bool invalid_param = false;
std::string arg;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
arg = argv[i];
@@ -154,7 +160,7 @@ int main(int argc, char * argv[]) {
}
size_t size = std::stoi(argv[i]);
if (size % 32 != 0) {
fprintf(stderr, "error: size %zu not divisible by 32\n", size);
LOG_ERR("error: size %zu not divisible by 32\n", size);
invalid_param = true;
break;
}
@@ -203,7 +209,7 @@ int main(int argc, char * argv[]) {
}
int alignment = std::stoi(argv[i]);
if (alignment < 0 || alignment > MAX_ALIGNMENT) {
fprintf(stderr, "error: alignment-offset must be less than %d\n", MAX_ALIGNMENT);
LOG_ERR("error: alignment-offset must be less than %d\n", MAX_ALIGNMENT);
invalid_param = true;
break;
}
@@ -215,7 +221,7 @@ int main(int argc, char * argv[]) {
}
int number = std::stoi(argv[i]);
if (number < 0 || number > MAX_ITERATIONS) {
fprintf(stderr, "error: iterations must be less than %d\n", MAX_ITERATIONS);
LOG_ERR("error: iterations must be less than %d\n", MAX_ITERATIONS);
invalid_param = true;
break;
}
@@ -223,15 +229,31 @@ int main(int argc, char * argv[]) {
} else if ((arg == "-h") || (arg == "--help")) {
usage(argv);
return 1;
} else if (arg[0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else {
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
LOG_ERR("error: unknown argument: %s\n", arg.c_str());
common_log_flush(common_log_main());
return 1;
}
}
if (invalid_param) {
fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());
LOG_ERR("error: invalid parameter for argument: %s\n", arg.c_str());
common_log_flush(common_log_main());
return 1;
}
common_argv.push_back(nullptr);
// the test's own "params" is declared above, so scope the parsed one
{
common_params params;
params.model.path = "."; // this test takes no model
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
}
LOG("%s: running\n", "test-quantize-perf");
if (params.test_sizes.empty()) {
params.test_sizes.push_back(L1_SIZE);
@@ -271,14 +293,14 @@ int main(int argc, char * argv[]) {
}
if (qfns_cpu->from_float && qfns->to_float) {
printf("%s\n", ggml_type_name(type));
LOG_INF(" running %s\n", ggml_type_name(type));
ggml_quantize_init(type);
if (params.op_quantize_row_q_reference) {
printf(" quantize_row_q_reference\n");
LOG_CNT(" quantize_row_q_reference\n");
for (size_t size : params.test_sizes) {
printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
LOG_CNT(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
auto quantize_fn = [&](void) -> float {
qfns->from_float_ref(test_data1, test_q1, size);
return test_q1[0];
@@ -286,13 +308,13 @@ int main(int argc, char * argv[]) {
size_t quantized_size = ggml_row_size(type, size);
benchmark_function(size, quantized_size, iterations, quantize_fn);
}
printf("\n");
LOG_CNT("\n");
}
if (params.op_quantize_row_q) {
printf(" quantize_row_q\n");
LOG_CNT(" quantize_row_q\n");
for (size_t size : params.test_sizes) {
printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
LOG_CNT(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
auto quantize_fn = [&](void) -> float {
qfns_cpu->from_float(test_data1, test_q1, size);
return test_q1[0];
@@ -300,14 +322,14 @@ int main(int argc, char * argv[]) {
size_t quantized_size = ggml_row_size(type, size);
benchmark_function(size, quantized_size, iterations, quantize_fn);
}
printf("\n");
LOG_CNT("\n");
}
if (params.op_dequantize_row_q) {
printf(" dequantize_row_q\n");
LOG_CNT(" dequantize_row_q\n");
qfns_cpu->from_float(test_data1, test_q1, largest);
for (size_t size : params.test_sizes) {
printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
LOG_CNT(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
auto quantize_fn = [&](void) -> float {
qfns->to_float(test_q1, test_out, size);
return test_out[0];
@@ -315,13 +337,13 @@ int main(int argc, char * argv[]) {
size_t quantized_size = ggml_row_size(type, size);
benchmark_function(size, quantized_size, iterations, quantize_fn);
}
printf("\n");
LOG_CNT("\n");
}
if (params.op_quantize_row_q_dot) {
printf(" quantize_row_q_dot\n");
LOG_CNT(" quantize_row_q_dot\n");
for (size_t size : params.test_sizes) {
printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
LOG_CNT(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
auto quantize_fn = [&](void) -> float {
const auto * vdot = ggml_get_type_traits_cpu(qfns_cpu->vec_dot_type);
vdot->from_float(test_data1, test_q1, size);
@@ -330,15 +352,15 @@ int main(int argc, char * argv[]) {
size_t quantized_size = ggml_row_size(type, size);
benchmark_function(size, quantized_size, iterations, quantize_fn);
}
printf("\n");
LOG_CNT("\n");
}
if (params.op_vec_dot_q) {
printf(" vec_dot_q\n");
LOG_CNT(" vec_dot_q\n");
qfns_cpu->from_float(test_data1, test_q1, largest);
qfns_cpu->from_float(test_data2, test_q2, largest);
for (size_t size : params.test_sizes) {
printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
LOG_CNT(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024));
auto quantize_fn = [&](void) -> float {
float result;
qfns_cpu->vec_dot(size, &result, 0, test_q1, 0, test_q2, 0, 1);
@@ -347,10 +369,13 @@ int main(int argc, char * argv[]) {
size_t quantized_size = ggml_row_size(type, size);
benchmark_function(size, quantized_size, iterations, quantize_fn);
}
printf("\n");
LOG_CNT("\n");
}
}
}
// this benchmark has no failure path, so reaching this point means it all passed
LOG("%s: %s\n", "test-quantize-perf", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-quantize-stats.cpp
+73 -40
View File
@@ -1,7 +1,9 @@
#include "llama.h"
#include "arg.h"
#include "build-info.h"
#include "common.h"
#include "log.h"
#include "../src/llama-model.h"
@@ -46,29 +48,31 @@ struct error_stats {
uint64_t error_histogram[HISTOGRAM_BUCKETS];
};
static void quantize_stats_print_usage(int /*argc*/, char ** argv) {
// print the usage text at the given level: INFO for -h, ERROR when it follows a bad argument
static void quantize_stats_print_usage(int /*argc*/, char ** argv, ggml_log_level level) {
quantize_stats_params params;
fprintf(stderr, "usage: %s [options]\n", argv[0]);
fprintf(stderr, "\n");
fprintf(stderr, "options:\n");
fprintf(stderr, " -h, --help show this help message and exit\n");
fprintf(stderr, " -m FNAME, --model FNAME\n");
fprintf(stderr, " model path (default: %s)\n", params.model.c_str());
fprintf(stderr, " -r, --reference\n");
fprintf(stderr, " use reference implementation (default: false)\n");
fprintf(stderr, " -v, --verbose\n");
fprintf(stderr, " verbose output (default: false)\n");
fprintf(stderr, " -p, --per-layer-stats\n");
fprintf(stderr, " print stats per layer (default: false)\n");
fprintf(stderr, " --histogram\n");
fprintf(stderr, " print error histogram (default: false)\n");
fprintf(stderr, " -l LAYER, --include-layer LAYER\n");
fprintf(stderr, " only test layers matching pattern\n");
fprintf(stderr, " -L LAYER, --exclude-layer LAYER\n");
fprintf(stderr, " exclude layers matching pattern\n");
fprintf(stderr, " -t TYPE, --type TYPE\n");
fprintf(stderr, " only test given type (q4_0, q4_1)\n");
fprintf(stderr, "\n");
const int verbosity = level == GGML_LOG_LEVEL_ERROR ? LOG_LEVEL_ERROR : LOG_LEVEL_INFO;
LOG_TMPL(level, verbosity, "usage: %s [options]\n", argv[0]);
LOG_TMPL(level, verbosity, "\n");
LOG_TMPL(level, verbosity, "options:\n");
LOG_TMPL(level, verbosity, " -h, --help show this help message and exit\n");
LOG_TMPL(level, verbosity, " -m FNAME, --model FNAME\n");
LOG_TMPL(level, verbosity, " model path (default: %s)\n", params.model.c_str());
LOG_TMPL(level, verbosity, " -r, --reference\n");
LOG_TMPL(level, verbosity, " use reference implementation (default: false)\n");
LOG_TMPL(level, verbosity, " -v, --verbose\n");
LOG_TMPL(level, verbosity, " verbose output (default: false)\n");
LOG_TMPL(level, verbosity, " -p, --per-layer-stats\n");
LOG_TMPL(level, verbosity, " print stats per layer (default: false)\n");
LOG_TMPL(level, verbosity, " --histogram\n");
LOG_TMPL(level, verbosity, " print error histogram (default: false)\n");
LOG_TMPL(level, verbosity, " -l LAYER, --include-layer LAYER\n");
LOG_TMPL(level, verbosity, " only test layers matching pattern\n");
LOG_TMPL(level, verbosity, " -L LAYER, --exclude-layer LAYER\n");
LOG_TMPL(level, verbosity, " exclude layers matching pattern\n");
LOG_TMPL(level, verbosity, " -t TYPE, --type TYPE\n");
LOG_TMPL(level, verbosity, " only test given type (q4_0, q4_1)\n");
LOG_TMPL(level, verbosity, "\n");
}
// Check if a layer is included/excluded by command line
@@ -121,14 +125,14 @@ static void print_error_stats(const std::string & name, const error_stats & stat
double rmse = sqrt(stats.total_error / (double) stats.num_samples);
double median = find_quantile(stats, .5);
double pct95 = find_quantile(stats, .95);
printf("%-50s: rmse %.8f, maxerr %.8f, 95pct<%.4f, median<%.4f\n", name.c_str(), rmse, stats.max_error, pct95, median);
LOG_INF("%-50s: rmse %.8f, maxerr %.8f, 95pct<%.4f, median<%.4f\n", name.c_str(), rmse, stats.max_error, pct95, median);
if (print_histogram) {
printf("Error distribution:\n");
LOG_INF("Error distribution:\n");
for (size_t i = 0; i < HISTOGRAM_BUCKETS; i++) {
double lower = i * HISTOGRAM_RANGE / HISTOGRAM_BUCKETS;
double upper = (i+1) * HISTOGRAM_RANGE / HISTOGRAM_BUCKETS;
if (i == HISTOGRAM_BUCKETS -1) upper = INFINITY;
printf("[%3.4f, %3.4f): %11" PRIu64 "\n", lower, upper, stats.error_histogram[i]);
LOG_INF("[%3.4f, %3.4f): %11" PRIu64 "\n", lower, upper, stats.error_histogram[i]);
}
}
}
@@ -226,6 +230,8 @@ static void test_roundtrip_on_layer(
}
int main(int argc, char ** argv) {
common_init();
ggml_time_init();
quantize_stats_params params;
@@ -235,11 +241,14 @@ int main(int argc, char ** argv) {
int max_thread = 0;
bool invalid_param = false;
std::string arg;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
arg = argv[i];
if (arg == "-h" || arg == "--help") {
quantize_stats_print_usage(argc, argv);
quantize_stats_print_usage(argc, argv, GGML_LOG_LEVEL_INFO);
common_log_flush(common_log_main());
exit(0);
} else if (arg == "-r" || arg == "--reference") {
params.reference = true;
@@ -280,7 +289,7 @@ int main(int argc, char ** argv) {
if (j < GGML_TYPE_COUNT) {
params.include_types.push_back((ggml_type) j);
} else {
fprintf(stderr, "error: %s not in list of types\n", argv[i]);
LOG_ERR("error: %s not in list of types\n", argv[i]);
invalid_param = true;
}
} else if (arg == "-n" || arg == "--num-threads") {
@@ -289,22 +298,38 @@ int main(int argc, char ** argv) {
break;
}
max_thread = atoi(argv[i]);
} else if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else {
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
quantize_stats_print_usage(argc, argv);
LOG_ERR("error: unknown argument: %s\n", arg.c_str());
quantize_stats_print_usage(argc, argv, GGML_LOG_LEVEL_ERROR);
common_log_flush(common_log_main());
return 1;
}
}
if (invalid_param) {
fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());
quantize_stats_print_usage(argc, argv);
LOG_ERR("error: invalid parameter for argument: %s\n", arg.c_str());
quantize_stats_print_usage(argc, argv, GGML_LOG_LEVEL_ERROR);
common_log_flush(common_log_main());
return 1;
}
common_argv.push_back(nullptr);
// the test's own "params" is declared above, so scope the parsed one
{
common_params params;
params.model.path = "."; // this test takes no model
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
}
LOG("%s: running\n", "test-quantize-stats");
llama_print_build_info(llama_version());
// load the model
fprintf(stderr, "Loading model\n");
LOG_INF("Loading model\n");
const int64_t t_main_start_us = ggml_time_us();
llama_model * model;
@@ -317,7 +342,9 @@ int main(int argc, char ** argv) {
model = llama_model_load_from_file(params.model.c_str(), mparams);
if (model == NULL) {
fprintf(stderr, "%s: error: failed to load model '%s'\n", __func__, params.model.c_str());
LOG_ERR("%s: error: failed to load model '%s'\n", __func__, params.model.c_str());
LOG("%s: %s\n", "test-quantize-stats", "FAILED");
common_log_flush(common_log_main());
return 1;
}
@@ -327,8 +354,10 @@ int main(int argc, char ** argv) {
ctx = llama_init_from_model(model, cparams);
if (ctx == NULL) {
fprintf(stderr, "%s: error: failed to create context with model '%s'\n", __func__, params.model.c_str());
LOG_ERR("%s: error: failed to create context with model '%s'\n", __func__, params.model.c_str());
llama_model_free(model);
LOG("%s: %s\n", "test-quantize-stats", "FAILED");
common_log_flush(common_log_main());
return 1;
}
}
@@ -344,15 +373,17 @@ int main(int argc, char ** argv) {
continue;
}
if (params.verbose) {
printf("%s: type %s, size %" PRId64 "\n", kv_tensor.first.c_str(), ggml_type_name(kv_tensor.second->type), ggml_nelements(kv_tensor.second));
LOG_INF("%s: type %s, size %" PRId64 "\n", kv_tensor.first.c_str(), ggml_type_name(kv_tensor.second->type), ggml_nelements(kv_tensor.second));
}
if (kv_tensor.second->type == GGML_TYPE_F16) {
is_f16 = true;
} else if (kv_tensor.second->type != GGML_TYPE_F32) {
fprintf(stderr, "%s: error: Quantization should be tested with a float model, "
LOG_ERR("%s: error: Quantization should be tested with a float model, "
"this model contains already quantized layers (%s is type %d)\n", __func__, kv_tensor.first.c_str(), kv_tensor.second->type);
llama_free(ctx);
llama_model_free(model);
LOG("%s: %s\n", "test-quantize-stats", "FAILED");
common_log_flush(common_log_main());
return 1;
}
included_layers++;
@@ -360,9 +391,9 @@ int main(int argc, char ** argv) {
}
if (is_f16) {
printf("note: source model is f16\n");
LOG_INF("note: source model is f16\n");
}
printf("testing %d layers with max size %" PRId64 "\n", included_layers, max_nelements);
LOG_INF("testing %d layers with max size %" PRId64 "\n", included_layers, max_nelements);
// allocate scratch space
std::vector<float> input_scratch;
std::vector<char> quantized_scratch;
@@ -378,7 +409,7 @@ int main(int argc, char ** argv) {
const auto * qfns_cpu = ggml_get_type_traits_cpu(type);
if (qfns_cpu->from_float && qfns->to_float) {
if (params.verbose) {
printf("testing %s ...\n", ggml_type_name(type));
LOG_INF("testing %s ...\n", ggml_type_name(type));
}
ggml_quantize_init(type);
@@ -390,7 +421,7 @@ int main(int argc, char ** argv) {
continue;
}
if (params.verbose) {
printf(" %s ...\n", kv_tensor.first.c_str());
LOG_INF(" %s ...\n", kv_tensor.first.c_str());
}
std::string layer_name { ggml_type_name(type) };
layer_name += "::" + kv_tensor.first;
@@ -423,5 +454,7 @@ int main(int argc, char ** argv) {
printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0);
}
LOG("%s: %s\n", "test-quantize-stats", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-reasoning-budget.cpp
+28 -13
View File
@@ -4,6 +4,10 @@
#include "llama.h"
#include "ggml.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#ifdef NDEBUG
#undef NDEBUG
#endif
@@ -85,7 +89,7 @@ static void test_reasoning_budget(
llama_sampler_accept(sampler, sequence[i]);
fprintf(stderr, " i=%zu: token=%d, finite_count=%zu, finite_token=%d\n", i, (int)sequence[i], finite_count, (int)finite_token);
LOG_TRC(" i=%zu: token=%d, finite_count=%zu, finite_token=%d\n", i, (int)sequence[i], finite_count, (int)finite_token);
if (finite_count == 1) {
if (actual_force_start == SIZE_MAX) {
@@ -103,28 +107,28 @@ static void test_reasoning_budget(
// Verify forcing occurred at expected positions
if (expected_force_start == SIZE_MAX) {
if (actual_force_start != SIZE_MAX) {
fprintf(stderr, "Test '%s' FAILED: Expected no forcing, but forcing occurred at %zu\n", test_name, actual_force_start);
LOG_ERR("Test '%s' FAILED: Expected no forcing, but forcing occurred at %zu\n", test_name, actual_force_start);
GGML_ASSERT(false && "Expected no forcing, but forcing occurred");
}
} else {
if (actual_force_start == SIZE_MAX) {
fprintf(stderr, "Test '%s' FAILED: Expected forcing but none occurred\n", test_name);
LOG_ERR("Test '%s' FAILED: Expected forcing but none occurred\n", test_name);
GGML_ASSERT(false && "Expected forcing but none occurred");
}
if (actual_force_start != expected_force_start) {
fprintf(stderr, "Test '%s' FAILED: Forcing started at %zu, expected %zu\n", test_name, actual_force_start, expected_force_start);
LOG_ERR("Test '%s' FAILED: Forcing started at %zu, expected %zu\n", test_name, actual_force_start, expected_force_start);
GGML_ASSERT(false && "Forcing started at wrong position");
}
}
if (expected_force_end != SIZE_MAX) {
if (actual_force_end < expected_force_end) {
fprintf(stderr, "Test '%s' FAILED: Forcing ended at %zu, expected >= %zu\n", test_name, actual_force_end, expected_force_end);
LOG_ERR("Test '%s' FAILED: Forcing ended at %zu, expected >= %zu\n", test_name, actual_force_end, expected_force_end);
GGML_ASSERT(false && "Forcing ended too early");
}
}
fprintf(stderr, " Test '%s' passed (force_start=%zu, force_end=%zu)\n", test_name, actual_force_start, actual_force_end);
LOG_INF(" Test '%s' passed (force_start=%zu, force_end=%zu)\n", test_name, actual_force_start, actual_force_end);
(void)sequence;
}
@@ -255,7 +259,7 @@ static void test_reasoning_budget_force_manual() {
// a null sampler is safely ignored
GGML_ASSERT(!common_reasoning_budget_force(nullptr));
fprintf(stderr, " Test 'manual force transition' passed\n");
LOG_INF(" Test 'manual force transition' passed\n");
}
static void test_reasoning_budget_end_match() {
@@ -330,7 +334,7 @@ static void test_reasoning_budget_end_match() {
// a null sampler is safely ignored
GGML_ASSERT(common_reasoning_budget_get_end_match(nullptr) == nullptr);
fprintf(stderr, " Test 'matched end sequence' passed\n");
LOG_INF(" Test 'matched end sequence' passed\n");
}
// UTF-8 boundary detection unit test
@@ -358,9 +362,18 @@ static void test_utf8_boundary_detection() {
GGML_ASSERT(common_utf8_is_complete(std::string("hello\xC3\xA9", 7))); // ASCII + complete 2-byte
}
int main(void) {
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-reasoning-budget");
// Reasoning budget sampler tests
printf("Testing reasoning budget sampler... ");
LOG_INF("Testing reasoning budget sampler...\n");
// Test 1: Basic budget with start/end tokens - no forcing (natural end before budget exhausted)
{
@@ -495,11 +508,13 @@ int main(void) {
test_reasoning_budget_force_manual();
test_reasoning_budget_end_match();
printf("OK (12 tests passed)\n");
LOG_CNT("OK (12 tests passed)\n");
printf("Testing UTF-8 boundary detection... ");
LOG_INF("Testing UTF-8 boundary detection...\n");
test_utf8_boundary_detection();
printf("OK\n");
LOG_CNT("OK\n");
LOG("%s: %s\n", "test-reasoning-budget", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-recurrent-state-rollback.cpp
+41 -30
View File
@@ -1,5 +1,6 @@
#include "arg.h"
#include "common.h"
#include "log.h"
#include "ggml-backend.h"
#include "llama.h"
@@ -67,7 +68,7 @@ static llama_context * init_ctx(llama_model * model, llama_context_params cparam
llama_get_memory(ctx)->state_write(collector);
llama_memory_clear(llama_get_memory(ctx), true);
if (collector.buffers.empty()) {
fprintf(stderr, "%s : no cache buffers found\n", __func__);
LOG_ERR("%s : no cache buffers found\n", __func__);
llama_free(ctx);
return nullptr;
}
@@ -117,7 +118,7 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode
llama_context * ctx_roll = make_ctx_multi();
llama_context * ctx_ref = make_ctx_multi();
if (ctx_roll == nullptr || ctx_ref == nullptr) {
fprintf(stderr, "%s : failed to init multi-seq contexts\n", __func__);
LOG_ERR("%s : failed to init multi-seq contexts\n", __func__);
return false;
}
@@ -127,7 +128,7 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode
};
if (llama_n_rs_seq(ctx_roll) < n_rollback) {
fprintf(stderr, "%s : skipping because n_rs_seq is too small\n", __func__);
LOG_WRN("%s : skipping because n_rs_seq is too small\n", __func__);
cleanup();
return true;
}
@@ -161,7 +162,7 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode
ok = ok && !llama_memory_seq_rm(llama_get_memory(ctx_roll), (llama_seq_id) s, p0 - 1, -1);
}
if (!ok) {
fprintf(stderr, "%s : multi-seq prefill/rollback failed\n", __func__);
LOG_ERR("%s : multi-seq prefill/rollback failed\n", __func__);
cleanup();
return false;
}
@@ -177,7 +178,7 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode
ok = ok && llama_decode(ctx_ref, batch) == 0;
llama_batch_free(batch);
if (!ok) {
fprintf(stderr, "%s : multi-seq replay decode failed\n", __func__);
LOG_ERR("%s : multi-seq replay decode failed\n", __func__);
cleanup();
return false;
}
@@ -193,7 +194,7 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode
const float * l_roll = llama_get_logits_ith(ctx_roll, i);
const float * l_ref = llama_get_logits_ith(ctx_ref, i);
if (l_roll == nullptr || l_ref == nullptr) {
fprintf(stderr, "%s : missing multi-seq logits at index %u\n", __func__, i);
LOG_ERR("%s : missing multi-seq logits at index %u\n", __func__, i);
cleanup();
return false;
}
@@ -208,13 +209,13 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode
}
if (diff_max > eps) {
fprintf(stderr, "%s : multi-seq split replay logits mismatch (max diff %g, first at seq %u pos %d)\n",
LOG_ERR("%s : multi-seq split replay logits mismatch (max diff %g, first at seq %u pos %d)\n",
__func__, (double) diff_max, seq_first, pos_first);
cleanup();
return false;
}
fprintf(stderr, "%s : multi-seq split replay matched (max diff %g)\n", __func__, (double) diff_max);
LOG_INF("%s : multi-seq split replay matched (max diff %g)\n", __func__, (double) diff_max);
// seq-1-only decodes must be independent of seq 0's content: diverge seq 0
// in ctx_ref only, then compare identical seq-1-only continuations bitwise
@@ -251,13 +252,13 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode
}
if (!ok || diff_tail > eps) {
fprintf(stderr, "%s : seq-1-only decode leaked seq 0 state (ok=%d, max diff %g)\n",
LOG_ERR("%s : seq-1-only decode leaked seq 0 state (ok=%d, max diff %g)\n",
__func__, ok ? 1 : 0, (double) diff_tail);
cleanup();
return false;
}
fprintf(stderr, "%s : seq-1-only decode independent of seq 0 (max diff %g)\n", __func__, (double) diff_tail);
LOG_INF("%s : seq-1-only decode independent of seq 0 (max diff %g)\n", __func__, (double) diff_tail);
cleanup();
return true;
}
@@ -269,12 +270,12 @@ static int test_rollback(const common_params & params, llama_model * model, uint
llama_context * ctx_src = make_ctx(params, model, fill);
llama_context * ctx_dst = make_ctx(params, model, fill);
if (ctx_src == nullptr || ctx_dst == nullptr) {
fprintf(stderr, "%s : failed to init contexts\n", __func__);
LOG_ERR("%s : failed to init contexts\n", __func__);
return 1;
}
if (llama_n_rs_seq(ctx_src) == 0) {
fprintf(stderr, "%s : skipping because n_rs_seq is disabled\n", __func__);
LOG_WRN("%s : skipping because n_rs_seq is disabled\n", __func__);
llama_free(ctx_src);
llama_free(ctx_dst);
return 0;
@@ -289,13 +290,13 @@ static int test_rollback(const common_params & params, llama_model * model, uint
const uint32_t n_rs_seq = llama_n_rs_seq(ctx_src);
constexpr uint32_t n_rollback = 3;
if (n_rs_seq < n_rollback) {
fprintf(stderr, "%s : skipping because n_rs_seq is too small\n", __func__);
LOG_WRN("%s : skipping because n_rs_seq is too small\n", __func__);
llama_free(ctx_src);
llama_free(ctx_dst);
return 0;
}
if (tokens.empty()) {
fprintf(stderr, "%s : not enough prompt tokens\n", __func__);
LOG_ERR("%s : not enough prompt tokens\n", __func__);
return 1;
}
tokens.resize(n_rs_seq + 1, tokens.back());
@@ -307,11 +308,11 @@ static int test_rollback(const common_params & params, llama_model * model, uint
// Replaying them crosses DSV4's ratio-4 compressor boundary.
// Rollback leaves the recurrent memory in a snapshot state (rs_idx != 0).
if (!decode_tokens(ctx_src, tokens, n_tokens)) {
fprintf(stderr, "%s : failed to decode prompt\n", __func__);
LOG_ERR("%s : failed to decode prompt\n", __func__);
return 1;
}
if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, rollback_pos, -1)) {
fprintf(stderr, "%s : rollback failed\n", __func__);
LOG_ERR("%s : rollback failed\n", __func__);
return 1;
}
@@ -327,21 +328,21 @@ static int test_rollback(const common_params & params, llama_model * model, uint
const llama_pos pos = rollback_pos + i;
if (!decode_one(ctx_src, tokens[pos], pos) ||
!decode_one(ctx_dst, tokens[pos], pos)) {
fprintf(stderr, "%s : %s replay failed at position %d\n", __func__, mode, pos);
LOG_ERR("%s : %s replay failed at position %d\n", __func__, mode, pos);
return false;
}
const float * logits_src = llama_get_logits_ith(ctx_src, 0);
const float * logits_dst = llama_get_logits_ith(ctx_dst, 0);
if (logits_src == nullptr || logits_dst == nullptr) {
fprintf(stderr, "%s : missing %s logits at position %d\n", __func__, mode, pos);
LOG_ERR("%s : missing %s logits at position %d\n", __func__, mode, pos);
return false;
}
logits_src_replay[i].assign(logits_src, logits_src + n_vocab);
for (int token = 0; token < n_vocab; ++token) {
if (logit_diff(logits_src[token], logits_dst[token]) > eps) {
fprintf(stderr, "%s : %s logits mismatch at position %d, token %d (%g != %g)\n",
LOG_ERR("%s : %s logits mismatch at position %d, token %d (%g != %g)\n",
__func__, mode, pos, token, (double) logits_src[token], (double) logits_dst[token]);
return false;
}
@@ -355,7 +356,7 @@ static int test_rollback(const common_params & params, llama_model * model, uint
if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, rollback_pos, -1) ||
!llama_memory_seq_rm(llama_get_memory(ctx_dst), 0, rollback_pos, -1)) {
fprintf(stderr, "%s : partial rollback failed\n", __func__);
LOG_ERR("%s : partial rollback failed\n", __func__);
return 1;
}
@@ -373,7 +374,7 @@ static int test_rollback(const common_params & params, llama_model * model, uint
// non-zero at load time. The restore must wipe that state and still match.
llama_context * ctx_dirty = make_ctx(params, model, fill);
if (ctx_dirty == nullptr) {
fprintf(stderr, "%s : failed to init dirty ctx\n", __func__);
LOG_ERR("%s : failed to init dirty ctx\n", __func__);
return 1;
}
@@ -385,11 +386,11 @@ static int test_rollback(const common_params & params, llama_model * model, uint
}
}
if (!decode_tokens(ctx_dirty, noise, n_tokens)) {
fprintf(stderr, "%s : dirty prompt decode failed\n", __func__);
LOG_ERR("%s : dirty prompt decode failed\n", __func__);
return 1;
}
if (!llama_memory_seq_rm(llama_get_memory(ctx_dirty), 0, rollback_pos, -1)) {
fprintf(stderr, "%s : dirty rollback failed\n", __func__);
LOG_ERR("%s : dirty rollback failed\n", __func__);
return 1;
}
@@ -398,26 +399,26 @@ static int test_rollback(const common_params & params, llama_model * model, uint
for (uint32_t i = 0; i < n_rollback; ++i) {
const llama_pos pos = rollback_pos + i;
if (!decode_one(ctx_dirty, tokens[pos], pos)) {
fprintf(stderr, "%s : dirty replay failed at position %d\n", __func__, pos);
LOG_ERR("%s : dirty replay failed at position %d\n", __func__, pos);
return 1;
}
const float * logits_dirty = llama_get_logits_ith(ctx_dirty, 0);
if (logits_dirty == nullptr) {
fprintf(stderr, "%s : missing dirty logits at position %d\n", __func__, pos);
LOG_ERR("%s : missing dirty logits at position %d\n", __func__, pos);
return 1;
}
for (int token = 0; token < n_vocab; ++token) {
if (logit_diff(logits_src_replay[i][token], logits_dirty[token]) > eps) {
fprintf(stderr, "%s : dirty-ctx logits mismatch at position %d, token %d (%g != %g)\n",
LOG_ERR("%s : dirty-ctx logits mismatch at position %d, token %d (%g != %g)\n",
__func__, pos, token, (double) logits_src_replay[i][token], (double) logits_dirty[token]);
return 1;
}
}
}
fprintf(stderr, "%s : recurrent rollback checkpoint restored successfully\n", __func__);
LOG_INF("%s : recurrent rollback checkpoint restored successfully\n", __func__);
llama_free(ctx_src);
llama_free(ctx_dst);
llama_free(ctx_dirty);
@@ -442,26 +443,36 @@ int main(int argc, char ** argv) {
return 1;
}
LOG("%s: running\n", "test-recurrent-state-rollback");
ggml_backend_load_all();
common_init_result_ptr llama_init = common_init_from_params(params);
llama_model * model = llama_init->model();
if (model == nullptr) {
fprintf(stderr, "%s : failed to init model\n", __func__);
LOG_ERR("%s : failed to init model\n", __func__);
LOG("%s: %s\n", "test-recurrent-state-rollback", "FAILED");
common_log_flush(common_log_main());
return 1;
}
if (!llama_model_is_recurrent(model) && !llama_model_is_hybrid(model)) {
fprintf(stderr, "%s : skipping for non-recurrent model\n", __func__);
LOG_WRN("%s : skipping for non-recurrent model\n", __func__);
LOG("%s: %s\n", "test-recurrent-state-rollback", "PASSED");
common_log_flush(common_log_main());
return 0;
}
for (uint8_t fill : { 0, 0x3e }) {
fprintf(stderr, "%s : testing with cache fill 0x%02x\n", __func__, fill);
LOG_INF("%s : testing with cache fill 0x%02x\n", __func__, fill);
if (test_rollback(params, model, fill) != 0) {
LOG("%s: %s\n", "test-recurrent-state-rollback", "FAILED");
common_log_flush(common_log_main());
return 1;
}
}
LOG("%s: %s\n", "test-recurrent-state-rollback", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-rope.cpp
+27 -7
View File
@@ -1,6 +1,10 @@
#include "ggml.h"
#include "ggml-cpu.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#include <cmath>
#include <cstdio>
#include <cstdlib>
@@ -124,7 +128,19 @@ static void ggml_graph_compute_helper(std::vector<uint8_t> & buf, ggml_cgraph *
ggml_graph_compute(graph, &plan);
}
int main(int /*argc*/, const char ** /*argv*/) {
int main(int argc, char ** argv) {
// the body below declares its own "params", so scope the parsed one
{
common_params params;
params.model.path = "."; // this test takes no model
common_init();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
}
LOG("%s: running\n", "test-rope");
struct ggml_init_params params = {
/* .mem_size = */ 128*1024*1024,
/* .mem_buffer = */ NULL,
@@ -245,12 +261,14 @@ int main(int /*argc*/, const char ** /*argv*/) {
// printf("%f %f\n", r1_data[i], r2_data[i]);
//}
printf("mode: %d\n", mode);
printf("sum0: %f\n", sum0);
printf("sum1: %f\n", sum1);
printf("diff: %f\n", diff);
printf("rel err: %f\n", diff / sum0);
printf("rel err: %f\n", diff / sum1);
LOG_INF("mode: %d\n", mode);
LOG_INF("sum0: %f\n", sum0);
LOG_INF("sum1: %f\n", sum1);
LOG_INF("diff: %f\n", diff);
LOG_INF("rel err: %f\n", diff / sum0);
LOG_INF("rel err: %f\n", diff / sum1);
// drain the queue, the checks below abort
common_log_flush(common_log_main());
GGML_ASSERT(diff / sum0 < 0.0001f);
GGML_ASSERT(diff / sum1 < 0.0001f);
@@ -259,5 +277,7 @@ int main(int /*argc*/, const char ** /*argv*/) {
ggml_free(ctx0);
LOG("%s: %s\n", "test-rope", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-rpc-multi-server.cpp
+37 -3
View File
@@ -4,12 +4,43 @@
#include "ggml-rpc.h"
#include "ggml.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#include <vector>
int main(int argc, char ** argv) {
GGML_ASSERT(argc == 3);
std::vector<const char *> endpoints;
{
common_params params;
params.model.path = "."; // this test takes no model
common_init();
// the test takes two positional endpoints, everything else goes to common_params_parse
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else {
endpoints.push_back(argv[i]);
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
}
LOG("%s: running\n", "test-rpc-multi-server");
GGML_ASSERT(endpoints.size() == 2);
ggml_backend_load_all();
const char * endpoint_a = argv[1];
const char * endpoint_b = argv[2];
const char * endpoint_a = endpoints[0];
const char * endpoint_b = endpoints[1];
ggml_backend_t backend_a = ggml_backend_rpc_init(endpoint_a, 0);
ggml_backend_t backend_b = ggml_backend_rpc_init(endpoint_b, 0);
@@ -43,5 +74,8 @@ int main(int argc, char ** argv) {
ggml_free(ctx);
ggml_backend_free(backend_b);
ggml_backend_free(backend_a);
LOG("%s: %s\n", "test-rpc-multi-server", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-rset-release.cpp
+33 -1
View File
@@ -5,11 +5,14 @@
#include "llama.h"
#include "common.h"
#include "arg.h"
#include "log.h"
#include <cstdint>
#include <mach/mach.h>
#include <mach/mach_host.h>
#include <unistd.h>
#include <vector>
static uint64_t wired_memory() {
vm_statistics64_data_t vmstat;
@@ -21,7 +24,34 @@ static uint64_t wired_memory() {
}
int main(int argc, char ** argv) {
auto * model_path = common_get_model_or_exit(argc, argv);
// the model path is this test's only positional argument
char * model_argv[2] = { argv[0], nullptr };
{
common_params params;
params.model.path = "."; // this test takes no model
common_init();
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else if (model_argv[1] == nullptr) {
model_argv[1] = argv[i];
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
}
// falls back to LLAMACPP_TEST_MODELFILE, or warns and exits if no model is given
auto * model_path = common_get_model_or_exit(model_argv[1] == nullptr ? 1 : 2, model_argv);
// that call exits when no model is given, so the verdict below is only reached with a model
LOG("%s: running\n", "test-rset-release");
llama_backend_init();
@@ -49,5 +79,7 @@ int main(int argc, char ** argv) {
llama_backend_free();
LOG("%s: %s\n", "test-rset-release", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-sampling.cpp
+50 -8
View File
@@ -1,6 +1,10 @@
#include "ggml.h"
#include "llama.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#ifdef NDEBUG
#undef NDEBUG
#endif
@@ -15,11 +19,11 @@ extern struct llama_sampler * llama_sampler_init_dry_testing(float dry_multiplie
static void dump(const llama_token_data_array * cur_p) {
for (size_t i = 0; i < cur_p->size; i++) {
printf("%d: %f (%f)\n", cur_p->data[i].id, cur_p->data[i].p, cur_p->data[i].logit);
LOG_CNT("%d: %f (%f)\n", cur_p->data[i].id, cur_p->data[i].p, cur_p->data[i].logit);
}
}
#define DUMP(__cur_p) do { printf("%s:%d (%s)\n", __FILE__, __LINE__, __func__); dump((__cur_p)); printf("-\n"); } while(0)
#define DUMP(__cur_p) do { LOG_CNT("%s:%d (%s)\n", __FILE__, __LINE__, __func__); dump((__cur_p)); LOG_CNT("-\n"); } while(0)
struct sampler_tester {
sampler_tester(size_t n_vocab) {
@@ -76,6 +80,8 @@ static llama_token sample_dist(llama_sampler * sampler, const std::vector<float>
}
static void test_dist_singleton_rng() {
LOG_INF(" running %s\n", __func__);
llama_sampler * singleton = llama_sampler_init_dist(4242);
llama_sampler * control = llama_sampler_init_dist(4242);
@@ -92,6 +98,8 @@ static void test_dist_singleton_rng() {
}
static void test_temp(const std::vector<float> & probs, const std::vector<float> & probs_expected, float temp) {
LOG_INF(" running %s\n", __func__);
sampler_tester tester(probs, probs_expected);
DUMP(&tester.cur_p);
@@ -103,6 +111,8 @@ static void test_temp(const std::vector<float> & probs, const std::vector<float>
}
static void test_temp_ext(const std::vector<float> & probs, const std::vector<float> & probs_expected, float temp, float delta, float exponent) {
LOG_INF(" running %s\n", __func__);
sampler_tester tester(probs, probs_expected);
DUMP(&tester.cur_p);
@@ -114,6 +124,8 @@ static void test_temp_ext(const std::vector<float> & probs, const std::vector<fl
}
static void test_top_k(const std::vector<float> & probs, const std::vector<float> & probs_expected, int k) {
LOG_INF(" running %s\n", __func__);
sampler_tester tester(probs, probs_expected);
DUMP(&tester.cur_p);
@@ -125,6 +137,8 @@ static void test_top_k(const std::vector<float> & probs, const std::vector<float
}
static void test_top_p(const std::vector<float> & probs, const std::vector<float> & probs_expected, float p) {
LOG_INF(" running %s\n", __func__);
sampler_tester tester(probs, probs_expected);
DUMP(&tester.cur_p);
@@ -136,6 +150,8 @@ static void test_top_p(const std::vector<float> & probs, const std::vector<float
}
static void test_min_p(const std::vector<float> & probs, const std::vector<float> & probs_expected, float p) {
LOG_INF(" running %s\n", __func__);
sampler_tester tester(probs, probs_expected);
DUMP(&tester.cur_p);
@@ -147,6 +163,8 @@ static void test_min_p(const std::vector<float> & probs, const std::vector<float
}
static void test_xtc(const std::vector<float> & probs, const std::vector<float> & probs_expected, float p, float t) {
LOG_INF(" running %s\n", __func__);
sampler_tester tester(probs, probs_expected);
DUMP(&tester.cur_p);
@@ -157,6 +175,8 @@ static void test_xtc(const std::vector<float> & probs, const std::vector<float>
}
static void test_typical(const std::vector<float> & probs, const std::vector<float> & probs_expected, float p) {
LOG_INF(" running %s\n", __func__);
sampler_tester tester(probs, probs_expected);
DUMP(&tester.cur_p);
@@ -170,6 +190,8 @@ static void test_penalties(
const std::vector<float> & probs, const std::vector<llama_token> & last_tokens,
const std::vector<float> & probs_expected, float repeat_penalty, float alpha_frequency, float alpha_presence
) {
LOG_INF(" running %s\n", __func__);
GGML_ASSERT(probs.size() == probs_expected.size());
sampler_tester tester(probs, probs_expected);
@@ -194,6 +216,8 @@ static void test_dry(
int dry_allowed_length, int dry_penalty_last_n,
const std::vector<std::vector<llama_token>> & seq_breakers
) {
LOG_INF(" running %s\n", __func__);
GGML_ASSERT(probs.size() == expected_probs.size());
sampler_tester tester(probs, expected_probs);
@@ -212,6 +236,8 @@ static void test_dry(
}
static void test_top_n_sigma(const std::vector<float> & probs, const std::vector<float> & probs_expected, int n) {
LOG_INF(" running %s\n", __func__);
sampler_tester tester(probs, probs_expected);
DUMP(&tester.cur_p);
@@ -224,6 +250,8 @@ static void test_top_n_sigma(const std::vector<float> & probs, const std::vector
static void test_sampler_queue(const size_t n_vocab, const std::string & samplers_sequence, const int top_k, const float top_p, const float min_p
) {
LOG_INF(" running %s\n", __func__);
sampler_tester tester(n_vocab);
llama_token min_token_id = 0;
@@ -293,7 +321,7 @@ static void test_sampler_queue(const size_t n_vocab, const std::string & sampler
}
}
printf("Sampler queue %3s OK with n_vocab=%05zu top_k=%5d top_p=%f min_p=%f\n",
LOG_CNT("Sampler queue %3s OK with n_vocab=%05zu top_k=%5d top_p=%f min_p=%f\n",
samplers_sequence.c_str(), n_vocab, top_k, top_p, min_p);
}
@@ -312,12 +340,14 @@ static void bench(llama_sampler * cnstr, const char * cnstr_name, const std::vec
}
const int64_t t_end = ggml_time_us();
llama_sampler_free(cnstr);
printf("%-43s: %8.3f us/iter\n", cnstr_name, (t_end - t_start) / (float)n_iter);
LOG_CNT("%-43s: %8.3f us/iter\n", cnstr_name, (t_end - t_start) / (float)n_iter);
}
#define BENCH(__cnstr, __data, __n_iter) bench((__cnstr), #__cnstr, (__data), (__n_iter))
static void test_perf() {
LOG_INF(" running %s\n", __func__);
const int n_vocab = 1 << 17;
std::vector<llama_token_data> data;
@@ -335,7 +365,16 @@ static void test_perf() {
BENCH(llama_sampler_init_xtc (1.0f, 0.1f, 1, 1), data, 32);
}
int main(void) {
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-sampling");
ggml_time_init();
test_dist_singleton_rng();
@@ -366,12 +405,12 @@ int main(void) {
test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f/0.4f}, 1.00f);
test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f/0.4f}, 1.05f);
printf("XTC should:\n");
LOG_CNT("XTC should:\n");
test_xtc({0.4f, 0.3f, 0.2f, 0.1f}, {0.1f}, 0.99f, 0.09f);
test_xtc({0.4f, 0.3f, 0.2f, 0.1f}, {0.2f, 0.1f}, 0.99f, 0.19f);
test_xtc({0.4f, 0.3f, 0.2f, 0.1f}, {0.3f, 0.2f, 0.1f}, 0.99f, 0.29f);
printf("XTC should not:\n");
LOG_CNT("XTC should not:\n");
test_xtc({0.4f, 0.3f, 0.2f, 0.1f}, {0.4f, 0.3f, 0.2f, 0.1f}, 0.99f, 0.39f);
test_typical({0.97f, 0.01f, 0.01f, 0.01f}, {0.97f}, 0.5f);
@@ -424,9 +463,12 @@ int main(void) {
test_sampler_queue(10000, "mkp", 100, 0.8f, 0.1f);
test_sampler_queue(10000, "mpk", 100, 0.8f, 0.1f);
printf("OK\n");
LOG_CNT("OK\n");
test_perf();
// the tests abort on failure, so reaching this point means they all passed
LOG("%s: %s\n", "test-sampling", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-save-load-state.cpp
+34 -20
View File
@@ -35,7 +35,8 @@ static llama_tokens generate_tokens(llama_context * ctx, llama_sampler * smpl, i
for (int i = 0; i < n_predict; i++) {
auto next_token = llama_sampler_sample(smpl, ctx, -1);
LOG("%d ", next_token);
// one row per generated token: pure noise, TRACE only
LOG_CNTV(LOG_LEVEL_TRACE, "%d ", next_token);
result.push_back(next_token);
common_batch_clear(batch.get());
@@ -48,6 +49,8 @@ static llama_tokens generate_tokens(llama_context * ctx, llama_sampler * smpl, i
n_past++;
}
LOG_CNTV(LOG_LEVEL_TRACE, "\n");
return result;
}
@@ -71,15 +74,13 @@ static llama_tokens test_baseline(struct llama_model * model, const struct commo
return {};
}
LOG("\n=== Test 1: baseline ===\n");
LOG_CNT("=== Test 1: baseline ===\n");
auto result = generate_tokens(ctx.get(), smpl.get(), n_past, params.n_predict, 0);
if (result.empty()) {
return {};
}
LOG("\n");
return result;
}
@@ -103,7 +104,7 @@ static bool test_seq_rm_isolated(
return false;
}
LOG("\n=== Test 2: sequence removal isolation ===\n");
LOG_CNT("=== Test 2: sequence removal isolation ===\n");
const size_t n_tokens = tokens.size() < 128 ? tokens.size() : 128;
for (llama_seq_id seq_id = 0; seq_id < 2; ++seq_id) {
@@ -156,7 +157,8 @@ static bool test_seq_rm_isolated(
return false;
}
LOG("PASS\n");
// per-sub-test verdict: TRACE, the per-model line already reports the result
LOG_TRC("PASS\n");
return true;
}
@@ -175,7 +177,7 @@ static bool test_state_load(struct llama_model * model, const struct common_para
auto smpl = llama_sampler_ptr{llama_sampler_chain_init(sparams)};
llama_sampler_chain_add(smpl.get(), llama_sampler_init_dist(params.sampling.seed));
LOG("\n=== Test 3: state load ===\n");
LOG_CNT("=== Test 3: state load ===\n");
// Load state from file
llama_tokens unused_sts(tokens.size());
@@ -206,7 +208,7 @@ static bool test_state_load(struct llama_model * model, const struct common_para
return false;
}
LOG("\nPASS\n");
LOG_TRC("PASS\n");
return true;
}
@@ -226,7 +228,7 @@ static bool test_seq_cp_host(struct llama_model * model, const struct common_par
auto smpl = llama_sampler_ptr{llama_sampler_chain_init(sparams)};
llama_sampler_chain_add(smpl.get(), llama_sampler_init_dist(params.sampling.seed));
LOG("\n=== Test 4: seq copy (host) ===\n");
LOG_CNT("=== Test 4: seq copy (host) ===\n");
// Load state from file
llama_tokens unused_sts(tokens.size());
@@ -278,7 +280,7 @@ static bool test_seq_cp_host(struct llama_model * model, const struct common_par
return false;
}
LOG("\nPASS\n");
LOG_TRC("PASS\n");
return true;
}
@@ -298,7 +300,7 @@ static bool test_seq_cp_device(struct llama_model * model, const struct common_p
auto smpl = llama_sampler_ptr{llama_sampler_chain_init(sparams)};
llama_sampler_chain_add(smpl.get(), llama_sampler_init_dist(params.sampling.seed));
LOG("\n=== Test 5: seq copy (device) ===\n");
LOG_CNT("=== Test 5: seq copy (device) ===\n");
// Load state from file
llama_tokens unused_sts(tokens.size());
@@ -350,7 +352,7 @@ static bool test_seq_cp_device(struct llama_model * model, const struct common_p
return false;
}
LOG("\nPASS\n");
LOG_TRC("PASS\n");
return true;
}
@@ -367,7 +369,7 @@ static bool test_seq_cp_scatter(struct llama_model * model, const struct common_
params_ctx.kv_unified = true;
auto ctx = llama_context_ptr{llama_init_from_model(model, params_ctx)};
LOG("\n=== Test %d: seq copy (%s, scatter) ===\n", test_num, on_device ? "device" : "host");
LOG_CNT("=== Test %d: seq copy (%s, scatter) ===\n", test_num, on_device ? "device" : "host");
const uint32_t flags = on_device ? LLAMA_STATE_SEQ_FLAGS_ON_DEVICE : LLAMA_STATE_SEQ_FLAGS_NONE;
@@ -444,7 +446,7 @@ static bool test_seq_cp_scatter(struct llama_model * model, const struct common_
return false;
}
LOG("\nPASS\n");
LOG_TRC("PASS\n");
return true;
}
@@ -455,7 +457,7 @@ static bool test_state_roundtrip(struct llama_model * model, const struct common
auto params_ctx = common_context_params_to_llama(params);
auto ctx = llama_context_ptr{llama_init_from_model(model, params_ctx)};
LOG("\n=== Test 8: state blob round-trip ===\n");
LOG_CNT("=== Test 8: state blob round-trip ===\n");
if (llama_decode(ctx.get(), llama_batch_get_one(const_cast<llama_token *>(tokens.data()), (int32_t) tokens.size()))) {
LOG_ERR("\n%s: failed to decode prompt\n", __func__);
@@ -503,7 +505,7 @@ static bool test_state_roundtrip(struct llama_model * model, const struct common
return false;
}
LOG("\nPASS\n");
LOG_TRC("PASS\n");
return true;
}
@@ -590,7 +592,7 @@ static bool run_save_load_tests_for_model(const std::string & model_path, const
return false;
}
LOG("\nAll tests passed.\n");
LOG_CNT("All tests passed.\n");
return true;
}
@@ -615,6 +617,7 @@ int main(int argc, char ** argv) {
if (strcmp(argv[i], "--models") == 0) {
if (i + 1 >= argc) {
LOG_ERR("%s: --models requires a directory argument\n", __func__);
common_log_flush(common_log_main());
return 1;
}
models_dir = argv[i + 1];
@@ -645,12 +648,16 @@ int main(int argc, char ** argv) {
params.n_predict = 16;
}
LOG("%s: running\n", "test-save-load-state");
ggml_backend_load_all();
if (!models_dir.empty()) {
// run the suite over every dummy model in the directory
if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) {
LOG_ERR("%s: models directory '%s' does not exist\n", __func__, models_dir.c_str());
LOG("%s: %s\n", "test-save-load-state", "FAILED");
common_log_flush(common_log_main());
return 1;
}
@@ -664,6 +671,8 @@ int main(int argc, char ** argv) {
if (models.empty()) {
LOG_ERR("%s: no .gguf models found in '%s'\n", __func__, models_dir.c_str());
LOG("%s: %s\n", "test-save-load-state", "FAILED");
common_log_flush(common_log_main());
return 1;
}
@@ -672,7 +681,7 @@ int main(int argc, char ** argv) {
size_t n_pass = 0;
size_t n_fail = 0;
for (const auto & model_path : models) {
LOG("\n================================================================\n");
LOG_CNT("================================================================\n");
LOG_INF("%s: model %s\n", __func__, model_path.c_str());
if (run_save_load_tests_for_model(model_path, params)) {
@@ -682,12 +691,17 @@ int main(int argc, char ** argv) {
}
}
LOG("\n================================================================\n");
LOG_CNT("================================================================\n");
LOG_INF("%s: summary: %zu passed, %zu failed (of %zu)\n", __func__, n_pass, n_fail, models.size());
LOG("%s: %s\n", "test-save-load-state", n_fail == 0 ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return n_fail == 0 ? 0 : 1;
}
// single-model mode
return run_save_load_tests_for_model(params.model.path, params) ? 0 : 1;
const bool success = run_save_load_tests_for_model(params.model.path, params);
LOG("%s: %s\n", "test-save-load-state", success ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return success ? 0 : 1;
}
tests/test-state-restore-fragmented.cpp
+27 -12
View File
@@ -8,6 +8,7 @@
#include "arg.h"
#include "common.h"
#include "log.h"
#include "llama.h"
#include <vector>
@@ -28,6 +29,8 @@ int main(int argc, char ** argv) {
return 1;
}
LOG("%s: running\n", "test-state-restore-fragmented");
// init
ggml_backend_load_all();
@@ -38,7 +41,9 @@ int main(int argc, char ** argv) {
llama_context * ctx = llama_init->context();
if (model == nullptr || ctx == nullptr) {
fprintf(stderr, "%s : failed to init\n", __func__);
LOG_ERR("%s : failed to init\n", __func__);
LOG("%s: %s\n", "test-state-restore-fragmented", "FAILED");
common_log_flush(common_log_main());
return 1;
}
@@ -58,26 +63,30 @@ int main(int argc, char ** argv) {
batch.logits[batch.n_tokens - 1] = true;
if (llama_decode(ctx, batch)) {
fprintf(stderr, "%s : failed to decode seq 0\n", __func__);
LOG_ERR("%s : failed to decode seq 0\n", __func__);
LOG("%s: %s\n", "test-state-restore-fragmented", "FAILED");
common_log_flush(common_log_main());
return 1;
}
fprintf(stderr, "%s : processed prompt on seq 0, 1, 2 (%zu tokens each)\n", __func__, tokens.size());
LOG_INF("%s : processed prompt on seq 0, 1, 2 (%zu tokens each)\n", __func__, tokens.size());
// Save state of seq 1
std::vector<uint8_t> seq_state(llama_state_seq_get_size(ctx, 1));
const size_t ncopy = llama_state_seq_get_data(ctx, seq_state.data(), seq_state.size(), 1);
if (ncopy != seq_state.size()) {
fprintf(stderr, "%s : failed to save seq 1 state\n", __func__);
LOG_ERR("%s : failed to save seq 1 state\n", __func__);
LOG("%s: %s\n", "test-state-restore-fragmented", "FAILED");
common_log_flush(common_log_main());
return 1;
}
fprintf(stderr, "%s : saved seq 1 state, %zu bytes\n", __func__, ncopy);
LOG_INF("%s : saved seq 1 state, %zu bytes\n", __func__, ncopy);
// clear seq 1 to create a "hole" in the KV cache (fragmentation)
// 0.20.20.20.2....
llama_memory_t mem = llama_get_memory(ctx);
llama_memory_seq_rm(mem, 1, -1, -1);
fprintf(stderr, "%s : cleared seq 1 to create fragmentation\n", __func__);
LOG_INF("%s : cleared seq 1 to create fragmentation\n", __func__);
// Now the cache has holes where seq 1 was
// This creates fragmentation - there's no contiguous block large enough
@@ -88,13 +97,15 @@ int main(int argc, char ** argv) {
// Before the fix, this would fail with "failed to find available cells in kv cache"
const size_t nset = llama_state_seq_set_data(ctx, seq_state.data(), seq_state.size(), 1);
if (nset != seq_state.size()) {
fprintf(stderr, "%s : FAILED to restore seq state into fragmented cache (got %zu, expected %zu)\n",
LOG_ERR("%s : FAILED to restore seq state into fragmented cache (got %zu, expected %zu)\n",
__func__, nset, seq_state.size());
fprintf(stderr, "%s : This is the bug - state restore fails with fragmented KV cache\n", __func__);
LOG_ERR("%s : This is the bug - state restore fails with fragmented KV cache\n", __func__);
LOG("%s: %s\n", "test-state-restore-fragmented", "FAILED");
llama_batch_free(batch);
common_log_flush(common_log_main());
return 1;
}
fprintf(stderr, "%s : restored state into seq 1, %zu bytes\n", __func__, nset);
LOG_INF("%s : restored state into seq 1, %zu bytes\n", __func__, nset);
// Verify we can decode with the restored state
// Generate one token to verify the restored state is usable
@@ -109,17 +120,21 @@ int main(int argc, char ** argv) {
common_batch_add(batch, next_token, (int)tokens.size(), {1}, true);
if (llama_decode(ctx, batch)) {
fprintf(stderr, "%s : failed to decode with restored state\n", __func__);
LOG_ERR("%s : failed to decode with restored state\n", __func__);
LOG("%s: %s\n", "test-state-restore-fragmented", "FAILED");
llama_sampler_free(smpl);
llama_batch_free(batch);
common_log_flush(common_log_main());
return 1;
}
fprintf(stderr, "%s : successfully decoded with restored state, generated: '%s'\n", __func__, next_token_str.c_str());
fprintf(stderr, "%s : SUCCESS - state restore works with fragmented KV cache\n", __func__);
LOG_INF("%s : successfully decoded with restored state, generated: '%s'\n", __func__, next_token_str.c_str());
LOG_INF("%s : SUCCESS - state restore works with fragmented KV cache\n", __func__);
llama_sampler_free(smpl);
llama_batch_free(batch);
LOG("%s: %s\n", "test-state-restore-fragmented", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-thread-safety.cpp
+14 -4
View File
@@ -22,6 +22,8 @@ int main(int argc, char ** argv) {
return 1;
}
LOG("%s: running\n", "test-thread-safety");
llama_backend_init();
llama_numa_init(params.numa);
@@ -56,6 +58,8 @@ int main(int argc, char ** argv) {
std::atomic<bool> failed = false;
for (int m = 0; m < num_models; ++m) {
LOG_INF(" running model %d/%d\n", m + 1, num_models);
auto mparams = common_model_params_to_llama(params);
if (m < gpu_dev_count) {
@@ -71,6 +75,8 @@ int main(int argc, char ** argv) {
llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);
if (model == NULL) {
LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.path.c_str());
LOG("%s: %s\n", "test-thread-safety", "FAILED");
common_log_flush(common_log_main());
return 1;
}
@@ -156,11 +162,15 @@ int main(int argc, char ** argv) {
thread.join();
}
if (failed) {
const bool ok = !failed;
if (!ok) {
LOG_ERR("One or more threads failed.\n");
return 1;
} else {
LOG_INF("All threads finished without errors.\n");
}
LOG_INF("All threads finished without errors.\n");
return 0;
LOG("%s: %s\n", "test-thread-safety", ok ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return ok ? 0 : 1;
}
tests/test-tokenizer-0.cpp
+75 -33
View File
@@ -2,6 +2,9 @@
#include "common.h"
#include "console.h"
#include "arg.h"
#include "log.h"
#include <cstdio>
#include <string>
#include <map>
@@ -63,7 +66,7 @@ static llama_tests read_tests(const std::string & fname_inp, const std::string &
std::ifstream ifs_inp(fname_inp);
if (!ifs_inp) {
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_inp.c_str());
LOG_ERR("%s : error: could not open file '%s'\n", __func__, fname_inp.c_str());
return tests;
}
@@ -71,7 +74,7 @@ static llama_tests read_tests(const std::string & fname_inp, const std::string &
std::ifstream ifs_out(fname_out);
if (!ifs_out) {
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str());
LOG_ERR("%s : error: could not open file '%s'\n", __func__, fname_out.c_str());
return tests;
}
@@ -96,7 +99,7 @@ static llama_tests read_tests(const std::string & fname_inp, const std::string &
}
if (sinp.size() != sout.size()) {
fprintf(stderr, "%s : error: input and output files have different number of tests\n", __func__);
LOG_ERR("%s : error: input and output files have different number of tests\n", __func__);
return tests;
}
@@ -124,22 +127,44 @@ static llama_tests read_tests(const std::string & fname_inp, const std::string &
}
int main(int argc, char **argv) {
if (argc < 2) {
fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]);
common_params params;
params.model.path = "."; // this test takes no model
common_init();
std::vector<std::string> positional;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else {
positional.push_back(argv[i]);
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
const std::string fname = argv[1];
if (positional.empty()) {
LOG_ERR("Usage: %s vocab-file [text-file]\n", argv[0]);
common_log_flush(common_log_main());
return 1;
}
LOG("%s: running\n", "test-tokenizer-0");
const std::string fname = positional[0];
const std::string fname_inp = fname + ".inp";
const std::string fname_out = fname + ".out";
std::string fname_text;
if (argc > 2) {
fname_text = argv[2];
if (positional.size() > 1) {
fname_text = positional[1];
}
fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str());
LOG_INF("%s : reading vocab from: '%s'\n", __func__, fname.c_str());
llama_model * model;
llama_context * ctx;
@@ -155,7 +180,9 @@ int main(int argc, char **argv) {
model = llama_model_load_from_file(fname.c_str(), mparams);
if (model == NULL) {
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
LOG_ERR("%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
LOG("%s: %s\n", "test-tokenizer-0", "FAILED");
common_log_flush(common_log_main());
return 1;
}
@@ -164,8 +191,10 @@ int main(int argc, char **argv) {
ctx = llama_init_from_model(model, cparams);
if (ctx == NULL) {
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
LOG_ERR("%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
llama_model_free(model);
LOG("%s: %s\n", "test-tokenizer-0", "FAILED");
common_log_flush(common_log_main());
return 1;
}
}
@@ -186,7 +215,9 @@ int main(int argc, char **argv) {
const auto res = read_tests(fname_inp, fname_out);
if (res.empty()) {
fprintf(stderr, "%s : error: no tests found\n", __func__);
LOG_ERR("%s : error: no tests found\n", __func__);
LOG("%s: %s\n", "test-tokenizer-0", "FAILED");
common_log_flush(common_log_main());
exit(1);
}
@@ -210,14 +241,15 @@ int main(int argc, char **argv) {
continue;
}
printf("\n");
printf("src: '%s'\n", test_kv.first.c_str());
printf("res: '%s'\n", common_detokenize(ctx, res).c_str());
printf("tok: ");
// per-data-item detail: INFO, so hidden under --errors-only
LOG_CNT("\n");
LOG_CNT("src: '%s'\n", test_kv.first.c_str());
LOG_CNT("res: '%s'\n", common_detokenize(ctx, res).c_str());
LOG_CNT("tok: ");
for (const auto & tok : res) {
printf("%d ", tok);
LOG_CNT("%d ", tok);
}
printf("\n");
LOG_CNT("\n");
bool correct = res.size() == test_kv.second.size();
for (int i = 0; i < (int) res.size() && correct; ++i) {
@@ -227,20 +259,21 @@ int main(int argc, char **argv) {
}
if (!correct) {
fprintf(stderr, "%s : failed test: '%s'\n", __func__, test_kv.first.c_str());
fprintf(stderr, "%s : detokenized to: '%s' instead of '%s'\n", __func__,
LOG_ERR("%s : failed test: '%s'\n", __func__, test_kv.first.c_str());
LOG_ERR("%s : detokenized to: '%s' instead of '%s'\n", __func__,
common_detokenize(ctx, res).c_str(),
common_detokenize(ctx, test_kv.second).c_str());
fprintf(stderr, "%s : expected tokens: ", __func__);
// partial line: LOG_CNTV adds no prefix, error level so --errors-only keeps it
LOG_ERR("%s : expected tokens: ", __func__);
for (const auto & t : test_kv.second) {
fprintf(stderr, "%6d '%s', ", t, common_token_to_piece(ctx, t).c_str());
LOG_CNTV(LOG_LEVEL_ERROR, "%6d '%s', ", t, common_token_to_piece(ctx, t).c_str());
}
fprintf(stderr, "\n");
fprintf(stderr, "%s : got tokens: ", __func__);
LOG_CNTV(LOG_LEVEL_ERROR, "\n");
LOG_ERR("%s : got tokens: ", __func__);
for (const auto & t : res) {
fprintf(stderr, "%6d '%s', ", t, common_token_to_piece(ctx, t).c_str());
LOG_CNTV(LOG_LEVEL_ERROR, "%6d '%s', ", t, common_token_to_piece(ctx, t).c_str());
}
fprintf(stderr, "\n");
LOG_CNTV(LOG_LEVEL_ERROR, "\n");
success = false;
}
@@ -254,19 +287,21 @@ int main(int argc, char **argv) {
// single threaded tokenization
if (!fname_text.empty()) {
fprintf(stderr, "%s : tokenizing: '%s'\n", __func__, fname_text.c_str());
LOG_INF("%s : tokenizing: '%s'\n", __func__, fname_text.c_str());
std::string text;
{
std::ifstream ifs(fname_text);
if (!ifs) {
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_text.c_str());
LOG_ERR("%s : error: could not open file '%s'\n", __func__, fname_text.c_str());
LOG("%s: %s\n", "test-tokenizer-0", "FAILED");
common_log_flush(common_log_main());
return 1;
}
text = std::string(std::istreambuf_iterator<char>(ifs), std::istreambuf_iterator<char>());
}
fprintf(stderr, "%s : text size: %zu\n", __func__, text.size());
LOG_INF("%s : text size: %zu\n", __func__, text.size());
std::vector<llama_token> res;
@@ -277,17 +312,19 @@ int main(int argc, char **argv) {
const auto t_end = ggml_time_us();
fprintf(stderr, "%s : tokenized in %.3f ms (cpp)\n", __func__, (t_end - t_start) / 1000.0);
LOG_INF("%s : tokenized in %.3f ms (cpp)\n", __func__, (t_end - t_start) / 1000.0);
}
fprintf(stderr, "%s : tokens: %zu\n", __func__, res.size());
LOG_INF("%s : tokens: %zu\n", __func__, res.size());
{
const std::string fname_out = fname_text + ".tokcpp";
std::ofstream ofs(fname_out);
if (!ofs) {
fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str());
LOG_ERR("%s : error: could not open file '%s'\n", __func__, fname_out.c_str());
LOG("%s: %s\n", "test-tokenizer-0", "FAILED");
common_log_flush(common_log_main());
return 1;
}
@@ -297,7 +334,7 @@ int main(int argc, char **argv) {
}
}
fprintf(stderr, "%s : tokens written to '%s'\n", __func__, (fname_text + ".tokcpp").c_str());
LOG_INF("%s : tokens written to '%s'\n", __func__, (fname_text + ".tokcpp").c_str());
}
llama_free(ctx);
@@ -305,8 +342,13 @@ int main(int argc, char **argv) {
llama_backend_free();
common_log_flush(common_log_main());
printf("\n");
printf("Tests %s\n", success ? "passed" : "failed");
LOG("%s: %s\n", "test-tokenizer-0", success ? "PASSED" : "FAILED");
common_log_flush(common_log_main());
return success ? 0 : 3;
}
tests/test-tokenizer-1-bpe.cpp
+57 -21
View File
@@ -2,6 +2,9 @@
#include "common.h"
#include "console.h"
#include "arg.h"
#include "log.h"
#include "../src/unicode.h"
#include <cassert>
@@ -15,25 +18,40 @@
#include <atomic>
int main(int argc, char **argv) {
if (argc < 2 || argc > 3) {
fprintf(stderr, "Usage: %s <vocab-file> [--ignore-merges]\n", argv[0]);
common_params params;
params.model.path = "."; // this test takes no model
common_init();
std::vector<std::string> positional;
bool ignore_merges = false;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (std::strcmp(argv[i], "--ignore-merges") == 0) {
ignore_merges = true;
} else if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else {
positional.push_back(argv[i]);
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
const std::string fname = argv[1];
bool ignore_merges = false;
if (argc == 3) {
if (std::strcmp(argv[2], "--ignore-merges") != 0) {
fprintf(stderr, "Usage: %s <vocab-file> [--ignore-merges]\n", argv[0]);
return 1;
}
ignore_merges = true;
if (positional.size() != 1) {
LOG_ERR("Usage: %s <vocab-file> [--ignore-merges]\n", argv[0]);
common_log_flush(common_log_main());
return 1;
}
fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str());
const std::string fname = positional[0];
LOG_INF("%s : reading vocab from: '%s'\n", __func__, fname.c_str());
if (ignore_merges) {
fprintf(stderr, "%s : ignoring merges for tokens inside vocab\n", __func__);
LOG_INF("%s : ignoring merges for tokens inside vocab\n", __func__);
}
llama_model * model;
@@ -50,7 +68,8 @@ int main(int argc, char **argv) {
model = llama_model_load_from_file(fname.c_str(), mparams);
if (model == NULL) {
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
LOG_ERR("%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
common_log_flush(common_log_main());
return 1;
}
@@ -59,8 +78,9 @@ int main(int argc, char **argv) {
ctx = llama_init_from_model(model, cparams);
if (ctx == NULL) {
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
LOG_ERR("%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
llama_model_free(model);
common_log_flush(common_log_main());
return 1;
}
}
@@ -69,9 +89,13 @@ int main(int argc, char **argv) {
//GGML_ASSERT(llama_vocab_type(vocab) == LLAMA_VOCAB_TYPE_BPE);
if (llama_vocab_type(vocab) != LLAMA_VOCAB_TYPE_BPE) {
// a vocab of another type is a skip (99), so it does not get a running line
common_log_flush(common_log_main());
return 99;
}
LOG("%s: running\n", "test-tokenizer-1-bpe");
#ifdef _WIN32
// We need this for unicode console support
console::init(false, false);
@@ -80,27 +104,33 @@ int main(int argc, char **argv) {
const int n_vocab = llama_vocab_n_tokens(vocab);
LOG_INF(" running vocab detokenize round-trip (%d tokens)\n", n_vocab);
for (int i = 0; i < n_vocab; ++i) {
std::string str = common_detokenize(ctx, std::vector<int>(1, i));
try {
auto cps = unicode_cpts_from_utf8(str);
std::vector<llama_token> tokens = common_tokenize(ctx, str, false, true);
if (ignore_merges && tokens.size() > 1) {
fprintf(stderr,
"%s : error: token %d detokenizes to '%s'(%zu) but "
LOG_ERR("%s : error: token %d detokenizes to '%s'(%zu) but "
"tokenization of this to multiple tokens: [",
__func__, i, str.c_str(), str.length());
fprintf(stderr, "%d", tokens[0]);
// partial line: LOG_CNTV adds no prefix, error level so --errors-only keeps it
LOG_CNTV(LOG_LEVEL_ERROR, "%d", tokens[0]);
for (size_t i = 1; i < tokens.size(); i++) {
fprintf(stderr, ", %d", tokens[i]);
LOG_CNTV(LOG_LEVEL_ERROR, ", %d", tokens[i]);
}
fprintf(stderr, "]\n");
LOG_CNTV(LOG_LEVEL_ERROR, "]\n");
LOG("%s: %s\n", "test-tokenizer-1-bpe", "FAILED");
common_log_flush(common_log_main());
return 2;
}
std::string check = common_detokenize(ctx, tokens);
if (check != str) {
fprintf(stderr, "%s : error: token %d detokenizes to '%s'(%zu) but tokenization of this detokenizes to '%s'(%zu)\n",
LOG_ERR("%s : error: token %d detokenizes to '%s'(%zu) but tokenization of this detokenizes to '%s'(%zu)\n",
__func__, i, str.c_str(), str.length(), check.c_str(), check.length());
LOG("%s: %s\n", "test-tokenizer-1-bpe", "FAILED");
common_log_flush(common_log_main());
return 2;
}
}
@@ -111,6 +141,8 @@ int main(int argc, char **argv) {
// unicode
{
LOG_INF(" running unicode codepoint round-trip\n");
const int nthread = std::thread::hardware_concurrency();
std::vector<std::thread> threads(nthread);
@@ -129,7 +161,7 @@ int main(int argc, char **argv) {
std::vector<llama_token> tokens = common_tokenize(ctx, str, false);
std::string check = common_detokenize(ctx, tokens);
if (cp != 9601 && str != check) {
fprintf(stderr, "error: codepoint 0x%x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n",
LOG_ERR("error: codepoint 0x%x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n",
cp, check.c_str(), check.length(), str.c_str(), str.length());
errcode = 3;
}
@@ -142,6 +174,8 @@ int main(int argc, char **argv) {
}
if (errcode) {
LOG("%s: %s\n", "test-tokenizer-1-bpe", "FAILED");
common_log_flush(common_log_main());
return errcode;
}
}
@@ -151,5 +185,7 @@ int main(int argc, char **argv) {
llama_backend_free();
LOG("%s: %s\n", "test-tokenizer-1-bpe", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-tokenizer-1-spm.cpp
+45 -8
View File
@@ -2,6 +2,9 @@
#include "common.h"
#include "console.h"
#include "arg.h"
#include "log.h"
#include "../src/unicode.h"
#include <cassert>
@@ -15,14 +18,32 @@
#include <atomic>
int main(int argc, char ** argv) {
if (argc < 2) {
fprintf(stderr, "Usage: %s <vocab-file>\n", argv[0]);
common_params params;
params.model.path = "."; // this test takes no model
common_init();
std::string fname;
std::vector<char *> common_argv;
common_argv.push_back(argv[0]);
for (int i = 1; i < argc; i++) {
if (argv[i][0] == '-') {
common_argv.push_back(argv[i]); // an option: let common_params_parse handle it
} else if (fname.empty()) {
fname = argv[i];
}
}
common_argv.push_back(nullptr);
if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
const std::string fname = argv[1];
if (fname.empty()) {
LOG_ERR("Usage: %s <vocab-file>\n", argv[0]);
common_log_flush(common_log_main());
return 1;
}
fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str());
LOG_INF("%s : reading vocab from: '%s'\n", __func__, fname.c_str());
llama_model * model;
llama_context * ctx;
@@ -38,7 +59,8 @@ int main(int argc, char ** argv) {
model = llama_model_load_from_file(fname.c_str(), mparams);
if (model == NULL) {
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
LOG_ERR("%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
common_log_flush(common_log_main());
return 1;
}
@@ -47,8 +69,9 @@ int main(int argc, char ** argv) {
ctx = llama_init_from_model(model, cparams);
if (ctx == NULL) {
fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
LOG_ERR("%s: error: failed to load vocab '%s'\n", __func__, fname.c_str());
llama_model_free(model);
common_log_flush(common_log_main());
return 1;
}
}
@@ -57,9 +80,13 @@ int main(int argc, char ** argv) {
//GGML_ASSERT(llama_vocab_type(model) == LLAMA_VOCAB_TYPE_SPM);
if (llama_vocab_type(vocab) != LLAMA_VOCAB_TYPE_SPM) {
// a vocab of another type is a skip (99), so it does not get a running line
common_log_flush(common_log_main());
return 99;
}
LOG("%s: running\n", "test-tokenizer-1-spm");
#ifdef _WIN32
// We need this for unicode console support
console::init(false, false);
@@ -68,19 +95,25 @@ int main(int argc, char ** argv) {
const int n_vocab = llama_vocab_n_tokens(vocab);
LOG_INF(" running vocab detokenize round-trip (%d tokens)\n", n_vocab);
for (int i = 0; i < n_vocab; ++i) {
std::string str = common_detokenize(ctx, std::vector<int>(1, i), true);
std::vector<llama_token> tokens = common_tokenize(ctx, str, false, true);
std::string check = common_detokenize(ctx, tokens);
if (check != str) {
fprintf(stderr, "%s : error: token %d detokenizes to '%s'(%zu) but tokenization of this detokenizes to '%s'(%zu)\n",
LOG_ERR("%s : error: token %d detokenizes to '%s'(%zu) but tokenization of this detokenizes to '%s'(%zu)\n",
__func__, i, str.c_str(), str.length(), check.c_str(), check.length());
LOG("%s: %s\n", "test-tokenizer-1-spm", "FAILED");
common_log_flush(common_log_main());
return 2;
}
}
// unicode
{
LOG_INF(" running unicode codepoint round-trip\n");
const int nthread = std::thread::hardware_concurrency();
std::vector<std::thread> threads(nthread);
@@ -99,7 +132,7 @@ int main(int argc, char ** argv) {
std::vector<llama_token> tokens = common_tokenize(ctx, str, false, true);
std::string check = common_detokenize(ctx, tokens);
if (cp != 9601 && str != check) {
fprintf(stderr, "error: codepoint 0x%x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n",
LOG_ERR("error: codepoint 0x%x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n",
cp, check.c_str(), check.length(), str.c_str(), str.length());
errcode = 3;
}
@@ -112,6 +145,8 @@ int main(int argc, char ** argv) {
}
if(errcode) {
LOG("%s: %s\n", "test-tokenizer-1-spm", "FAILED");
common_log_flush(common_log_main());
return errcode;
}
}
@@ -121,5 +156,7 @@ int main(int argc, char ** argv) {
llama_backend_free();
LOG("%s: %s\n", "test-tokenizer-1-spm", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/test-unicode.cpp
+21 -4
View File
@@ -1,10 +1,23 @@
#include "../src/unicode.h"
#include "arg.h"
#include "common.h"
#include "log.h"
#include <cstdio>
#include <string>
#include <vector>
int main() {
int main(int argc, char ** argv) {
common_params params;
params.model.path = "."; // this test takes no model
common_init();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) {
return 1;
}
LOG("%s: running\n", "test-unicode");
const std::vector<std::string> regex_exprs = {
"[~][A-Za-z]+| ?[\\p{S}]+|\\s+",
};
@@ -12,13 +25,17 @@ int main() {
const auto actual = unicode_regex_split(" ~foo", regex_exprs, false);
if (actual != expected) {
fprintf(stderr, "unexpected split:");
LOG_ERR("unexpected split:");
for (const auto & piece : actual) {
fprintf(stderr, " [%s]", piece.c_str());
LOG_ERR(" [%s]", piece.c_str());
}
fprintf(stderr, "\n");
LOG_ERR("\n");
LOG("%s: %s\n", "test-unicode", "FAILED");
common_log_flush(common_log_main());
return 1;
}
LOG("%s: %s\n", "test-unicode", "PASSED");
common_log_flush(common_log_main());
return 0;
}
tests/testing.h
+22 -2
View File
@@ -2,6 +2,10 @@
#include "common.h"
// "log.h" cannot be included here: clip-impl.h (pulled in by test-mtmd-impl.cpp) defines the
// LOG_* macros unconditionally, which conflicts with log.h and is an error with -Werror
int common_log_get_verbosity_thold(void);
#include <chrono>
#include <exception>
#include <iostream>
@@ -28,9 +32,16 @@ struct testing {
std::string skip_reason;
static constexpr std::size_t status_column = 80;
// LOG_LEVEL_WARN, see common/log.h
static constexpr int log_level_warn = 2;
explicit testing(std::ostream &os = std::cout) : out(os) {}
// hide the per-test detail when only warnings and errors are requested
bool show_details() const {
return common_log_get_verbosity_thold() > log_level_warn;
}
std::string indent() const {
if (stack.empty()) {
return "";
@@ -89,6 +100,10 @@ struct testing {
}
void print_result(const std::string &label, int new_failures, int new_assertions, const std::string &extra = "", bool was_skipped = false) const {
if (new_failures == 0 && !show_details()) {
return; // hide [PASS]/[SKIP], always report failures
}
std::string line = indent() + label;
std::string details;
@@ -131,7 +146,9 @@ struct testing {
}
++tests;
out << indent() << name << "\n";
if (show_details()) {
out << indent() << name << "\n";
}
int before_failures = failures;
int before_assertions = assertions;
@@ -174,7 +191,9 @@ struct testing {
}
++tests;
out << indent() << "[bench] " << name << "\n";
if (show_details()) {
out << indent() << "[bench] " << name << "\n";
}
int before_failures = failures;
int before_assertions = assertions;
@@ -263,6 +282,7 @@ struct testing {
out << "failures : " << failures << "\n";
out << "exceptions : " << exceptions << "\n";
out << "skipped : " << skipped << "\n";
out << (failures == 0 ? "PASSED" : "FAILED") << "\n";
return failures == 0 ? 0 : 1;
}
};
tools/rpc/CMakeLists.txt
+1 -1
View File
@@ -5,7 +5,7 @@ target_compile_features(${TARGET} PRIVATE cxx_std_17)
if (LLAMA_BUILD_TESTS AND UNIX AND NOT GGML_BACKEND_DL)
add_executable(test-rpc-multi-server ${PROJECT_SOURCE_DIR}/tests/test-rpc-multi-server.cpp)
target_link_libraries(test-rpc-multi-server PRIVATE ggml ggml-rpc)
target_link_libraries(test-rpc-multi-server PRIVATE ggml ggml-rpc llama-common)
target_include_directories(test-rpc-multi-server PRIVATE ${PROJECT_SOURCE_DIR}/ggml/src)
add_test(
NAME test-rpc-multi-server
tools/server/tests/conftest.py
+16 -1
View File
@@ -2,6 +2,21 @@ import os
import pytest
from filelock import FileLock
from utils import *
import utils
def pytest_addoption(parser):
parser.addoption(
"--errors-only",
action="store_true",
default=False,
help="Run llama-server with WARN log verbosity, hiding INFO/TRACE/DEBUG logs",
)
@pytest.fixture(scope="session", autouse=True)
def configure_errors_only(request):
utils.errors_only = request.config.getoption("--errors-only")
@pytest.fixture(scope="session", autouse=True)
@@ -26,7 +41,7 @@ def stop_server_after_each_test():
@pytest.fixture(scope="session", autouse=True)
def load_server_presets(configure_worker_port, tmp_path_factory):
def load_server_presets(configure_worker_port, configure_errors_only, tmp_path_factory):
# this will be run once per test session, before any tests
# serialize model downloads across parallel workers.
tools/server/tests/tests.sh
+3 -3
View File
@@ -8,12 +8,12 @@ set -eu
WORKERS="${PYTEST_WORKERS:-auto}"
if [ $# -lt 1 ]
if [ $# -lt 1 ] || [ "$1" = "--errors-only" ]
then
if [[ "${SLOW_TESTS:-0}" == 1 ]]; then
pytest --durations=30 -v -x -n "${WORKERS}" --dist=worksteal
pytest --durations=30 -v -x -n "${WORKERS}" --dist=worksteal "$@"
else
pytest --durations=30 -v -x -n "${WORKERS}" --dist=worksteal -m "not slow"
pytest --durations=30 -v -x -n "${WORKERS}" --dist=worksteal -m "not slow" "$@"
fi
else
pytest --durations=30 -n "${WORKERS}" --dist=worksteal "$@"
tools/server/tests/utils.py
+5
View File
@@ -34,6 +34,9 @@ DEFAULT_HTTP_TIMEOUT = 60
# per-request timeout, a hung server fails the test instead of stalling the CI for hours
DEFAULT_REQUEST_TIMEOUT = 600
# set by conftest from --errors-only, makes servers run with WARN log verbosity
errors_only: bool = False
class ServerResponse:
headers: dict
@@ -295,6 +298,8 @@ class ServerProcess:
if self.gcp_compat:
env["AIP_MODE"] = "PREDICTION"
env["AIP_HTTP_PORT"] = str(self.server_port)
if errors_only and not self.debug:
server_args.extend(["--log-verbosity", "2"])
args = [str(arg) for arg in [server_path, *server_args]]
print(f"tests: starting server with: {' '.join(args)}")