From 10532516c32789bd2524db8cd847c747f7b12dbe Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Thu, 17 Sep 2026 18:47:59 +0200 Subject: [PATCH] CI/tests: refactor argument handling, add `--errors-only` to emit only WARN/ERR messages --- .github/workflows/fusion.yml | 4 +- ci/run.sh | 30 +-- common/arg.cpp | 8 + common/common.cpp | 4 +- .../snapdragon/qdc/tests/linux/run_linux.sh | 2 +- .../qdc/tests/run_backend_ops_posix.py | 2 +- src/llama-vocab.cpp | 2 +- tests/CMakeLists.txt | 4 +- tests/test-alloc.cpp | 29 +- tests/test-arg-parser.cpp | 55 ++-- tests/test-autorelease.cpp | 34 ++- tests/test-backend-ops.cpp | 133 +++++---- tests/test-backend-sampler.cpp | 255 +++++++++++------- tests/test-barrier.cpp | 84 ++++-- tests/test-batch-alloc.cpp | 33 ++- tests/test-c.c | 6 +- tests/test-chat-analysis.cpp | 136 ++++++---- tests/test-chat-auto-parser.cpp | 163 +++++++---- tests/test-chat-peg-parser.cpp | 34 ++- tests/test-chat-template.cpp | 41 ++- tests/test-chat.cpp | 98 ++++--- tests/test-col2im-1d.cpp | 25 +- tests/test-export-graph-ops.cpp | 22 ++ tests/test-fusion.cpp | 40 ++- tests/test-gbnf-validator.cpp | 48 +++- tests/test-gguf-model-data.cpp | 105 +++++--- tests/test-gguf.cpp | 202 +++++++++----- tests/test-grammar-integration.cpp | 86 ++++-- tests/test-grammar-llguidance.cpp | 71 +++-- tests/test-grammar-parser.cpp | 77 +++--- tests/test-jinja.cpp | 22 +- tests/test-json-schema-to-grammar.cpp | 39 ++- tests/test-json-schema.cpp | 35 ++- tests/test-llama-archs.cpp | 84 ++++-- tests/test-llama-grammar.cpp | 26 +- tests/test-log.cpp | 22 +- tests/test-model-load-cancel.cpp | 43 ++- tests/test-model-resolution.cpp | 16 +- tests/test-mtmd-c-api.c | 6 + tests/test-mtmd-impl.cpp | 25 +- tests/test-opt.cpp | 92 +++++-- tests/test-peg-parser.cpp | 36 ++- tests/test-quant-type-selection.cpp | 85 ++++-- tests/test-quantize-fns.cpp | 41 ++- tests/test-quantize-perf.cpp | 75 ++++-- tests/test-quantize-stats.cpp | 113 +++++--- tests/test-reasoning-budget.cpp | 41 ++- tests/test-recurrent-state-rollback.cpp | 71 ++--- tests/test-rope.cpp | 34 ++- tests/test-rpc-multi-server.cpp | 40 ++- tests/test-rset-release.cpp | 34 ++- tests/test-sampling.cpp | 58 +++- tests/test-save-load-state.cpp | 54 ++-- tests/test-state-restore-fragmented.cpp | 39 ++- tests/test-thread-safety.cpp | 18 +- tests/test-tokenizer-0.cpp | 108 +++++--- tests/test-tokenizer-1-bpe.cpp | 78 ++++-- tests/test-tokenizer-1-spm.cpp | 53 +++- tests/test-unicode.cpp | 25 +- tests/testing.h | 24 +- tools/rpc/CMakeLists.txt | 2 +- tools/server/tests/conftest.py | 17 +- tools/server/tests/tests.sh | 6 +- tools/server/tests/utils.py | 5 + 64 files changed, 2352 insertions(+), 948 deletions(-) diff --git a/.github/workflows/fusion.yml b/.github/workflows/fusion.yml index 7c8596467a..cebfedf4dc 100644 --- a/.github/workflows/fusion.yml +++ b/.github/workflows/fusion.yml @@ -63,9 +63,9 @@ jobs: id: generate_models run: | rm -rf build-ci-models && mkdir -p build-ci-models - ./build/bin/test-llama-archs -o build-ci-models + ./build/bin/test-llama-archs -o build-ci-models --errors-only - name: Test fusion id: test_fusion run: | - ./build/bin/test-fusion --models build-ci-models --device MTL0 --check tests/fusion/MTL.csv + ./build/bin/test-fusion --models build-ci-models --device MTL0 --check tests/fusion/MTL.csv --errors-only diff --git a/ci/run.sh b/ci/run.sh index 0595fac5a1..df110f335c 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -278,17 +278,17 @@ function gg_run_test_llama_archs_tensor_split { set -e if [ ! -z ${GG_BUILD_CUDA} ]; then - GGML_CUDA_DEVICES=1 ./build-ci-release/bin/test-llama-archs -s 1 2>&1 - GGML_CUDA_DEVICES=2 ./build-ci-release/bin/test-llama-archs -s 1 2>&1 - GGML_CUDA_DEVICES=3 ./build-ci-release/bin/test-llama-archs -s 1 2>&1 - GGML_CUDA_DEVICES=4 ./build-ci-release/bin/test-llama-archs -s 1 2>&1 + GGML_CUDA_DEVICES=1 ./build-ci-release/bin/test-llama-archs -s 1 --errors-only 2>&1 + GGML_CUDA_DEVICES=2 ./build-ci-release/bin/test-llama-archs -s 1 --errors-only 2>&1 + GGML_CUDA_DEVICES=3 ./build-ci-release/bin/test-llama-archs -s 1 --errors-only 2>&1 + GGML_CUDA_DEVICES=4 ./build-ci-release/bin/test-llama-archs -s 1 --errors-only 2>&1 fi if [ ! -z ${GG_BUILD_METAL} ]; then - GGML_METAL_DEVICES=1 ./build-ci-release/bin/test-llama-archs -s 1 2>&1 - GGML_METAL_DEVICES=2 ./build-ci-release/bin/test-llama-archs -s 1 2>&1 - GGML_METAL_DEVICES=3 ./build-ci-release/bin/test-llama-archs -s 1 2>&1 - GGML_METAL_DEVICES=4 ./build-ci-release/bin/test-llama-archs -s 1 2>&1 + GGML_METAL_DEVICES=1 ./build-ci-release/bin/test-llama-archs -s 1 --errors-only 2>&1 + GGML_METAL_DEVICES=2 ./build-ci-release/bin/test-llama-archs -s 1 --errors-only 2>&1 + GGML_METAL_DEVICES=3 ./build-ci-release/bin/test-llama-archs -s 1 --errors-only 2>&1 + GGML_METAL_DEVICES=4 ./build-ci-release/bin/test-llama-archs -s 1 --errors-only 2>&1 fi set +e @@ -307,7 +307,7 @@ function gg_run_test_llama_archs_models { rm -rf build-ci-models && mkdir -p build-ci-models # generate the dummy models used by the model-dependent tests - ./build-ci-release/bin/test-llama-archs -o build-ci-models 2>&1 + ./build-ci-release/bin/test-llama-archs -o build-ci-models --errors-only 2>&1 fi set +e @@ -455,10 +455,10 @@ function gg_run_qwen3_0_6b { (time ./bin/llama-imatrix --model ${model_f16} -f ${wiki_test} -ngl 99 -c 1024 -b 512 --chunks 2 ) 2>&1 | tee -a $OUT/${ci}-imatrix.log - (time ./bin/test-save-load-state --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload) 2>&1 | tee -a $OUT/${ci}-save-load-state.log - (time ./bin/test-save-load-state --model ${model_q4_0} -ngl 10 -c 1024 -fa on --no-op-offload) 2>&1 | tee -a $OUT/${ci}-save-load-state.log - (time ./bin/test-save-load-state --model ${model_q4_0} -ngl 99 -c 1024 -fa off ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log - (time ./bin/test-save-load-state --model ${model_q4_0} -ngl 99 -c 1024 -fa on ) 2>&1 | tee -a $OUT/${ci}-save-load-state.log + (time ./bin/test-save-load-state --model ${model_q4_0} -ngl 10 -c 1024 -fa off --no-op-offload --errors-only) 2>&1 | tee -a $OUT/${ci}-save-load-state.log + (time ./bin/test-save-load-state --model ${model_q4_0} -ngl 10 -c 1024 -fa on --no-op-offload --errors-only) 2>&1 | tee -a $OUT/${ci}-save-load-state.log + (time ./bin/test-save-load-state --model ${model_q4_0} -ngl 99 -c 1024 -fa off --errors-only) 2>&1 | tee -a $OUT/${ci}-save-load-state.log + (time ./bin/test-save-load-state --model ${model_q4_0} -ngl 99 -c 1024 -fa on --errors-only) 2>&1 | tee -a $OUT/${ci}-save-load-state.log function check_ppl { qnt="$1" @@ -676,9 +676,9 @@ function gg_run_test_backend_ops { # TODO: reduce the test-backend-ops timeout to 1800s if [ ! -z ${GG_BUILD_HIGH_PERF} ]; then - (time timeout 3600 ./bin/test-backend-ops ${args_extra} -b CPU) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log + (time timeout 3600 ./bin/test-backend-ops ${args_extra} -b CPU --errors-only) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log else - (time timeout 3600 ./bin/test-backend-ops ${args_extra} ) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log + (time timeout 3600 ./bin/test-backend-ops ${args_extra} --errors-only) 2>&1 | tee -a $OUT/${ci}-test-backend-ops.log fi set +e diff --git a/common/arg.cpp b/common/arg.cpp index c4c4e143c9..b5ca9545ea 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -3937,6 +3937,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex common_log_set_verbosity_thold(value); } ).set_env("LLAMA_ARG_LOG_VERBOSITY")); + add_opt(common_arg( + {"--errors-only"}, + "Set verbosity threshold to WARN (only print warnings and errors)", + [](common_params & params) { + params.verbosity = LOG_LEVEL_WARN; + common_log_set_verbosity_thold(LOG_LEVEL_WARN); + } + )); add_opt(common_arg( {"--log-prefix"}, {"--no-log-prefix"}, diff --git a/common/common.cpp b/common/common.cpp index d8319cd9ac..d7860fd32d 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1573,7 +1573,9 @@ char * common_get_model_or_exit(int argc, char * argv[]) { char * path = getenv("LLAMACPP_TEST_MODELFILE"); if (!path || strlen(path) == 0) { - fprintf(stderr, "\033[33mWARNING: No model file provided. Skipping this test. Set LLAMACPP_TEST_MODELFILE= to silence this warning and run this test.\n\033[0m"); + const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN; + fprintf(stderr, "%sWARNING: No model file provided. Skipping this test. Set LLAMACPP_TEST_MODELFILE= to silence this warning and run this test.\n%s", + use_color ? LOG_COL_YELLOW : "", use_color ? LOG_COL_DEFAULT : ""); exit(EXIT_SUCCESS); } diff --git a/scripts/snapdragon/qdc/tests/linux/run_linux.sh b/scripts/snapdragon/qdc/tests/linux/run_linux.sh index 11083f5213..5c06c8cb9f 100644 --- a/scripts/snapdragon/qdc/tests/linux/run_linux.sh +++ b/scripts/snapdragon/qdc/tests/linux/run_linux.sh @@ -197,7 +197,7 @@ run_backend_ops_case() { esac echo "=== [backend-ops:$dtype] test-backend-ops -b HTP0 -o MUL_MAT ===" timeout 600 env GGML_HEXAGON_NDEV=1 GGML_HEXAGON_HOSTBUF=0 ./bin/test-backend-ops \ - -b HTP0 -o MUL_MAT -p "$pattern" \ + -b HTP0 -o MUL_MAT -p "$pattern" --errors-only \ > "$log" 2>&1 local rc=$? note_timeout_if_triggered "$rc" 600 "$log" diff --git a/scripts/snapdragon/qdc/tests/run_backend_ops_posix.py b/scripts/snapdragon/qdc/tests/run_backend_ops_posix.py index f2f870f131..20f7336a05 100644 --- a/scripts/snapdragon/qdc/tests/run_backend_ops_posix.py +++ b/scripts/snapdragon/qdc/tests/run_backend_ops_posix.py @@ -32,7 +32,7 @@ def test_backend_ops_htp0(type_a): pattern = f"type_a={type_a}" result = run_snapdragon( - ["test-backend-ops", "-b", "HTP0", "-o", "MUL_MAT", "-p", pattern], + ["test-backend-ops", "-b", "HTP0", "-o", "MUL_MAT", "-p", pattern, "--errors-only"], ) write_qdc_log(f"backend_ops_{type_a}.log", result.stdout or "") assert result.returncode == 0, ( diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index ee65faf23e..7dab882bdf 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -1955,7 +1955,7 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { // read vocab size from metadata uint32_t n_tokens = 0; if (ml.get_key(LLM_KV_VOCAB_SIZE, n_tokens, false)) { - LLAMA_LOG_WARN("%s: adding %u dummy tokens\n", __func__, n_tokens); + LLAMA_LOG_INFO("%s: adding %u dummy tokens\n", __func__, n_tokens); id_to_token.resize(n_tokens); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a398344c89..b661f94450 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -40,7 +40,7 @@ function(llama_test target) add_test( NAME ${TEST_NAME} WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY} - COMMAND $ + COMMAND $ --errors-only ${LLAMA_TEST_ARGS}) set_property(TEST ${TEST_NAME} PROPERTY LABELS ${LLAMA_TEST_LABEL}) @@ -110,7 +110,7 @@ function(llama_build_and_test source) add_test( NAME ${TEST_TARGET} WORKING_DIRECTORY ${LLAMA_TEST_WORKING_DIRECTORY} - COMMAND $ + COMMAND $ --errors-only ${LLAMA_TEST_ARGS}) set_property(TEST ${TEST_TARGET} PROPERTY LABELS ${LLAMA_TEST_LABEL}) diff --git a/tests/test-alloc.cpp b/tests/test-alloc.cpp index 8f1a98aa03..27e1c118f9 100644 --- a/tests/test-alloc.cpp +++ b/tests/test-alloc.cpp @@ -4,6 +4,10 @@ #include "../ggml/src/ggml-impl.h" #include "ggml.h" +#include "arg.h" +#include "common.h" +#include "log.h" + #include #include #include @@ -189,7 +193,7 @@ static int get_leaf_id(ggml_cgraph * graph, const char * tensor_name) { return i; } } - fprintf(stderr, "leaf not found: %s\n", tensor_name); + LOG_ERR("leaf not found: %s\n", tensor_name); return -1; } @@ -199,7 +203,7 @@ static int get_node_id(ggml_cgraph * graph, const char * tensor_name) { return i; } } - fprintf(stderr, "node not found: %s", tensor_name); + LOG_ERR("node not found: %s", tensor_name); return -1; } @@ -651,13 +655,23 @@ static void test_graph_optimize_alloc_dep() { } static void run(const char * name, void (*f)()) { - printf("%s ", name); - fflush(stdout); + LOG_INF(" running %s\n", name); + // drain the queue, so the name is on disk if f() aborts + common_log_flush(common_log_main()); f(); - printf("PASSED\n"); + LOG_CNT("PASSED\n"); } -int main() { +int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-alloc"); + run("test_max_size_too_many_tensors", test_max_size_too_many_tensors); run("test_max_size_tensor_too_large", test_max_size_tensor_too_large); run("test_tensor_larger_than_max_size", test_tensor_larger_than_max_size); @@ -672,5 +686,8 @@ int main() { run("test_buffer_size_zero", test_buffer_size_zero); run("test_reallocation", test_reallocation); run("test_graph_optimize_alloc_dep", test_graph_optimize_alloc_dep); + + LOG("%s: %s\n", "test-alloc", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-arg-parser.cpp b/tests/test-arg-parser.cpp index e0907631ab..4488a48fbf 100644 --- a/tests/test-arg-parser.cpp +++ b/tests/test-arg-parser.cpp @@ -2,6 +2,7 @@ #include "common.h" #include "download.h" #include "llama.h" +#include "log.h" #include "speculative.h" #include @@ -101,7 +102,7 @@ static void test(void) { assert(draft.n_outputs_max_per_seq == 1); } - printf("test-arg-parser: make sure there is no duplicated arguments in any examples\n\n"); + LOG_INF("test-arg-parser: make sure there is no duplicated arguments in any examples\n\n"); for (int ex = 0; ex < LLAMA_EXAMPLE_COUNT; ex++) { try { auto ctx_arg = common_params_parser_init(params, (enum llama_example)ex); @@ -114,7 +115,7 @@ static void test(void) { if (seen_args.find(arg) == seen_args.end()) { seen_args.insert(arg); } else { - fprintf(stderr, "test-arg-parser: found different handlers for the same argument: %s", arg.c_str()); + LOG_ERR("test-arg-parser: found different handlers for the same argument: %s", arg.c_str()); exit(1); } } @@ -123,7 +124,7 @@ static void test(void) { if (seen_env_vars.find(env) == seen_env_vars.end()) { seen_env_vars.insert(env); } else { - fprintf(stderr, "test-arg-parser: found different handlers for the same env var: %s", env.c_str()); + LOG_ERR("test-arg-parser: found different handlers for the same env var: %s", env.c_str()); exit(1); } } @@ -138,7 +139,7 @@ static void test(void) { const std::string last(opt.args.back()); if (first.length() > last.length()) { - fprintf(stderr, "test-arg-parser: shorter argument should come before longer one: %s, %s\n", + LOG_ERR("test-arg-parser: shorter argument should come before longer one: %s, %s\n", first.c_str(), last.c_str()); assert(false); } @@ -150,14 +151,15 @@ static void test(void) { const std::string last(opt.args_neg.back()); if (first.length() > last.length()) { - fprintf(stderr, "test-arg-parser: shorter negated argument should come before longer one: %s, %s\n", + LOG_ERR("test-arg-parser: shorter negated argument should come before longer one: %s, %s\n", first.c_str(), last.c_str()); assert(false); } } } } catch (std::exception & e) { - printf("%s\n", e.what()); + LOG_ERR("%s\n", e.what()); + common_log_flush(common_log_main()); assert(false); } } @@ -172,7 +174,7 @@ static void test(void) { std::vector argv; - printf("test-arg-parser: test invalid usage\n\n"); + LOG_INF("test-arg-parser: test invalid usage\n\n"); // missing value argv = {"binary_name", "-m"}; @@ -229,7 +231,7 @@ static void test(void) { argv = {"binary_name", "-lm", "hello"}; assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON)); - printf("test-arg-parser: test valid usage\n\n"); + LOG_INF("test-arg-parser: test valid usage\n\n"); argv = {"binary_name", "-m", "model_file.gguf"}; assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON)); @@ -305,9 +307,9 @@ static void test(void) { // skip this part on windows, because setenv is not supported #ifdef _WIN32 - printf("test-arg-parser: skip on windows build\n"); + LOG_INF("test-arg-parser: skip on windows build\n"); #else - printf("test-arg-parser: test environment variables (valid + invalid usages)\n\n"); + LOG_INF("test-arg-parser: test environment variables (valid + invalid usages)\n\n"); setenv("LLAMA_ARG_THREADS", "blah", true); argv = {"binary_name"}; @@ -344,7 +346,7 @@ static void test(void) { assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON)); assert(params.load_mode == LLAMA_LOAD_MODE_DIRECT_IO); - printf("test-arg-parser: test negated environment variables\n\n"); + LOG_INF("test-arg-parser: test negated environment variables\n\n"); setenv("LLAMA_ARG_LOAD_MODE", "none", true); setenv("LLAMA_ARG_NO_PERF", "1", true); // legacy format @@ -353,7 +355,7 @@ static void test(void) { assert(params.load_mode == LLAMA_LOAD_MODE_NONE); assert(params.no_perf == true); - printf("test-arg-parser: test environment variables being overwritten\n\n"); + LOG_INF("test-arg-parser: test environment variables being overwritten\n\n"); setenv("LLAMA_ARG_MODEL", "blah.gguf", true); setenv("LLAMA_ARG_THREADS", "1010", true); @@ -363,12 +365,12 @@ static void test(void) { assert(params.cpuparams.n_threads == 1010); #endif // _WIN32 - printf("test-arg-parser: test download functions\n\n"); + LOG_INF("test-arg-parser: test download functions\n\n"); const char * GOOD_URL = "http://ggml.ai/"; const char * BAD_URL = "http://ggml.ai/404"; { - printf("test-arg-parser: test good URL\n\n"); + LOG_INF("test-arg-parser: test good URL\n\n"); auto res = common_remote_get_content(GOOD_URL, {}); assert(res.first == 200); assert(res.second.size() > 0); @@ -377,32 +379,45 @@ static void test(void) { } { - printf("test-arg-parser: test bad URL\n\n"); + LOG_INF("test-arg-parser: test bad URL\n\n"); auto res = common_remote_get_content(BAD_URL, {}); assert(res.first == 404); } { - printf("test-arg-parser: test max size error\n"); + LOG_INF("test-arg-parser: test max size error\n"); common_remote_params params; params.max_size = 1; try { common_remote_get_content(GOOD_URL, params); assert(false && "it should throw an error"); } catch (std::exception & e) { - printf(" expected error: %s\n\n", e.what()); + LOG_INF(" expected error: %s\n\n", e.what()); } } - printf("test-arg-parser: all tests OK\n\n"); + LOG_INF("test-arg-parser: all tests OK\n\n"); } -int main(void) { +int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-arg-parser"); + try { test(); } catch (std::exception & e) { - fprintf(stderr, "test-arg-parser: exception: %s\n", e.what()); + LOG_ERR("test-arg-parser: exception: %s\n", e.what()); + LOG("%s: %s\n", "test-arg-parser", "FAILED"); + common_log_flush(common_log_main()); return 1; } + LOG("%s: %s\n", "test-arg-parser", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-autorelease.cpp b/tests/test-autorelease.cpp index 370428809f..742b438233 100644 --- a/tests/test-autorelease.cpp +++ b/tests/test-autorelease.cpp @@ -1,13 +1,43 @@ // ref: https://github.com/ggml-org/llama.cpp/issues/4952#issuecomment-1892864763 #include +#include #include "llama.h" #include "common.h" +#include "arg.h" +#include "log.h" // This creates a new context inside a pthread and then tries to exit cleanly. int main(int argc, char ** argv) { - auto * model_path = common_get_model_or_exit(argc, argv); + // the model path is this test's only positional argument + char * model_argv[2] = { argv[0], nullptr }; + + { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + + std::vector common_argv; + common_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { + if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else if (model_argv[1] == nullptr) { + model_argv[1] = argv[i]; + } + } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + } + + // falls back to LLAMACPP_TEST_MODELFILE, or warns and exits if no model is given + auto * model_path = common_get_model_or_exit(model_argv[1] == nullptr ? 1 : 2, model_argv); + + // that call exits when no model is given, so the verdict below is only reached with a model + LOG("%s: running\n", "test-autorelease"); std::thread([&model_path]() { llama_backend_init(); @@ -18,5 +48,7 @@ int main(int argc, char ** argv) { llama_backend_free(); }).join(); + LOG("%s: %s\n", "test-autorelease", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index dc529a352b..2307c54d5f 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -20,6 +20,10 @@ #include "ggml-backend.h" #include "ggml-cpp.h" +#include "arg.h" +#include "common.h" +#include "log.h" + #include #include #include @@ -833,6 +837,14 @@ struct printer { }; struct console_printer : public printer { + // the printer is created after the CLI has been parsed, so the threshold is final here: + // no ANSI escapes at WARN or lower (--errors-only) + const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN; + const char * const col_ok = use_color ? "\033[1;32m" : ""; + const char * const col_fail = use_color ? "\033[1;31m" : ""; + const char * const col_blue = use_color ? "\033[1;34m" : ""; + const char * const col_end = use_color ? "\033[0m" : ""; + void print_test_result(const test_result & result) override { if (result.test_mode == "test") { print_test_console(result); @@ -844,21 +856,21 @@ struct console_printer : public printer { } void print_operation(const test_operation_info & info) override { - printf(" %s(%s): ", info.op_name.c_str(), info.op_params.c_str()); + LOG_CNT(" %s(%s): ", info.op_name.c_str(), info.op_params.c_str()); fflush(stdout); // Handle large tensor skip first if (info.is_large_tensor_skip) { - printf("skipping large tensors for speed \n"); + LOG_CNT("skipping large tensors for speed \n"); return; } // Handle not supported status if (info.status == test_status_t::NOT_SUPPORTED) { if (!info.failure_reason.empty()) { - printf("not supported [%s]\n", info.failure_reason.c_str()); + LOG_CNT("not supported [%s]\n", info.failure_reason.c_str()); } else { - printf("not supported [%s]\n", info.backend_name.c_str()); + LOG_CNT("not supported [%s]\n", info.backend_name.c_str()); } return; } @@ -866,52 +878,52 @@ struct console_printer : public printer { // Handle errors and additional information if (info.has_error) { if (info.error_component == "allocation") { - fprintf(stderr, "failed to allocate tensors [%s] ", info.backend_name.c_str()); + LOG_ERR("failed to allocate tensors [%s] ", info.backend_name.c_str()); } else if (info.error_component == "backend") { - fprintf(stderr, " Failed to initialize %s backend\n", info.backend_name.c_str()); + LOG_ERR(" Failed to initialize %s backend\n", info.backend_name.c_str()); } else { - fprintf(stderr, "Error in %s: %s\n", info.error_component.c_str(), info.error_details.c_str()); + LOG_ERR("Error in %s: %s\n", info.error_component.c_str(), info.error_details.c_str()); } } // Handle gradient info if (info.has_gradient_info) { - printf("[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", info.op_name.c_str(), info.gradient_index, - info.gradient_param_name.c_str(), info.gradient_value); + LOG_CNT("[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", info.op_name.c_str(), info.gradient_index, + info.gradient_param_name.c_str(), info.gradient_value); } // Handle MAA error if (info.has_maa_error) { - printf("[%s] MAA = %.9f > %.9f ", info.op_name.c_str(), info.maa_error, info.maa_threshold); + LOG_CNT("[%s] MAA = %.9f > %.9f ", info.op_name.c_str(), info.maa_error, info.maa_threshold); } // Handle compare failure if (info.is_compare_failure) { - printf("compare failed "); + LOG_CNT("compare failed "); } // Print final status if (info.status == test_status_t::OK) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } } void print_summary(const test_summary_info & info) override { if (info.is_backend_summary) { - printf("%zu/%zu backends passed\n", info.tests_passed, info.tests_total); + LOG_CNT("%zu/%zu backends passed\n", info.tests_passed, info.tests_total); } else { - printf(" %zu/%zu tests passed\n", info.tests_passed, info.tests_total); + LOG_CNT(" %zu/%zu tests passed\n", info.tests_passed, info.tests_total); } } void print_backend_status(const backend_status_info & info) override { - printf(" Backend %s: ", info.backend_name.c_str()); + LOG_CNT(" Backend %s: ", info.backend_name.c_str()); if (info.status == test_status_t::OK) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } } @@ -920,30 +932,30 @@ struct console_printer : public printer { } void print_backend_init(const backend_init_info & info) override { - printf("Backend %zu/%zu: %s\n", info.device_index + 1, info.total_devices, info.device_name.c_str()); + LOG_INF("Backend %zu/%zu: %s\n", info.device_index + 1, info.total_devices, info.device_name.c_str()); if (info.skipped) { - printf(" %s\n", info.skip_reason.c_str()); + LOG_INF(" %s\n", info.skip_reason.c_str()); return; } if (!info.description.empty()) { - printf(" Device description: %s\n", info.description.c_str()); + LOG_INF(" Device description: %s\n", info.description.c_str()); } if (info.has_memory_info) { - printf(" Device memory: %zu MB (%zu MB free)\n", info.memory_total_mb, info.memory_free_mb); + LOG_INF(" Device memory: %zu MB (%zu MB free)\n", info.memory_total_mb, info.memory_free_mb); } - printf("\n"); + LOG_CNT("\n"); } void print_overall_summary(const overall_summary_info & info) override { printf("%zu/%zu backends passed\n", info.backends_passed, info.backends_total); if (info.all_passed) { - printf("\033[1;32mOK\033[0m\n"); + printf("%sOK%s\n", col_ok, col_end); } else { - printf("\033[1;31mFAIL\033[0m\n"); + printf("%sFAIL%s\n", col_fail, col_end); } } @@ -960,28 +972,30 @@ struct console_printer : public printer { private: void print_test_console(const test_result & result) { - printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str()); + LOG_CNT(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str()); fflush(stdout); if (!result.supported) { - printf("not supported [%s] ", result.backend_name.c_str()); - printf("\n"); + LOG_CNT("not supported [%s] ", result.backend_name.c_str()); + LOG_CNT("\n"); return; } if (result.passed) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } } void print_perf_console(const test_result & result) { - int len = printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str()); + const std::string label = " " + result.op_name + "(" + result.op_params + "): "; + int len = (int) label.size(); + LOG_CNT("%s", label.c_str()); fflush(stdout); if (!result.supported) { - printf("not supported\n"); + LOG_CNT("not supported\n"); return; } @@ -991,9 +1005,9 @@ struct console_printer : public printer { if (last - len < 5) { last += align; } - printf("%*s", last - len, ""); + LOG_CNT("%*s", last - len, ""); - printf(" %8d runs - %8.2f us/run - ", result.n_runs, result.time_us); + LOG_CNT(" %8d runs - %8.2f us/run - ", result.n_runs, result.time_us); if (result.flops > 0) { auto format_flops = [](double flops) -> std::string { @@ -1010,22 +1024,22 @@ struct console_printer : public printer { return buf; }; uint64_t op_flops_per_run = result.flops * result.time_us / 1e6; - printf("%s/run - \033[1;34m%sS\033[0m", format_flops(op_flops_per_run).c_str(), - format_flops(result.flops).c_str()); + LOG_CNT("%s/run - %s%sS%s", format_flops(op_flops_per_run).c_str(), col_blue, + format_flops(result.flops).c_str(), col_end); } else { - printf("%8zu kB/run - \033[1;34m%7.2f GB/s\033[0m", result.memory_kb, result.bandwidth_gb_s); + LOG_CNT("%8zu kB/run - %s%7.2f GB/s%s", result.memory_kb, col_blue, result.bandwidth_gb_s, col_end); } - printf("\n"); + LOG_CNT("\n"); } void print_support_console(const test_result & result) { - printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str()); + LOG_CNT(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str()); fflush(stdout); if (result.supported) { - printf("\033[1;32mSUPPORTED\033[0m\n"); + LOG_CNT("%sSUPPORTED%s\n", col_ok, col_end); } else { - printf("\033[1;31mNOT SUPPORTED\033[0m\n"); + LOG_CNT("%sNOT SUPPORTED%s\n", col_fail, col_end); } } }; @@ -1595,7 +1609,7 @@ struct test_case { // warmup run ggml_status status = ggml_backend_graph_compute(backend, gf); if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + LOG_ERR("%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); return false; } @@ -1650,7 +1664,7 @@ struct test_case { int64_t start_time = ggml_time_us(); ggml_status status = ggml_backend_graph_compute(backend, gf); if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + LOG_ERR("%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); return false; } int64_t end_time = ggml_time_us(); @@ -1841,12 +1855,12 @@ struct test_case { ggml_status status = ggml_backend_graph_compute(backend, gf); if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + LOG_ERR("%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); return false; } status = ggml_backend_graph_compute(backend, gb); if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + LOG_ERR("%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); return false; } @@ -1900,7 +1914,7 @@ struct test_case { ggml_backend_tensor_set(t, &xiu, i*sizeof(float), sizeof(float)); status = ggml_backend_graph_compute(backend, gf); if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + LOG_ERR("%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); return false; } ggml_backend_tensor_get(out, &fu, 0, ggml_nbytes(out)); @@ -1908,7 +1922,7 @@ struct test_case { ggml_backend_tensor_set(t, &xid, i*sizeof(float), sizeof(float)); status = ggml_backend_graph_compute(backend, gf); if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + LOG_ERR("%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); return false; } ggml_backend_tensor_get(out, &fd, 0, ggml_nbytes(out)); @@ -1917,7 +1931,7 @@ struct test_case { ggml_backend_tensor_set(t, &xiuh, i*sizeof(float), sizeof(float)); status = ggml_backend_graph_compute(backend, gf); if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + LOG_ERR("%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); return false; } ggml_backend_tensor_get(out, &fuh, 0, ggml_nbytes(out)); @@ -1925,7 +1939,7 @@ struct test_case { ggml_backend_tensor_set(t, &xidh, i*sizeof(float), sizeof(float)); status = ggml_backend_graph_compute(backend, gf); if (status != GGML_STATUS_SUCCESS) { - fprintf(stderr, "%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); + LOG_ERR("%s: ggml_backend_graph_compute failed. status=%s \n", __func__, ggml_status_to_string(status)); return false; } ggml_backend_tensor_get(out, &fdh, 0, ggml_nbytes(out)); @@ -11918,6 +11932,10 @@ static void usage(char ** argv) { } int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + test_mode mode = MODE_TEST; output_formats output_format = CONSOLE; const char * op_names_filter = nullptr; @@ -11926,6 +11944,9 @@ int main(int argc, char ** argv) { const char * test_file_path = nullptr; int parallel_workers = 1; + std::vector common_argv; + common_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "test") == 0) { mode = MODE_TEST; @@ -11990,12 +12011,21 @@ int main(int argc, char ** argv) { usage(argv); return 1; } + } else if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it } else { usage(argv); return 1; } } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-backend-ops"); + // load and enumerate backends ggml_backend_load_all(); @@ -12056,12 +12086,17 @@ int main(int argc, char ** argv) { output_printer->print_footer(); } + common_log_flush(common_log_main()); output_printer->print_overall_summary( overall_summary_info(n_ok, ggml_backend_dev_count(), n_ok == ggml_backend_dev_count())); if (n_ok != ggml_backend_dev_count()) { + LOG("%s: %s\n", "test-backend-ops", "FAILED"); + common_log_flush(common_log_main()); return 1; } + LOG("%s: %s\n", "test-backend-ops", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-backend-sampler.cpp b/tests/test-backend-sampler.cpp index c23e7248d5..48d28ecfc9 100644 --- a/tests/test-backend-sampler.cpp +++ b/tests/test-backend-sampler.cpp @@ -3,6 +3,9 @@ #include "llama-cpp.h" #include "common.h" +#include "arg.h" +#include "log.h" + #ifdef NDEBUG #undef NDEBUG #endif @@ -40,7 +43,7 @@ static llama_model_ptr load_model(const test_args & args) { devs[0] = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU); if (devs[0] == nullptr) { - fprintf(stderr, "Error: GPU requested but not available\n"); + LOG_ERR("Error: GPU requested but not available\n"); return nullptr; } @@ -50,13 +53,13 @@ static llama_model_ptr load_model(const test_args & args) { mparams.n_gpu_layers = 0; } else { - fprintf(stderr, "Error: invalid device '%s'\n", args.device.c_str()); + LOG_ERR("Error: invalid device '%s'\n", args.device.c_str()); return nullptr; } mparams.devices = devs; - fprintf(stderr, "Using device: %s\n", ggml_backend_dev_name(devs[0])); + LOG_INF("Using device: %s\n", ggml_backend_dev_name(devs[0])); } llama_model_ptr res; @@ -64,7 +67,7 @@ static llama_model_ptr load_model(const test_args & args) { res.reset(llama_model_load_from_file(args.model.c_str(), mparams)); if (!res) { - fprintf(stderr, "Warning: failed to load model '%s', skipping test\n", args.model.c_str()); + LOG_WRN("Warning: failed to load model '%s', skipping test\n", args.model.c_str()); return nullptr; } @@ -140,7 +143,7 @@ struct test_context { prompt_tokens.data(), prompt_tokens.size(), false, false); if (n_tokens < 0) { - fprintf(stderr, "Warning: tokenization failed for seq_id %d\n", seq_id); + LOG_WRN("Warning: tokenization failed for seq_id %d\n", seq_id); llama_batch_free(batch); return false; } @@ -162,19 +165,19 @@ struct test_context { } - printf("Batch contents:\n"); - printf("n_tokens: %d\n", batch.n_tokens); + LOG_CNT("Batch contents:\n"); + LOG_CNT("n_tokens: %d\n", batch.n_tokens); for (int i = 0; i < batch.n_tokens; i++) { - printf("token[%d]: tok=%-5d, pos=%d, n_seq_id=%d, seq_ids=[", i, batch.token[i], batch.pos[i], batch.n_seq_id[i]); + LOG_CNT("token[%d]: tok=%-5d, pos=%d, n_seq_id=%d, seq_ids=[", i, batch.token[i], batch.pos[i], batch.n_seq_id[i]); for (int j = 0; j < batch.n_seq_id[i]; j++) { - printf("%d%s", batch.seq_id[i][j], j < batch.n_seq_id[i]-1 ? ", " : ""); + LOG_CNT("%d%s", batch.seq_id[i][j], j < batch.n_seq_id[i]-1 ? ", " : ""); } - printf("], logits=%d\n", batch.logits[i]); + LOG_CNT("], logits=%d\n", batch.logits[i]); } if (llama_decode(ctx.get(), batch) != 0) { - fprintf(stderr, "Warning: llama_decode failed\n"); + LOG_WRN("Warning: llama_decode failed\n"); llama_batch_free(batch); return false; } @@ -194,7 +197,7 @@ struct test_context { int32_t idx_for_seq(llama_seq_id seq_id) { auto it = last_batch_info.find(seq_id); if (it == last_batch_info.end()) { - fprintf(stderr, "Error: no batch index found for seq_id %d\n", seq_id); + LOG_ERR("Error: no batch index found for seq_id %d\n", seq_id); return -1; } return it->second; @@ -218,7 +221,7 @@ struct test_context { common_batch_add(batch, token, pos, { seq_id }, true); if (llama_decode(ctx.get(), batch) != 0) { - fprintf(stderr, "Warning: llama_decode failed for token %d in seq %d\n", token, seq_id); + LOG_WRN("Warning: llama_decode failed for token %d in seq %d\n", token, seq_id); llama_batch_free(batch); return false; } @@ -242,7 +245,7 @@ struct test_context { } if (llama_decode(ctx.get(), batch) != 0) { - fprintf(stderr, "Warning: llama_decode failed for batch tokens\n"); + LOG_WRN("Warning: llama_decode failed for batch tokens\n"); llama_batch_free(batch); return false; } @@ -352,17 +355,17 @@ static void test_backend_greedy_sampling(const test_params & params) { int32_t batch_idx = test_ctx.idx_for_seq(seq_id); llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx); - printf("greedy sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str()); + LOG_CNT("greedy sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str()); GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab); token = llama_get_sampled_token_ith(test_ctx.ctx.get(), -1); - printf("greedy sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str()); + LOG_CNT("greedy sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str()); GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab); for (int i = 0; i < 10; i++) { int32_t loop_idx = test_ctx.idx_for_seq(seq_id); llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), loop_idx); - printf("Generation step %d: token id:%d, string: %s\n", i, token, test_ctx.token_to_piece(token, false).c_str()); + LOG_CNT("Generation step %d: token id:%d, string: %s\n", i, token, test_ctx.token_to_piece(token, false).c_str()); if (!test_ctx.decode_token(token, 0)) { GGML_ASSERT(false && "Failed to decode token"); } @@ -388,14 +391,14 @@ static void test_backend_top_k_sampling(const test_params & params) { float * logits = llama_get_sampled_logits_ith(test_ctx.ctx.get(), batch_idx); uint32_t n_logits = llama_get_sampled_logits_count_ith(test_ctx.ctx.get(), batch_idx); for (size_t i = 0; i < n_logits; ++i) { - printf("top_k logit[%zu] = %.6f\n", i, logits[i]); + LOG_CNT("top_k logit[%zu] = %.6f\n", i, logits[i]); } llama_token * candidates = llama_get_sampled_candidates_ith(test_ctx.ctx.get(), batch_idx); uint32_t n_candidates = llama_get_sampled_candidates_count_ith(test_ctx.ctx.get(), batch_idx); for (size_t i = 0; i < n_candidates; ++i) { - printf("top_k candidate[%zu] = %d : %s\n", i, candidates[i], - test_ctx.token_to_piece(candidates[i], false).c_str()); + LOG_CNT("top_k candidate[%zu] = %d : %s\n", i, candidates[i], + test_ctx.token_to_piece(candidates[i], false).c_str()); } // Sample using CPU sampler for verification that it is possible to do hybrid @@ -408,7 +411,7 @@ static void test_backend_top_k_sampling(const test_params & params) { llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx); GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab); - printf("backend top-k hybrid sampling test PASSED\n"); + LOG_CNT("backend top-k hybrid sampling test PASSED\n"); } static void test_backend_temp_sampling(const test_params & params) { @@ -447,7 +450,7 @@ static void test_backend_temp_sampling(const test_params & params) { llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx); const std::string token_str = test_ctx.token_to_piece(token, false); - printf("Sequence 0 sampled token id:%d, string: '%s'\n", token, token_str.c_str()); + LOG_CNT("Sequence 0 sampled token id:%d, string: '%s'\n", token, token_str.c_str()); GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab); } @@ -463,14 +466,14 @@ static void test_backend_temp_sampling(const test_params & params) { llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx); const std::string token_str = test_ctx.token_to_piece(token, false); - printf("Sequence 1 sampled token id:%d, string: '%s'\n", token, token_str.c_str()); + LOG_CNT("Sequence 1 sampled token id:%d, string: '%s'\n", token, token_str.c_str()); GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab); } } // lambda for testing non-positive temperature values. auto test_argmax_temp = [&](float temp) { - printf("\nTesting temperature = %.1f\n", temp); + LOG_CNT("\nTesting temperature = %.1f\n", temp); int seq_id = 0; struct llama_sampler_chain_params backend_chain_params = llama_sampler_chain_default_params(); @@ -496,7 +499,7 @@ static void test_backend_temp_sampling(const test_params & params) { test_argmax_temp(0.0f); test_argmax_temp(-1.0f); - printf("backend temp sampling test PASSED\n"); + LOG_CNT("backend temp sampling test PASSED\n"); } static void test_backend_temp_ext_sampling(const test_params & params) { @@ -529,7 +532,7 @@ static void test_backend_temp_ext_sampling(const test_params & params) { // lambda for testing non-positive temp/delta/exponent values. auto test_argmax_temp = [&](float temp, float delta, float exponent) { - printf("\nTesting temperature = %.1f, delta = %1.f, exponent = %1.f\n", temp, delta, exponent); + LOG_CNT("\nTesting temperature = %.1f, delta = %1.f, exponent = %1.f\n", temp, delta, exponent); int seq_id = 0; struct llama_sampler_chain_params backend_chain_params = llama_sampler_chain_default_params(); @@ -561,7 +564,7 @@ static void test_backend_temp_ext_sampling(const test_params & params) { test_argmax_temp(-1.0f, 0.3f, 2.0f); // Greedy (temp<0) test_argmax_temp(0.8f, 0.0f, 2.0f); // Temperature scaling - printf("backend temp_ext sampling test PASSED\n"); + LOG_CNT("backend temp_ext sampling test PASSED\n"); } static void test_backend_min_p_sampling(const test_params & params) { @@ -600,20 +603,20 @@ static void test_backend_min_p_sampling(const test_params & params) { llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx); const std::string token_str = test_ctx.token_to_piece(token, false); - printf("min-p cpu sampled token id:%d, string: '%s'\n", token, token_str.c_str()); + LOG_CNT("min-p cpu sampled token id:%d, string: '%s'\n", token, token_str.c_str()); GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab); // Decode and sample 10 more tokens for (int i = 0; i < 10; i++) { int32_t loop_idx = test_ctx.idx_for_seq(seq_id); llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), loop_idx); - printf("min-p gen step %d: token id :%5.d, string: %s\n", i, token, test_ctx.token_to_piece(token, false).c_str()); + LOG_CNT("min-p gen step %d: token id :%5.d, string: %s\n", i, token, test_ctx.token_to_piece(token, false).c_str()); if (!test_ctx.decode_token(token, 0)) { GGML_ASSERT(false && "Failed to decode token"); } } - printf("min-p sampling test PASSED\n"); + LOG_CNT("min-p sampling test PASSED\n"); } static void test_backend_top_p_sampling(const test_params & params) { @@ -652,18 +655,18 @@ static void test_backend_top_p_sampling(const test_params & params) { llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx); const std::string token_str = test_ctx.token_to_piece(token, false); - printf("top-p cpu sampled token id:%d, string: '%s'\n", token, token_str.c_str()); + LOG_CNT("top-p cpu sampled token id:%d, string: '%s'\n", token, token_str.c_str()); GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab); // Decode and sample 10 more tokens for (int i = 0; i < 10; i++) { int32_t loop_idx = test_ctx.idx_for_seq(seq_id); llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), loop_idx); - printf("top-p gen step %d: token id :%5.d, string: %s\n", i, token, test_ctx.token_to_piece(token, false).c_str()); + LOG_CNT("top-p gen step %d: token id :%5.d, string: %s\n", i, token, test_ctx.token_to_piece(token, false).c_str()); test_ctx.decode_token(token, 0); } - printf("top-p sampling test PASSED\n"); + LOG_CNT("top-p sampling test PASSED\n"); } static void test_backend_multi_sequence_sampling(const test_params & params) { @@ -697,7 +700,7 @@ static void test_backend_multi_sequence_sampling(const test_params & params) { int32_t batch_idx = test_ctx.idx_for_seq(0); llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx); const std::string token_str = test_ctx.token_to_piece(token, false); - printf("Seq 0 sampled token id=%d, string='%s'\n", token, token_str.c_str()); + LOG_CNT("Seq 0 sampled token id=%d, string='%s'\n", token, token_str.c_str()); GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab); } @@ -706,12 +709,12 @@ static void test_backend_multi_sequence_sampling(const test_params & params) { int32_t batch_idx= test_ctx.idx_for_seq(1); llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx); const std::string token_str = test_ctx.token_to_piece(token, false); - printf("Seq 1 sampled token id=%d, string='%s'\n", token, token_str.c_str()); + LOG_CNT("Seq 1 sampled token id=%d, string='%s'\n", token, token_str.c_str()); GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab); } // Generate tokens for each sequence - printf("\nMulti-sequence generation:\n"); + LOG_CNT("\nMulti-sequence generation:\n"); for (int step = 0; step < 4; step++) { std::map tokens; @@ -719,7 +722,7 @@ static void test_backend_multi_sequence_sampling(const test_params & params) { int32_t idx = test_ctx.idx_for_seq(seq_id); llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), idx); const std::string token_str = test_ctx.token_to_piece(token, false); - printf(" Seq %d, step %d: token id=%d, string='%s'\n", seq_id, step, token, token_str.c_str()); + LOG_CNT(" Seq %d, step %d: token id=%d, string='%s'\n", seq_id, step, token, token_str.c_str()); tokens[seq_id] = token; } @@ -729,7 +732,7 @@ static void test_backend_multi_sequence_sampling(const test_params & params) { } } - printf("backend multi-sequence sampling test PASSED\n"); + LOG_CNT("backend multi-sequence sampling test PASSED\n"); } static void test_backend_dist_sampling(const test_params & params) { @@ -749,15 +752,15 @@ static void test_backend_dist_sampling(const test_params & params) { int32_t batch_idx = test_ctx.idx_for_seq(seq_id); llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx); - printf("dist sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str()); + LOG_CNT("dist sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str()); GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab); //GGML_ASSERT(llama_get_sampled_logits_ith(test_ctx.ctx.get(), batch_idx) == nullptr); token = llama_get_sampled_token_ith(test_ctx.ctx.get(), -1); - printf("dist sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str()); + LOG_CNT("dist sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str()); GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab); - printf("backend dist sampling test PASSED\n"); + LOG_CNT("backend dist sampling test PASSED\n"); } static void test_backend_dist_sampling_and_cpu(const test_params & params) { @@ -784,10 +787,10 @@ static void test_backend_dist_sampling_and_cpu(const test_params & params) { llama_token backend_token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx); llama_token cpu_token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx); - printf("dist & cpu sampled id:%d, string:'%s'\n", cpu_token, test_ctx.token_to_piece(cpu_token, false).c_str()); + LOG_CNT("dist & cpu sampled id:%d, string:'%s'\n", cpu_token, test_ctx.token_to_piece(cpu_token, false).c_str()); GGML_ASSERT(backend_token == cpu_token); - printf("backend dist & cpu sampling test PASSED\n"); + LOG_CNT("backend dist & cpu sampling test PASSED\n"); } static void test_backend_logit_bias_sampling(const test_params & params) { @@ -809,7 +812,7 @@ static void test_backend_logit_bias_sampling(const test_params & params) { //logit_bias.push_back({ bias_token, +100.0f }); logit_bias.push_back({ bias_token, +10.0f }); - printf("biasing token piece '%s' -> token id %d\n", piece.c_str(), bias_token); + LOG_CNT("biasing token piece '%s' -> token id %d\n", piece.c_str(), bias_token); struct llama_sampler_chain_params backend_chain_params = llama_sampler_chain_default_params(); llama_sampler_ptr backend_sampler_chain(llama_sampler_chain_init(backend_chain_params)); @@ -830,10 +833,11 @@ static void test_backend_logit_bias_sampling(const test_params & params) { } llama_token backend_token = llama_get_sampled_token_ith(test_ctx.ctx.get(), test_ctx.idx_for_seq(seq_id)); - printf("sampled token = %d, expected = %d\n", backend_token, bias_token); + LOG_CNT("sampled token = %d, expected = %d\n", backend_token, bias_token); + common_log_flush(common_log_main()); // the assert below aborts GGML_ASSERT(backend_token == bias_token); - printf("backend logit bias sampling test PASSED\n"); + LOG_CNT("backend logit bias sampling test PASSED\n"); } static void accept_prompt(llama_sampler * smpl, const llama_vocab * vocab, const std::string & prompt) { @@ -1014,7 +1018,7 @@ static sampler_comparison_stats compare_sampler_outputs( const float logit = actual.logits[i]; if (!seen.insert(token).second || std::isnan(logit)) { if (result.n_mismatch < 5) { - printf("%s token %d has invalid backend output\n", name, token); + LOG_CNT("%s token %d has invalid backend output\n", name, token); } ++result.n_mismatch; continue; @@ -1026,7 +1030,7 @@ static sampler_comparison_stats compare_sampler_outputs( ++result.n_masked; } else if (!allow_extra_candidates) { if (result.n_mismatch < 5) { - printf("%s token %d was not masked\n", name, token); + LOG_CNT("%s token %d was not masked\n", name, token); } ++result.n_mismatch; } @@ -1037,8 +1041,8 @@ static sampler_comparison_stats compare_sampler_outputs( result.max_diff = std::max(result.max_diff, diff); if (!std::isfinite(logit) || diff > 1e-3f) { if (result.n_mismatch < 5) { - printf("%s mismatch token %d: cpu=%.6f backend=%.6f diff=%.6f\n", - name, token, it->second, logit, diff); + LOG_CNT("%s mismatch token %d: cpu=%.6f backend=%.6f diff=%.6f\n", + name, token, it->second, logit, diff); } ++result.n_mismatch; } @@ -1047,14 +1051,15 @@ static sampler_comparison_stats compare_sampler_outputs( for (const auto & item : expected) { if (seen.find(item.first) == seen.end()) { if (result.n_mismatch < 5) { - printf("%s missing backend token %d\n", name, item.first); + LOG_CNT("%s missing backend token %d\n", name, item.first); } ++result.n_mismatch; } } - printf("%s logits: max_diff=%.6f n_masked=%d n_mismatch=%d\n", - name, result.max_diff, result.n_masked, result.n_mismatch); + LOG_CNT("%s logits: max_diff=%.6f n_masked=%d n_mismatch=%d\n", + name, result.max_diff, result.n_masked, result.n_mismatch); + common_log_flush(common_log_main()); // the callers abort right after this on a mismatch return result; } @@ -1311,10 +1316,10 @@ static void compare_masking_penalties_logits( } static void test_backend_penalties_sampling(const test_params & params) { - printf("Testing backend penalties (repeat + freq + presence)\n"); + LOG_CNT("Testing backend penalties (repeat + freq + presence)\n"); compare_penalties_logits(params, 64, 1.1f, 0.5f, 0.25f, "Hello Hello world"); - printf("Testing backend penalties with penalty_last_n > 64\n"); + LOG_CNT("Testing backend penalties with penalty_last_n > 64\n"); const auto * vocab = llama_model_get_vocab(params.model.get()); std::vector tokens(8); int32_t n_tok = llama_tokenize(vocab, "a", 1, tokens.data(), (int32_t) tokens.size(), false, false); @@ -1332,52 +1337,52 @@ static void test_backend_penalties_sampling(const test_params & params) { } }); - printf("Testing backend penalties without filler entries\n"); + LOG_CNT("Testing backend penalties without filler entries\n"); compare_penalties_logits(params, 64, 1.1f, 0.5f, 0.25f, "Hello", [](llama_sampler * smpl) { for (llama_token token = 0; token < 64; ++token) { llama_sampler_accept(smpl, token); } }); - printf("Testing backend top-k followed by penalties\n"); + LOG_CNT("Testing backend top-k followed by penalties\n"); compare_top_k_penalties_logits(params, 8, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter); - printf("Testing backend penalties followed by top-k\n"); + LOG_CNT("Testing backend penalties followed by top-k\n"); compare_top_k_penalties_logits(params, 8, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::before_filter); - printf("Testing backend top-p followed by penalties\n"); + LOG_CNT("Testing backend top-p followed by penalties\n"); compare_masking_penalties_logits(params, "top-p", []() { return llama_sampler_init_top_p(0.9f, 0); }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, true); - printf("Testing backend top-p followed by penalties with a large history window\n"); + LOG_CNT("Testing backend top-p followed by penalties with a large history window\n"); compare_masking_penalties_logits(params, "top-p large-window", []() { return llama_sampler_init_top_p(0.9f, 0); }, 4096, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, true); - printf("Testing backend penalties followed by top-p\n"); + LOG_CNT("Testing backend penalties followed by top-p\n"); compare_masking_penalties_logits(params, "top-p", []() { return llama_sampler_init_top_p(0.9f, 0); }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::before_filter, true); - printf("Testing backend min-p followed by penalties\n"); + LOG_CNT("Testing backend min-p followed by penalties\n"); compare_masking_penalties_logits(params, "min-p", []() { return llama_sampler_init_min_p(0.1f, 0); }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, false); - printf("Testing backend penalties followed by min-p\n"); + LOG_CNT("Testing backend penalties followed by min-p\n"); compare_masking_penalties_logits(params, "min-p", []() { return llama_sampler_init_min_p(0.1f, 0); }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::before_filter, false); - printf("Testing backend top-p followed by penalties with empty history\n"); + LOG_CNT("Testing backend top-p followed by penalties with empty history\n"); compare_masking_penalties_logits(params, "top-p empty", []() { return llama_sampler_init_top_p(0.9f, 0); }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, true, false); - printf("Testing backend top-p followed by individual penalties\n"); + LOG_CNT("Testing backend top-p followed by individual penalties\n"); compare_masking_penalties_logits(params, "top-p repeat", []() { return llama_sampler_init_top_p(0.9f, 0); }, 64, 1.1f, 0.0f, 0.0f, "Hello", penalties_position::after_filter, true); @@ -1388,10 +1393,10 @@ static void test_backend_penalties_sampling(const test_params & params) { return llama_sampler_init_top_p(0.9f, 0); }, 64, 1.0f, 0.0f, 0.25f, "Hello", penalties_position::after_filter, true); - printf("Testing backend penalty parameter values\n"); + LOG_CNT("Testing backend penalty parameter values\n"); test_penalty_parameter_values(params); - printf("backend penalties sampling test PASSED\n"); + LOG_CNT("backend penalties sampling test PASSED\n"); } // This test verifies that it is possible to have two different backend samplers, @@ -1427,7 +1432,7 @@ static void test_backend_mixed_sampling(const test_params & params) { int32_t batch_idx = test_ctx.idx_for_seq(0); llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx); const std::string token_str = test_ctx.token_to_piece(token, false); - printf("sampled token id=%d, string='%s'\n", token, token_str.c_str()); + LOG_CNT("sampled token id=%d, string='%s'\n", token, token_str.c_str()); GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab); //GGML_ASSERT(llama_get_sampled_logits_ith(test_ctx.ctx.get(), batch_idx) == nullptr); //GGML_ASSERT(llama_get_sampled_logits_count_ith(test_ctx.ctx.get(), batch_idx) == 0); @@ -1443,7 +1448,7 @@ static void test_backend_mixed_sampling(const test_params & params) { GGML_ASSERT(llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx) == LLAMA_TOKEN_NULL); } - printf("backend mixed sampling test PASSED\n"); + LOG_CNT("backend mixed sampling test PASSED\n"); } static void test_backend_set_sampler(const test_params & params) { @@ -1466,11 +1471,11 @@ static void test_backend_set_sampler(const test_params & params) { // Sample using backend sampler configured above llama_token backend_token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx); const std::string backend_token_str = test_ctx.token_to_piece(backend_token, false); - printf("dist sampled token = %d, string='%s'\n", backend_token, backend_token_str.c_str()); + LOG_CNT("dist sampled token = %d, string='%s'\n", backend_token, backend_token_str.c_str()); // Now clear the backend sampler for this sequence. llama_set_sampler(test_ctx.ctx.get(), seq_id, nullptr); - printf("Cleared backend sampler for seq_id %d\n", seq_id); + LOG_CNT("Cleared backend sampler for seq_id %d\n", seq_id); // Sample using CPU sampler struct llama_sampler_chain_params chain_params = llama_sampler_chain_default_params(); @@ -1490,7 +1495,7 @@ static void test_backend_set_sampler(const test_params & params) { // Sample the token using the CPU sampler chain. llama_token token2 = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), seq_id); const std::string token2_str = test_ctx.token_to_piece(token2, false); - printf("CPU sampled token after clearing backend sampler: id=%d, string='%s'\n", token2, token2_str.c_str()); + LOG_CNT("CPU sampled token after clearing backend sampler: id=%d, string='%s'\n", token2, token2_str.c_str()); std::map tokens2 = { { seq_id, token2}, }; // Set a new backend sampler for the sequence. @@ -1506,9 +1511,9 @@ static void test_backend_set_sampler(const test_params & params) { llama_token new_backend_token = llama_get_sampled_token_ith(test_ctx.ctx.get(), test_ctx.idx_for_seq(seq_id)); const std::string new_backend_token_str = test_ctx.token_to_piece(new_backend_token, false); - printf("dist sampled token = %d, string='%s'\n", new_backend_token, new_backend_token_str.c_str()); + LOG_CNT("dist sampled token = %d, string='%s'\n", new_backend_token, new_backend_token_str.c_str()); - printf("backend set sampler test PASSED\n"); + LOG_CNT("backend set sampler test PASSED\n"); } static void test_backend_cpu_mixed_batch(const test_params & params) { @@ -1538,7 +1543,7 @@ static void test_backend_cpu_mixed_batch(const test_params & params) { int32_t batch_idx = test_ctx.idx_for_seq(0); llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx); const std::string token_str = test_ctx.token_to_piece(token, false); - printf("Seq 0 (backend) sampled token id=%d, string='%s'\n", token, token_str.c_str()); + LOG_CNT("Seq 0 (backend) sampled token id=%d, string='%s'\n", token, token_str.c_str()); GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab); } @@ -1555,7 +1560,7 @@ static void test_backend_cpu_mixed_batch(const test_params & params) { llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx); const std::string token_str = test_ctx.token_to_piece(token, false); - printf("Seq 1 (CPU) sampled token id=%d, string='%s'\n", token, token_str.c_str()); + LOG_CNT("Seq 1 (CPU) sampled token id=%d, string='%s'\n", token, token_str.c_str()); GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab); } @@ -1592,11 +1597,11 @@ static void test_backend_cpu_mixed_batch(const test_params & params) { int32_t batch_idx = test_ctx.idx_for_seq(0); llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx); const std::string token_str = test_ctx.token_to_piece(token, false); - printf("re-added backend sampled token id=%d, string='%s'\n", token, token_str.c_str()); + LOG_CNT("re-added backend sampled token id=%d, string='%s'\n", token, token_str.c_str()); GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab); } - printf("backend-cpu mixed batch test PASSED\n"); + LOG_CNT("backend-cpu mixed batch test PASSED\n"); } static void test_backend_multi_output_limit(const test_params & params) { @@ -1612,14 +1617,14 @@ static void test_backend_multi_output_limit(const test_params & params) { common_batch_add(batch, llama_vocab_bos(test_ctx.vocab), i, { seq_id }, true); } - printf(">>> test_backend_multi_output_limit expected error start:\n"); + LOG_CNT(">>> test_backend_multi_output_limit expected error start:\n"); const int ret = llama_decode(test_ctx.ctx.get(), batch); GGML_ASSERT(ret != 0 && "llama_decode should reject outputs above the per-sequence limit"); - printf("<<< test_backend_multi_output_limit expected error end.\n"); + LOG_CNT("<<< test_backend_multi_output_limit expected error end.\n"); llama_batch_free(batch); - printf("backend multi-output limit test PASSED\n"); + LOG_CNT("backend multi-output limit test PASSED\n"); } static void test_backend_multi_sequence_multi_output_dist(const test_params & params) { @@ -1708,7 +1713,7 @@ static void test_backend_multi_sequence_multi_output_dist(const test_params & pa llama_batch_free(batch); - printf("backend multi-sequence multi-output dist test PASSED\n"); + LOG_CNT("backend multi-sequence multi-output dist test PASSED\n"); } static void test_backend_multi_output_dist_transaction(const test_params & params) { @@ -1778,7 +1783,7 @@ static void test_backend_multi_output_dist_transaction(const test_params & param verify_random(0, randoms[2]); llama_batch_free(batch); - printf("backend multi-output dist transaction test PASSED\n"); + LOG_CNT("backend multi-output dist transaction test PASSED\n"); } static void test_backend_multi_output_sampling_chain(const test_params & params) { @@ -1928,7 +1933,7 @@ static void test_backend_multi_output_sampling_chain(const test_params & params) GGML_ASSERT(llama_decode(test_ctx.ctx.get(), batch) == 0); llama_batch_free(batch); - printf("backend multi-output sampling chain test PASSED\n"); + LOG_CNT("backend multi-output sampling chain test PASSED\n"); } static void test_backend_multi_output_cpu_suffix(const test_params & params) { @@ -1990,7 +1995,7 @@ static void test_backend_multi_output_cpu_suffix(const test_params & params) { llama_batch_free(batch); } - printf("backend multi-output CPU suffix test PASSED\n"); + LOG_CNT("backend multi-output CPU suffix test PASSED\n"); } struct backend_test_case { @@ -2029,7 +2034,8 @@ static test_args parse_cli(int argc, char ** argv) { if (std::strcmp(arg, "--test") == 0) { if (i + 1 >= argc) { - fprintf(stderr, "--test expects a value\n"); + LOG_ERR("--test expects a value\n"); + common_log_flush(common_log_main()); exit(EXIT_FAILURE); } out.test = argv[++i]; @@ -2041,7 +2047,8 @@ static test_args parse_cli(int argc, char ** argv) { } if (std::strcmp(arg, "--model") == 0) { if (i + 1 >= argc) { - fprintf(stderr, "--model expects a value\n"); + LOG_ERR("--model expects a value\n"); + common_log_flush(common_log_main()); exit(EXIT_FAILURE); } out.model = argv[++i]; @@ -2053,7 +2060,8 @@ static test_args parse_cli(int argc, char ** argv) { } if (std::strcmp(arg, "--device") == 0) { if (i + 1 >= argc) { - fprintf(stderr, "--device expects a value (cpu or gpu)\n"); + LOG_ERR("--device expects a value (cpu or gpu)\n"); + common_log_flush(common_log_main()); exit(EXIT_FAILURE); } out.device = argv[++i]; @@ -2068,12 +2076,14 @@ static test_args parse_cli(int argc, char ** argv) { continue; } - fprintf(stderr, "Unexpected argument: %s\n", arg); + LOG_ERR("Unexpected argument: %s\n", arg); + common_log_flush(common_log_main()); exit(EXIT_FAILURE); } if (out.device != "cpu" && out.device != "gpu" && out.device != "auto") { - fprintf(stderr, "Invalid device '%s'. Must be 'cpu', 'gpu' or 'auto'\n", out.device.c_str()); + LOG_ERR("Invalid device '%s'. Must be 'cpu', 'gpu' or 'auto'\n", out.device.c_str()); + common_log_flush(common_log_main()); exit(EXIT_FAILURE); } @@ -2091,10 +2101,12 @@ static std::vector collect_tests_to_run(const std::st } } if (selected.empty()) { - fprintf(stderr, "Unknown test '%s'. Available tests:\n", requested.c_str()); + LOG_ERR("Unknown test '%s'. Available tests:\n", requested.c_str()); for (const auto & test : BACKEND_TESTS) { - fprintf(stderr, " %s\n", test.name.c_str()); + LOG_ERR(" %s\n", test.name.c_str()); } + LOG("%s: %s\n", "test-backend-sampler", "FAILED"); + common_log_flush(common_log_main()); exit(EXIT_FAILURE); } } else { @@ -2108,7 +2120,7 @@ static std::vector collect_tests_to_run(const std::st test.name == "mixed" || test.name == "top_p" || test.name == "multi_output_sampling_chain" || test.name == "multi_output_cpu") { - fprintf(stderr, "Skipping test '%s' on HIP backend (no backend TOP_K support)\n", test.name.c_str()); + LOG_WRN("Skipping test '%s' on HIP backend (no backend TOP_K support)\n", test.name.c_str()); continue; } #endif // GGML_USE_HIP @@ -2117,7 +2129,7 @@ static std::vector collect_tests_to_run(const std::st } if (selected.empty()) { - fprintf(stderr, "No backend sampling tests selected. Use --test= to pick one.\n"); + LOG_WRN("No backend sampling tests selected. Use --test= to pick one.\n"); } return selected; @@ -2125,32 +2137,75 @@ static std::vector collect_tests_to_run(const std::st static void run_tests(const std::vector & tests, const test_params & args) { for (const auto & test : tests) { - fprintf(stderr, "\n=== %s ===\n", test->name.c_str()); + LOG_INF("\n=== %s ===\n", test->name.c_str()); try { test->fn(args); } catch (const std::exception & e) { - fprintf(stderr, "Error running test '%s': %s\n", test->name.c_str(), e.what()); + LOG_ERR("Error running test '%s': %s\n", test->name.c_str(), e.what()); + LOG("%s: %s\n", "test-backend-sampler", "FAILED"); + common_log_flush(common_log_main()); exit(EXIT_FAILURE); } } } int main(int argc, char ** argv) { - test_args args = parse_cli(argc, argv); + test_args args; + + { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + + // the test's own options and the model path are handled here, everything else goes to common_params_parse + std::vector test_argv; + std::vector common_argv; + test_argv.push_back(argv[0]); + common_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "--test") == 0 || strcmp(argv[i], "--model") == 0 || strcmp(argv[i], "--device") == 0) { + test_argv.push_back(argv[i]); + if (i + 1 < argc) { + test_argv.push_back(argv[++i]); // the option value + } + continue; + } + if (strncmp(argv[i], "--test=", 7) == 0 || strncmp(argv[i], "--model=", 8) == 0 || strncmp(argv[i], "--device=", 9) == 0) { + test_argv.push_back(argv[i]); + continue; + } + if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else { + test_argv.push_back(argv[i]); // the model path + } + } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + test_argv.push_back(nullptr); + args = parse_cli((int) test_argv.size() - 1, test_argv.data()); + } if (args.model.empty()) { args.model = common_get_model_or_exit(1, argv); } + LOG("%s: running\n", "test-backend-sampler"); + { std::ifstream file(args.model); if (!file.is_open()) { - fprintf(stderr, "no model '%s' found\n", args.model.c_str()); + LOG_ERR("no model '%s' found\n", args.model.c_str()); + LOG("%s: %s\n", "test-backend-sampler", "FAILED"); + common_log_flush(common_log_main()); return EXIT_FAILURE; } } - fprintf(stderr, "using '%s'\n", args.model.c_str()); + LOG_INF("using '%s'\n", args.model.c_str()); llama_backend_init(); @@ -2163,5 +2218,7 @@ int main(int argc, char ** argv) { run_tests(tests, params); } + LOG("%s: %s\n", "test-backend-sampler", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-barrier.cpp b/tests/test-barrier.cpp index 61f73adfd2..0cc20fe636 100644 --- a/tests/test-barrier.cpp +++ b/tests/test-barrier.cpp @@ -1,11 +1,15 @@ #include "ggml.h" #include "ggml-cpu.h" +#include "arg.h" +#include "common.h" +#include "log.h" + #include -#include #include #include #include +#include #include #include @@ -40,7 +44,8 @@ static void test_barrier(int n_threads, int n_rounds) { struct ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads); struct ggml_threadpool* threadpool = ggml_threadpool_new(&tpp); if (!threadpool) { - fprintf(stderr, "threadpool create failed : n_threads %d\n", n_threads); + LOG_ERR("threadpool create failed : n_threads %d\n", n_threads); + common_log_flush(common_log_main()); exit(1); } @@ -50,11 +55,10 @@ static void test_barrier(int n_threads, int n_rounds) { std::vector work_data(cplan.work_size); cplan.work_data = work_data.data(); - std::cerr << "graph-compute with" - << "\n n_threads: " << n_threads - << "\n n_nodes: " << n_nodes - << "\n n_rounds: " << n_rounds - << "\n"; + LOG_INF("graph-compute with" + "\n n_threads: %d" + "\n n_nodes: %d" + "\n n_rounds: %d\n", n_threads, n_nodes, n_rounds); // ggml_graph_print(gf); // Warmup @@ -70,10 +74,10 @@ static void test_barrier(int n_threads, int n_rounds) { auto usec = std::chrono::duration_cast(t1-t0).count(); auto nsec = std::chrono::duration_cast(t1-t0).count(); - std::cerr << "graph-compute took " << usec << " usec " - << "\n " << (float) usec / n_rounds << " usec per-iter" - << "\n " << (float) nsec / (n_rounds * n_nodes) << " nsec per-node" - << "\n"; + LOG_INF("graph-compute took %lld usec " + "\n %g usec per-iter" + "\n %g nsec per-node\n", + (long long) usec, (float) usec / n_rounds, (float) nsec / (n_rounds * n_nodes)); ggml_threadpool_free(threadpool); ggml_free(ctx); @@ -108,15 +112,15 @@ static void test_active(int n_threads, int n_rounds) { struct ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads); struct ggml_threadpool* threadpool = ggml_threadpool_new(&tpp); if (!threadpool) { - fprintf(stderr, "threadpool create failed : n_threads %d\n", n_threads); + LOG_ERR("threadpool create failed : n_threads %d\n", n_threads); + common_log_flush(common_log_main()); exit(1); } - std::cerr << "graph-compute with" - << "\n n_threads: " << n_threads - << "\n n_nodes: " << n_nodes - << "\n n_rounds: " << n_rounds - << "\n"; + LOG_INF("graph-compute with" + "\n n_threads: %d" + "\n n_nodes: %d" + "\n n_rounds: %d\n", n_threads, n_nodes, n_rounds); // ggml_graph_print(gf); // In this test we keep changing the number of threads every 4th iteration @@ -181,16 +185,17 @@ static void test_multi_graph(int n_threads, int n_rounds) { struct ggml_threadpool_params tpp = ggml_threadpool_params_default(n_threads); struct ggml_threadpool* threadpool = ggml_threadpool_new(&tpp); if (!threadpool) { - fprintf(stderr, "threadpool create failed : n_threads %d\n", n_threads); + LOG_ERR("threadpool create failed : n_threads %d\n", n_threads); + common_log_flush(common_log_main()); exit(1); } - std::cerr << "graph-compute with" - << "\n gf0 n_nodes: " << ggml_graph_n_nodes(gf0) - << "\n gf1 n_nodes: " << ggml_graph_n_nodes(gf1) - << "\n n_threads: " << n_threads - << "\n n_rounds: " << n_rounds - << "\n"; + LOG_INF("graph-compute with" + "\n gf0 n_nodes: %d" + "\n gf1 n_nodes: %d" + "\n n_threads: %d" + "\n n_rounds: %d\n", + ggml_graph_n_nodes(gf0), ggml_graph_n_nodes(gf1), n_threads, n_rounds); // In this test we keep changing the number of threads every 4th iteration // and we compute two graphs back to back to test graph frequent graph switching @@ -214,23 +219,46 @@ static void test_multi_graph(int n_threads, int n_rounds) { int main(int argc, char *argv[]) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + + // this test takes n_threads and n_rounds as positional arguments + std::vector positional; + std::vector common_argv; + common_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { + if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else { + positional.push_back(argv[i]); + } + } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } int n_threads = std::max(1, std::min(4, (int) std::thread::hardware_concurrency())); int n_rounds = 100; - if (argc > 1) { - n_threads = std::atoi(argv[1]); + if (positional.size() > 0) { + n_threads = std::atoi(positional[0].c_str()); } - if (argc > 2) { - n_rounds = std::atoi(argv[2]); + if (positional.size() > 1) { + n_rounds = std::atoi(positional[1].c_str()); } + LOG("%s: running\n", "test-barrier"); + test_barrier(n_threads, n_rounds); test_active(n_threads, n_rounds * 100); test_multi_graph(n_threads, n_rounds * 10); + LOG("%s: %s\n", "test-barrier", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-batch-alloc.cpp b/tests/test-batch-alloc.cpp index 66d29d6f51..fe8755debd 100644 --- a/tests/test-batch-alloc.cpp +++ b/tests/test-batch-alloc.cpp @@ -1,6 +1,8 @@ +#include "arg.h" #include "testing.h" #include "llama.h" +#include "log.h" #include "../src/llama-batch.h" #include "../src/llama-memory.h" @@ -651,6 +653,28 @@ static void test_mrope(testing & t) { } int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + + // this test takes an optional filter as its only positional argument + std::string filter; + std::vector common_argv; + common_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { + if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else if (filter.empty()) { + filter = argv[i]; + } + } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-batch-alloc"); + testing t; const char * verbose = getenv("LLAMA_TEST_VERBOSE"); @@ -661,8 +685,8 @@ int main(int argc, char ** argv) { llama_log_set([](ggml_log_level, const char *, void *) {}, nullptr); } - if (argc > 1) { - t.set_filter(argv[1]); + if (!filter.empty()) { + t.set_filter(filter); } t.test("init", test_init); @@ -670,5 +694,8 @@ int main(int argc, char ** argv) { t.test("keep_tail", test_keep_tail); t.test("mrope", test_mrope); - return t.summary(); + const int rc = t.summary(); + LOG("%s: %s\n", "test-batch-alloc", rc == 0 ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); + return rc; } diff --git a/tests/test-c.c b/tests/test-c.c index a05071080a..8b58933160 100644 --- a/tests/test-c.c +++ b/tests/test-c.c @@ -1,3 +1,7 @@ #include "llama.h" -int main(void) {} +int main(void) { + printf("test-c: running\n"); + printf("test-c: PASSED\n"); + return 0; +} diff --git a/tests/test-chat-analysis.cpp b/tests/test-chat-analysis.cpp index 42ad7a0724..d18aeb6068 100644 --- a/tests/test-chat-analysis.cpp +++ b/tests/test-chat-analysis.cpp @@ -1,6 +1,8 @@ #include "chat-auto-parser.h" #include "chat-auto-parser-helpers.h" #include "chat.h" +#include "arg.h" +#include "common.h" #include "log.h" #include "jinja/caps.h" #include "jinja/runtime.h" @@ -84,16 +86,16 @@ static std::string read_file(const std::string & path) { } static void print_usage(const char * program_name) { - LOG_ERR("Debug the auto-parser's differential analysis: render a template with/without tools, reasoning, etc. and show the diffs.\n"); - LOG_ERR("\nUsage: %s [options]\n", program_name); - LOG_ERR("\nOptions:\n"); - LOG_ERR(" --template Analyze specific template from test suite (e.g., 'deepseek' or 'DeepSeek-V3.1')\n"); - LOG_ERR(" --template-file Analyze custom template file\n"); - LOG_ERR(" --all Analyze all templates from test suite (default when no arguments are given)\n"); - LOG_ERR("\nExamples:\n"); - LOG_ERR(" %s --all\n", program_name); - LOG_ERR(" %s --template deepseek\n", program_name); - LOG_ERR(" %s --template-file my-template.jinja\n", program_name); + LOG_INF("Debug the auto-parser's differential analysis: render a template with/without tools, reasoning, etc. and show the diffs.\n"); + LOG_INF("\nUsage: %s [options]\n", program_name); + LOG_INF("\nOptions:\n"); + LOG_INF(" --template Analyze specific template from test suite (e.g., 'deepseek' or 'DeepSeek-V3.1')\n"); + LOG_INF(" --template-file Analyze custom template file\n"); + LOG_INF(" --all Analyze all templates from test suite (default when no arguments are given)\n"); + LOG_INF("\nExamples:\n"); + LOG_INF(" %s --all\n", program_name); + LOG_INF(" %s --template deepseek\n", program_name); + LOG_INF(" %s --template-file my-template.jinja\n", program_name); } static bool parse_options(int argc, char ** argv, analysis_options & opts) { @@ -270,15 +272,15 @@ static json make_assistant_one_tool_with_reasoning() { } static void print_diff_split(const std::string & title, const diff_split & diff) { - LOG_ERR("\n%s=== %s ===%s\n", ANSI_CYAN, title.c_str(), ANSI_RESET); - LOG_ERR("%sCommon Prefix:%s '%s'\n", ANSI_PREFIX, ANSI_RESET, diff.prefix.c_str()); - LOG_ERR("%sCommon Suffix:%s '%s'\n", ANSI_SUFFIX, ANSI_RESET, diff.suffix.c_str()); - LOG_ERR("%sLeft (difference):%s '%s'\n", ANSI_GREEN, ANSI_RESET, diff.left.c_str()); - LOG_ERR("%sRight (difference):%s '%s'\n", ANSI_ORANGE, ANSI_RESET, diff.right.c_str()); + LOG_INF("\n%s=== %s ===%s\n", ANSI_CYAN, title.c_str(), ANSI_RESET); + LOG_INF("%sCommon Prefix:%s '%s'\n", ANSI_PREFIX, ANSI_RESET, diff.prefix.c_str()); + LOG_INF("%sCommon Suffix:%s '%s'\n", ANSI_SUFFIX, ANSI_RESET, diff.suffix.c_str()); + LOG_INF("%sLeft (difference):%s '%s'\n", ANSI_GREEN, ANSI_RESET, diff.left.c_str()); + LOG_INF("%sRight (difference):%s '%s'\n", ANSI_ORANGE, ANSI_RESET, diff.right.c_str()); } static void check_reasoning_variables(const common_chat_template & tmpl) { - LOG_ERR("\n%s=== Checking Reasoning Variables ===%s\n", ANSI_CYAN, ANSI_RESET); + LOG_INF("\n%s=== Checking Reasoning Variables ===%s\n", ANSI_CYAN, ANSI_RESET); try { // Create a list of candidate reasoning/thinking variable names to probe @@ -355,11 +357,11 @@ static void check_reasoning_variables(const common_chat_template & tmpl) { } if (accessed_vars.empty()) { - LOG_ERR("%sNo reasoning/thinking-related variables were queried by the template%s\n", ANSI_GRAY, ANSI_RESET); + LOG_INF("%sNo reasoning/thinking-related variables were queried by the template%s\n", ANSI_GRAY, ANSI_RESET); } else { - LOG_ERR("Template queries the following reasoning/thinking-related variables:\n"); + LOG_INF("Template queries the following reasoning/thinking-related variables:\n"); for (const auto & var : accessed_vars) { - LOG_ERR(" %s- %s%s\n", ANSI_ORANGE, var.c_str(), ANSI_RESET); + LOG_INF(" %s- %s%s\n", ANSI_ORANGE, var.c_str(), ANSI_RESET); } } @@ -369,12 +371,12 @@ static void check_reasoning_variables(const common_chat_template & tmpl) { } static void analyze_template(const std::string & template_path) { - LOG_ERR("\n"); - LOG_ERR("%s", ANSI_PURPLE); - LOG_ERR("================================================================================\n"); - LOG_ERR(" ANALYZING TEMPLATE: %s\n", template_path.c_str()); - LOG_ERR("================================================================================\n"); - LOG_ERR("%s", ANSI_RESET); + LOG_INF("\n"); + LOG_INF("%s", ANSI_PURPLE); + LOG_INF("================================================================================\n"); + LOG_INF(" ANALYZING TEMPLATE: %s\n", template_path.c_str()); + LOG_INF("================================================================================\n"); + LOG_INF("%s", ANSI_RESET); std::string template_source; try { @@ -389,14 +391,14 @@ static void analyze_template(const std::string & template_path) { json tools = build_tools_definition(); // ===== CAPABILITIES ANALYSIS ===== - LOG_ERR("\n%s=== Template Capabilities (from jinja::caps) ===%s\n", ANSI_CYAN, ANSI_RESET); + LOG_INF("\n%s=== Template Capabilities (from jinja::caps) ===%s\n", ANSI_CYAN, ANSI_RESET); auto caps = chat_template.original_caps(); - LOG_ERR("%ssupports_tools:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_tools ? "true" : "false"); - LOG_ERR("%ssupports_tool_calls:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_tool_calls ? "true" : "false"); - LOG_ERR("%ssupports_system_role:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_system_role ? "true" : "false"); - LOG_ERR("%ssupports_parallel_tool_calls:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_parallel_tool_calls ? "true" : "false"); - LOG_ERR("%ssupports_typed_content:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_typed_content ? "true" : "false"); - LOG_ERR("%ssupports_string_content:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_string_content ? "true" : "false"); + LOG_INF("%ssupports_tools:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_tools ? "true" : "false"); + LOG_INF("%ssupports_tool_calls:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_tool_calls ? "true" : "false"); + LOG_INF("%ssupports_system_role:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_system_role ? "true" : "false"); + LOG_INF("%ssupports_parallel_tool_calls:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_parallel_tool_calls ? "true" : "false"); + LOG_INF("%ssupports_typed_content:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_typed_content ? "true" : "false"); + LOG_INF("%ssupports_string_content:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_string_content ? "true" : "false"); // ===== DIFFERENTIAL ANALYSIS ===== @@ -584,32 +586,68 @@ static void analyze_template(const std::string & template_path) { } int main(int argc, char ** argv) { - // Set log level to capture all output - common_log_set_verbosity_thold(99); + common_params params; + params.model.path = "."; // placeholder so common_params_parse does not require --model + common_init(); - analysis_options opts; - if (!parse_options(argc, argv, opts)) { + // this tool handles --all/--template/--template-file and -h itself; every other option goes to the common parser + std::vector common_argv; + std::vector own_argv; + common_argv.push_back(argv[0]); + own_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { + const std::string arg = argv[i]; + if (arg == "--all" || arg == "-h" || arg == "--help") { + own_argv.push_back(argv[i]); + } else if (arg == "--template" || arg == "--template-file") { + own_argv.push_back(argv[i]); + if (i + 1 < argc) { + own_argv.push_back(argv[++i]); + } + } else if (arg[0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else { + own_argv.push_back(argv[i]); // unrecognised token: parse_options reports it + } + } + common_argv.push_back(nullptr); + own_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { return 1; } - LOG_ERR("\n"); - LOG_ERR("%s", ANSI_PURPLE); - LOG_ERR("================================================================================\n"); - LOG_ERR(" TEMPLATE ANALYSIS TOOL\n"); - LOG_ERR("================================================================================\n"); - LOG_ERR("%s", ANSI_RESET); - LOG_ERR("Analyzing %s%zu%s template(s)\n", ANSI_CYAN, opts.template_paths.size(), ANSI_RESET); + // capture all output unless the user asked for a lower verbosity threshold + common_log_set_verbosity_thold(params.verbosity < LOG_DEFAULT_LLAMA ? params.verbosity : 99); + + analysis_options opts; + if (!parse_options((int) own_argv.size() - 1, own_argv.data(), opts)) { + common_log_flush(common_log_main()); + return 1; + } + + LOG("%s: running\n", "test-chat-analysis"); + + LOG_INF("\n"); + LOG_INF("%s", ANSI_PURPLE); + LOG_INF("================================================================================\n"); + LOG_INF(" TEMPLATE ANALYSIS TOOL\n"); + LOG_INF("================================================================================\n"); + LOG_INF("%s", ANSI_RESET); + LOG_INF("Analyzing %s%zu%s template(s)\n", ANSI_CYAN, opts.template_paths.size(), ANSI_RESET); for (const auto & path : opts.template_paths) { + LOG_INF(" running %s\n", path.c_str()); analyze_template(path); } - LOG_ERR("\n"); - LOG_ERR("%s", ANSI_GREEN); - LOG_ERR("================================================================================\n"); - LOG_ERR(" ANALYSIS COMPLETE\n"); - LOG_ERR("================================================================================\n"); - LOG_ERR("%s", ANSI_RESET); + LOG_INF("\n"); + LOG_INF("%s", ANSI_GREEN); + LOG_INF("================================================================================\n"); + LOG_INF(" ANALYSIS COMPLETE\n"); + LOG_INF("================================================================================\n"); + LOG_INF("%s", ANSI_RESET); + LOG("%s: %s\n", "test-chat-analysis", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-chat-auto-parser.cpp b/tests/test-chat-auto-parser.cpp index 5aa9482512..c64396fdcf 100644 --- a/tests/test-chat-auto-parser.cpp +++ b/tests/test-chat-auto-parser.cpp @@ -4,6 +4,8 @@ #include "chat.h" #include "gguf.h" #include "jinja/runtime.h" +#include "arg.h" +#include "common.h" #include "log.h" #include "peg-parser.h" #include "testing.h" @@ -16,6 +18,7 @@ #include #include #include +#include using namespace autoparser; @@ -175,24 +178,24 @@ static std::string read_gguf_chat_template(const std::string & path) { } static void print_usage(const char * program_name) { - LOG_ERR("Test the chat template auto-parser; also usable as a debug tool that shows the generated PEG parser, GBNF grammar and triggers for a given template.\n"); - LOG_ERR("\nUsage: %s [filter_regex] run the automated tests (default)\n", program_name); - LOG_ERR(" %s [options] debug a single template\n", program_name); - LOG_ERR("\nDebug mode options:\n"); - LOG_ERR(" --no-tools Disable tool definitions\n"); - LOG_ERR(" --force-tool-call Set tool calls to forced\n"); - LOG_ERR(" --parallel-tool-calls=0|1 Set parallel_tool_calls (default: 1)\n"); - LOG_ERR(" --generation-prompt=0|1 Set add_generation_prompt (default: 1)\n"); - LOG_ERR(" --enable-reasoning=0|1 Enable reasoning parsing (default: 1)\n"); - LOG_ERR(" --output=MODE Output mode: analysis, template, both (default: both)\n"); - LOG_ERR(" --debug-jinja Enable Jinja fine-grained debug\n"); - LOG_ERR(" --input-message=TYPE Message type to render:\n"); - LOG_ERR(" content_only, reasoning_content, tool_call_only,\n"); - LOG_ERR(" content_tool_call, reasoning_tool_call,\n"); - LOG_ERR(" content_fake_tool_call, all\n"); - LOG_ERR("\nExamples:\n"); - LOG_ERR(" %s template.jinja --input-message=all --generation-prompt=1\n", program_name); - LOG_ERR(" %s template.jinja --output=template --input-message=tool_call_only\n", program_name); + LOG_INF("Test the chat template auto-parser; also usable as a debug tool that shows the generated PEG parser, GBNF grammar and triggers for a given template.\n"); + LOG_INF("\nUsage: %s [filter_regex] run the automated tests (default)\n", program_name); + LOG_INF(" %s [options] debug a single template\n", program_name); + LOG_INF("\nDebug mode options:\n"); + LOG_INF(" --no-tools Disable tool definitions\n"); + LOG_INF(" --force-tool-call Set tool calls to forced\n"); + LOG_INF(" --parallel-tool-calls=0|1 Set parallel_tool_calls (default: 1)\n"); + LOG_INF(" --generation-prompt=0|1 Set add_generation_prompt (default: 1)\n"); + LOG_INF(" --enable-reasoning=0|1 Enable reasoning parsing (default: 1)\n"); + LOG_INF(" --output=MODE Output mode: analysis, template, both (default: both)\n"); + LOG_INF(" --debug-jinja Enable Jinja fine-grained debug\n"); + LOG_INF(" --input-message=TYPE Message type to render:\n"); + LOG_INF(" content_only, reasoning_content, tool_call_only,\n"); + LOG_INF(" content_tool_call, reasoning_tool_call,\n"); + LOG_INF(" content_fake_tool_call, all\n"); + LOG_INF("\nExamples:\n"); + LOG_INF(" %s template.jinja --input-message=all --generation-prompt=1\n", program_name); + LOG_INF(" %s template.jinja --output=template --input-message=tool_call_only\n", program_name); } static bool parse_bool_option(const std::string & value) { @@ -336,8 +339,8 @@ static void render_scenario(const common_chat_template & tmpl, const json & tools, bool add_generation_prompt, bool enable_thinking) { - LOG_ERR("\n=== Scenario: %s ===\n", scenario_name.c_str()); - LOG_ERR("add_generation_prompt: %s, enable_thinking: %s\n", add_generation_prompt ? "true" : "false", + LOG_INF("\n=== Scenario: %s ===\n", scenario_name.c_str()); + LOG_INF("add_generation_prompt: %s, enable_thinking: %s\n", add_generation_prompt ? "true" : "false", enable_thinking ? "true" : "false"); // When add_generation_prompt is true, add a trailing user message to trigger the prompt @@ -349,7 +352,7 @@ static void render_scenario(const common_chat_template & tmpl, }); } - LOG_ERR("Messages:\n%s\n", final_messages.dump(2).c_str()); + LOG_TRC("Messages:\n%s\n", final_messages.dump(2).c_str()); try { generation_params inputs; @@ -363,9 +366,9 @@ static void render_scenario(const common_chat_template & tmpl, std::string output = common_chat_template_direct_apply(tmpl, inputs); - LOG_ERR("\n--- Rendered Output ---\n"); - LOG_ERR("%s\n", output.c_str()); - LOG_ERR("--- End Output (length: %zu) ---\n", output.length()); + LOG_INF("\n--- Rendered Output ---\n"); + LOG_INF("%s\n", output.c_str()); + LOG_INF("--- End Output (length: %zu) ---\n", output.length()); } catch (const std::exception & e) { LOG_ERR("Rendering failed: %s\n", e.what()); } @@ -395,7 +398,7 @@ static void render_all_scenarios(const common_chat_template & tmpl, // Also render with add_generation_prompt=true to show the prompt ending if (message_type == input_message_type::ALL) { - LOG_ERR("\n\n=== Generation Prompt Scenarios (add_generation_prompt=true) ===\n"); + LOG_INF("\n\n=== Generation Prompt Scenarios (add_generation_prompt=true) ===\n"); json prompt_messages = json::array({ user_msg }); render_scenario(tmpl, "generation_prompt_only", prompt_messages, tools, true, enable_thinking); @@ -438,8 +441,8 @@ static int debug_single_template(const debug_options & opts) { return 1; } - LOG_ERR("Analyzing template: %s\n", opts.template_path.c_str()); - LOG_ERR("Options: with_tools=%s, generation_prompt=%s, enable_reasoning=%s\n", opts.with_tools ? "true" : "false", + LOG_INF("Analyzing template: %s\n", opts.template_path.c_str()); + LOG_INF("Options: with_tools=%s, generation_prompt=%s, enable_reasoning=%s\n", opts.with_tools ? "true" : "false", opts.generation_prompt ? "true" : "false", opts.enable_reasoning ? "true" : "false"); try { @@ -451,17 +454,17 @@ static int debug_single_template(const debug_options & opts) { common_chat_params parser_data; if (std::optional spec_tmpl = common_chat_try_specialized_template(chat_template, template_source, params)) { - LOG_ERR("\n"); - LOG_ERR("This template uses a specialized parser, analysis results will not be available.\n"); + LOG_INF("\n"); + LOG_INF("This template uses a specialized parser, analysis results will not be available.\n"); parser_data = *spec_tmpl; } else { // Render template scenarios if requested if (opts.input_message != input_message_type::NONE && (opts.mode == output_mode::TEMPLATE || opts.mode == output_mode::BOTH)) { - LOG_ERR("\n"); - LOG_ERR("================================================================================\n"); - LOG_ERR(" TEMPLATE RENDERING OUTPUT\n"); - LOG_ERR("================================================================================\n"); + LOG_INF("\n"); + LOG_INF("================================================================================\n"); + LOG_INF(" TEMPLATE RENDERING OUTPUT\n"); + LOG_INF("================================================================================\n"); render_all_scenarios(chat_template, tools, opts.generation_prompt, opts.enable_reasoning, opts.input_message); @@ -469,10 +472,10 @@ static int debug_single_template(const debug_options & opts) { // Output analysis if requested if (opts.mode == output_mode::ANALYSIS || opts.mode == output_mode::BOTH) { - LOG_ERR("\n"); - LOG_ERR("================================================================================\n"); - LOG_ERR(" TEMPLATE ANALYSIS\n"); - LOG_ERR("================================================================================\n"); + LOG_INF("\n"); + LOG_INF("================================================================================\n"); + LOG_INF(" TEMPLATE ANALYSIS\n"); + LOG_INF("================================================================================\n"); struct autoparser analysis; analysis.analyze_template(chat_template); @@ -483,25 +486,25 @@ static int debug_single_template(const debug_options & opts) { } if (!std::empty(parser_data.parser)) { - LOG_ERR("\n=== Generated Parser ===\n"); + LOG_INF("\n=== Generated Parser ===\n"); common_peg_arena arena; arena.load(parser_data.parser); - LOG_ERR("%s\n", arena.dump(arena.root()).c_str()); + LOG_INF("%s\n", arena.dump(arena.root()).c_str()); - LOG_ERR("\n=== Generated Grammar ===\n"); - LOG_ERR("%s\n", parser_data.grammar.c_str()); + LOG_INF("\n=== Generated Grammar ===\n"); + LOG_INF("%s\n", parser_data.grammar.c_str()); - LOG_ERR("\n=== Generated Lazy Grammar ===\n"); - LOG_ERR("%d\n", parser_data.grammar_lazy); + LOG_INF("\n=== Generated Lazy Grammar ===\n"); + LOG_INF("%d\n", parser_data.grammar_lazy); - LOG_ERR("\n=== Generated Grammar Triggers ===\n"); + LOG_INF("\n=== Generated Grammar Triggers ===\n"); for (const common_grammar_trigger & cgt : parser_data.grammar_triggers) { - LOG_ERR("Token: %d | Type: %d | Value: %s\n", cgt.token, cgt.type, cgt.value.c_str()); + LOG_INF("Token: %d | Type: %d | Value: %s\n", cgt.token, cgt.type, cgt.value.c_str()); } - LOG_ERR("\n=== Preserved Tokens ===\n"); + LOG_INF("\n=== Preserved Tokens ===\n"); for (const std::string & token : parser_data.preserved_tokens) { - LOG_ERR(" '%s'\n", token.c_str()); + LOG_INF(" '%s'\n", token.c_str()); } } } catch (const std::exception & e) { @@ -513,20 +516,59 @@ static int debug_single_template(const debug_options & opts) { } int main(int argc, char * argv[]) { - if (argc > 1) { - std::string arg = argv[1]; + common_params params; + params.model.path = "."; // placeholder so common_params_parse does not require --model + common_init(); + + // this tool handles -h/--help, its debug options and one positional argument + // (template path or test filter) itself; every other option goes to the common parser + std::vector common_argv; + std::vector own_argv; + common_argv.push_back(argv[0]); + own_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { + const std::string arg = argv[i]; + if (arg == "-h" || arg == "--help" || + arg == "--force-tool-call" || arg == "--debug-jinja" || arg == "--no-tools" || + arg.rfind("--parallel-tool-calls=", 0) == 0 || + arg.rfind("--generation-prompt=", 0) == 0 || + arg.rfind("--enable-reasoning=", 0) == 0 || + arg.rfind("--output=", 0) == 0 || + arg.rfind("--input-message=", 0) == 0) { + own_argv.push_back(argv[i]); + } else if (arg[0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else { + own_argv.push_back(argv[i]); // positional: template path or test filter + } + } + common_argv.push_back(nullptr); + own_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + // capture all output unless the user asked for a lower verbosity threshold + const int own_verbosity = params.verbosity < LOG_DEFAULT_LLAMA ? params.verbosity : 99; + const int own_argc = (int) own_argv.size() - 1; + char ** own_argv_data = own_argv.data(); + + if (own_argc > 1) { + std::string arg = own_argv_data[1]; if (arg == "-h" || arg == "--help") { - common_log_set_verbosity_thold(99); - print_usage(argv[0]); + common_log_set_verbosity_thold(own_verbosity); + print_usage(own_argv_data[0]); + common_log_flush(common_log_main()); return 0; } // debug mode: if the first argument is an existing file, analyze that template instead of running the automated tests if (std::filesystem::is_regular_file(arg)) { - common_log_set_verbosity_thold(99); + common_log_set_verbosity_thold(own_verbosity); debug_options opts; - if (!parse_debug_options(argc, argv, opts)) { + if (!parse_debug_options(own_argc, own_argv_data, opts)) { + common_log_flush(common_log_main()); return 1; } @@ -534,17 +576,21 @@ int main(int argc, char * argv[]) { jinja::enable_debug(true); } - return debug_single_template(opts); + const int rc = debug_single_template(opts); + common_log_flush(common_log_main()); + return rc; } } + LOG("%s: running\n", "test-chat-auto-parser"); + testing t(std::cout); t.verbose = true; // usage: test-chat-auto-parser [filter_regex] - if (argc > 1) { - t.set_filter(argv[1]); + if (own_argc > 1) { + t.set_filter(own_argv_data[1]); } t.test("diff_split", test_calculate_diff_split); @@ -565,7 +611,10 @@ int main(int argc, char * argv[]) { t.test("bailing_v3", test_bailing_v3_tool_format); t.test("role_markers_all_templates", test_role_markers_all_templates); - return t.summary(); + const int rc = t.summary(); + LOG("%s: %s\n", "test-chat-auto-parser", rc == 0 ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); + return rc; } static void test_marker_separation(testing & t) { diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cpp index 9d15796f7a..3226b014da 100644 --- a/tests/test-chat-peg-parser.cpp +++ b/tests/test-chat-peg-parser.cpp @@ -1,7 +1,9 @@ +#include "arg.h" #include "chat-peg-parser.h" #include "chat.h" #include "common.h" #include "json-schema-to-grammar.h" +#include "log.h" #include "peg-parser.h" #include "testing.h" #include "peg-parser/simple-tokenize.h" @@ -10,6 +12,7 @@ #include #include #include +#include #include "json.h" @@ -25,9 +28,31 @@ static void test_tagged_peg_parser(testing & t); static void test_permute(testing & t); int main(int argc, char * argv[]) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + + // this test takes an optional filter as its only positional argument + std::string filter; + std::vector common_argv; + common_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { + if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else if (filter.empty()) { + filter = argv[i]; + } + } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-chat-peg-parser"); + testing t(std::cout); - if (argc >= 2) { - t.set_filter(argv[1]); + if (!filter.empty()) { + t.set_filter(filter); } const char * verbose = getenv("LLAMA_TEST_VERBOSE"); @@ -43,7 +68,10 @@ int main(int argc, char * argv[]) { t.test("tagged peg parser", test_tagged_peg_parser); t.test("permute", test_permute); - return t.summary(); + const int rc = t.summary(); + LOG("%s: %s\n", "test-chat-peg-parser", rc == 0 ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); + return rc; } static json create_tools() { diff --git a/tests/test-chat-template.cpp b/tests/test-chat-template.cpp index a477180cd0..df00332a2a 100644 --- a/tests/test-chat-template.cpp +++ b/tests/test-chat-template.cpp @@ -13,7 +13,9 @@ #include #include "llama.h" +#include "arg.h" #include "common.h" +#include "log.h" #include "chat.h" #include "jinja/runtime.h" #include "jinja/parser.h" @@ -113,6 +115,10 @@ static std::string DEFAULT_JSON_WITH_TOOLS = R"({ int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + std::vector args(argv, argv + argc); std::string tmpl_path; @@ -123,6 +129,8 @@ int main(int argc, char ** argv) { bool use_common = true; bool dump_prog = false; + std::vector common_argv; + common_argv.push_back(argv[0]); for (size_t i = 1; i < args.size(); i++) { if (args[i] == "--help" || args[i] == "-h") { std::cout << HELP << "\n"; @@ -142,6 +150,8 @@ int main(int argc, char ** argv) { use_common = false; } else if (args[i] == "--dump-prog") { dump_prog = true; + } else if (args[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it } else if (tmpl_path.empty()) { tmpl_path = args[i]; } else { @@ -150,16 +160,27 @@ int main(int argc, char ** argv) { return 1; } } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-chat-template"); if (tmpl_path.empty()) { - return main_automated_tests(); + const int rc = main_automated_tests(); + LOG("%s: %s\n", "test-chat-template", rc == 0 ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); + return rc; } json input_json; if (!json_path.empty()) { std::ifstream json_file(json_path); if (!json_file) { - std::cerr << "Error: Could not open JSON file: " << json_path << "\n"; + LOG_ERR("Error: Could not open JSON file: %s\n", json_path.c_str()); + LOG("%s: %s\n", "test-chat-template", "FAILED"); + common_log_flush(common_log_main()); return 1; } std::string content = std::string( @@ -180,10 +201,14 @@ int main(int argc, char ** argv) { std::istreambuf_iterator()); run_single(contents, input_json, use_common, dump_prog, output_path); } else { - std::cerr << "Error: PATH_TO_TEMPLATE is not a valid file or directory: " << tmpl_path << "\n"; + LOG_ERR("Error: PATH_TO_TEMPLATE is not a valid file or directory: %s\n", tmpl_path.c_str()); + LOG("%s: %s\n", "test-chat-template", "FAILED"); + common_log_flush(common_log_main()); return 1; } + LOG("%s: %s\n", "test-chat-template", "PASSED"); + common_log_flush(common_log_main()); return 0; } @@ -197,7 +222,7 @@ void run_multiple(const std::string& dir_path, bool stop_on_first_fail, const js // only process .jinja files if (entry.path().extension() == ".jinja" && entry.is_regular_file()) { test_count++; - std::cout << "\n\n=== RUNNING TEMPLATE FILE: " << entry.path().string() << " ===\n"; + LOG_INF("=== RUNNING TEMPLATE FILE: %s ===\n", entry.path().string().c_str()); std::ifstream infile(entry.path()); std::string contents((std::istreambuf_iterator(infile)), std::istreambuf_iterator()); try { @@ -662,9 +687,9 @@ int main_automated_tests(void) { assert(res > 0); supported_tmpl.resize(res); res = llama_chat_builtin_templates(supported_tmpl.data(), supported_tmpl.size()); - std::cout << "Built-in chat templates:\n"; + LOG_INF("Built-in chat templates:\n"); for (const auto *tmpl : supported_tmpl) { - std::cout << " " << tmpl << "\n"; + LOG_CNT(" %s\n", tmpl); } // test invalid chat template @@ -673,7 +698,7 @@ int main_automated_tests(void) { const auto add_generation_prompt = true; for (const auto & test_case : test_cases) { - std::cout << "\n\n=== " << test_case.name << " ===\n\n"; + LOG_INF("=== %s ===\n", test_case.name.c_str()); auto conv = conversation; conv.insert(conv.end(), test_case.extra_conversation.begin(), test_case.extra_conversation.end()); formatted_chat.resize(2048); @@ -705,7 +730,7 @@ int main_automated_tests(void) { if (!test_case.supported_with_jinja) { continue; } - std::cout << "\n\n=== " << test_case.name << " (jinja) ===\n\n"; + LOG_INF("=== %s (jinja) ===\n", test_case.name.c_str()); try { auto msgs = messages; for (const auto & msg : test_case.extra_conversation) { diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 30a7237e31..d937293751 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -10,6 +10,7 @@ #include "../tools/server/server-chat.h" #include "chat-auto-parser.h" #include "chat.h" +#include "arg.h" #include "common.h" #include "ggml.h" #include "log.h" @@ -1118,6 +1119,8 @@ static bool g_force_reconstruction_test = false; static void test_peg_parser(common_chat_templates * tmpls, const std::function & init, bool detailed_debug) { + const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN; + // UTF-8-safe truncation helper (same as in test_parser_with_streaming) constexpr auto utf8_truncate_safe_len = [](const std::string_view s) -> size_t { auto len = s.size(); @@ -1442,10 +1445,11 @@ static void test_peg_parser(common_chat_templates * tmpls, size_t ctx_start = diff_pos > 60 ? diff_pos - 60 : 0; size_t ctx_end_e = std::min(expected_text.size(), diff_pos + 40); size_t ctx_end_r = std::min(reconstruction_params.prompt.size(), diff_pos + 40); - LOG_ERR("\x1b[31m[RECONSTRUCTION FAIL]\x1b[0m " + LOG_ERR("%s[RECONSTRUCTION FAIL]%s " "first diff at byte %zu (expected len=%zu, reconstructed len=%zu)\n" " expected: ...%s...\n" " reconstructed: ...%s...\n", + use_color ? "\x1b[31m" : "", use_color ? "\x1b[0m" : "", diff_pos, expected_text.size(), reconstruction_params.prompt.size(), expected_text.substr(ctx_start, ctx_end_e - ctx_start).c_str(), reconstruction_params.prompt.substr(ctx_start, ctx_end_r - ctx_start).c_str()); @@ -1461,6 +1465,18 @@ static void test_peg_parser(common_chat_templates * tmpls, } } +// Case-insensitive substring match against the --template filter +static bool template_matches_filter(const std::string & template_path) { + if (g_template_filter.empty()) { + return true; + } + std::string template_path_lower = template_path; + std::string filter_lower = g_template_filter; + std::transform(template_path_lower.begin(), template_path_lower.end(), template_path_lower.begin(), ::tolower); + std::transform(filter_lower.begin(), filter_lower.end(), filter_lower.begin(), ::tolower); + return template_path_lower.find(filter_lower) != std::string::npos; +} + // Fluent builder for PEG parser tests class peg_test_builder; @@ -1474,7 +1490,11 @@ class peg_tester { explicit peg_tester(const std::string & template_path, const bool detailed_debug = false) : tmpls_(read_templates(template_path)), template_path_(template_path), - detailed_debug_(detailed_debug) {} + detailed_debug_(detailed_debug) { + if (template_matches_filter(template_path_)) { + LOG_INF(" running %s\n", template_path_.c_str()); + } + } const std::string & template_path() const { return template_path_; } @@ -1571,19 +1591,11 @@ class peg_test_builder { // Execute the test void run() { // Check template filter - if (!g_template_filter.empty()) { - // Case-insensitive substring match - std::string template_path_lower = tester_.template_path(); - std::string filter_lower = g_template_filter; - std::transform(template_path_lower.begin(), template_path_lower.end(), template_path_lower.begin(), - ::tolower); - std::transform(filter_lower.begin(), filter_lower.end(), filter_lower.begin(), ::tolower); - if (template_path_lower.find(filter_lower) == std::string::npos) { - // Skip this test - return; - } + if (!template_matches_filter(tester_.template_path())) { + // Skip this test + return; } - LOG_INF("\n\x1b[38;5;126m[%s]\x1b[0m\n%s\n\n", tester_.template_path().c_str(), tc_.input.c_str()); + LOG_TRC("\n\x1b[38;5;126m[%s]\x1b[0m\n%s\n\n", tester_.template_path().c_str(), tc_.input.c_str()); test_peg_parser(tester_.tmpls_.get(), [this](peg_test_case & t) { t = tc_; }, tester_.detailed_debug_); } }; @@ -7017,7 +7029,7 @@ static void test_developer_role_to_system_workaround() { if (params.prompt.find("<|system|>You are a helpful developer assistant.<|end|>") == std::string::npos) { throw std::runtime_error("Test failed: system message not found in output"); } - LOG_ERR("Test 1 passed: developer role changed to system\n"); + LOG_INF("Test 1 passed: developer role changed to system\n"); } } @@ -7321,42 +7333,61 @@ static void test_msg_diffs_compute() { } int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + bool detailed_debug = false; bool only_run_filtered = false; // Check for --template and --detailed flags + std::vector positional; + std::vector common_argv; + common_argv.push_back(argv[0]); for (int i = 1; i < argc; i++) { std::string arg = argv[i]; - if (arg == "--template" && i + 1 < argc) { - g_template_filter = argv[++i]; - // Only run PEG parser tests with the filter - only_run_filtered = true; - } - if (arg == "--detailed") { + if (arg == "--template") { + if (i + 1 < argc) { + g_template_filter = argv[++i]; + // Only run PEG parser tests with the filter + only_run_filtered = true; + } + } else if (arg == "--detailed") { detailed_debug = true; - common_log_set_verbosity_thold(999); - } - if (arg == "--force-reconstruction-test") { + } else if (arg == "--force-reconstruction-test") { g_force_reconstruction_test = true; only_run_filtered = true; + } else if (arg[0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else { + positional.push_back(arg); // .jinja template path } } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + // keep the detailed output unless the user asked for a lower verbosity threshold + if (detailed_debug) { + common_log_set_verbosity_thold(params.verbosity < LOG_DEFAULT_LLAMA ? params.verbosity : 999); + } + + LOG("%s: running\n", "test-chat"); if (only_run_filtered) { test_template_output_peg_parsers(detailed_debug); std::cout << "\n[chat] All template tests passed!" << '\n'; + LOG("%s: %s\n", "test-chat", "PASSED"); + common_log_flush(common_log_main()); return 0; } #ifndef _WIN32 // Check if any argument is a .jinja file (for template format detection mode) bool has_jinja_files = false; - for (int i = 1; i < argc; i++) { - std::string arg = argv[i]; - if (arg == "--detailed") { - continue; - } - if (arg.size() >= 6 && arg.rfind(".jinja") == arg.size() - 6) { + for (const auto & path : positional) { + if (path.size() >= 6 && path.rfind(".jinja") == path.size() - 6) { has_jinja_files = true; break; } @@ -7373,9 +7404,8 @@ int main(int argc, char ** argv) { std::cout << "| Template | Format |\n"; std::cout << "|----------|--------|\n"; - for (int i = 1; i < argc; i++) { + for (const auto & path : positional) { try { - std::string path = argv[i]; if (path.rfind(".jinja") != path.size() - 6) { std::cerr << "Skipping non-jinja file: " << path << '\n'; continue; @@ -7386,7 +7416,7 @@ int main(int argc, char ** argv) { const auto * format = common_chat_format_name(common_chat_templates_apply(tmpls.get(), inputs).format); std::cout << "| " << name << " | " << format << " |\n"; } catch (const std::exception & e) { - std::cerr << "Failed to process " << argv[i] << ": " << e.what() << '\n'; + std::cerr << "Failed to process " << path << ": " << e.what() << '\n'; } } } else @@ -7407,5 +7437,7 @@ int main(int argc, char ** argv) { test_template_output_peg_parsers(detailed_debug); std::cout << "\n[chat] All tests passed!" << '\n'; } + LOG("%s: %s\n", "test-chat", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-col2im-1d.cpp b/tests/test-col2im-1d.cpp index f1d36479b3..154ab5594c 100644 --- a/tests/test-col2im-1d.cpp +++ b/tests/test-col2im-1d.cpp @@ -13,6 +13,10 @@ #include "ggml.h" #include "ggml-cpu.h" +#include "arg.h" +#include "common.h" +#include "log.h" + #include #include #include @@ -86,7 +90,16 @@ static double nmse_cropped(const float * y, const float * ref, int64_t T_out, in return num / (den + 1e-30); } -int main(void) { +int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-col2im-1d"); + int fails = 0; for (const col2im_case & c : CASES) { @@ -148,12 +161,16 @@ int main(void) { if (!ok) { fails++; } - printf("col2im_1d K=%2d OC=%2d T_in=%3d s0=%d p0=%d: nmse f32=%.2e f16=%.2e bf16=%.2e %s\n", + LOG_INF("col2im_1d K=%2d OC=%2d T_in=%3d s0=%d p0=%d: nmse f32=%.2e f16=%.2e bf16=%.2e %s\n", (int) c.K, (int) c.OC, (int) c.T_in, c.s0, c.p0, e32, e16, ebf, ok ? "OK" : "FAIL"); ggml_free(ctx); } - printf(fails == 0 ? "all col2im_1d checks passed\n" : "%d col2im_1d checks FAILED\n", fails); - return fails == 0 ? 0 : 1; + const bool ok = fails == 0; + + printf(ok ? "all col2im_1d checks passed\n" : "%d col2im_1d checks FAILED\n", fails); + LOG("%s: %s\n", "test-col2im-1d", ok ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); + return ok ? 0 : 1; } diff --git a/tests/test-export-graph-ops.cpp b/tests/test-export-graph-ops.cpp index 46ded13985..f30917d380 100644 --- a/tests/test-export-graph-ops.cpp +++ b/tests/test-export-graph-ops.cpp @@ -135,6 +135,8 @@ int main(int argc, char ** argv) { return 1; } + LOG("%s: running\n", "test-export-graph-ops"); + // Load CPU-only ggml_backend_dev_t cpu_device = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); params.devices = { cpu_device, nullptr }; @@ -154,6 +156,8 @@ int main(int argc, char ** argv) { ctx = init_result->context(); if (!ctx) { LOG_ERR("failed to initialize params\n"); + LOG("%s: %s\n", "test-export-graph-ops", "FAILED"); + common_log_flush(common_log_main()); return 1; } } else { @@ -166,6 +170,8 @@ int main(int argc, char ** argv) { gguf_context_ptr gguf_ctx = gguf_fetch_gguf_ctx(hf_repo, hf_quant); if (!gguf_ctx) { LOG_ERR("failed to fetch GGUF metadata from %s\n", hf_repo.c_str()); + LOG("%s: %s\n", "test-export-graph-ops", "FAILED"); + common_log_flush(common_log_main()); return 1; } @@ -177,6 +183,8 @@ int main(int argc, char ** argv) { if (!model) { LOG_ERR("failed to create llama_model from %s\n", hf_repo.c_str()); + LOG("%s: %s\n", "test-export-graph-ops", "FAILED"); + common_log_flush(common_log_main()); return 1; } @@ -186,10 +194,14 @@ int main(int argc, char ** argv) { if (!ctx) { LOG_ERR("failed to create llama_context\n"); + LOG("%s: %s\n", "test-export-graph-ops", "FAILED"); + common_log_flush(common_log_main()); return 1; } #else LOG_ERR("test-export-graph-ops compiled without HF fetch support\n"); + LOG("%s: %s\n", "test-export-graph-ops", "FAILED"); + common_log_flush(common_log_main()); return 1; #endif } @@ -197,11 +209,15 @@ int main(int argc, char ** argv) { const uint32_t n_seqs = llama_n_seq_max(ctx); const uint32_t n_tokens = std::min(llama_n_ctx(ctx), llama_n_ubatch(ctx)); + LOG_INF(" running %s\n", params.model.get_name().c_str()); + std::set tests; auto * gf_pp = llama_graph_reserve(ctx, n_tokens, n_seqs, n_tokens); if (!gf_pp) { LOG_ERR("failed to reserve prompt processing graph\n"); + LOG("%s: %s\n", "test-export-graph-ops", "FAILED"); + common_log_flush(common_log_main()); return 1; } extract_graph_ops(gf_pp, "pp", tests); @@ -209,6 +225,8 @@ int main(int argc, char ** argv) { auto * gf_tg = llama_graph_reserve(ctx, n_seqs, n_seqs, n_seqs); if (!gf_tg) { LOG_ERR("failed to reserve token generation graph\n"); + LOG("%s: %s\n", "test-export-graph-ops", "FAILED"); + common_log_flush(common_log_main()); return 1; } extract_graph_ops(gf_tg, "tg", tests); @@ -219,6 +237,8 @@ int main(int argc, char ** argv) { if (!f.is_open()) { LOG_ERR("unable to open output file: %s\n", params.out_file.c_str()); + LOG("%s: %s\n", "test-export-graph-ops", "FAILED"); + common_log_flush(common_log_main()); return 1; } @@ -226,5 +246,7 @@ int main(int argc, char ** argv) { test.serialize(f); } + LOG("%s: %s\n", "test-export-graph-ops", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-fusion.cpp b/tests/test-fusion.cpp index 467248f0f1..a524b6c024 100644 --- a/tests/test-fusion.cpp +++ b/tests/test-fusion.cpp @@ -16,6 +16,7 @@ // test-fusion --models DIR --device MTL0 --check baseline.csv # validate against it // test-fusion --model FILE --device MTL0 --check baseline.csv # validate a single model +#include "arg.h" #include "common.h" #include "log.h" #include "llama-cpp.h" @@ -224,17 +225,24 @@ static void usage(const char * argv0) { } int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes its models from --models/--model + common_init(); + std::string models_dir; std::string model_file; std::string device_name; std::string record_path; std::string check_path; + std::vector common_argv; + common_argv.push_back(argv[0]); for (int i = 1; i < argc; i++) { const std::string arg = argv[i]; const auto next = [&](const char * name) -> std::string { if (i + 1 >= argc) { LOG_ERR("%s: %s requires an argument\n", __func__, name); + common_log_flush(common_log_main()); exit(1); } return argv[++i]; @@ -248,39 +256,56 @@ int main(int argc, char ** argv) { else if (arg == "--device"){ device_name = next("--device"); } else if (arg == "--record"){ record_path = next("--record"); } else if (arg == "--check") { check_path = next("--check"); } - else { + else if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else { LOG_ERR("%s: unknown argument: %s\n", __func__, arg.c_str()); + common_log_flush(common_log_main()); return 1; } } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } if (device_name.empty()) { LOG_ERR("%s: --device NAME is required\n", __func__); + common_log_flush(common_log_main()); return 1; } if (models_dir.empty() && model_file.empty()) { LOG_ERR("%s: --models DIR or --model FILE is required\n", __func__); + common_log_flush(common_log_main()); return 1; } if (!models_dir.empty() && !model_file.empty()) { LOG_ERR("%s: --models DIR and --model FILE are mutually exclusive\n", __func__); + common_log_flush(common_log_main()); return 1; } if (!record_path.empty() && !check_path.empty()) { LOG_ERR("%s: --record and --check are mutually exclusive\n", __func__); + common_log_flush(common_log_main()); return 1; } + LOG("%s: running\n", "test-fusion"); + std::vector models; if (!model_file.empty()) { if (!std::filesystem::is_regular_file(model_file)) { LOG_ERR("%s: model file '%s' does not exist\n", __func__, model_file.c_str()); + LOG("%s: %s\n", "test-fusion", "FAILED"); + common_log_flush(common_log_main()); return 1; } models.push_back(model_file); } else { if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) { LOG_ERR("%s: models directory '%s' does not exist\n", __func__, models_dir.c_str()); + LOG("%s: %s\n", "test-fusion", "FAILED"); + common_log_flush(common_log_main()); return 1; } for (const auto & entry : std::filesystem::directory_iterator(models_dir)) { @@ -292,17 +317,20 @@ int main(int argc, char ** argv) { if (models.empty()) { LOG_ERR("%s: no .gguf models found in '%s'\n", __func__, models_dir.c_str()); + LOG("%s: %s\n", "test-fusion", "FAILED"); + common_log_flush(common_log_main()); return 1; } } - common_init(); ggml_backend_load_all(); ggml_backend_dev_t dev = ggml_backend_dev_by_name(device_name.c_str()); if (!dev) { LOG_WRN("%s: device '%s' not found - skipping (baseline is device-specific)\n", __func__, device_name.c_str()); + LOG("%s: %s\n", "test-fusion", "PASSED"); + common_log_flush(common_log_main()); return 0; } @@ -324,6 +352,8 @@ int main(int argc, char ** argv) { LOG_ERR("%s: device '%s' does not export the generic fusion debugging API " "(ggml_backend_fusion_*) - cannot run the fusion regression test\n", __func__, device_name.c_str()); + LOG("%s: %s\n", "test-fusion", "FAILED"); + common_log_flush(common_log_main()); return 1; } @@ -340,6 +370,8 @@ int main(int argc, char ** argv) { std::ifstream in(check_path); if (!in) { LOG_ERR("%s: cannot open baseline '%s'\n", __func__, check_path.c_str()); + LOG("%s: %s\n", "test-fusion", "FAILED"); + common_log_flush(common_log_main()); return 1; } std::string line; @@ -368,6 +400,8 @@ int main(int argc, char ** argv) { const size_t seed = 1; for (const auto & model_path : models) { + LOG_INF(" running %s\n", model_path.c_str()); + const std::string arch = get_arch(model_path); const bool moe = arch.find("moe") != std::string::npos; @@ -560,6 +594,8 @@ int main(int argc, char ** argv) { __func__, models_dir.c_str(), argv[0], device_name.c_str(), models_dir.c_str(), check_path.c_str()); } + LOG("%s: %s\n", "test-fusion", n_bad == 0 ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); return n_bad; } } diff --git a/tests/test-gbnf-validator.cpp b/tests/test-gbnf-validator.cpp index 6547eec32f..a0411502c7 100644 --- a/tests/test-gbnf-validator.cpp +++ b/tests/test-gbnf-validator.cpp @@ -1,6 +1,9 @@ #include "../src/unicode.h" #include "../src/llama-grammar.h" +#include "arg.h" +#include "log.h" + #include #include #include @@ -37,33 +40,61 @@ static bool llama_grammar_validate(struct llama_grammar * grammar, const std::st } static void print_error_message(const std::string & input_str, size_t error_pos, const std::string & error_msg) { + // no ANSI escapes at WARN or lower (--errors-only) + const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN; + const char * const col_pos = use_color ? "\033[1;31m" : ""; + const char * const col_rest = use_color ? "\033[0;31m" : ""; + const char * const col_end = use_color ? "\033[0m" : ""; fprintf(stdout, "Input string is invalid according to the grammar.\n"); fprintf(stdout, "Error: %s at position %zu\n", error_msg.c_str(), error_pos); fprintf(stdout, "\n"); fprintf(stdout, "Input string:\n"); fprintf(stdout, "%s", input_str.substr(0, error_pos).c_str()); if (error_pos < input_str.size()) { - fprintf(stdout, "\033[1;31m%c", input_str[error_pos]); + fprintf(stdout, "%s%c", col_pos, input_str[error_pos]); if (error_pos+1 < input_str.size()) { - fprintf(stdout, "\033[0;31m%s", input_str.substr(error_pos+1).c_str()); + fprintf(stdout, "%s%s", col_rest, input_str.substr(error_pos+1).c_str()); } - fprintf(stdout, "\033[0m\n"); + fprintf(stdout, "%s\n", col_end); } } int main(int argc, char** argv) { - if (argc != 3) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + + std::vector positional; + std::vector common_argv; + common_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { + if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else { + positional.push_back(argv[i]); + } + } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + if (positional.size() != 2) { fprintf(stdout, "Usage: %s \n", argv[0]); return 1; } - const std::string grammar_filename = argv[1]; - const std::string input_filename = argv[2]; + LOG("%s: running\n", "test-gbnf-validator"); + + const std::string grammar_filename = positional[0]; + const std::string input_filename = positional[1]; // Read the GBNF grammar file FILE* grammar_file = fopen(grammar_filename.c_str(), "r"); if (!grammar_file) { fprintf(stdout, "Failed to open grammar file: %s\n", grammar_filename.c_str()); + LOG("%s: %s\n", "test-gbnf-validator", "FAILED"); + common_log_flush(common_log_main()); return 1; } @@ -79,6 +110,8 @@ int main(int argc, char** argv) { llama_grammar * grammar = llama_grammar_init_impl(nullptr, grammar_str.c_str(), "root", false, nullptr, 0, nullptr, 0); if (grammar == nullptr) { fprintf(stdout, "Failed to initialize llama_grammar\n"); + LOG("%s: %s\n", "test-gbnf-validator", "FAILED"); + common_log_flush(common_log_main()); return 1; } // Read the input file @@ -105,5 +138,8 @@ int main(int argc, char** argv) { // Clean up llama_grammar_free_impl(grammar); + LOG("%s: %s\n", "test-gbnf-validator", "PASSED"); + common_log_flush(common_log_main()); + return 0; } diff --git a/tests/test-gguf-model-data.cpp b/tests/test-gguf-model-data.cpp index e05c4b8976..e45a753a79 100644 --- a/tests/test-gguf-model-data.cpp +++ b/tests/test-gguf-model-data.cpp @@ -1,39 +1,56 @@ #include "gguf-model-data.h" +#include "arg.h" +#include "common.h" +#include "log.h" + #include #define TEST_ASSERT(cond, msg) \ do { \ if (!(cond)) { \ - fprintf(stderr, "FAIL: %s (line %d): %s\n", #cond, __LINE__, msg); \ + LOG_ERR("FAIL: %s (line %d): %s\n", #cond, __LINE__, msg); \ + LOG("%s: %s\n", "test-gguf-model-data", "FAILED"); \ + common_log_flush(common_log_main()); \ return 1; \ } \ } while (0) -int main() { - fprintf(stderr, "=== test-gguf-model-data ===\n"); +int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-gguf-model-data"); + LOG_INF("=== test-gguf-model-data ===\n"); // Fetch Qwen3-0.6B Q8_0 metadata + LOG_INF(" running %s\n", "ggml-org/Qwen3-0.6B-GGUF (Q8_0)"); auto result = gguf_fetch_model_meta("ggml-org/Qwen3-0.6B-GGUF", "Q8_0"); if (!result.has_value()) { - fprintf(stderr, "SKIP: could not fetch model metadata (no network or HTTP disabled)\n"); + LOG_WRN("SKIP: could not fetch model metadata (no network or HTTP disabled)\n"); + LOG("%s: %s\n", "test-gguf-model-data", "PASSED"); + common_log_flush(common_log_main()); return 0; } const auto & model = result.value(); - fprintf(stderr, "Architecture: %s\n", model.architecture.c_str()); - fprintf(stderr, "n_embd: %u\n", model.n_embd); - fprintf(stderr, "n_ff: %u\n", model.n_ff); - fprintf(stderr, "n_vocab: %u\n", model.n_vocab); - fprintf(stderr, "n_layer: %u\n", model.n_layer); - fprintf(stderr, "n_head: %u\n", model.n_head); - fprintf(stderr, "n_head_kv: %u\n", model.n_head_kv); - fprintf(stderr, "n_expert: %u\n", model.n_expert); - fprintf(stderr, "n_embd_head_k: %u\n", model.n_embd_head_k); - fprintf(stderr, "n_embd_head_v: %u\n", model.n_embd_head_v); - fprintf(stderr, "tensors: %zu\n", model.tensors.size()); + LOG_INF("Architecture: %s\n", model.architecture.c_str()); + LOG_INF("n_embd: %u\n", model.n_embd); + LOG_INF("n_ff: %u\n", model.n_ff); + LOG_INF("n_vocab: %u\n", model.n_vocab); + LOG_INF("n_layer: %u\n", model.n_layer); + LOG_INF("n_head: %u\n", model.n_head); + LOG_INF("n_head_kv: %u\n", model.n_head_kv); + LOG_INF("n_expert: %u\n", model.n_expert); + LOG_INF("n_embd_head_k: %u\n", model.n_embd_head_k); + LOG_INF("n_embd_head_v: %u\n", model.n_embd_head_v); + LOG_INF("tensors: %zu\n", model.tensors.size()); // Verify architecture TEST_ASSERT(model.architecture == "qwen3", "expected architecture 'qwen3'"); @@ -83,24 +100,27 @@ int main() { TEST_ASSERT(result2->tensors.size() == model.tensors.size(), "cached result should match"); // Test a split MoE model without specifying quant (should default to Q8_0) + LOG_INF(" running %s\n", "ggml-org/GLM-4.6V-GGUF"); auto result3 = gguf_fetch_model_meta("ggml-org/GLM-4.6V-GGUF"); if (!result3.has_value()) { - fprintf(stderr, "SKIP: could not fetch GLM-4.6V metadata (no network?)\n"); + LOG_WRN("SKIP: could not fetch GLM-4.6V metadata (no network?)\n"); + LOG("%s: %s\n", "test-gguf-model-data", "PASSED"); + common_log_flush(common_log_main()); return 0; } const auto & model3 = result3.value(); - fprintf(stderr, "Architecture: %s\n", model3.architecture.c_str()); - fprintf(stderr, "n_embd: %u\n", model3.n_embd); - fprintf(stderr, "n_ff: %u\n", model3.n_ff); - fprintf(stderr, "n_vocab: %u\n", model3.n_vocab); - fprintf(stderr, "n_layer: %u\n", model3.n_layer); - fprintf(stderr, "n_head: %u\n", model3.n_head); - fprintf(stderr, "n_head_kv: %u\n", model3.n_head_kv); - fprintf(stderr, "n_expert: %u\n", model3.n_expert); - fprintf(stderr, "n_embd_head_k: %u\n", model3.n_embd_head_k); - fprintf(stderr, "n_embd_head_v: %u\n", model3.n_embd_head_v); - fprintf(stderr, "tensors: %zu\n", model3.tensors.size()); + LOG_INF("Architecture: %s\n", model3.architecture.c_str()); + LOG_INF("n_embd: %u\n", model3.n_embd); + LOG_INF("n_ff: %u\n", model3.n_ff); + LOG_INF("n_vocab: %u\n", model3.n_vocab); + LOG_INF("n_layer: %u\n", model3.n_layer); + LOG_INF("n_head: %u\n", model3.n_head); + LOG_INF("n_head_kv: %u\n", model3.n_head_kv); + LOG_INF("n_expert: %u\n", model3.n_expert); + LOG_INF("n_embd_head_k: %u\n", model3.n_embd_head_k); + LOG_INF("n_embd_head_v: %u\n", model3.n_embd_head_v); + LOG_INF("tensors: %zu\n", model3.tensors.size()); // Verify architecture TEST_ASSERT(model3.architecture == "glm4moe", "expected architecture 'glm4moe'"); @@ -117,24 +137,27 @@ int main() { TEST_ASSERT(model3.tensors.size() == 780, "expected tensor count == 780"); // Test a hybrid-attention model with array-valued head counts + LOG_INF(" running %s\n", "ggml-org/Step-3.5-Flash-GGUF (Q4_K)"); auto result4 = gguf_fetch_model_meta("ggml-org/Step-3.5-Flash-GGUF", "Q4_K"); if (!result4.has_value()) { - fprintf(stderr, "FAIL: could not fetch Step-3.5-Flash metadata\n"); + LOG_ERR("FAIL: could not fetch Step-3.5-Flash metadata\n"); + LOG("%s: %s\n", "test-gguf-model-data", "FAILED"); + common_log_flush(common_log_main()); return 1; } const auto & model4 = result4.value(); - fprintf(stderr, "Architecture: %s\n", model4.architecture.c_str()); - fprintf(stderr, "n_embd: %u\n", model4.n_embd); - fprintf(stderr, "n_ff: %u\n", model4.n_ff); - fprintf(stderr, "n_vocab: %u\n", model4.n_vocab); - fprintf(stderr, "n_layer: %u\n", model4.n_layer); - fprintf(stderr, "n_head: %u\n", model4.n_head); - fprintf(stderr, "n_head_kv: %u\n", model4.n_head_kv); - fprintf(stderr, "n_expert: %u\n", model4.n_expert); - fprintf(stderr, "n_embd_head_k: %u\n", model4.n_embd_head_k); - fprintf(stderr, "n_embd_head_v: %u\n", model4.n_embd_head_v); - fprintf(stderr, "tensors: %zu\n", model4.tensors.size()); + LOG_INF("Architecture: %s\n", model4.architecture.c_str()); + LOG_INF("n_embd: %u\n", model4.n_embd); + LOG_INF("n_ff: %u\n", model4.n_ff); + LOG_INF("n_vocab: %u\n", model4.n_vocab); + LOG_INF("n_layer: %u\n", model4.n_layer); + LOG_INF("n_head: %u\n", model4.n_head); + LOG_INF("n_head_kv: %u\n", model4.n_head_kv); + LOG_INF("n_expert: %u\n", model4.n_expert); + LOG_INF("n_embd_head_k: %u\n", model4.n_embd_head_k); + LOG_INF("n_embd_head_v: %u\n", model4.n_embd_head_v); + LOG_INF("tensors: %zu\n", model4.tensors.size()); TEST_ASSERT(model4.architecture == "step35", "expected architecture 'step35'"); @@ -149,6 +172,8 @@ int main() { TEST_ASSERT(model4.n_vocab == 128896, "expected n_vocab == 128896"); TEST_ASSERT(model4.tensors.size() == 754, "expected tensor count == 754"); - fprintf(stderr, "=== ALL TESTS PASSED ===\n"); + LOG_INF("=== ALL TESTS PASSED ===\n"); + LOG("%s: %s\n", "test-gguf-model-data", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-gguf.cpp b/tests/test-gguf.cpp index fc636186f4..58ff4f1d77 100644 --- a/tests/test-gguf.cpp +++ b/tests/test-gguf.cpp @@ -3,6 +3,10 @@ #include "../ggml/src/ggml-impl.h" #include "gguf.h" +#include "arg.h" +#include "common.h" +#include "log.h" + #include #include #include @@ -748,6 +752,14 @@ static bool handcrafted_check_tensor_data(const gguf_context * gguf_ctx, const u } static std::pair test_handcrafted_file(const unsigned int seed) { + LOG_INF(" running %s\n", __func__); + + // no ANSI escapes at WARN or lower (--errors-only) + const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN; + const char * const col_ok = use_color ? "\033[1;32m" : ""; + const char * const col_fail = use_color ? "\033[1;31m" : ""; + const char * const col_end = use_color ? "\033[0m" : ""; + int npass = 0; int ntest = 0; @@ -791,7 +803,7 @@ static std::pair test_handcrafted_file(const unsigned int seed) { }; for (enum handcrafted_file_type hft : hfts) { - printf("%s: handcrafted_file_type=%s\n", __func__, handcrafted_file_type_name(hft).c_str()); + LOG_CNT("%s: handcrafted_file_type=%s\n", __func__, handcrafted_file_type_name(hft).c_str()); FILE * file = get_handcrafted_file(seed, hft); #ifdef _WIN32 @@ -813,24 +825,24 @@ static std::pair test_handcrafted_file(const unsigned int seed) { struct gguf_context * gguf_ctx = gguf_init_from_file_ptr(file, gguf_params); if (expect_context_not_null(hft)) { - printf("%s: - context_not_null: ", __func__); + LOG_CNT("%s: - context_not_null: ", __func__); } else { - printf("%s: - context_null: ", __func__); + LOG_CNT("%s: - context_null: ", __func__); } if (bool(gguf_ctx) == expect_context_not_null(hft)) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ntest++; if (hft >= offset_has_data && !expect_context_not_null(hft)) { - printf("%s: - no_dangling_ggml_context_pointer: ", __func__); + LOG_CNT("%s: - no_dangling_ggml_context_pointer: ", __func__); if (ctx) { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } else { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } ntest++; @@ -839,23 +851,23 @@ static std::pair test_handcrafted_file(const unsigned int seed) { const bool alignment_defined = hft == HANDCRAFTED_TENSORS_CUSTOM_ALIGN || hft == HANDCRAFTED_DATA_CUSTOM_ALIGN; if (expect_context_not_null(hft)) { - printf("%s: - check_header: ", __func__); + LOG_CNT("%s: - check_header: ", __func__); if (handcrafted_check_header(gguf_ctx, seed, hft >= offset_has_kv, hft >= offset_has_tensors, alignment_defined)) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ntest++; } if (expect_context_not_null(hft) && hft >= offset_has_kv) { - printf("%s: - check_kv: ", __func__); + LOG_CNT("%s: - check_kv: ", __func__); if (handcrafted_check_kv(gguf_ctx, seed, hft >= offset_has_tensors, alignment_defined)) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ntest++; } @@ -863,23 +875,23 @@ static std::pair test_handcrafted_file(const unsigned int seed) { // HANDCRAFTED_TENSORS_ZERO_DIM deliberately mangles the tensor shapes to 0 elements, // so only assert that it loads without crashing; skip the exact-geometry comparison. if (expect_context_not_null(hft) && hft >= offset_has_tensors && hft != HANDCRAFTED_TENSORS_ZERO_DIM) { - printf("%s: - check_tensors: ", __func__); + LOG_CNT("%s: - check_tensors: ", __func__); if (handcrafted_check_tensors(gguf_ctx, seed)) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ntest++; } if (expect_context_not_null(hft) && hft >= offset_has_data) { - printf("%s: - check_tensor_data: ", __func__); + LOG_CNT("%s: - check_tensor_data: ", __func__); if (handcrafted_check_tensor_data(gguf_ctx, seed, file)) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ntest++; } @@ -889,7 +901,7 @@ static std::pair test_handcrafted_file(const unsigned int seed) { ggml_free(ctx); gguf_free(gguf_ctx); } - printf("\n"); + LOG_CNT("\n"); } @@ -1185,10 +1197,16 @@ static std::pair test_roundtrip( ggml_backend_dev_t dev, const unsigned int seed, const bool only_meta, const roundtrip_read_mode read_mode) { ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr); - printf("%s: device=%s, backend=%s, only_meta=%s, read_mode=%s\n", + LOG_INF(" running %s: device=%s, backend=%s, only_meta=%s, read_mode=%s\n", __func__, ggml_backend_dev_description(dev), ggml_backend_name(backend), only_meta ? "yes" : "no", roundtrip_read_mode_name(read_mode)); + // no ANSI escapes at WARN or lower (--errors-only) + const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN; + const char * const col_ok = use_color ? "\033[1;32m" : ""; + const char * const col_fail = use_color ? "\033[1;31m" : ""; + const char * const col_end = use_color ? "\033[0m" : ""; + int npass = 0; int ntest = 0; @@ -1239,76 +1257,76 @@ static std::pair test_roundtrip( gguf_ctx_1 = gguf_init_from_file_ptr(file, gguf_params); } - printf("%s: same_version: ", __func__); + LOG_CNT("%s: same_version: ", __func__); if (gguf_get_version(gguf_ctx_0) == gguf_get_version(gguf_ctx_1)) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ntest++; - printf("%s: same_n_kv: ", __func__); + LOG_CNT("%s: same_n_kv: ", __func__); if (gguf_get_n_kv(gguf_ctx_0) == gguf_get_n_kv(gguf_ctx_1)) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ntest++; - printf("%s: same_n_tensors: ", __func__); + LOG_CNT("%s: same_n_tensors: ", __func__); if (gguf_get_n_tensors(gguf_ctx_0) == gguf_get_n_tensors(gguf_ctx_1)) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ntest++; - printf("%s: all_orig_kv_in_read: ", __func__); + LOG_CNT("%s: all_orig_kv_in_read: ", __func__); if (all_kv_in_other(gguf_ctx_0, gguf_ctx_1)) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ntest++; - printf("%s: all_read_kv_in_orig: ", __func__); + LOG_CNT("%s: all_read_kv_in_orig: ", __func__); if (all_kv_in_other(gguf_ctx_1, gguf_ctx_0)) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ntest++; - printf("%s: all_orig_tensors_in_read: ", __func__); + LOG_CNT("%s: all_orig_tensors_in_read: ", __func__); if (all_tensors_in_other(gguf_ctx_0, gguf_ctx_1)) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ntest++; - printf("%s: all_read_tensors_in_orig: ", __func__); + LOG_CNT("%s: all_read_tensors_in_orig: ", __func__); if (all_tensors_in_other(gguf_ctx_1, gguf_ctx_0)) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ntest++; if (!only_meta) { - printf("%s: same_tensor_data: ", __func__); + LOG_CNT("%s: same_tensor_data: ", __func__); if (same_tensor_data(ctx_0, ctx_1)) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ntest++; } @@ -1321,13 +1339,20 @@ static std::pair test_roundtrip( ggml_backend_free(backend); fclose(file); - printf("\n"); + LOG_CNT("\n"); return std::make_pair(npass, ntest); } static std::pair test_gguf_set_kv(ggml_backend_dev_t dev, const unsigned int seed) { ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr); - printf("%s: device=%s, backend=%s\n", __func__, ggml_backend_dev_description(dev), ggml_backend_name(backend)); + LOG_INF(" running %s: device=%s, backend=%s\n", + __func__, ggml_backend_dev_description(dev), ggml_backend_name(backend)); + + // no ANSI escapes at WARN or lower (--errors-only) + const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN; + const char * const col_ok = use_color ? "\033[1;32m" : ""; + const char * const col_fail = use_color ? "\033[1;31m" : ""; + const char * const col_end = use_color ? "\033[0m" : ""; int npass = 0; int ntest = 0; @@ -1357,50 +1382,50 @@ static std::pair test_gguf_set_kv(ggml_backend_dev_t dev, const unsign gguf_set_kv(gguf_ctx_1, gguf_ctx_0); gguf_set_kv(gguf_ctx_2, gguf_ctx_0); - printf("%s: same_n_kv: ", __func__); + LOG_CNT("%s: same_n_kv: ", __func__); if (gguf_get_n_kv(gguf_ctx_0) == gguf_get_n_kv(gguf_ctx_2)) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ntest++; - printf("%s: all_kv_0_in_1: ", __func__); + LOG_CNT("%s: all_kv_0_in_1: ", __func__); if (all_kv_in_other(gguf_ctx_0, gguf_ctx_1)) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ntest++; - printf("%s: all_kv_0_in_2: ", __func__); + LOG_CNT("%s: all_kv_0_in_2: ", __func__); if (all_kv_in_other(gguf_ctx_0, gguf_ctx_2)) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ntest++; gguf_set_kv(gguf_ctx_0, gguf_ctx_1); - printf("%s: same_n_kv_after_double_copy: ", __func__); + LOG_CNT("%s: same_n_kv_after_double_copy: ", __func__); if (gguf_get_n_kv(gguf_ctx_0) == gguf_get_n_kv(gguf_ctx_1)) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ntest++; - printf("%s: all_kv_1_in_0_after_double_copy: ", __func__); + LOG_CNT("%s: all_kv_1_in_0_after_double_copy: ", __func__); if (all_kv_in_other(gguf_ctx_1, gguf_ctx_0)) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ntest++; @@ -1413,7 +1438,7 @@ static std::pair test_gguf_set_kv(ggml_backend_dev_t dev, const unsign gguf_free(gguf_ctx_2); ggml_backend_free(backend); - printf("\n"); + LOG_CNT("\n"); return std::make_pair(npass, ntest); } @@ -1423,17 +1448,39 @@ static void print_usage() { } int main(int argc, char ** argv) { - if (argc > 2) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + + // this test takes an optional seed as its only positional argument + std::vector positional; + std::vector common_argv; + common_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { + if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else { + positional.push_back(argv[i]); + } + } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + if (positional.size() > 1) { print_usage(); return 1; } std::random_device rd; - const unsigned int seed = argc < 2 ? rd() : std::stoi(argv[1]); + const unsigned int seed = positional.empty() ? rd() : std::stoi(positional[0]); + + LOG("%s: running\n", "test-gguf"); // Initialize ggml backends early so the prints aren't interleaved with the test results: ggml_backend_dev_count(); - fprintf(stderr, "\n"); + LOG_CNT("\n"); // bare separator, LOG_CNT adds no prefix int npass = 0; int ntest = 0; @@ -1469,11 +1516,22 @@ int main(int argc, char ** argv) { } } + const bool ok = npass == ntest; + + // no ANSI escapes at WARN or lower (--errors-only) + const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN; + const char * const col_ok = use_color ? "\033[1;32m" : ""; + const char * const col_fail = use_color ? "\033[1;31m" : ""; + const char * const col_end = use_color ? "\033[0m" : ""; + + common_log_flush(common_log_main()); printf("%d/%d tests passed\n", npass, ntest); - if (npass != ntest) { - printf("\033[1;31mFAIL\033[0m\n"); - return 1; + if (!ok) { + printf("%sFAIL%s\n", col_fail, col_end); + } else { + printf("%sOK%s\n", col_ok, col_end); } - printf("\033[1;32mOK\033[0m\n"); - return 0; + LOG("%s: %s\n", "test-gguf", ok ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); + return ok ? 0 : 1; } diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp index 2af96f8b47..3ceaa9a31a 100644 --- a/tests/test-grammar-integration.cpp +++ b/tests/test-grammar-integration.cpp @@ -9,6 +9,10 @@ #include "json.h" +#include "arg.h" +#include "common.h" +#include "log.h" + #include #include #include @@ -24,14 +28,14 @@ static llama_grammar * build_grammar(const std::string & grammar_str) { } static bool test_build_grammar_fails(const std::string & grammar_str) { - fprintf(stderr, "⚫ Testing failure for grammar: %s\n", grammar_str.c_str()); + LOG_INF("⚫ Testing failure for grammar: %s\n", grammar_str.c_str()); bool grammar_fails = false; llama_grammar * grammar = build_grammar(grammar_str); if (grammar != nullptr) { - fprintf(stderr, " ❌ Expected build failure, but succeeded\n"); + LOG_ERR(" ❌ Expected build failure, but succeeded\n"); } else { grammar_fails = true; - fprintf(stdout, " ✅︎\n"); + LOG_CNT(" ✅︎\n"); } return grammar_fails; } @@ -114,8 +118,8 @@ static bool match_string(const std::string & input, llama_grammar * grammar) { } static void test(const std::string & test_desc, const std::string & grammar_str, const std::vector & passing_strings, const std::vector & failing_strings) { - fprintf(stderr, "⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str()); - fflush(stderr); + LOG_INF("⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str()); + common_log_flush(common_log_main()); auto * grammar = build_grammar(grammar_str); @@ -124,17 +128,18 @@ static void test(const std::string & test_desc, const std::string & grammar_str, llama_grammar_stacks & stacks_cur = llama_grammar_get_stacks(grammar); - fprintf(stderr, " 🔵 Valid strings:\n"); + LOG_INF(" 🔵 Valid strings:\n"); // Passing strings for (const auto & test_string : passing_strings) { - fprintf(stderr, " \"%s\" ", test_string.c_str()); - fflush(stderr); + // partial line, completed by the next message: LOG_CNT adds no prefix + LOG_CNT(" \"%s\" ", test_string.c_str()); + common_log_flush(common_log_main()); bool matched = match_string(test_string, grammar); if (!matched) { - fprintf(stderr, "❌ (failed to match)\n"); + LOG_ERR("❌ (failed to match)\n"); // DEBUG: Write strings to files so that we can analyze more easily with gbnf-validator program to see exactly where things failed. // DEBUG: Write the grammar_str to test-grammar-integration.grammar.gbnf @@ -151,9 +156,9 @@ static void test(const std::string & test_desc, const std::string & grammar_str, fclose(string_file); } - fprintf(stderr, "\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following command: ./llama-gbnf-validator test-grammar-integration.grammar.gbnf test-grammar-integration.string.txt\n\n"); + LOG_ERR("\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following command: ./llama-gbnf-validator test-grammar-integration.grammar.gbnf test-grammar-integration.string.txt\n\n"); } else { - fprintf(stdout, "✅︎\n"); + LOG_CNT("✅︎\n"); } assert(matched); @@ -162,19 +167,19 @@ static void test(const std::string & test_desc, const std::string & grammar_str, stacks_cur = stacks_org; } - fprintf(stderr, " 🟠 Invalid strings:\n"); + LOG_INF(" 🟠 Invalid strings:\n"); // Failing strings for (const auto & test_string : failing_strings) { - fprintf(stderr, " \"%s\" ", test_string.c_str()); - fflush(stderr); + LOG_CNT(" \"%s\" ", test_string.c_str()); + common_log_flush(common_log_main()); bool matched = match_string(test_string, grammar); if (matched) { - fprintf(stderr, "❌ (incorrectly matched)\n"); + LOG_ERR("❌ (incorrectly matched)\n"); } else { - fprintf(stdout, "✅︎\n"); + LOG_CNT("✅︎\n"); } assert(!matched); @@ -193,6 +198,7 @@ static void test_schema(const std::string & test_desc, const std::string & schem } static void test_simple_grammar() { + LOG_INF(" running %s\n", __func__); test_schema( "min 0", R"""({ @@ -510,6 +516,7 @@ static void test_simple_grammar() { } static void test_complex_grammar() { + LOG_INF(" running %s\n", __func__); // Test case for a more complex grammar, with both failure strings and success strings test_grammar( "medium complexity grammar", @@ -599,6 +606,7 @@ static void test_complex_grammar() { } static void test_special_chars() { + LOG_INF(" running %s\n", __func__); // A collection of tests to exercise special characters such as "." test_grammar( "special characters", @@ -626,6 +634,7 @@ static void test_special_chars() { } static void test_quantifiers() { + LOG_INF(" running %s\n", __func__); // A collection of tests to exercise * + and ? quantifiers test_grammar( @@ -809,7 +818,8 @@ static void test_quantifiers() { } static void test_failure_missing_root() { - fprintf(stderr, "⚫ Testing missing root node:\n"); + LOG_INF(" running %s\n", __func__); + LOG_INF("⚫ Testing missing root node:\n"); // Test case for a grammar that is missing a root rule const std::string grammar_str = R"""( rot ::= expr @@ -825,11 +835,12 @@ static void test_failure_missing_root() { // Ensure we do NOT have a root node assert(parsed_grammar.symbol_ids.find("root") == parsed_grammar.symbol_ids.end()); - fprintf(stderr, " ✅︎ Passed\n"); + LOG_INF(" ✅︎ Passed\n"); } static void test_failure_missing_reference() { - fprintf(stderr, "⚫ Testing missing reference node:\n"); + LOG_INF(" running %s\n", __func__); + LOG_INF("⚫ Testing missing reference node:\n"); // Test case for a grammar that is missing a referenced rule const std::string grammar_str = @@ -838,6 +849,8 @@ static void test_failure_missing_reference() { term ::= numero number ::= [0-9]+)"""; + // the parser below writes to stderr directly, so drain the log first + common_log_flush(common_log_main()); fprintf(stderr, " Expected error: "); llama_grammar_parser parsed_grammar; @@ -847,11 +860,12 @@ static void test_failure_missing_reference() { assert(parsed_grammar.rules.empty()); fprintf(stderr, " End of expected error.\n"); - fprintf(stderr, " ✅︎ Passed\n"); + LOG_INF(" ✅︎ Passed\n"); } static void test_failure_left_recursion() { - fprintf(stderr, "⚫ Testing left recursion detection:\n"); + LOG_INF(" running %s\n", __func__); + LOG_INF("⚫ Testing left recursion detection:\n"); // Test simple left recursion detection const std::string simple_str = R"""(root ::= "a" | root "a")"""; @@ -879,11 +893,12 @@ static void test_failure_left_recursion() { empty ::= "blah" | )"""; assert(test_build_grammar_fails(hardest_str)); - fprintf(stderr, " ✅︎ Passed\n"); + LOG_INF(" ✅︎ Passed\n"); } static void test_failure_missing_root_symbol() { - fprintf(stderr, "⚫ Testing missing root symbol:\n"); + LOG_INF(" running %s\n", __func__); + LOG_INF("⚫ Testing missing root symbol:\n"); const std::string grammar_str = R"""( root ::= "foobar" @@ -892,11 +907,12 @@ static void test_failure_missing_root_symbol() { llama_grammar * failure_result = build_grammar_with_root(grammar_str, "nonexistent"); assert(failure_result == nullptr); - fprintf(stderr, " ✅︎ Passed\n"); + LOG_INF(" ✅︎ Passed\n"); } static void test_custom_root_symbol_check() { - fprintf(stderr, "⚫ Testing custom root symbol check:\n"); + LOG_INF(" running %s\n", __func__); + LOG_INF("⚫ Testing custom root symbol check:\n"); const std::string custom_root_grammar_str = R"""( foobar ::= "foobar" @@ -909,10 +925,11 @@ static void test_custom_root_symbol_check() { assert(success_result != nullptr); llama_grammar_free_impl(success_result); - fprintf(stderr, " ✅︎ Passed\n"); + LOG_INF(" ✅︎ Passed\n"); } static void test_json_schema() { + LOG_INF(" running %s\n", __func__); // Note that this is similar to the regular grammar tests, // but we convert each json schema to a grammar before parsing. // Otherwise, this test structure is the same. @@ -1478,8 +1495,16 @@ static void test_json_schema() { ); } -int main() { - fprintf(stdout, "Running grammar integration tests...\n"); +int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-grammar-integration"); + test_simple_grammar(); test_complex_grammar(); test_special_chars(); @@ -1490,6 +1515,9 @@ int main() { test_failure_missing_root_symbol(); test_custom_root_symbol_check(); test_json_schema(); - fprintf(stdout, "All tests passed.\n"); + common_log_flush(common_log_main()); + + // the tests abort on failure, so reaching this point means they all passed + LOG("%s: %s\n", "test-grammar-integration", "PASSED"); return 0; } diff --git a/tests/test-grammar-llguidance.cpp b/tests/test-grammar-llguidance.cpp index 34746c200c..415d3834be 100644 --- a/tests/test-grammar-llguidance.cpp +++ b/tests/test-grammar-llguidance.cpp @@ -4,6 +4,9 @@ #include "sampling.h" +#include "arg.h" +#include "log.h" + #include #include #include @@ -49,22 +52,23 @@ static bool match_string(const std::string & input, llama_sampler * grammar) { static void test(const std::string & test_desc, const std::string & grammar_str, const std::vector & passing_strings, const std::vector & failing_strings) { - fprintf(stderr, "⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str()); - fflush(stderr); + LOG_INF("⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str()); + common_log_flush(common_log_main()); auto * grammar = llama_sampler_init_llg(vocab, "lark", grammar_str.c_str()); - fprintf(stderr, " 🔵 Valid strings:\n"); + LOG_INF(" 🔵 Valid strings:\n"); // Passing strings for (const auto & test_string : passing_strings) { - fprintf(stderr, " \"%s\" ", test_string.c_str()); - fflush(stderr); + // partial line, completed by the next message: LOG_CNT adds no prefix + LOG_CNT(" \"%s\" ", test_string.c_str()); + common_log_flush(common_log_main()); bool matched = match_string(test_string, grammar); if (!matched) { - fprintf(stderr, "❌ (failed to match)\n"); + LOG_ERR("❌ (failed to match)\n"); // DEBUG: Write strings to files so that we can analyze more easily with gbnf-validator program to see exactly where things failed. // DEBUG: Write the grammar_str to test-grammar-integration.grammar.gbnf @@ -81,8 +85,7 @@ static void test(const std::string & test_desc, const std::string & grammar_str, fclose(string_file); } - fprintf(stderr, - "\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following " + LOG_ERR("\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following " "command: ./test-gbnf-validator test-grammar-integration.grammar.gbnf " "test-grammar-integration.string.txt\n\n"); } else { @@ -92,17 +95,18 @@ static void test(const std::string & test_desc, const std::string & grammar_str, assert(matched); } - fprintf(stderr, " 🟠 Invalid strings:\n"); + LOG_INF(" 🟠 Invalid strings:\n"); // Failing strings for (const auto & test_string : failing_strings) { - fprintf(stderr, " \"%s\" ", test_string.c_str()); - fflush(stderr); + // partial line, completed by the next message: LOG_CNT adds no prefix + LOG_CNT(" \"%s\" ", test_string.c_str()); + common_log_flush(common_log_main()); bool matched = match_string(test_string, grammar); if (matched) { - fprintf(stderr, "❌ (incorrectly matched)\n"); + LOG_ERR("❌ (incorrectly matched)\n"); } else { fprintf(stdout, "✅︎\n"); } @@ -1124,11 +1128,11 @@ start: /[A-Z ]*/)"; for (const auto token : tokens) { one_hot(tok_arr, token); - fprintf(stderr, "applying token: %d\n", token); + LOG_DBG("applying token: %d\n", token); llama_sampler_apply(sampler, &tok_arr); auto idx = tok_arr.selected; - fprintf(stderr, " -> %d %f\n", cur[idx].id, cur[idx].logit); + LOG_DBG(" -> %d %f\n", cur[idx].id, cur[idx].logit); assert(cur[tok_arr.selected].id == token); llama_sampler_accept(sampler, token); } @@ -1144,17 +1148,35 @@ start: /[A-Z ]*/)"; assert(cur[tok_arr.selected].id == tok_eos); } -int main(int argc, const char ** argv) { - fprintf(stdout, "Running llguidance integration tests...\n"); +int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); - if (argc != 2) { - fprintf(stderr, "Usage: %s \n", argv[0]); + std::string vocab_file; + std::vector common_argv; + common_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { + if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else if (vocab_file.empty()) { + vocab_file = argv[i]; + } + } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { return 1; } - const char * vocab_file = argv[1]; + fprintf(stdout, "Running llguidance integration tests...\n"); - fprintf(stderr, "reading vocab from: '%s'\n", vocab_file); + if (vocab_file.empty()) { + LOG_ERR("Usage: %s \n", argv[0]); + common_log_flush(common_log_main()); + return 1; + } + + LOG_INF("reading vocab from: '%s'\n", vocab_file.c_str()); llama_model * model; llama_context * ctx; @@ -1167,10 +1189,11 @@ int main(int argc, const char ** argv) { mparams.vocab_only = true; - model = llama_model_load_from_file(vocab_file, mparams); + model = llama_model_load_from_file(vocab_file.c_str(), mparams); if (model == NULL) { - fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, vocab_file); + LOG_ERR("%s: error: failed to load vocab '%s'\n", __func__, vocab_file.c_str()); + common_log_flush(common_log_main()); return 1; } @@ -1180,8 +1203,9 @@ int main(int argc, const char ** argv) { ctx = llama_init_from_model(model, cparams); if (ctx == NULL) { - fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, vocab_file); + LOG_ERR("%s: error: failed to load vocab '%s'\n", __func__, vocab_file.c_str()); llama_model_free(model); + common_log_flush(common_log_main()); return 1; } } @@ -1199,6 +1223,7 @@ int main(int argc, const char ** argv) { llama_free(ctx); llama_model_free(model); + common_log_flush(common_log_main()); fprintf(stdout, "All tests passed.\n"); return 0; } diff --git a/tests/test-grammar-parser.cpp b/tests/test-grammar-parser.cpp index 2ddf25bb69..bcaeea4f39 100644 --- a/tests/test-grammar-parser.cpp +++ b/tests/test-grammar-parser.cpp @@ -7,6 +7,10 @@ // TODO: shold not include libllama sources #include "../src/llama-grammar.h" +#include "arg.h" +#include "common.h" +#include "log.h" + #include static const char * type_str(llama_gretype type) { @@ -33,52 +37,53 @@ static void verify_parsing(const char *grammar_bytes, const std::vectorfirst.c_str(), it->second); + LOG_CNT(" {\"%s\", %u},\n", it->first.c_str(), it->second); } - fprintf(stderr, " }, {\n"); + LOG_CNT(" }, {\n"); for (size_t i_rule = 0; i_rule < parsed_grammar.rules.size(); i_rule++) { - fprintf(stderr, " // %s (index %zu)\n", symbol_names[i_rule].c_str(), i_rule); + LOG_CNT(" // %s (index %zu)\n", symbol_names[i_rule].c_str(), i_rule); auto & rule = parsed_grammar.rules[i_rule]; for (uint32_t i = 0; i < rule.size(); i++) { std::string rule_str; - fprintf(stderr, " {%s, ", type_str(rule[i].type)); + LOG_CNT(" {%s, ", type_str(rule[i].type)); if (rule[i].type == LLAMA_GRETYPE_CHAR || rule[i].type == LLAMA_GRETYPE_CHAR_ALT || rule[i].type == LLAMA_GRETYPE_CHAR_NOT || rule[i].type == LLAMA_GRETYPE_CHAR_RNG_UPPER) { char c = rule[i].value; if (c == '\n') { - fprintf(stderr, "'\\n'"); + LOG_CNT("'\\n'"); } else if (c == '\t') { - fprintf(stderr, "'\\t'"); + LOG_CNT("'\\t'"); } else if (c == '\r') { - fprintf(stderr, "'\\r'"); + LOG_CNT("'\\r'"); } else if (c == '\0') { - fprintf(stderr, "'\\0'"); + LOG_CNT("'\\0'"); } else { - fprintf(stderr, "'%c'", c); + LOG_CNT("'%c'", c); } } else if (rule[i].type == LLAMA_GRETYPE_RULE_REF) { - fprintf(stderr, "/* %s */ %u", symbol_names[rule[i].value].c_str(), rule[i].value); + LOG_CNT("/* %s */ %u", symbol_names[rule[i].value].c_str(), rule[i].value); } else { - fprintf(stderr, "%u", rule[i].value); + LOG_CNT("%u", rule[i].value); } - fprintf(stderr, "},\n"); + LOG_CNT("},\n"); } } - fprintf(stderr, " });\n"); + LOG_CNT(" });\n"); }; if (getenv("TEST_GRAMMAR_PARSER_PRINT_ALL")) { print_all(); - fprintf(stderr, "\n"); + LOG_CNT("\n"); return; } - fprintf(stderr, "Testing grammar:%s\n", grammar_bytes); + LOG_INF("Testing grammar:%s\n", grammar_bytes); if (parsed_grammar.symbol_ids.size() != expected.size()) { - fprintf(stderr, "Code to update expectation (set TEST_GRAMMAR_PARSER_PRINT_ALL=1 to print all):\n"); + LOG_ERR("Code to update expectation (set TEST_GRAMMAR_PARSER_PRINT_ALL=1 to print all):\n"); print_all(); assert(parsed_grammar.symbol_ids.size() == expected.size()); } @@ -92,11 +97,11 @@ static void verify_parsing(const char *grammar_bytes, const std::vector #include #include +#include +#include "arg.h" #include "json.h" #include "subproc.h" @@ -12,6 +14,7 @@ #include "jinja/utils.h" #include "jinja/caps.h" +#include "log.h" #include "testing.h" using json = common_json; @@ -41,6 +44,10 @@ static void test_fuzzing(testing & t); static bool g_python_mode = false; int main(int argc, char *argv[]) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + testing t(std::cout); t.verbose = true; @@ -49,14 +56,24 @@ int main(int argc, char *argv[]) { // only use this for cross-checking, not for correctness // note: the implementation of this flag is basic, only intented to be used by maintainers + std::vector common_argv; + common_argv.push_back(argv[0]); for (int i = 1; i < argc; i++) { std::string arg = argv[i]; if (arg == "-py") { g_python_mode = true; + } else if (arg[0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it } else { t.set_filter(arg); } } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-jinja"); t.test("whitespace control", test_whitespace_control); t.test("conditionals", test_conditionals); @@ -80,7 +97,10 @@ int main(int argc, char *argv[]) { t.test("fuzzing", test_fuzzing); } - return t.summary(); + const int rc = t.summary(); + LOG("%s: %s\n", "test-jinja", rc == 0 ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); + return rc; } static void test_whitespace_control(testing & t) { diff --git a/tests/test-json-schema-to-grammar.cpp b/tests/test-json-schema-to-grammar.cpp index 4c4206c6e6..20fa5dee40 100755 --- a/tests/test-json-schema-to-grammar.cpp +++ b/tests/test-json-schema-to-grammar.cpp @@ -8,6 +8,10 @@ #include "json.h" +#include "arg.h" +#include "common.h" +#include "log.h" + #include #include @@ -30,12 +34,12 @@ struct TestCase { std::string expected_grammar; void _print_failure_header() const { - fprintf(stderr, "#\n# Test '%s' failed.\n#\n%s\n", name.c_str(), schema.c_str()); + LOG_ERR("#\n# Test '%s' failed.\n#\n%s\n", name.c_str(), schema.c_str()); } void verify(const std::string & actual_grammar) const { if (trim(actual_grammar) != trim(expected_grammar)) { _print_failure_header(); - fprintf(stderr, "# EXPECTED:\n%s\n# ACTUAL:\n%s\n", expected_grammar.c_str(), actual_grammar.c_str()); + LOG_ERR("# EXPECTED:\n%s\n# ACTUAL:\n%s\n", expected_grammar.c_str(), actual_grammar.c_str()); assert(false); } } @@ -48,24 +52,24 @@ struct TestCase { } } catch (const std::runtime_error & ex) { _print_failure_header(); - fprintf(stderr, "# GRAMMAR ERROR: %s\n", ex.what()); + LOG_ERR("# GRAMMAR ERROR: %s\n", ex.what()); assert(false); } } void verify_status(TestCaseStatus status) const { if (status != expected_status) { _print_failure_header(); - fprintf(stderr, "# EXPECTED STATUS: %s\n", expected_status == SUCCESS ? "SUCCESS" : "FAILURE"); - fprintf(stderr, "# ACTUAL STATUS: %s\n", status == SUCCESS ? "SUCCESS" : "FAILURE"); + LOG_ERR("# EXPECTED STATUS: %s\n", expected_status == SUCCESS ? "SUCCESS" : "FAILURE"); + LOG_ERR("# ACTUAL STATUS: %s\n", status == SUCCESS ? "SUCCESS" : "FAILURE"); assert(false); } } }; static void test_all(const std::string & title, std::function runner) { - fprintf(stderr, "#\n# %s\n#\n", title.c_str()); + LOG_INF("#\n# %s\n#\n", title.c_str()); auto test = [&](const TestCase & tc) { - fprintf(stderr, "- %s%s\n", tc.name.c_str(), tc.expected_status == FAILURE ? " (failure expected)" : ""); + LOG_INF("- %s%s\n", tc.name.c_str(), tc.expected_status == FAILURE ? " (failure expected)" : ""); runner(tc); }; @@ -1510,20 +1514,29 @@ static void test_all(const std::string & title, std::function @@ -7,6 +10,7 @@ #include #include #include +#include static common_chat_schema_document parse(const std::string & schema) { return common_chat_schema_from_json(common_json::parse(schema)); @@ -484,9 +488,31 @@ static void test_errors(testing & t) { } int main(int argc, char * argv[]) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + + // this test takes an optional filter as its only positional argument + std::string filter; + std::vector common_argv; + common_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { + if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else if (filter.empty()) { + filter = argv[i]; + } + } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-json-schema"); + testing t(std::cout); - if (argc >= 2) { - t.set_filter(argv[1]); + if (!filter.empty()) { + t.set_filter(filter); } const char * verbose = getenv("LLAMA_TEST_VERBOSE"); @@ -509,5 +535,8 @@ int main(int argc, char * argv[]) { t.test("value_types", test_value_types); t.test("errors", test_errors); - return t.summary(); + const int rc = t.summary(); + LOG("%s: %s\n", "test-json-schema", rc == 0 ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); + return rc; } diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 568f7234c1..bf6d33118b 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -1,3 +1,4 @@ +#include "arg.h" #include "common.h" #include "log.h" #include "ggml-backend.h" @@ -718,18 +719,24 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in const std::string template_row_cfg = std::string("|%" + std::to_string(max_arch_name_length) + "s|%") + std::to_string(max_device_label_length) + "s|%6s|"; const std::string template_row_res = "%15s %10s|%20s|\n"; + // no ANSI escapes at WARN or lower (--errors-only) + const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN; + const std::string status_skip = use_color ? "\033[1;33mSKIP\033[0m" : "SKIP"; + const std::string status_ok = use_color ? "\033[1;32mOK\033[0m" : "OK"; + const std::string status_fail = use_color ? "\033[1;31mFAIL\033[0m" : "FAIL"; + bool all_ok = true; common_log_flush(common_log_main()); - printf(template_header.c_str(), "Model arch.", "Device", "Config", "NMSE vs. CPU", "Roundtrip"); - printf("|"); + LOG_CNT(template_header.c_str(), "Model arch.", "Device", "Config", "NMSE vs. CPU", "Roundtrip"); + LOG_CNT("|"); for (size_t i = 0; i < max_arch_name_length; i++) { - printf("-"); + LOG_CNT("-"); } - printf("|"); + LOG_CNT("|"); for (size_t i = 0; i < max_device_label_length; i++) { - printf("-"); + LOG_CNT("-"); } - printf("|------|---------------|---------|\n"); + LOG_CNT("|------|---------------|---------|\n"); for (const llm_arch & arch : llm_arch_all()) { if (arch == LLM_ARCH_UNKNOWN) { continue; @@ -761,14 +768,14 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in std::vector logits_cpu; for (device_config & dc : dev_configs) { // print test config first; should anything fail during model loading or inference, at least we know which test case caused it - printf(template_row_cfg.c_str(), + LOG_CNT(template_row_cfg.c_str(), llm_arch_name(arch), dc.label.c_str(), config_name.c_str()); - fflush(stdout); + common_log_flush(common_log_main()); std::pair model_and_ctx_dev; std::vector logits_dev; - std::string status_nmse = "\033[1;33mSKIP\033[0m"; - std::string status_roundtrip = "\033[1;33mSKIP\033[0m"; + std::string status_nmse = status_skip; + std::string status_roundtrip = status_skip; char nmse_str[12] = {0}; bool skip = !arch_supported(arch) || (dc.split_mode == LLAMA_SPLIT_MODE_TENSOR && dc.devs.empty()); @@ -782,10 +789,10 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in logits_dev = get_logits(model_and_ctx_dev.first.get(), model_and_ctx_dev.second.get(), tokens, encode); const double nmse_val = nmse(logits_cpu, logits_dev); snprintf(nmse_str, sizeof(nmse_str), "(%.2e)", nmse_val); - status_nmse = "\033[1;32mOK\033[0m"; + status_nmse = status_ok; if (nmse_val > 1e-4) { all_ok = false; - status_nmse = "\033[1;31mFAIL\033[0m"; + status_nmse = status_fail; } } @@ -803,12 +810,12 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in auto model_and_ctx_roundtrip = get_model_and_ctx(nullptr, file, seed, dc.devs, dc.split_mode, encode); const std::vector logits_roundtrip = get_logits( model_and_ctx_roundtrip.first.get(), model_and_ctx_roundtrip.second.get(), tokens, encode); - status_roundtrip = "\033[1;32mOK\033[0m"; + status_roundtrip = status_ok; GGML_ASSERT(logits_roundtrip.size() == logits_dev.size()); for (size_t i = 0; i < logits_roundtrip.size(); i++) { if (logits_roundtrip[i] != logits_dev[i]) { all_ok = false; - status_roundtrip = "\033[1;31mFAIL\033[0m"; + status_roundtrip = status_fail; break; } } @@ -816,7 +823,7 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in } // log the results for this test case - printf(template_row_res.c_str(), + LOG_CNT(template_row_res.c_str(), status_nmse.c_str(), nmse_str, status_roundtrip.c_str()); } } @@ -826,8 +833,8 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const in } int main(int argc, char ** argv) { - // init the logger at max verbosity. filter with a custom callback respecting the user-configure verbosity - common_log_set_verbosity_thold(LOG_LEVEL_DEBUG); + common_params params; + params.model.path = "."; // this test takes no model common_init(); std::random_device rd; @@ -838,58 +845,79 @@ int main(int argc, char ** argv) { int verbosity = LOG_LEVEL_ERROR; + std::vector common_argv; + common_argv.push_back(argv[0]); for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { usage(argv); return 0; - } - if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--arch") == 0) { + } else if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--arch") == 0) { if (i + 1 < argc) { const std::string arch_name = argv[++i]; arch = llm_arch_from_string(arch_name); if (arch == LLM_ARCH_UNKNOWN) { LOG_ERR("%s: unkown LLM architecture: %s\n", __func__, arch_name.c_str()); + common_log_flush(common_log_main()); return 1; } } else { usage(argv); return 1; } - } - if (strcmp(argv[i], "-s") == 0 || strcmp(argv[i], "--seed") == 0) { + } else if (strcmp(argv[i], "-s") == 0 || strcmp(argv[i], "--seed") == 0) { if (i + 1 < argc) { seed = std::stoull(argv[++i]); } else { usage(argv); return 1; } - } - if (strcmp(argv[i], "-v") == 0) { + } else if (strcmp(argv[i], "-v") == 0) { if (i + 1 < argc) { verbosity = std::stoull(argv[++i]); } else { usage(argv); return 1; } - } - if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--out") == 0) { + } else if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--out") == 0) { if (i + 1 < argc) { out = argv[++i]; } else { usage(argv); return 1; } + } else if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it } } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + // the parser sets the threshold from the parsed verbosity; keep the max verbosity so that the + // test's own -v N stays the effective filter, unless the user asked for a lower threshold (--errors-only) + common_log_set_verbosity_thold(params.verbosity < LOG_DEFAULT_LLAMA ? params.verbosity : LOG_LEVEL_DEBUG); + + LOG("%s: running\n", "test-llama-archs"); + common_log_flush(common_log_main()); // the seed line below goes to stdout directly, so drain the start line first + printf("%s: using seed %zu\n", __func__, seed); try { if (!out.empty()) { - return save_models(arch, seed, verbosity, out); + const int rc = save_models(arch, seed, verbosity, out); + LOG("%s: %s\n", "test-llama-archs", rc == 0 ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); + return rc; } - return test_backends(arch, seed, verbosity); + const int rc = test_backends(arch, seed, verbosity); + LOG("%s: %s\n", "test-llama-archs", rc == 0 ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); + return rc; } catch (const std::exception & err) { - fprintf(stderr, "encountered runtime error: %s\n", err.what()); + LOG_ERR("encountered runtime error: %s\n", err.what()); + LOG("%s: %s\n", "test-llama-archs", "FAILED"); + common_log_flush(common_log_main()); return -1; } } diff --git a/tests/test-llama-grammar.cpp b/tests/test-llama-grammar.cpp index 25f432a2f5..87d63800ef 100644 --- a/tests/test-llama-grammar.cpp +++ b/tests/test-llama-grammar.cpp @@ -6,11 +6,24 @@ #include "../src/llama-grammar.h" +#include "arg.h" +#include "common.h" +#include "log.h" + #include #include -int main() +int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-llama-grammar"); + llama_grammar_parser parsed_grammar; std::vector> expected = { @@ -183,10 +196,10 @@ int main() // pretty print error message before asserting if (expected_element.type != element->type || expected_element.value != element->value) { - fprintf(stderr, "index: %d\n", index); - fprintf(stderr, "expected_element: %d, %u\n", expected_element.type, expected_element.value); - fprintf(stderr, "actual_element: %d, %u\n", element->type, element->value); - fprintf(stderr, "expected_element != actual_element\n"); + LOG_ERR("index: %d\n", index); + LOG_ERR("expected_element: %d, %u\n", expected_element.type, expected_element.value); + LOG_ERR("actual_element: %d, %u\n", element->type, element->value); + LOG_ERR("expected_element != actual_element\n"); } assert(expected_element.type == element->type && expected_element.value == element->value); @@ -402,5 +415,8 @@ int main() llama_grammar_free_impl(grammar); + // the test aborts on a mismatch, so reaching this point means it passed + LOG("%s: %s\n", "test-llama-grammar", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-log.cpp b/tests/test-log.cpp index ae4a6606bd..3f74bf607f 100644 --- a/tests/test-log.cpp +++ b/tests/test-log.cpp @@ -1,9 +1,20 @@ +#include "arg.h" +#include "common.h" #include "log.h" #include #include -int main() { +int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-log"); + const int n_thread = 8; std::thread threads[n_thread]; @@ -12,13 +23,12 @@ int main() { const int n_msg = 1000; for (int j = 0; j < n_msg; j++) { - const int log_type = std::rand() % 4; + const int log_type = std::rand() % 3; switch (log_type) { case 0: LOG_INF("Thread %d: %d\n", i, j); break; - case 1: LOG_WRN("Thread %d: %d\n", i, j); break; - case 2: LOG_ERR("Thread %d: %d\n", i, j); break; - case 3: LOG_DBG("Thread %d: %d\n", i, j); break; + case 1: LOG_TRC("Thread %d: %d\n", i, j); break; + case 2: LOG_DBG("Thread %d: %d\n", i, j); break; default: break; } @@ -35,6 +45,8 @@ int main() { threads[i].join(); } + LOG("%s: %s\n", "test-log", "PASSED"); + common_log_flush(common_log_main()); // We explicitly free the logger singleton to avoid hanging on Windows // related to timing issues of thread startup and DLL teardown diff --git a/tests/test-model-load-cancel.cpp b/tests/test-model-load-cancel.cpp index f8139b26d7..5427e87b33 100644 --- a/tests/test-model-load-cancel.cpp +++ b/tests/test-model-load-cancel.cpp @@ -1,17 +1,50 @@ #include "llama.h" #include "common.h" +#include "arg.h" +#include "log.h" #include +#include + +int main(int argc, char ** argv) { + // the model path is this test's only positional argument + char * model_argv[2] = { argv[0], nullptr }; + + { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + + std::vector common_argv; + common_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { + if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else if (model_argv[1] == nullptr) { + model_argv[1] = argv[i]; + } + } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + } + + // falls back to LLAMACPP_TEST_MODELFILE, or warns and exits if no model is given + auto * model_path = common_get_model_or_exit(model_argv[1] == nullptr ? 1 : 2, model_argv); + + // that call exits when no model is given, so the verdict below is only reached with a model + LOG("%s: running\n", "test-model-load-cancel"); -int main(int argc, char *argv[] ) { - auto * model_path = common_get_model_or_exit(argc, argv); auto * file = fopen(model_path, "r"); if (file == nullptr) { - fprintf(stderr, "no model at '%s' found\n", model_path); + LOG_ERR("no model at '%s' found\n", model_path); + LOG("%s: %s\n", "test-model-load-cancel", "FAILED"); + common_log_flush(common_log_main()); return EXIT_FAILURE; } - fprintf(stderr, "using '%s'\n", model_path); + LOG_INF("using '%s'\n", model_path); fclose(file); llama_backend_init(); @@ -23,5 +56,7 @@ int main(int argc, char *argv[] ) { }; auto * model = llama_model_load_from_file(model_path, params); llama_backend_free(); + LOG("%s: %s\n", "test-model-load-cancel", model == nullptr ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); return model == nullptr ? EXIT_SUCCESS : EXIT_FAILURE; } diff --git a/tests/test-model-resolution.cpp b/tests/test-model-resolution.cpp index 5191e77514..20fe2a7b00 100644 --- a/tests/test-model-resolution.cpp +++ b/tests/test-model-resolution.cpp @@ -468,7 +468,16 @@ static void test_task_assembly() { g_repos.clear(); } -int main(void) { +int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-model-resolution"); + // unbuffered, so a crash cannot swallow the reports already printed setvbuf(stdout, nullptr, _IONBF, 0); setvbuf(stderr, nullptr, _IONBF, 0); @@ -502,5 +511,10 @@ int main(void) { std::filesystem::remove_all(cache_dir); printf("test-model-resolution: all tests OK\n"); + + // the log worker was paused above, restart it for the verdict + common_log_resume(common_log_main()); + LOG("%s: %s\n", "test-model-resolution", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-mtmd-c-api.c b/tests/test-mtmd-c-api.c index 664c56cf80..c766fc427a 100644 --- a/tests/test-mtmd-c-api.c +++ b/tests/test-mtmd-c-api.c @@ -6,7 +6,10 @@ #include "mtmd.h" #include "mtmd-helper.h" +// this test is C, it cannot use the common LOG_* macros int main(void) { + printf("%s: running\n", "test-mtmd-c-api"); + printf("\n\nTesting libmtmd C API...\n"); printf("--------\n\n"); @@ -17,6 +20,7 @@ int main(void) { if (!chunks) { fprintf(stderr, "Failed to create input chunks\n"); + printf("%s: %s\n", "test-mtmd-c-api", "FAILED"); return 1; } @@ -168,5 +172,7 @@ int main(void) { printf("\n\nDONE: test libmtmd C API...\n"); + printf("%s: %s\n", "test-mtmd-c-api", "PASSED"); + return 0; } diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp index 2ec6b23915..9ce451f49b 100644 --- a/tests/test-mtmd-impl.cpp +++ b/tests/test-mtmd-impl.cpp @@ -1,3 +1,4 @@ +#include "arg.h" #include "testing.h" #include "mtmd-image.h" @@ -142,17 +143,37 @@ MAKE_TEST(test_temporal_merge_grouping) { // int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + testing t(std::cout); t.verbose = true; // usage: test-mtmd-impl [filter_regex] + std::vector common_argv; + common_argv.push_back(argv[0]); for (int i = 1; i < argc; i++) { - t.set_filter(argv[i]); + if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else { + t.set_filter(argv[i]); + } } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + // log.h cannot be included here (clip-impl.h redefines LOG_*), so print the start line directly + printf("%s: running\n", "test-mtmd-impl"); for (const auto & e : test_registry::all()) { t.test(e.name, e.fn); } - return t.summary(); + const int rc = t.summary(); + // log.h cannot be included here (clip-impl.h redefines LOG_*), so print the result line directly + printf("%s: %s\n", "test-mtmd-impl", rc == 0 ? "PASSED" : "FAILED"); + return rc; } diff --git a/tests/test-opt.cpp b/tests/test-opt.cpp index 8dcb4a7dbf..7b4399b272 100644 --- a/tests/test-opt.cpp +++ b/tests/test-opt.cpp @@ -5,6 +5,10 @@ #include "ggml-backend.h" #include "ggml-opt.h" +#include "arg.h" +#include "common.h" +#include "log.h" + #include #include #include @@ -171,25 +175,47 @@ static void helper_free_ctx_data(struct helper_ctx_data ctx_data) { ggml_opt_dataset_free(ctx_data.dataset_unsupervised); } +// verdict of a single sub-test; INFO, so hidden with --errors-only +static void print_ok_detail(bool subtest_ok) { + const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN; + const char * const col_ok = use_color ? "\033[1;32m" : ""; + const char * const col_fail = use_color ? "\033[1;31m" : ""; + const char * const col_end = use_color ? "\033[0m" : ""; + if (subtest_ok) { + LOG_CNT("%sOK%s\n", col_ok, col_end); + } else { + LOG_CNT("%sFAIL%s\n", col_fail, col_end); + } +} + +// verdict of the whole test; always visible static void print_ok(bool subtest_ok) { - printf(subtest_ok ? "\033[1;32mOK\033[0m\n" : "\033[1;31mFAIL\033[0m\n"); + const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN; + const char * const col_ok = use_color ? "\033[1;32m" : ""; + const char * const col_fail = use_color ? "\033[1;31m" : ""; + const char * const col_end = use_color ? "\033[0m" : ""; + if (subtest_ok) { + printf("%sOK%s\n", col_ok, col_end); + } else { + printf("%sFAIL%s\n", col_fail, col_end); + } } static void helper_after_test( enum ggml_opt_optimizer_type optim, const char * func, const bool high_level, const std::string options, const std::string subtest, const bool subtest_ok, int & ntest, int & npass) { - printf(" %s(high_level=%s%s, subtest=%s, optimizer=%s): ", - func, high_level ? "yes" : "no", options.c_str(), subtest.c_str(), ggml_opt_optimizer_name(optim)); - print_ok(subtest_ok); + LOG_CNT(" %s(high_level=%s%s, subtest=%s, optimizer=%s): ", + func, high_level ? "yes" : "no", options.c_str(), subtest.c_str(), ggml_opt_optimizer_name(optim)); + print_ok_detail(subtest_ok); if (subtest_ok) npass++; ntest++; } static void print_ok(const char * func, bool subtest_ok, int & npass, int & ntest, const char * args = "") { - printf(" %s(%s): ", func, args); - print_ok(subtest_ok); + LOG_CNT(" %s(%s): ", func, args); + print_ok_detail(subtest_ok); if (subtest_ok) npass++; ++ntest; @@ -263,13 +289,13 @@ static std::pair test_dataset( } } - printf(" %s(shuffle=%s, ndata_shard=%" PRId64 ", ndata_batch=%" PRId64 "): ", - __func__, shuffle ? "yes" : "no", ndata_shard, ndata_batch); + LOG_CNT(" %s(shuffle=%s, ndata_shard=%" PRId64 ", ndata_batch=%" PRId64 "): ", + __func__, shuffle ? "yes" : "no", ndata_shard, ndata_batch); if (subtest_ok) { - printf("\033[1;32mOK\033[0m\n"); + print_ok_detail(true); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + print_ok_detail(false); } ntest++; } @@ -310,12 +336,12 @@ static std::pair test_grad( subtest_ok = false; } } - printf(" %s(): ", __func__); + LOG_CNT(" %s(): ", __func__); if (subtest_ok) { - printf("\033[1;32mOK\033[0m\n"); + print_ok_detail(true); npass++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + print_ok_detail(false); } ntest++; } @@ -896,7 +922,16 @@ static std::pair test_backend( } -int main(void) { +int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-opt"); + ggml_log_set(nullptr, nullptr); ggml_backend_load_all(); const size_t dev_count = ggml_backend_dev_count(); @@ -931,12 +966,12 @@ int main(void) { backends_modded.data(), nullptr, backends_modded.size(), GGML_DEFAULT_GRAPH_SIZE, false, true); char const* devname = ggml_backend_dev_name(devs[i]); - printf("Backend %zu/%zu: %s\n", i + 1, dev_count, devname); - printf(" Device description: %s\n", ggml_backend_dev_description(devs[i])); + LOG_INF("Backend %zu/%zu: %s\n", i + 1, dev_count, devname); + LOG_INF(" Device description: %s\n", ggml_backend_dev_description(devs[i])); size_t free, total; // NOLINT ggml_backend_dev_memory(devs[i], &free, &total); - printf(" Device memory: %zu MB (%zu MB free)\n", total / 1024 / 1024, free / 1024 / 1024); - printf("\n"); + LOG_INF(" Device memory: %zu MB (%zu MB free)\n", total / 1024 / 1024, free / 1024 / 1024); + LOG_CNT("\n"); bool skip; { @@ -973,21 +1008,27 @@ int main(void) { std::pair result; if (!skip) { result = test_backend(backend_sched, backends[i], optim); - printf(" %d/%d tests passed\n", result.first, result.second); + LOG_CNT(" %d/%d tests passed\n", result.first, result.second); } - printf(" Backend %s %s: ", ggml_backend_name(backends[i]), ggml_opt_optimizer_name(optim)); + const bool use_color = common_log_get_verbosity_thold() > LOG_LEVEL_WARN; + const char * const col_ok = use_color ? "\033[1;32m" : ""; + const char * const col_fail = use_color ? "\033[1;31m" : ""; + const char * const col_skip = use_color ? "\033[0;33m" : ""; + const char * const col_end = use_color ? "\033[0m" : ""; + + LOG_CNT(" Backend %s %s: ", ggml_backend_name(backends[i]), ggml_opt_optimizer_name(optim)); if (skip) { - printf("\033[0;33mSKIPPED\033[0m\n"); + LOG_CNT("%sSKIPPED%s\n", col_skip, col_end); n_ok++; } else if (result.first == result.second) { - printf("\033[1;32mOK\033[0m\n"); + LOG_CNT("%sOK%s\n", col_ok, col_end); n_ok++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + LOG_CNT("%sFAIL%s\n", col_fail, col_end); } ++n_total; - printf("\n"); + LOG_CNT("\n"); ggml_backend_sched_free(backend_sched); } } @@ -996,8 +1037,11 @@ int main(void) { ggml_backend_free(backend); } + common_log_flush(common_log_main()); printf("%zu/%zu backend*optimizer passed\n", n_ok, n_total); bool ok = n_ok == n_total; print_ok(ok); + LOG("%s: %s\n", "test-opt", ok ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); return ok ? 0 : 1; } diff --git a/tests/test-peg-parser.cpp b/tests/test-peg-parser.cpp index 7d22d77612..c3ee87fbb3 100644 --- a/tests/test-peg-parser.cpp +++ b/tests/test-peg-parser.cpp @@ -1,13 +1,40 @@ +#include "arg.h" +#include "common.h" +#include "log.h" + #include #include #include +#include #include "peg-parser/tests.h" int main(int argc, char *argv[]) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + + // this test takes an optional filter as its only positional argument + std::string filter; + std::vector common_argv; + common_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { + if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else if (filter.empty()) { + filter = argv[i]; + } + } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-peg-parser"); + testing t(std::cout); - if (argc >= 2) { - t.set_filter(argv[1]); + if (!filter.empty()) { + t.set_filter(filter); } const char * verbose = getenv("LLAMA_TEST_VERBOSE"); @@ -22,5 +49,8 @@ int main(int argc, char *argv[]) { t.test("serialization", test_json_serialization); t.test("python-dict", test_python_dict_parser); - return t.summary(); + const int rc = t.summary(); + LOG("%s: %s\n", "test-peg-parser", rc == 0 ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); + return rc; } diff --git a/tests/test-quant-type-selection.cpp b/tests/test-quant-type-selection.cpp index 1696ec164c..f241720579 100644 --- a/tests/test-quant-type-selection.cpp +++ b/tests/test-quant-type-selection.cpp @@ -2,6 +2,8 @@ #include "ggml-cpp.h" #include "gguf-model-data.h" #include "llama.h" +#include "arg.h" +#include "log.h" #include #include @@ -121,7 +123,7 @@ static bool parse_snapshot_file(const std::string & path, std::vector sections; if (!parse_snapshot_file(snapshot_path, sections)) { - printf(" SKIP (could not read snapshot file: %s)\n\n", snapshot_path.c_str()); + LOG_INF(" SKIP (could not read snapshot file: %s)\n\n", snapshot_path.c_str()); llama_quant_free(qs); llama_model_free(model); total_skip++; @@ -469,9 +476,9 @@ static int run_remote_tests(const std::string & snapshot_dir, const char * argv0 } } - printf(" %s %s: %d/%d ftype sections passed (%d tensors)\n", model_fail == 0 ? "PASS" : "FAIL", name.c_str(), - model_pass, model_pass + model_fail, (int) mt.tensors.size()); - printf("\n"); + LOG_INF(" %s %s: %d/%d ftype sections passed (%d tensors)\n", model_fail == 0 ? "PASS" : "FAIL", name.c_str(), + model_pass, model_pass + model_fail, (int) mt.tensors.size()); + LOG_CNT("\n"); if (model_fail == 0) { total_pass++; @@ -498,23 +505,43 @@ static int run_remote_tests(const std::string & snapshot_dir, const char * argv0 } int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + std::string snapshot_dir = SNAPSHOT_DIR; bool generate = false; + std::vector common_argv; + common_argv.push_back(argv[0]); for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "--generate") == 0) { generate = true; } else if (strcmp(argv[i], "--snapshot-dir") == 0 && i + 1 < argc) { snapshot_dir = argv[++i]; + } else if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it } } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-quant-type-selection"); if (generate) { - return run_generate(snapshot_dir); + const int rc = run_generate(snapshot_dir); + LOG("%s: %s\n", "test-quant-type-selection", rc == 0 ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); + return rc; } // suppress llama log warnings during test (e.g. tensor type fallback messages) llama_log_set([](enum ggml_log_level, const char *, void *) {}, nullptr); - return run_remote_tests(snapshot_dir, argv[0]); + const int rc = run_remote_tests(snapshot_dir, argv[0]); + LOG("%s: %s\n", "test-quant-type-selection", rc == 0 ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); + return rc; } diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index 570fca89a8..9b6cae429e 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -2,6 +2,8 @@ #include "ggml.h" #include "ggml-cpu.h" +#include "arg.h" +#include "log.h" #undef NDEBUG #include @@ -148,8 +150,8 @@ static int test_vec_dot_f32(bool verbose) { const bool failed = !(error < MAX_QUANTIZATION_REFERENCE_ERROR); num_failed += failed; if (failed || verbose) { - printf(" f32 vec_dot n=%4d: %s (ref=%f got=%f err=%f)\n", - n, RESULT_STR[failed], ref, result, error); + LOG_INF(" f32 vec_dot n=%4d: %s (ref=%f got=%f err=%f)\n", + n, RESULT_STR[failed], ref, result, error); } } return num_failed; @@ -182,7 +184,7 @@ static int test_vec_dot_q(bool verbose) { const ggml_type ei = (ggml_type)i; - printf("Testing %s\n", ggml_type_name((ggml_type) i)); + LOG_INF("Testing %s\n", ggml_type_name((ggml_type) i)); ggml_quantize_init(ei); if (qfns_cpu->from_float && qfns->to_float) { @@ -201,14 +203,14 @@ static int test_vec_dot_q(bool verbose) { bool failed = !(total_error < max_quantization_error); num_failed += failed; if (failed || verbose) { - printf("%5s absolute quantization error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], total_error); + LOG_INF("%5s absolute quantization error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], total_error); } const float reference_error = reference_quantization_error(qfns, qfns_cpu, test_size, test_data.data()); failed = !(reference_error < MAX_QUANTIZATION_REFERENCE_ERROR); num_failed += failed; if (failed || verbose) { - printf("%5s reference implementation error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], reference_error); + LOG_INF("%5s reference implementation error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], reference_error); } const float vec_dot_error = dot_product_error(qfns_cpu, type, test_size, test_data.data(), test_data2.data(), nullptr, nullptr, 1); @@ -225,7 +227,7 @@ static int test_vec_dot_q(bool verbose) { failed = !(vec_dot_error < max_allowed_error); num_failed += failed; if (failed || verbose) { - printf("%5s dot product error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error); + LOG_INF("%5s dot product error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error); } // Test nrc=2 path for types that support it @@ -234,7 +236,7 @@ static int test_vec_dot_q(bool verbose) { failed = !(vec_dot_error_nrc2 < max_allowed_error); num_failed += failed; if (failed || verbose) { - printf("%5s dot product error (nrc=2): %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error_nrc2); + LOG_INF("%5s dot product error (nrc=2): %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error_nrc2); } } } @@ -244,19 +246,34 @@ static int test_vec_dot_q(bool verbose) { } int main(int argc, char * argv[]) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + bool verbose = false; + std::vector common_argv; + common_argv.push_back(argv[0]); std::string arg; for (int i = 1; i < argc; i++) { arg = argv[i]; if (arg == "-v") { - verbose = true; + verbose = true; // this test's own -v, not the common --verbose + } else if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it } else { - fprintf(stderr, "error: unknown argument: %s\n", arg.c_str()); + LOG_ERR("error: unknown argument: %s\n", arg.c_str()); + common_log_flush(common_log_main()); return 1; } } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-quantize-fns"); ggml_cpu_init(); @@ -265,9 +282,13 @@ int main(int argc, char * argv[]) { num_failed += test_vec_dot_f32(verbose); num_failed += test_vec_dot_q(verbose); + const bool ok = num_failed == 0; + if (num_failed || verbose) { printf("%d tests failed\n", num_failed); } - return num_failed > 0; + LOG("%s: %s\n", "test-quantize-fns", ok ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); + return ok ? 0 : 1; } diff --git a/tests/test-quantize-perf.cpp b/tests/test-quantize-perf.cpp index cac0782dee..fd316fec59 100644 --- a/tests/test-quantize-perf.cpp +++ b/tests/test-quantize-perf.cpp @@ -2,6 +2,8 @@ #include "ggml.h" #include "ggml-cpu.h" +#include "arg.h" +#include "log.h" #undef NDEBUG #include @@ -101,10 +103,10 @@ static void benchmark_function(size_t size, size_t q_size, int64_t iterations, c min_time_us = std::min(min_time_us, end_time - start_time); } - printf(" min cycles/%d vals : %9.2f\n", QK, QK * min_time_cycles / (float) size); - printf(" avg cycles/%d vals : %9.2f\n", QK, QK * total_time_cycles / (float) (size * iterations)); - printf(" float32 throughput : %9.2f GB/s\n", gigabytes_per_second(4 * size * iterations, total_time_us)); - printf(" quantized throughput : %9.2f GB/s\n", gigabytes_per_second(q_size * iterations, total_time_us)); + LOG_CNT(" min cycles/%d vals : %9.2f\n", QK, QK * min_time_cycles / (float) size); + LOG_CNT(" avg cycles/%d vals : %9.2f\n", QK, QK * total_time_cycles / (float) (size * iterations)); + LOG_CNT(" float32 throughput : %9.2f GB/s\n", gigabytes_per_second(4 * size * iterations, total_time_us)); + LOG_CNT(" quantized throughput : %9.2f GB/s\n", gigabytes_per_second(q_size * iterations, total_time_us)); } static void usage(char * argv[]) { @@ -138,12 +140,16 @@ static void usage(char * argv[]) { } int main(int argc, char * argv[]) { + common_init(); + quantize_perf_params params {}; // read command line bool invalid_param = false; std::string arg; + std::vector common_argv; + common_argv.push_back(argv[0]); for (int i = 1; i < argc; i++) { arg = argv[i]; @@ -154,7 +160,7 @@ int main(int argc, char * argv[]) { } size_t size = std::stoi(argv[i]); if (size % 32 != 0) { - fprintf(stderr, "error: size %zu not divisible by 32\n", size); + LOG_ERR("error: size %zu not divisible by 32\n", size); invalid_param = true; break; } @@ -203,7 +209,7 @@ int main(int argc, char * argv[]) { } int alignment = std::stoi(argv[i]); if (alignment < 0 || alignment > MAX_ALIGNMENT) { - fprintf(stderr, "error: alignment-offset must be less than %d\n", MAX_ALIGNMENT); + LOG_ERR("error: alignment-offset must be less than %d\n", MAX_ALIGNMENT); invalid_param = true; break; } @@ -215,7 +221,7 @@ int main(int argc, char * argv[]) { } int number = std::stoi(argv[i]); if (number < 0 || number > MAX_ITERATIONS) { - fprintf(stderr, "error: iterations must be less than %d\n", MAX_ITERATIONS); + LOG_ERR("error: iterations must be less than %d\n", MAX_ITERATIONS); invalid_param = true; break; } @@ -223,15 +229,31 @@ int main(int argc, char * argv[]) { } else if ((arg == "-h") || (arg == "--help")) { usage(argv); return 1; + } else if (arg[0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it } else { - fprintf(stderr, "error: unknown argument: %s\n", arg.c_str()); + LOG_ERR("error: unknown argument: %s\n", arg.c_str()); + common_log_flush(common_log_main()); return 1; } } if (invalid_param) { - fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str()); + LOG_ERR("error: invalid parameter for argument: %s\n", arg.c_str()); + common_log_flush(common_log_main()); return 1; } + common_argv.push_back(nullptr); + + // the test's own "params" is declared above, so scope the parsed one + { + common_params params; + params.model.path = "."; // this test takes no model + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + } + + LOG("%s: running\n", "test-quantize-perf"); if (params.test_sizes.empty()) { params.test_sizes.push_back(L1_SIZE); @@ -271,14 +293,14 @@ int main(int argc, char * argv[]) { } if (qfns_cpu->from_float && qfns->to_float) { - printf("%s\n", ggml_type_name(type)); + LOG_INF(" running %s\n", ggml_type_name(type)); ggml_quantize_init(type); if (params.op_quantize_row_q_reference) { - printf(" quantize_row_q_reference\n"); + LOG_CNT(" quantize_row_q_reference\n"); for (size_t size : params.test_sizes) { - printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024)); + LOG_CNT(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024)); auto quantize_fn = [&](void) -> float { qfns->from_float_ref(test_data1, test_q1, size); return test_q1[0]; @@ -286,13 +308,13 @@ int main(int argc, char * argv[]) { size_t quantized_size = ggml_row_size(type, size); benchmark_function(size, quantized_size, iterations, quantize_fn); } - printf("\n"); + LOG_CNT("\n"); } if (params.op_quantize_row_q) { - printf(" quantize_row_q\n"); + LOG_CNT(" quantize_row_q\n"); for (size_t size : params.test_sizes) { - printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024)); + LOG_CNT(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024)); auto quantize_fn = [&](void) -> float { qfns_cpu->from_float(test_data1, test_q1, size); return test_q1[0]; @@ -300,14 +322,14 @@ int main(int argc, char * argv[]) { size_t quantized_size = ggml_row_size(type, size); benchmark_function(size, quantized_size, iterations, quantize_fn); } - printf("\n"); + LOG_CNT("\n"); } if (params.op_dequantize_row_q) { - printf(" dequantize_row_q\n"); + LOG_CNT(" dequantize_row_q\n"); qfns_cpu->from_float(test_data1, test_q1, largest); for (size_t size : params.test_sizes) { - printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024)); + LOG_CNT(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024)); auto quantize_fn = [&](void) -> float { qfns->to_float(test_q1, test_out, size); return test_out[0]; @@ -315,13 +337,13 @@ int main(int argc, char * argv[]) { size_t quantized_size = ggml_row_size(type, size); benchmark_function(size, quantized_size, iterations, quantize_fn); } - printf("\n"); + LOG_CNT("\n"); } if (params.op_quantize_row_q_dot) { - printf(" quantize_row_q_dot\n"); + LOG_CNT(" quantize_row_q_dot\n"); for (size_t size : params.test_sizes) { - printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024)); + LOG_CNT(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024)); auto quantize_fn = [&](void) -> float { const auto * vdot = ggml_get_type_traits_cpu(qfns_cpu->vec_dot_type); vdot->from_float(test_data1, test_q1, size); @@ -330,15 +352,15 @@ int main(int argc, char * argv[]) { size_t quantized_size = ggml_row_size(type, size); benchmark_function(size, quantized_size, iterations, quantize_fn); } - printf("\n"); + LOG_CNT("\n"); } if (params.op_vec_dot_q) { - printf(" vec_dot_q\n"); + LOG_CNT(" vec_dot_q\n"); qfns_cpu->from_float(test_data1, test_q1, largest); qfns_cpu->from_float(test_data2, test_q2, largest); for (size_t size : params.test_sizes) { - printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024)); + LOG_CNT(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024)); auto quantize_fn = [&](void) -> float { float result; qfns_cpu->vec_dot(size, &result, 0, test_q1, 0, test_q2, 0, 1); @@ -347,10 +369,13 @@ int main(int argc, char * argv[]) { size_t quantized_size = ggml_row_size(type, size); benchmark_function(size, quantized_size, iterations, quantize_fn); } - printf("\n"); + LOG_CNT("\n"); } } } + // this benchmark has no failure path, so reaching this point means it all passed + LOG("%s: %s\n", "test-quantize-perf", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-quantize-stats.cpp b/tests/test-quantize-stats.cpp index e07d75b7e7..4e5c286839 100644 --- a/tests/test-quantize-stats.cpp +++ b/tests/test-quantize-stats.cpp @@ -1,7 +1,9 @@ #include "llama.h" +#include "arg.h" #include "build-info.h" #include "common.h" +#include "log.h" #include "../src/llama-model.h" @@ -46,29 +48,31 @@ struct error_stats { uint64_t error_histogram[HISTOGRAM_BUCKETS]; }; -static void quantize_stats_print_usage(int /*argc*/, char ** argv) { +// print the usage text at the given level: INFO for -h, ERROR when it follows a bad argument +static void quantize_stats_print_usage(int /*argc*/, char ** argv, ggml_log_level level) { quantize_stats_params params; - fprintf(stderr, "usage: %s [options]\n", argv[0]); - fprintf(stderr, "\n"); - fprintf(stderr, "options:\n"); - fprintf(stderr, " -h, --help show this help message and exit\n"); - fprintf(stderr, " -m FNAME, --model FNAME\n"); - fprintf(stderr, " model path (default: %s)\n", params.model.c_str()); - fprintf(stderr, " -r, --reference\n"); - fprintf(stderr, " use reference implementation (default: false)\n"); - fprintf(stderr, " -v, --verbose\n"); - fprintf(stderr, " verbose output (default: false)\n"); - fprintf(stderr, " -p, --per-layer-stats\n"); - fprintf(stderr, " print stats per layer (default: false)\n"); - fprintf(stderr, " --histogram\n"); - fprintf(stderr, " print error histogram (default: false)\n"); - fprintf(stderr, " -l LAYER, --include-layer LAYER\n"); - fprintf(stderr, " only test layers matching pattern\n"); - fprintf(stderr, " -L LAYER, --exclude-layer LAYER\n"); - fprintf(stderr, " exclude layers matching pattern\n"); - fprintf(stderr, " -t TYPE, --type TYPE\n"); - fprintf(stderr, " only test given type (q4_0, q4_1)\n"); - fprintf(stderr, "\n"); + const int verbosity = level == GGML_LOG_LEVEL_ERROR ? LOG_LEVEL_ERROR : LOG_LEVEL_INFO; + LOG_TMPL(level, verbosity, "usage: %s [options]\n", argv[0]); + LOG_TMPL(level, verbosity, "\n"); + LOG_TMPL(level, verbosity, "options:\n"); + LOG_TMPL(level, verbosity, " -h, --help show this help message and exit\n"); + LOG_TMPL(level, verbosity, " -m FNAME, --model FNAME\n"); + LOG_TMPL(level, verbosity, " model path (default: %s)\n", params.model.c_str()); + LOG_TMPL(level, verbosity, " -r, --reference\n"); + LOG_TMPL(level, verbosity, " use reference implementation (default: false)\n"); + LOG_TMPL(level, verbosity, " -v, --verbose\n"); + LOG_TMPL(level, verbosity, " verbose output (default: false)\n"); + LOG_TMPL(level, verbosity, " -p, --per-layer-stats\n"); + LOG_TMPL(level, verbosity, " print stats per layer (default: false)\n"); + LOG_TMPL(level, verbosity, " --histogram\n"); + LOG_TMPL(level, verbosity, " print error histogram (default: false)\n"); + LOG_TMPL(level, verbosity, " -l LAYER, --include-layer LAYER\n"); + LOG_TMPL(level, verbosity, " only test layers matching pattern\n"); + LOG_TMPL(level, verbosity, " -L LAYER, --exclude-layer LAYER\n"); + LOG_TMPL(level, verbosity, " exclude layers matching pattern\n"); + LOG_TMPL(level, verbosity, " -t TYPE, --type TYPE\n"); + LOG_TMPL(level, verbosity, " only test given type (q4_0, q4_1)\n"); + LOG_TMPL(level, verbosity, "\n"); } // Check if a layer is included/excluded by command line @@ -121,14 +125,14 @@ static void print_error_stats(const std::string & name, const error_stats & stat double rmse = sqrt(stats.total_error / (double) stats.num_samples); double median = find_quantile(stats, .5); double pct95 = find_quantile(stats, .95); - printf("%-50s: rmse %.8f, maxerr %.8f, 95pct<%.4f, median<%.4f\n", name.c_str(), rmse, stats.max_error, pct95, median); + LOG_INF("%-50s: rmse %.8f, maxerr %.8f, 95pct<%.4f, median<%.4f\n", name.c_str(), rmse, stats.max_error, pct95, median); if (print_histogram) { - printf("Error distribution:\n"); + LOG_INF("Error distribution:\n"); for (size_t i = 0; i < HISTOGRAM_BUCKETS; i++) { double lower = i * HISTOGRAM_RANGE / HISTOGRAM_BUCKETS; double upper = (i+1) * HISTOGRAM_RANGE / HISTOGRAM_BUCKETS; if (i == HISTOGRAM_BUCKETS -1) upper = INFINITY; - printf("[%3.4f, %3.4f): %11" PRIu64 "\n", lower, upper, stats.error_histogram[i]); + LOG_INF("[%3.4f, %3.4f): %11" PRIu64 "\n", lower, upper, stats.error_histogram[i]); } } } @@ -226,6 +230,8 @@ static void test_roundtrip_on_layer( } int main(int argc, char ** argv) { + common_init(); + ggml_time_init(); quantize_stats_params params; @@ -235,11 +241,14 @@ int main(int argc, char ** argv) { int max_thread = 0; bool invalid_param = false; std::string arg; + std::vector common_argv; + common_argv.push_back(argv[0]); for (int i = 1; i < argc; i++) { arg = argv[i]; if (arg == "-h" || arg == "--help") { - quantize_stats_print_usage(argc, argv); + quantize_stats_print_usage(argc, argv, GGML_LOG_LEVEL_INFO); + common_log_flush(common_log_main()); exit(0); } else if (arg == "-r" || arg == "--reference") { params.reference = true; @@ -280,7 +289,7 @@ int main(int argc, char ** argv) { if (j < GGML_TYPE_COUNT) { params.include_types.push_back((ggml_type) j); } else { - fprintf(stderr, "error: %s not in list of types\n", argv[i]); + LOG_ERR("error: %s not in list of types\n", argv[i]); invalid_param = true; } } else if (arg == "-n" || arg == "--num-threads") { @@ -289,22 +298,38 @@ int main(int argc, char ** argv) { break; } max_thread = atoi(argv[i]); + } else if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it } else { - fprintf(stderr, "error: unknown argument: %s\n", arg.c_str()); - quantize_stats_print_usage(argc, argv); + LOG_ERR("error: unknown argument: %s\n", arg.c_str()); + quantize_stats_print_usage(argc, argv, GGML_LOG_LEVEL_ERROR); + common_log_flush(common_log_main()); return 1; } } if (invalid_param) { - fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str()); - quantize_stats_print_usage(argc, argv); + LOG_ERR("error: invalid parameter for argument: %s\n", arg.c_str()); + quantize_stats_print_usage(argc, argv, GGML_LOG_LEVEL_ERROR); + common_log_flush(common_log_main()); return 1; } + common_argv.push_back(nullptr); + + // the test's own "params" is declared above, so scope the parsed one + { + common_params params; + params.model.path = "."; // this test takes no model + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + } + + LOG("%s: running\n", "test-quantize-stats"); llama_print_build_info(llama_version()); // load the model - fprintf(stderr, "Loading model\n"); + LOG_INF("Loading model\n"); const int64_t t_main_start_us = ggml_time_us(); llama_model * model; @@ -317,7 +342,9 @@ int main(int argc, char ** argv) { model = llama_model_load_from_file(params.model.c_str(), mparams); if (model == NULL) { - fprintf(stderr, "%s: error: failed to load model '%s'\n", __func__, params.model.c_str()); + LOG_ERR("%s: error: failed to load model '%s'\n", __func__, params.model.c_str()); + LOG("%s: %s\n", "test-quantize-stats", "FAILED"); + common_log_flush(common_log_main()); return 1; } @@ -327,8 +354,10 @@ int main(int argc, char ** argv) { ctx = llama_init_from_model(model, cparams); if (ctx == NULL) { - fprintf(stderr, "%s: error: failed to create context with model '%s'\n", __func__, params.model.c_str()); + LOG_ERR("%s: error: failed to create context with model '%s'\n", __func__, params.model.c_str()); llama_model_free(model); + LOG("%s: %s\n", "test-quantize-stats", "FAILED"); + common_log_flush(common_log_main()); return 1; } } @@ -344,15 +373,17 @@ int main(int argc, char ** argv) { continue; } if (params.verbose) { - printf("%s: type %s, size %" PRId64 "\n", kv_tensor.first.c_str(), ggml_type_name(kv_tensor.second->type), ggml_nelements(kv_tensor.second)); + LOG_INF("%s: type %s, size %" PRId64 "\n", kv_tensor.first.c_str(), ggml_type_name(kv_tensor.second->type), ggml_nelements(kv_tensor.second)); } if (kv_tensor.second->type == GGML_TYPE_F16) { is_f16 = true; } else if (kv_tensor.second->type != GGML_TYPE_F32) { - fprintf(stderr, "%s: error: Quantization should be tested with a float model, " + LOG_ERR("%s: error: Quantization should be tested with a float model, " "this model contains already quantized layers (%s is type %d)\n", __func__, kv_tensor.first.c_str(), kv_tensor.second->type); llama_free(ctx); llama_model_free(model); + LOG("%s: %s\n", "test-quantize-stats", "FAILED"); + common_log_flush(common_log_main()); return 1; } included_layers++; @@ -360,9 +391,9 @@ int main(int argc, char ** argv) { } if (is_f16) { - printf("note: source model is f16\n"); + LOG_INF("note: source model is f16\n"); } - printf("testing %d layers with max size %" PRId64 "\n", included_layers, max_nelements); + LOG_INF("testing %d layers with max size %" PRId64 "\n", included_layers, max_nelements); // allocate scratch space std::vector input_scratch; std::vector quantized_scratch; @@ -378,7 +409,7 @@ int main(int argc, char ** argv) { const auto * qfns_cpu = ggml_get_type_traits_cpu(type); if (qfns_cpu->from_float && qfns->to_float) { if (params.verbose) { - printf("testing %s ...\n", ggml_type_name(type)); + LOG_INF("testing %s ...\n", ggml_type_name(type)); } ggml_quantize_init(type); @@ -390,7 +421,7 @@ int main(int argc, char ** argv) { continue; } if (params.verbose) { - printf(" %s ...\n", kv_tensor.first.c_str()); + LOG_INF(" %s ...\n", kv_tensor.first.c_str()); } std::string layer_name { ggml_type_name(type) }; layer_name += "::" + kv_tensor.first; @@ -423,5 +454,7 @@ int main(int argc, char ** argv) { printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0); } + LOG("%s: %s\n", "test-quantize-stats", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-reasoning-budget.cpp b/tests/test-reasoning-budget.cpp index 3bcc77e173..3aa4166d0c 100644 --- a/tests/test-reasoning-budget.cpp +++ b/tests/test-reasoning-budget.cpp @@ -4,6 +4,10 @@ #include "llama.h" #include "ggml.h" +#include "arg.h" +#include "common.h" +#include "log.h" + #ifdef NDEBUG #undef NDEBUG #endif @@ -85,7 +89,7 @@ static void test_reasoning_budget( llama_sampler_accept(sampler, sequence[i]); - fprintf(stderr, " i=%zu: token=%d, finite_count=%zu, finite_token=%d\n", i, (int)sequence[i], finite_count, (int)finite_token); + LOG_TRC(" i=%zu: token=%d, finite_count=%zu, finite_token=%d\n", i, (int)sequence[i], finite_count, (int)finite_token); if (finite_count == 1) { if (actual_force_start == SIZE_MAX) { @@ -103,28 +107,28 @@ static void test_reasoning_budget( // Verify forcing occurred at expected positions if (expected_force_start == SIZE_MAX) { if (actual_force_start != SIZE_MAX) { - fprintf(stderr, "Test '%s' FAILED: Expected no forcing, but forcing occurred at %zu\n", test_name, actual_force_start); + LOG_ERR("Test '%s' FAILED: Expected no forcing, but forcing occurred at %zu\n", test_name, actual_force_start); GGML_ASSERT(false && "Expected no forcing, but forcing occurred"); } } else { if (actual_force_start == SIZE_MAX) { - fprintf(stderr, "Test '%s' FAILED: Expected forcing but none occurred\n", test_name); + LOG_ERR("Test '%s' FAILED: Expected forcing but none occurred\n", test_name); GGML_ASSERT(false && "Expected forcing but none occurred"); } if (actual_force_start != expected_force_start) { - fprintf(stderr, "Test '%s' FAILED: Forcing started at %zu, expected %zu\n", test_name, actual_force_start, expected_force_start); + LOG_ERR("Test '%s' FAILED: Forcing started at %zu, expected %zu\n", test_name, actual_force_start, expected_force_start); GGML_ASSERT(false && "Forcing started at wrong position"); } } if (expected_force_end != SIZE_MAX) { if (actual_force_end < expected_force_end) { - fprintf(stderr, "Test '%s' FAILED: Forcing ended at %zu, expected >= %zu\n", test_name, actual_force_end, expected_force_end); + LOG_ERR("Test '%s' FAILED: Forcing ended at %zu, expected >= %zu\n", test_name, actual_force_end, expected_force_end); GGML_ASSERT(false && "Forcing ended too early"); } } - fprintf(stderr, " Test '%s' passed (force_start=%zu, force_end=%zu)\n", test_name, actual_force_start, actual_force_end); + LOG_INF(" Test '%s' passed (force_start=%zu, force_end=%zu)\n", test_name, actual_force_start, actual_force_end); (void)sequence; } @@ -255,7 +259,7 @@ static void test_reasoning_budget_force_manual() { // a null sampler is safely ignored GGML_ASSERT(!common_reasoning_budget_force(nullptr)); - fprintf(stderr, " Test 'manual force transition' passed\n"); + LOG_INF(" Test 'manual force transition' passed\n"); } static void test_reasoning_budget_end_match() { @@ -330,7 +334,7 @@ static void test_reasoning_budget_end_match() { // a null sampler is safely ignored GGML_ASSERT(common_reasoning_budget_get_end_match(nullptr) == nullptr); - fprintf(stderr, " Test 'matched end sequence' passed\n"); + LOG_INF(" Test 'matched end sequence' passed\n"); } // UTF-8 boundary detection unit test @@ -358,9 +362,18 @@ static void test_utf8_boundary_detection() { GGML_ASSERT(common_utf8_is_complete(std::string("hello\xC3\xA9", 7))); // ASCII + complete 2-byte } -int main(void) { +int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-reasoning-budget"); + // Reasoning budget sampler tests - printf("Testing reasoning budget sampler... "); + LOG_INF("Testing reasoning budget sampler...\n"); // Test 1: Basic budget with start/end tokens - no forcing (natural end before budget exhausted) { @@ -495,11 +508,13 @@ int main(void) { test_reasoning_budget_force_manual(); test_reasoning_budget_end_match(); - printf("OK (12 tests passed)\n"); + LOG_CNT("OK (12 tests passed)\n"); - printf("Testing UTF-8 boundary detection... "); + LOG_INF("Testing UTF-8 boundary detection...\n"); test_utf8_boundary_detection(); - printf("OK\n"); + LOG_CNT("OK\n"); + LOG("%s: %s\n", "test-reasoning-budget", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-recurrent-state-rollback.cpp b/tests/test-recurrent-state-rollback.cpp index ef05de67d0..9c7c23c457 100644 --- a/tests/test-recurrent-state-rollback.cpp +++ b/tests/test-recurrent-state-rollback.cpp @@ -1,5 +1,6 @@ #include "arg.h" #include "common.h" +#include "log.h" #include "ggml-backend.h" #include "llama.h" @@ -67,7 +68,7 @@ static llama_context * init_ctx(llama_model * model, llama_context_params cparam llama_get_memory(ctx)->state_write(collector); llama_memory_clear(llama_get_memory(ctx), true); if (collector.buffers.empty()) { - fprintf(stderr, "%s : no cache buffers found\n", __func__); + LOG_ERR("%s : no cache buffers found\n", __func__); llama_free(ctx); return nullptr; } @@ -117,7 +118,7 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode llama_context * ctx_roll = make_ctx_multi(); llama_context * ctx_ref = make_ctx_multi(); if (ctx_roll == nullptr || ctx_ref == nullptr) { - fprintf(stderr, "%s : failed to init multi-seq contexts\n", __func__); + LOG_ERR("%s : failed to init multi-seq contexts\n", __func__); return false; } @@ -127,7 +128,7 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode }; if (llama_n_rs_seq(ctx_roll) < n_rollback) { - fprintf(stderr, "%s : skipping because n_rs_seq is too small\n", __func__); + LOG_WRN("%s : skipping because n_rs_seq is too small\n", __func__); cleanup(); return true; } @@ -161,7 +162,7 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode ok = ok && !llama_memory_seq_rm(llama_get_memory(ctx_roll), (llama_seq_id) s, p0 - 1, -1); } if (!ok) { - fprintf(stderr, "%s : multi-seq prefill/rollback failed\n", __func__); + LOG_ERR("%s : multi-seq prefill/rollback failed\n", __func__); cleanup(); return false; } @@ -177,7 +178,7 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode ok = ok && llama_decode(ctx_ref, batch) == 0; llama_batch_free(batch); if (!ok) { - fprintf(stderr, "%s : multi-seq replay decode failed\n", __func__); + LOG_ERR("%s : multi-seq replay decode failed\n", __func__); cleanup(); return false; } @@ -193,7 +194,7 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode const float * l_roll = llama_get_logits_ith(ctx_roll, i); const float * l_ref = llama_get_logits_ith(ctx_ref, i); if (l_roll == nullptr || l_ref == nullptr) { - fprintf(stderr, "%s : missing multi-seq logits at index %u\n", __func__, i); + LOG_ERR("%s : missing multi-seq logits at index %u\n", __func__, i); cleanup(); return false; } @@ -208,13 +209,13 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode } if (diff_max > eps) { - fprintf(stderr, "%s : multi-seq split replay logits mismatch (max diff %g, first at seq %u pos %d)\n", + LOG_ERR("%s : multi-seq split replay logits mismatch (max diff %g, first at seq %u pos %d)\n", __func__, (double) diff_max, seq_first, pos_first); cleanup(); return false; } - fprintf(stderr, "%s : multi-seq split replay matched (max diff %g)\n", __func__, (double) diff_max); + LOG_INF("%s : multi-seq split replay matched (max diff %g)\n", __func__, (double) diff_max); // seq-1-only decodes must be independent of seq 0's content: diverge seq 0 // in ctx_ref only, then compare identical seq-1-only continuations bitwise @@ -251,13 +252,13 @@ static bool test_multi_seq_split_replay(const common_params & params, llama_mode } if (!ok || diff_tail > eps) { - fprintf(stderr, "%s : seq-1-only decode leaked seq 0 state (ok=%d, max diff %g)\n", + LOG_ERR("%s : seq-1-only decode leaked seq 0 state (ok=%d, max diff %g)\n", __func__, ok ? 1 : 0, (double) diff_tail); cleanup(); return false; } - fprintf(stderr, "%s : seq-1-only decode independent of seq 0 (max diff %g)\n", __func__, (double) diff_tail); + LOG_INF("%s : seq-1-only decode independent of seq 0 (max diff %g)\n", __func__, (double) diff_tail); cleanup(); return true; } @@ -269,12 +270,12 @@ static int test_rollback(const common_params & params, llama_model * model, uint llama_context * ctx_src = make_ctx(params, model, fill); llama_context * ctx_dst = make_ctx(params, model, fill); if (ctx_src == nullptr || ctx_dst == nullptr) { - fprintf(stderr, "%s : failed to init contexts\n", __func__); + LOG_ERR("%s : failed to init contexts\n", __func__); return 1; } if (llama_n_rs_seq(ctx_src) == 0) { - fprintf(stderr, "%s : skipping because n_rs_seq is disabled\n", __func__); + LOG_WRN("%s : skipping because n_rs_seq is disabled\n", __func__); llama_free(ctx_src); llama_free(ctx_dst); return 0; @@ -289,13 +290,13 @@ static int test_rollback(const common_params & params, llama_model * model, uint const uint32_t n_rs_seq = llama_n_rs_seq(ctx_src); constexpr uint32_t n_rollback = 3; if (n_rs_seq < n_rollback) { - fprintf(stderr, "%s : skipping because n_rs_seq is too small\n", __func__); + LOG_WRN("%s : skipping because n_rs_seq is too small\n", __func__); llama_free(ctx_src); llama_free(ctx_dst); return 0; } if (tokens.empty()) { - fprintf(stderr, "%s : not enough prompt tokens\n", __func__); + LOG_ERR("%s : not enough prompt tokens\n", __func__); return 1; } tokens.resize(n_rs_seq + 1, tokens.back()); @@ -307,11 +308,11 @@ static int test_rollback(const common_params & params, llama_model * model, uint // Replaying them crosses DSV4's ratio-4 compressor boundary. // Rollback leaves the recurrent memory in a snapshot state (rs_idx != 0). if (!decode_tokens(ctx_src, tokens, n_tokens)) { - fprintf(stderr, "%s : failed to decode prompt\n", __func__); + LOG_ERR("%s : failed to decode prompt\n", __func__); return 1; } if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, rollback_pos, -1)) { - fprintf(stderr, "%s : rollback failed\n", __func__); + LOG_ERR("%s : rollback failed\n", __func__); return 1; } @@ -327,21 +328,21 @@ static int test_rollback(const common_params & params, llama_model * model, uint const llama_pos pos = rollback_pos + i; if (!decode_one(ctx_src, tokens[pos], pos) || !decode_one(ctx_dst, tokens[pos], pos)) { - fprintf(stderr, "%s : %s replay failed at position %d\n", __func__, mode, pos); + LOG_ERR("%s : %s replay failed at position %d\n", __func__, mode, pos); return false; } const float * logits_src = llama_get_logits_ith(ctx_src, 0); const float * logits_dst = llama_get_logits_ith(ctx_dst, 0); if (logits_src == nullptr || logits_dst == nullptr) { - fprintf(stderr, "%s : missing %s logits at position %d\n", __func__, mode, pos); + LOG_ERR("%s : missing %s logits at position %d\n", __func__, mode, pos); return false; } logits_src_replay[i].assign(logits_src, logits_src + n_vocab); for (int token = 0; token < n_vocab; ++token) { if (logit_diff(logits_src[token], logits_dst[token]) > eps) { - fprintf(stderr, "%s : %s logits mismatch at position %d, token %d (%g != %g)\n", + LOG_ERR("%s : %s logits mismatch at position %d, token %d (%g != %g)\n", __func__, mode, pos, token, (double) logits_src[token], (double) logits_dst[token]); return false; } @@ -355,7 +356,7 @@ static int test_rollback(const common_params & params, llama_model * model, uint if (!llama_memory_seq_rm(llama_get_memory(ctx_src), 0, rollback_pos, -1) || !llama_memory_seq_rm(llama_get_memory(ctx_dst), 0, rollback_pos, -1)) { - fprintf(stderr, "%s : partial rollback failed\n", __func__); + LOG_ERR("%s : partial rollback failed\n", __func__); return 1; } @@ -373,7 +374,7 @@ static int test_rollback(const common_params & params, llama_model * model, uint // non-zero at load time. The restore must wipe that state and still match. llama_context * ctx_dirty = make_ctx(params, model, fill); if (ctx_dirty == nullptr) { - fprintf(stderr, "%s : failed to init dirty ctx\n", __func__); + LOG_ERR("%s : failed to init dirty ctx\n", __func__); return 1; } @@ -385,11 +386,11 @@ static int test_rollback(const common_params & params, llama_model * model, uint } } if (!decode_tokens(ctx_dirty, noise, n_tokens)) { - fprintf(stderr, "%s : dirty prompt decode failed\n", __func__); + LOG_ERR("%s : dirty prompt decode failed\n", __func__); return 1; } if (!llama_memory_seq_rm(llama_get_memory(ctx_dirty), 0, rollback_pos, -1)) { - fprintf(stderr, "%s : dirty rollback failed\n", __func__); + LOG_ERR("%s : dirty rollback failed\n", __func__); return 1; } @@ -398,26 +399,26 @@ static int test_rollback(const common_params & params, llama_model * model, uint for (uint32_t i = 0; i < n_rollback; ++i) { const llama_pos pos = rollback_pos + i; if (!decode_one(ctx_dirty, tokens[pos], pos)) { - fprintf(stderr, "%s : dirty replay failed at position %d\n", __func__, pos); + LOG_ERR("%s : dirty replay failed at position %d\n", __func__, pos); return 1; } const float * logits_dirty = llama_get_logits_ith(ctx_dirty, 0); if (logits_dirty == nullptr) { - fprintf(stderr, "%s : missing dirty logits at position %d\n", __func__, pos); + LOG_ERR("%s : missing dirty logits at position %d\n", __func__, pos); return 1; } for (int token = 0; token < n_vocab; ++token) { if (logit_diff(logits_src_replay[i][token], logits_dirty[token]) > eps) { - fprintf(stderr, "%s : dirty-ctx logits mismatch at position %d, token %d (%g != %g)\n", + LOG_ERR("%s : dirty-ctx logits mismatch at position %d, token %d (%g != %g)\n", __func__, pos, token, (double) logits_src_replay[i][token], (double) logits_dirty[token]); return 1; } } } - fprintf(stderr, "%s : recurrent rollback checkpoint restored successfully\n", __func__); + LOG_INF("%s : recurrent rollback checkpoint restored successfully\n", __func__); llama_free(ctx_src); llama_free(ctx_dst); llama_free(ctx_dirty); @@ -442,26 +443,36 @@ int main(int argc, char ** argv) { return 1; } + LOG("%s: running\n", "test-recurrent-state-rollback"); + ggml_backend_load_all(); common_init_result_ptr llama_init = common_init_from_params(params); llama_model * model = llama_init->model(); if (model == nullptr) { - fprintf(stderr, "%s : failed to init model\n", __func__); + LOG_ERR("%s : failed to init model\n", __func__); + LOG("%s: %s\n", "test-recurrent-state-rollback", "FAILED"); + common_log_flush(common_log_main()); return 1; } if (!llama_model_is_recurrent(model) && !llama_model_is_hybrid(model)) { - fprintf(stderr, "%s : skipping for non-recurrent model\n", __func__); + LOG_WRN("%s : skipping for non-recurrent model\n", __func__); + LOG("%s: %s\n", "test-recurrent-state-rollback", "PASSED"); + common_log_flush(common_log_main()); return 0; } for (uint8_t fill : { 0, 0x3e }) { - fprintf(stderr, "%s : testing with cache fill 0x%02x\n", __func__, fill); + LOG_INF("%s : testing with cache fill 0x%02x\n", __func__, fill); if (test_rollback(params, model, fill) != 0) { + LOG("%s: %s\n", "test-recurrent-state-rollback", "FAILED"); + common_log_flush(common_log_main()); return 1; } } + LOG("%s: %s\n", "test-recurrent-state-rollback", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-rope.cpp b/tests/test-rope.cpp index 801e4cd827..d4474127c4 100644 --- a/tests/test-rope.cpp +++ b/tests/test-rope.cpp @@ -1,6 +1,10 @@ #include "ggml.h" #include "ggml-cpu.h" +#include "arg.h" +#include "common.h" +#include "log.h" + #include #include #include @@ -124,7 +128,19 @@ static void ggml_graph_compute_helper(std::vector & buf, ggml_cgraph * ggml_graph_compute(graph, &plan); } -int main(int /*argc*/, const char ** /*argv*/) { +int main(int argc, char ** argv) { + // the body below declares its own "params", so scope the parsed one + { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + } + + LOG("%s: running\n", "test-rope"); + struct ggml_init_params params = { /* .mem_size = */ 128*1024*1024, /* .mem_buffer = */ NULL, @@ -245,12 +261,14 @@ int main(int /*argc*/, const char ** /*argv*/) { // printf("%f %f\n", r1_data[i], r2_data[i]); //} - printf("mode: %d\n", mode); - printf("sum0: %f\n", sum0); - printf("sum1: %f\n", sum1); - printf("diff: %f\n", diff); - printf("rel err: %f\n", diff / sum0); - printf("rel err: %f\n", diff / sum1); + LOG_INF("mode: %d\n", mode); + LOG_INF("sum0: %f\n", sum0); + LOG_INF("sum1: %f\n", sum1); + LOG_INF("diff: %f\n", diff); + LOG_INF("rel err: %f\n", diff / sum0); + LOG_INF("rel err: %f\n", diff / sum1); + // drain the queue, the checks below abort + common_log_flush(common_log_main()); GGML_ASSERT(diff / sum0 < 0.0001f); GGML_ASSERT(diff / sum1 < 0.0001f); @@ -259,5 +277,7 @@ int main(int /*argc*/, const char ** /*argv*/) { ggml_free(ctx0); + LOG("%s: %s\n", "test-rope", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-rpc-multi-server.cpp b/tests/test-rpc-multi-server.cpp index 4502e2ce71..7ea3e1faa8 100644 --- a/tests/test-rpc-multi-server.cpp +++ b/tests/test-rpc-multi-server.cpp @@ -4,12 +4,43 @@ #include "ggml-rpc.h" #include "ggml.h" +#include "arg.h" +#include "common.h" +#include "log.h" + +#include + int main(int argc, char ** argv) { - GGML_ASSERT(argc == 3); + std::vector endpoints; + + { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + + // the test takes two positional endpoints, everything else goes to common_params_parse + std::vector common_argv; + common_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { + if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else { + endpoints.push_back(argv[i]); + } + } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + } + + LOG("%s: running\n", "test-rpc-multi-server"); + + GGML_ASSERT(endpoints.size() == 2); ggml_backend_load_all(); - const char * endpoint_a = argv[1]; - const char * endpoint_b = argv[2]; + const char * endpoint_a = endpoints[0]; + const char * endpoint_b = endpoints[1]; ggml_backend_t backend_a = ggml_backend_rpc_init(endpoint_a, 0); ggml_backend_t backend_b = ggml_backend_rpc_init(endpoint_b, 0); @@ -43,5 +74,8 @@ int main(int argc, char ** argv) { ggml_free(ctx); ggml_backend_free(backend_b); ggml_backend_free(backend_a); + + LOG("%s: %s\n", "test-rpc-multi-server", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-rset-release.cpp b/tests/test-rset-release.cpp index c60801c115..0b84b7d951 100644 --- a/tests/test-rset-release.cpp +++ b/tests/test-rset-release.cpp @@ -5,11 +5,14 @@ #include "llama.h" #include "common.h" +#include "arg.h" +#include "log.h" #include #include #include #include +#include static uint64_t wired_memory() { vm_statistics64_data_t vmstat; @@ -21,7 +24,34 @@ static uint64_t wired_memory() { } int main(int argc, char ** argv) { - auto * model_path = common_get_model_or_exit(argc, argv); + // the model path is this test's only positional argument + char * model_argv[2] = { argv[0], nullptr }; + + { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + + std::vector common_argv; + common_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { + if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else if (model_argv[1] == nullptr) { + model_argv[1] = argv[i]; + } + } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + } + + // falls back to LLAMACPP_TEST_MODELFILE, or warns and exits if no model is given + auto * model_path = common_get_model_or_exit(model_argv[1] == nullptr ? 1 : 2, model_argv); + + // that call exits when no model is given, so the verdict below is only reached with a model + LOG("%s: running\n", "test-rset-release"); llama_backend_init(); @@ -49,5 +79,7 @@ int main(int argc, char ** argv) { llama_backend_free(); + LOG("%s: %s\n", "test-rset-release", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-sampling.cpp b/tests/test-sampling.cpp index 353a5a1a1d..f029d49d9f 100644 --- a/tests/test-sampling.cpp +++ b/tests/test-sampling.cpp @@ -1,6 +1,10 @@ #include "ggml.h" #include "llama.h" +#include "arg.h" +#include "common.h" +#include "log.h" + #ifdef NDEBUG #undef NDEBUG #endif @@ -15,11 +19,11 @@ extern struct llama_sampler * llama_sampler_init_dry_testing(float dry_multiplie static void dump(const llama_token_data_array * cur_p) { for (size_t i = 0; i < cur_p->size; i++) { - printf("%d: %f (%f)\n", cur_p->data[i].id, cur_p->data[i].p, cur_p->data[i].logit); + LOG_CNT("%d: %f (%f)\n", cur_p->data[i].id, cur_p->data[i].p, cur_p->data[i].logit); } } -#define DUMP(__cur_p) do { printf("%s:%d (%s)\n", __FILE__, __LINE__, __func__); dump((__cur_p)); printf("-\n"); } while(0) +#define DUMP(__cur_p) do { LOG_CNT("%s:%d (%s)\n", __FILE__, __LINE__, __func__); dump((__cur_p)); LOG_CNT("-\n"); } while(0) struct sampler_tester { sampler_tester(size_t n_vocab) { @@ -76,6 +80,8 @@ static llama_token sample_dist(llama_sampler * sampler, const std::vector } static void test_dist_singleton_rng() { + LOG_INF(" running %s\n", __func__); + llama_sampler * singleton = llama_sampler_init_dist(4242); llama_sampler * control = llama_sampler_init_dist(4242); @@ -92,6 +98,8 @@ static void test_dist_singleton_rng() { } static void test_temp(const std::vector & probs, const std::vector & probs_expected, float temp) { + LOG_INF(" running %s\n", __func__); + sampler_tester tester(probs, probs_expected); DUMP(&tester.cur_p); @@ -103,6 +111,8 @@ static void test_temp(const std::vector & probs, const std::vector } static void test_temp_ext(const std::vector & probs, const std::vector & probs_expected, float temp, float delta, float exponent) { + LOG_INF(" running %s\n", __func__); + sampler_tester tester(probs, probs_expected); DUMP(&tester.cur_p); @@ -114,6 +124,8 @@ static void test_temp_ext(const std::vector & probs, const std::vector & probs, const std::vector & probs_expected, int k) { + LOG_INF(" running %s\n", __func__); + sampler_tester tester(probs, probs_expected); DUMP(&tester.cur_p); @@ -125,6 +137,8 @@ static void test_top_k(const std::vector & probs, const std::vector & probs, const std::vector & probs_expected, float p) { + LOG_INF(" running %s\n", __func__); + sampler_tester tester(probs, probs_expected); DUMP(&tester.cur_p); @@ -136,6 +150,8 @@ static void test_top_p(const std::vector & probs, const std::vector & probs, const std::vector & probs_expected, float p) { + LOG_INF(" running %s\n", __func__); + sampler_tester tester(probs, probs_expected); DUMP(&tester.cur_p); @@ -147,6 +163,8 @@ static void test_min_p(const std::vector & probs, const std::vector & probs, const std::vector & probs_expected, float p, float t) { + LOG_INF(" running %s\n", __func__); + sampler_tester tester(probs, probs_expected); DUMP(&tester.cur_p); @@ -157,6 +175,8 @@ static void test_xtc(const std::vector & probs, const std::vector } static void test_typical(const std::vector & probs, const std::vector & probs_expected, float p) { + LOG_INF(" running %s\n", __func__); + sampler_tester tester(probs, probs_expected); DUMP(&tester.cur_p); @@ -170,6 +190,8 @@ static void test_penalties( const std::vector & probs, const std::vector & last_tokens, const std::vector & probs_expected, float repeat_penalty, float alpha_frequency, float alpha_presence ) { + LOG_INF(" running %s\n", __func__); + GGML_ASSERT(probs.size() == probs_expected.size()); sampler_tester tester(probs, probs_expected); @@ -194,6 +216,8 @@ static void test_dry( int dry_allowed_length, int dry_penalty_last_n, const std::vector> & seq_breakers ) { + LOG_INF(" running %s\n", __func__); + GGML_ASSERT(probs.size() == expected_probs.size()); sampler_tester tester(probs, expected_probs); @@ -212,6 +236,8 @@ static void test_dry( } static void test_top_n_sigma(const std::vector & probs, const std::vector & probs_expected, int n) { + LOG_INF(" running %s\n", __func__); + sampler_tester tester(probs, probs_expected); DUMP(&tester.cur_p); @@ -224,6 +250,8 @@ static void test_top_n_sigma(const std::vector & probs, const std::vector static void test_sampler_queue(const size_t n_vocab, const std::string & samplers_sequence, const int top_k, const float top_p, const float min_p ) { + LOG_INF(" running %s\n", __func__); + sampler_tester tester(n_vocab); llama_token min_token_id = 0; @@ -293,7 +321,7 @@ static void test_sampler_queue(const size_t n_vocab, const std::string & sampler } } - printf("Sampler queue %3s OK with n_vocab=%05zu top_k=%5d top_p=%f min_p=%f\n", + LOG_CNT("Sampler queue %3s OK with n_vocab=%05zu top_k=%5d top_p=%f min_p=%f\n", samplers_sequence.c_str(), n_vocab, top_k, top_p, min_p); } @@ -312,12 +340,14 @@ static void bench(llama_sampler * cnstr, const char * cnstr_name, const std::vec } const int64_t t_end = ggml_time_us(); llama_sampler_free(cnstr); - printf("%-43s: %8.3f us/iter\n", cnstr_name, (t_end - t_start) / (float)n_iter); + LOG_CNT("%-43s: %8.3f us/iter\n", cnstr_name, (t_end - t_start) / (float)n_iter); } #define BENCH(__cnstr, __data, __n_iter) bench((__cnstr), #__cnstr, (__data), (__n_iter)) static void test_perf() { + LOG_INF(" running %s\n", __func__); + const int n_vocab = 1 << 17; std::vector data; @@ -335,7 +365,16 @@ static void test_perf() { BENCH(llama_sampler_init_xtc (1.0f, 0.1f, 1, 1), data, 32); } -int main(void) { +int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-sampling"); + ggml_time_init(); test_dist_singleton_rng(); @@ -366,12 +405,12 @@ int main(void) { test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f/0.4f}, 1.00f); test_min_p({0.1f, 0.2f, 0.3f, 0.4f}, {0.4f/0.4f}, 1.05f); - printf("XTC should:\n"); + LOG_CNT("XTC should:\n"); test_xtc({0.4f, 0.3f, 0.2f, 0.1f}, {0.1f}, 0.99f, 0.09f); test_xtc({0.4f, 0.3f, 0.2f, 0.1f}, {0.2f, 0.1f}, 0.99f, 0.19f); test_xtc({0.4f, 0.3f, 0.2f, 0.1f}, {0.3f, 0.2f, 0.1f}, 0.99f, 0.29f); - printf("XTC should not:\n"); + LOG_CNT("XTC should not:\n"); test_xtc({0.4f, 0.3f, 0.2f, 0.1f}, {0.4f, 0.3f, 0.2f, 0.1f}, 0.99f, 0.39f); test_typical({0.97f, 0.01f, 0.01f, 0.01f}, {0.97f}, 0.5f); @@ -424,9 +463,12 @@ int main(void) { test_sampler_queue(10000, "mkp", 100, 0.8f, 0.1f); test_sampler_queue(10000, "mpk", 100, 0.8f, 0.1f); - printf("OK\n"); + LOG_CNT("OK\n"); test_perf(); + // the tests abort on failure, so reaching this point means they all passed + LOG("%s: %s\n", "test-sampling", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-save-load-state.cpp b/tests/test-save-load-state.cpp index 74d1ba6c21..dff59701e1 100644 --- a/tests/test-save-load-state.cpp +++ b/tests/test-save-load-state.cpp @@ -35,7 +35,8 @@ static llama_tokens generate_tokens(llama_context * ctx, llama_sampler * smpl, i for (int i = 0; i < n_predict; i++) { auto next_token = llama_sampler_sample(smpl, ctx, -1); - LOG("%d ", next_token); + // one row per generated token: pure noise, TRACE only + LOG_CNTV(LOG_LEVEL_TRACE, "%d ", next_token); result.push_back(next_token); common_batch_clear(batch.get()); @@ -48,6 +49,8 @@ static llama_tokens generate_tokens(llama_context * ctx, llama_sampler * smpl, i n_past++; } + LOG_CNTV(LOG_LEVEL_TRACE, "\n"); + return result; } @@ -71,15 +74,13 @@ static llama_tokens test_baseline(struct llama_model * model, const struct commo return {}; } - LOG("\n=== Test 1: baseline ===\n"); + LOG_CNT("=== Test 1: baseline ===\n"); auto result = generate_tokens(ctx.get(), smpl.get(), n_past, params.n_predict, 0); if (result.empty()) { return {}; } - LOG("\n"); - return result; } @@ -103,7 +104,7 @@ static bool test_seq_rm_isolated( return false; } - LOG("\n=== Test 2: sequence removal isolation ===\n"); + LOG_CNT("=== Test 2: sequence removal isolation ===\n"); const size_t n_tokens = tokens.size() < 128 ? tokens.size() : 128; for (llama_seq_id seq_id = 0; seq_id < 2; ++seq_id) { @@ -156,7 +157,8 @@ static bool test_seq_rm_isolated( return false; } - LOG("PASS\n"); + // per-sub-test verdict: TRACE, the per-model line already reports the result + LOG_TRC("PASS\n"); return true; } @@ -175,7 +177,7 @@ static bool test_state_load(struct llama_model * model, const struct common_para auto smpl = llama_sampler_ptr{llama_sampler_chain_init(sparams)}; llama_sampler_chain_add(smpl.get(), llama_sampler_init_dist(params.sampling.seed)); - LOG("\n=== Test 3: state load ===\n"); + LOG_CNT("=== Test 3: state load ===\n"); // Load state from file llama_tokens unused_sts(tokens.size()); @@ -206,7 +208,7 @@ static bool test_state_load(struct llama_model * model, const struct common_para return false; } - LOG("\nPASS\n"); + LOG_TRC("PASS\n"); return true; } @@ -226,7 +228,7 @@ static bool test_seq_cp_host(struct llama_model * model, const struct common_par auto smpl = llama_sampler_ptr{llama_sampler_chain_init(sparams)}; llama_sampler_chain_add(smpl.get(), llama_sampler_init_dist(params.sampling.seed)); - LOG("\n=== Test 4: seq copy (host) ===\n"); + LOG_CNT("=== Test 4: seq copy (host) ===\n"); // Load state from file llama_tokens unused_sts(tokens.size()); @@ -278,7 +280,7 @@ static bool test_seq_cp_host(struct llama_model * model, const struct common_par return false; } - LOG("\nPASS\n"); + LOG_TRC("PASS\n"); return true; } @@ -298,7 +300,7 @@ static bool test_seq_cp_device(struct llama_model * model, const struct common_p auto smpl = llama_sampler_ptr{llama_sampler_chain_init(sparams)}; llama_sampler_chain_add(smpl.get(), llama_sampler_init_dist(params.sampling.seed)); - LOG("\n=== Test 5: seq copy (device) ===\n"); + LOG_CNT("=== Test 5: seq copy (device) ===\n"); // Load state from file llama_tokens unused_sts(tokens.size()); @@ -350,7 +352,7 @@ static bool test_seq_cp_device(struct llama_model * model, const struct common_p return false; } - LOG("\nPASS\n"); + LOG_TRC("PASS\n"); return true; } @@ -367,7 +369,7 @@ static bool test_seq_cp_scatter(struct llama_model * model, const struct common_ params_ctx.kv_unified = true; auto ctx = llama_context_ptr{llama_init_from_model(model, params_ctx)}; - LOG("\n=== Test %d: seq copy (%s, scatter) ===\n", test_num, on_device ? "device" : "host"); + LOG_CNT("=== Test %d: seq copy (%s, scatter) ===\n", test_num, on_device ? "device" : "host"); const uint32_t flags = on_device ? LLAMA_STATE_SEQ_FLAGS_ON_DEVICE : LLAMA_STATE_SEQ_FLAGS_NONE; @@ -444,7 +446,7 @@ static bool test_seq_cp_scatter(struct llama_model * model, const struct common_ return false; } - LOG("\nPASS\n"); + LOG_TRC("PASS\n"); return true; } @@ -455,7 +457,7 @@ static bool test_state_roundtrip(struct llama_model * model, const struct common auto params_ctx = common_context_params_to_llama(params); auto ctx = llama_context_ptr{llama_init_from_model(model, params_ctx)}; - LOG("\n=== Test 8: state blob round-trip ===\n"); + LOG_CNT("=== Test 8: state blob round-trip ===\n"); if (llama_decode(ctx.get(), llama_batch_get_one(const_cast(tokens.data()), (int32_t) tokens.size()))) { LOG_ERR("\n%s: failed to decode prompt\n", __func__); @@ -503,7 +505,7 @@ static bool test_state_roundtrip(struct llama_model * model, const struct common return false; } - LOG("\nPASS\n"); + LOG_TRC("PASS\n"); return true; } @@ -590,7 +592,7 @@ static bool run_save_load_tests_for_model(const std::string & model_path, const return false; } - LOG("\nAll tests passed.\n"); + LOG_CNT("All tests passed.\n"); return true; } @@ -615,6 +617,7 @@ int main(int argc, char ** argv) { if (strcmp(argv[i], "--models") == 0) { if (i + 1 >= argc) { LOG_ERR("%s: --models requires a directory argument\n", __func__); + common_log_flush(common_log_main()); return 1; } models_dir = argv[i + 1]; @@ -645,12 +648,16 @@ int main(int argc, char ** argv) { params.n_predict = 16; } + LOG("%s: running\n", "test-save-load-state"); + ggml_backend_load_all(); if (!models_dir.empty()) { // run the suite over every dummy model in the directory if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) { LOG_ERR("%s: models directory '%s' does not exist\n", __func__, models_dir.c_str()); + LOG("%s: %s\n", "test-save-load-state", "FAILED"); + common_log_flush(common_log_main()); return 1; } @@ -664,6 +671,8 @@ int main(int argc, char ** argv) { if (models.empty()) { LOG_ERR("%s: no .gguf models found in '%s'\n", __func__, models_dir.c_str()); + LOG("%s: %s\n", "test-save-load-state", "FAILED"); + common_log_flush(common_log_main()); return 1; } @@ -672,7 +681,7 @@ int main(int argc, char ** argv) { size_t n_pass = 0; size_t n_fail = 0; for (const auto & model_path : models) { - LOG("\n================================================================\n"); + LOG_CNT("================================================================\n"); LOG_INF("%s: model %s\n", __func__, model_path.c_str()); if (run_save_load_tests_for_model(model_path, params)) { @@ -682,12 +691,17 @@ int main(int argc, char ** argv) { } } - LOG("\n================================================================\n"); + LOG_CNT("================================================================\n"); LOG_INF("%s: summary: %zu passed, %zu failed (of %zu)\n", __func__, n_pass, n_fail, models.size()); + LOG("%s: %s\n", "test-save-load-state", n_fail == 0 ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); return n_fail == 0 ? 0 : 1; } // single-model mode - return run_save_load_tests_for_model(params.model.path, params) ? 0 : 1; + const bool success = run_save_load_tests_for_model(params.model.path, params); + LOG("%s: %s\n", "test-save-load-state", success ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); + return success ? 0 : 1; } diff --git a/tests/test-state-restore-fragmented.cpp b/tests/test-state-restore-fragmented.cpp index d5548afba1..e1c3d6f834 100644 --- a/tests/test-state-restore-fragmented.cpp +++ b/tests/test-state-restore-fragmented.cpp @@ -8,6 +8,7 @@ #include "arg.h" #include "common.h" +#include "log.h" #include "llama.h" #include @@ -28,6 +29,8 @@ int main(int argc, char ** argv) { return 1; } + LOG("%s: running\n", "test-state-restore-fragmented"); + // init ggml_backend_load_all(); @@ -38,7 +41,9 @@ int main(int argc, char ** argv) { llama_context * ctx = llama_init->context(); if (model == nullptr || ctx == nullptr) { - fprintf(stderr, "%s : failed to init\n", __func__); + LOG_ERR("%s : failed to init\n", __func__); + LOG("%s: %s\n", "test-state-restore-fragmented", "FAILED"); + common_log_flush(common_log_main()); return 1; } @@ -58,26 +63,30 @@ int main(int argc, char ** argv) { batch.logits[batch.n_tokens - 1] = true; if (llama_decode(ctx, batch)) { - fprintf(stderr, "%s : failed to decode seq 0\n", __func__); + LOG_ERR("%s : failed to decode seq 0\n", __func__); + LOG("%s: %s\n", "test-state-restore-fragmented", "FAILED"); + common_log_flush(common_log_main()); return 1; } - fprintf(stderr, "%s : processed prompt on seq 0, 1, 2 (%zu tokens each)\n", __func__, tokens.size()); + LOG_INF("%s : processed prompt on seq 0, 1, 2 (%zu tokens each)\n", __func__, tokens.size()); // Save state of seq 1 std::vector seq_state(llama_state_seq_get_size(ctx, 1)); const size_t ncopy = llama_state_seq_get_data(ctx, seq_state.data(), seq_state.size(), 1); if (ncopy != seq_state.size()) { - fprintf(stderr, "%s : failed to save seq 1 state\n", __func__); + LOG_ERR("%s : failed to save seq 1 state\n", __func__); + LOG("%s: %s\n", "test-state-restore-fragmented", "FAILED"); + common_log_flush(common_log_main()); return 1; } - fprintf(stderr, "%s : saved seq 1 state, %zu bytes\n", __func__, ncopy); + LOG_INF("%s : saved seq 1 state, %zu bytes\n", __func__, ncopy); // clear seq 1 to create a "hole" in the KV cache (fragmentation) // 0.20.20.20.2.... llama_memory_t mem = llama_get_memory(ctx); llama_memory_seq_rm(mem, 1, -1, -1); - fprintf(stderr, "%s : cleared seq 1 to create fragmentation\n", __func__); + LOG_INF("%s : cleared seq 1 to create fragmentation\n", __func__); // Now the cache has holes where seq 1 was // This creates fragmentation - there's no contiguous block large enough @@ -88,13 +97,15 @@ int main(int argc, char ** argv) { // Before the fix, this would fail with "failed to find available cells in kv cache" const size_t nset = llama_state_seq_set_data(ctx, seq_state.data(), seq_state.size(), 1); if (nset != seq_state.size()) { - fprintf(stderr, "%s : FAILED to restore seq state into fragmented cache (got %zu, expected %zu)\n", + LOG_ERR("%s : FAILED to restore seq state into fragmented cache (got %zu, expected %zu)\n", __func__, nset, seq_state.size()); - fprintf(stderr, "%s : This is the bug - state restore fails with fragmented KV cache\n", __func__); + LOG_ERR("%s : This is the bug - state restore fails with fragmented KV cache\n", __func__); + LOG("%s: %s\n", "test-state-restore-fragmented", "FAILED"); llama_batch_free(batch); + common_log_flush(common_log_main()); return 1; } - fprintf(stderr, "%s : restored state into seq 1, %zu bytes\n", __func__, nset); + LOG_INF("%s : restored state into seq 1, %zu bytes\n", __func__, nset); // Verify we can decode with the restored state // Generate one token to verify the restored state is usable @@ -109,17 +120,21 @@ int main(int argc, char ** argv) { common_batch_add(batch, next_token, (int)tokens.size(), {1}, true); if (llama_decode(ctx, batch)) { - fprintf(stderr, "%s : failed to decode with restored state\n", __func__); + LOG_ERR("%s : failed to decode with restored state\n", __func__); + LOG("%s: %s\n", "test-state-restore-fragmented", "FAILED"); llama_sampler_free(smpl); llama_batch_free(batch); + common_log_flush(common_log_main()); return 1; } - fprintf(stderr, "%s : successfully decoded with restored state, generated: '%s'\n", __func__, next_token_str.c_str()); - fprintf(stderr, "%s : SUCCESS - state restore works with fragmented KV cache\n", __func__); + LOG_INF("%s : successfully decoded with restored state, generated: '%s'\n", __func__, next_token_str.c_str()); + LOG_INF("%s : SUCCESS - state restore works with fragmented KV cache\n", __func__); llama_sampler_free(smpl); llama_batch_free(batch); + LOG("%s: %s\n", "test-state-restore-fragmented", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-thread-safety.cpp b/tests/test-thread-safety.cpp index d0b5946e2c..ee29601eac 100644 --- a/tests/test-thread-safety.cpp +++ b/tests/test-thread-safety.cpp @@ -22,6 +22,8 @@ int main(int argc, char ** argv) { return 1; } + LOG("%s: running\n", "test-thread-safety"); + llama_backend_init(); llama_numa_init(params.numa); @@ -56,6 +58,8 @@ int main(int argc, char ** argv) { std::atomic failed = false; for (int m = 0; m < num_models; ++m) { + LOG_INF(" running model %d/%d\n", m + 1, num_models); + auto mparams = common_model_params_to_llama(params); if (m < gpu_dev_count) { @@ -71,6 +75,8 @@ int main(int argc, char ** argv) { llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams); if (model == NULL) { LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.path.c_str()); + LOG("%s: %s\n", "test-thread-safety", "FAILED"); + common_log_flush(common_log_main()); return 1; } @@ -156,11 +162,15 @@ int main(int argc, char ** argv) { thread.join(); } - if (failed) { + const bool ok = !failed; + + if (!ok) { LOG_ERR("One or more threads failed.\n"); - return 1; + } else { + LOG_INF("All threads finished without errors.\n"); } - LOG_INF("All threads finished without errors.\n"); - return 0; + LOG("%s: %s\n", "test-thread-safety", ok ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); + return ok ? 0 : 1; } diff --git a/tests/test-tokenizer-0.cpp b/tests/test-tokenizer-0.cpp index 37f8312c46..c0643998fd 100644 --- a/tests/test-tokenizer-0.cpp +++ b/tests/test-tokenizer-0.cpp @@ -2,6 +2,9 @@ #include "common.h" #include "console.h" +#include "arg.h" +#include "log.h" + #include #include #include @@ -63,7 +66,7 @@ static llama_tests read_tests(const std::string & fname_inp, const std::string & std::ifstream ifs_inp(fname_inp); if (!ifs_inp) { - fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_inp.c_str()); + LOG_ERR("%s : error: could not open file '%s'\n", __func__, fname_inp.c_str()); return tests; } @@ -71,7 +74,7 @@ static llama_tests read_tests(const std::string & fname_inp, const std::string & std::ifstream ifs_out(fname_out); if (!ifs_out) { - fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str()); + LOG_ERR("%s : error: could not open file '%s'\n", __func__, fname_out.c_str()); return tests; } @@ -96,7 +99,7 @@ static llama_tests read_tests(const std::string & fname_inp, const std::string & } if (sinp.size() != sout.size()) { - fprintf(stderr, "%s : error: input and output files have different number of tests\n", __func__); + LOG_ERR("%s : error: input and output files have different number of tests\n", __func__); return tests; } @@ -124,22 +127,44 @@ static llama_tests read_tests(const std::string & fname_inp, const std::string & } int main(int argc, char **argv) { - if (argc < 2) { - fprintf(stderr, "Usage: %s vocab-file [text-file]\n", argv[0]); + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + + std::vector positional; + std::vector common_argv; + common_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { + if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else { + positional.push_back(argv[i]); + } + } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { return 1; } - const std::string fname = argv[1]; + if (positional.empty()) { + LOG_ERR("Usage: %s vocab-file [text-file]\n", argv[0]); + common_log_flush(common_log_main()); + return 1; + } + + LOG("%s: running\n", "test-tokenizer-0"); + + const std::string fname = positional[0]; const std::string fname_inp = fname + ".inp"; const std::string fname_out = fname + ".out"; std::string fname_text; - if (argc > 2) { - fname_text = argv[2]; + if (positional.size() > 1) { + fname_text = positional[1]; } - fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str()); + LOG_INF("%s : reading vocab from: '%s'\n", __func__, fname.c_str()); llama_model * model; llama_context * ctx; @@ -155,7 +180,9 @@ int main(int argc, char **argv) { model = llama_model_load_from_file(fname.c_str(), mparams); if (model == NULL) { - fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); + LOG_ERR("%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); + LOG("%s: %s\n", "test-tokenizer-0", "FAILED"); + common_log_flush(common_log_main()); return 1; } @@ -164,8 +191,10 @@ int main(int argc, char **argv) { ctx = llama_init_from_model(model, cparams); if (ctx == NULL) { - fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); + LOG_ERR("%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); llama_model_free(model); + LOG("%s: %s\n", "test-tokenizer-0", "FAILED"); + common_log_flush(common_log_main()); return 1; } } @@ -186,7 +215,9 @@ int main(int argc, char **argv) { const auto res = read_tests(fname_inp, fname_out); if (res.empty()) { - fprintf(stderr, "%s : error: no tests found\n", __func__); + LOG_ERR("%s : error: no tests found\n", __func__); + LOG("%s: %s\n", "test-tokenizer-0", "FAILED"); + common_log_flush(common_log_main()); exit(1); } @@ -210,14 +241,15 @@ int main(int argc, char **argv) { continue; } - printf("\n"); - printf("src: '%s'\n", test_kv.first.c_str()); - printf("res: '%s'\n", common_detokenize(ctx, res).c_str()); - printf("tok: "); + // per-data-item detail: INFO, so hidden under --errors-only + LOG_CNT("\n"); + LOG_CNT("src: '%s'\n", test_kv.first.c_str()); + LOG_CNT("res: '%s'\n", common_detokenize(ctx, res).c_str()); + LOG_CNT("tok: "); for (const auto & tok : res) { - printf("%d ", tok); + LOG_CNT("%d ", tok); } - printf("\n"); + LOG_CNT("\n"); bool correct = res.size() == test_kv.second.size(); for (int i = 0; i < (int) res.size() && correct; ++i) { @@ -227,20 +259,21 @@ int main(int argc, char **argv) { } if (!correct) { - fprintf(stderr, "%s : failed test: '%s'\n", __func__, test_kv.first.c_str()); - fprintf(stderr, "%s : detokenized to: '%s' instead of '%s'\n", __func__, + LOG_ERR("%s : failed test: '%s'\n", __func__, test_kv.first.c_str()); + LOG_ERR("%s : detokenized to: '%s' instead of '%s'\n", __func__, common_detokenize(ctx, res).c_str(), common_detokenize(ctx, test_kv.second).c_str()); - fprintf(stderr, "%s : expected tokens: ", __func__); + // partial line: LOG_CNTV adds no prefix, error level so --errors-only keeps it + LOG_ERR("%s : expected tokens: ", __func__); for (const auto & t : test_kv.second) { - fprintf(stderr, "%6d '%s', ", t, common_token_to_piece(ctx, t).c_str()); + LOG_CNTV(LOG_LEVEL_ERROR, "%6d '%s', ", t, common_token_to_piece(ctx, t).c_str()); } - fprintf(stderr, "\n"); - fprintf(stderr, "%s : got tokens: ", __func__); + LOG_CNTV(LOG_LEVEL_ERROR, "\n"); + LOG_ERR("%s : got tokens: ", __func__); for (const auto & t : res) { - fprintf(stderr, "%6d '%s', ", t, common_token_to_piece(ctx, t).c_str()); + LOG_CNTV(LOG_LEVEL_ERROR, "%6d '%s', ", t, common_token_to_piece(ctx, t).c_str()); } - fprintf(stderr, "\n"); + LOG_CNTV(LOG_LEVEL_ERROR, "\n"); success = false; } @@ -254,19 +287,21 @@ int main(int argc, char **argv) { // single threaded tokenization if (!fname_text.empty()) { - fprintf(stderr, "%s : tokenizing: '%s'\n", __func__, fname_text.c_str()); + LOG_INF("%s : tokenizing: '%s'\n", __func__, fname_text.c_str()); std::string text; { std::ifstream ifs(fname_text); if (!ifs) { - fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_text.c_str()); + LOG_ERR("%s : error: could not open file '%s'\n", __func__, fname_text.c_str()); + LOG("%s: %s\n", "test-tokenizer-0", "FAILED"); + common_log_flush(common_log_main()); return 1; } text = std::string(std::istreambuf_iterator(ifs), std::istreambuf_iterator()); } - fprintf(stderr, "%s : text size: %zu\n", __func__, text.size()); + LOG_INF("%s : text size: %zu\n", __func__, text.size()); std::vector res; @@ -277,17 +312,19 @@ int main(int argc, char **argv) { const auto t_end = ggml_time_us(); - fprintf(stderr, "%s : tokenized in %.3f ms (cpp)\n", __func__, (t_end - t_start) / 1000.0); + LOG_INF("%s : tokenized in %.3f ms (cpp)\n", __func__, (t_end - t_start) / 1000.0); } - fprintf(stderr, "%s : tokens: %zu\n", __func__, res.size()); + LOG_INF("%s : tokens: %zu\n", __func__, res.size()); { const std::string fname_out = fname_text + ".tokcpp"; std::ofstream ofs(fname_out); if (!ofs) { - fprintf(stderr, "%s : error: could not open file '%s'\n", __func__, fname_out.c_str()); + LOG_ERR("%s : error: could not open file '%s'\n", __func__, fname_out.c_str()); + LOG("%s: %s\n", "test-tokenizer-0", "FAILED"); + common_log_flush(common_log_main()); return 1; } @@ -297,7 +334,7 @@ int main(int argc, char **argv) { } } - fprintf(stderr, "%s : tokens written to '%s'\n", __func__, (fname_text + ".tokcpp").c_str()); + LOG_INF("%s : tokens written to '%s'\n", __func__, (fname_text + ".tokcpp").c_str()); } llama_free(ctx); @@ -305,8 +342,13 @@ int main(int argc, char **argv) { llama_backend_free(); + common_log_flush(common_log_main()); + printf("\n"); printf("Tests %s\n", success ? "passed" : "failed"); + LOG("%s: %s\n", "test-tokenizer-0", success ? "PASSED" : "FAILED"); + common_log_flush(common_log_main()); + return success ? 0 : 3; } diff --git a/tests/test-tokenizer-1-bpe.cpp b/tests/test-tokenizer-1-bpe.cpp index 505dbfdb93..d6db4cb7ee 100644 --- a/tests/test-tokenizer-1-bpe.cpp +++ b/tests/test-tokenizer-1-bpe.cpp @@ -2,6 +2,9 @@ #include "common.h" #include "console.h" +#include "arg.h" +#include "log.h" + #include "../src/unicode.h" #include @@ -15,25 +18,40 @@ #include int main(int argc, char **argv) { - if (argc < 2 || argc > 3) { - fprintf(stderr, "Usage: %s [--ignore-merges]\n", argv[0]); + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + + std::vector positional; + bool ignore_merges = false; + std::vector common_argv; + common_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { + if (std::strcmp(argv[i], "--ignore-merges") == 0) { + ignore_merges = true; + } else if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else { + positional.push_back(argv[i]); + } + } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { return 1; } - const std::string fname = argv[1]; - bool ignore_merges = false; - if (argc == 3) { - if (std::strcmp(argv[2], "--ignore-merges") != 0) { - fprintf(stderr, "Usage: %s [--ignore-merges]\n", argv[0]); - return 1; - } - ignore_merges = true; + if (positional.size() != 1) { + LOG_ERR("Usage: %s [--ignore-merges]\n", argv[0]); + common_log_flush(common_log_main()); + return 1; } - fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str()); + const std::string fname = positional[0]; + + LOG_INF("%s : reading vocab from: '%s'\n", __func__, fname.c_str()); if (ignore_merges) { - fprintf(stderr, "%s : ignoring merges for tokens inside vocab\n", __func__); + LOG_INF("%s : ignoring merges for tokens inside vocab\n", __func__); } llama_model * model; @@ -50,7 +68,8 @@ int main(int argc, char **argv) { model = llama_model_load_from_file(fname.c_str(), mparams); if (model == NULL) { - fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); + LOG_ERR("%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); + common_log_flush(common_log_main()); return 1; } @@ -59,8 +78,9 @@ int main(int argc, char **argv) { ctx = llama_init_from_model(model, cparams); if (ctx == NULL) { - fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); + LOG_ERR("%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); llama_model_free(model); + common_log_flush(common_log_main()); return 1; } } @@ -69,9 +89,13 @@ int main(int argc, char **argv) { //GGML_ASSERT(llama_vocab_type(vocab) == LLAMA_VOCAB_TYPE_BPE); if (llama_vocab_type(vocab) != LLAMA_VOCAB_TYPE_BPE) { + // a vocab of another type is a skip (99), so it does not get a running line + common_log_flush(common_log_main()); return 99; } + LOG("%s: running\n", "test-tokenizer-1-bpe"); + #ifdef _WIN32 // We need this for unicode console support console::init(false, false); @@ -80,27 +104,33 @@ int main(int argc, char **argv) { const int n_vocab = llama_vocab_n_tokens(vocab); + LOG_INF(" running vocab detokenize round-trip (%d tokens)\n", n_vocab); + for (int i = 0; i < n_vocab; ++i) { std::string str = common_detokenize(ctx, std::vector(1, i)); try { auto cps = unicode_cpts_from_utf8(str); std::vector tokens = common_tokenize(ctx, str, false, true); if (ignore_merges && tokens.size() > 1) { - fprintf(stderr, - "%s : error: token %d detokenizes to '%s'(%zu) but " + LOG_ERR("%s : error: token %d detokenizes to '%s'(%zu) but " "tokenization of this to multiple tokens: [", __func__, i, str.c_str(), str.length()); - fprintf(stderr, "%d", tokens[0]); + // partial line: LOG_CNTV adds no prefix, error level so --errors-only keeps it + LOG_CNTV(LOG_LEVEL_ERROR, "%d", tokens[0]); for (size_t i = 1; i < tokens.size(); i++) { - fprintf(stderr, ", %d", tokens[i]); + LOG_CNTV(LOG_LEVEL_ERROR, ", %d", tokens[i]); } - fprintf(stderr, "]\n"); + LOG_CNTV(LOG_LEVEL_ERROR, "]\n"); + LOG("%s: %s\n", "test-tokenizer-1-bpe", "FAILED"); + common_log_flush(common_log_main()); return 2; } std::string check = common_detokenize(ctx, tokens); if (check != str) { - fprintf(stderr, "%s : error: token %d detokenizes to '%s'(%zu) but tokenization of this detokenizes to '%s'(%zu)\n", + LOG_ERR("%s : error: token %d detokenizes to '%s'(%zu) but tokenization of this detokenizes to '%s'(%zu)\n", __func__, i, str.c_str(), str.length(), check.c_str(), check.length()); + LOG("%s: %s\n", "test-tokenizer-1-bpe", "FAILED"); + common_log_flush(common_log_main()); return 2; } } @@ -111,6 +141,8 @@ int main(int argc, char **argv) { // unicode { + LOG_INF(" running unicode codepoint round-trip\n"); + const int nthread = std::thread::hardware_concurrency(); std::vector threads(nthread); @@ -129,7 +161,7 @@ int main(int argc, char **argv) { std::vector tokens = common_tokenize(ctx, str, false); std::string check = common_detokenize(ctx, tokens); if (cp != 9601 && str != check) { - fprintf(stderr, "error: codepoint 0x%x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n", + LOG_ERR("error: codepoint 0x%x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n", cp, check.c_str(), check.length(), str.c_str(), str.length()); errcode = 3; } @@ -142,6 +174,8 @@ int main(int argc, char **argv) { } if (errcode) { + LOG("%s: %s\n", "test-tokenizer-1-bpe", "FAILED"); + common_log_flush(common_log_main()); return errcode; } } @@ -151,5 +185,7 @@ int main(int argc, char **argv) { llama_backend_free(); + LOG("%s: %s\n", "test-tokenizer-1-bpe", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-tokenizer-1-spm.cpp b/tests/test-tokenizer-1-spm.cpp index 8e370d2c7b..1c1dd2d195 100644 --- a/tests/test-tokenizer-1-spm.cpp +++ b/tests/test-tokenizer-1-spm.cpp @@ -2,6 +2,9 @@ #include "common.h" #include "console.h" +#include "arg.h" +#include "log.h" + #include "../src/unicode.h" #include @@ -15,14 +18,32 @@ #include int main(int argc, char ** argv) { - if (argc < 2) { - fprintf(stderr, "Usage: %s \n", argv[0]); + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + + std::string fname; + std::vector common_argv; + common_argv.push_back(argv[0]); + for (int i = 1; i < argc; i++) { + if (argv[i][0] == '-') { + common_argv.push_back(argv[i]); // an option: let common_params_parse handle it + } else if (fname.empty()) { + fname = argv[i]; + } + } + common_argv.push_back(nullptr); + if (!common_params_parse((int) common_argv.size() - 1, common_argv.data(), params, LLAMA_EXAMPLE_COMMON)) { return 1; } - const std::string fname = argv[1]; + if (fname.empty()) { + LOG_ERR("Usage: %s \n", argv[0]); + common_log_flush(common_log_main()); + return 1; + } - fprintf(stderr, "%s : reading vocab from: '%s'\n", __func__, fname.c_str()); + LOG_INF("%s : reading vocab from: '%s'\n", __func__, fname.c_str()); llama_model * model; llama_context * ctx; @@ -38,7 +59,8 @@ int main(int argc, char ** argv) { model = llama_model_load_from_file(fname.c_str(), mparams); if (model == NULL) { - fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); + LOG_ERR("%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); + common_log_flush(common_log_main()); return 1; } @@ -47,8 +69,9 @@ int main(int argc, char ** argv) { ctx = llama_init_from_model(model, cparams); if (ctx == NULL) { - fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); + LOG_ERR("%s: error: failed to load vocab '%s'\n", __func__, fname.c_str()); llama_model_free(model); + common_log_flush(common_log_main()); return 1; } } @@ -57,9 +80,13 @@ int main(int argc, char ** argv) { //GGML_ASSERT(llama_vocab_type(model) == LLAMA_VOCAB_TYPE_SPM); if (llama_vocab_type(vocab) != LLAMA_VOCAB_TYPE_SPM) { + // a vocab of another type is a skip (99), so it does not get a running line + common_log_flush(common_log_main()); return 99; } + LOG("%s: running\n", "test-tokenizer-1-spm"); + #ifdef _WIN32 // We need this for unicode console support console::init(false, false); @@ -68,19 +95,25 @@ int main(int argc, char ** argv) { const int n_vocab = llama_vocab_n_tokens(vocab); + LOG_INF(" running vocab detokenize round-trip (%d tokens)\n", n_vocab); + for (int i = 0; i < n_vocab; ++i) { std::string str = common_detokenize(ctx, std::vector(1, i), true); std::vector tokens = common_tokenize(ctx, str, false, true); std::string check = common_detokenize(ctx, tokens); if (check != str) { - fprintf(stderr, "%s : error: token %d detokenizes to '%s'(%zu) but tokenization of this detokenizes to '%s'(%zu)\n", + LOG_ERR("%s : error: token %d detokenizes to '%s'(%zu) but tokenization of this detokenizes to '%s'(%zu)\n", __func__, i, str.c_str(), str.length(), check.c_str(), check.length()); + LOG("%s: %s\n", "test-tokenizer-1-spm", "FAILED"); + common_log_flush(common_log_main()); return 2; } } // unicode { + LOG_INF(" running unicode codepoint round-trip\n"); + const int nthread = std::thread::hardware_concurrency(); std::vector threads(nthread); @@ -99,7 +132,7 @@ int main(int argc, char ** argv) { std::vector tokens = common_tokenize(ctx, str, false, true); std::string check = common_detokenize(ctx, tokens); if (cp != 9601 && str != check) { - fprintf(stderr, "error: codepoint 0x%x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n", + LOG_ERR("error: codepoint 0x%x detokenizes to '%s'(%zu) instead of '%s'(%zu)\n", cp, check.c_str(), check.length(), str.c_str(), str.length()); errcode = 3; } @@ -112,6 +145,8 @@ int main(int argc, char ** argv) { } if(errcode) { + LOG("%s: %s\n", "test-tokenizer-1-spm", "FAILED"); + common_log_flush(common_log_main()); return errcode; } } @@ -121,5 +156,7 @@ int main(int argc, char ** argv) { llama_backend_free(); + LOG("%s: %s\n", "test-tokenizer-1-spm", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/test-unicode.cpp b/tests/test-unicode.cpp index 2347d9000a..ec1e1eae03 100644 --- a/tests/test-unicode.cpp +++ b/tests/test-unicode.cpp @@ -1,10 +1,23 @@ #include "../src/unicode.h" +#include "arg.h" +#include "common.h" +#include "log.h" + #include #include #include -int main() { +int main(int argc, char ** argv) { + common_params params; + params.model.path = "."; // this test takes no model + common_init(); + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_COMMON)) { + return 1; + } + + LOG("%s: running\n", "test-unicode"); + const std::vector regex_exprs = { "[~][A-Za-z]+| ?[\\p{S}]+|\\s+", }; @@ -12,13 +25,17 @@ int main() { const auto actual = unicode_regex_split(" ~foo", regex_exprs, false); if (actual != expected) { - fprintf(stderr, "unexpected split:"); + LOG_ERR("unexpected split:"); for (const auto & piece : actual) { - fprintf(stderr, " [%s]", piece.c_str()); + LOG_ERR(" [%s]", piece.c_str()); } - fprintf(stderr, "\n"); + LOG_ERR("\n"); + LOG("%s: %s\n", "test-unicode", "FAILED"); + common_log_flush(common_log_main()); return 1; } + LOG("%s: %s\n", "test-unicode", "PASSED"); + common_log_flush(common_log_main()); return 0; } diff --git a/tests/testing.h b/tests/testing.h index 891d78530a..7a7bdea30b 100644 --- a/tests/testing.h +++ b/tests/testing.h @@ -2,6 +2,10 @@ #include "common.h" +// "log.h" cannot be included here: clip-impl.h (pulled in by test-mtmd-impl.cpp) defines the +// LOG_* macros unconditionally, which conflicts with log.h and is an error with -Werror +int common_log_get_verbosity_thold(void); + #include #include #include @@ -28,9 +32,16 @@ struct testing { std::string skip_reason; static constexpr std::size_t status_column = 80; + // LOG_LEVEL_WARN, see common/log.h + static constexpr int log_level_warn = 2; explicit testing(std::ostream &os = std::cout) : out(os) {} + // hide the per-test detail when only warnings and errors are requested + bool show_details() const { + return common_log_get_verbosity_thold() > log_level_warn; + } + std::string indent() const { if (stack.empty()) { return ""; @@ -89,6 +100,10 @@ struct testing { } void print_result(const std::string &label, int new_failures, int new_assertions, const std::string &extra = "", bool was_skipped = false) const { + if (new_failures == 0 && !show_details()) { + return; // hide [PASS]/[SKIP], always report failures + } + std::string line = indent() + label; std::string details; @@ -131,7 +146,9 @@ struct testing { } ++tests; - out << indent() << name << "\n"; + if (show_details()) { + out << indent() << name << "\n"; + } int before_failures = failures; int before_assertions = assertions; @@ -174,7 +191,9 @@ struct testing { } ++tests; - out << indent() << "[bench] " << name << "\n"; + if (show_details()) { + out << indent() << "[bench] " << name << "\n"; + } int before_failures = failures; int before_assertions = assertions; @@ -263,6 +282,7 @@ struct testing { out << "failures : " << failures << "\n"; out << "exceptions : " << exceptions << "\n"; out << "skipped : " << skipped << "\n"; + out << (failures == 0 ? "PASSED" : "FAILED") << "\n"; return failures == 0 ? 0 : 1; } }; diff --git a/tools/rpc/CMakeLists.txt b/tools/rpc/CMakeLists.txt index 2891c7d034..e1fe9e0144 100644 --- a/tools/rpc/CMakeLists.txt +++ b/tools/rpc/CMakeLists.txt @@ -5,7 +5,7 @@ target_compile_features(${TARGET} PRIVATE cxx_std_17) if (LLAMA_BUILD_TESTS AND UNIX AND NOT GGML_BACKEND_DL) add_executable(test-rpc-multi-server ${PROJECT_SOURCE_DIR}/tests/test-rpc-multi-server.cpp) - target_link_libraries(test-rpc-multi-server PRIVATE ggml ggml-rpc) + target_link_libraries(test-rpc-multi-server PRIVATE ggml ggml-rpc llama-common) target_include_directories(test-rpc-multi-server PRIVATE ${PROJECT_SOURCE_DIR}/ggml/src) add_test( NAME test-rpc-multi-server diff --git a/tools/server/tests/conftest.py b/tools/server/tests/conftest.py index 69c4fd687f..3e40aad96e 100644 --- a/tools/server/tests/conftest.py +++ b/tools/server/tests/conftest.py @@ -2,6 +2,21 @@ import os import pytest from filelock import FileLock from utils import * +import utils + + +def pytest_addoption(parser): + parser.addoption( + "--errors-only", + action="store_true", + default=False, + help="Run llama-server with WARN log verbosity, hiding INFO/TRACE/DEBUG logs", + ) + + +@pytest.fixture(scope="session", autouse=True) +def configure_errors_only(request): + utils.errors_only = request.config.getoption("--errors-only") @pytest.fixture(scope="session", autouse=True) @@ -26,7 +41,7 @@ def stop_server_after_each_test(): @pytest.fixture(scope="session", autouse=True) -def load_server_presets(configure_worker_port, tmp_path_factory): +def load_server_presets(configure_worker_port, configure_errors_only, tmp_path_factory): # this will be run once per test session, before any tests # serialize model downloads across parallel workers. diff --git a/tools/server/tests/tests.sh b/tools/server/tests/tests.sh index 1d1415663b..0557a6105c 100755 --- a/tools/server/tests/tests.sh +++ b/tools/server/tests/tests.sh @@ -8,12 +8,12 @@ set -eu WORKERS="${PYTEST_WORKERS:-auto}" -if [ $# -lt 1 ] +if [ $# -lt 1 ] || [ "$1" = "--errors-only" ] then if [[ "${SLOW_TESTS:-0}" == 1 ]]; then - pytest --durations=30 -v -x -n "${WORKERS}" --dist=worksteal + pytest --durations=30 -v -x -n "${WORKERS}" --dist=worksteal "$@" else - pytest --durations=30 -v -x -n "${WORKERS}" --dist=worksteal -m "not slow" + pytest --durations=30 -v -x -n "${WORKERS}" --dist=worksteal -m "not slow" "$@" fi else pytest --durations=30 -n "${WORKERS}" --dist=worksteal "$@" diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py index 826aef2d5b..0fbd3e735e 100644 --- a/tools/server/tests/utils.py +++ b/tools/server/tests/utils.py @@ -34,6 +34,9 @@ DEFAULT_HTTP_TIMEOUT = 60 # per-request timeout, a hung server fails the test instead of stalling the CI for hours DEFAULT_REQUEST_TIMEOUT = 600 +# set by conftest from --errors-only, makes servers run with WARN log verbosity +errors_only: bool = False + class ServerResponse: headers: dict @@ -295,6 +298,8 @@ class ServerProcess: if self.gcp_compat: env["AIP_MODE"] = "PREDICTION" env["AIP_HTTP_PORT"] = str(self.server_port) + if errors_only and not self.debug: + server_args.extend(["--log-verbosity", "2"]) args = [str(arg) for arg in [server_path, *server_args]] print(f"tests: starting server with: {' '.join(args)}")