From b7bc3225e82e032cb3ecfea23ddee9a7c2b6bc89 Mon Sep 17 00:00:00 2001 From: Xiaodong Ye Date: Wed, 25 Jun 2025 17:46:54 +0800 Subject: [PATCH 01/12] test-backend-ops: add support for specifying output format Signed-off-by: Xiaodong Ye --- tests/test-backend-ops.cpp | 559 ++++++++++++++++++++++++++++++------- 1 file changed, 457 insertions(+), 102 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 5c3db854cb498..1ece814c7cc36 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -24,10 +24,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -317,6 +319,317 @@ enum test_mode { MODE_GRAD, }; +// Output format support similar to llama-bench +enum output_formats { CONSOLE, SQL }; + +static const char * output_format_str(output_formats format) { + switch (format) { + case CONSOLE: + return "console"; + case SQL: + return "sql"; + default: + GGML_ABORT("invalid output format"); + } +} + +static bool output_format_from_str(const std::string & s, output_formats & format) { + if (s == "console") { + format = CONSOLE; + } else if (s == "sql") { + format = SQL; + } else { + return false; + } + return true; +} + +// Test result structure for SQL output +struct test_result { + std::string test_time; + std::string backend_name; + std::string op_name; + std::string op_params; + std::string test_mode; + bool supported; + bool passed; + std::string error_message; + double time_us; + double flops_per_sec; + double bandwidth_gb_s; + size_t memory_kb; + int n_runs; + + test_result() { + // Initialize with default values + time_us = 0.0; + flops_per_sec = 0.0; + bandwidth_gb_s = 0.0; + memory_kb = 0; + n_runs = 0; + supported = false; + passed = false; + + // Set test time + time_t t = time(NULL); + char buf[32]; + std::strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&t)); + test_time = buf; + } + + static const std::vector & get_fields() { + static const std::vector fields = { + "test_time", "backend_name", "op_name", "op_params", "test_mode", + "supported", "passed", "error_message", "time_us", "flops_per_sec", + "bandwidth_gb_s", "memory_kb", "n_runs" + }; + return fields; + } + + enum field_type { STRING, BOOL, INT, FLOAT }; + + static field_type get_field_type(const std::string & field) { + if (field == "supported" || field == "passed") { + return BOOL; + } + if (field == "memory_kb" || field == "n_runs") { + return INT; + } + if (field == "time_us" || field == "flops_per_sec" || field == "bandwidth_gb_s") { + return FLOAT; + } + return STRING; + } + + std::vector get_values() const { + return { + test_time, + backend_name, + op_name, + op_params, + test_mode, + std::to_string(supported), + std::to_string(passed), + error_message, + std::to_string(time_us), + std::to_string(flops_per_sec), + std::to_string(bandwidth_gb_s), + std::to_string(memory_kb), + std::to_string(n_runs) + }; + } +}; + +// Printer classes for different output formats +struct printer { + virtual ~printer() {} + FILE * fout = stdout; + virtual void print_header() {} + virtual void print_test_result(const test_result & result) = 0; + virtual void print_footer() {} + + // General purpose output methods + virtual void print_info(const char * format, ...) = 0; + virtual void print_error(const char * format, ...) = 0; + virtual void print_device_info(const char * format, ...) = 0; + virtual void print_test_summary(const char * format, ...) = 0; + virtual void print_status_ok() = 0; + virtual void print_status_fail() = 0; +}; + +struct console_printer : public printer { + void print_test_result(const test_result & result) override { + if (result.test_mode == "test") { + print_test_console(result); + } else if (result.test_mode == "perf") { + print_perf_console(result); + } + } + + void print_info(const char * format, ...) override { + va_list args; + va_start(args, format); + vprintf(format, args); + va_end(args); + } + + void print_error(const char * format, ...) override { + va_list args; + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); + } + + void print_device_info(const char * format, ...) override { + va_list args; + va_start(args, format); + vprintf(format, args); + va_end(args); + } + + void print_test_summary(const char * format, ...) override { + va_list args; + va_start(args, format); + vprintf(format, args); + va_end(args); + } + + void print_status_ok() override { + printf("\033[1;32mOK\033[0m\n"); + } + + void print_status_fail() override { + printf("\033[1;31mFAIL\033[0m\n"); + } + +private: + void print_test_console(const test_result & result) { + printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str()); + fflush(stdout); + + if (!result.supported) { + printf("not supported [%s] ", result.backend_name.c_str()); + printf("\n"); + return; + } + + if (result.passed) { + printf("\033[1;32mOK\033[0m\n"); + } else { + printf("\033[1;31mFAIL\033[0m\n"); + } + } + + void print_perf_console(const test_result & result) { + int len = printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str()); + fflush(stdout); + + if (!result.supported) { + printf("not supported\n"); + return; + } + + // align while also leaving some margin for variations in parameters + int align = 8; + int last = (len + align - 1) / align * align; + if (last - len < 5) { + last += align; + } + printf("%*s", last - len, ""); + + printf(" %8d runs - %8.2f us/run - ", + result.n_runs, + result.time_us); + + if (result.flops_per_sec > 0) { + auto format_flops = [](double flops) -> std::string { + char buf[256]; + if (flops >= 1e12) { + snprintf(buf, sizeof(buf), "%6.2f TFLOP", flops / 1e12); + } else if (flops >= 1e9) { + snprintf(buf, sizeof(buf), "%6.2f GFLOP", flops / 1e9); + } else if (flops >= 1e6) { + snprintf(buf, sizeof(buf), "%6.2f MFLOP", flops / 1e6); + } else { + snprintf(buf, sizeof(buf), "%6.2f KFLOP", flops / 1e3); + } + return buf; + }; + uint64_t op_flops_per_run = result.flops_per_sec * result.time_us / 1e6; + printf("%s/run - \033[1;34m%sS\033[0m", + format_flops(op_flops_per_run).c_str(), + format_flops(result.flops_per_sec).c_str()); + } else { + printf("%8zu kB/run - \033[1;34m%7.2f GB/s\033[0m", + result.memory_kb, + result.bandwidth_gb_s); + } + printf("\n"); + } +}; + +struct sql_printer : public printer { + static std::string get_sql_field_type(const std::string & field) { + switch (test_result::get_field_type(field)) { + case test_result::STRING: + return "TEXT"; + case test_result::BOOL: + case test_result::INT: + return "INTEGER"; + case test_result::FLOAT: + return "REAL"; + default: + GGML_ABORT("invalid field type"); + } + } + + void print_header() override { + std::vector fields = test_result::get_fields(); + fprintf(fout, "CREATE TABLE IF NOT EXISTS test_results (\n"); + for (size_t i = 0; i < fields.size(); i++) { + fprintf(fout, " %s %s%s\n", fields[i].c_str(), get_sql_field_type(fields[i]).c_str(), + i < fields.size() - 1 ? "," : ""); + } + fprintf(fout, ");\n\n"); + } + + void print_test_result(const test_result & result) override { + fprintf(fout, "INSERT INTO test_results ("); + std::vector fields = test_result::get_fields(); + for (size_t i = 0; i < fields.size(); i++) { + fprintf(fout, "%s%s", fields[i].c_str(), i < fields.size() - 1 ? ", " : ""); + } + fprintf(fout, ") VALUES ("); + std::vector values = result.get_values(); + for (size_t i = 0; i < values.size(); i++) { + fprintf(fout, "'%s'%s", values[i].c_str(), i < values.size() - 1 ? ", " : ""); + } + fprintf(fout, ");\n"); + } + + // SQL printer ignores general output - only outputs test results + void print_info(const char * format, ...) override { + // Do nothing - SQL format only outputs test results + (void)format; + } + + void print_error(const char * format, ...) override { + // Still output errors to stderr for SQL format + va_list args; + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); + } + + void print_device_info(const char * format, ...) override { + // Do nothing - SQL format only outputs test results + (void)format; + } + + void print_test_summary(const char * format, ...) override { + // Do nothing - SQL format only outputs test results + (void)format; + } + + void print_status_ok() override { + // Do nothing - SQL format only outputs test results + } + + void print_status_fail() override { + // Do nothing - SQL format only outputs test results + } +}; + +static std::unique_ptr create_printer(output_formats format) { + switch (format) { + case CONSOLE: + return std::make_unique(); + case SQL: + return std::make_unique(); + } + GGML_ABORT("invalid output format"); +} + struct test_case { virtual ~test_case() {} @@ -434,7 +747,7 @@ struct test_case { return t; } - bool eval(ggml_backend_t backend1, ggml_backend_t backend2, const char * op_name) { + bool eval(ggml_backend_t backend1, ggml_backend_t backend2, const char * op_name, printer * output_printer = nullptr) { mode = MODE_TEST; ggml_init_params params = { @@ -451,29 +764,44 @@ struct test_case { add_sentinel(ctx); ggml_tensor * out = build_graph(ctx); - - if (op_name != nullptr && op_desc(out) != op_name) { + std::string current_op_name = op_desc(out); + if (op_name != nullptr && current_op_name != op_name) { //printf(" %s: skipping\n", op_desc(out).c_str()); ggml_free(ctx); return true; } - printf(" %s(%s): ", op_desc(out).c_str(), vars().c_str()); - fflush(stdout); - // check if the backends support the ops bool supported = true; for (ggml_backend_t backend : {backend1, backend2}) { for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { if (!ggml_backend_supports_op(backend, t)) { - printf("not supported [%s] ", ggml_backend_name(backend)); supported = false; break; } } } + if (!supported) { - printf("\n"); + // Create test result for unsupported operation + test_result result; + result.backend_name = ggml_backend_name(backend1); + result.op_name = current_op_name; + result.op_params = vars(); + result.test_mode = "test"; + result.supported = false; + result.passed = false; + result.error_message = "not supported"; + result.time_us = 0.0; + result.flops_per_sec = 0.0; + result.bandwidth_gb_s = 0.0; + result.memory_kb = 0; + result.n_runs = 0; + + if (output_printer) { + output_printer->print_test_result(result); + } + ggml_free(ctx); return true; } @@ -578,24 +906,34 @@ struct test_case { const bool cmp_ok = ggml_backend_compare_graph_backend(backend1, backend2, gf, callback, &ud, run_whole_graph() ? out : nullptr); - if (!cmp_ok) { - printf("compare failed "); - } - ggml_backend_buffer_free(buf); ggml_free(ctx); - if (ud.ok && cmp_ok) { - printf("\033[1;32mOK\033[0m\n"); - return true; + // Create test result + bool test_passed = ud.ok && cmp_ok; + test_result result; + result.backend_name = ggml_backend_name(backend1); + result.op_name = current_op_name; + result.op_params = vars(); + result.test_mode = "test"; + result.supported = supported; + result.passed = test_passed; + result.error_message = test_passed ? "" : (!cmp_ok ? "compare failed" : "test failed"); + result.time_us = 0.0; // Not measured in test mode + result.flops_per_sec = 0.0; + result.bandwidth_gb_s = 0.0; + result.memory_kb = 0; + result.n_runs = 0; + + if (output_printer) { + output_printer->print_test_result(result); } - printf("\033[1;31mFAIL\033[0m\n"); - return false; + return test_passed; } - bool eval_perf(ggml_backend_t backend, const char * op_name) { + bool eval_perf(ggml_backend_t backend, const char * op_name, printer * output_printer = nullptr) { mode = MODE_PERF; static const size_t graph_nodes = 8192; @@ -609,28 +947,35 @@ struct test_case { GGML_ASSERT(ctx); ggml_tensor * out = build_graph(ctx.get()); - - if (op_name != nullptr && op_desc(out) != op_name) { + std::string current_op_name = op_desc(out); + if (op_name != nullptr && current_op_name != op_name) { //printf(" %s: skipping\n", op_desc(out).c_str()); return true; } - int len = printf(" %s(%s): ", op_desc(out).c_str(), vars().c_str()); - fflush(stdout); - // check if backends support op if (!ggml_backend_supports_op(backend, out)) { - printf("not supported\n"); - return true; - } + // Create test result for unsupported performance test + test_result result; + result.backend_name = ggml_backend_name(backend); + result.op_name = current_op_name; + result.op_params = vars(); + result.test_mode = "perf"; + result.supported = false; + result.passed = false; + result.error_message = "not supported"; + result.time_us = 0.0; + result.flops_per_sec = 0.0; + result.bandwidth_gb_s = 0.0; + result.memory_kb = 0; + result.n_runs = 0; + + if (output_printer) { + output_printer->print_test_result(result); + } - // align while also leaving some margin for variations in parameters - int align = 8; - int last = (len + align - 1) / align * align; - if (last - len < 5) { - last += align; + return true; } - printf("%*s", last - len, ""); // allocate ggml_backend_buffer_ptr buf(ggml_backend_alloc_ctx_tensors(ctx.get(), backend)); // smart ptr @@ -715,40 +1060,29 @@ struct test_case { total_runs += n_runs; } while (total_time_us < 1000*1000); // run for at least 1 second - printf(" %8d runs - %8.2f us/run - ", - total_runs, - (double)total_time_us / total_runs); - - if (op_flops(out) > 0) { - double flops_per_sec = (op_flops(out) * total_runs) / (total_time_us / 1e6); - auto format_flops = [](double flops) -> std::string { - char buf[256]; - if (flops >= 1e12) { - snprintf(buf, sizeof(buf), "%6.2f TFLOP", flops / 1e12); - } else if (flops >= 1e9) { - snprintf(buf, sizeof(buf), "%6.2f GFLOP", flops / 1e9); - } else if (flops >= 1e6) { - snprintf(buf, sizeof(buf), "%6.2f MFLOP", flops / 1e6); - } else { - snprintf(buf, sizeof(buf), "%6.2f KFLOP", flops / 1e3); - } - return buf; - }; - printf("%s/run - \033[1;34m%sS\033[0m", - format_flops(op_flops(out)).c_str(), - format_flops(flops_per_sec).c_str()); - - } else { - printf("%8zu kB/run - \033[1;34m%7.2f GB/s\033[0m", - op_size(out) / 1024, - total_mem / (total_time_us / 1e6) / 1024.0 / 1024.0 / 1024.0); + // Create test result + test_result result; + result.backend_name = ggml_backend_name(backend); + result.op_name = current_op_name; + result.op_params = vars(); + result.test_mode = "perf"; + result.supported = true; // If we got this far, it's supported + result.passed = true; // Performance tests don't fail + result.error_message = ""; + result.time_us = (double)total_time_us / total_runs; + result.flops_per_sec = (op_flops(out) > 0) ? (op_flops(out) * total_runs) / (total_time_us / 1e6) : 0.0; + result.bandwidth_gb_s = (op_flops(out) == 0) ? total_mem / (total_time_us / 1e6) / 1024.0 / 1024.0 / 1024.0 : 0.0; + result.memory_kb = op_size(out) / 1024; + result.n_runs = total_runs; + + if (output_printer) { + output_printer->print_test_result(result); } - printf("\n"); return true; } - bool eval_grad(ggml_backend_t backend, const char * op_name) { + bool eval_grad(ggml_backend_t backend, const char * op_name, printer * output_printer = nullptr) { mode = MODE_GRAD; const std::vector expect = grad_expect(); @@ -766,15 +1100,14 @@ struct test_case { ggml_tensor * out = build_graph(ctx.get()); if ((op_name != nullptr && op_desc(out) != op_name) || out->op == GGML_OP_OPT_STEP_ADAMW) { - //printf(" %s: skipping\n", op_desc(out).c_str()); + //output_printer->print_info(" %s: skipping\n", op_desc(out).c_str()); return true; } - printf(" %s(%s): ", op_desc(out).c_str(), vars().c_str()); - fflush(stdout); + output_printer->print_info(" %s(%s): ", op_desc(out).c_str(), vars().c_str()); if (out->type != GGML_TYPE_F32) { - printf("not supported [%s->type != FP32]\n", out->name); + output_printer->print_info("not supported [%s->type != FP32]\n", out->name); return true; } @@ -783,25 +1116,25 @@ struct test_case { bool any_params = false; for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { if (!ggml_backend_supports_op(backend, t)) { - printf("not supported [%s] ", ggml_backend_name(backend)); + output_printer->print_info("not supported [%s] ", ggml_backend_name(backend)); supported = false; break; } if ((t->flags & GGML_TENSOR_FLAG_PARAM)) { any_params = true; if (t->type != GGML_TYPE_F32) { - printf("not supported [%s->type != FP32] ", t->name); + output_printer->print_info("not supported [%s->type != FP32] ", t->name); supported = false; break; } } } if (!any_params) { - printf("not supported [%s] \n", op_desc(out).c_str()); + output_printer->print_info("not supported [%s] \n", op_desc(out).c_str()); supported = false; } if (!supported) { - printf("\n"); + output_printer->print_info("\n"); return true; } @@ -812,7 +1145,7 @@ struct test_case { } } if (ngrads > grad_nmax()) { - printf("skipping large tensors for speed \n"); + output_printer->print_info("skipping large tensors for speed \n"); return true; } @@ -835,25 +1168,25 @@ struct test_case { for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { if (!ggml_backend_supports_op(backend, t)) { - printf("not supported [%s] ", ggml_backend_name(backend)); + output_printer->print_info("not supported [%s] ", ggml_backend_name(backend)); supported = false; break; } if ((t->flags & GGML_TENSOR_FLAG_PARAM) && t->type != GGML_TYPE_F32) { - printf("not supported [%s->type != FP32] ", t->name); + output_printer->print_info("not supported [%s->type != FP32] ", t->name); supported = false; break; } } if (!supported) { - printf("\n"); + output_printer->print_info("\n"); return true; } // allocate ggml_backend_buffer_ptr buf(ggml_backend_alloc_ctx_tensors(ctx.get(), backend)); // smart ptr if (buf == NULL) { - printf("failed to allocate tensors [%s] ", ggml_backend_name(backend)); + output_printer->print_error("failed to allocate tensors [%s] ", ggml_backend_name(backend)); return false; } @@ -891,7 +1224,7 @@ struct test_case { for (int64_t i = 0; i < ne; ++i) { // gradient algebraic // check for nans if (!std::isfinite(ga[i])) { - printf("[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", ggml_op_desc(t), i, bn, ga[i]); + output_printer->print_info("[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", ggml_op_desc(t), i, bn, ga[i]); ok = false; break; } @@ -959,7 +1292,7 @@ struct test_case { const double err = mean_abs_asymm(gn.data(), ga.data(), gn.size(), expect); if (err > max_maa_err()) { - printf("[%s] MAA = %.9f > %.9f ", ggml_op_desc(t), err, max_maa_err()); + output_printer->print_info("[%s] MAA = %.9f > %.9f ", ggml_op_desc(t), err, max_maa_err()); ok = false; break; } @@ -969,15 +1302,15 @@ struct test_case { } if (!ok) { - printf("compare failed "); + output_printer->print_info("compare failed "); } if (ok) { - printf("\033[1;32mOK\033[0m\n"); + output_printer->print_status_ok(); return true; } - printf("\033[1;31mFAIL\033[0m\n"); + output_printer->print_status_fail(); return false; } }; @@ -4985,7 +5318,7 @@ static std::vector> make_test_cases_perf() { return test_cases; } -static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op_name, const char * params_filter) { +static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op_name, const char * params_filter, printer * output_printer = nullptr) { auto filter_test_cases = [](std::vector> & test_cases, const char * params_filter) { if (params_filter == nullptr) { return; @@ -5008,17 +5341,17 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op filter_test_cases(test_cases, params_filter); ggml_backend_t backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, NULL); if (backend_cpu == NULL) { - printf(" Failed to initialize CPU backend\n"); + output_printer->print_error(" Failed to initialize CPU backend\n"); return false; } size_t n_ok = 0; for (auto & test : test_cases) { - if (test->eval(backend, backend_cpu, op_name)) { + if (test->eval(backend, backend_cpu, op_name, output_printer)) { n_ok++; } } - printf(" %zu/%zu tests passed\n", n_ok, test_cases.size()); + output_printer->print_test_summary(" %zu/%zu tests passed\n", n_ok, test_cases.size()); ggml_backend_free(backend_cpu); @@ -5030,11 +5363,11 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op filter_test_cases(test_cases, params_filter); size_t n_ok = 0; for (auto & test : test_cases) { - if (test->eval_grad(backend, op_name)) { + if (test->eval_grad(backend, op_name, output_printer)) { n_ok++; } } - printf(" %zu/%zu tests passed\n", n_ok, test_cases.size()); + output_printer->print_test_summary(" %zu/%zu tests passed\n", n_ok, test_cases.size()); return n_ok == test_cases.size(); } @@ -5043,7 +5376,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op auto test_cases = make_test_cases_perf(); filter_test_cases(test_cases, params_filter); for (auto & test : test_cases) { - test->eval_perf(backend, op_name); + test->eval_perf(backend, op_name, output_printer); } return true; } @@ -5052,16 +5385,18 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op } static void usage(char ** argv) { - printf("Usage: %s [mode] [-o ] [-b ] [-p ]\n", argv[0]); + printf("Usage: %s [mode] [-o ] [-b ] [-p ] [--output ]\n", argv[0]); printf(" valid modes:\n"); printf(" - test (default, compare with CPU backend for correctness)\n"); printf(" - grad (compare gradients from backpropagation with method of finite differences)\n"); printf(" - perf (performance evaluation)\n"); printf(" op names for -o are as given by ggml_op_desc() (e.g. ADD, MUL_MAT, etc)\n"); + printf(" --output specifies output format (default: console)\n"); } int main(int argc, char ** argv) { test_mode mode = MODE_TEST; + output_formats output_format = CONSOLE; const char * op_name_filter = nullptr; const char * backend_filter = nullptr; const char * params_filter = nullptr; @@ -5094,6 +5429,16 @@ int main(int argc, char ** argv) { usage(argv); return 1; } + } else if (strcmp(argv[i], "--output") == 0) { + if (i + 1 < argc) { + if (!output_format_from_str(argv[++i], output_format)) { + usage(argv); + return 1; + } + } else { + usage(argv); + return 1; + } } else { usage(argv); return 1; @@ -5103,23 +5448,29 @@ int main(int argc, char ** argv) { // load and enumerate backends ggml_backend_load_all(); - printf("Testing %zu devices\n\n", ggml_backend_dev_count()); + // Create printer for output format + std::unique_ptr output_printer = create_printer(output_format); + if (output_printer) { + output_printer->print_header(); + } + + output_printer->print_info("Testing %zu devices\n\n", ggml_backend_dev_count()); size_t n_ok = 0; for (size_t i = 0; i < ggml_backend_dev_count(); i++) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); - printf("Backend %zu/%zu: %s\n", i + 1, ggml_backend_dev_count(), ggml_backend_dev_name(dev)); + output_printer->print_device_info("Backend %zu/%zu: %s\n", i + 1, ggml_backend_dev_count(), ggml_backend_dev_name(dev)); if (backend_filter != NULL && strcmp(backend_filter, ggml_backend_dev_name(dev)) != 0) { - printf(" Skipping\n"); + output_printer->print_device_info(" Skipping\n"); n_ok++; continue; } if (backend_filter == NULL && ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU && mode != MODE_GRAD) { - printf(" Skipping CPU backend\n"); + output_printer->print_device_info(" Skipping CPU backend\n"); n_ok++; continue; } @@ -5134,36 +5485,40 @@ int main(int argc, char ** argv) { ggml_backend_set_n_threads_fn(backend, std::thread::hardware_concurrency()); } - printf(" Device description: %s\n", ggml_backend_dev_description(dev)); + output_printer->print_device_info(" Device description: %s\n", ggml_backend_dev_description(dev)); size_t free, total; // NOLINT ggml_backend_dev_memory(dev, &free, &total); - printf(" Device memory: %zu MB (%zu MB free)\n", total / 1024 / 1024, free / 1024 / 1024); - printf("\n"); + output_printer->print_device_info(" Device memory: %zu MB (%zu MB free)\n", total / 1024 / 1024, free / 1024 / 1024); + output_printer->print_device_info("\n"); - bool ok = test_backend(backend, mode, op_name_filter, params_filter); + bool ok = test_backend(backend, mode, op_name_filter, params_filter, output_printer.get()); - printf(" Backend %s: ", ggml_backend_name(backend)); + output_printer->print_device_info(" Backend %s: ", ggml_backend_name(backend)); if (ok) { - printf("\033[1;32mOK\033[0m\n"); + output_printer->print_status_ok(); n_ok++; } else { - printf("\033[1;31mFAIL\033[0m\n"); + output_printer->print_status_fail(); } - printf("\n"); + output_printer->print_device_info("\n"); ggml_backend_free(backend); } ggml_quantize_free(); - printf("%zu/%zu backends passed\n", n_ok, ggml_backend_dev_count()); + if (output_printer) { + output_printer->print_footer(); + } + + output_printer->print_test_summary("%zu/%zu backends passed\n", n_ok, ggml_backend_dev_count()); if (n_ok != ggml_backend_dev_count()) { - printf("\033[1;31mFAIL\033[0m\n"); + output_printer->print_status_fail(); return 1; } - printf("\033[1;32mOK\033[0m\n"); + output_printer->print_status_ok(); return 0; } From 8ec85718afb77670787c81054622b2bce322c867 Mon Sep 17 00:00:00 2001 From: Xiaodong Ye Date: Thu, 26 Jun 2025 12:39:14 +0800 Subject: [PATCH 02/12] Address review comments Signed-off-by: Xiaodong Ye --- tests/test-backend-ops.cpp | 107 ++++++++++++++----------------------- 1 file changed, 40 insertions(+), 67 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 1ece814c7cc36..0b671b0f615e4 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -355,7 +355,7 @@ struct test_result { bool passed; std::string error_message; double time_us; - double flops_per_sec; + double flops; double bandwidth_gb_s; size_t memory_kb; int n_runs; @@ -363,7 +363,7 @@ struct test_result { test_result() { // Initialize with default values time_us = 0.0; - flops_per_sec = 0.0; + flops = 0.0; bandwidth_gb_s = 0.0; memory_kb = 0; n_runs = 0; @@ -377,10 +377,24 @@ struct test_result { test_time = buf; } + test_result(const std::string& backend_name, const std::string& op_name, const std::string& op_params, + const std::string& test_mode, bool supported, bool passed, const std::string& error_message = "", + double time_us = 0.0, double flops = 0.0, double bandwidth_gb_s = 0.0, + size_t memory_kb = 0, int n_runs = 0) + : backend_name(backend_name), op_name(op_name), op_params(op_params), test_mode(test_mode), + supported(supported), passed(passed), error_message(error_message), time_us(time_us), + flops(flops), bandwidth_gb_s(bandwidth_gb_s), memory_kb(memory_kb), n_runs(n_runs) { + // Set test time + time_t t = time(NULL); + char buf[32]; + std::strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&t)); + test_time = buf; + } + static const std::vector & get_fields() { static const std::vector fields = { "test_time", "backend_name", "op_name", "op_params", "test_mode", - "supported", "passed", "error_message", "time_us", "flops_per_sec", + "supported", "passed", "error_message", "time_us", "flops", "bandwidth_gb_s", "memory_kb", "n_runs" }; return fields; @@ -395,7 +409,7 @@ struct test_result { if (field == "memory_kb" || field == "n_runs") { return INT; } - if (field == "time_us" || field == "flops_per_sec" || field == "bandwidth_gb_s") { + if (field == "time_us" || field == "flops" || field == "bandwidth_gb_s") { return FLOAT; } return STRING; @@ -412,7 +426,7 @@ struct test_result { std::to_string(passed), error_message, std::to_string(time_us), - std::to_string(flops_per_sec), + std::to_string(flops), std::to_string(bandwidth_gb_s), std::to_string(memory_kb), std::to_string(n_runs) @@ -521,7 +535,7 @@ struct console_printer : public printer { result.n_runs, result.time_us); - if (result.flops_per_sec > 0) { + if (result.flops > 0) { auto format_flops = [](double flops) -> std::string { char buf[256]; if (flops >= 1e12) { @@ -531,14 +545,14 @@ struct console_printer : public printer { } else if (flops >= 1e6) { snprintf(buf, sizeof(buf), "%6.2f MFLOP", flops / 1e6); } else { - snprintf(buf, sizeof(buf), "%6.2f KFLOP", flops / 1e3); + snprintf(buf, sizeof(buf), "%6.2f kFLOP", flops / 1e3); } return buf; }; - uint64_t op_flops_per_run = result.flops_per_sec * result.time_us / 1e6; + uint64_t op_flops_per_run = result.flops * result.time_us / 1e6; printf("%s/run - \033[1;34m%sS\033[0m", format_flops(op_flops_per_run).c_str(), - format_flops(result.flops_per_sec).c_str()); + format_flops(result.flops).c_str()); } else { printf("%8zu kB/run - \033[1;34m%7.2f GB/s\033[0m", result.memory_kb, @@ -565,7 +579,7 @@ struct sql_printer : public printer { void print_header() override { std::vector fields = test_result::get_fields(); - fprintf(fout, "CREATE TABLE IF NOT EXISTS test_results (\n"); + fprintf(fout, "CREATE TABLE IF NOT EXISTS test_backend_ops (\n"); for (size_t i = 0; i < fields.size(); i++) { fprintf(fout, " %s %s%s\n", fields[i].c_str(), get_sql_field_type(fields[i]).c_str(), i < fields.size() - 1 ? "," : ""); @@ -574,7 +588,7 @@ struct sql_printer : public printer { } void print_test_result(const test_result & result) override { - fprintf(fout, "INSERT INTO test_results ("); + fprintf(fout, "INSERT INTO test_backend_ops ("); std::vector fields = test_result::get_fields(); for (size_t i = 0; i < fields.size(); i++) { fprintf(fout, "%s%s", fields[i].c_str(), i < fields.size() - 1 ? ", " : ""); @@ -602,21 +616,17 @@ struct sql_printer : public printer { } void print_device_info(const char * format, ...) override { - // Do nothing - SQL format only outputs test results (void)format; } void print_test_summary(const char * format, ...) override { - // Do nothing - SQL format only outputs test results (void)format; } void print_status_ok() override { - // Do nothing - SQL format only outputs test results } void print_status_fail() override { - // Do nothing - SQL format only outputs test results } }; @@ -784,19 +794,8 @@ struct test_case { if (!supported) { // Create test result for unsupported operation - test_result result; - result.backend_name = ggml_backend_name(backend1); - result.op_name = current_op_name; - result.op_params = vars(); - result.test_mode = "test"; - result.supported = false; - result.passed = false; - result.error_message = "not supported"; - result.time_us = 0.0; - result.flops_per_sec = 0.0; - result.bandwidth_gb_s = 0.0; - result.memory_kb = 0; - result.n_runs = 0; + test_result result(ggml_backend_name(backend1), current_op_name, vars(), "test", + false, false, "not supported"); if (output_printer) { output_printer->print_test_result(result); @@ -912,19 +911,9 @@ struct test_case { // Create test result bool test_passed = ud.ok && cmp_ok; - test_result result; - result.backend_name = ggml_backend_name(backend1); - result.op_name = current_op_name; - result.op_params = vars(); - result.test_mode = "test"; - result.supported = supported; - result.passed = test_passed; - result.error_message = test_passed ? "" : (!cmp_ok ? "compare failed" : "test failed"); - result.time_us = 0.0; // Not measured in test mode - result.flops_per_sec = 0.0; - result.bandwidth_gb_s = 0.0; - result.memory_kb = 0; - result.n_runs = 0; + std::string error_msg = test_passed ? "" : (!cmp_ok ? "compare failed" : "test failed"); + test_result result(ggml_backend_name(backend1), current_op_name, vars(), "test", + supported, test_passed, error_msg); if (output_printer) { output_printer->print_test_result(result); @@ -956,19 +945,8 @@ struct test_case { // check if backends support op if (!ggml_backend_supports_op(backend, out)) { // Create test result for unsupported performance test - test_result result; - result.backend_name = ggml_backend_name(backend); - result.op_name = current_op_name; - result.op_params = vars(); - result.test_mode = "perf"; - result.supported = false; - result.passed = false; - result.error_message = "not supported"; - result.time_us = 0.0; - result.flops_per_sec = 0.0; - result.bandwidth_gb_s = 0.0; - result.memory_kb = 0; - result.n_runs = 0; + test_result result(ggml_backend_name(backend), current_op_name, vars(), "perf", + false, false, "not supported"); if (output_printer) { output_printer->print_test_result(result); @@ -1061,19 +1039,14 @@ struct test_case { } while (total_time_us < 1000*1000); // run for at least 1 second // Create test result - test_result result; - result.backend_name = ggml_backend_name(backend); - result.op_name = current_op_name; - result.op_params = vars(); - result.test_mode = "perf"; - result.supported = true; // If we got this far, it's supported - result.passed = true; // Performance tests don't fail - result.error_message = ""; - result.time_us = (double)total_time_us / total_runs; - result.flops_per_sec = (op_flops(out) > 0) ? (op_flops(out) * total_runs) / (total_time_us / 1e6) : 0.0; - result.bandwidth_gb_s = (op_flops(out) == 0) ? total_mem / (total_time_us / 1e6) / 1024.0 / 1024.0 / 1024.0 : 0.0; - result.memory_kb = op_size(out) / 1024; - result.n_runs = total_runs; + double avg_time_us = (double)total_time_us / total_runs; + double calculated_flops = (op_flops(out) > 0) ? (op_flops(out) * total_runs) / (total_time_us / 1e6) : 0.0; + double calculated_bandwidth = (op_flops(out) == 0) ? total_mem / (total_time_us / 1e6) / 1024.0 / 1024.0 / 1024.0 : 0.0; + size_t calculated_memory_kb = op_size(out) / 1024; + + test_result result(ggml_backend_name(backend), current_op_name, vars(), "perf", + true, true, "", avg_time_us, calculated_flops, calculated_bandwidth, + calculated_memory_kb, total_runs); if (output_printer) { output_printer->print_test_result(result); From 3cabda82505a25b486b7e0ddc647c8543cc5fa5b Mon Sep 17 00:00:00 2001 From: Xiaodong Ye Date: Thu, 26 Jun 2025 12:39:42 +0800 Subject: [PATCH 03/12] Add build_commit and build_number in test_result Signed-off-by: Xiaodong Ye --- tests/test-backend-ops.cpp | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 0b671b0f615e4..52c301f01af6e 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -38,6 +38,10 @@ #include #include +// External declarations for build info +extern int LLAMA_BUILD_NUMBER; +extern const char *LLAMA_COMMIT; + static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) { size_t nels = ggml_nelements(tensor); std::vector data(nels); @@ -347,6 +351,8 @@ static bool output_format_from_str(const std::string & s, output_formats & forma // Test result structure for SQL output struct test_result { std::string test_time; + std::string build_commit; + int build_number; std::string backend_name; std::string op_name; std::string op_params; @@ -375,6 +381,10 @@ struct test_result { char buf[32]; std::strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&t)); test_time = buf; + + // Set build info + build_commit = LLAMA_COMMIT; + build_number = LLAMA_BUILD_NUMBER; } test_result(const std::string& backend_name, const std::string& op_name, const std::string& op_params, @@ -389,11 +399,15 @@ struct test_result { char buf[32]; std::strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&t)); test_time = buf; + + // Set build info + build_commit = LLAMA_COMMIT; + build_number = LLAMA_BUILD_NUMBER; } static const std::vector & get_fields() { static const std::vector fields = { - "test_time", "backend_name", "op_name", "op_params", "test_mode", + "test_time", "build_commit", "build_number", "backend_name", "op_name", "op_params", "test_mode", "supported", "passed", "error_message", "time_us", "flops", "bandwidth_gb_s", "memory_kb", "n_runs" }; @@ -406,7 +420,7 @@ struct test_result { if (field == "supported" || field == "passed") { return BOOL; } - if (field == "memory_kb" || field == "n_runs") { + if (field == "memory_kb" || field == "n_runs" || field == "build_number") { return INT; } if (field == "time_us" || field == "flops" || field == "bandwidth_gb_s") { @@ -418,6 +432,8 @@ struct test_result { std::vector get_values() const { return { test_time, + build_commit, + std::to_string(build_number), backend_name, op_name, op_params, From 2a5cd8ed34e0b4e7d8c6751dcadbabec65355df6 Mon Sep 17 00:00:00 2001 From: Xiaodong Ye Date: Tue, 1 Jul 2025 17:38:31 +0800 Subject: [PATCH 04/12] Address review comments Signed-off-by: Xiaodong Ye --- tests/test-backend-ops.cpp | 164 +++++++++++++++---------------------- 1 file changed, 67 insertions(+), 97 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 52c301f01af6e..dfe6f6725e637 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -451,20 +451,20 @@ struct test_result { }; // Printer classes for different output formats +enum class message_type { + INFO, + ERROR, + STATUS_OK, + STATUS_FAIL +}; + struct printer { virtual ~printer() {} FILE * fout = stdout; virtual void print_header() {} virtual void print_test_result(const test_result & result) = 0; virtual void print_footer() {} - - // General purpose output methods - virtual void print_info(const char * format, ...) = 0; - virtual void print_error(const char * format, ...) = 0; - virtual void print_device_info(const char * format, ...) = 0; - virtual void print_test_summary(const char * format, ...) = 0; - virtual void print_status_ok() = 0; - virtual void print_status_fail() = 0; + virtual void print_message(message_type type, const char * format, ...) = 0; }; struct console_printer : public printer { @@ -476,42 +476,28 @@ struct console_printer : public printer { } } - void print_info(const char * format, ...) override { - va_list args; - va_start(args, format); - vprintf(format, args); - va_end(args); - } - - void print_error(const char * format, ...) override { + void print_message(message_type type, const char * format, ...) override { va_list args; va_start(args, format); - vfprintf(stderr, format, args); - va_end(args); - } - void print_device_info(const char * format, ...) override { - va_list args; - va_start(args, format); - vprintf(format, args); - va_end(args); - } + switch (type) { + case message_type::INFO: + vprintf(format, args); + break; + case message_type::ERROR: + vfprintf(stderr, format, args); + break; + case message_type::STATUS_OK: + printf("\033[1;32mOK\033[0m\n"); + break; + case message_type::STATUS_FAIL: + printf("\033[1;31mFAIL\033[0m\n"); + break; + } - void print_test_summary(const char * format, ...) override { - va_list args; - va_start(args, format); - vprintf(format, args); va_end(args); } - void print_status_ok() override { - printf("\033[1;32mOK\033[0m\n"); - } - - void print_status_fail() override { - printf("\033[1;31mFAIL\033[0m\n"); - } - private: void print_test_console(const test_result & result) { printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str()); @@ -617,32 +603,16 @@ struct sql_printer : public printer { fprintf(fout, ");\n"); } - // SQL printer ignores general output - only outputs test results - void print_info(const char * format, ...) override { - // Do nothing - SQL format only outputs test results - (void)format; - } - - void print_error(const char * format, ...) override { - // Still output errors to stderr for SQL format - va_list args; - va_start(args, format); - vfprintf(stderr, format, args); - va_end(args); - } - - void print_device_info(const char * format, ...) override { - (void)format; - } - - void print_test_summary(const char * format, ...) override { - (void)format; - } - - void print_status_ok() override { - } - - void print_status_fail() override { + // SQL printer ignores most output types - only outputs test results and errors + void print_message(message_type type, const char * format, ...) override { + if (type == message_type::ERROR) { + // Still output errors to stderr for SQL format + va_list args; + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); + } + // All other message types are ignored in SQL format } }; @@ -1089,14 +1059,14 @@ struct test_case { ggml_tensor * out = build_graph(ctx.get()); if ((op_name != nullptr && op_desc(out) != op_name) || out->op == GGML_OP_OPT_STEP_ADAMW) { - //output_printer->print_info(" %s: skipping\n", op_desc(out).c_str()); + //output_printer->print_message(message_type::INFO, " %s: skipping\n", op_desc(out).c_str()); return true; } - output_printer->print_info(" %s(%s): ", op_desc(out).c_str(), vars().c_str()); + output_printer->print_message(message_type::INFO, " %s(%s): ", op_desc(out).c_str(), vars().c_str()); if (out->type != GGML_TYPE_F32) { - output_printer->print_info("not supported [%s->type != FP32]\n", out->name); + output_printer->print_message(message_type::INFO, "not supported [%s->type != FP32]\n", out->name); return true; } @@ -1105,25 +1075,25 @@ struct test_case { bool any_params = false; for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { if (!ggml_backend_supports_op(backend, t)) { - output_printer->print_info("not supported [%s] ", ggml_backend_name(backend)); + output_printer->print_message(message_type::INFO, "not supported [%s] ", ggml_backend_name(backend)); supported = false; break; } if ((t->flags & GGML_TENSOR_FLAG_PARAM)) { any_params = true; if (t->type != GGML_TYPE_F32) { - output_printer->print_info("not supported [%s->type != FP32] ", t->name); + output_printer->print_message(message_type::INFO, "not supported [%s->type != FP32] ", t->name); supported = false; break; } } } if (!any_params) { - output_printer->print_info("not supported [%s] \n", op_desc(out).c_str()); + output_printer->print_message(message_type::INFO, "not supported [%s] \n", op_desc(out).c_str()); supported = false; } if (!supported) { - output_printer->print_info("\n"); + output_printer->print_message(message_type::INFO, "\n"); return true; } @@ -1134,7 +1104,7 @@ struct test_case { } } if (ngrads > grad_nmax()) { - output_printer->print_info("skipping large tensors for speed \n"); + output_printer->print_message(message_type::INFO, "skipping large tensors for speed \n"); return true; } @@ -1157,25 +1127,25 @@ struct test_case { for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { if (!ggml_backend_supports_op(backend, t)) { - output_printer->print_info("not supported [%s] ", ggml_backend_name(backend)); + output_printer->print_message(message_type::INFO, "not supported [%s] ", ggml_backend_name(backend)); supported = false; break; } if ((t->flags & GGML_TENSOR_FLAG_PARAM) && t->type != GGML_TYPE_F32) { - output_printer->print_info("not supported [%s->type != FP32] ", t->name); + output_printer->print_message(message_type::INFO, "not supported [%s->type != FP32] ", t->name); supported = false; break; } } if (!supported) { - output_printer->print_info("\n"); + output_printer->print_message(message_type::INFO, "\n"); return true; } // allocate ggml_backend_buffer_ptr buf(ggml_backend_alloc_ctx_tensors(ctx.get(), backend)); // smart ptr if (buf == NULL) { - output_printer->print_error("failed to allocate tensors [%s] ", ggml_backend_name(backend)); + output_printer->print_message(message_type::ERROR, "failed to allocate tensors [%s] ", ggml_backend_name(backend)); return false; } @@ -1213,7 +1183,7 @@ struct test_case { for (int64_t i = 0; i < ne; ++i) { // gradient algebraic // check for nans if (!std::isfinite(ga[i])) { - output_printer->print_info("[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", ggml_op_desc(t), i, bn, ga[i]); + output_printer->print_message(message_type::INFO, "[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", ggml_op_desc(t), i, bn, ga[i]); ok = false; break; } @@ -1281,7 +1251,7 @@ struct test_case { const double err = mean_abs_asymm(gn.data(), ga.data(), gn.size(), expect); if (err > max_maa_err()) { - output_printer->print_info("[%s] MAA = %.9f > %.9f ", ggml_op_desc(t), err, max_maa_err()); + output_printer->print_message(message_type::INFO, "[%s] MAA = %.9f > %.9f ", ggml_op_desc(t), err, max_maa_err()); ok = false; break; } @@ -1291,15 +1261,15 @@ struct test_case { } if (!ok) { - output_printer->print_info("compare failed "); + output_printer->print_message(message_type::INFO, "compare failed "); } if (ok) { - output_printer->print_status_ok(); + output_printer->print_message(message_type::STATUS_OK, ""); return true; } - output_printer->print_status_fail(); + output_printer->print_message(message_type::STATUS_FAIL, ""); return false; } }; @@ -5330,7 +5300,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op filter_test_cases(test_cases, params_filter); ggml_backend_t backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, NULL); if (backend_cpu == NULL) { - output_printer->print_error(" Failed to initialize CPU backend\n"); + output_printer->print_message(message_type::ERROR, " Failed to initialize CPU backend\n"); return false; } @@ -5340,7 +5310,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op n_ok++; } } - output_printer->print_test_summary(" %zu/%zu tests passed\n", n_ok, test_cases.size()); + output_printer->print_message(message_type::INFO, " %zu/%zu tests passed\n", n_ok, test_cases.size()); ggml_backend_free(backend_cpu); @@ -5356,7 +5326,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op n_ok++; } } - output_printer->print_test_summary(" %zu/%zu tests passed\n", n_ok, test_cases.size()); + output_printer->print_message(message_type::INFO, " %zu/%zu tests passed\n", n_ok, test_cases.size()); return n_ok == test_cases.size(); } @@ -5443,23 +5413,23 @@ int main(int argc, char ** argv) { output_printer->print_header(); } - output_printer->print_info("Testing %zu devices\n\n", ggml_backend_dev_count()); + output_printer->print_message(message_type::INFO, "Testing %zu devices\n\n", ggml_backend_dev_count()); size_t n_ok = 0; for (size_t i = 0; i < ggml_backend_dev_count(); i++) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); - output_printer->print_device_info("Backend %zu/%zu: %s\n", i + 1, ggml_backend_dev_count(), ggml_backend_dev_name(dev)); + output_printer->print_message(message_type::INFO, "Backend %zu/%zu: %s\n", i + 1, ggml_backend_dev_count(), ggml_backend_dev_name(dev)); if (backend_filter != NULL && strcmp(backend_filter, ggml_backend_dev_name(dev)) != 0) { - output_printer->print_device_info(" Skipping\n"); + output_printer->print_message(message_type::INFO, " Skipping\n"); n_ok++; continue; } if (backend_filter == NULL && ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU && mode != MODE_GRAD) { - output_printer->print_device_info(" Skipping CPU backend\n"); + output_printer->print_message(message_type::INFO, " Skipping CPU backend\n"); n_ok++; continue; } @@ -5474,23 +5444,23 @@ int main(int argc, char ** argv) { ggml_backend_set_n_threads_fn(backend, std::thread::hardware_concurrency()); } - output_printer->print_device_info(" Device description: %s\n", ggml_backend_dev_description(dev)); + output_printer->print_message(message_type::INFO, " Device description: %s\n", ggml_backend_dev_description(dev)); size_t free, total; // NOLINT ggml_backend_dev_memory(dev, &free, &total); - output_printer->print_device_info(" Device memory: %zu MB (%zu MB free)\n", total / 1024 / 1024, free / 1024 / 1024); - output_printer->print_device_info("\n"); + output_printer->print_message(message_type::INFO, " Device memory: %zu MB (%zu MB free)\n", total / 1024 / 1024, free / 1024 / 1024); + output_printer->print_message(message_type::INFO, "\n"); bool ok = test_backend(backend, mode, op_name_filter, params_filter, output_printer.get()); - output_printer->print_device_info(" Backend %s: ", ggml_backend_name(backend)); + output_printer->print_message(message_type::INFO, " Backend %s: ", ggml_backend_name(backend)); if (ok) { - output_printer->print_status_ok(); + output_printer->print_message(message_type::STATUS_OK, ""); n_ok++; } else { - output_printer->print_status_fail(); + output_printer->print_message(message_type::STATUS_FAIL, ""); } - output_printer->print_device_info("\n"); + output_printer->print_message(message_type::INFO, "\n"); ggml_backend_free(backend); } @@ -5501,13 +5471,13 @@ int main(int argc, char ** argv) { output_printer->print_footer(); } - output_printer->print_test_summary("%zu/%zu backends passed\n", n_ok, ggml_backend_dev_count()); + output_printer->print_message(message_type::INFO, "%zu/%zu backends passed\n", n_ok, ggml_backend_dev_count()); if (n_ok != ggml_backend_dev_count()) { - output_printer->print_status_fail(); + output_printer->print_message(message_type::STATUS_FAIL, ""); return 1; } - output_printer->print_status_ok(); + output_printer->print_message(message_type::STATUS_OK, ""); return 0; } From d3f68bd8d429d602b5f85f87c5ca76bad0e716f8 Mon Sep 17 00:00:00 2001 From: Xiaodong Ye Date: Wed, 2 Jul 2025 10:09:49 +0800 Subject: [PATCH 05/12] refactor Signed-off-by: Xiaodong Ye --- tests/test-backend-ops.cpp | 408 ++++++++++++++++++++++++++++++------- 1 file changed, 337 insertions(+), 71 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index dfe6f6725e637..6bdf4203f4c8c 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -451,11 +451,137 @@ struct test_result { }; // Printer classes for different output formats -enum class message_type { - INFO, - ERROR, - STATUS_OK, - STATUS_FAIL +struct test_operation_info { + std::string op_name; + std::string op_params; + std::string backend_name; + bool supported = true; + std::string failure_reason; + + test_operation_info() = default; + test_operation_info(const std::string& op_name, const std::string& op_params, const std::string& backend_name, bool supported = true, const std::string& failure_reason = "") + : op_name(op_name), op_params(op_params), backend_name(backend_name), supported(supported), failure_reason(failure_reason) {} +}; + +struct device_info { + size_t device_index; + size_t total_devices; + std::string device_name; + std::string description; + size_t memory_total_mb; + size_t memory_free_mb; + bool skipped = false; + std::string skip_reason; + + device_info() = default; + device_info(size_t device_index, size_t total_devices, const std::string& device_name, const std::string& description = "", + size_t memory_total_mb = 0, size_t memory_free_mb = 0, bool skipped = false, const std::string& skip_reason = "") + : device_index(device_index), total_devices(total_devices), device_name(device_name), description(description), + memory_total_mb(memory_total_mb), memory_free_mb(memory_free_mb), skipped(skipped), skip_reason(skip_reason) {} +}; + +struct test_summary_info { + size_t tests_passed; + size_t tests_total; + bool is_backend_summary = false; // true for backend summary, false for test summary + + test_summary_info() = default; + test_summary_info(size_t tests_passed, size_t tests_total, bool is_backend_summary = false) + : tests_passed(tests_passed), tests_total(tests_total), is_backend_summary(is_backend_summary) {} +}; + +struct error_info { + std::string component; // e.g., "backend", "allocation", "gradient" + std::string details; + std::string backend_name; + + error_info() = default; + error_info(const std::string& component, const std::string& details, const std::string& backend_name = "") + : component(component), details(details), backend_name(backend_name) {} +}; + +struct test_status_info { + bool passed; + std::string details; // optional additional info + + test_status_info() = default; + test_status_info(bool passed, const std::string& details = "") + : passed(passed), details(details) {} +}; + +struct testing_start_info { + size_t device_count; + + testing_start_info() = default; + testing_start_info(size_t device_count) : device_count(device_count) {} +}; + +struct backend_init_info { + size_t device_index; + size_t total_devices; + std::string device_name; + bool skipped = false; + std::string skip_reason; + std::string description; + size_t memory_total_mb = 0; + size_t memory_free_mb = 0; + bool has_memory_info = false; + + backend_init_info() = default; + backend_init_info(size_t device_index, size_t total_devices, const std::string& device_name, bool skipped = false, + const std::string& skip_reason = "", const std::string& description = "", + size_t memory_total_mb = 0, size_t memory_free_mb = 0, bool has_memory_info = false) + : device_index(device_index), total_devices(total_devices), device_name(device_name), skipped(skipped), + skip_reason(skip_reason), description(description), memory_total_mb(memory_total_mb), + memory_free_mb(memory_free_mb), has_memory_info(has_memory_info) {} +}; + +struct backend_status_info { + std::string backend_name; + bool passed; + + backend_status_info() = default; + backend_status_info(const std::string& backend_name, bool passed) + : backend_name(backend_name), passed(passed) {} +}; + +struct overall_summary_info { + size_t backends_passed; + size_t backends_total; + bool all_passed; + + overall_summary_info() = default; + overall_summary_info(size_t backends_passed, size_t backends_total, bool all_passed) + : backends_passed(backends_passed), backends_total(backends_total), all_passed(all_passed) {} +}; + +struct gradient_info { + std::string op_desc; + int64_t index; + std::string param_name; + float value; + + gradient_info() = default; + gradient_info(const std::string& op_desc, int64_t index, const std::string& param_name, float value) + : op_desc(op_desc), index(index), param_name(param_name), value(value) {} +}; + +struct maa_error_info { + std::string op_desc; + double error; + double threshold; + + maa_error_info() = default; + maa_error_info(const std::string& op_desc, double error, double threshold) + : op_desc(op_desc), error(error), threshold(threshold) {} +}; + +struct compare_failure_info { + // Empty for now - just indicates compare failure +}; + +struct large_tensor_skip_info { + // Empty for now - just indicates large tensor skip }; struct printer { @@ -464,7 +590,14 @@ struct printer { virtual void print_header() {} virtual void print_test_result(const test_result & result) = 0; virtual void print_footer() {} - virtual void print_message(message_type type, const char * format, ...) = 0; + + template + void print_message(const T& data) { + print_message_impl(&data, typeid(T).name()); + } + +protected: + virtual void print_message_impl(const void* data, const char* type_name) = 0; }; struct console_printer : public printer { @@ -476,29 +609,161 @@ struct console_printer : public printer { } } - void print_message(message_type type, const char * format, ...) override { - va_list args; - va_start(args, format); +protected: + void print_message_impl(const void* data, const char* type_name) override { + std::string type_str(type_name); + + if (type_str.find("test_operation_info") != std::string::npos) { + handle_message(*static_cast(data)); + } else if (type_str.find("device_info") != std::string::npos) { + handle_message(*static_cast(data)); + } else if (type_str.find("test_summary_info") != std::string::npos) { + handle_message(*static_cast(data)); + } else if (type_str.find("error_info") != std::string::npos) { + handle_message(*static_cast(data)); + } else if (type_str.find("test_status_info") != std::string::npos) { + handle_message(*static_cast(data)); + } else if (type_str.find("testing_start_info") != std::string::npos) { + handle_message(*static_cast(data)); + } else if (type_str.find("backend_init_info") != std::string::npos) { + handle_message(*static_cast(data)); + } else if (type_str.find("backend_status_info") != std::string::npos) { + handle_message(*static_cast(data)); + } else if (type_str.find("overall_summary_info") != std::string::npos) { + handle_message(*static_cast(data)); + } else if (type_str.find("gradient_info") != std::string::npos) { + handle_message(*static_cast(data)); + } else if (type_str.find("maa_error_info") != std::string::npos) { + handle_message(*static_cast(data)); + } else if (type_str.find("compare_failure_info") != std::string::npos) { + handle_message(*static_cast(data)); + } else if (type_str.find("large_tensor_skip_info") != std::string::npos) { + handle_message(*static_cast(data)); + } + } - switch (type) { - case message_type::INFO: - vprintf(format, args); - break; - case message_type::ERROR: - vfprintf(stderr, format, args); - break; - case message_type::STATUS_OK: - printf("\033[1;32mOK\033[0m\n"); - break; - case message_type::STATUS_FAIL: - printf("\033[1;31mFAIL\033[0m\n"); - break; +private: + void handle_message(const test_operation_info& info) { + printf(" %s(%s): ", info.op_name.c_str(), info.op_params.c_str()); + fflush(stdout); + + if (!info.supported) { + if (!info.failure_reason.empty()) { + printf("not supported [%s]\n", info.failure_reason.c_str()); + } else { + printf("not supported [%s]\n", info.backend_name.c_str()); + } + } + } + + void handle_message(const device_info& info) { + if (info.device_index == 0) { + // This is the first device, don't print the header info here + } + + printf("Backend %zu/%zu: %s\n", info.device_index + 1, info.total_devices, info.device_name.c_str()); + + if (info.skipped) { + printf(" %s\n", info.skip_reason.c_str()); + return; + } + + if (!info.description.empty()) { + printf(" Device description: %s\n", info.description.c_str()); } - va_end(args); + if (info.memory_total_mb > 0) { + printf(" Device memory: %zu MB (%zu MB free)\n", info.memory_total_mb, info.memory_free_mb); + } + + printf("\n"); + } + + void handle_message(const test_summary_info& info) { + if (info.is_backend_summary) { + printf("%zu/%zu backends passed\n", info.tests_passed, info.tests_total); + } else { + printf(" %zu/%zu tests passed\n", info.tests_passed, info.tests_total); + } + } + + void handle_message(const error_info& info) { + if (info.component == "allocation") { + fprintf(stderr, "failed to allocate tensors [%s] ", info.backend_name.c_str()); + } else if (info.component == "backend") { + fprintf(stderr, " Failed to initialize %s backend\n", info.backend_name.c_str()); + } else { + fprintf(stderr, "Error in %s: %s\n", info.component.c_str(), info.details.c_str()); + } + } + + void handle_message(const test_status_info& info) { + if (info.passed) { + printf("\033[1;32mOK\033[0m\n"); + } else { + printf("\033[1;31mFAIL\033[0m\n"); + } + } + + void handle_message(const testing_start_info& info) { + printf("Testing %zu devices\n\n", info.device_count); + } + + void handle_message(const backend_init_info& info) { + printf("Backend %zu/%zu: %s\n", info.device_index + 1, info.total_devices, info.device_name.c_str()); + + if (info.skipped) { + printf(" %s\n", info.skip_reason.c_str()); + return; + } + + if (!info.description.empty()) { + printf(" Device description: %s\n", info.description.c_str()); + } + + if (info.has_memory_info) { + printf(" Device memory: %zu MB (%zu MB free)\n", info.memory_total_mb, info.memory_free_mb); + } + + printf("\n"); + } + + void handle_message(const backend_status_info& info) { + printf(" Backend %s: ", info.backend_name.c_str()); + if (info.passed) { + printf("\033[1;32mOK\033[0m\n"); + } else { + printf("\033[1;31mFAIL\033[0m\n"); + } + } + + void handle_message(const overall_summary_info& info) { + printf("%zu/%zu backends passed\n", info.backends_passed, info.backends_total); + if (info.all_passed) { + printf("\033[1;32mOK\033[0m\n"); + } else { + printf("\033[1;31mFAIL\033[0m\n"); + } + } + + void handle_message(const gradient_info& info) { + printf("[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", info.op_desc.c_str(), info.index, info.param_name.c_str(), info.value); + } + + void handle_message(const maa_error_info& info) { + printf("[%s] MAA = %.9f > %.9f ", info.op_desc.c_str(), info.error, info.threshold); + } + + void handle_message(const compare_failure_info& info) { + (void)info; // unused + printf("compare failed "); + } + + void handle_message(const large_tensor_skip_info& info) { + (void)info; // unused + printf("skipping large tensors for speed \n"); } -private: void print_test_console(const test_result & result) { printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str()); fflush(stdout); @@ -603,16 +868,24 @@ struct sql_printer : public printer { fprintf(fout, ");\n"); } - // SQL printer ignores most output types - only outputs test results and errors - void print_message(message_type type, const char * format, ...) override { - if (type == message_type::ERROR) { - // Still output errors to stderr for SQL format - va_list args; - va_start(args, format); - vfprintf(stderr, format, args); - va_end(args); +protected: + // Implementation that checks type and dispatches to appropriate handler + void print_message_impl(const void* data, const char* type_name) override { + // Use string comparison to identify types and cast accordingly + std::string type_str(type_name); + + if (type_str.find("error_info") != std::string::npos) { + handle_message(*static_cast(data)); + } + } + +private: + void handle_message(const error_info& info) { + // Only output critical errors that affect SQL generation + if (info.component == "backend" || info.component == "allocation") { + fprintf(stderr, "Critical error in %s: %s\n", info.component.c_str(), + info.backend_name.empty() ? info.details.c_str() : info.backend_name.c_str()); } - // All other message types are ignored in SQL format } }; @@ -1059,41 +1332,46 @@ struct test_case { ggml_tensor * out = build_graph(ctx.get()); if ((op_name != nullptr && op_desc(out) != op_name) || out->op == GGML_OP_OPT_STEP_ADAMW) { - //output_printer->print_message(message_type::INFO, " %s: skipping\n", op_desc(out).c_str()); return true; } - output_printer->print_message(message_type::INFO, " %s(%s): ", op_desc(out).c_str(), vars().c_str()); - if (out->type != GGML_TYPE_F32) { - output_printer->print_message(message_type::INFO, "not supported [%s->type != FP32]\n", out->name); + output_printer->print_message(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), false, out->name + std::string("->type != FP32"))); return true; } + // Print operation info first + output_printer->print_message(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend))); + // check if the backend supports the ops bool supported = true; bool any_params = false; + std::string failure_reason; + for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { if (!ggml_backend_supports_op(backend, t)) { - output_printer->print_message(message_type::INFO, "not supported [%s] ", ggml_backend_name(backend)); supported = false; + failure_reason = ggml_backend_name(backend); break; } if ((t->flags & GGML_TENSOR_FLAG_PARAM)) { any_params = true; if (t->type != GGML_TYPE_F32) { - output_printer->print_message(message_type::INFO, "not supported [%s->type != FP32] ", t->name); supported = false; + failure_reason = std::string(t->name) + "->type != FP32"; break; } } } if (!any_params) { - output_printer->print_message(message_type::INFO, "not supported [%s] \n", op_desc(out).c_str()); supported = false; + failure_reason = op_desc(out); + } + + if (!supported) { + output_printer->print_message(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), false, failure_reason)); } if (!supported) { - output_printer->print_message(message_type::INFO, "\n"); return true; } @@ -1104,7 +1382,7 @@ struct test_case { } } if (ngrads > grad_nmax()) { - output_printer->print_message(message_type::INFO, "skipping large tensors for speed \n"); + output_printer->print_message(large_tensor_skip_info{}); return true; } @@ -1127,25 +1405,24 @@ struct test_case { for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { if (!ggml_backend_supports_op(backend, t)) { - output_printer->print_message(message_type::INFO, "not supported [%s] ", ggml_backend_name(backend)); + output_printer->print_message(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), false, ggml_backend_name(backend))); supported = false; break; } if ((t->flags & GGML_TENSOR_FLAG_PARAM) && t->type != GGML_TYPE_F32) { - output_printer->print_message(message_type::INFO, "not supported [%s->type != FP32] ", t->name); + output_printer->print_message(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), false, std::string(t->name) + "->type != FP32")); supported = false; break; } } if (!supported) { - output_printer->print_message(message_type::INFO, "\n"); return true; } // allocate ggml_backend_buffer_ptr buf(ggml_backend_alloc_ctx_tensors(ctx.get(), backend)); // smart ptr if (buf == NULL) { - output_printer->print_message(message_type::ERROR, "failed to allocate tensors [%s] ", ggml_backend_name(backend)); + output_printer->print_message(error_info("allocation", "", ggml_backend_name(backend))); return false; } @@ -1183,7 +1460,7 @@ struct test_case { for (int64_t i = 0; i < ne; ++i) { // gradient algebraic // check for nans if (!std::isfinite(ga[i])) { - output_printer->print_message(message_type::INFO, "[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", ggml_op_desc(t), i, bn, ga[i]); + output_printer->print_message(gradient_info(ggml_op_desc(t), i, bn, ga[i])); ok = false; break; } @@ -1251,7 +1528,7 @@ struct test_case { const double err = mean_abs_asymm(gn.data(), ga.data(), gn.size(), expect); if (err > max_maa_err()) { - output_printer->print_message(message_type::INFO, "[%s] MAA = %.9f > %.9f ", ggml_op_desc(t), err, max_maa_err()); + output_printer->print_message(maa_error_info(ggml_op_desc(t), err, max_maa_err())); ok = false; break; } @@ -1261,15 +1538,15 @@ struct test_case { } if (!ok) { - output_printer->print_message(message_type::INFO, "compare failed "); + output_printer->print_message(compare_failure_info{}); } + output_printer->print_message(test_status_info(ok)); + if (ok) { - output_printer->print_message(message_type::STATUS_OK, ""); return true; } - output_printer->print_message(message_type::STATUS_FAIL, ""); return false; } }; @@ -5300,7 +5577,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op filter_test_cases(test_cases, params_filter); ggml_backend_t backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, NULL); if (backend_cpu == NULL) { - output_printer->print_message(message_type::ERROR, " Failed to initialize CPU backend\n"); + output_printer->print_message(error_info("backend", "Failed to initialize CPU backend", "CPU")); return false; } @@ -5310,7 +5587,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op n_ok++; } } - output_printer->print_message(message_type::INFO, " %zu/%zu tests passed\n", n_ok, test_cases.size()); + output_printer->print_message(test_summary_info(n_ok, test_cases.size(), false)); ggml_backend_free(backend_cpu); @@ -5326,7 +5603,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op n_ok++; } } - output_printer->print_message(message_type::INFO, " %zu/%zu tests passed\n", n_ok, test_cases.size()); + output_printer->print_message(test_summary_info(n_ok, test_cases.size(), false)); return n_ok == test_cases.size(); } @@ -5413,23 +5690,21 @@ int main(int argc, char ** argv) { output_printer->print_header(); } - output_printer->print_message(message_type::INFO, "Testing %zu devices\n\n", ggml_backend_dev_count()); + output_printer->print_message(testing_start_info(ggml_backend_dev_count())); size_t n_ok = 0; for (size_t i = 0; i < ggml_backend_dev_count(); i++) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); - output_printer->print_message(message_type::INFO, "Backend %zu/%zu: %s\n", i + 1, ggml_backend_dev_count(), ggml_backend_dev_name(dev)); - if (backend_filter != NULL && strcmp(backend_filter, ggml_backend_dev_name(dev)) != 0) { - output_printer->print_message(message_type::INFO, " Skipping\n"); + output_printer->print_message(backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), true, "Skipping")); n_ok++; continue; } if (backend_filter == NULL && ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU && mode != MODE_GRAD) { - output_printer->print_message(message_type::INFO, " Skipping CPU backend\n"); + output_printer->print_message(backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), true, "Skipping CPU backend")); n_ok++; continue; } @@ -5444,23 +5719,16 @@ int main(int argc, char ** argv) { ggml_backend_set_n_threads_fn(backend, std::thread::hardware_concurrency()); } - output_printer->print_message(message_type::INFO, " Device description: %s\n", ggml_backend_dev_description(dev)); size_t free, total; // NOLINT ggml_backend_dev_memory(dev, &free, &total); - output_printer->print_message(message_type::INFO, " Device memory: %zu MB (%zu MB free)\n", total / 1024 / 1024, free / 1024 / 1024); - output_printer->print_message(message_type::INFO, "\n"); + output_printer->print_message(backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), false, "", ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024, true)); bool ok = test_backend(backend, mode, op_name_filter, params_filter, output_printer.get()); - output_printer->print_message(message_type::INFO, " Backend %s: ", ggml_backend_name(backend)); if (ok) { - output_printer->print_message(message_type::STATUS_OK, ""); n_ok++; - } else { - output_printer->print_message(message_type::STATUS_FAIL, ""); } - - output_printer->print_message(message_type::INFO, "\n"); + output_printer->print_message(backend_status_info(ggml_backend_name(backend), ok)); ggml_backend_free(backend); } @@ -5471,13 +5739,11 @@ int main(int argc, char ** argv) { output_printer->print_footer(); } - output_printer->print_message(message_type::INFO, "%zu/%zu backends passed\n", n_ok, ggml_backend_dev_count()); + output_printer->print_message(overall_summary_info(n_ok, ggml_backend_dev_count(), n_ok == ggml_backend_dev_count())); if (n_ok != ggml_backend_dev_count()) { - output_printer->print_message(message_type::STATUS_FAIL, ""); return 1; } - output_printer->print_message(message_type::STATUS_OK, ""); return 0; } From a4b376f7c6c5d03c4f2ac4ba99d4f7ebd64838b8 Mon Sep 17 00:00:00 2001 From: Xiaodong Ye Date: Thu, 3 Jul 2025 08:24:28 +0800 Subject: [PATCH 06/12] Get build commit from ggml_commit() Signed-off-by: Xiaodong Ye --- tests/test-backend-ops.cpp | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 6bdf4203f4c8c..fec6fbd8ad03c 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -38,10 +38,6 @@ #include #include -// External declarations for build info -extern int LLAMA_BUILD_NUMBER; -extern const char *LLAMA_COMMIT; - static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) { size_t nels = ggml_nelements(tensor); std::vector data(nels); @@ -352,7 +348,6 @@ static bool output_format_from_str(const std::string & s, output_formats & forma struct test_result { std::string test_time; std::string build_commit; - int build_number; std::string backend_name; std::string op_name; std::string op_params; @@ -383,8 +378,7 @@ struct test_result { test_time = buf; // Set build info - build_commit = LLAMA_COMMIT; - build_number = LLAMA_BUILD_NUMBER; + build_commit = ggml_commit(); } test_result(const std::string& backend_name, const std::string& op_name, const std::string& op_params, @@ -401,15 +395,14 @@ struct test_result { test_time = buf; // Set build info - build_commit = LLAMA_COMMIT; - build_number = LLAMA_BUILD_NUMBER; + build_commit = ggml_commit(); } static const std::vector & get_fields() { static const std::vector fields = { - "test_time", "build_commit", "build_number", "backend_name", "op_name", "op_params", "test_mode", - "supported", "passed", "error_message", "time_us", "flops", - "bandwidth_gb_s", "memory_kb", "n_runs" + "test_time", "build_commit", "backend_name", "op_name", "op_params", + "test_mode", "supported", "passed", "error_message", "time_us", + "flops", "bandwidth_gb_s", "memory_kb", "n_runs" }; return fields; } @@ -420,7 +413,7 @@ struct test_result { if (field == "supported" || field == "passed") { return BOOL; } - if (field == "memory_kb" || field == "n_runs" || field == "build_number") { + if (field == "memory_kb" || field == "n_runs") { return INT; } if (field == "time_us" || field == "flops" || field == "bandwidth_gb_s") { @@ -433,7 +426,6 @@ struct test_result { return { test_time, build_commit, - std::to_string(build_number), backend_name, op_name, op_params, From fb835fa25bb4d33a88fe7596377f87be9ccf85ca Mon Sep 17 00:00:00 2001 From: Xiaodong Ye Date: Thu, 3 Jul 2025 14:33:00 +0800 Subject: [PATCH 07/12] Merge errors into test_operation_info && address review comments Signed-off-by: Xiaodong Ye --- tests/test-backend-ops.cpp | 311 +++++++++++++++++-------------------- 1 file changed, 144 insertions(+), 167 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index fec6fbd8ad03c..f0270e8869919 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -442,34 +442,89 @@ struct test_result { } }; +// Enum for test status +enum class test_status_t { + NOT_SUPPORTED, + OK, + FAIL +}; + // Printer classes for different output formats struct test_operation_info { std::string op_name; std::string op_params; std::string backend_name; - bool supported = true; + test_status_t status = test_status_t::OK; std::string failure_reason; + // Additional information fields that were previously in separate structs + std::string error_component; + std::string error_details; + + // Gradient info + int64_t gradient_index = -1; + std::string gradient_param_name; + float gradient_value = 0.0f; + + // MAA error info + double maa_error = 0.0; + double maa_threshold = 0.0; + + // Flags for different types of information + bool has_error = false; + bool has_gradient_info = false; + bool has_maa_error = false; + bool is_compare_failure = false; + bool is_large_tensor_skip = false; + test_operation_info() = default; - test_operation_info(const std::string& op_name, const std::string& op_params, const std::string& backend_name, bool supported = true, const std::string& failure_reason = "") - : op_name(op_name), op_params(op_params), backend_name(backend_name), supported(supported), failure_reason(failure_reason) {} -}; + test_operation_info(const std::string& op_name, const std::string& op_params, const std::string& backend_name, + test_status_t status = test_status_t::OK, const std::string& failure_reason = "") + : op_name(op_name), op_params(op_params), backend_name(backend_name), status(status), failure_reason(failure_reason) {} -struct device_info { - size_t device_index; - size_t total_devices; - std::string device_name; - std::string description; - size_t memory_total_mb; - size_t memory_free_mb; - bool skipped = false; - std::string skip_reason; + // Set error information + void set_error(const std::string& component, const std::string& details) { + has_error = true; + error_component = component; + error_details = details; + if (status == test_status_t::OK) { + status = test_status_t::FAIL; + } + } + + // Set gradient information + void set_gradient_info(int64_t index, const std::string& param_name, float value) { + has_gradient_info = true; + gradient_index = index; + gradient_param_name = param_name; + gradient_value = value; + if (status == test_status_t::OK) { + status = test_status_t::FAIL; + } + } + + // Set MAA error information + void set_maa_error(double error, double threshold) { + has_maa_error = true; + maa_error = error; + maa_threshold = threshold; + if (status == test_status_t::OK) { + status = test_status_t::FAIL; + } + } + + // Set compare failure + void set_compare_failure() { + is_compare_failure = true; + if (status == test_status_t::OK) { + status = test_status_t::FAIL; + } + } - device_info() = default; - device_info(size_t device_index, size_t total_devices, const std::string& device_name, const std::string& description = "", - size_t memory_total_mb = 0, size_t memory_free_mb = 0, bool skipped = false, const std::string& skip_reason = "") - : device_index(device_index), total_devices(total_devices), device_name(device_name), description(description), - memory_total_mb(memory_total_mb), memory_free_mb(memory_free_mb), skipped(skipped), skip_reason(skip_reason) {} + // Set large tensor skip + void set_large_tensor_skip() { + is_large_tensor_skip = true; + } }; struct test_summary_info { @@ -482,25 +537,6 @@ struct test_summary_info { : tests_passed(tests_passed), tests_total(tests_total), is_backend_summary(is_backend_summary) {} }; -struct error_info { - std::string component; // e.g., "backend", "allocation", "gradient" - std::string details; - std::string backend_name; - - error_info() = default; - error_info(const std::string& component, const std::string& details, const std::string& backend_name = "") - : component(component), details(details), backend_name(backend_name) {} -}; - -struct test_status_info { - bool passed; - std::string details; // optional additional info - - test_status_info() = default; - test_status_info(bool passed, const std::string& details = "") - : passed(passed), details(details) {} -}; - struct testing_start_info { size_t device_count; @@ -530,11 +566,11 @@ struct backend_init_info { struct backend_status_info { std::string backend_name; - bool passed; + test_status_t status; backend_status_info() = default; - backend_status_info(const std::string& backend_name, bool passed) - : backend_name(backend_name), passed(passed) {} + backend_status_info(const std::string& backend_name, test_status_t status) + : backend_name(backend_name), status(status) {} }; struct overall_summary_info { @@ -547,35 +583,6 @@ struct overall_summary_info { : backends_passed(backends_passed), backends_total(backends_total), all_passed(all_passed) {} }; -struct gradient_info { - std::string op_desc; - int64_t index; - std::string param_name; - float value; - - gradient_info() = default; - gradient_info(const std::string& op_desc, int64_t index, const std::string& param_name, float value) - : op_desc(op_desc), index(index), param_name(param_name), value(value) {} -}; - -struct maa_error_info { - std::string op_desc; - double error; - double threshold; - - maa_error_info() = default; - maa_error_info(const std::string& op_desc, double error, double threshold) - : op_desc(op_desc), error(error), threshold(threshold) {} -}; - -struct compare_failure_info { - // Empty for now - just indicates compare failure -}; - -struct large_tensor_skip_info { - // Empty for now - just indicates large tensor skip -}; - struct printer { virtual ~printer() {} FILE * fout = stdout; @@ -607,14 +614,8 @@ struct console_printer : public printer { if (type_str.find("test_operation_info") != std::string::npos) { handle_message(*static_cast(data)); - } else if (type_str.find("device_info") != std::string::npos) { - handle_message(*static_cast(data)); } else if (type_str.find("test_summary_info") != std::string::npos) { handle_message(*static_cast(data)); - } else if (type_str.find("error_info") != std::string::npos) { - handle_message(*static_cast(data)); - } else if (type_str.find("test_status_info") != std::string::npos) { - handle_message(*static_cast(data)); } else if (type_str.find("testing_start_info") != std::string::npos) { handle_message(*static_cast(data)); } else if (type_str.find("backend_init_info") != std::string::npos) { @@ -623,14 +624,8 @@ struct console_printer : public printer { handle_message(*static_cast(data)); } else if (type_str.find("overall_summary_info") != std::string::npos) { handle_message(*static_cast(data)); - } else if (type_str.find("gradient_info") != std::string::npos) { - handle_message(*static_cast(data)); - } else if (type_str.find("maa_error_info") != std::string::npos) { - handle_message(*static_cast(data)); - } else if (type_str.find("compare_failure_info") != std::string::npos) { - handle_message(*static_cast(data)); - } else if (type_str.find("large_tensor_skip_info") != std::string::npos) { - handle_message(*static_cast(data)); + } else { + GGML_ABORT("unknown message type: %s", type_name); } } @@ -639,36 +634,55 @@ struct console_printer : public printer { printf(" %s(%s): ", info.op_name.c_str(), info.op_params.c_str()); fflush(stdout); - if (!info.supported) { + // Handle large tensor skip first + if (info.is_large_tensor_skip) { + printf("skipping large tensors for speed \n"); + return; + } + + // Handle not supported status + if (info.status == test_status_t::NOT_SUPPORTED) { if (!info.failure_reason.empty()) { printf("not supported [%s]\n", info.failure_reason.c_str()); } else { printf("not supported [%s]\n", info.backend_name.c_str()); } + return; } - } - void handle_message(const device_info& info) { - if (info.device_index == 0) { - // This is the first device, don't print the header info here + // Handle errors and additional information + if (info.has_error) { + if (info.error_component == "allocation") { + fprintf(stderr, "failed to allocate tensors [%s] ", info.backend_name.c_str()); + } else if (info.error_component == "backend") { + fprintf(stderr, " Failed to initialize %s backend\n", info.backend_name.c_str()); + } else { + fprintf(stderr, "Error in %s: %s\n", info.error_component.c_str(), info.error_details.c_str()); + } } - printf("Backend %zu/%zu: %s\n", info.device_index + 1, info.total_devices, info.device_name.c_str()); - - if (info.skipped) { - printf(" %s\n", info.skip_reason.c_str()); - return; + // Handle gradient info + if (info.has_gradient_info) { + printf("[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", + info.op_name.c_str(), info.gradient_index, info.gradient_param_name.c_str(), info.gradient_value); } - if (!info.description.empty()) { - printf(" Device description: %s\n", info.description.c_str()); + // Handle MAA error + if (info.has_maa_error) { + printf("[%s] MAA = %.9f > %.9f ", info.op_name.c_str(), info.maa_error, info.maa_threshold); } - if (info.memory_total_mb > 0) { - printf(" Device memory: %zu MB (%zu MB free)\n", info.memory_total_mb, info.memory_free_mb); + // Handle compare failure + if (info.is_compare_failure) { + printf("compare failed "); } - printf("\n"); + // Print final status + if (info.status == test_status_t::OK) { + printf("\033[1;32mOK\033[0m\n"); + } else { + printf("\033[1;31mFAIL\033[0m\n"); + } } void handle_message(const test_summary_info& info) { @@ -679,18 +693,9 @@ struct console_printer : public printer { } } - void handle_message(const error_info& info) { - if (info.component == "allocation") { - fprintf(stderr, "failed to allocate tensors [%s] ", info.backend_name.c_str()); - } else if (info.component == "backend") { - fprintf(stderr, " Failed to initialize %s backend\n", info.backend_name.c_str()); - } else { - fprintf(stderr, "Error in %s: %s\n", info.component.c_str(), info.details.c_str()); - } - } - - void handle_message(const test_status_info& info) { - if (info.passed) { + void handle_message(const backend_status_info& info) { + printf(" Backend %s: ", info.backend_name.c_str()); + if (info.status == test_status_t::OK) { printf("\033[1;32mOK\033[0m\n"); } else { printf("\033[1;31mFAIL\033[0m\n"); @@ -720,15 +725,6 @@ struct console_printer : public printer { printf("\n"); } - void handle_message(const backend_status_info& info) { - printf(" Backend %s: ", info.backend_name.c_str()); - if (info.passed) { - printf("\033[1;32mOK\033[0m\n"); - } else { - printf("\033[1;31mFAIL\033[0m\n"); - } - } - void handle_message(const overall_summary_info& info) { printf("%zu/%zu backends passed\n", info.backends_passed, info.backends_total); if (info.all_passed) { @@ -738,24 +734,6 @@ struct console_printer : public printer { } } - void handle_message(const gradient_info& info) { - printf("[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", info.op_desc.c_str(), info.index, info.param_name.c_str(), info.value); - } - - void handle_message(const maa_error_info& info) { - printf("[%s] MAA = %.9f > %.9f ", info.op_desc.c_str(), info.error, info.threshold); - } - - void handle_message(const compare_failure_info& info) { - (void)info; // unused - printf("compare failed "); - } - - void handle_message(const large_tensor_skip_info& info) { - (void)info; // unused - printf("skipping large tensors for speed \n"); - } - void print_test_console(const test_result & result) { printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str()); fflush(stdout); @@ -863,21 +841,10 @@ struct sql_printer : public printer { protected: // Implementation that checks type and dispatches to appropriate handler void print_message_impl(const void* data, const char* type_name) override { - // Use string comparison to identify types and cast accordingly - std::string type_str(type_name); - - if (type_str.find("error_info") != std::string::npos) { - handle_message(*static_cast(data)); - } - } - -private: - void handle_message(const error_info& info) { - // Only output critical errors that affect SQL generation - if (info.component == "backend" || info.component == "allocation") { - fprintf(stderr, "Critical error in %s: %s\n", info.component.c_str(), - info.backend_name.empty() ? info.details.c_str() : info.backend_name.c_str()); - } + // SQL printer doesn't need to handle message types for now + // All necessary output is handled through print_test_result + (void)data; + (void)type_name; } }; @@ -1328,7 +1295,7 @@ struct test_case { } if (out->type != GGML_TYPE_F32) { - output_printer->print_message(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), false, out->name + std::string("->type != FP32"))); + output_printer->print_message(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), test_status_t::NOT_SUPPORTED, out->name + std::string("->type != FP32"))); return true; } @@ -1361,9 +1328,7 @@ struct test_case { } if (!supported) { - output_printer->print_message(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), false, failure_reason)); - } - if (!supported) { + output_printer->print_message(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), test_status_t::NOT_SUPPORTED, failure_reason)); return true; } @@ -1374,7 +1339,9 @@ struct test_case { } } if (ngrads > grad_nmax()) { - output_printer->print_message(large_tensor_skip_info{}); + test_operation_info info(op_desc(out), vars(), ggml_backend_name(backend)); + info.set_large_tensor_skip(); + output_printer->print_message(info); return true; } @@ -1397,12 +1364,12 @@ struct test_case { for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { if (!ggml_backend_supports_op(backend, t)) { - output_printer->print_message(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), false, ggml_backend_name(backend))); + output_printer->print_message(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), test_status_t::NOT_SUPPORTED, ggml_backend_name(backend))); supported = false; break; } if ((t->flags & GGML_TENSOR_FLAG_PARAM) && t->type != GGML_TYPE_F32) { - output_printer->print_message(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), false, std::string(t->name) + "->type != FP32")); + output_printer->print_message(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), test_status_t::NOT_SUPPORTED, std::string(t->name) + "->type != FP32")); supported = false; break; } @@ -1414,7 +1381,9 @@ struct test_case { // allocate ggml_backend_buffer_ptr buf(ggml_backend_alloc_ctx_tensors(ctx.get(), backend)); // smart ptr if (buf == NULL) { - output_printer->print_message(error_info("allocation", "", ggml_backend_name(backend))); + test_operation_info info(op_desc(out), vars(), ggml_backend_name(backend)); + info.set_error("allocation", ""); + output_printer->print_message(info); return false; } @@ -1452,7 +1421,9 @@ struct test_case { for (int64_t i = 0; i < ne; ++i) { // gradient algebraic // check for nans if (!std::isfinite(ga[i])) { - output_printer->print_message(gradient_info(ggml_op_desc(t), i, bn, ga[i])); + test_operation_info info(op_desc(out), vars(), ggml_backend_name(backend)); + info.set_gradient_info(i, bn, ga[i]); + output_printer->print_message(info); ok = false; break; } @@ -1520,7 +1491,9 @@ struct test_case { const double err = mean_abs_asymm(gn.data(), ga.data(), gn.size(), expect); if (err > max_maa_err()) { - output_printer->print_message(maa_error_info(ggml_op_desc(t), err, max_maa_err())); + test_operation_info info(op_desc(out), vars(), ggml_backend_name(backend)); + info.set_maa_error(err, max_maa_err()); + output_printer->print_message(info); ok = false; break; } @@ -1529,11 +1502,13 @@ struct test_case { } } + // Create final test result + test_operation_info final_info(op_desc(out), vars(), ggml_backend_name(backend)); if (!ok) { - output_printer->print_message(compare_failure_info{}); + final_info.set_compare_failure(); } - - output_printer->print_message(test_status_info(ok)); + final_info.status = ok ? test_status_t::OK : test_status_t::FAIL; + output_printer->print_message(final_info); if (ok) { return true; @@ -5569,7 +5544,9 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op filter_test_cases(test_cases, params_filter); ggml_backend_t backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, NULL); if (backend_cpu == NULL) { - output_printer->print_message(error_info("backend", "Failed to initialize CPU backend", "CPU")); + test_operation_info info("", "", "CPU"); + info.set_error("backend", "Failed to initialize CPU backend"); + output_printer->print_message(info); return false; } @@ -5720,7 +5697,7 @@ int main(int argc, char ** argv) { if (ok) { n_ok++; } - output_printer->print_message(backend_status_info(ggml_backend_name(backend), ok)); + output_printer->print_message(backend_status_info(ggml_backend_name(backend), ok ? test_status_t::OK : test_status_t::FAIL)); ggml_backend_free(backend); } From bbca39f237b9b5387567c7f455a4965cb4b3db8e Mon Sep 17 00:00:00 2001 From: Xiaodong Ye Date: Thu, 3 Jul 2025 14:56:55 +0800 Subject: [PATCH 08/12] Address review comments Signed-off-by: Xiaodong Ye --- tests/test-backend-ops.cpp | 132 +++++++++++++++++++++++-------------- 1 file changed, 84 insertions(+), 48 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index f0270e8869919..8b694fa1bcbf1 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -449,8 +449,28 @@ enum class test_status_t { FAIL }; +// Forward declarations for the visitor pattern +struct message_visitor; + +// Base class for all message types that can be printed +struct message_data { + virtual ~message_data() {} + virtual void accept(message_visitor& visitor) const = 0; +}; + +// Message visitor interface +struct message_visitor { + virtual ~message_visitor() {} + virtual void visit(const struct test_operation_info& info) = 0; + virtual void visit(const struct test_summary_info& info) = 0; + virtual void visit(const struct testing_start_info& info) = 0; + virtual void visit(const struct backend_init_info& info) = 0; + virtual void visit(const struct backend_status_info& info) = 0; + virtual void visit(const struct overall_summary_info& info) = 0; +}; + // Printer classes for different output formats -struct test_operation_info { +struct test_operation_info : public message_data { std::string op_name; std::string op_params; std::string backend_name; @@ -482,6 +502,10 @@ struct test_operation_info { test_status_t status = test_status_t::OK, const std::string& failure_reason = "") : op_name(op_name), op_params(op_params), backend_name(backend_name), status(status), failure_reason(failure_reason) {} + void accept(message_visitor& visitor) const override { + visitor.visit(*this); + } + // Set error information void set_error(const std::string& component, const std::string& details) { has_error = true; @@ -527,7 +551,7 @@ struct test_operation_info { } }; -struct test_summary_info { +struct test_summary_info : public message_data { size_t tests_passed; size_t tests_total; bool is_backend_summary = false; // true for backend summary, false for test summary @@ -535,16 +559,24 @@ struct test_summary_info { test_summary_info() = default; test_summary_info(size_t tests_passed, size_t tests_total, bool is_backend_summary = false) : tests_passed(tests_passed), tests_total(tests_total), is_backend_summary(is_backend_summary) {} + + void accept(message_visitor& visitor) const override { + visitor.visit(*this); + } }; -struct testing_start_info { +struct testing_start_info : public message_data { size_t device_count; testing_start_info() = default; testing_start_info(size_t device_count) : device_count(device_count) {} + + void accept(message_visitor& visitor) const override { + visitor.visit(*this); + } }; -struct backend_init_info { +struct backend_init_info : public message_data { size_t device_index; size_t total_devices; std::string device_name; @@ -562,18 +594,26 @@ struct backend_init_info { : device_index(device_index), total_devices(total_devices), device_name(device_name), skipped(skipped), skip_reason(skip_reason), description(description), memory_total_mb(memory_total_mb), memory_free_mb(memory_free_mb), has_memory_info(has_memory_info) {} + + void accept(message_visitor& visitor) const override { + visitor.visit(*this); + } }; -struct backend_status_info { +struct backend_status_info : public message_data { std::string backend_name; test_status_t status; backend_status_info() = default; backend_status_info(const std::string& backend_name, test_status_t status) : backend_name(backend_name), status(status) {} + + void accept(message_visitor& visitor) const override { + visitor.visit(*this); + } }; -struct overall_summary_info { +struct overall_summary_info : public message_data { size_t backends_passed; size_t backends_total; bool all_passed; @@ -581,22 +621,22 @@ struct overall_summary_info { overall_summary_info() = default; overall_summary_info(size_t backends_passed, size_t backends_total, bool all_passed) : backends_passed(backends_passed), backends_total(backends_total), all_passed(all_passed) {} + + void accept(message_visitor& visitor) const override { + visitor.visit(*this); + } }; -struct printer { +struct printer : public message_visitor { virtual ~printer() {} FILE * fout = stdout; virtual void print_header() {} virtual void print_test_result(const test_result & result) = 0; virtual void print_footer() {} - template - void print_message(const T& data) { - print_message_impl(&data, typeid(T).name()); + void print_message(const message_data& data) { + data.accept(*this); } - -protected: - virtual void print_message_impl(const void* data, const char* type_name) = 0; }; struct console_printer : public printer { @@ -608,29 +648,8 @@ struct console_printer : public printer { } } -protected: - void print_message_impl(const void* data, const char* type_name) override { - std::string type_str(type_name); - - if (type_str.find("test_operation_info") != std::string::npos) { - handle_message(*static_cast(data)); - } else if (type_str.find("test_summary_info") != std::string::npos) { - handle_message(*static_cast(data)); - } else if (type_str.find("testing_start_info") != std::string::npos) { - handle_message(*static_cast(data)); - } else if (type_str.find("backend_init_info") != std::string::npos) { - handle_message(*static_cast(data)); - } else if (type_str.find("backend_status_info") != std::string::npos) { - handle_message(*static_cast(data)); - } else if (type_str.find("overall_summary_info") != std::string::npos) { - handle_message(*static_cast(data)); - } else { - GGML_ABORT("unknown message type: %s", type_name); - } - } - -private: - void handle_message(const test_operation_info& info) { + // Visitor pattern implementations + void visit(const test_operation_info& info) override { printf(" %s(%s): ", info.op_name.c_str(), info.op_params.c_str()); fflush(stdout); @@ -685,7 +704,7 @@ struct console_printer : public printer { } } - void handle_message(const test_summary_info& info) { + void visit(const test_summary_info& info) override { if (info.is_backend_summary) { printf("%zu/%zu backends passed\n", info.tests_passed, info.tests_total); } else { @@ -693,7 +712,7 @@ struct console_printer : public printer { } } - void handle_message(const backend_status_info& info) { + void visit(const backend_status_info& info) override { printf(" Backend %s: ", info.backend_name.c_str()); if (info.status == test_status_t::OK) { printf("\033[1;32mOK\033[0m\n"); @@ -702,11 +721,11 @@ struct console_printer : public printer { } } - void handle_message(const testing_start_info& info) { + void visit(const testing_start_info& info) override { printf("Testing %zu devices\n\n", info.device_count); } - void handle_message(const backend_init_info& info) { + void visit(const backend_init_info& info) override { printf("Backend %zu/%zu: %s\n", info.device_index + 1, info.total_devices, info.device_name.c_str()); if (info.skipped) { @@ -725,7 +744,7 @@ struct console_printer : public printer { printf("\n"); } - void handle_message(const overall_summary_info& info) { + void visit(const overall_summary_info& info) override { printf("%zu/%zu backends passed\n", info.backends_passed, info.backends_total); if (info.all_passed) { printf("\033[1;32mOK\033[0m\n"); @@ -734,6 +753,7 @@ struct console_printer : public printer { } } +private: void print_test_console(const test_result & result) { printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str()); fflush(stdout); @@ -838,13 +858,29 @@ struct sql_printer : public printer { fprintf(fout, ");\n"); } -protected: - // Implementation that checks type and dispatches to appropriate handler - void print_message_impl(const void* data, const char* type_name) override { - // SQL printer doesn't need to handle message types for now - // All necessary output is handled through print_test_result - (void)data; - (void)type_name; + // Visitor pattern implementations - SQL printer doesn't need to handle message types for now + void visit(const test_operation_info& info) override { + (void)info; + } + + void visit(const test_summary_info& info) override { + (void)info; + } + + void visit(const testing_start_info& info) override { + (void)info; + } + + void visit(const backend_init_info& info) override { + (void)info; + } + + void visit(const backend_status_info& info) override { + (void)info; + } + + void visit(const overall_summary_info& info) override { + (void)info; } }; From a3252a0d90e6a0662cbb5804ba951744f5e5f254 Mon Sep 17 00:00:00 2001 From: Xiaodong Ye Date: Thu, 3 Jul 2025 15:47:33 +0800 Subject: [PATCH 09/12] Address review comments Signed-off-by: Xiaodong Ye --- tests/test-backend-ops.cpp | 64 +++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 8b694fa1bcbf1..fb8231c117b6f 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -442,13 +442,6 @@ struct test_result { } }; -// Enum for test status -enum class test_status_t { - NOT_SUPPORTED, - OK, - FAIL -}; - // Forward declarations for the visitor pattern struct message_visitor; @@ -470,6 +463,12 @@ struct message_visitor { }; // Printer classes for different output formats +enum class test_status_t { + NOT_SUPPORTED, + OK, + FAIL +}; + struct test_operation_info : public message_data { std::string op_name; std::string op_params; @@ -637,6 +636,32 @@ struct printer : public message_visitor { void print_message(const message_data& data) { data.accept(*this); } + + // Default implementations for all visit methods (no-op) + // Derived classes can override only the ones they care about + void visit(const test_operation_info& info) override { + (void)info; + } + + void visit(const test_summary_info& info) override { + (void)info; + } + + void visit(const testing_start_info& info) override { + (void)info; + } + + void visit(const backend_init_info& info) override { + (void)info; + } + + void visit(const backend_status_info& info) override { + (void)info; + } + + void visit(const overall_summary_info& info) override { + (void)info; + } }; struct console_printer : public printer { @@ -857,31 +882,6 @@ struct sql_printer : public printer { } fprintf(fout, ");\n"); } - - // Visitor pattern implementations - SQL printer doesn't need to handle message types for now - void visit(const test_operation_info& info) override { - (void)info; - } - - void visit(const test_summary_info& info) override { - (void)info; - } - - void visit(const testing_start_info& info) override { - (void)info; - } - - void visit(const backend_init_info& info) override { - (void)info; - } - - void visit(const backend_status_info& info) override { - (void)info; - } - - void visit(const overall_summary_info& info) override { - (void)info; - } }; static std::unique_ptr create_printer(output_formats format) { From e8bc633ef9bf31c1d0dd1739d81c7f6f6673e846 Mon Sep 17 00:00:00 2001 From: slaren Date: Thu, 3 Jul 2025 12:49:51 +0200 Subject: [PATCH 10/12] remove visitor nonsense --- tests/test-backend-ops.cpp | 144 ++++++++++--------------------------- 1 file changed, 38 insertions(+), 106 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index fb8231c117b6f..70ab6f95ea638 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -442,26 +442,6 @@ struct test_result { } }; -// Forward declarations for the visitor pattern -struct message_visitor; - -// Base class for all message types that can be printed -struct message_data { - virtual ~message_data() {} - virtual void accept(message_visitor& visitor) const = 0; -}; - -// Message visitor interface -struct message_visitor { - virtual ~message_visitor() {} - virtual void visit(const struct test_operation_info& info) = 0; - virtual void visit(const struct test_summary_info& info) = 0; - virtual void visit(const struct testing_start_info& info) = 0; - virtual void visit(const struct backend_init_info& info) = 0; - virtual void visit(const struct backend_status_info& info) = 0; - virtual void visit(const struct overall_summary_info& info) = 0; -}; - // Printer classes for different output formats enum class test_status_t { NOT_SUPPORTED, @@ -469,7 +449,7 @@ enum class test_status_t { FAIL }; -struct test_operation_info : public message_data { +struct test_operation_info { std::string op_name; std::string op_params; std::string backend_name; @@ -501,10 +481,6 @@ struct test_operation_info : public message_data { test_status_t status = test_status_t::OK, const std::string& failure_reason = "") : op_name(op_name), op_params(op_params), backend_name(backend_name), status(status), failure_reason(failure_reason) {} - void accept(message_visitor& visitor) const override { - visitor.visit(*this); - } - // Set error information void set_error(const std::string& component, const std::string& details) { has_error = true; @@ -550,7 +526,7 @@ struct test_operation_info : public message_data { } }; -struct test_summary_info : public message_data { +struct test_summary_info { size_t tests_passed; size_t tests_total; bool is_backend_summary = false; // true for backend summary, false for test summary @@ -558,24 +534,16 @@ struct test_summary_info : public message_data { test_summary_info() = default; test_summary_info(size_t tests_passed, size_t tests_total, bool is_backend_summary = false) : tests_passed(tests_passed), tests_total(tests_total), is_backend_summary(is_backend_summary) {} - - void accept(message_visitor& visitor) const override { - visitor.visit(*this); - } }; -struct testing_start_info : public message_data { +struct testing_start_info { size_t device_count; testing_start_info() = default; testing_start_info(size_t device_count) : device_count(device_count) {} - - void accept(message_visitor& visitor) const override { - visitor.visit(*this); - } }; -struct backend_init_info : public message_data { +struct backend_init_info { size_t device_index; size_t total_devices; std::string device_name; @@ -593,26 +561,18 @@ struct backend_init_info : public message_data { : device_index(device_index), total_devices(total_devices), device_name(device_name), skipped(skipped), skip_reason(skip_reason), description(description), memory_total_mb(memory_total_mb), memory_free_mb(memory_free_mb), has_memory_info(has_memory_info) {} - - void accept(message_visitor& visitor) const override { - visitor.visit(*this); - } }; -struct backend_status_info : public message_data { +struct backend_status_info { std::string backend_name; test_status_t status; backend_status_info() = default; backend_status_info(const std::string& backend_name, test_status_t status) : backend_name(backend_name), status(status) {} - - void accept(message_visitor& visitor) const override { - visitor.visit(*this); - } }; -struct overall_summary_info : public message_data { +struct overall_summary_info { size_t backends_passed; size_t backends_total; bool all_passed; @@ -620,48 +580,20 @@ struct overall_summary_info : public message_data { overall_summary_info() = default; overall_summary_info(size_t backends_passed, size_t backends_total, bool all_passed) : backends_passed(backends_passed), backends_total(backends_total), all_passed(all_passed) {} - - void accept(message_visitor& visitor) const override { - visitor.visit(*this); - } }; -struct printer : public message_visitor { +struct printer { virtual ~printer() {} FILE * fout = stdout; virtual void print_header() {} virtual void print_test_result(const test_result & result) = 0; virtual void print_footer() {} - - void print_message(const message_data& data) { - data.accept(*this); - } - - // Default implementations for all visit methods (no-op) - // Derived classes can override only the ones they care about - void visit(const test_operation_info& info) override { - (void)info; - } - - void visit(const test_summary_info& info) override { - (void)info; - } - - void visit(const testing_start_info& info) override { - (void)info; - } - - void visit(const backend_init_info& info) override { - (void)info; - } - - void visit(const backend_status_info& info) override { - (void)info; - } - - void visit(const overall_summary_info& info) override { - (void)info; - } + virtual void print_operation(const test_operation_info & info) { (void) info; } + virtual void print_summary(const test_summary_info & info) { (void) info; } + virtual void print_testing_start(const testing_start_info & info) { (void) info; } + virtual void print_backend_init(const backend_init_info & info) { (void) info; } + virtual void print_backend_status(const backend_status_info & info) { (void) info; } + virtual void print_overall_summary(const overall_summary_info & info) { (void) info; } }; struct console_printer : public printer { @@ -674,7 +606,7 @@ struct console_printer : public printer { } // Visitor pattern implementations - void visit(const test_operation_info& info) override { + void print_operation(const test_operation_info& info) override { printf(" %s(%s): ", info.op_name.c_str(), info.op_params.c_str()); fflush(stdout); @@ -729,7 +661,7 @@ struct console_printer : public printer { } } - void visit(const test_summary_info& info) override { + void print_summary(const test_summary_info& info) override { if (info.is_backend_summary) { printf("%zu/%zu backends passed\n", info.tests_passed, info.tests_total); } else { @@ -737,7 +669,7 @@ struct console_printer : public printer { } } - void visit(const backend_status_info& info) override { + void print_backend_status(const backend_status_info& info) override { printf(" Backend %s: ", info.backend_name.c_str()); if (info.status == test_status_t::OK) { printf("\033[1;32mOK\033[0m\n"); @@ -746,11 +678,11 @@ struct console_printer : public printer { } } - void visit(const testing_start_info& info) override { + void print_testing_start(const testing_start_info& info) override { printf("Testing %zu devices\n\n", info.device_count); } - void visit(const backend_init_info& info) override { + void print_backend_init(const backend_init_info& info) override { printf("Backend %zu/%zu: %s\n", info.device_index + 1, info.total_devices, info.device_name.c_str()); if (info.skipped) { @@ -769,7 +701,7 @@ struct console_printer : public printer { printf("\n"); } - void visit(const overall_summary_info& info) override { + void print_overall_summary(const overall_summary_info& info) override { printf("%zu/%zu backends passed\n", info.backends_passed, info.backends_total); if (info.all_passed) { printf("\033[1;32mOK\033[0m\n"); @@ -1331,12 +1263,12 @@ struct test_case { } if (out->type != GGML_TYPE_F32) { - output_printer->print_message(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), test_status_t::NOT_SUPPORTED, out->name + std::string("->type != FP32"))); + output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), test_status_t::NOT_SUPPORTED, out->name + std::string("->type != FP32"))); return true; } // Print operation info first - output_printer->print_message(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend))); + output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend))); // check if the backend supports the ops bool supported = true; @@ -1364,7 +1296,7 @@ struct test_case { } if (!supported) { - output_printer->print_message(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), test_status_t::NOT_SUPPORTED, failure_reason)); + output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), test_status_t::NOT_SUPPORTED, failure_reason)); return true; } @@ -1377,7 +1309,7 @@ struct test_case { if (ngrads > grad_nmax()) { test_operation_info info(op_desc(out), vars(), ggml_backend_name(backend)); info.set_large_tensor_skip(); - output_printer->print_message(info); + output_printer->print_operation(info); return true; } @@ -1400,12 +1332,12 @@ struct test_case { for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { if (!ggml_backend_supports_op(backend, t)) { - output_printer->print_message(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), test_status_t::NOT_SUPPORTED, ggml_backend_name(backend))); + output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), test_status_t::NOT_SUPPORTED, ggml_backend_name(backend))); supported = false; break; } if ((t->flags & GGML_TENSOR_FLAG_PARAM) && t->type != GGML_TYPE_F32) { - output_printer->print_message(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), test_status_t::NOT_SUPPORTED, std::string(t->name) + "->type != FP32")); + output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), test_status_t::NOT_SUPPORTED, std::string(t->name) + "->type != FP32")); supported = false; break; } @@ -1419,7 +1351,7 @@ struct test_case { if (buf == NULL) { test_operation_info info(op_desc(out), vars(), ggml_backend_name(backend)); info.set_error("allocation", ""); - output_printer->print_message(info); + output_printer->print_operation(info); return false; } @@ -1459,7 +1391,7 @@ struct test_case { if (!std::isfinite(ga[i])) { test_operation_info info(op_desc(out), vars(), ggml_backend_name(backend)); info.set_gradient_info(i, bn, ga[i]); - output_printer->print_message(info); + output_printer->print_operation(info); ok = false; break; } @@ -1529,7 +1461,7 @@ struct test_case { if (err > max_maa_err()) { test_operation_info info(op_desc(out), vars(), ggml_backend_name(backend)); info.set_maa_error(err, max_maa_err()); - output_printer->print_message(info); + output_printer->print_operation(info); ok = false; break; } @@ -1544,7 +1476,7 @@ struct test_case { final_info.set_compare_failure(); } final_info.status = ok ? test_status_t::OK : test_status_t::FAIL; - output_printer->print_message(final_info); + output_printer->print_operation(final_info); if (ok) { return true; @@ -5582,7 +5514,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op if (backend_cpu == NULL) { test_operation_info info("", "", "CPU"); info.set_error("backend", "Failed to initialize CPU backend"); - output_printer->print_message(info); + output_printer->print_operation(info); return false; } @@ -5592,7 +5524,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op n_ok++; } } - output_printer->print_message(test_summary_info(n_ok, test_cases.size(), false)); + output_printer->print_summary(test_summary_info(n_ok, test_cases.size(), false)); ggml_backend_free(backend_cpu); @@ -5608,7 +5540,7 @@ static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op n_ok++; } } - output_printer->print_message(test_summary_info(n_ok, test_cases.size(), false)); + output_printer->print_summary(test_summary_info(n_ok, test_cases.size(), false)); return n_ok == test_cases.size(); } @@ -5695,7 +5627,7 @@ int main(int argc, char ** argv) { output_printer->print_header(); } - output_printer->print_message(testing_start_info(ggml_backend_dev_count())); + output_printer->print_testing_start(testing_start_info(ggml_backend_dev_count())); size_t n_ok = 0; @@ -5703,13 +5635,13 @@ int main(int argc, char ** argv) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); if (backend_filter != NULL && strcmp(backend_filter, ggml_backend_dev_name(dev)) != 0) { - output_printer->print_message(backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), true, "Skipping")); + output_printer->print_backend_init(backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), true, "Skipping")); n_ok++; continue; } if (backend_filter == NULL && ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU && mode != MODE_GRAD) { - output_printer->print_message(backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), true, "Skipping CPU backend")); + output_printer->print_backend_init(backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), true, "Skipping CPU backend")); n_ok++; continue; } @@ -5726,14 +5658,14 @@ int main(int argc, char ** argv) { size_t free, total; // NOLINT ggml_backend_dev_memory(dev, &free, &total); - output_printer->print_message(backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), false, "", ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024, true)); + output_printer->print_backend_init(backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), false, "", ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024, true)); bool ok = test_backend(backend, mode, op_name_filter, params_filter, output_printer.get()); if (ok) { n_ok++; } - output_printer->print_message(backend_status_info(ggml_backend_name(backend), ok ? test_status_t::OK : test_status_t::FAIL)); + output_printer->print_backend_status(backend_status_info(ggml_backend_name(backend), ok ? test_status_t::OK : test_status_t::FAIL)); ggml_backend_free(backend); } @@ -5744,7 +5676,7 @@ int main(int argc, char ** argv) { output_printer->print_footer(); } - output_printer->print_message(overall_summary_info(n_ok, ggml_backend_dev_count(), n_ok == ggml_backend_dev_count())); + output_printer->print_overall_summary(overall_summary_info(n_ok, ggml_backend_dev_count(), n_ok == ggml_backend_dev_count())); if (n_ok != ggml_backend_dev_count()) { return 1; From 6db8a27cbd6cfbc54db02215af1e1607337c3197 Mon Sep 17 00:00:00 2001 From: Xiaodong Ye Date: Thu, 3 Jul 2025 20:54:36 +0800 Subject: [PATCH 11/12] remove visitor comment Signed-off-by: Xiaodong Ye --- tests/test-backend-ops.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 70ab6f95ea638..c38897acc8771 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -605,7 +605,6 @@ struct console_printer : public printer { } } - // Visitor pattern implementations void print_operation(const test_operation_info& info) override { printf(" %s(%s): ", info.op_name.c_str(), info.op_params.c_str()); fflush(stdout); From 7359eed2ca21c933c56fc31e5b53534fc10efc61 Mon Sep 17 00:00:00 2001 From: Xiaodong Ye Date: Fri, 4 Jul 2025 10:14:37 +0800 Subject: [PATCH 12/12] Address review comments Signed-off-by: Xiaodong Ye --- tests/test-backend-ops.cpp | 326 +++++++++++++++++++++---------------- 1 file changed, 183 insertions(+), 143 deletions(-) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index c38897acc8771..12e0b9aa14df9 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -352,28 +352,28 @@ struct test_result { std::string op_name; std::string op_params; std::string test_mode; - bool supported; - bool passed; + bool supported; + bool passed; std::string error_message; - double time_us; - double flops; - double bandwidth_gb_s; - size_t memory_kb; - int n_runs; + double time_us; + double flops; + double bandwidth_gb_s; + size_t memory_kb; + int n_runs; test_result() { // Initialize with default values - time_us = 0.0; - flops = 0.0; + time_us = 0.0; + flops = 0.0; bandwidth_gb_s = 0.0; - memory_kb = 0; - n_runs = 0; - supported = false; - passed = false; + memory_kb = 0; + n_runs = 0; + supported = false; + passed = false; // Set test time time_t t = time(NULL); - char buf[32]; + char buf[32]; std::strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&t)); test_time = buf; @@ -381,16 +381,25 @@ struct test_result { build_commit = ggml_commit(); } - test_result(const std::string& backend_name, const std::string& op_name, const std::string& op_params, - const std::string& test_mode, bool supported, bool passed, const std::string& error_message = "", - double time_us = 0.0, double flops = 0.0, double bandwidth_gb_s = 0.0, - size_t memory_kb = 0, int n_runs = 0) - : backend_name(backend_name), op_name(op_name), op_params(op_params), test_mode(test_mode), - supported(supported), passed(passed), error_message(error_message), time_us(time_us), - flops(flops), bandwidth_gb_s(bandwidth_gb_s), memory_kb(memory_kb), n_runs(n_runs) { + test_result(const std::string & backend_name, const std::string & op_name, const std::string & op_params, + const std::string & test_mode, bool supported, bool passed, const std::string & error_message = "", + double time_us = 0.0, double flops = 0.0, double bandwidth_gb_s = 0.0, size_t memory_kb = 0, + int n_runs = 0) : + backend_name(backend_name), + op_name(op_name), + op_params(op_params), + test_mode(test_mode), + supported(supported), + passed(passed), + error_message(error_message), + time_us(time_us), + flops(flops), + bandwidth_gb_s(bandwidth_gb_s), + memory_kb(memory_kb), + n_runs(n_runs) { // Set test time time_t t = time(NULL); - char buf[32]; + char buf[32]; std::strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&t)); test_time = buf; @@ -400,9 +409,8 @@ struct test_result { static const std::vector & get_fields() { static const std::vector fields = { - "test_time", "build_commit", "backend_name", "op_name", "op_params", - "test_mode", "supported", "passed", "error_message", "time_us", - "flops", "bandwidth_gb_s", "memory_kb", "n_runs" + "test_time", "build_commit", "backend_name", "op_name", "op_params", "test_mode", "supported", + "passed", "error_message", "time_us", "flops", "bandwidth_gb_s", "memory_kb", "n_runs" }; return fields; } @@ -423,80 +431,79 @@ struct test_result { } std::vector get_values() const { - return { - test_time, - build_commit, - backend_name, - op_name, - op_params, - test_mode, - std::to_string(supported), - std::to_string(passed), - error_message, - std::to_string(time_us), - std::to_string(flops), - std::to_string(bandwidth_gb_s), - std::to_string(memory_kb), - std::to_string(n_runs) - }; + return { test_time, + build_commit, + backend_name, + op_name, + op_params, + test_mode, + std::to_string(supported), + std::to_string(passed), + error_message, + std::to_string(time_us), + std::to_string(flops), + std::to_string(bandwidth_gb_s), + std::to_string(memory_kb), + std::to_string(n_runs) }; } }; // Printer classes for different output formats -enum class test_status_t { - NOT_SUPPORTED, - OK, - FAIL -}; +enum class test_status_t { NOT_SUPPORTED, OK, FAIL }; struct test_operation_info { - std::string op_name; - std::string op_params; - std::string backend_name; + std::string op_name; + std::string op_params; + std::string backend_name; test_status_t status = test_status_t::OK; - std::string failure_reason; + std::string failure_reason; // Additional information fields that were previously in separate structs std::string error_component; std::string error_details; // Gradient info - int64_t gradient_index = -1; + int64_t gradient_index = -1; std::string gradient_param_name; - float gradient_value = 0.0f; + float gradient_value = 0.0f; // MAA error info - double maa_error = 0.0; + double maa_error = 0.0; double maa_threshold = 0.0; // Flags for different types of information - bool has_error = false; - bool has_gradient_info = false; - bool has_maa_error = false; - bool is_compare_failure = false; + bool has_error = false; + bool has_gradient_info = false; + bool has_maa_error = false; + bool is_compare_failure = false; bool is_large_tensor_skip = false; test_operation_info() = default; - test_operation_info(const std::string& op_name, const std::string& op_params, const std::string& backend_name, - test_status_t status = test_status_t::OK, const std::string& failure_reason = "") - : op_name(op_name), op_params(op_params), backend_name(backend_name), status(status), failure_reason(failure_reason) {} + + test_operation_info(const std::string & op_name, const std::string & op_params, const std::string & backend_name, + test_status_t status = test_status_t::OK, const std::string & failure_reason = "") : + op_name(op_name), + op_params(op_params), + backend_name(backend_name), + status(status), + failure_reason(failure_reason) {} // Set error information - void set_error(const std::string& component, const std::string& details) { - has_error = true; + void set_error(const std::string & component, const std::string & details) { + has_error = true; error_component = component; - error_details = details; + error_details = details; if (status == test_status_t::OK) { status = test_status_t::FAIL; } } // Set gradient information - void set_gradient_info(int64_t index, const std::string& param_name, float value) { - has_gradient_info = true; - gradient_index = index; + void set_gradient_info(int64_t index, const std::string & param_name, float value) { + has_gradient_info = true; + gradient_index = index; gradient_param_name = param_name; - gradient_value = value; + gradient_value = value; if (status == test_status_t::OK) { status = test_status_t::FAIL; } @@ -505,7 +512,7 @@ struct test_operation_info { // Set MAA error information void set_maa_error(double error, double threshold) { has_maa_error = true; - maa_error = error; + maa_error = error; maa_threshold = threshold; if (status == test_status_t::OK) { status = test_status_t::FAIL; @@ -521,78 +528,102 @@ struct test_operation_info { } // Set large tensor skip - void set_large_tensor_skip() { - is_large_tensor_skip = true; - } + void set_large_tensor_skip() { is_large_tensor_skip = true; } }; struct test_summary_info { size_t tests_passed; size_t tests_total; - bool is_backend_summary = false; // true for backend summary, false for test summary + bool is_backend_summary = false; // true for backend summary, false for test summary test_summary_info() = default; - test_summary_info(size_t tests_passed, size_t tests_total, bool is_backend_summary = false) - : tests_passed(tests_passed), tests_total(tests_total), is_backend_summary(is_backend_summary) {} + + test_summary_info(size_t tests_passed, size_t tests_total, bool is_backend_summary = false) : + tests_passed(tests_passed), + tests_total(tests_total), + is_backend_summary(is_backend_summary) {} }; struct testing_start_info { size_t device_count; testing_start_info() = default; + testing_start_info(size_t device_count) : device_count(device_count) {} }; struct backend_init_info { - size_t device_index; - size_t total_devices; + size_t device_index; + size_t total_devices; std::string device_name; - bool skipped = false; + bool skipped = false; std::string skip_reason; std::string description; - size_t memory_total_mb = 0; - size_t memory_free_mb = 0; - bool has_memory_info = false; + size_t memory_total_mb = 0; + size_t memory_free_mb = 0; + bool has_memory_info = false; backend_init_info() = default; - backend_init_info(size_t device_index, size_t total_devices, const std::string& device_name, bool skipped = false, - const std::string& skip_reason = "", const std::string& description = "", - size_t memory_total_mb = 0, size_t memory_free_mb = 0, bool has_memory_info = false) - : device_index(device_index), total_devices(total_devices), device_name(device_name), skipped(skipped), - skip_reason(skip_reason), description(description), memory_total_mb(memory_total_mb), - memory_free_mb(memory_free_mb), has_memory_info(has_memory_info) {} + + backend_init_info(size_t device_index, size_t total_devices, const std::string & device_name, bool skipped = false, + const std::string & skip_reason = "", const std::string & description = "", + size_t memory_total_mb = 0, size_t memory_free_mb = 0, bool has_memory_info = false) : + device_index(device_index), + total_devices(total_devices), + device_name(device_name), + skipped(skipped), + skip_reason(skip_reason), + description(description), + memory_total_mb(memory_total_mb), + memory_free_mb(memory_free_mb), + has_memory_info(has_memory_info) {} }; struct backend_status_info { - std::string backend_name; + std::string backend_name; test_status_t status; backend_status_info() = default; - backend_status_info(const std::string& backend_name, test_status_t status) - : backend_name(backend_name), status(status) {} + + backend_status_info(const std::string & backend_name, test_status_t status) : + backend_name(backend_name), + status(status) {} }; struct overall_summary_info { size_t backends_passed; size_t backends_total; - bool all_passed; + bool all_passed; overall_summary_info() = default; - overall_summary_info(size_t backends_passed, size_t backends_total, bool all_passed) - : backends_passed(backends_passed), backends_total(backends_total), all_passed(all_passed) {} + + overall_summary_info(size_t backends_passed, size_t backends_total, bool all_passed) : + backends_passed(backends_passed), + backends_total(backends_total), + all_passed(all_passed) {} }; struct printer { virtual ~printer() {} + FILE * fout = stdout; + virtual void print_header() {} + virtual void print_test_result(const test_result & result) = 0; + virtual void print_footer() {} + virtual void print_operation(const test_operation_info & info) { (void) info; } + virtual void print_summary(const test_summary_info & info) { (void) info; } + virtual void print_testing_start(const testing_start_info & info) { (void) info; } + virtual void print_backend_init(const backend_init_info & info) { (void) info; } + virtual void print_backend_status(const backend_status_info & info) { (void) info; } + virtual void print_overall_summary(const overall_summary_info & info) { (void) info; } }; @@ -605,7 +636,7 @@ struct console_printer : public printer { } } - void print_operation(const test_operation_info& info) override { + void print_operation(const test_operation_info & info) override { printf(" %s(%s): ", info.op_name.c_str(), info.op_params.c_str()); fflush(stdout); @@ -638,8 +669,8 @@ struct console_printer : public printer { // Handle gradient info if (info.has_gradient_info) { - printf("[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", - info.op_name.c_str(), info.gradient_index, info.gradient_param_name.c_str(), info.gradient_value); + printf("[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", info.op_name.c_str(), info.gradient_index, + info.gradient_param_name.c_str(), info.gradient_value); } // Handle MAA error @@ -660,7 +691,7 @@ struct console_printer : public printer { } } - void print_summary(const test_summary_info& info) override { + void print_summary(const test_summary_info & info) override { if (info.is_backend_summary) { printf("%zu/%zu backends passed\n", info.tests_passed, info.tests_total); } else { @@ -668,7 +699,7 @@ struct console_printer : public printer { } } - void print_backend_status(const backend_status_info& info) override { + void print_backend_status(const backend_status_info & info) override { printf(" Backend %s: ", info.backend_name.c_str()); if (info.status == test_status_t::OK) { printf("\033[1;32mOK\033[0m\n"); @@ -677,11 +708,11 @@ struct console_printer : public printer { } } - void print_testing_start(const testing_start_info& info) override { + void print_testing_start(const testing_start_info & info) override { printf("Testing %zu devices\n\n", info.device_count); } - void print_backend_init(const backend_init_info& info) override { + void print_backend_init(const backend_init_info & info) override { printf("Backend %zu/%zu: %s\n", info.device_index + 1, info.total_devices, info.device_name.c_str()); if (info.skipped) { @@ -700,7 +731,7 @@ struct console_printer : public printer { printf("\n"); } - void print_overall_summary(const overall_summary_info& info) override { + void print_overall_summary(const overall_summary_info & info) override { printf("%zu/%zu backends passed\n", info.backends_passed, info.backends_total); if (info.all_passed) { printf("\033[1;32mOK\033[0m\n"); @@ -709,7 +740,7 @@ struct console_printer : public printer { } } -private: + private: void print_test_console(const test_result & result) { printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str()); fflush(stdout); @@ -738,15 +769,13 @@ struct console_printer : public printer { // align while also leaving some margin for variations in parameters int align = 8; - int last = (len + align - 1) / align * align; + int last = (len + align - 1) / align * align; if (last - len < 5) { last += align; } printf("%*s", last - len, ""); - printf(" %8d runs - %8.2f us/run - ", - result.n_runs, - result.time_us); + printf(" %8d runs - %8.2f us/run - ", result.n_runs, result.time_us); if (result.flops > 0) { auto format_flops = [](double flops) -> std::string { @@ -763,13 +792,10 @@ struct console_printer : public printer { return buf; }; uint64_t op_flops_per_run = result.flops * result.time_us / 1e6; - printf("%s/run - \033[1;34m%sS\033[0m", - format_flops(op_flops_per_run).c_str(), - format_flops(result.flops).c_str()); + printf("%s/run - \033[1;34m%sS\033[0m", format_flops(op_flops_per_run).c_str(), + format_flops(result.flops).c_str()); } else { - printf("%8zu kB/run - \033[1;34m%7.2f GB/s\033[0m", - result.memory_kb, - result.bandwidth_gb_s); + printf("%8zu kB/run - \033[1;34m%7.2f GB/s\033[0m", result.memory_kb, result.bandwidth_gb_s); } printf("\n"); } @@ -942,7 +968,7 @@ struct test_case { return t; } - bool eval(ggml_backend_t backend1, ggml_backend_t backend2, const char * op_name, printer * output_printer = nullptr) { + bool eval(ggml_backend_t backend1, ggml_backend_t backend2, const char * op_name, printer * output_printer) { mode = MODE_TEST; ggml_init_params params = { @@ -1095,10 +1121,10 @@ struct test_case { ggml_free(ctx); // Create test result - bool test_passed = ud.ok && cmp_ok; - std::string error_msg = test_passed ? "" : (!cmp_ok ? "compare failed" : "test failed"); - test_result result(ggml_backend_name(backend1), current_op_name, vars(), "test", - supported, test_passed, error_msg); + bool test_passed = ud.ok && cmp_ok; + std::string error_msg = test_passed ? "" : (!cmp_ok ? "compare failed" : "test failed"); + test_result result(ggml_backend_name(backend1), current_op_name, vars(), "test", supported, test_passed, + error_msg); if (output_printer) { output_printer->print_test_result(result); @@ -1107,7 +1133,7 @@ struct test_case { return test_passed; } - bool eval_perf(ggml_backend_t backend, const char * op_name, printer * output_printer = nullptr) { + bool eval_perf(ggml_backend_t backend, const char * op_name, printer * output_printer) { mode = MODE_PERF; static const size_t graph_nodes = 8192; @@ -1120,8 +1146,8 @@ struct test_case { ggml_context_ptr ctx(ggml_init(params)); // smart ptr GGML_ASSERT(ctx); - ggml_tensor * out = build_graph(ctx.get()); - std::string current_op_name = op_desc(out); + ggml_tensor * out = build_graph(ctx.get()); + std::string current_op_name = op_desc(out); if (op_name != nullptr && current_op_name != op_name) { //printf(" %s: skipping\n", op_desc(out).c_str()); return true; @@ -1130,8 +1156,8 @@ struct test_case { // check if backends support op if (!ggml_backend_supports_op(backend, out)) { // Create test result for unsupported performance test - test_result result(ggml_backend_name(backend), current_op_name, vars(), "perf", - false, false, "not supported"); + test_result result(ggml_backend_name(backend), current_op_name, vars(), "perf", false, false, + "not supported"); if (output_printer) { output_printer->print_test_result(result); @@ -1224,14 +1250,14 @@ struct test_case { } while (total_time_us < 1000*1000); // run for at least 1 second // Create test result - double avg_time_us = (double)total_time_us / total_runs; + double avg_time_us = (double) total_time_us / total_runs; double calculated_flops = (op_flops(out) > 0) ? (op_flops(out) * total_runs) / (total_time_us / 1e6) : 0.0; - double calculated_bandwidth = (op_flops(out) == 0) ? total_mem / (total_time_us / 1e6) / 1024.0 / 1024.0 / 1024.0 : 0.0; + double calculated_bandwidth = + (op_flops(out) == 0) ? total_mem / (total_time_us / 1e6) / 1024.0 / 1024.0 / 1024.0 : 0.0; size_t calculated_memory_kb = op_size(out) / 1024; - test_result result(ggml_backend_name(backend), current_op_name, vars(), "perf", - true, true, "", avg_time_us, calculated_flops, calculated_bandwidth, - calculated_memory_kb, total_runs); + test_result result(ggml_backend_name(backend), current_op_name, vars(), "perf", true, true, "", avg_time_us, + calculated_flops, calculated_bandwidth, calculated_memory_kb, total_runs); if (output_printer) { output_printer->print_test_result(result); @@ -1240,7 +1266,7 @@ struct test_case { return true; } - bool eval_grad(ggml_backend_t backend, const char * op_name, printer * output_printer = nullptr) { + bool eval_grad(ggml_backend_t backend, const char * op_name, printer * output_printer) { mode = MODE_GRAD; const std::vector expect = grad_expect(); @@ -1262,7 +1288,9 @@ struct test_case { } if (out->type != GGML_TYPE_F32) { - output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), test_status_t::NOT_SUPPORTED, out->name + std::string("->type != FP32"))); + output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), + test_status_t::NOT_SUPPORTED, + out->name + std::string("->type != FP32"))); return true; } @@ -1270,32 +1298,33 @@ struct test_case { output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend))); // check if the backend supports the ops - bool supported = true; - bool any_params = false; + bool supported = true; + bool any_params = false; std::string failure_reason; for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { if (!ggml_backend_supports_op(backend, t)) { - supported = false; + supported = false; failure_reason = ggml_backend_name(backend); break; } if ((t->flags & GGML_TENSOR_FLAG_PARAM)) { any_params = true; if (t->type != GGML_TYPE_F32) { - supported = false; + supported = false; failure_reason = std::string(t->name) + "->type != FP32"; break; } } } if (!any_params) { - supported = false; + supported = false; failure_reason = op_desc(out); } if (!supported) { - output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), test_status_t::NOT_SUPPORTED, failure_reason)); + output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), + test_status_t::NOT_SUPPORTED, failure_reason)); return true; } @@ -1331,12 +1360,16 @@ struct test_case { for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { if (!ggml_backend_supports_op(backend, t)) { - output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), test_status_t::NOT_SUPPORTED, ggml_backend_name(backend))); + output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), + test_status_t::NOT_SUPPORTED, + ggml_backend_name(backend))); supported = false; break; } if ((t->flags & GGML_TENSOR_FLAG_PARAM) && t->type != GGML_TYPE_F32) { - output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), test_status_t::NOT_SUPPORTED, std::string(t->name) + "->type != FP32")); + output_printer->print_operation(test_operation_info(op_desc(out), vars(), ggml_backend_name(backend), + test_status_t::NOT_SUPPORTED, + std::string(t->name) + "->type != FP32")); supported = false; break; } @@ -5488,7 +5521,8 @@ static std::vector> make_test_cases_perf() { return test_cases; } -static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op_name, const char * params_filter, printer * output_printer = nullptr) { +static bool test_backend(ggml_backend_t backend, test_mode mode, const char * op_name, const char * params_filter, + printer * output_printer) { auto filter_test_cases = [](std::vector> & test_cases, const char * params_filter) { if (params_filter == nullptr) { return; @@ -5634,13 +5668,15 @@ int main(int argc, char ** argv) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); if (backend_filter != NULL && strcmp(backend_filter, ggml_backend_dev_name(dev)) != 0) { - output_printer->print_backend_init(backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), true, "Skipping")); + output_printer->print_backend_init( + backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), true, "Skipping")); n_ok++; continue; } if (backend_filter == NULL && ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU && mode != MODE_GRAD) { - output_printer->print_backend_init(backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), true, "Skipping CPU backend")); + output_printer->print_backend_init(backend_init_info( + i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), true, "Skipping CPU backend")); n_ok++; continue; } @@ -5655,16 +5691,19 @@ int main(int argc, char ** argv) { ggml_backend_set_n_threads_fn(backend, std::thread::hardware_concurrency()); } - size_t free, total; // NOLINT + size_t free, total; // NOLINT ggml_backend_dev_memory(dev, &free, &total); - output_printer->print_backend_init(backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), false, "", ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024, true)); + output_printer->print_backend_init(backend_init_info(i, ggml_backend_dev_count(), ggml_backend_dev_name(dev), + false, "", ggml_backend_dev_description(dev), + total / 1024 / 1024, free / 1024 / 1024, true)); bool ok = test_backend(backend, mode, op_name_filter, params_filter, output_printer.get()); if (ok) { n_ok++; } - output_printer->print_backend_status(backend_status_info(ggml_backend_name(backend), ok ? test_status_t::OK : test_status_t::FAIL)); + output_printer->print_backend_status( + backend_status_info(ggml_backend_name(backend), ok ? test_status_t::OK : test_status_t::FAIL)); ggml_backend_free(backend); } @@ -5675,7 +5714,8 @@ int main(int argc, char ** argv) { output_printer->print_footer(); } - output_printer->print_overall_summary(overall_summary_info(n_ok, ggml_backend_dev_count(), n_ok == ggml_backend_dev_count())); + output_printer->print_overall_summary( + overall_summary_info(n_ok, ggml_backend_dev_count(), n_ok == ggml_backend_dev_count())); if (n_ok != ggml_backend_dev_count()) { return 1;