Add option to print and save all results, when performing multiple test iterations (#298)

* Add option to print all histograms instead of only BEST and WORST

This functionallity is needed since sometimes theres a need to see a
specific run's results.
Controlled by a new flag: --print-all-hists yes

* Fixed release action steps with minimal changes (#299)

* using ubuntu-22.04 on runner of release (#300)

* Remove ubuntu:bionic logic from smoke-test-packages and removed usage of actions/download-artifact@v3. (#301)

* Renamed --print-all-hists to --print-all-runs. Ensured the change is retro-compatible. Added test to confirm it.

* Fixed extra space on run check

---------

Co-authored-by: Filipe Oliveira (Personal) <[email protected]>

Author: Yuval-Ariel

memtier_benchmark.cpp

@@ -159,7 +159,8 @@ static void config_print(FILE *file, struct benchmark_config *cfg)
         "wait-ratio = %u:%u\n"
         "num-slaves = %u-%u\n"
         "wait-timeout = %u-%u\n"
-        "json-out-file = %s\n",
+        "json-out-file = %s\n"
+        "print-all-runs = %s\n",
         cfg->server,
         cfg->port,
         cfg->unix_socket,
@@ -209,7 +210,8 @@ static void config_print(FILE *file, struct benchmark_config *cfg)
         cfg->wait_ratio.a, cfg->wait_ratio.b,
         cfg->num_slaves.min, cfg->num_slaves.max,
         cfg->wait_timeout.min, cfg->wait_timeout.max,
-        cfg->json_out_file);
+        cfg->json_out_file,
+        cfg->print_all_runs ? "yes" : "no");
 }
 
 static void config_print_to_json(json_handler * jsonhandler, struct benchmark_config *cfg)
@@ -267,6 +269,7 @@ static void config_print_to_json(json_handler * jsonhandler, struct benchmark_co
     jsonhandler->write_obj("wait-ratio"        ,"\"%u:%u\"",    cfg->wait_ratio.a, cfg->wait_ratio.b);
     jsonhandler->write_obj("num-slaves"        ,"\"%u:%u\"",    cfg->num_slaves.min, cfg->num_slaves.max);
     jsonhandler->write_obj("wait-timeout"      ,"\"%u-%u\"",   	cfg->wait_timeout.min, cfg->wait_timeout.max);
+    jsonhandler->write_obj("print-all-runs"   ,"\"%s\"",       cfg->print_all_runs ? "true" : "false");
 
     jsonhandler->close_nesting();
 }
@@ -403,6 +406,7 @@ static int config_parse_args(int argc, char *argv[], struct benchmark_config *cf
         o_show_config,
         o_hide_histogram,
         o_print_percentiles,
+        o_print_all_runs,
         o_distinct_client_seed,
         o_randomize,
         o_client_stats,
@@ -456,6 +460,7 @@ static int config_parse_args(int argc, char *argv[], struct benchmark_config *cf
         { "show-config",                0, 0, o_show_config },
         { "hide-histogram",             0, 0, o_hide_histogram },
         { "print-percentiles",          1, 0, o_print_percentiles },
+        { "print-all-runs",            0, 0, o_print_all_runs },
         { "distinct-client-seed",       0, 0, o_distinct_client_seed },
         { "randomize",                  0, 0, o_randomize },
         { "requests",                   1, 0, 'n' },
@@ -587,6 +592,9 @@ static int config_parse_args(int argc, char *argv[], struct benchmark_config *cf
                         return -1;
                     }
                     break;
+                case o_print_all_runs:
+                    cfg->print_all_runs = true;
+                    break;
                 case o_distinct_client_seed:
                     cfg->distinct_client_seed++;
                     break;
@@ -977,6 +985,7 @@ void usage() {
             "      --show-config              Print detailed configuration before running\n"
             "      --hide-histogram           Don't print detailed latency histogram\n"
             "      --print-percentiles        Specify which percentiles info to print on the results table (by default prints percentiles: 50,99,99.9)\n"
+            "      --print-all-runs           When performing multiple test iterations, print and save results for all iterations\n"
             "      --cluster-mode             Run client in cluster mode\n"
             "  -h, --help                     Display this help\n"
             "  -v, --version                  Display version information\n"
@@ -1652,7 +1661,16 @@ int main(int argc, char *argv[])
         }
 
         // If more than 1 run was used, compute best, worst and average
+        // Furthermore, if print_all_runs is enabled we save separate histograms per run
         if (cfg.run_count > 1) {
+            // User wants to see a separate histogram per run
+            if (cfg.print_all_runs) {
+                for (auto i = 0U; i < all_stats.size(); i++) {
+                    auto run_title = std::string("RUN #") + std::to_string(i + 1) + " RESULTS";
+                    all_stats[i].print(outfile, &cfg, run_title.c_str(), jsonhandler);
+                }
+            }
+            // User wants the best and worst
             unsigned int min_ops_sec = (unsigned int) -1;
             unsigned int max_ops_sec = 0;
             run_stats* worst = NULL;
@@ -1669,7 +1687,6 @@ int main(int argc, char *argv[])
                     best = &(*i);
                 }
             }
-
             // Best results:
             best->print(outfile, &cfg, "BEST RUN RESULTS", jsonhandler);
             // worst results:

memtier_benchmark.h

@@ -63,6 +63,7 @@ struct benchmark_config {
     int show_config;
     int hide_histogram;
     config_quantiles print_percentiles;
+    bool print_all_runs;
     int distinct_client_seed;
     int randomize;
     int next_client_idx;

tests/tests_oss_simple_flow.py

@@ -407,6 +407,51 @@ def test_default_set_get_3_runs(env):
     assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count, overall_request_count)
 
 
+
+# run each test on different env
+def test_print_all_runs(env):
+    run_count = 5
+    benchmark_specs = {"name": env.testName, "args": ['--print-all-runs','--run-count={}'.format(run_count)]}
+    addTLSArgs(benchmark_specs, env)
+    config = get_default_memtier_config()
+    master_nodes_list = env.getMasterNodesList()
+    overall_expected_request_count = get_expected_request_count(config) * run_count
+
+    add_required_env_arguments(benchmark_specs, config, env, master_nodes_list)
+
+    # Create a temporary directory
+    test_dir = tempfile.mkdtemp()
+
+    config = RunConfig(test_dir, env.testName, config, {})
+    ensure_clean_benchmark_folder(config.results_dir)
+
+    benchmark = Benchmark.from_json(config, benchmark_specs)
+
+    # benchmark.run() returns True if the return code of memtier_benchmark was 0
+    memtier_ok = benchmark.run()
+
+    master_nodes_connections = env.getOSSMasterNodesConnectionList()
+    merged_command_stats = {'cmdstat_set': {'calls': 0}, 'cmdstat_get': {'calls': 0}}
+    overall_request_count = agg_info_commandstats(master_nodes_connections, merged_command_stats)
+    assert_minimum_memtier_outcomes(config, env, memtier_ok, overall_expected_request_count, overall_request_count)
+
+    json_filename = '{0}/mb.json'.format(config.results_dir)
+    ## Assert that all BW metrics are properly stored and calculated
+    with open(json_filename) as results_json:
+        results_dict = json.load(results_json)
+        print_all_runs = results_dict["configuration"]["print-all-runs"]
+        env.assertTrue(print_all_runs)
+        for run_count in range(1, run_count+1):
+            # assert the run infomation exists
+            env.assertTrue(f"RUN #{run_count} RESULTS" in results_dict)
+        
+        # ensure best, worst, and aggregate results are present
+        env.assertTrue("BEST RUN RESULTS" in results_dict)
+        env.assertTrue("WORST RUN RESULTS" in results_dict)
+        env.assertTrue(f"AGGREGATED AVERAGE RESULTS ({run_count} runs)" in results_dict)
+        # all stats should only exist on a single run json
+        env.assertTrue("ALL STATS" not in results_dict)
+
 def test_default_arbitrary_command_pubsub(env):
     benchmark_specs = {"name": env.testName, "args": []}
     addTLSArgs(benchmark_specs, env)