refactor: Add mem module to allow tests to override allocators.

This will allow us to do more interesting things with memory allocation
within toxcore, and allow fuzzers to explore various allocation failure
paths.

Author: iphydf

CMakeLists.txt

@@ -268,6 +268,8 @@ set(toxcore_SOURCES
   toxcore/logger.h
   toxcore/Messenger.c
   toxcore/Messenger.h
+  toxcore/mem.c
+  toxcore/mem.h
   toxcore/mono_time.c
   toxcore/mono_time.h
   toxcore/net_crypto.c
@@ -434,6 +436,7 @@ unit_test(toxcore bin_pack)
 unit_test(toxcore crypto_core)
 unit_test(toxcore group_announce)
 unit_test(toxcore group_moderation)
+unit_test(toxcore mem)
 unit_test(toxcore mono_time)
 unit_test(toxcore ping_array)
 unit_test(toxcore tox)

auto_tests/TCP_test.c

@@ -48,15 +48,19 @@ static void test_basic(void)
     Mono_Time *mono_time = mono_time_new(nullptr, nullptr);
     const Random *rng = system_random();
     ck_assert(rng != nullptr);
+    const Network *ns = system_network();
+    ck_assert(ns != nullptr);
+    const Memory *mem = system_memory();
+    ck_assert(mem != nullptr);
+
     Logger *logger = logger_new();
     logger_callback_log(logger, print_debug_logger, nullptr, nullptr);
 
     // Attempt to create a new TCP_Server instance.
     uint8_t self_public_key[CRYPTO_PUBLIC_KEY_SIZE];
     uint8_t self_secret_key[CRYPTO_SECRET_KEY_SIZE];
     crypto_new_keypair(rng, self_public_key, self_secret_key);
-    const Network *ns = system_network();
-    TCP_Server *tcp_s = new_TCP_server(logger, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr, nullptr);
+    TCP_Server *tcp_s = new_TCP_server(logger, mem, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr, nullptr);
     ck_assert_msg(tcp_s != nullptr, "Failed to create a TCP relay server.");
     ck_assert_msg(tcp_server_listen_count(tcp_s) == NUM_PORTS,
                   "Failed to bind a TCP relay server to all %d attempted ports.", NUM_PORTS);
@@ -302,11 +306,14 @@ static void test_some(void)
     ck_assert(rng != nullptr);
     Logger *logger = logger_new();
     const Network *ns = system_network();
+    ck_assert(ns != nullptr);
+    const Memory *mem = system_memory();
+    ck_assert(mem != nullptr);
 
     uint8_t self_public_key[CRYPTO_PUBLIC_KEY_SIZE];
     uint8_t self_secret_key[CRYPTO_SECRET_KEY_SIZE];
     crypto_new_keypair(rng, self_public_key, self_secret_key);
-    TCP_Server *tcp_s = new_TCP_server(logger, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr, nullptr);
+    TCP_Server *tcp_s = new_TCP_server(logger, mem, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr, nullptr);
     ck_assert_msg(tcp_s != nullptr, "Failed to create TCP relay server");
     ck_assert_msg(tcp_server_listen_count(tcp_s) == NUM_PORTS, "Failed to bind to all ports.");
 
@@ -492,12 +499,15 @@ static void test_client(void)
     const Random *rng = system_random();
     ck_assert(rng != nullptr);
     Logger *logger = logger_new();
+    const Network *ns = system_network();
+    ck_assert(ns != nullptr);
+    const Memory *mem = system_memory();
+    ck_assert(mem != nullptr);
 
     uint8_t self_public_key[CRYPTO_PUBLIC_KEY_SIZE];
     uint8_t self_secret_key[CRYPTO_SECRET_KEY_SIZE];
     crypto_new_keypair(rng, self_public_key, self_secret_key);
-    const Network *ns = system_network();
-    TCP_Server *tcp_s = new_TCP_server(logger, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr, nullptr);
+    TCP_Server *tcp_s = new_TCP_server(logger, mem, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr, nullptr);
     ck_assert_msg(tcp_s != nullptr, "Failed to create a TCP relay server.");
     ck_assert_msg(tcp_server_listen_count(tcp_s) == NUM_PORTS, "Failed to bind the relay server to all ports.");
 
@@ -699,22 +709,25 @@ static void test_tcp_connection(void)
     const Random *rng = system_random();
     ck_assert(rng != nullptr);
     const Network *ns = system_network();
+    ck_assert(ns != nullptr);
+    const Memory *mem = system_memory();
+    ck_assert(mem != nullptr);
 
     tcp_data_callback_called = 0;
     uint8_t self_public_key[CRYPTO_PUBLIC_KEY_SIZE];
     uint8_t self_secret_key[CRYPTO_SECRET_KEY_SIZE];
     crypto_new_keypair(rng, self_public_key, self_secret_key);
-    TCP_Server *tcp_s = new_TCP_server(logger, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr, nullptr);
+    TCP_Server *tcp_s = new_TCP_server(logger, mem, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr, nullptr);
     ck_assert_msg(pk_equal(tcp_server_public_key(tcp_s), self_public_key), "Wrong public key");
 
     TCP_Proxy_Info proxy_info;
     proxy_info.proxy_type = TCP_PROXY_NONE;
     crypto_new_keypair(rng, self_public_key, self_secret_key);
-    TCP_Connections *tc_1 = new_tcp_connections(logger, rng, ns, mono_time, self_secret_key, &proxy_info);
+    TCP_Connections *tc_1 = new_tcp_connections(logger, mem, rng, ns, mono_time, self_secret_key, &proxy_info);
     ck_assert_msg(pk_equal(tcp_connections_public_key(tc_1), self_public_key), "Wrong public key");
 
     crypto_new_keypair(rng, self_public_key, self_secret_key);
-    TCP_Connections *tc_2 = new_tcp_connections(logger, rng, ns, mono_time, self_secret_key, &proxy_info);
+    TCP_Connections *tc_2 = new_tcp_connections(logger, mem, rng, ns, mono_time, self_secret_key, &proxy_info);
     ck_assert_msg(pk_equal(tcp_connections_public_key(tc_2), self_public_key), "Wrong public key");
 
     IP_Port ip_port_tcp_s;
@@ -808,24 +821,27 @@ static void test_tcp_connection2(void)
     const Random *rng = system_random();
     ck_assert(rng != nullptr);
     const Network *ns = system_network();
+    ck_assert(ns != nullptr);
+    const Memory *mem = system_memory();
+    ck_assert(mem != nullptr);
 
     tcp_oobdata_callback_called = 0;
     tcp_data_callback_called = 0;
 
     uint8_t self_public_key[CRYPTO_PUBLIC_KEY_SIZE];
     uint8_t self_secret_key[CRYPTO_SECRET_KEY_SIZE];
     crypto_new_keypair(rng, self_public_key, self_secret_key);
-    TCP_Server *tcp_s = new_TCP_server(logger, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr, nullptr);
+    TCP_Server *tcp_s = new_TCP_server(logger, mem, rng, ns, USE_IPV6, NUM_PORTS, ports, self_secret_key, nullptr, nullptr);
     ck_assert_msg(pk_equal(tcp_server_public_key(tcp_s), self_public_key), "Wrong public key");
 
     TCP_Proxy_Info proxy_info;
     proxy_info.proxy_type = TCP_PROXY_NONE;
     crypto_new_keypair(rng, self_public_key, self_secret_key);
-    TCP_Connections *tc_1 = new_tcp_connections(logger, rng, ns, mono_time, self_secret_key, &proxy_info);
+    TCP_Connections *tc_1 = new_tcp_connections(logger, mem, rng, ns, mono_time, self_secret_key, &proxy_info);
     ck_assert_msg(pk_equal(tcp_connections_public_key(tc_1), self_public_key), "Wrong public key");
 
     crypto_new_keypair(rng, self_public_key, self_secret_key);
-    TCP_Connections *tc_2 = new_tcp_connections(logger, rng, ns, mono_time, self_secret_key, &proxy_info);
+    TCP_Connections *tc_2 = new_tcp_connections(logger, mem, rng, ns, mono_time, self_secret_key, &proxy_info);
     ck_assert_msg(pk_equal(tcp_connections_public_key(tc_2), self_public_key), "Wrong public key");
 
     IP_Port ip_port_tcp_s;

auto_tests/announce_test.c

@@ -54,14 +54,17 @@ static void test_store_data(void)
     ck_assert(rng != nullptr);
     const Network *ns = system_network();
     ck_assert(ns != nullptr);
+    const Memory *mem = system_memory();
+    ck_assert(mem != nullptr);
+
     Logger *log = logger_new();
     ck_assert(log != nullptr);
     logger_callback_log(log, print_debug_logger, nullptr, nullptr);
     Mono_Time *mono_time = mono_time_new(nullptr, nullptr);
     Networking_Core *net = new_networking_no_udp(log, ns);
-    DHT *dht = new_dht(log, rng, ns, mono_time, net, true, true);
+    DHT *dht = new_dht(log, mem, rng, ns, mono_time, net, true, true);
     Forwarding *forwarding = new_forwarding(log, rng, mono_time, dht);
-    Announcements *announce = new_announcements(log, rng, mono_time, forwarding);
+    Announcements *announce = new_announcements(log, mem, rng, mono_time, forwarding);
     ck_assert(announce != nullptr);
 
     /* Just to prevent CI from complaining that set_synch_offset is unused: */

auto_tests/forwarding_test.c

@@ -116,6 +116,8 @@ static Forwarding_Subtox *new_forwarding_subtox(bool no_udp, uint32_t *index, ui
     ck_assert(rng != nullptr);
     const Network *ns = system_network();
     ck_assert(ns != nullptr);
+    const Memory *mem = system_memory();
+    ck_assert(mem != nullptr);
 
     if (no_udp) {
         subtox->net = new_networking_no_udp(subtox->log, ns);
@@ -124,15 +126,15 @@ static Forwarding_Subtox *new_forwarding_subtox(bool no_udp, uint32_t *index, ui
         subtox->net = new_networking_ex(subtox->log, ns, &ip, port, port, nullptr);
     }
 
-    subtox->dht = new_dht(subtox->log, rng, ns, subtox->mono_time, subtox->net, true, true);
+    subtox->dht = new_dht(subtox->log, mem, rng, ns, subtox->mono_time, subtox->net, true, true);
 
     const TCP_Proxy_Info inf = {{{{0}}}};
-    subtox->c = new_net_crypto(subtox->log, rng, ns, subtox->mono_time, subtox->dht, &inf);
+    subtox->c = new_net_crypto(subtox->log, mem, rng, ns, subtox->mono_time, subtox->dht, &inf);
 
     subtox->forwarding = new_forwarding(subtox->log, rng, subtox->mono_time, subtox->dht);
     ck_assert(subtox->forwarding != nullptr);
 
-    subtox->announce = new_announcements(subtox->log, rng, subtox->mono_time, subtox->forwarding);
+    subtox->announce = new_announcements(subtox->log, mem, rng, subtox->mono_time, subtox->forwarding);
     ck_assert(subtox->announce != nullptr);
 
     return subtox;

auto_tests/onion_test.c

@@ -224,20 +224,23 @@ static void test_basic(void)
 {
     uint32_t index[] = { 1, 2, 3 };
     const Network *ns = system_network();
+    ck_assert(ns != nullptr);
+    const Memory *mem = system_memory();
+    ck_assert(mem != nullptr);
+    const Random *rng = system_random();
+    ck_assert(rng != nullptr);
 
     Logger *log1 = logger_new();
     logger_callback_log(log1, print_debug_logger, nullptr, &index[0]);
     Logger *log2 = logger_new();
     logger_callback_log(log2, print_debug_logger, nullptr, &index[1]);
 
-    const Random *rng = system_random();
-    ck_assert(rng != nullptr);
     Mono_Time *mono_time1 = mono_time_new(nullptr, nullptr);
     Mono_Time *mono_time2 = mono_time_new(nullptr, nullptr);
 
     IP ip = get_loopback();
-    Onion *onion1 = new_onion(log1, mono_time1, rng, new_dht(log1, rng, ns, mono_time1, new_networking(log1, ns, &ip, 36567), true, false));
-    Onion *onion2 = new_onion(log2, mono_time2, rng, new_dht(log2, rng, ns, mono_time2, new_networking(log2, ns, &ip, 36568), true, false));
+    Onion *onion1 = new_onion(log1, mono_time1, rng, new_dht(log1, mem, rng, ns, mono_time1, new_networking(log1, ns, &ip, 36567), true, false));
+    Onion *onion2 = new_onion(log2, mono_time2, rng, new_dht(log2, mem, rng, ns, mono_time2, new_networking(log2, ns, &ip, 36568), true, false));
     ck_assert_msg((onion1 != nullptr) && (onion2 != nullptr), "Onion failed initializing.");
     networking_registerhandler(onion2->net, NET_PACKET_ANNOUNCE_REQUEST, &handle_test_1, onion2);
 
@@ -333,7 +336,7 @@ static void test_basic(void)
 
     Mono_Time *mono_time3 = mono_time_new(nullptr, nullptr);
 
-    Onion *onion3 = new_onion(log3, mono_time3, rng, new_dht(log3, rng, ns, mono_time3, new_networking(log3, ns, &ip, 36569), true, false));
+    Onion *onion3 = new_onion(log3, mono_time3, rng, new_dht(log3, mem, rng, ns, mono_time3, new_networking(log3, ns, &ip, 36569), true, false));
     ck_assert_msg((onion3 != nullptr), "Onion failed initializing.");
 
     random_nonce(rng, nonce);
@@ -400,7 +403,7 @@ typedef struct {
     Onion_Client *onion_c;
 } Onions;
 
-static Onions *new_onions(const Random *rng, uint16_t port, uint32_t *index)
+static Onions *new_onions(const Memory *mem, const Random *rng, uint16_t port, uint32_t *index)
 {
     IP ip = get_loopback();
     ip.ip.v6.uint8[15] = 1;
@@ -437,7 +440,7 @@ static Onions *new_onions(const Random *rng, uint16_t port, uint32_t *index)
         return nullptr;
     }
 
-    DHT *dht = new_dht(on->log, rng, ns, on->mono_time, net, true, false);
+    DHT *dht = new_dht(on->log, mem, rng, ns, on->mono_time, net, true, false);
 
     if (!dht) {
         kill_networking(net);
@@ -471,7 +474,7 @@ static Onions *new_onions(const Random *rng, uint16_t port, uint32_t *index)
     }
 
     TCP_Proxy_Info inf = {{{{0}}}};
-    on->onion_c = new_onion_client(on->log, rng, on->mono_time, new_net_crypto(on->log, rng, ns, on->mono_time, dht, &inf));
+    on->onion_c = new_onion_client(on->log, rng, on->mono_time, new_net_crypto(on->log, mem, rng, ns, on->mono_time, dht, &inf));
 
     if (!on->onion_c) {
         kill_onion_announce(on->onion_a);
@@ -576,10 +579,12 @@ static void test_announce(void)
     Onions *onions[NUM_ONIONS];
     const Random *rng = system_random();
     ck_assert(rng != nullptr);
+    const Memory *mem = system_memory();
+    ck_assert(mem != nullptr);
 
     for (i = 0; i < NUM_ONIONS; ++i) {
         index[i] = i + 1;
-        onions[i] = new_onions(rng, i + 36655, &index[i]);
+        onions[i] = new_onions(mem, rng, i + 36655, &index[i]);
         ck_assert_msg(onions[i] != nullptr, "Failed to create onions. %u", i);
     }
 

other/DHT_bootstrap.c

@@ -148,7 +148,8 @@ int main(int argc, char *argv[])
     const uint16_t start_port = PORT;
     const uint16_t end_port = start_port + (TOX_PORTRANGE_TO - TOX_PORTRANGE_FROM);
     const Network *ns = system_network();
-    DHT *dht = new_dht(logger, rng, ns, mono_time, new_networking_ex(logger, ns, &ip, start_port, end_port, nullptr), true, true);
+    const Memory *mem = system_memory();
+    DHT *dht = new_dht(logger, mem, rng, ns, mono_time, new_networking_ex(logger, ns, &ip, start_port, end_port, nullptr), true, true);
     Onion *onion = new_onion(logger, mono_time, rng, dht);
     Forwarding *forwarding = new_forwarding(logger, rng, mono_time, dht);
     GC_Announces_List *gc_announces_list = new_gca_list();
@@ -173,7 +174,7 @@ int main(int argc, char *argv[])
 #ifdef TCP_RELAY_ENABLED
 #define NUM_PORTS 3
     uint16_t ports[NUM_PORTS] = {443, 3389, PORT};
-    TCP_Server *tcp_s = new_TCP_server(logger, rng, ns, ipv6enabled, NUM_PORTS, ports, dht_get_self_secret_key(dht), onion, forwarding);
+    TCP_Server *tcp_s = new_TCP_server(logger, mem, rng, ns, ipv6enabled, NUM_PORTS, ports, dht_get_self_secret_key(dht), onion, forwarding);
 
     if (tcp_s == nullptr) {
         printf("TCP server failed to initialize.\n");

other/bootstrap_daemon/docker/tox-bootstrapd.sha256

@@ -1 +1 @@
-23e142729a72618462018b25c830dbad5b023d0fa3de9ae53d376f43dc46b481  /usr/local/bin/tox-bootstrapd
+94b8f80261340bb6debb79ff6cfeed6c238aa304ffee8297791e7153f42b9536  /usr/local/bin/tox-bootstrapd

other/bootstrap_daemon/src/tox-bootstrapd.c

@@ -322,8 +322,9 @@ int main(int argc, char *argv[])
 
     mono_time_update(mono_time);
 
+    const Memory *mem = system_memory();
     const Random *rng = system_random();
-    DHT *const dht = new_dht(logger, rng, ns, mono_time, net, true, enable_lan_discovery);
+    DHT *const dht = new_dht(logger, mem, rng, ns, mono_time, net, true, enable_lan_discovery);
 
     if (dht == nullptr) {
         log_write(LOG_LEVEL_ERROR, "Couldn't initialize Tox DHT instance. Exiting.\n");
@@ -350,7 +351,7 @@ int main(int argc, char *argv[])
         return 1;
     }
 
-    Announcements *announce = new_announcements(logger, rng, mono_time, forwarding);
+    Announcements *announce = new_announcements(logger, mem, rng, mono_time, forwarding);
 
     if (announce == nullptr) {
         log_write(LOG_LEVEL_ERROR, "Couldn't initialize DHT announcements. Exiting.\n");
@@ -476,7 +477,8 @@ int main(int argc, char *argv[])
             return 1;
         }
 
-        tcp_server = new_TCP_server(logger, rng, ns, enable_ipv6, tcp_relay_port_count, tcp_relay_ports,
+        tcp_server = new_TCP_server(logger, mem, rng, ns, enable_ipv6,
+                                    tcp_relay_port_count, tcp_relay_ports,
                                     dht_get_self_secret_key(dht), onion, forwarding);
 
         free(tcp_relay_ports);

testing/Messenger_test.c

@@ -102,7 +102,7 @@ int main(int argc, char *argv[])
     Messenger_Options options = {0};
     options.ipv6enabled = ipv6enabled;
     Messenger_Error err;
-    m = new_messenger(mono_time, system_random(), system_network(), &options, &err);
+    m = new_messenger(mono_time, system_memory(), system_random(), system_network(), &options, &err);
 
     if (!m) {
         fprintf(stderr, "Failed to allocate messenger datastructure: %d\n", err);

testing/fuzzing/BUILD.bazel

@@ -28,6 +28,7 @@ cc_library(
 
 cc_fuzz_test(
     name = "bootstrap_fuzz_test",
+    #size = "small",
     srcs = ["bootstrap_harness.cc"],
     copts = ["-UNDEBUG"],
     corpus = ["//tools/toktok-fuzzer/corpus:bootstrap_fuzzer"],
@@ -42,6 +43,7 @@ cc_fuzz_test(
 
 cc_fuzz_test(
     name = "e2e_fuzz_test",
+    #size = "small",
     srcs = ["e2e_fuzz_test.cc"],
     copts = ["-UNDEBUG"],
     corpus = ["//tools/toktok-fuzzer/corpus:e2e_fuzz_test"],
@@ -57,6 +59,7 @@ cc_fuzz_test(
 
 cc_fuzz_test(
     name = "toxsave_fuzz_test",
+    #size = "small",
     srcs = ["toxsave_harness.cc"],
     copts = ["-UNDEBUG"],
     corpus = ["//tools/toktok-fuzzer/corpus:toxsave_fuzzer"],
@@ -89,6 +92,7 @@ fuzzing_binary(
 
 cc_fuzz_test(
     name = "protodump_reduce",
+    #size = "small",
     srcs = ["protodump_reduce.cc"],
     copts = ["-UNDEBUG"],
     deps = [