retry when failed to save/load in SwapVM

src/extensions/empty_private_settings.h

@@ -46,10 +46,10 @@ static const unsigned char oracleMachineIPs[][4] = {
 #endif
 
 // Virtual memory settings for logging
-#define LOG_BUFFER_PAGE_SIZE 300000000ULL
-#define PMAP_LOG_PAGE_SIZE 30000000ULL
+#define LOG_BUFFER_PAGE_SIZE (300000000ULL / 2)
+#define PMAP_LOG_PAGE_SIZE (30000000ULL / 2)
 #define IMAP_LOG_PAGE_SIZE 10000ULL
-#define VM_NUM_CACHE_PAGE 8
+#define VM_NUM_CACHE_PAGE 4
 
 #if ENABLE_QUBIC_LOGGING_EVENT
 // DO NOT MODIFY THIS AREA UNLESS YOU ARE DEVELOPING LOGGING FEATURES

src/platform/virtual_memory.h

@@ -9,6 +9,7 @@
 
 #include "four_q.h"
 #include "kangaroo_twelve.h"
+#include <lib/platform_common/sleep.h>
 
 template <class T>
 inline constexpr const T& max(const T& left, const T& right)
@@ -22,6 +23,10 @@ inline constexpr const T& min(const T& left, const T& right)
     return (left < right) ? left : right;
 }
 
+// bounded retry: 100ms, 200ms, 400ms, 800ms, 1600ms (~3.1s max)
+static constexpr int SWAPVM_IO_MAX_ATTEMPTS = 5;
+static constexpr unsigned int SWAPVM_IO_INITIAL_DELAY_MS = 100;
+
 // an util to use disk as RAM to reduce hardware requirement for qubic core node
 // this VirtualMemory doesn't (yet) support amend operation. That means data stay persisted once they are written
 // template variables meaning:
@@ -722,18 +727,41 @@ class SwapVirtualMemory : private VirtualMemory<T, prefixName, pageDirectory, pa
         }
         unsigned char *pageBuffer = (unsigned char*)cache[cache_idx];
 
-#if defined(NO_UEFI) && !defined(REAL_NODE)
-        auto sz = save(pageName, pageSize, (unsigned char*)pageBuffer, pageDir);
-#else
-        auto sz = save(pageName, pageSize, (unsigned char*)pageBuffer, pageDir);
-#endif
+        // bounded retry on save() failure
+        unsigned long long sz = 0;
+        unsigned int delayMs = SWAPVM_IO_INITIAL_DELAY_MS;
+        for (int attempt = 0; attempt < SWAPVM_IO_MAX_ATTEMPTS; attempt++)
+        {
+            sz = save(pageName, pageSize, (unsigned char*)pageBuffer, pageDir);
+            if (sz == pageSize)
+                break;
+
+            setText(message, L"swapVM writePageToDisk failed (attempt ");
+            appendNumber(message, (unsigned long long)(attempt + 1), false);
+            appendText(message, L"/");
+            appendNumber(message, (unsigned long long)SWAPVM_IO_MAX_ATTEMPTS, false);
+            appendText(message, L") page ");
+            appendNumber(message, pageId, true);
+            logToConsole(message);
+
+            if (attempt + 1 < SWAPVM_IO_MAX_ATTEMPTS)
+            {
+                sleepMilliseconds(delayMs);
+                delayMs *= 2;
+            }
+        }
 
-#if !defined(NDEBUG)
         if (sz != pageSize)
         {
-            addDebugMessage(L"Failed to store virtualMemory to disk. Old data maybe lost");
+            setText(message, L"Fatal: swapVM writePageToDisk exhausted retries | page ");
+            appendNumber(message, pageId, true);
+            appendText(message, L" | prefix ");
+            appendText(message, pageDir);
+            logToConsole(message);
+            exit(1);
         }
-        else
+
+#if !defined(NDEBUG)
         {
             CHAR16 debugMsg[128];
             unsigned long long tmp = prefixName;
@@ -749,6 +777,7 @@ class SwapVirtualMemory : private VirtualMemory<T, prefixName, pageDirectory, pa
         }
 #endif
 
+        // only mark written on confirmed success
         isPageWrittenToDisk[pageId] = true;
     }
 
@@ -782,8 +811,30 @@ class SwapVirtualMemory : private VirtualMemory<T, prefixName, pageDirectory, pa
         unsigned long long sz = 0;
         if (isPageWrittenToDisk[pageId])
         {
-            sz = load(pageName, pageSize, (unsigned char*)cache[cache_page_id], pageDir);
-        } else
+            // bounded retry on load() failure
+            unsigned int delayMs = SWAPVM_IO_INITIAL_DELAY_MS;
+            for (int attempt = 0; attempt < SWAPVM_IO_MAX_ATTEMPTS; attempt++)
+            {
+                sz = load(pageName, pageSize, (unsigned char*)cache[cache_page_id], pageDir);
+                if (sz == pageSize)
+                    break;
+
+                setText(message, L"swapVM loadPage failed (attempt ");
+                appendNumber(message, (unsigned long long)(attempt + 1), false);
+                appendText(message, L"/");
+                appendNumber(message, (unsigned long long)SWAPVM_IO_MAX_ATTEMPTS, false);
+                appendText(message, L") page ");
+                appendNumber(message, pageId, true);
+                logToConsole(message);
+
+                if (attempt + 1 < SWAPVM_IO_MAX_ATTEMPTS)
+                {
+                    sleepMilliseconds(delayMs);
+                    delayMs *= 2;
+                }
+            }
+        }
+        else
         {
             sz = pageSize;
             setMem(cache[cache_page_id], pageSize, 0);

src/private_settings.h

@@ -1406,10 +1406,10 @@ static const unsigned char oracleMachineIPs[][4] = {
 #endif
 
 // Virtual memory settings for logging
-#define LOG_BUFFER_PAGE_SIZE 300000000ULL
-#define PMAP_LOG_PAGE_SIZE 30000000ULL
+#define LOG_BUFFER_PAGE_SIZE (300000000ULL / 2)
+#define PMAP_LOG_PAGE_SIZE (30000000ULL / 2)
 #define IMAP_LOG_PAGE_SIZE 10000ULL
-#define VM_NUM_CACHE_PAGE 8
+#define VM_NUM_CACHE_PAGE 4
 
 #if ENABLE_QUBIC_LOGGING_EVENT
 // DO NOT MODIFY THIS AREA UNLESS YOU ARE DEVELOPING LOGGING FEATURES

src/public_settings.h

@@ -48,7 +48,7 @@
 #define TICK_DURATION_FOR_ALLOCATION_MS 350
 #define TRANSACTION_SPARSENESS 3
 // Number of ticks that are stored in the pending txs pool. This also defines how many ticks in advance a tx can be registered.
-#define PENDING_TXS_POOL_NUM_TICKS (1000 * 60 * 10ULL / TICK_DURATION_FOR_ALLOCATION_MS) // 10 minutes
+#define PENDING_TXS_POOL_NUM_TICKS (1000 * 60 * 3ULL / TICK_DURATION_FOR_ALLOCATION_MS) // 3 minutes
 #endif
 
 #ifdef TESTNET