👷 Add mutation tests

Problem:
- Quis custodiet ipsos custodies? (Who tests the tests?)

Solution:
- Mutation testing tests the tests.

Note:
- To run mutation tests, build the `mull_tests` target.
- Mull tests are run as a CI job but this doesn't gate `merge_ok`.
- The CI job runs mutations tests on ubuntu-22.04, LLVM 15, mull v0.23.0. This
  is the current "latest version" taking into consideration the constraints. When
  we run on ubuntu-24.04, we can use LLVM 17 for mull. And later versions as
  they are supported.

.github/workflows/unit_tests.yml

@@ -15,6 +15,7 @@ env:
   DEFAULT_CXX_STANDARD: 20
   DEFAULT_LLVM_VERSION: 18
   DEFAULT_GCC_VERSION: 13
+  MULL_LLVM_VERSION: 15
 
 concurrency:
   group: ${{ github.head_ref || github.run_id }}
@@ -348,6 +349,52 @@ jobs:
           echo "</details>" >> $GITHUB_STEP_SUMMARY
           test $FAILSIZE = "0"
 
+  mutate:
+    runs-on: ${{ github.repository_owner == 'intel' && 'intel-' || '' }}ubuntu-22.04
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+
+      - name: Install build tools
+        run: |
+          wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && sudo ./llvm.sh ${{env.MULL_LLVM_VERSION}}
+          sudo apt install -y ninja-build
+
+      - name: Install mull
+        env:
+          MULL_VERSION: 0.23.0
+        run: |
+          wget https://github.com/mull-project/mull/releases/download/${{env.MULL_VERSION}}/Mull-${{env.MULL_LLVM_VERSION}}-${{env.MULL_VERSION}}-LLVM-${{env.MULL_LLVM_VERSION}}.0-ubuntu-22.04.deb
+          sudo dpkg -i Mull-${{env.MULL_LLVM_VERSION}}-${{env.MULL_VERSION}}-LLVM-${{env.MULL_LLVM_VERSION}}.0-ubuntu-22.04.deb
+
+      - name: Restore CPM cache
+        env:
+          cache-name: cpm-cache-0
+        id: cpm-cache-restore
+        uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: ~/cpm-cache
+          key: ${{runner.os}}-${{env.cache-name}}-${{ hashFiles('**/CMakeLists.txt', 'cmake/**') }}
+          restore-keys: |
+            ${{runner.os}}-${{env.cache-name}}-
+
+      - name: Configure CMake
+        env:
+          CC: "/usr/lib/llvm-${{env.MULL_LLVM_VERSION}}/bin/clang"
+          CXX: "/usr/lib/llvm-${{env.MULL_LLVM_VERSION}}/bin/clang++"
+        run: cmake -B ${{github.workspace}}/build -DCMAKE_CXX_STANDARD=${{env.DEFAULT_CXX_STANDARD}} -DCPM_SOURCE_CACHE=~/cpm-cache
+
+      - name: Save CPM cache
+        env:
+          cache-name: cpm-cache-0
+        if: steps.cpm-cache-restore.outputs.cache-hit != 'true'
+        uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: ~/cpm-cache
+          key: ${{runner.os}}-${{env.cache-name}}-${{ hashFiles('**/CMakeLists.txt', 'cmake/**') }}
+
+      - name: Build and run mull tests
+        run: cmake --build build -t mull_tests
+
   merge_ok:
     runs-on: ${{ github.repository_owner == 'intel' && 'intel-' || '' }}ubuntu-22.04
     needs: [build_and_test, quality_checks_pass, sanitize, valgrind]

mull.yml

@@ -0,0 +1,7 @@
+mutators:
+  - cxx_all
+ignoreMutators:
+  - cxx_remove_void_call
+excludePaths:
+  - .*/include/stdx/.*
+timeout: 10000

test/CMakeLists.txt

@@ -1,5 +1,8 @@
 function(add_tests)
-    foreach(name ${ARGN})
+    set(multiValueArgs FILES MULL_EXCLUSIONS)
+    cmake_parse_arguments(TEST "" "" "${multiValueArgs}" ${ARGN})
+
+    foreach(name ${TEST_FILES})
         add_unit_test(
             "${name}_test"
             CATCH2
@@ -8,10 +11,14 @@ function(add_tests)
             LIBRARIES
             warnings
             stdx)
+        if(NOT name IN_LIST TEST_MULL_EXCLUSIONS)
+            add_mull_test("${name}_test" EXCLUDE_CTEST)
+        endif()
     endforeach()
 endfunction()
 
 add_tests(
+    FILES
     algorithm
     always_false
     bind
@@ -53,7 +60,7 @@ add_tests(
     udls)
 
 if(${CMAKE_CXX_STANDARD} GREATER_EQUAL 20)
-    add_tests(ct_format ct_string indexed_tuple tuple tuple_algorithms)
+    add_tests(FILES ct_format ct_string indexed_tuple tuple tuple_algorithms)
 endif()
 
 add_subdirectory(fail)

test/ct_format.cpp

@@ -67,20 +67,20 @@ TEST_CASE("format a compile-time enum argument", "[ct_format]") {
 }
 
 TEST_CASE("format a runtime argument", "[ct_format]") {
-    auto x = 42;
+    auto x = 17;
     CHECK(stdx::ct_format<"Hello {}">(x) ==
-          stdx::format_result{"Hello {}"_cts, stdx::make_tuple(42)});
-    static_assert(stdx::ct_format<"Hello {}">(42) ==
-                  stdx::format_result{"Hello {}"_cts, stdx::make_tuple(42)});
+          stdx::format_result{"Hello {}"_cts, stdx::make_tuple(17)});
+    static_assert(stdx::ct_format<"Hello {}">(17) ==
+                  stdx::format_result{"Hello {}"_cts, stdx::make_tuple(17)});
 }
 
 TEST_CASE("format a compile-time and a runtime argument (1)", "[ct_format]") {
-    auto x = 42;
+    auto x = 17;
     CHECK(stdx::ct_format<"Hello {} {}">(CX_VALUE(int), x) ==
-          stdx::format_result{"Hello int {}"_cts, stdx::make_tuple(42)});
+          stdx::format_result{"Hello int {}"_cts, stdx::make_tuple(17)});
     static_assert(
-        stdx::ct_format<"Hello {} {}">(CX_VALUE(int), 42) ==
-        stdx::format_result{"Hello int {}"_cts, stdx::make_tuple(42)});
+        stdx::ct_format<"Hello {} {}">(CX_VALUE(int), 17) ==
+        stdx::format_result{"Hello int {}"_cts, stdx::make_tuple(17)});
 }
 
 TEST_CASE("format a compile-time and a runtime argument (2)", "[ct_format]") {
@@ -141,13 +141,13 @@ TEST_CASE("format_to a different type", "[ct_format]") {
     static_assert(stdx::ct_format<"{}", string_constant>(CX_VALUE("A"sv)) ==
                   string_constant<char, 'A'>{});
 
-    auto x = 42;
+    auto x = 17;
     CHECK(stdx::ct_format<"{}", string_constant>(x) ==
           stdx::format_result{string_constant<char, '{', '}'>{},
-                              stdx::make_tuple(42)});
-    static_assert(stdx::ct_format<"{}", string_constant>(42) ==
+                              stdx::make_tuple(17)});
+    static_assert(stdx::ct_format<"{}", string_constant>(17) ==
                   stdx::format_result{string_constant<char, '{', '}'>{},
-                                      stdx::make_tuple(42)});
+                                      stdx::make_tuple(17)});
 }
 
 TEST_CASE("format a string-type argument", "[ct_format]") {

test/cx_map.cpp

@@ -44,8 +44,10 @@ TEST_CASE("const get", "[cx_map]") {
 TEST_CASE("update existing key", "[cx_map]") {
     auto t = stdx::cx_map<int, int, 64>{};
     t.put(13, 500);
-    t.put(13, 700);
+    REQUIRE(t.contains(13));
+    CHECK(t.get(13) == 500);
 
+    t.put(13, 700);
     REQUIRE(t.contains(13));
     CHECK(t.get(13) == 700);
 }

test/cx_multimap.cpp

@@ -50,13 +50,25 @@ TEST_CASE("erase values", "[cx_multimap]") {
 
     t.put(60, 1);
     t.put(60, 2);
-    t.put(60, 3);
+    t.put(61, 3);
+    t.put(62, 4);
+    CHECK(t.size() == 3);
 
-    t.erase(60, 2);
-    CHECK(t.size() == 1);
+    CHECK(t.erase(60, 1));
+    CHECK(t.size() == 3);
+    CHECK(not t.contains(60, 1));
+    CHECK(t.contains(60, 2));
+
+    CHECK(t.erase(60, 2));
+    CHECK(t.size() == 2);
     CHECK(not t.contains(60, 2));
+    CHECK(t.contains(61, 3));
+    CHECK(t.contains(62, 4));
 
-    t.erase(60);
+    t.erase(61);
+    CHECK(t.size() == 1);
+    CHECK(t.contains(62, 4));
+    t.erase(62);
     CHECK(t.empty());
 }
 

test/cx_set.cpp

@@ -45,8 +45,8 @@ TEST_CASE("insert multiple", "[cx_set]") {
     stdx::cx_set<int, 64> t;
 
     t.insert(10);
-    t.insert(10);
-
+    CHECK(t.size() == 1);
+    CHECK(not t.insert(10));
     CHECK(t.size() == 1);
     CHECK(not t.empty());
     CHECK(t.contains(10));
@@ -56,7 +56,7 @@ TEST_CASE("erase all", "[cx_set]") {
     stdx::cx_set<int, 64> t;
 
     t.insert(10);
-    t.erase(10);
+    CHECK(t.erase(10) == 1);
 
     CHECK(t.size() == 0);
     CHECK(t.empty());
@@ -67,6 +67,7 @@ TEST_CASE("erase some", "[cx_set]") {
     stdx::cx_set<int, 64> t;
 
     t.insert(10);
+    CHECK(t.contains(10));
     t.insert(11);
     t.erase(10);
 

test/cx_vector.cpp

@@ -172,9 +172,9 @@ TEST_CASE("resize_and_overwrite", "[cx_vector]") {
     stdx::cx_vector<int, 5> v{1, 2, 3, 4, 5};
     resize_and_overwrite(v, [](int *dest, std::size_t max_size) {
         CHECK(max_size == 5);
-        *dest = 42;
+        *dest = 17;
         return 1u;
     });
     REQUIRE(v.size() == 1u);
-    CHECK(v[0] == 42);
+    CHECK(v[0] == 17);
 }

test/function_traits.cpp

@@ -170,28 +170,29 @@ TEST_CASE("generic lambda arity", "[function_traits]") {
 }
 
 namespace {
-bool called_1{};
-bool called_2{};
+int called_1{};
+int called_2{};
 
 template <typename F>
 constexpr auto call_f(F f, stdx::priority_t<1>) -> stdx::return_t<F> {
-    called_1 = true;
+    ++called_1;
     return f();
 }
 
 template <typename F> constexpr auto call_f(F f, stdx::priority_t<0>) -> void {
-    called_2 = true;
+    ++called_2;
     f(0);
 }
 } // namespace
 
 TEST_CASE("SFINAE friendly", "[function_traits]") {
-    called_1 = false;
-    called_2 = false;
+    called_1 = 0;
+    called_2 = 0;
     auto f1 = []() -> int { return 1; };
     auto f2 = [](auto) -> void {};
     call_f(f1, stdx::priority<1>);
-    CHECK(called_1);
+    CHECK(called_1 == 1);
+    CHECK(called_2 == 0);
     call_f(f2, stdx::priority<1>);
-    CHECK(called_2);
+    CHECK(called_2 == 1);
 }

test/intrusive_forward_list.cpp

@@ -240,23 +240,23 @@ TEST_CASE("checked operation clears pointers on clear", "[intrusive_list]") {
 
 namespace {
 #if __cplusplus >= 202002L
-bool compile_time_call{};
+int compile_time_calls{};
 #else
-bool runtime_call{};
+int runtime_calls{};
 #endif
 
 struct injected_handler {
 #if __cplusplus >= 202002L
     template <stdx::ct_string Why, typename... Ts>
     static auto panic(Ts &&...) noexcept -> void {
         static_assert(std::string_view{Why} == "bad list node!");
-        compile_time_call = true;
+        ++compile_time_calls;
     }
 #else
     template <typename Why, typename... Ts>
     static auto panic(Why why, Ts &&...) noexcept -> void {
         CHECK(std::string_view{why} == "bad list node!");
-        runtime_call = true;
+        ++runtime_calls;
     }
 #endif
 };
@@ -270,31 +270,31 @@ TEST_CASE("checked panic when pushing populated node", "[intrusive_list]") {
     int_node n{5};
 
     n.next = &n;
-    compile_time_call = false;
+    compile_time_calls = 0;
     list.push_back(&n);
-    CHECK(compile_time_call);
+    CHECK(compile_time_calls == 1);
     list.pop_front();
 
     n.next = &n;
-    compile_time_call = false;
+    compile_time_calls = 0;
     list.push_front(&n);
-    CHECK(compile_time_call);
+    CHECK(compile_time_calls == 1);
 }
 #else
 TEST_CASE("checked panic when pushing populated node", "[intrusive_list]") {
     stdx::intrusive_forward_list<int_node> list{};
     int_node n{5};
 
     n.next = &n;
-    runtime_call = false;
+    runtime_calls = 0;
     list.push_back(&n);
-    CHECK(runtime_call);
+    CHECK(runtime_calls == 1);
     list.pop_front();
 
     n.next = &n;
-    runtime_call = false;
+    runtime_calls = 0;
     list.push_front(&n);
-    CHECK(runtime_call);
+    CHECK(runtime_calls == 1);
 }
 #endif