Skip to content

Commit 7976eaf

Browse files
elbenomjcaisse-intel
elbeno
authored and
mjcaisse-intel
committed
👷 Add mutation tests
Problem: - Quis custodiet ipsos custodies? (Who tests the tests?) Solution: - Mutation testing tests the tests. Note: - To run mutation tests, build the `mull_tests` target. - Mull tests are run as a CI job but this doesn't gate `merge_ok`. - The CI job runs mutations tests on ubuntu-22.04, LLVM 15, mull v0.23.0. This is the current "latest version" taking into consideration the constraints. When we run on ubuntu-24.04, we can use LLVM 17 for mull. And later versions as they are supported.
1 parent 5cb910f commit 7976eaf

19 files changed

+233
-125
lines changed

‎.github/workflows/unit_tests.yml

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ env:
1515
DEFAULT_CXX_STANDARD: 20
1616
DEFAULT_LLVM_VERSION: 18
1717
DEFAULT_GCC_VERSION: 13
18+
MULL_LLVM_VERSION: 15
1819

1920
concurrency:
2021
group: ${{ github.head_ref || github.run_id }}
@@ -348,6 +349,52 @@ jobs:
348349
echo "</details>" >> $GITHUB_STEP_SUMMARY
349350
test $FAILSIZE = "0"
350351
352+
mutate:
353+
runs-on: ${{ github.repository_owner == 'intel' && 'intel-' || '' }}ubuntu-22.04
354+
steps:
355+
- uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
356+
357+
- name: Install build tools
358+
run: |
359+
wget https://apt.llvm.org/llvm.sh && chmod +x llvm.sh && sudo ./llvm.sh ${{env.MULL_LLVM_VERSION}}
360+
sudo apt install -y ninja-build
361+
362+
- name: Install mull
363+
env:
364+
MULL_VERSION: 0.23.0
365+
run: |
366+
wget https://github.com/mull-project/mull/releases/download/${{env.MULL_VERSION}}/Mull-${{env.MULL_LLVM_VERSION}}-${{env.MULL_VERSION}}-LLVM-${{env.MULL_LLVM_VERSION}}.0-ubuntu-22.04.deb
367+
sudo dpkg -i Mull-${{env.MULL_LLVM_VERSION}}-${{env.MULL_VERSION}}-LLVM-${{env.MULL_LLVM_VERSION}}.0-ubuntu-22.04.deb
368+
369+
- name: Restore CPM cache
370+
env:
371+
cache-name: cpm-cache-0
372+
id: cpm-cache-restore
373+
uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
374+
with:
375+
path: ~/cpm-cache
376+
key: ${{runner.os}}-${{env.cache-name}}-${{ hashFiles('**/CMakeLists.txt', 'cmake/**') }}
377+
restore-keys: |
378+
${{runner.os}}-${{env.cache-name}}-
379+
380+
- name: Configure CMake
381+
env:
382+
CC: "/usr/lib/llvm-${{env.MULL_LLVM_VERSION}}/bin/clang"
383+
CXX: "/usr/lib/llvm-${{env.MULL_LLVM_VERSION}}/bin/clang++"
384+
run: cmake -B ${{github.workspace}}/build -DCMAKE_CXX_STANDARD=${{env.DEFAULT_CXX_STANDARD}} -DCPM_SOURCE_CACHE=~/cpm-cache
385+
386+
- name: Save CPM cache
387+
env:
388+
cache-name: cpm-cache-0
389+
if: steps.cpm-cache-restore.outputs.cache-hit != 'true'
390+
uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
391+
with:
392+
path: ~/cpm-cache
393+
key: ${{runner.os}}-${{env.cache-name}}-${{ hashFiles('**/CMakeLists.txt', 'cmake/**') }}
394+
395+
- name: Build and run mull tests
396+
run: cmake --build build -t mull_tests
397+
351398
merge_ok:
352399
runs-on: ${{ github.repository_owner == 'intel' && 'intel-' || '' }}ubuntu-22.04
353400
needs: [build_and_test, quality_checks_pass, sanitize, valgrind]

‎mull.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
mutators:
2+
- cxx_all
3+
ignoreMutators:
4+
- cxx_remove_void_call
5+
excludePaths:
6+
- .*/include/stdx/.*
7+
timeout: 10000

‎test/CMakeLists.txt

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
function(add_tests)
2-
foreach(name ${ARGN})
2+
set(multiValueArgs FILES MULL_EXCLUSIONS)
3+
cmake_parse_arguments(TEST "" "" "${multiValueArgs}" ${ARGN})
4+
5+
foreach(name ${TEST_FILES})
36
add_unit_test(
47
"${name}_test"
58
CATCH2
@@ -8,10 +11,14 @@ function(add_tests)
811
LIBRARIES
912
warnings
1013
stdx)
14+
if(NOT name IN_LIST TEST_MULL_EXCLUSIONS)
15+
add_mull_test("${name}_test" EXCLUDE_CTEST)
16+
endif()
1117
endforeach()
1218
endfunction()
1319

1420
add_tests(
21+
FILES
1522
algorithm
1623
always_false
1724
bind
@@ -53,7 +60,7 @@ add_tests(
5360
udls)
5461

5562
if(${CMAKE_CXX_STANDARD} GREATER_EQUAL 20)
56-
add_tests(ct_format ct_string indexed_tuple tuple tuple_algorithms)
63+
add_tests(FILES ct_format ct_string indexed_tuple tuple tuple_algorithms)
5764
endif()
5865

5966
add_subdirectory(fail)

‎test/ct_format.cpp

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -67,20 +67,20 @@ TEST_CASE("format a compile-time enum argument", "[ct_format]") {
6767
}
6868

6969
TEST_CASE("format a runtime argument", "[ct_format]") {
70-
auto x = 42;
70+
auto x = 17;
7171
CHECK(stdx::ct_format<"Hello {}">(x) ==
72-
stdx::format_result{"Hello {}"_cts, stdx::make_tuple(42)});
73-
static_assert(stdx::ct_format<"Hello {}">(42) ==
74-
stdx::format_result{"Hello {}"_cts, stdx::make_tuple(42)});
72+
stdx::format_result{"Hello {}"_cts, stdx::make_tuple(17)});
73+
static_assert(stdx::ct_format<"Hello {}">(17) ==
74+
stdx::format_result{"Hello {}"_cts, stdx::make_tuple(17)});
7575
}
7676

7777
TEST_CASE("format a compile-time and a runtime argument (1)", "[ct_format]") {
78-
auto x = 42;
78+
auto x = 17;
7979
CHECK(stdx::ct_format<"Hello {} {}">(CX_VALUE(int), x) ==
80-
stdx::format_result{"Hello int {}"_cts, stdx::make_tuple(42)});
80+
stdx::format_result{"Hello int {}"_cts, stdx::make_tuple(17)});
8181
static_assert(
82-
stdx::ct_format<"Hello {} {}">(CX_VALUE(int), 42) ==
83-
stdx::format_result{"Hello int {}"_cts, stdx::make_tuple(42)});
82+
stdx::ct_format<"Hello {} {}">(CX_VALUE(int), 17) ==
83+
stdx::format_result{"Hello int {}"_cts, stdx::make_tuple(17)});
8484
}
8585

8686
TEST_CASE("format a compile-time and a runtime argument (2)", "[ct_format]") {
@@ -141,13 +141,13 @@ TEST_CASE("format_to a different type", "[ct_format]") {
141141
static_assert(stdx::ct_format<"{}", string_constant>(CX_VALUE("A"sv)) ==
142142
string_constant<char, 'A'>{});
143143

144-
auto x = 42;
144+
auto x = 17;
145145
CHECK(stdx::ct_format<"{}", string_constant>(x) ==
146146
stdx::format_result{string_constant<char, '{', '}'>{},
147-
stdx::make_tuple(42)});
148-
static_assert(stdx::ct_format<"{}", string_constant>(42) ==
147+
stdx::make_tuple(17)});
148+
static_assert(stdx::ct_format<"{}", string_constant>(17) ==
149149
stdx::format_result{string_constant<char, '{', '}'>{},
150-
stdx::make_tuple(42)});
150+
stdx::make_tuple(17)});
151151
}
152152

153153
TEST_CASE("format a string-type argument", "[ct_format]") {

‎test/cx_map.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,10 @@ TEST_CASE("const get", "[cx_map]") {
4444
TEST_CASE("update existing key", "[cx_map]") {
4545
auto t = stdx::cx_map<int, int, 64>{};
4646
t.put(13, 500);
47-
t.put(13, 700);
47+
REQUIRE(t.contains(13));
48+
CHECK(t.get(13) == 500);
4849

50+
t.put(13, 700);
4951
REQUIRE(t.contains(13));
5052
CHECK(t.get(13) == 700);
5153
}

‎test/cx_multimap.cpp

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,13 +50,25 @@ TEST_CASE("erase values", "[cx_multimap]") {
5050

5151
t.put(60, 1);
5252
t.put(60, 2);
53-
t.put(60, 3);
53+
t.put(61, 3);
54+
t.put(62, 4);
55+
CHECK(t.size() == 3);
5456

55-
t.erase(60, 2);
56-
CHECK(t.size() == 1);
57+
CHECK(t.erase(60, 1));
58+
CHECK(t.size() == 3);
59+
CHECK(not t.contains(60, 1));
60+
CHECK(t.contains(60, 2));
61+
62+
CHECK(t.erase(60, 2));
63+
CHECK(t.size() == 2);
5764
CHECK(not t.contains(60, 2));
65+
CHECK(t.contains(61, 3));
66+
CHECK(t.contains(62, 4));
5867

59-
t.erase(60);
68+
t.erase(61);
69+
CHECK(t.size() == 1);
70+
CHECK(t.contains(62, 4));
71+
t.erase(62);
6072
CHECK(t.empty());
6173
}
6274

‎test/cx_set.cpp

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ TEST_CASE("insert multiple", "[cx_set]") {
4545
stdx::cx_set<int, 64> t;
4646

4747
t.insert(10);
48-
t.insert(10);
49-
48+
CHECK(t.size() == 1);
49+
CHECK(not t.insert(10));
5050
CHECK(t.size() == 1);
5151
CHECK(not t.empty());
5252
CHECK(t.contains(10));
@@ -56,7 +56,7 @@ TEST_CASE("erase all", "[cx_set]") {
5656
stdx::cx_set<int, 64> t;
5757

5858
t.insert(10);
59-
t.erase(10);
59+
CHECK(t.erase(10) == 1);
6060

6161
CHECK(t.size() == 0);
6262
CHECK(t.empty());
@@ -67,6 +67,7 @@ TEST_CASE("erase some", "[cx_set]") {
6767
stdx::cx_set<int, 64> t;
6868

6969
t.insert(10);
70+
CHECK(t.contains(10));
7071
t.insert(11);
7172
t.erase(10);
7273

‎test/cx_vector.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,9 +172,9 @@ TEST_CASE("resize_and_overwrite", "[cx_vector]") {
172172
stdx::cx_vector<int, 5> v{1, 2, 3, 4, 5};
173173
resize_and_overwrite(v, [](int *dest, std::size_t max_size) {
174174
CHECK(max_size == 5);
175-
*dest = 42;
175+
*dest = 17;
176176
return 1u;
177177
});
178178
REQUIRE(v.size() == 1u);
179-
CHECK(v[0] == 42);
179+
CHECK(v[0] == 17);
180180
}

‎test/function_traits.cpp

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -170,28 +170,29 @@ TEST_CASE("generic lambda arity", "[function_traits]") {
170170
}
171171

172172
namespace {
173-
bool called_1{};
174-
bool called_2{};
173+
int called_1{};
174+
int called_2{};
175175

176176
template <typename F>
177177
constexpr auto call_f(F f, stdx::priority_t<1>) -> stdx::return_t<F> {
178-
called_1 = true;
178+
++called_1;
179179
return f();
180180
}
181181

182182
template <typename F> constexpr auto call_f(F f, stdx::priority_t<0>) -> void {
183-
called_2 = true;
183+
++called_2;
184184
f(0);
185185
}
186186
} // namespace
187187

188188
TEST_CASE("SFINAE friendly", "[function_traits]") {
189-
called_1 = false;
190-
called_2 = false;
189+
called_1 = 0;
190+
called_2 = 0;
191191
auto f1 = []() -> int { return 1; };
192192
auto f2 = [](auto) -> void {};
193193
call_f(f1, stdx::priority<1>);
194-
CHECK(called_1);
194+
CHECK(called_1 == 1);
195+
CHECK(called_2 == 0);
195196
call_f(f2, stdx::priority<1>);
196-
CHECK(called_2);
197+
CHECK(called_2 == 1);
197198
}

‎test/intrusive_forward_list.cpp

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -240,23 +240,23 @@ TEST_CASE("checked operation clears pointers on clear", "[intrusive_list]") {
240240

241241
namespace {
242242
#if __cplusplus >= 202002L
243-
bool compile_time_call{};
243+
int compile_time_calls{};
244244
#else
245-
bool runtime_call{};
245+
int runtime_calls{};
246246
#endif
247247

248248
struct injected_handler {
249249
#if __cplusplus >= 202002L
250250
template <stdx::ct_string Why, typename... Ts>
251251
static auto panic(Ts &&...) noexcept -> void {
252252
static_assert(std::string_view{Why} == "bad list node!");
253-
compile_time_call = true;
253+
++compile_time_calls;
254254
}
255255
#else
256256
template <typename Why, typename... Ts>
257257
static auto panic(Why why, Ts &&...) noexcept -> void {
258258
CHECK(std::string_view{why} == "bad list node!");
259-
runtime_call = true;
259+
++runtime_calls;
260260
}
261261
#endif
262262
};
@@ -270,31 +270,31 @@ TEST_CASE("checked panic when pushing populated node", "[intrusive_list]") {
270270
int_node n{5};
271271

272272
n.next = &n;
273-
compile_time_call = false;
273+
compile_time_calls = 0;
274274
list.push_back(&n);
275-
CHECK(compile_time_call);
275+
CHECK(compile_time_calls == 1);
276276
list.pop_front();
277277

278278
n.next = &n;
279-
compile_time_call = false;
279+
compile_time_calls = 0;
280280
list.push_front(&n);
281-
CHECK(compile_time_call);
281+
CHECK(compile_time_calls == 1);
282282
}
283283
#else
284284
TEST_CASE("checked panic when pushing populated node", "[intrusive_list]") {
285285
stdx::intrusive_forward_list<int_node> list{};
286286
int_node n{5};
287287

288288
n.next = &n;
289-
runtime_call = false;
289+
runtime_calls = 0;
290290
list.push_back(&n);
291-
CHECK(runtime_call);
291+
CHECK(runtime_calls == 1);
292292
list.pop_front();
293293

294294
n.next = &n;
295-
runtime_call = false;
295+
runtime_calls = 0;
296296
list.push_front(&n);
297-
CHECK(runtime_call);
297+
CHECK(runtime_calls == 1);
298298
}
299299
#endif
300300

0 commit comments

Comments
 (0)