diff --git a/gaitmap-bench/tests/test_result_loading.py b/gaitmap-bench/tests/test_result_loading.py
new file mode 100644
index 0000000..cd4b823
--- /dev/null
+++ b/gaitmap-bench/tests/test_result_loading.py
@@ -0,0 +1,16 @@
+from gaitmap_bench import is_config_set, config
+from gaitmap_bench.docu_utils import set_docs_config
+from gaitmap_challenges.results import get_all_result_paths
+from gaitmap_challenges.stride_segmentation.egait_segmentation_validation_2014_original_label import Challenge
+import pytest
+
+
+@pytest.fixture(scope="module", autouse=True)
+def _config():
+    is_config_set() or set_docs_config()
+
+
+def test_multi_version_results(recwarn):
+    assert len(recwarn) == 0
+    get_all_result_paths(Challenge, config().results_dir)
+    assert len(recwarn) == 0
diff --git a/gaitmap-challenges/gaitmap_challenges/results.py b/gaitmap-challenges/gaitmap_challenges/results.py
index b259cec..a6821bd 100644
--- a/gaitmap-challenges/gaitmap_challenges/results.py
+++ b/gaitmap-challenges/gaitmap_challenges/results.py
@@ -85,9 +85,11 @@ def get_all_result_paths(
         ) from e
 
     entries = {}
+    versions = {}
     for run in folder.rglob("metadata.json"):
         meta = load_run_metadata(run.parent)
         entries.setdefault(tuple(meta["entry_name"]), []).append(run.parent)
+        versions.setdefault(tuple(meta["entry_name"]), []).append(meta["challenge_version"])
 
     if len(entries) == 0:
         raise ValueError(
@@ -97,16 +99,22 @@ def get_all_result_paths(
 
     sorted_entries = {}
     for name, entry_list in entries.items():
-        parents = {e.parent for e in entry_list}
-        if len(parents) > 1:
-            warnings.warn(
-                f"We found results from the same entry name ({name}) in different folders. "
-                "This could indicate that you forgot to correctly name one of your entries when you saved a "
-                "run, or that files/folders where manually copied around. "
-                "Both can lead to issues. "
-                "Please double check your results folder.",
-                stacklevel=2,
-            )
+        versions_per_entry = versions[name]
+        parents_per_version = {}
+        for version, entry in zip(versions_per_entry, entry_list):
+            parents_per_version.setdefault(version, []).append(entry.parent)
+
+        for version, parents in parents_per_version.items():
+            if len(set(parents)) > 1:
+                warnings.warn(
+                    f"We found results from the same entry name ({name}) for the same challenge version "
+                    f"({version}) in different folders. "
+                    "This could indicate that you forgot to correctly name one of your entries when you saved a "
+                    "run, or that files/folders where manually copied around. "
+                    "Both can lead to issues. "
+                    "Please double check your results folder.",
+                    stacklevel=1,
+                )
         sorted_entries[name] = sorted(entry_list, key=lambda e: e.name, reverse=False)
 
     return entries