twister: remove fallback testcase if cases added at runtime

The fallback testcase, which is the scenario name from the yaml file is
added if we did not discover any testcases via parsing. It can be
removed if we discover testcaes at runtime or otherwise it is added as
skipped...

Signed-off-by: Anas Nashif <[email protected]>

Author: nashif

scripts/pylib/twister/twisterlib.py

@@ -1705,13 +1705,15 @@ def __eq__(self, other):
                  sorted(other.ztest_suite_names)))
 
 class TestCase(DisablePyTestCollectionMixin):
+
     def __init__(self, name=None, testsuite=None):
         self.duration = 0
         self.name = name
         self.status = None
         self.reason = None
         self.testsuite = testsuite
         self.output = ""
+        self.freeform = False
 
     def __lt__(self, other):
         return self.name < other.name
@@ -1781,8 +1783,9 @@ def __init__(self, testsuite_root, workdir, name):
         self.ztest_suite_names = []
 
 
-    def add_testcase(self, name):
+    def add_testcase(self, name, freeform=False):
         tc = TestCase(name=name, testsuite=self)
+        tc.freeform = freeform
         self.testcases.append(tc)
 
     @staticmethod
@@ -2058,7 +2061,7 @@ def parse_subcases(self, test_path):
                 self.add_testcase(name)
 
             if not subcases:
-                self.add_testcase(self.id)
+                self.add_testcase(self.id, freeform=True)
 
         self.ztest_suite_names = ztest_suite_names
 
@@ -2113,7 +2116,7 @@ def __init__(self, testsuite, platform, outdir):
     # Fix an issue with copying objects from testsuite, need better solution.
     def init_cases(self):
         for c in self.testsuite.testcases:
-            self.add_testcase(c.name)
+            self.add_testcase(c.name, freeform=c.freeform)
 
     def _get_run_id(self):
         """ generate run id from instance unique identifier and a random
@@ -2150,8 +2153,9 @@ def set_case_status_by_name(self, name, status, reason=None):
             tc.reason = reason
         return tc
 
-    def add_testcase(self, name):
+    def add_testcase(self, name, freeform=False):
         tc = TestCase(name=name)
+        tc.freeform = freeform
         self.testcases.append(tc)
         return tc
 
@@ -4124,7 +4128,6 @@ def json_report(self, filename, version="NA"):
             if instance.status is not None:
                 suite["execution_time"] =  f"{float(handler_time):.2f}"
 
-
             testcases = []
 
             if len(instance.testcases) == 1:
@@ -4133,6 +4136,12 @@ def json_report(self, filename, version="NA"):
                 single_case_duration = 0
 
             for case in instance.testcases:
+                # freeform was set when no sub testcases were parsed, however,
+                # if we discover those at runtime, the fallback testcase wont be
+                # needed anymore and can be removed from the output, it does
+                # not have a status and would otherwise be reported as skipped.
+                if case.freeform and case.status is None and len(instance.testcases) > 1:
+                    continue
                 testcase = {}
                 testcase['identifier'] = case.name
                 if instance.status: