improve reporting of failing tests and tests missing a result file

Author: dirk-thomas

ament_cmake_test/cmake/run_test.py

@@ -26,6 +26,7 @@
 import subprocess
 from xml.etree.ElementTree import ElementTree
 from xml.etree.ElementTree import ParseError
+from xml.sax.saxutils import quoteattr
 
 
 def main(argv=sys.argv[1:]):
@@ -74,66 +75,93 @@ def main(argv=sys.argv[1:]):
         args.result_file,
         'The test did not generate a result file.'
     )
-    with open(args.result_file, 'w') as f:
-        f.write(failure_result_file)
+    with open(args.result_file, 'w') as h:
+        h.write(failure_result_file)
 
-    print("-- run_test.py: invoke following command in '%s':\n - %s" %
+    print("-- run_test.py: invoking following command in '%s':\n - %s" %
           (os.getcwd(), ' '.join(args.command)))
 
+    # collect output / exception to generate more detailed result file
+    # if the command fails to generate it
+    output = ''
     h = None
     if args.output_file:
         h = open(args.output_file, 'wb')
     try:
         proc = subprocess.Popen(args.command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
         for line in proc.stdout:
-            print(line.decode(), end='')
+            decoded_line = line.decode()
+            print(decoded_line, end='')
+            output += decoded_line
             if h:
                 h.write(line)
+        proc.wait()
         rc = proc.returncode
+        print('-- run_test.py: return code', rc, file=sys.stderr if rc else sys.stdout)
+    except Exception as e:
+        print('-- run_test.py: invocation failed:', str(e), file=sys.stderr)
+        output += str(e)
+        rc = 1
     finally:
         if h:
             h.close()
 
-    print("-- run_test.py: verify result file '%s'" % args.result_file)
-
-    if os.path.exists(args.result_file):
-        # if result file exists ensure that it contains valid xml
-        # unit test suites are not good about screening out
-        # illegal unicode characters
-        tree = None
-        try:
-            tree = ElementTree(None, args.result_file)
-        except ParseError as e:
-            modified = _tidy_xml(args.result_file)
-            if not modified:
-                print("Invalid XML in result file '%s': %s" %
-                      (args.result_file, str(e)), file=sys.stderr)
-            else:
-                try:
-                    tree = ElementTree(None, args.result_file)
-                except ParseError as e:
-                    print("Invalid XML in result file '%s' "
-                          "(even after trying to tidy it): %s" %
-                          (args.result_file, str(e)), file=sys.stderr)
-
-        if not tree:
-            # set error code when result file is not parsable
-            rc = 1
-        else:
-            # set error code when result file contains errors or failures
-            root = tree.getroot()
-            num_errors = int(root.attrib.get('errors', 0))
-            num_failures = int(root.attrib.get('failures', 0))
-            if num_errors or num_failures:
-                rc = 1
-
-    elif not rc and args.generate_result_on_success:
+    if not rc and args.generate_result_on_success:
         # generate result file with one passed test
         # if it was expected that no result file was generated
         # and the command returned with code zero
+        print("-- run_test.py: generate result file '%s' with successful test" % args.result_file)
         success_result_file = _generate_result(args.result_file)
-        with open(args.result_file, 'w') as f:
-            f.write(success_result_file)
+        with open(args.result_file, 'w') as h:
+            h.write(success_result_file)
+
+    elif os.path.exists(args.result_file):
+        # check if content of result file has actually changed
+        with open(args.result_file, 'r') as h:
+            not_changed = h.read() == failure_result_file
+
+        if not_changed:
+            print("-- run_test.py: generate result file '%s' with failed test" % args.result_file,
+                  file=sys.stderr)
+            # regenerate result file to include output / exception of the invoked command
+            failure_result_file = _generate_result(
+                args.result_file,
+                'The test did not generate a result file:\n\n' + output
+            )
+            with open(args.result_file, 'w') as h:
+                h.write(failure_result_file)
+
+        else:
+            print("-- run_test.py: verify result file '%s'" % args.result_file)
+            # if result file exists ensure that it contains valid xml
+            # unit test suites are not good about screening out
+            # illegal unicode characters
+            tree = None
+            try:
+                tree = ElementTree(None, args.result_file)
+            except ParseError as e:
+                modified = _tidy_xml(args.result_file)
+                if not modified:
+                    print("Invalid XML in result file '%s': %s" %
+                          (args.result_file, str(e)), file=sys.stderr)
+                else:
+                    try:
+                        tree = ElementTree(None, args.result_file)
+                    except ParseError as e:
+                        print("Invalid XML in result file '%s' "
+                              "(even after trying to tidy it): %s" %
+                              (args.result_file, str(e)), file=sys.stderr)
+
+            if not tree:
+                # set error code when result file is not parsable
+                rc = 1
+            else:
+                # set error code when result file contains errors or failures
+                root = tree.getroot()
+                num_errors = int(root.attrib.get('errors', 0))
+                num_failures = int(root.attrib.get('failures', 0))
+                if num_errors or num_failures:
+                    rc = 1
 
     # ensure that a result file exists at the end
     if not rc and not os.path.exists(args.result_file):
@@ -147,24 +175,22 @@ def main(argv=sys.argv[1:]):
 def _generate_result(result_file, failure_message=None):
     pkgname = os.path.basename(os.path.dirname(result_file))
     testname = os.path.splitext(os.path.basename(result_file))[0]
-    name = '%s__%s' % (pkgname, testname)
-    failure_message = '<failure message="%s" type=""/>' % failure_message \
+    name = '%s.%s' % (pkgname, testname)
+    failure_message = '<failure message=%s/>' % quoteattr(failure_message) \
         if failure_message else ''
     return '''<?xml version="1.0" encoding="UTF-8"?>
 <testsuite tests="1" failures="%d" time="1" errors="0" name="%s">
-  <testcase name="%s" status="run" time="1" classname="Results">
+  <testcase name="missing_result" status="run" time="1" classname="%s">
     %s
   </testcase>
-</testsuite>''' % (1 if failure_message else 0,
-                   name,
-                   name,
-                   failure_message)
+</testsuite>\n''' % \
+        (1 if failure_message else 0, name, name, failure_message)
 
 
 def _tidy_xml(filename):
     assert os.path.isfile(filename)
 
-    # try reading utf-8 firth then iso
+    # try reading utf-8 first then iso
     # this is ugly but the files in question do not declare a unicode type
     data = None
     for encoding in ['utf-8', 'iso8859-1']:
@@ -200,8 +226,8 @@ def _tidy_xml(filename):
     for match in SAFE_XML_REGEX.finditer(data):
         data = data[:match.start()] + '?' + data[match.end():]
 
-    with open(filename, 'w') as f:
-        f.write(data)
+    with open(filename, 'w') as h:
+        h.write(data)
     return True