Redesign to support verification that a test file has been correctly

converted to 3c-regtest.

`llvm-lit b9_allsafestructp.c` passes, and
`./3c-regtest-unconvert.py b9_allsafestructp.c` prints its content as of
the test-command-refactoring.base branch.

Author: mattmccutchen-cci

clang/test/3C/3c-regtest-unconvert.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+#
+# 3c-regtest-unconvert.py can be used to verify that an existing 3C regression
+# test with a written-out RUN script has been correctly converted to use
+# 3c-regtest.py.
+#
+# Usage: 3c-regtest-unconvert.py TEST_FILE
+#
+# This verifies that the TEST_FILE contains a 3c-regtest.py RUN command in
+# canonical form and then prints the TEST_FILE with this RUN command replaced by
+# the generated RUN script. The output should be identical to the test file
+# before conversion to 3c-regtest.py.
+
+import sys
+import re
+import argparse
+
+import script_generator
+
+parser = argparse.ArgumentParser(description='''\
+Converts a 3C regression test that uses 3c-regtest.py back to one with a
+written-out RUN script so you can verify that a test was correctly converted to
+use 3c-regtest.py.\
+''')
+# TODO: Add help
+parser.add_argument('test_file')
+
+argobj = parser.parse_args()
+
+with open(argobj.test_file) as f:
+    lines = f.readlines()
+
+m = re.search(r"\A// RUN: %S/3c-regtest\.py (.*) %s -t %t --clang '%clang'\n\Z", lines[0])
+if m is None:
+    sys.exit('The first line of %s is not a canonical 3c-regtest.py RUN line.' % argobj.test_file)
+test_type_flags_joined = m.group(1)
+
+# FUTURE: Will we need to handle quoting?
+test_type_flags = test_type_flags_joined.split(' ')
+# XXX: This just exits on error. We'd like to add a more meaningful message, but
+# the default Python version on gamera (2.7.18) is too old to support
+# exit_on_error=False.
+test_type_argobj = script_generator.parser.parse_args(test_type_flags + [argobj.test_file])
+
+run_lines = ['// RUN: %s\n' % cmd for cmd in script_generator.generate_commands(test_type_argobj)]
+new_lines = run_lines + lines[1:]
+sys.stdout.write(''.join(new_lines))

clang/test/3C/3c-regtest.py

@@ -1,59 +1,52 @@
 #!/usr/bin/env python
 #
-# Usage: 3c-regtest -t TMPNAME [options...] SRC_FILE
+# 3c-regtest.py: Run a 3C regression test using a standard RUN script.
 #
-# TMPNAME corresponds to %t and SRC_FILE corresponds to %s. Both are required.
+# 3c-regtest.py is intended to be invoked by a single RUN command in the test
+# file. The canonical form is:
 #
-# --subst can be used for things such as %clang, for example (assuming single
-# --quotes are removed by the shell):
+# // RUN: %S/3c-regtest.py TEST_TYPE_FLAGS %s -t %t --clang '%clang'
 #
-# --subst %clang 'clang -some-flag'
+# 3c-regtest.py generates a RUN script based on the TEST_TYPE_FLAGS (using
+# script_generator.py) and runs it using code from `lit`.
 #
-# (Note: A literal % has to be represented as %% in a RUN line. If we instead
-# established the convention of automatically prepending the % here, then the
-# RUN line would trip the "Do not use 'clang' in tests, use '%clang'." error.)
-#
-# Example RUN line:
-#
-# // RUN: %S/3c-regtest.py -t %t --subst %%clang '%clang' %s
-#
-# Soon, we'll add options for different kinds of 3C regression tests.
+# The -t and --clang flags are used to pass substitution values from the outer
+# `lit` configuration so that 3c-regtest.py knows what to substitute for
+# occurrences of %t and %clang in its script. We'll add flags like this for all
+# % codes that appear in the scripts.
 
-# TODO: Add Windows compatibility code once we have an easy way to test on Windows.
+# TODO: Add Windows compatibility code once we have an easy way to test on
+# Windows.
 
 import sys
 import os
 import platform
 import argparse
 
+import script_generator
+
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)) +
                 '/../../../llvm/utils/lit')
 import lit.TestRunner
 
-print "NOTICE: cwd is %s" % os.getcwd()
-
-def die(msg):
-    sys.stderr.write('Error: %s\n' % msg)
-    sys.exit(1)
+parser = argparse.ArgumentParser(description='Run a 3C regression test.',
+                                 parents=[script_generator.parser])
+# Substitution arguments. The test file name is already in
+# script_generator.parser.
+parser.add_argument('-t')
+parser.add_argument('--clang')
 
-parser = argparse.ArgumentParser(description='Run a 3C regression test.')
-# TODO: Add help
-parser.add_argument('test_file')
-parser.add_argument('-t', required=True)
-parser.add_argument('--subst', action='append', nargs=2, default=[])
-args = parser.parse_args()
+argobj = parser.parse_args()
 
-test_dir = os.path.dirname(args.test_file)
-if test_dir == '':
-    test_dir = '.'
+test_dir = os.path.dirname(os.path.abspath(argobj.test_file))
 
-tmpName = args.t
-tmpNameSuffix = '.tmp'
-if tmpName.endswith(tmpNameSuffix):
-    tmpBase = tmpName[:-len(tmpNameSuffix)]
-else:
-    die('-t argument %s does not end with %s' % (tmpName, tmpNameSuffix))
+tmpName = argobj.t
+tmpBase = script_generator.remove_suffix(tmpName, '.tmp')
+if tmpBase is None:
+    sys.exit('-t argument %s does not end with .tmp' % tmpName)
 
+# `lit` supports more substitutions, but these are the only ones needed by the
+# tests that use 3c-regtest.py so far.
 substitutions = [
     # #_MARKER_# is a hack copied from getDefaultSubstitutions in
     # llvm/utils/lit/lit/TestRunner.py. To explain it a bit more fully:
@@ -66,23 +59,14 @@ def die(msg):
     # over the input from left to right, replacing codes as they are found, but
     # apparently that wasn't worth the extra code in `lit`.
     ('%%', '#_MARKER_#'),
-    ('%s', args.test_file),
+    ('%s', argobj.test_file),
     ('%S', test_dir),
     ('%t', tmpName),
+    ('%clang', argobj.clang),
+    ('#_MARKER_#', '%')
 ]
-substitutions.extend(args.subst)
-substitutions.append(('#_MARKER_#', '%'))
 
-# Starting with processor.py because it's always the same.
-commands = [
-    # FIXME: 'foo.c' + 'hecked.c' is a terrible hack; find the right way to do this.
-    '3c -alltypes -addcr %s -- | FileCheck -match-full-lines -check-prefixes="CHECK_ALL","CHECK" %s',
-    '3c -addcr %s -- | FileCheck -match-full-lines -check-prefixes="CHECK_NOALL","CHECK" %s',
-    '3c -addcr %s -- | %clang -c -fcheckedc-extension -x c -o /dev/null -',
-    '3c -output-postfix=checked -alltypes %s',
-    '3c -alltypes %shecked.c -- | count 0',
-    'rm %shecked.c',
-]
+commands = script_generator.generate_commands(argobj)
 commands = lit.TestRunner.applySubstitutions(commands, substitutions)
 
 class FakeTestConfig:

clang/test/3C/b9_allsafestructp.c

@@ -1,4 +1,4 @@
-// RUN: %S/3c-regtest.py -t %t --subst %%clang '%clang' %s
+// RUN: %S/3c-regtest.py --predefined-script processor %s -t %t --clang '%clang'
 #include <stddef.h>
 #include <stddef.h>
 extern _Itype_for_any(T) void *calloc(size_t nmemb, size_t size) : itype(_Array_ptr<T>) byte_count(nmemb * size);

clang/test/3C/script_generator.py

@@ -0,0 +1,53 @@
+# script_generator.py: Generates an unexpanded RUN script for a test file based
+# on its name and flags that specify the kind of testing to be done. Used by
+# both 3c-regtest.py and 3c-regtest-unconvert.py.
+
+import os
+import re
+import argparse
+
+def remove_suffix(string, suffix):
+    return string[:-len(suffix)] if string.endswith(suffix) else None
+
+parser = argparse.ArgumentParser(
+    # This will be shown as if it is a program name runnable by the user. It
+    # isn't, but give the user a clue what this argument parser represents.
+    prog='<3c-regtest script generator>',
+    # Without this, the parser in 3c-regtest.py raises an error about
+    # conflicting --help options. That's silly; one would expect argparse to
+    # handle this specially.
+    add_help=False
+)
+# TODO: Add help
+parser.add_argument('test_file')
+# Future: Redesign in terms of kinds of testing rather than merely selecting one
+# of the existing 5 scripts of the test updating programs.
+parser.add_argument('--predefined-script', required=True,
+                    choices=['processor'])
+
+# Unlike all the other % codes that are substituted by `lit`, %B is a special
+# code that generate_script (below) replaces with the basename of the test file
+# without the `.c` extension. This reproduces some of the scripts in existing
+# test files.
+predefined_scripts = {
+    'processor': '''\
+3c -alltypes -addcr %s -- | FileCheck -match-full-lines -check-prefixes="CHECK_ALL","CHECK" %s
+3c -addcr %s -- | FileCheck -match-full-lines -check-prefixes="CHECK_NOALL","CHECK" %s
+3c -addcr %s -- | %clang -c -fcheckedc-extension -x c -o /dev/null -
+3c -output-postfix=checked -alltypes %s
+3c -alltypes %S/%B.checked.c -- | count 0
+rm %S/%B.checked.c
+''',
+}
+
+# `argobj` should be an argument object returned by `parser` or a parser based
+# on it. Returns a list of command strings.
+def generate_commands(argobj):
+    test_file = argobj.test_file
+    # FUTURE: If this assumption is no longer valid, update this code.
+    test_file_basename_without_ext = remove_suffix(os.path.basename(test_file), '.c')
+    if test_file_basename_without_ext is None:
+        sys.exit('Test filename does not end in `.c`.')
+    script = remove_suffix(predefined_scripts[argobj.predefined_script], '\n')
+    script = re.sub('%B', test_file_basename_without_ext, script)
+    return script.split('\n')