From 658df958cb0c2f486a8da931f160de8d38e65247 Mon Sep 17 00:00:00 2001
From: "Brian St. Pierre" <brian@bstpierre.org>
Date: Fri, 28 Sep 2018 22:22:03 -0400
Subject: [PATCH] Add support for running list of tests in manifest

Adding `"test_list": ["test1", "test2"]` to the keys in the manifest
declares a list of STF test targets.

Running `p4app test myapp.p4app` will run all of the tests in the list, and
report a summary at the end, in the form:

    TEST SUMMARY
    test1: PASS
    test2: FAIL
---
 docker/scripts/p4apprunner.py | 44 +++++++++++++++++++++++++++++------
 p4app                         | 27 +++++++++++++++++++++
 2 files changed, 64 insertions(+), 7 deletions(-)

diff --git a/docker/scripts/p4apprunner.py b/docker/scripts/p4apprunner.py
index 30f35b7..a081acf 100755
--- a/docker/scripts/p4apprunner.py
+++ b/docker/scripts/p4apprunner.py
@@ -29,6 +29,8 @@
                     action='store_true', required=False, default=False)
 parser.add_argument('--build-only', help='Compile the program, but do not run it.',
                     action='store_true', required=False, default=False)
+parser.add_argument('--tests', help='Run all tests listed in the manifest.',
+                    action='store_true', required=False, default=False)
 parser.add_argument('--json', help='Use this compiled JSON file instead of compiling.',
                     type=str, action='store', required=False, default=None)
 parser.add_argument('--manifest', help='Path to manifest file.',
@@ -52,13 +54,19 @@ def run_command(command):
     return os.WEXITSTATUS(os.system(command))
 
 class Manifest:
-    def __init__(self, program_file, language, target, target_config):
+    def __init__(self, program_file, language, target, targets, tests=()):
         self.program_file = program_file
         self.language = language
         self.target = target
-        self.target_config = target_config
+        self.targets = targets
+        self.tests = tests
+
+    @property
+    def target_config(self):
+        return self.targets[self.target]
 
-def read_manifest(manifest_file):
+
+def read_manifest(manifest_file, target):
     manifest = json.load(manifest_file, object_pairs_hook=OrderedDict)
 
     if 'program' not in manifest:
@@ -75,8 +83,8 @@ def read_manifest(manifest_file):
         log_error('No targets defined in manifest.')
         sys.exit(1)
 
-    if args.target is not None:
-        chosen_target = args.target
+    if target is not None:
+        chosen_target = target
     elif 'default-target' in manifest:
         chosen_target = manifest['default-target']
     else:
@@ -86,7 +94,11 @@ def read_manifest(manifest_file):
         log_error('Target not found in manifest:', chosen_target)
         sys.exit(1)
 
-    return Manifest(program_file, language, chosen_target, manifest['targets'][chosen_target])
+    return Manifest(program_file, language,
+                    target=chosen_target,
+                    targets=manifest['targets'],
+                    tests=manifest.get('test_list', []))
+
 
 def get_program_name(program_file):
     return os.path.basename(program_file).rstrip('.p4')
@@ -137,6 +149,21 @@ def run_compile_bmv2(manifest):
 
     return output_file
 
+
+def run_tests(manifest):
+    results = {}
+    for test in manifest.tests:
+        print('{0}\n!!! Running test: {1}\n{0}\n'.format(
+            '!' * 70, test))
+        manifest.target = test
+        rv = run_stf(manifest)
+        results[test] = rv
+    print('TEST SUMMARY')
+    for test, rv in results.iteritems():
+        print('{}: {}'.format(test, 'FAIL' if rv else 'PASS'))
+    return 0
+
+
 def run_mininet(manifest):
     output_file = run_compile_bmv2(manifest)
 
@@ -292,7 +319,7 @@ def main():
 
     log('Reading package manifest.')
     with open(args.manifest, 'r') as manifest_file:
-        manifest = read_manifest(manifest_file)
+        manifest = read_manifest(manifest_file, args.target)
 
     # Dispatch to the backend implementation for this target.
     backend = manifest.target
@@ -302,6 +329,8 @@ def main():
     if args.build_only or backend == 'compile-bmv2':
         build_only(manifest)
         rc = 0
+    elif args.tests:
+        rc = run_tests(manifest)
     elif backend == 'mininet':
         rc = run_mininet(manifest)
     elif backend == 'multiswitch':
@@ -317,4 +346,5 @@ def main():
     sys.exit(rc)
 
 if __name__ == '__main__':
+    print('p4apprunner {}'.format(sys.argv))
     main()
diff --git a/p4app b/p4app
index 3290940..28f0346 100755
--- a/p4app
+++ b/p4app
@@ -84,6 +84,30 @@ function run-command {
   return $rc
 }
 
+function test-command {
+  # Run the .p4app package provided by the user.
+  if [ -d "$1" ]; then
+    # The user passed the package as a directory. Tar it up and pass it to the
+    # container.
+    PACKAGE_DIR=$(normalize_path "$1")
+    APP_FILE=$(mktemp /tmp/p4app.tar.gz.XXXXXX)
+    tar -czf "$APP_FILE" -C "$PACKAGE_DIR" .
+    run-p4app "$APP_FILE" --tests "${@:2}"
+    rc=$?
+    rm "$APP_FILE"
+  elif [ -f "$1" ]; then
+    # The user passed the package as a file. We'll assume it's already a .tar.gz
+    # archive; just pass it to the container as-is.
+    APP_FILE=$(get_abs_filename "$1")
+    run-p4app "$APP_FILE" --tests "${@:2}"
+    rc=$?
+  else
+    echo "Couldn't read p4app package: $1"
+    exit 1
+  fi
+  return $rc
+}
+
 function pack-command {
   # Compress the provided .p4app package.
   if [ -d "$1" ]; then
@@ -166,6 +190,9 @@ case "$1" in
   "run")
     run-command "${@:2}"
     ;;
+  "test")
+    test-command "${@:2}"
+    ;;
   "build")
     build-command "${@:2}"
     ;;