Merge pull request #20 from Shaikh-Ubaid/integration_tests

Support Integration tests

Author: ubaidsk

.github/workflows/CI.yml

@@ -55,18 +55,6 @@ jobs:
         run: |
             which clang
 
-            lc examples/expr2.c --backend c --extra-arg="-Isrc/runtime/include" --
-
-            lc examples/expr2.c --backend wasm --extra-arg="-Isrc/runtime/include" --
-
-            lc examples/expr2.c --backend llvm --extra-arg="-Isrc/runtime/include" --
-
-            lc tests/array_01.cpp --backend=llvm --extra-arg="-Isrc/runtime/include" --
-
-            lc tests/array_02.cpp --backend=llvm --extra-arg="-Isrc/runtime/include" --
-
-            lc tests/array_02.cpp --backend=llvm --extra-arg="-Isrc/runtime/include" --
-
             lc tests/array_03.cpp --backend=llvm --extra-arg="-I$CONDA_PREFIX/include" --
 
             # Test generating object files
@@ -77,3 +65,10 @@ jobs:
             # Test including iostream and span headers
             lc tests/span_01.cpp --ast-dump --ast-dump-file="tests/span_01.ast" --extra-arg="-std=c++20" --
             cat tests/span_01.ast
+
+      - name: Test3 (Linux / macOS)
+        shell: bash -l -e {0}
+        if: contains(matrix.os, 'ubuntu') || contains(matrix.os, 'macos')
+        run: |
+            ./integration_tests/run_tests.py -b gcc llvm wasm c
+            ./integration_tests/run_tests.py -b gcc llvm wasm c -f

.gitignore

@@ -31,6 +31,7 @@ fa
 fa2
 */bin/lc
 *vscode*
+integration_tests/test-*
 
 ## Byte-compiled / optimized / DLL files
 __pycache__/

cmake/UserOverride.cmake

@@ -0,0 +1,26 @@
+# This overrides the default CMake Debug and Release compiler options.
+# The user can still specify different options by setting the
+# CMAKE_CXX_FLAGS_[RELEASE,DEBUG] variables (on the command line or in the
+# CMakeList.txt). This files serves as better CMake defaults and should only be
+# modified if the default values are to be changed. Project specific compiler
+# flags should be set in the CMakeList.txt by setting the CMAKE_CXX_FLAGS_*
+# variables.
+
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    # g++
+    set(common "-Wall -Wextra")
+    set(CMAKE_CXX_FLAGS_RELEASE_INIT "${common} -O3 -funroll-loops -DNDEBUG")
+    set(CMAKE_CXX_FLAGS_DEBUG_INIT   "${common} -g -ggdb")
+elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
+    # icpc
+    set(common "-Wall")
+    set(CMAKE_CXX_FLAGS_RELEASE_INIT "${common} -xHOST -O3")
+    set(CMAKE_CXX_FLAGS_DEBUG_INIT   "${common} -g -O0")
+elseif (CMAKE_CXX_COMPILER_ID MATCHES Clang)
+    # clang
+    set(common "-Wall -Wextra")
+    set(CMAKE_CXX_FLAGS_RELEASE_INIT "${common} -O3 -funroll-loops -DNDEBUG")
+    set(CMAKE_CXX_FLAGS_DEBUG_INIT   "${common} -g -ggdb")
+elseif (CMAKE_CXX_COMPILER_ID STREQUAL "PGI")
+    # pgcpp
+endif ()

integration_tests/CMakeLists.txt

@@ -0,0 +1,183 @@
+cmake_minimum_required(VERSION 3.10 FATAL_ERROR)
+
+project(lc C CXX)
+
+if (NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE Debug
+        CACHE STRING "Build type (Debug, Release)" FORCE)
+endif ()
+if (NOT (CMAKE_BUILD_TYPE STREQUAL "Debug" OR
+        CMAKE_BUILD_TYPE STREQUAL "Release"))
+    message("${CMAKE_BUILD_TYPE}")
+    message(FATAL_ERROR "CMAKE_BUILD_TYPE must be one of: Debug, Release (current value: '${CMAKE_BUILD_TYPE}')")
+endif ()
+
+find_program(LC NAMES lc)
+
+set(LC_RTL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../src/runtime)
+set(LC_RTL_HEADER_DIR ${LC_RTL_DIR}/impure)
+
+find_path(LC_RTLIB_DIR lfortran_intrinsics.h
+    HINTS ${LC_RTL_HEADER_DIR})
+find_library(LC_RTLIB_LIBRARY lc_runtime_static
+    HINTS ${LC_RTL_DIR})
+add_library(lc_runtime INTERFACE IMPORTED)
+set_property(TARGET lc_runtime PROPERTY INTERFACE_INCLUDE_DIRECTORIES
+    ${LC_RTLIB_DIR})
+set_property(TARGET lc_runtime PROPERTY INTERFACE_LINK_LIBRARIES
+    ${LC_RTLIB_LIBRARY})
+target_link_libraries(lc_runtime INTERFACE m)
+
+set(LC_BACKEND no CACHE STRING "Only compile the LC subset for the given backend")
+set(FAST no CACHE BOOL "Run supported tests with --fast")
+
+enable_testing()
+
+message("\n")
+message("Configuration results")
+message("---------------------")
+message("Fortran compiler: ${CMAKE_Fortran_COMPILER}")
+message("C compiler      : ${CMAKE_C_COMPILER}")
+message("Build type: ${CMAKE_BUILD_TYPE}")
+if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+    message("Fortran compiler flags: ${CMAKE_Fortran_FLAGS_DEBUG}")
+    message("C compiler flags      : ${CMAKE_C_FLAGS_DEBUG}")
+else ()
+    message("Fortran compiler flags: ${CMAKE_Fortran_FLAGS_RELEASE}")
+    message("C compiler flags      : ${CMAKE_C_FLAGS_RELEASE}")
+endif ()
+message("Installation prefix: ${CMAKE_INSTALL_PREFIX}")
+message("LC_BACKEND: ${LC_BACKEND}")
+message("FAST: ${FAST}")
+
+
+macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_EXTRA_ARGS RUN_COPY_TO_BIN RUN_GCC_ARGS)
+    set(fail ${${RUN_FAIL}})
+    set(name ${${RUN_NAME}})
+    set(file_name ${${RUN_FILE_NAME}})
+    set(labels ${${RUN_LABELS}})
+    set(extra_files ${${RUN_EXTRAFILES}})
+    set(extra_args ${${RUN_EXTRA_ARGS}})
+    set(copy_to_bin ${${RUN_COPY_TO_BIN}})
+    set(gcc_args ${${RUN_GCC_ARGS}})
+
+    if (NOT name)
+        message(FATAL_ERROR "Must specify the NAME argument")
+    endif()
+
+    if (LC_BACKEND)
+        if (${LC_BACKEND} IN_LIST labels)
+            # Test is supported by the given LC backend
+            set(ADD_TEST ON)
+        else()
+            # Test is not supported by the given LC backend
+            set(ADD_TEST OFF)
+        endif()
+    else()
+        # GCC
+        if ("gcc" IN_LIST labels)
+            set(ADD_TEST ON)
+        else()
+            set(ADD_TEST OFF)
+        endif()
+    endif()
+
+    if (ADD_TEST)
+        if ((LC_BACKEND STREQUAL "llvm") OR (LC_BACKEND STREQUAL "cpp") OR (LC_BACKEND STREQUAL "x86")
+                OR (LC_BACKEND STREQUAL "c") OR (LC_BACKEND STREQUAL "fortran"))
+            add_custom_command(
+                OUTPUT ${name}.o
+                COMMAND ${LC} -c ${extra_args} ${CMAKE_CURRENT_SOURCE_DIR}/${file_name} -o ${name}.o --extra-arg -I${CMAKE_CURRENT_SOURCE_DIR}/../src/runtime/include --
+                DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${file_name}
+                VERBATIM)
+            add_executable(${name} ${name}.o ${extra_files})
+            set_target_properties(${name} PROPERTIES LINKER_LANGUAGE C)
+            target_link_libraries(${name} lc_runtime)
+            add_test(${name} ${CURRENT_BINARY_DIR}/${name})
+        elseif (LC_BACKEND STREQUAL "wasm")
+            # wasm test
+            execute_process(COMMAND ${LC} ${extra_args} --backend=wasm ${CMAKE_CURRENT_SOURCE_DIR}/${file_name} -o ${name}
+                --extra-arg -I${CMAKE_CURRENT_SOURCE_DIR}/../src/runtime/include --)
+            find_program(WASM_EXEC_RUNTIME node)
+            execute_process(COMMAND "${WASM_EXEC_RUNTIME}" --version
+                            OUTPUT_VARIABLE WASM_EXEC_VERSION
+                            OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+            string(REGEX REPLACE "v([0-9]+)\\.([0-9]+)\\.([0-9]+)" "\\1" NODE_MAJOR_VERSION "${WASM_EXEC_VERSION}")
+
+            if (NODE_MAJOR_VERSION LESS 16)
+                message(STATUS "${WASM_EXEC_RUNTIME} version: ${WASM_EXEC_VERSION}")
+                set(WASM_EXEC_FLAGS "--experimental-wasm-bigint")
+            endif()
+            set(WASM_EXEC_FLAGS ${WASM_EXEC_FLAGS} "--experimental-wasi-unstable-preview1")
+            add_test(${name} ${WASM_EXEC_RUNTIME} ${WASM_EXEC_FLAGS} ${CURRENT_BINARY_DIR}/${name}.js)
+        else ()
+            add_executable(${name} ${file_name} ${extra_files})
+            target_compile_options(${name} PUBLIC ${gcc_args})
+            add_test(${name} ${CURRENT_BINARY_DIR}/${name})
+        endif()
+
+        if (labels)
+            set_tests_properties(${name} PROPERTIES LABELS "${labels}")
+        endif()
+
+        if (fail)
+            set_tests_properties(${name} PROPERTIES WILL_FAIL TRUE)
+        endif()
+
+        if (copy_to_bin)
+            file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/${copy_to_bin} DESTINATION ${CURRENT_BINARY_DIR})
+        endif()
+    endif()
+endmacro(RUN_UTIL)
+
+macro(RUN)
+    set(options FAIL NOFAST)
+    set(oneValueArgs NAME INCLUDE_PATH  COPY_TO_BIN)
+    set(multiValueArgs LABELS EXTRAFILES EXTRA_ARGS GCC_ARGS)
+    cmake_parse_arguments(RUN "${options}" "${oneValueArgs}"
+                        "${multiValueArgs}" ${ARGN} )
+
+    foreach(b ${RUN_LABELS})
+        if (NOT (b MATCHES "^(llvm|llvm2|llvm_rtlib|gcc|c|cpp|x86|wasm|gfortran|llvmImplicit|llvmStackArray|fortran|c_nopragma|llvm_nopragma)$"))
+            message(FATAL_ERROR "Unsupported backend: ${b}")
+        endif()
+    endforeach()
+
+    set(RUN_FILE_NAME ${RUN_NAME})
+
+    if (RUN_INCLUDE_PATH)
+        # Only one include path supported for now
+        # Later add support for multiple include paths by looping over and appending to extra args
+        set(RUN_EXTRA_ARGS ${RUN_EXTRA_ARGS} -I${CMAKE_CURRENT_SOURCE_DIR}/${RUN_INCLUDE_PATH})
+        set(RUN_GCC_ARGS ${RUN_GCC_ARGS} -I${CMAKE_CURRENT_SOURCE_DIR}/${RUN_INCLUDE_PATH})
+    endif()
+
+    if (NOT FAST)
+        RUN_UTIL(RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_EXTRA_ARGS RUN_COPY_TO_BIN RUN_GCC_ARGS)
+    endif()
+
+    if ((FAST) AND (NOT RUN_NOFAST))
+        set(RUN_EXTRA_ARGS ${RUN_EXTRA_ARGS} --fast)
+        set(RUN_NAME "${RUN_NAME}_FAST")
+        list(REMOVE_ITEM RUN_LABELS gfortran) # remove gfortran from --fast test
+        RUN_UTIL(RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_LABELS RUN_EXTRAFILES RUN_EXTRA_ARGS RUN_COPY_TO_BIN RUN_GCC_ARGS)
+    endif()
+endmacro(RUN)
+
+# The supported test labels are:
+#
+# gcc      --- compile with Gcc
+# fortran       --- convert to Fortran code and compile with GFortran
+# llvm          --- compile with LC, only generate binary code at the very end
+# llvm2         --- compile with LC, generate object files
+# llvm_rtlib    --- compile with LC loading ASR runtime library, generate object files
+# cpp           --- compile to C++, compile C++ to binary
+# x86           --- compile to x86 binary directly
+# wasm          --- compile to WASM binary directly
+
+RUN(NAME expr1.c LABELS gcc c wasm llvm NOFAST)
+
+# arrays
+RUN(NAME array_01.cpp LABELS gcc llvm NOFAST)
+RUN(NAME array_02.cpp LABELS gcc llvm NOFAST)

tests/array_01.cpp renamed to integration_tests/array_01.cpp

@@ -0,0 +1,8 @@
+#include <stdio.h>
+
+int main() {
+    int x;
+    x = (2+3)*5;
+    printf("%d\n", x);
+    return 0;
+}

tests/array_02.cpp renamed to integration_tests/array_02.cpp

@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+
+import argparse
+import subprocess as sp
+import os
+
+# Initialization
+NO_OF_THREADS = 8 # default no of threads is 8
+SUPPORTED_BACKENDS = ['llvm', 'llvm2', 'llvm_rtlib', 'c', 'cpp', 'x86', 'wasm',
+                      'gcc', 'llvmImplicit', 'llvmStackArray', 'fortran',
+                      'c_nopragma', 'llvm_nopragma']
+BASE_DIR = os.path.dirname(os.path.realpath(__file__))
+LC_PATH = f"{BASE_DIR}/../src/bin:$PATH"
+
+fast_tests = "no"
+
+def run_cmd(cmd, cwd=None):
+    print(f"+ {cmd}")
+    process = sp.run(cmd, shell=True, cwd=cwd)
+    if process.returncode != 0:
+        print("Command failed.")
+        exit(1)
+
+def run_test(backend):
+    run_cmd(f"mkdir {BASE_DIR}/test-{backend}")
+    cwd=f"{BASE_DIR}/test-{backend}"
+    common=f" -DCURRENT_BINARY_DIR={BASE_DIR}/test-{backend} -S {BASE_DIR} -B {BASE_DIR}/test-{backend}"
+    if backend == "gcc":
+        run_cmd(f"cmake" + common,
+                cwd=cwd)
+    else:
+        run_cmd(f"cmake -DLC_BACKEND={backend} -DFAST={fast_tests}" + common,
+                cwd=cwd)
+    run_cmd(f"make -j{NO_OF_THREADS}", cwd=cwd)
+    run_cmd(f"ctest -j{NO_OF_THREADS} --output-on-failure", cwd=cwd)
+
+
+def test_backend(backend):
+    if backend not in SUPPORTED_BACKENDS:
+        print(f"Unsupported Backend: {backend}\n")
+        return
+    run_test(backend)
+
+def check_module_names():
+    from glob import glob
+    import re
+    mod = re.compile("(module|MODULE) (\w+)")
+    files = glob("*.f90")
+    module_names = []
+    file_names = []
+    for file in files:
+        f = open(file).read()
+        s = mod.search(f)
+        if s:
+            module_names.append(s.group(2))
+            file_names.append(file)
+    for i in range(len(module_names)):
+        name = module_names[i]
+        if name in module_names[i+1:]:
+            print("FAIL: Found a duplicate module name")
+            print("Name:", name)
+            print("Filename:", file_names[i])
+            raise Exception("Duplicate module names")
+    print("OK: All module names are unique")
+
+def get_args():
+    parser = argparse.ArgumentParser(description="LFortran Integration Test Suite")
+    parser.add_argument("-j", "-n", "--no_of_threads", type=int,
+                help="Parallel testing on given number of threads")
+    parser.add_argument("-b", "--backends", nargs="*", default=["llvm"], type=str,
+                help="Test the requested backends (%s)" % \
+                        ", ".join(SUPPORTED_BACKENDS))
+    parser.add_argument("-f", "--fast", action='store_true',
+                help="Run supported tests with --fast")
+    parser.add_argument("-m", action='store_true',
+                help="Check that all module names are unique")
+    return parser.parse_args()
+
+def main():
+    args = get_args()
+
+    if args.m:
+        check_module_names()
+        return
+
+    # Setup
+    global NO_OF_THREADS, fast_tests
+    os.environ["PATH"] += os.pathsep + LC_PATH
+    # delete previously created directories (if any)
+    for backend in SUPPORTED_BACKENDS:
+        run_cmd(f"rm -rf {BASE_DIR}/test-{backend}")
+
+    NO_OF_THREADS = args.no_of_threads or NO_OF_THREADS
+    fast_tests = "yes" if args.fast else "no"
+    for backend in args.backends:
+        test_backend(backend)
+
+if __name__ == "__main__":
+    main()

integration_tests/expr1.c

@@ -1,11 +1,11 @@
 {
-    "basename": "asr-array_01-bf5ef80",
+    "basename": "asr-array_01-9c6ecba",
     "cmd": "lc --asr-dump --no-color {infile} -o {outfile} -extra-arg=\"-Isrc/runtime/include\"",
-    "infile": "tests/array_01.cpp",
+    "infile": "tests/../integration_tests/array_01.cpp",
     "infile_hash": "deb860e61919d1bd2fbc7abeeed41124193f4284de0838d4ba94fa09",
     "outfile": null,
     "outfile_hash": null,
-    "stdout": "asr-array_01-bf5ef80.stdout",
+    "stdout": "asr-array_01-9c6ecba.stdout",
     "stdout_hash": "141c00a55703c54d1d113b441b052bd1ac7dfc8cc1fb73b243532bfd",
     "stderr": null,
     "stderr_hash": null,

integration_tests/run_tests.py

@@ -1,11 +1,11 @@
 {
-    "basename": "asr-array_02-0507073",
+    "basename": "asr-array_02-e00d30a",
     "cmd": "lc --asr-dump --no-color {infile} -o {outfile} -extra-arg=\"-Isrc/runtime/include\"",
-    "infile": "tests/array_02.cpp",
+    "infile": "tests/../integration_tests/array_02.cpp",
     "infile_hash": "cb968d294be0d86c0f49b3364b7a3aa45b53edcc59349301d5f298ed",
     "outfile": null,
     "outfile_hash": null,
-    "stdout": "asr-array_02-0507073.stdout",
+    "stdout": "asr-array_02-e00d30a.stdout",
     "stdout_hash": "330d65993b5445881264552aaa9b7415b7be4a04a3ec05f5b8ae3d92",
     "stderr": null,
     "stderr_hash": null,