diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0685b60e3e..a8b96ae6d2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -121,8 +121,9 @@ jobs: - name: Install trgen shell: pwsh run: | - dotnet tool install -g trgen --version 0.16.3 - dotnet tool install -g trwdog --version 0.16.3 + dotnet tool install -g trgen --version 0.19.0 + dotnet tool install -g triconv --version 0.19.0 + dotnet tool install -g trwdog --version 0.19.0 if ("${{ matrix.os }}" -eq "ubuntu-latest") { echo "$HOME/.dotnet/tools" >> $env:GITHUB_PATH } diff --git a/.gitignore b/.gitignore index f4449bcb7a..c9f8ecd2e1 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,8 @@ # Ignore downloaded temporaries create by _scripts/get-antlr.ps1 antlr4-*-complete.jar + +# Ignore untracked .errors and .tree files. If it was +# checked in, it is tracked, and cannot be ignored. +/**/*.errors +/**/*.tree diff --git a/_scripts/readme.md b/_scripts/readme.md index 547741f72e..950f4a74aa 100644 --- a/_scripts/readme.md +++ b/_scripts/readme.md @@ -3,12 +3,11 @@ This directory contains scripts for CI testing. * regtest.sh -- This is a Bash-based script to test any target. To -run, cd to grammar-vs/, then type `bash regtest.sh ` where `` is CSharp, Java, Cpp, Dart, Go, or one -of the other targets. To test the target, you will need the NET SDK installed, -access to the internet, and to toolchain for the target you want to test. -For CSharp, it will download the Antlr4 tool and runtime. For the other -targets, you will need to download the Antlr4 tool antlr-4.9.3-complete.jar and -place it in /tmp. +run, cd to grammar-vs/ or a grammar directory, then type `bash regtest.sh ` +where `` is Antlr4cs, Cpp, CSharp, Dart, Go, Java, PHP, or Python3. +To test the target, you will need the NET SDK installed (required for the trgen +driver generator), access to the internet, and to toolchain for the target +you want to test. You will also need to set the Antlr4 tool. * test.ps1 -- this is a Powershell script for testing, similar to regtest.sh. diff --git a/_scripts/really-run.ps1 b/_scripts/really-run.ps1 index 932f2ee740..1c443ab537 100644 --- a/_scripts/really-run.ps1 +++ b/_scripts/really-run.ps1 @@ -14,8 +14,9 @@ $antlrPath = _scripts/get-antlr.ps1 "4.11.1" # Set up env as it is used in test script. echo "antlr_path=$antlrPath" >> $env:GITHUB_ENV -dotnet tool install -g trgen --version 0.16.3 -dotnet tool install -g trwdog --version 0.16.3 +dotnet tool install -g trgen --version 0.19.0 +dotnet tool install -g triconv --version 0.19.0 +dotnet tool install -g trwdog --version 0.19.0 # Call test script. $env:ANTLR_JAR_PATH="$antlrPath" diff --git a/_scripts/regtest.sh b/_scripts/regtest.sh index 20a5db5476..cd02711ef5 100644 --- a/_scripts/regtest.sh +++ b/_scripts/regtest.sh @@ -97,7 +97,7 @@ add() { fi } -rm -rf `find . -name Generated -type d` +rm -rf `find . -name 'Generated*' -type d` build() { @@ -108,7 +108,7 @@ build() echo Building $testname echo "" # Assert(cwd is valid) - pushd "$x/Generated" + pushd "$x/Generated-$target" if [[ $? != "0" ]] then echo "$1 is not a valid directory" @@ -136,14 +136,14 @@ test() echo Testing $testname echo "" # Assert(cwd is valid) - pushd "$x/Generated" + pushd "$x/Generated-$target" if [[ $? != "0" ]] then echo "$1 is not a valid directory" exit 1 fi date1=$(date +"%s") - make test + bash test.sh status="$?" date2=$(date +"%s") DIFF=$(($date2-$date1)) @@ -166,9 +166,10 @@ setupdeps() if [ $? != "0" ] then echo "Setting up trgen and antlr jar." - dotnet tool install -g trgen --version 0.16.3 - dotnet tool install -g trxml2 --version 0.16.3 - dotnet tool install -g trwdog --version 0.16.3 + dotnet tool install -g trgen --version 0.19.0 + dotnet tool install -g triconv --version 0.19.0 + dotnet tool install -g trxml2 --version 0.19.0 + dotnet tool install -g trwdog --version 0.19.0 case "${unameOut}" in Linux*) curl 'https://repo1.maven.org/maven2/org/antlr/antlr4/4.11.1/antlr4-4.11.1-complete.jar' -o /tmp/antlr4-4.11.1-complete.jar;; Darwin*) curl 'https://repo1.maven.org/maven2/org/antlr/antlr4/4.11.1/antlr4-4.11.1-complete.jar' -o /tmp/antlr4-4.11.1-complete.jar;; @@ -185,7 +186,7 @@ part1() { date # 1) Generate driver source code from poms. - rm -rf `find . -name Generated -type d` + rm -rf `find . -name Generated-$target -type d` echo "Generating drivers." if [[ "$invert" == "" ]] then @@ -208,7 +209,7 @@ part2() esac echo prefix $prefix echo bft $build_file_type - build_files=`find $prefix -type f -name $build_file_type | grep Generated` + build_files=`find $prefix -type f -name $build_file_type | grep Generated-$target` echo bf $build_files for build_file in $build_files do @@ -229,7 +230,7 @@ part3() # 3) Test generated parser on examples. echo "Parsing." date - build_files=`find $prefix -type f -name $build_file_type | grep Generated` + build_files=`find $prefix -type f -name $build_file_type | grep Generated-$target` echo bf $build_files for build_file in $build_files do diff --git a/_scripts/skip-go.txt b/_scripts/skip-go.txt index 293d2e8c79..2ee86cd38c 100644 --- a/_scripts/skip-go.txt +++ b/_scripts/skip-go.txt @@ -3,6 +3,7 @@ antlr/antlr2 antlr/antlr3 antlr/antlr4 apex +asm/asm8086 asn/asn_3gpp clif cobol85 @@ -11,6 +12,7 @@ csharp dart2 databank edif300 +fasta fortran77 gdscript glsl @@ -73,6 +75,7 @@ sql/hive/v2 sql/hive/v3 sql/plsql sql/postgresql +sql/tsql stringtemplate swift/swift2 swift/swift3 diff --git a/_scripts/skip-javascript.txt b/_scripts/skip-javascript.txt index 2d6405856d..f1892c5e36 100644 --- a/_scripts/skip-javascript.txt +++ b/_scripts/skip-javascript.txt @@ -35,6 +35,7 @@ r rego rexx rust +scss sql/hive/v2 sql/hive/v3 sql/mariadb diff --git a/_scripts/skip-python3.txt b/_scripts/skip-python3.txt index 92b95bd4e6..d016ec153c 100644 --- a/_scripts/skip-python3.txt +++ b/_scripts/skip-python3.txt @@ -33,6 +33,7 @@ javascript/ecmascript javascript/jsx javascript/typescript joss +kirikiri-tjs kotlin/kotlin kotlin/kotlin-formal kuka @@ -84,6 +85,8 @@ sparql sql/hive/v2 sql/hive/v3 sql/mariadb +sql/mysql +sql/plsql sql/postgresql sql/sqlite sql/tsql diff --git a/_scripts/templates/Antlr4cs/CaseChangingCharStream.cs b/_scripts/templates/Antlr4cs/CaseChangingCharStream.cs index 32574b3ee1..b1c4622dd6 100644 --- a/_scripts/templates/Antlr4cs/CaseChangingCharStream.cs +++ b/_scripts/templates/Antlr4cs/CaseChangingCharStream.cs @@ -1,4 +1,4 @@ -// Template generated code from trgen +// Generated from trgen /* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. * Use of this file is governed by the BSD 3-clause license that diff --git a/_scripts/templates/Antlr4cs/ErrorListener.cs b/_scripts/templates/Antlr4cs/ErrorListener.cs index e6328e8071..06a542444e 100644 --- a/_scripts/templates/Antlr4cs/ErrorListener.cs +++ b/_scripts/templates/Antlr4cs/ErrorListener.cs @@ -1,35 +1,39 @@ +// Generated from trgen + +using Antlr4.Runtime; +using Antlr4.Runtime.Misc; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + namespace { - public class ErrorListener : IAntlrErrorListener + public class ErrorListener\ : IAntlrErrorListener\ { - /// - /// Provides a default instance of - /// - /// . - /// - public static readonly ErrorListener Instance = new ErrorListener(); + public bool had_error; + bool _quiet; + bool _tee; + TextWriter _out; + + public ErrorListener(bool quiet, bool tee, TextWriter @out) + { + _quiet = quiet; + _tee = tee; + _out = @out; + } - /// - /// - ///

- /// This implementation prints messages to - /// - /// containing the - /// values of - /// - /// , - /// - /// , and - /// - /// using - /// the following format.

- /// 
-        /// line line:charPositionInLine msg
-        ///
- /// - public virtual void SyntaxError(IRecognizer recognizer, Symbol offendingSymbol, int line, int charPositionInLine, string msg, RecognitionException e) + public virtual void SyntaxError(IRecognizer recognizer, S offendingSymbol, int line, int col, string msg, RecognitionException e) { - System.Console.Out.WriteLine("line " + line + ":" + charPositionInLine + " " + msg); + had_error = true; + if (_tee) + { + _out.WriteLine("line " + line + ":" + col + " " + msg); + } + if (!_quiet) + { + System.Console.Error.WriteLine("line " + line + ":" + col + " " + msg); + } } } -} \ No newline at end of file +} diff --git a/_scripts/templates/Antlr4cs/Program.cs b/_scripts/templates/Antlr4cs/Program.cs deleted file mode 100644 index fea837862a..0000000000 --- a/_scripts/templates/Antlr4cs/Program.cs +++ /dev/null @@ -1,121 +0,0 @@ -// Template generated code from trgen - -using Antlr4.Runtime; -using Antlr4.Runtime.Tree; -using System; -using System.IO; -using System.Linq; -using System.Text; -using System.Runtime.CompilerServices; - -namespace -{ - -public class Program -{ - public static Parser Parser { get; set; } - public static Lexer Lexer { get; set; } - public static ITokenStream TokenStream { get; set; } - public static IParseTree Tree { get; set; } - public static string StartSymbol { get; set; } = ""; - public static string Input { get; set; } - - static void Main(string[] args) - { - bool show_tree = false; - bool show_tokens = false; - bool old = false; - bool two_byte = false; - string file_name = null; - string input = null; - for (int i = 0; i \< args.Length; ++i) - { - if (args[i].Equals("-tokens")) - { - show_tokens = true; - continue; - } - else if (args[i].Equals("-two-byte")) - { - two_byte = true; - continue; - } - else if (args[i].Equals("-old")) - { - old = true; - continue; - } - else if (args[i].Equals("-tree")) - { - show_tree = true; - continue; - } - else if (args[i].Equals("-input")) - input = args[++i]; - else if (args[i].Equals("-file")) - file_name = args[++i]; - } - ICharStream str = null; - if (input == null && file_name == null) - { - StringBuilder sb = new StringBuilder(); - int ch; - while ((ch = System.Console.Read()) != -1) - { - sb.Append((char)ch); - } - input = sb.ToString(); - str = new Antlr4.Runtime.AntlrInputStream( - new MemoryStream(Encoding.UTF8.GetBytes(input ?? ""))); - } - else if (input != null) - { - str = new Antlr4.Runtime.AntlrInputStream( - new MemoryStream(Encoding.UTF8.GetBytes(input ?? ""))); - } else if (file_name != null) - { - FileStream fs = new FileStream(file_name, FileMode.Open); - str = new Antlr4.Runtime.AntlrInputStream(fs); - } - - str = new Antlr4.Runtime.CaseChangingCharStream(str, "" == "Upper"); -< endif > - var lexer = new Test.(str); - if (show_tokens) - { - StringBuilder new_s = new StringBuilder(); - for (int i = 0; ; ++i) - { - var ro_token = lexer.NextToken(); - var token = (CommonToken)ro_token; - token.TokenIndex = i; - new_s.AppendLine(token.ToString()); - if (token.Type == Antlr4.Runtime.TokenConstants.Eof) - break; - } - System.Console.Error.WriteLine(new_s.ToString()); - lexer.Reset(); - } - var tokens = new CommonTokenStream(lexer); - var parser = new Test.(tokens); - DateTime before = DateTime.Now; - var tree = parser.(); - DateTime after = DateTime.Now; - System.Console.Error.WriteLine("Time: " + (after - before)); - if (parser.NumberOfSyntaxErrors > 0) - { - System.Console.Error.WriteLine("Parse failed."); - } - else - { - System.Console.Error.WriteLine("Parse succeeded."); - if (show_tree) - { - System.Console.Out.WriteLine(tree.ToStringTree(parser)); - } - } - System.Environment.Exit(parser.NumberOfSyntaxErrors > 0 ? 1 : 0); - } -} - -} diff --git a/_scripts/templates/Antlr4cs/Test.cs b/_scripts/templates/Antlr4cs/Test.cs new file mode 100644 index 0000000000..0eccf512be --- /dev/null +++ b/_scripts/templates/Antlr4cs/Test.cs @@ -0,0 +1,234 @@ +// Generated from trgen + +using Antlr4.Runtime; +using Antlr4.Runtime.Atn; +using Antlr4.Runtime.Tree; +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Runtime.CompilerServices; +using System.Collections.Generic; +using System.Net; +namespace +{ + +public class Program +{ + public static Parser Parser { get; set; } + public static Lexer Lexer { get; set; } + public static ITokenStream TokenStream { get; set; } + public static IParseTree Tree { get; set; } + public static string StartSymbol { get; set; } = ""; + public static string Input { get; set; } + + static bool tee = false; + static bool show_profile = false; + static bool show_tree = false; + static bool show_tokens = false; + static bool show_trace = false; + static bool old = false; + static bool two_byte = false; + static int exit_code = 0; + static Encoding encoding = null; + static int string_instance = 0; + static string prefix = ""; + static bool quiet = false; + + static void Main(string[] args) + { + List\ is_fns = new List\(); + List\ inputs = new List\(); + for (int i = 0; i \< args.Length; ++i) + { + if (args[i].Equals("-profile")) + { + show_profile = true; + } + else if (args[i].Equals("-tokens")) + { + show_tokens = true; + } + else if (args[i].Equals("-two-byte")) + { + two_byte = true; + } + else if (args[i].Equals("-old")) + { + old = true; + } + else if (args[i].Equals("-tree")) + { + show_tree = true; + } + else if (args[i].Equals("-prefix")) + { + prefix = args[++i] + " "; + } + else if (args[i].Equals("-input")) + { + inputs.Add(args[++i]); + is_fns.Add(false); + } + else if (args[i].Equals("-tee")) + { + tee = true; + } + else if (args[i].Equals("-encoding")) + { + ++i; + encoding = Encoding.GetEncoding( + args[i], + new EncoderReplacementFallback("(unknown)"), + new DecoderReplacementFallback("(error)")); + if (encoding == null) + throw new Exception(@"Unknown encoding. Must be an Internet Assigned Numbers Authority (IANA) code page name. https://www.iana.org/assignments/character-sets/character-sets.xhtml"); + } + else if (args[i] == "-x") + { + for (; ; ) + { + var line = System.Console.In.ReadLine(); + line = line?.Trim(); + if (line == null || line == "") + { + break; + } + inputs.Add(line); + is_fns.Add(true); + } + } + else if (args[i] == "-q") + { + quiet = true; + } + else if (args[i] == "-trace") + { + show_trace = true; + } + else + { + inputs.Add(args[i]); + is_fns.Add(true); + } + } + if (inputs.Count() == 0) + { + ParseStdin(); + } + else + { + DateTime before = DateTime.Now; + for (int f = 0; f \< inputs.Count(); ++f) + { + if (is_fns[f]) + ParseFilename(inputs[f], f); + else + ParseString(inputs[f], f); + } + DateTime after = DateTime.Now; + if (!quiet) + { + System.Console.Error.WriteLine("Total Time: " + (after - before).TotalSeconds); + } + } + Environment.ExitCode = exit_code; + } + + static void ParseStdin() + { + StringBuilder sb = new StringBuilder(); + int ch; + while ((ch = System.Console.Read()) != -1) + { + sb.Append((char)ch); + } + var input = sb.ToString(); + var str = new Antlr4.Runtime.AntlrInputStream( + new MemoryStream(Encoding.UTF8.GetBytes(input ?? ""))); + DoParse(str, "stdin", 0); + } + + static void ParseString(string input, int row_number) + { + ICharStream str = null; + str = new Antlr4.Runtime.AntlrInputStream( + new MemoryStream(Encoding.UTF8.GetBytes(input ?? ""))); + DoParse(str, "string" + string_instance++, row_number); + } + + static void ParseFilename(string input, int row_number) + { + ICharStream str = null; + FileStream fs = new FileStream(input, FileMode.Open); + str = new Antlr4.Runtime.AntlrInputStream(fs); + DoParse(str, input, row_number); + } + + static void DoParse(ICharStream str, string input_name, int row_number) + { + + str = new Antlr4.Runtime.CaseChangingCharStream(str, "" == "Upper"); + + var lexer = new Test.(str); + if (show_tokens) + { + StringBuilder new_s = new StringBuilder(); + for (int i = 0; ; ++i) + { + var ro_token = lexer.NextToken(); + var token = (CommonToken)ro_token; + token.TokenIndex = i; + new_s.AppendLine(token.ToString()); + if (token.Type == Antlr4.Runtime.TokenConstants.Eof) + break; + } + System.Console.Error.WriteLine(new_s.ToString()); + lexer.Reset(); + } + var tokens = new CommonTokenStream(lexer); + var parser = new Test.(tokens); + var output = tee ? new StreamWriter(input_name + ".errors") : System.Console.Error; + var listener_lexer = new ErrorListener\(quiet, tee, output); + var listener_parser = new ErrorListener\(quiet, tee, output); + lexer.RemoveErrorListeners(); + parser.RemoveErrorListeners(); + lexer.AddErrorListener(listener_lexer); + parser.AddErrorListener(listener_parser); + if (show_trace) + { + // parser.Trace = true; + // ATN tracing missing. + } + DateTime before = DateTime.Now; + var tree = parser.(); + DateTime after = DateTime.Now; + var result = ""; + if (parser.NumberOfSyntaxErrors > 0) + { + result = "fail"; + exit_code = 1; + } + else + { + result = "success"; + } + if (show_tree) + { + if (tee) + { + System.IO.File.WriteAllText(input_name + ".tree", tree.ToStringTree(parser)); + } else + { + System.Console.Error.WriteLine(tree.ToStringTree(parser)); + } + } + if (!quiet) + { + System.Console.Error.WriteLine(prefix + "Antlr4cs " + row_number + " " + input_name + " " + result + " " + (after - before).TotalSeconds); + } + if (tee) output.Close(); + } +} + +} diff --git a/_scripts/templates/Antlr4cs/Test.csproj b/_scripts/templates/Antlr4cs/Test.csproj index 2ea47eb955..b480e355d3 100644 --- a/_scripts/templates/Antlr4cs/Test.csproj +++ b/_scripts/templates/Antlr4cs/Test.csproj @@ -1,4 +1,4 @@ -\ +\ \ \ \net6.0\ diff --git a/_scripts/templates/Antlr4cs/build.ps1 b/_scripts/templates/Antlr4cs/build.ps1 new file mode 100644 index 0000000000..9fc944240c --- /dev/null +++ b/_scripts/templates/Antlr4cs/build.ps1 @@ -0,0 +1,7 @@ +# Generated from trgen +if (Test-Path -Path transformGrammar.py -PathType Leaf) { + $(& python3 transformGrammar.py ) 2>&1 | Write-Host +} + +$(& dotnet build; $compile_exit_code = $LASTEXITCODE) | Write-Host +exit $compile_exit_code diff --git a/_scripts/templates/Antlr4cs/build.sh b/_scripts/templates/Antlr4cs/build.sh new file mode 100644 index 0000000000..b6525d0ec2 --- /dev/null +++ b/_scripts/templates/Antlr4cs/build.sh @@ -0,0 +1,4 @@ +# Generated from trgen +if [ -f transformGrammar.py ]; then python3 transformGrammar.py ; fi +dotnet restore +dotnet build diff --git a/_scripts/templates/Antlr4cs/clean.ps1 b/_scripts/templates/Antlr4cs/clean.ps1 new file mode 100644 index 0000000000..0dc470b356 --- /dev/null +++ b/_scripts/templates/Antlr4cs/clean.ps1 @@ -0,0 +1,5 @@ +# Generated from trgen +$(& dotnet clean; $status = $LASTEXITCODE) | Write-Host +$(& Remove-Item bin -Recurse -Force ) 2>&1 | Out-Null +$(& Remove-Item obj -Recurse -Force ) 2>&1 | Out-Null +exit 0 diff --git a/_scripts/templates/Antlr4cs/clean.sh b/_scripts/templates/Antlr4cs/clean.sh new file mode 100644 index 0000000000..f802018372 --- /dev/null +++ b/_scripts/templates/Antlr4cs/clean.sh @@ -0,0 +1,4 @@ +# Generated from trgen +dotnet clean +rm -rf bin obj +rm -f }> diff --git a/_scripts/templates/Antlr4cs/makefile b/_scripts/templates/Antlr4cs/makefile index 308be769d5..4a10a46f0d 100644 --- a/_scripts/templates/Antlr4cs/makefile +++ b/_scripts/templates/Antlr4cs/makefile @@ -1,12 +1,10 @@ -# Template generated code from trgen -GENERATED = }> +# Generated from trgen default: - dotnet restore - dotnet build + bash build.sh run: - trwdog dotnet run $(RUNARGS) + @trwdog dotnet run $(RUNARGS) clean: - dotnet clean - rm -rf bin obj -test: + bash clean.sh +FORCE: ; +test: FORCE bash test.sh diff --git a/_scripts/templates/Antlr4cs/test.ps1 b/_scripts/templates/Antlr4cs/test.ps1 new file mode 100644 index 0000000000..eb799ca56a --- /dev/null +++ b/_scripts/templates/Antlr4cs/test.ps1 @@ -0,0 +1,149 @@ +# Generated from trgen + +$TestDirectory = "../../" +Write-Host "Test cases here: $TestDirectory" + +# People often specify a test file directory, but sometimes no +# tests are provided. Git won't check in an empty directory. +# Test if the test file directory does not exist, or it is just +# an empty directory. +if (!(Test-Path -Path "$TestDirectory")) { + Write-Host "No test cases provided." + exit 0 +} elseif (!(Test-Path "$TestDirectory/*")) { + Write-Host "No test cases provided." + exit 0 +} + +# Get a list of test files from the test directory. Do not include any +# .errors or .tree files. Pay close attention to remove only file names +# that end with the suffix .errors or .tree. +if (Test-Path -Path "tests.txt" -PathType Leaf) { + Remove-Item "tests.txt" +} +$files = New-Object System.Collections.Generic.List[string] +foreach ($item in Get-ChildItem $TestDirectory -Recurse) { + $file = $item.fullname + $ext = $item.Extension + if (Test-Path $file -PathType Container) { + continue + } elseif ($ext -eq ".errors") { + continue + } elseif ($ext -eq ".tree") { + continue + } else { + $(& triconv -f utf-8 $file ; $last = $LASTEXITCODE ) | Out-Null + if ($last -ne 0) + { + continue + } + $files.Add($item) + Write-Host "Test case: $item" + } +} +foreach ($file in $files) { + Add-Content "tests.txt" $file +} +if (-not(Test-Path -Path "tests.txt" -PathType Leaf)) { + Write-Host "No test cases provided." + exit 0 +} + +# Parse all input files. +get-content "tests.txt" | trwdog dotnet run -q -x -tee -tree *> parse.txt +$status = $LASTEXITCODE + +# trwdog returns 255 if it cannot spawn the process. This could happen +# if the environment for running the program does not exist, or the +# program did not build. +if ( $status -eq 255 ) { + Write-Host "Test failed." + Get-Content $file | Write-Host + exit 1 +} + +# Any parse errors will be put in .errors files. But, if there's any +# output from the program in stdout or stderr, it's all bad news. +$size = (Get-Item -Path "parse.txt").Length +if ( $size -eq 0 ) { +} else { + Write-Host "Test failed." + Get-Content "parse.txt" | Write-Host + exit 1 +} + +# Check if any .errors/.tree files have changed. That's not good. +$message = git diff --exit-code --name-only $TestDirectory +$updated = $LASTEXITCODE +Write-Host $message + +# Check if any untracked .errors files are not empty. +$new_errors_txt = New-Object System.Collections.Generic.List[string] +$new_errors2_txt = git ls-files --exclude-standard -o --ignored $TestDirectory +$new_errors = $LASTEXITCODE + +# Gather up all untracked .errors file output. These are new errors +# and must be reported as a parse fail. +if ( ! [String]::IsNullOrWhiteSpace($new_errors2_txt) ) { + $new_errors3_txt = $new_errors2_txt.Split("\n\r\t ") +} else { + $new_errors3_txt = [System.Collections.Arraylist]@() +} +if (Test-Path -Path "new_errors.txt" -PathType Leaf) { + Remove-Item "new_errors.txt" +} +New-Item -Path . -Name "new_errors.txt" -ItemType "file" -Value "" | Out-Null +foreach ($s in $new_errors3_txt) { + if ( [String]::IsNullOrWhiteSpace($s) ) { + continue + } + $ext = $item.Extension + if (! $s.EndsWith(".errors")) { + continue + } + $file = $s + $size = (Get-Item -Path $file).Length + if ( $size -eq 0 ) { + } else { + $new_errors_txt.Add($item) + Add-Content -Path "new_errors.txt" -Value "$item" + ((Get-Content $file) -join "`n") + "`n" | Add-Content -Path "new_errors.txt" + } +} + +# If "git diff" reported an exit code of 129, it is because +# the directory containing the grammar is not in a repo. In this +# case, assume parse error code as the defacto result. +if ( $updated -eq 129 ) { + Write-Host "Grammar outside a git repository. Assuming parse exit code." + if ( $status -eq 0 ) { + Write-Host "Test succeeded." + } else { + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + } + $err = $status + exit 1 +} + +# "Git diff" reported a difference. Redo the "git diff" to print out all +# the differences. Also, output any untracked, non-zero length .errors files. +if ( $updated -eq 1 ) { + Write-Host "Difference in output." + git diff . | Write-Host + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + exit 1 +} + +# If there's non-zero length .errors flies that are new, report them +# as errors in the parse. +if ( $new_errors_txt.Count -gt 0 ) { + Write-Host "New errors in output." + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + exit 1 +} + +Write-Host "Test succeeded." +exit 0 diff --git a/_scripts/templates/Antlr4cs/test.sh b/_scripts/templates/Antlr4cs/test.sh index cf88cfe8ab..ab1bdeccb1 100644 --- a/_scripts/templates/Antlr4cs/test.sh +++ b/_scripts/templates/Antlr4cs/test.sh @@ -1,58 +1,149 @@ -# Template generated code from trgen -err=0 +# Generated from trgen + +# People often specify a test file directory, but sometimes no +# tests are provided. Git won't check in an empty directory. +# Test if the test file directory does not exist, or it is just +# an empty directory. +if [ ! -d ../ ] +then + echo "No test cases provided." + exit 0 +elif [ ! "$(ls -A ../)" ] +then + echo "No test cases provided." + exit 0 +fi + SAVEIFS=$IFS IFS=$(echo -en "\n\b") -parse_out_file=`mktemp --tmpdir=. parse_out.XXXXXXXXXX` -for g in `find ../ -type f | grep -v '.errors$' | grep -v '.tree$'` + +# Get a list of test files from the test directory. Do not include any +# .errors or .tree files. Pay close attention to remove only file names +# that end with the suffix .errors or .tree. +files2=`find ../ -type f | grep -v '.errors$' | grep -v '.tree$'` +files=() +for f in $files2 do - file="$g" - x1="${g##*.}" - if [ "$x1" != "errors" ] - then - echo "$file" - trwdog ./bin/Debug/net6.0/ -file "$file" -tree > "$parse_out_file" - status="$?" - if [ -f "$file".errors ] + triconv -f utf-8 $f > /dev/null 2>&1 + if [ "$?" = "0" ] then - if [ "$status" = "0" ] - then - echo Expected parse fail. - err=1 - break - else - echo Expected. - diff \<(tr -d "\r\n" \< "$file".errors) \<(tr -d "\r\n" \< "$parse_out_file") - status="$?" - if [ "$status" = "0" ] - then - echo Parse errors match succeeded. - else - echo Expected parse errors match. - err=1 - break - fi - fi - else # No .errors file - if [ "$status" != "0" ] - then - err=1 - break - fi - if [ -f "$file".tree ] - then - diff \<(tr -d "\r\n" \< "$file".tree) \<(tr -d "\r\n" \< "$parse_out_file") - status="$?" - if [ "$status" = "0" ] + files+=( $f ) + fi +done + +# Parse all input files. +echo "${files[*]}" | trwdog ./bin/Debug/net6.0/Test.exeTest -q -x -tee -tree > parse.txt 2>&1 +status=$? + +# trwdog returns 255 if it cannot spawn the process. This could happen +# if the environment for running the program does not exist, or the +# program did not build. +if [ "$status" = "255" ] +then + echo "Test failed." + cat parse.txt + exit 1 +fi + +# Any parse errors will be put in .errors files. But, if there's any +# output from the program in stdout or stderr, it's all bad news. +if [ -s parse.txt ] +then + echo "Test failed." + cat parse.txt + exit 1 +fi + +# rm -rf `find ../ -type f -name '*.errors' -o -name '*.tree' -size 0` + +# For Unix environments, convert the newline in the .errors and .trees +# to Unix style. +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) machine=Linux;; + Darwin*) machine=Mac;; + CYGWIN*) machine=Cygwin;; + MINGW*) machine=MinGw;; + *) machine="UNKNOWN:${unameOut}" +esac +if [[ "$machine" == "MinGw" || "$machine" == "Msys" || "$machine" == "Cygwin" || "#machine" == "Linux" ]] +then + gen=`find ../ -type f -name '*.errors' -o -name '*.tree'` + if [ "$gen" != "" ] + then + dos2unix $gen + fi +fi + +old=`pwd` +cd ../ + +# Check if any files in the test files directory have changed. +git diff --exit-code --name-only . > $old/updated.txt 2>&1 +updated=$? + +# Check if any untracked .errors files. +git ls-files --exclude-standard -o --ignored > $old/new_errors2.txt 2>&1 +new_errors=$? + +# Gather up all untracked .errors file output. These are new errors +# and must be reported as a parse fail. +rm -f $old/new_errors.txt +touch $old/new_errors.txt +for f in `cat $old/new_errors2.txt` +do + ext=${f##*.} + ext=".$ext" + if [ "$ext" = ".errors" ] + then + if [ -s $f ] then - echo Parse tree match succeeded. - else - echo Expected parse tree match. - err=1 - break + echo $f >> $old/new_errors.txt + cat $f >> $old/new_errors.txt fi - fi fi - fi done -rm "$parse_out_file" -exit $err + +# If "git diff" reported an exit code of 129, it is because +# the directory containing the grammar is not in a repo. In this +# case, assume parse error code as the defacto result. +if [ "$updated" = "129" ] +then + echo "Grammar outside a git repository. Assuming parse exit code." + if [ "$status" = 0 ] + then + echo "Test succeeded." + else + cat $old/new_errors.txt + echo "Test failed." + fi + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit $status +fi + +# "Git diff" reported a difference. Redo the "git diff" to print out all +# the differences. Also, output any untracked, non-zero length .errors files. +if [ "$updated" = "1" ] +then + echo "Difference in output." + git diff . + cat $old/new_errors.txt + echo "Test failed." + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit 1 +fi + +# If there's non-zero length .errors flies that are new, report them +# as errors in the parse. +if [ -s $old/new_errors.txt ] +then + echo "New errors in output." + cat $old/new_errors.txt + echo "Test failed." + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit 1 +fi + +echo "Test succeeded." +rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt +exit 0 diff --git a/_scripts/templates/Antlr4cs/tester.psm1 b/_scripts/templates/Antlr4cs/tester.psm1 deleted file mode 100644 index dd00ee1064..0000000000 --- a/_scripts/templates/Antlr4cs/tester.psm1 +++ /dev/null @@ -1,54 +0,0 @@ -# Template generated code from trgen -function Build-Grammar { - $msg = dotnet build -o CSharp - return @{ - Message = $msg - Success = $LASTEXITCODE -eq 0 - } -} - -function Test-Case { - param ( - $InputFile, - $TokenFile, - $TreeFile, - $ErrorFile - ) - # Save input and output character encodings and switch to UTF-8. - $oldInputEncoding = [console]::InputEncoding - $oldOutputEncoding = [console]::OutputEncoding - $OutputEncoding = [console]::InputEncoding = [console]::OutputEncoding = New-Object System.Text.UTF8Encoding - - $parseOutFile = $InputFile + ".out" - $o = trwdog dotnet CSharp/Test.dll -file $InputFile -tree | Out-File -LiteralPath "$parseOutFile" -Encoding UTF8 - $parseOk = $LASTEXITCODE -eq 0 - $treeMatch = $true - if ($errorFile) { - # If we expected errors, then a failed parse was a successful test. - if (!$parseOk) { - Write-Host "Expected." - # Confirm that the errors we received are the ones we expected. - $expectedData = (Get-Content $errorFile -Encoding UTF8) -join "" -replace "\r\n","" - $actualData = (Get-Content $parseOutFile -Encoding UTF8) -join "" -replace "\r\n","" - $parseOk = ($actualData -eq $expectedData) - if ($parseOk) { - Write-Host "Error list match succeeded." - } else { - Write-Host "Expected error list match." -ForegroundColor Red - } - } - } else { - if ($parseOk -and (Test-Path $TreeFile)) { - # Confirm that the parse tree we received is the one we expected. - $expectedData = Get-Content $TreeFile -Encoding UTF8 - $actualData = Get-Content $parseOutFile -Encoding UTF8 - $treeMatch = ($actualData -eq $expectedData) - } - } - # Restore input and output character encodings. - [console]::InputEncoding = $oldInputEncoding - [console]::OutputEncoding = $oldOutputEncoding - - Remove-Item $parseOutFile - return $parseOk, $treeMatch -} \ No newline at end of file diff --git a/_scripts/templates/CSharp/CaseChangingCharStream.cs b/_scripts/templates/CSharp/CaseChangingCharStream.cs deleted file mode 100644 index 0e82d67eb6..0000000000 --- a/_scripts/templates/CSharp/CaseChangingCharStream.cs +++ /dev/null @@ -1,113 +0,0 @@ -// Template generated code from trgen - -/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. - * Use of this file is governed by the BSD 3-clause license that - * can be found in the LICENSE.txt file in the project root. - */ -using System; -using Antlr4.Runtime.Misc; - -namespace Antlr4.Runtime -{ - /// \ - /// This class supports case-insensitive lexing by wrapping an existing - /// \ and forcing the lexer to see either upper or - /// lowercase characters. Grammar literals should then be either upper or - /// lower case such as 'BEGIN' or 'begin'. The text of the character - /// stream is unaffected. Example: input 'BeGiN' would match lexer rule - /// 'BEGIN' if constructor parameter upper=true but getText() would return - /// 'BeGiN'. - /// \ - public class CaseChangingCharStream : ICharStream - { - private ICharStream stream; - private bool upper; - - /// \ - /// Constructs a new CaseChangingCharStream wrapping the given \ forcing - /// all characters to upper case or lower case. - /// \ - /// \The stream to wrap.\ - /// \If true force each symbol to upper - /// case, otherwise force to lower.\ - public CaseChangingCharStream(ICharStream stream, bool upper) - { - this.stream = stream; - this.upper = upper; - } - - public int Index - { - get - { - return stream.Index; - } - } - - public int Size - { - get - { - return stream.Size; - } - } - - public string SourceName - { - get - { - return stream.SourceName; - } - } - - public void Consume() - { - stream.Consume(); - } - - [return: NotNull] - public string GetText(Interval interval) - { - return stream.GetText(interval); - } - - public int LA(int i) - { - int c = stream.LA(i); - - if (c \<= 0) - { - return c; - } - - char o = (char)c; - - if (upper) - { - return (int)char.ToUpperInvariant(o); - } - - return (int)char.ToLowerInvariant(o); - } - - public int Mark() - { - return stream.Mark(); - } - - public void Release(int marker) - { - stream.Release(marker); - } - - public void Seek(int index) - { - stream.Seek(index); - } - - public override string ToString() - { - return stream.ToString(); - } - } -} diff --git a/_scripts/templates/CSharp/Encodings.cs b/_scripts/templates/CSharp/Encodings.cs index 9230927810..d99175f9b9 100644 --- a/_scripts/templates/CSharp/Encodings.cs +++ b/_scripts/templates/CSharp/Encodings.cs @@ -1,4 +1,4 @@ -// Template generated code from trgen +// Generated from trgen using Antlr4.Runtime; using Antlr4.Runtime.Misc; diff --git a/_scripts/templates/CSharp/ErrorListener.cs b/_scripts/templates/CSharp/ErrorListener.cs index 0703546477..318cc778cc 100644 --- a/_scripts/templates/CSharp/ErrorListener.cs +++ b/_scripts/templates/CSharp/ErrorListener.cs @@ -1,4 +1,4 @@ -// Template generated code from trgen +// Generated from trgen using Antlr4.Runtime; using Antlr4.Runtime.Misc; @@ -7,20 +7,34 @@ using System.IO; using System.Linq; -public class ErrorListener\ : ConsoleErrorListener\ +namespace +{ +public class ErrorListener\ : IAntlrErrorListener\< S> { public bool had_error; bool _quiet; + bool _tee; + TextWriter _out; - public ErrorListener(bool quiet = false) + public ErrorListener(bool quiet, bool tee, TextWriter @out) { _quiet = quiet; + _tee = tee; + _out = @out; } - public override void SyntaxError(TextWriter output, IRecognizer recognizer, S offendingSymbol, int line, + public void SyntaxError(TextWriter output, IRecognizer recognizer, S offendingSymbol, int line, int col, string msg, RecognitionException e) { had_error = true; - if (!_quiet) System.Console.Out.WriteLine("line " + line + ":" + col + " " + msg); + if (_tee) + { + _out.WriteLine("line " + line + ":" + col + " " + msg); + } + if (!_quiet) + { + System.Console.Error.WriteLine("line " + line + ":" + col + " " + msg); + } } } +} diff --git a/_scripts/templates/CSharp/Program.cs b/_scripts/templates/CSharp/Program.cs deleted file mode 100644 index f06c934faf..0000000000 --- a/_scripts/templates/CSharp/Program.cs +++ /dev/null @@ -1,198 +0,0 @@ -// Template generated code from trgen - -using Antlr4.Runtime; -using Antlr4.Runtime.Tree; -using System; -using System.IO; -using System.Linq; -using System.Text; -using System.Runtime.CompilerServices; - -public class Program -{ - public static Parser { get; set; } - public static Lexer Lexer { get; set; } - public static ITokenStream TokenStream { get; set; } - public static IParseTree Tree { get; set; } - public static string StartSymbol { get; set; } = ""; - public static string Input { get; set; } - public static void SetupParse2(string input) - { - ICharStream str = new AntlrInputStream(input); - - str = new Antlr4.Runtime.CaseChangingCharStream(str, "" == "Upper"); - - var lexer = new (str); - Lexer = lexer; - var tokens = new CommonTokenStream(lexer); - TokenStream = tokens; - var parser = new (tokens); - Parser = parser; - var listener_lexer = new ErrorListener\(); - var listener_parser = new ErrorListener\(); - lexer.RemoveErrorListeners(); - parser.RemoveErrorListeners(); - lexer.AddErrorListener(listener_lexer); - parser.AddErrorListener(listener_parser); - } - public static IParseTree Parse2() - { - var tree = Parser.(); - Input = Lexer.InputStream.ToString(); - TokenStream = Parser.TokenStream; - Tree = tree; - return tree; - } - public static IParseTree Parse(string input) - { - ICharStream str = new AntlrInputStream(input); - - str = new Antlr4.Runtime.CaseChangingCharStream(str, "" == "Upper"); - < endif > - var lexer = new (str); - Lexer = lexer; - var tokens = new CommonTokenStream(lexer); - TokenStream = tokens; - var parser = new (tokens); - Parser = parser; - var listener_lexer = new ErrorListener\(); - var listener_parser = new ErrorListener\(); - lexer.RemoveErrorListeners(); - parser.RemoveErrorListeners(); - lexer.AddErrorListener(listener_lexer); - parser.AddErrorListener(listener_parser); - var tree = parser.(); - Input = lexer.InputStream.ToString(); - TokenStream = parser.TokenStream; - Tree = tree; - return tree; - } - - static void Main(string[] args) - { - bool show_profile = false; - bool show_tree = false; - bool show_tokens = false; - bool old = false; - bool two_byte = false; - string file_name = null; - string input = null; - System.Text.Encoding encoding = null; - for (int i = 0; i \< args.Length; ++i) - { - if (args[i].Equals("-profile")) - { - show_profile = true; - continue; - } - else if (args[i].Equals("-tokens")) - { - show_tokens = true; - continue; - } - else if (args[i].Equals("-two-byte")) - { - two_byte = true; - continue; - } - else if (args[i].Equals("-old")) - { - old = true; - continue; - } - else if (args[i].Equals("-tree")) - { - show_tree = true; - continue; - } - else if (args[i].Equals("-input")) - input = args[++i]; - else if (args[i].Equals("-file")) - file_name = args[++i]; - else if (args[i].Equals("-encoding")) - { - ++i; - encoding = Encoding.GetEncoding( - args[i], - new EncoderReplacementFallback("(unknown)"), - new DecoderReplacementFallback("(error)")); - if (encoding == null) - throw new Exception(@"Unknown encoding. Must be an Internet Assigned Numbers Authority (IANA) code page name. https://www.iana.org/assignments/character-sets/character-sets.xhtml"); - } - } - ICharStream str = null; - if (input == null && file_name == null) - { - str = CharStreams.fromStream(System.Console.OpenStandardInput()); - } else if (input != null) - { - str = CharStreams.fromString(input); - } else if (file_name != null) - { - if (two_byte) - str = new TwoByteCharStream(file_name); - else if (old) - { - FileStream fs = new FileStream(file_name, FileMode.Open); - str = new Antlr4.Runtime.AntlrInputStream(fs); - } - else if (encoding == null) - str = CharStreams.fromPath(file_name); - else - str = CharStreams.fromPath(file_name, encoding); - } - - str = new Antlr4.Runtime.CaseChangingCharStream(str, "" == "Upper"); - - var lexer = new (str); - if (show_tokens) - { - StringBuilder new_s = new StringBuilder(); - for (int i = 0; ; ++i) - { - var ro_token = lexer.NextToken(); - var token = (CommonToken)ro_token; - token.TokenIndex = i; - new_s.AppendLine(token.ToString()); - if (token.Type == Antlr4.Runtime.TokenConstants.EOF) - break; - } - System.Console.Error.WriteLine(new_s.ToString()); - lexer.Reset(); - } - var tokens = new CommonTokenStream(lexer); - var parser = new (tokens); - var listener_lexer = new ErrorListener\(); - var listener_parser = new ErrorListener\(); - lexer.RemoveErrorListeners(); - parser.RemoveErrorListeners(); - lexer.AddErrorListener(listener_lexer); - parser.AddErrorListener(listener_parser); - DateTime before = DateTime.Now; - if (show_profile) - { - parser.Profile = true; - } - var tree = parser.(); - DateTime after = DateTime.Now; - System.Console.Error.WriteLine("Time: " + (after - before)); - if (listener_lexer.had_error || listener_parser.had_error) - { - // Listener will have already printed the error(s) to stdout. - System.Console.Error.WriteLine("Parse failed."); - } - else - { - System.Console.Error.WriteLine("Parse succeeded."); - if (show_tree) - { - System.Console.Out.WriteLine(tree.ToStringTree(parser)); - } - } - if (show_profile) - { - System.Console.Out.WriteLine(String.Join(",\n\r", parser.ParseInfo.getDecisionInfo().Select(d => d.ToString()))); - } - System.Environment.Exit(listener_lexer.had_error || listener_parser.had_error ? 1 : 0); - } -} diff --git a/_scripts/templates/CSharp/Test.cs b/_scripts/templates/CSharp/Test.cs new file mode 100644 index 0000000000..2a5074281e --- /dev/null +++ b/_scripts/templates/CSharp/Test.cs @@ -0,0 +1,296 @@ +// Generated from trgen + +using Antlr4.Runtime; +using Antlr4.Runtime.Atn; +using Antlr4.Runtime.Tree; +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Runtime.CompilerServices; +using System.Collections.Generic; +using System.Net.Sockets; +namespace +{ + +public class Program +{ + public static Parser { get; set; } + public static ErrorListener\ ParserErrorListener { get; set; } + public static Lexer Lexer { get; set; } + public static ErrorListener\ LexerErrorListener { get; set; } + public static ITokenStream TokenStream { get; set; } + public static ICharStream CharStream { get; set; } + public static IParseTree Tree { get; set; } + public static string StartSymbol { get; set; } = ""; + public static string Input { get; set; } + public static void SetupParse2(string input, bool quiet = false) + { + ICharStream str = new AntlrInputStream(input); + CharStream = str; + var lexer = new (str); + Lexer = lexer; + var tokens = new CommonTokenStream(lexer); + TokenStream = tokens; + var parser = new (tokens); + Parser = parser; + var listener_lexer = new ErrorListener\(false, false, System.Console.Error); + var listener_parser = new ErrorListener\(false, false, System.Console.Error); + LexerErrorListener = listener_lexer; + ParserErrorListener = listener_parser; + lexer.RemoveErrorListeners(); + parser.RemoveErrorListeners(); + lexer.AddErrorListener(listener_lexer); + parser.AddErrorListener(listener_parser); + } + + public static IParseTree Parse2() + { + var tree = Parser.(); + Input = Lexer.InputStream.ToString(); + TokenStream = Parser.TokenStream; + Tree = tree; + return tree; + } + + public static bool AnyErrors() + { + return ParserErrorListener.had_error || LexerErrorListener.had_error; + } + + public static IParseTree Parse(string input) + { + ICharStream str = new AntlrInputStream(input); + CharStream = str; + var lexer = new (str); + Lexer = lexer; + var tokens = new CommonTokenStream(lexer); + TokenStream = tokens; + var parser = new (tokens); + Parser = parser; + var listener_lexer = new ErrorListener\(false, false, System.Console.Error); + var listener_parser = new ErrorListener\(false, false, System.Console.Error); + lexer.RemoveErrorListeners(); + parser.RemoveErrorListeners(); + lexer.AddErrorListener(listener_lexer); + parser.AddErrorListener(listener_parser); + var tree = parser.(); + Input = lexer.InputStream.ToString(); + TokenStream = parser.TokenStream; + Tree = tree; + return tree; + } + + static bool tee = false; + static bool show_profile = false; + static bool show_tree = false; + static bool show_tokens = false; + static bool show_trace = false; + static bool old = false; + static bool two_byte = false; + static int exit_code = 0; + static Encoding encoding = null; + static int string_instance = 0; + static string prefix = ""; + static bool quiet = false; + + static void Main(string[] args) + { + List\ is_fns = new List\(); + List\ inputs = new List\(); + for (int i = 0; i \< args.Length; ++i) + { + if (args[i] == "-profile") + { + show_profile = true; + } + else if (args[i] == "-tokens") + { + show_tokens = true; + } + else if (args[i] == "-two-byte") + { + two_byte = true; + } + else if (args[i] == "-old") + { + old = true; + } + else if (args[i] == "-tree") + { + show_tree = true; + } + else if (args[i] == "-prefix") + { + prefix = args[++i] + " "; + } + else if (args[i] == "-input") + { + inputs.Add(args[++i]); + is_fns.Add(false); + } + else if (args[i] == "-tee") + { + tee = true; + } + else if (args[i] == "-encoding") + { + ++i; + encoding = Encoding.GetEncoding( + args[i], + new EncoderReplacementFallback("(unknown)"), + new DecoderReplacementFallback("(error)")); + if (encoding == null) + throw new Exception(@"Unknown encoding. Must be an Internet Assigned Numbers Authority (IANA) code page name. https://www.iana.org/assignments/character-sets/character-sets.xhtml"); + } + else if (args[i] == "-x") + { + for (; ; ) + { + var line = System.Console.In.ReadLine(); + line = line?.Trim(); + if (line == null || line == "") + { + break; + } + inputs.Add(line); + is_fns.Add(true); + } + } + else if (args[i] == "-q") + { + quiet = true; + } + else if (args[i] == "-trace") + { + show_trace = true; + } + else + { + inputs.Add(args[i]); + is_fns.Add(true); + } + } + if (inputs.Count() == 0) + { + ParseStdin(); + } + else + { + DateTime before = DateTime.Now; + for (int f = 0; f \< inputs.Count(); ++f) + { + if (is_fns[f]) + ParseFilename(inputs[f], f); + else + ParseString(inputs[f], f); + } + DateTime after = DateTime.Now; + if (!quiet) System.Console.Error.WriteLine("Total Time: " + (after - before).TotalSeconds); + } + Environment.ExitCode = exit_code; + } + + static void ParseStdin() + { + ICharStream str = null; + str = CharStreams.fromStream(System.Console.OpenStandardInput()); + DoParse(str, "stdin", 0); + } + + static void ParseString(string input, int row_number) + { + ICharStream str = null; + str = CharStreams.fromString(input); + DoParse(str, "string" + string_instance++, row_number); + } + + static void ParseFilename(string input, int row_number) + { + ICharStream str = null; + if (two_byte) + str = new TwoByteCharStream(input); + else if (old) + { + FileStream fs = new FileStream(input, FileMode.Open); + str = new Antlr4.Runtime.AntlrInputStream(fs); + } + else if (encoding == null) + str = CharStreams.fromPath(input); + else + str = CharStreams.fromPath(input, encoding); + DoParse(str, input, row_number); + } + + static void DoParse(ICharStream str, string input_name, int row_number) + { + var lexer = new (str); + if (show_tokens) + { + StringBuilder new_s = new StringBuilder(); + for (int i = 0; ; ++i) + { + var ro_token = lexer.NextToken(); + var token = (CommonToken)ro_token; + token.TokenIndex = i; + new_s.AppendLine(token.ToString()); + if (token.Type == Antlr4.Runtime.TokenConstants.EOF) + break; + } + System.Console.Error.WriteLine(new_s.ToString()); + lexer.Reset(); + } + var tokens = new CommonTokenStream(lexer); + var parser = new (tokens); + var output = tee ? new StreamWriter(input_name + ".errors") : System.Console.Error; + var listener_lexer = new ErrorListener\(quiet, tee, output); + var listener_parser = new ErrorListener\(quiet, tee, output); + lexer.RemoveErrorListeners(); + parser.RemoveErrorListeners(); + lexer.AddErrorListener(listener_lexer); + parser.AddErrorListener(listener_parser); + if (show_profile) + { + parser.Profile = true; + } + if (show_trace) + { + parser.Trace = true; +// ParserATNSimulator.trace_atn_sim = true; + } + DateTime before = DateTime.Now; + var tree = parser.(); + DateTime after = DateTime.Now; + var result = ""; + if (listener_lexer.had_error || listener_parser.had_error) + { + result = "fail"; + exit_code = 1; + } + else + { + result = "success"; + } + if (show_tree) + { + if (tee) + { + System.IO.File.WriteAllText(input_name + ".tree", tree.ToStringTree(parser)); + } else + { + System.Console.Error.WriteLine(tree.ToStringTree(parser)); + } + } + if (show_profile) + { + System.Console.Error.WriteLine(String.Join(",\n\r", parser.ParseInfo.getDecisionInfo().Select(d => d.ToString()))); + } + if (!quiet) + { + System.Console.Error.WriteLine(prefix + "CSharp " + row_number + " " + input_name + " " + result + " " + (after - before).TotalSeconds); + } + if (tee) output.Close(); + } +} + +} \ No newline at end of file diff --git a/_scripts/templates/CSharp/Test.csproj b/_scripts/templates/CSharp/Test.csproj index df31954bcf..9a095f5cab 100644 --- a/_scripts/templates/CSharp/Test.csproj +++ b/_scripts/templates/CSharp/Test.csproj @@ -1,4 +1,4 @@ -\ +\ \ \ \net6.0\ @@ -13,7 +13,7 @@ } > \ \ \ - \ + \ \ \ \PackageReference\ diff --git a/_scripts/templates/CSharp/build.ps1 b/_scripts/templates/CSharp/build.ps1 new file mode 100644 index 0000000000..9fc944240c --- /dev/null +++ b/_scripts/templates/CSharp/build.ps1 @@ -0,0 +1,7 @@ +# Generated from trgen +if (Test-Path -Path transformGrammar.py -PathType Leaf) { + $(& python3 transformGrammar.py ) 2>&1 | Write-Host +} + +$(& dotnet build; $compile_exit_code = $LASTEXITCODE) | Write-Host +exit $compile_exit_code diff --git a/_scripts/templates/CSharp/build.sh b/_scripts/templates/CSharp/build.sh new file mode 100644 index 0000000000..85c91cecfa --- /dev/null +++ b/_scripts/templates/CSharp/build.sh @@ -0,0 +1,6 @@ +# Generated from trgen +set -e +if [ -f transformGrammar.py ]; then python3 transformGrammar.py ; fi +dotnet restore +dotnet build +exit 0 diff --git a/_scripts/templates/CSharp/clean.ps1 b/_scripts/templates/CSharp/clean.ps1 new file mode 100644 index 0000000000..0dc470b356 --- /dev/null +++ b/_scripts/templates/CSharp/clean.ps1 @@ -0,0 +1,5 @@ +# Generated from trgen +$(& dotnet clean; $status = $LASTEXITCODE) | Write-Host +$(& Remove-Item bin -Recurse -Force ) 2>&1 | Out-Null +$(& Remove-Item obj -Recurse -Force ) 2>&1 | Out-Null +exit 0 diff --git a/_scripts/templates/CSharp/clean.sh b/_scripts/templates/CSharp/clean.sh new file mode 100644 index 0000000000..f802018372 --- /dev/null +++ b/_scripts/templates/CSharp/clean.sh @@ -0,0 +1,4 @@ +# Generated from trgen +dotnet clean +rm -rf bin obj +rm -f }> diff --git a/_scripts/templates/CSharp/makefile b/_scripts/templates/CSharp/makefile index 69e3ad9df1..4a10a46f0d 100644 --- a/_scripts/templates/CSharp/makefile +++ b/_scripts/templates/CSharp/makefile @@ -1,12 +1,10 @@ -# Generated code from trgen -GENERATED = }> +# Generated from trgen default: - dotnet restore - dotnet build + bash build.sh run: - trwdog dotnet run $(RUNARGS) + @trwdog dotnet run $(RUNARGS) clean: - dotnet clean - rm -rf bin obj -test: + bash clean.sh +FORCE: ; +test: FORCE bash test.sh diff --git a/_scripts/templates/CSharp/test.ps1 b/_scripts/templates/CSharp/test.ps1 new file mode 100644 index 0000000000..eb799ca56a --- /dev/null +++ b/_scripts/templates/CSharp/test.ps1 @@ -0,0 +1,149 @@ +# Generated from trgen + +$TestDirectory = "../../" +Write-Host "Test cases here: $TestDirectory" + +# People often specify a test file directory, but sometimes no +# tests are provided. Git won't check in an empty directory. +# Test if the test file directory does not exist, or it is just +# an empty directory. +if (!(Test-Path -Path "$TestDirectory")) { + Write-Host "No test cases provided." + exit 0 +} elseif (!(Test-Path "$TestDirectory/*")) { + Write-Host "No test cases provided." + exit 0 +} + +# Get a list of test files from the test directory. Do not include any +# .errors or .tree files. Pay close attention to remove only file names +# that end with the suffix .errors or .tree. +if (Test-Path -Path "tests.txt" -PathType Leaf) { + Remove-Item "tests.txt" +} +$files = New-Object System.Collections.Generic.List[string] +foreach ($item in Get-ChildItem $TestDirectory -Recurse) { + $file = $item.fullname + $ext = $item.Extension + if (Test-Path $file -PathType Container) { + continue + } elseif ($ext -eq ".errors") { + continue + } elseif ($ext -eq ".tree") { + continue + } else { + $(& triconv -f utf-8 $file ; $last = $LASTEXITCODE ) | Out-Null + if ($last -ne 0) + { + continue + } + $files.Add($item) + Write-Host "Test case: $item" + } +} +foreach ($file in $files) { + Add-Content "tests.txt" $file +} +if (-not(Test-Path -Path "tests.txt" -PathType Leaf)) { + Write-Host "No test cases provided." + exit 0 +} + +# Parse all input files. +get-content "tests.txt" | trwdog dotnet run -q -x -tee -tree *> parse.txt +$status = $LASTEXITCODE + +# trwdog returns 255 if it cannot spawn the process. This could happen +# if the environment for running the program does not exist, or the +# program did not build. +if ( $status -eq 255 ) { + Write-Host "Test failed." + Get-Content $file | Write-Host + exit 1 +} + +# Any parse errors will be put in .errors files. But, if there's any +# output from the program in stdout or stderr, it's all bad news. +$size = (Get-Item -Path "parse.txt").Length +if ( $size -eq 0 ) { +} else { + Write-Host "Test failed." + Get-Content "parse.txt" | Write-Host + exit 1 +} + +# Check if any .errors/.tree files have changed. That's not good. +$message = git diff --exit-code --name-only $TestDirectory +$updated = $LASTEXITCODE +Write-Host $message + +# Check if any untracked .errors files are not empty. +$new_errors_txt = New-Object System.Collections.Generic.List[string] +$new_errors2_txt = git ls-files --exclude-standard -o --ignored $TestDirectory +$new_errors = $LASTEXITCODE + +# Gather up all untracked .errors file output. These are new errors +# and must be reported as a parse fail. +if ( ! [String]::IsNullOrWhiteSpace($new_errors2_txt) ) { + $new_errors3_txt = $new_errors2_txt.Split("\n\r\t ") +} else { + $new_errors3_txt = [System.Collections.Arraylist]@() +} +if (Test-Path -Path "new_errors.txt" -PathType Leaf) { + Remove-Item "new_errors.txt" +} +New-Item -Path . -Name "new_errors.txt" -ItemType "file" -Value "" | Out-Null +foreach ($s in $new_errors3_txt) { + if ( [String]::IsNullOrWhiteSpace($s) ) { + continue + } + $ext = $item.Extension + if (! $s.EndsWith(".errors")) { + continue + } + $file = $s + $size = (Get-Item -Path $file).Length + if ( $size -eq 0 ) { + } else { + $new_errors_txt.Add($item) + Add-Content -Path "new_errors.txt" -Value "$item" + ((Get-Content $file) -join "`n") + "`n" | Add-Content -Path "new_errors.txt" + } +} + +# If "git diff" reported an exit code of 129, it is because +# the directory containing the grammar is not in a repo. In this +# case, assume parse error code as the defacto result. +if ( $updated -eq 129 ) { + Write-Host "Grammar outside a git repository. Assuming parse exit code." + if ( $status -eq 0 ) { + Write-Host "Test succeeded." + } else { + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + } + $err = $status + exit 1 +} + +# "Git diff" reported a difference. Redo the "git diff" to print out all +# the differences. Also, output any untracked, non-zero length .errors files. +if ( $updated -eq 1 ) { + Write-Host "Difference in output." + git diff . | Write-Host + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + exit 1 +} + +# If there's non-zero length .errors flies that are new, report them +# as errors in the parse. +if ( $new_errors_txt.Count -gt 0 ) { + Write-Host "New errors in output." + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + exit 1 +} + +Write-Host "Test succeeded." +exit 0 diff --git a/_scripts/templates/CSharp/test.sh b/_scripts/templates/CSharp/test.sh index f88e98ce0c..ab1bdeccb1 100644 --- a/_scripts/templates/CSharp/test.sh +++ b/_scripts/templates/CSharp/test.sh @@ -1,58 +1,149 @@ -# Template generated code from trgen -err=0 +# Generated from trgen + +# People often specify a test file directory, but sometimes no +# tests are provided. Git won't check in an empty directory. +# Test if the test file directory does not exist, or it is just +# an empty directory. +if [ ! -d ../ ] +then + echo "No test cases provided." + exit 0 +elif [ ! "$(ls -A ../)" ] +then + echo "No test cases provided." + exit 0 +fi + SAVEIFS=$IFS IFS=$(echo -en "\n\b") -parse_out_file=`mktemp --tmpdir=. parse_out.XXXXXXXXXX` -for g in `find ../ -type f | grep -v '.errors$' | grep -v '.tree$'` + +# Get a list of test files from the test directory. Do not include any +# .errors or .tree files. Pay close attention to remove only file names +# that end with the suffix .errors or .tree. +files2=`find ../ -type f | grep -v '.errors$' | grep -v '.tree$'` +files=() +for f in $files2 do - file="$g" - x1="${g##*.}" - if [ "$x1" != "errors" ] - then - echo "$file" - cat "$file" | trwdog ./bin/Debug/net6.0/ -tree > "$parse_out_file" - status="$?" - if [ -f "$file".errors ] + triconv -f utf-8 $f > /dev/null 2>&1 + if [ "$?" = "0" ] then - if [ "$status" = "0" ] - then - echo Expected parse fail. - err=1 - break - else - echo Expected. - diff \<(tr -d "\r\n" \< "$file".errors) \<(tr -d "\r\n" \< "$parse_out_file") - status="$?" - if [ "$status" = "0" ] - then - echo Parse errors match succeeded. - else - echo Expected parse errors match. - err=1 - break - fi - fi - else # No .errors file - if [ "$status" != "0" ] - then - err=1 - break - fi - if [ -f "$file".tree ] - then - diff \<(tr -d "\r\n" \< "$file".tree) \<(tr -d "\r\n" \< "$parse_out_file") - status="$?" - if [ "$status" = "0" ] + files+=( $f ) + fi +done + +# Parse all input files. +echo "${files[*]}" | trwdog ./bin/Debug/net6.0/Test.exeTest -q -x -tee -tree > parse.txt 2>&1 +status=$? + +# trwdog returns 255 if it cannot spawn the process. This could happen +# if the environment for running the program does not exist, or the +# program did not build. +if [ "$status" = "255" ] +then + echo "Test failed." + cat parse.txt + exit 1 +fi + +# Any parse errors will be put in .errors files. But, if there's any +# output from the program in stdout or stderr, it's all bad news. +if [ -s parse.txt ] +then + echo "Test failed." + cat parse.txt + exit 1 +fi + +# rm -rf `find ../ -type f -name '*.errors' -o -name '*.tree' -size 0` + +# For Unix environments, convert the newline in the .errors and .trees +# to Unix style. +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) machine=Linux;; + Darwin*) machine=Mac;; + CYGWIN*) machine=Cygwin;; + MINGW*) machine=MinGw;; + *) machine="UNKNOWN:${unameOut}" +esac +if [[ "$machine" == "MinGw" || "$machine" == "Msys" || "$machine" == "Cygwin" || "#machine" == "Linux" ]] +then + gen=`find ../ -type f -name '*.errors' -o -name '*.tree'` + if [ "$gen" != "" ] + then + dos2unix $gen + fi +fi + +old=`pwd` +cd ../ + +# Check if any files in the test files directory have changed. +git diff --exit-code --name-only . > $old/updated.txt 2>&1 +updated=$? + +# Check if any untracked .errors files. +git ls-files --exclude-standard -o --ignored > $old/new_errors2.txt 2>&1 +new_errors=$? + +# Gather up all untracked .errors file output. These are new errors +# and must be reported as a parse fail. +rm -f $old/new_errors.txt +touch $old/new_errors.txt +for f in `cat $old/new_errors2.txt` +do + ext=${f##*.} + ext=".$ext" + if [ "$ext" = ".errors" ] + then + if [ -s $f ] then - echo Parse tree match succeeded. - else - echo Expected parse tree match. - err=1 - break + echo $f >> $old/new_errors.txt + cat $f >> $old/new_errors.txt fi - fi fi - fi done -rm "$parse_out_file" -exit $err + +# If "git diff" reported an exit code of 129, it is because +# the directory containing the grammar is not in a repo. In this +# case, assume parse error code as the defacto result. +if [ "$updated" = "129" ] +then + echo "Grammar outside a git repository. Assuming parse exit code." + if [ "$status" = 0 ] + then + echo "Test succeeded." + else + cat $old/new_errors.txt + echo "Test failed." + fi + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit $status +fi + +# "Git diff" reported a difference. Redo the "git diff" to print out all +# the differences. Also, output any untracked, non-zero length .errors files. +if [ "$updated" = "1" ] +then + echo "Difference in output." + git diff . + cat $old/new_errors.txt + echo "Test failed." + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit 1 +fi + +# If there's non-zero length .errors flies that are new, report them +# as errors in the parse. +if [ -s $old/new_errors.txt ] +then + echo "New errors in output." + cat $old/new_errors.txt + echo "Test failed." + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit 1 +fi + +echo "Test succeeded." +rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt +exit 0 diff --git a/_scripts/templates/CSharp/tester.psm1 b/_scripts/templates/CSharp/tester.psm1 deleted file mode 100644 index 291c61c5cf..0000000000 --- a/_scripts/templates/CSharp/tester.psm1 +++ /dev/null @@ -1,56 +0,0 @@ -# Template generated code from trgen -# Template generated code from trgen -function Build-Grammar { - $msg = dotnet build -o CSharp - return @{ - Message = $msg - Success = $LASTEXITCODE -eq 0 - } -} - -function Test-Case { - param ( - $InputFile, - $TokenFile, - $TreeFile, - $ErrorFile - ) - # Save input and output character encodings and switch to UTF-8. - $oldInputEncoding = [console]::InputEncoding - $oldOutputEncoding = [console]::OutputEncoding - $OutputEncoding = [console]::InputEncoding = [console]::OutputEncoding = New-Object System.Text.UTF8Encoding - - $parseOutFile = $InputFile + ".out" - $o = trwdog dotnet CSharp/Test.dll -file $InputFile -tree | Out-File -LiteralPath "$parseOutFile" -Encoding UTF8 - - $parseOk = $LASTEXITCODE -eq 0 - $treeMatch = $true - if ($errorFile) { - # If we expected errors, then a failed parse was a successful test. - if (!$parseOk) { - Write-Host "Expected." - # Confirm that the errors we received are the ones we expected. - $expectedData = (Get-Content $errorFile -Encoding UTF8) -join "" -replace "\r\n","" - $actualData = (Get-Content $parseOutFile -Encoding UTF8) -join "" -replace "\r\n","" - $parseOk = ($actualData -eq $expectedData) - if ($parseOk) { - Write-Host "Error list match succeeded." - } else { - Write-Host "Expected error list match." -ForegroundColor Red - } - } - } else { - if ($parseOk -and (Test-Path $TreeFile)) { - # Confirm that the parse tree we received is the one we expected. - $expectedData = Get-Content $TreeFile -Encoding UTF8 - $actualData = Get-Content $parseOutFile -Encoding UTF8 - $treeMatch = ($actualData -eq $expectedData) - } - } - # Restore input and output character encodings. - [console]::InputEncoding = $oldInputEncoding - [console]::OutputEncoding = $oldOutputEncoding - - Remove-Item $parseOutFile - return $parseOk, $treeMatch -} \ No newline at end of file diff --git a/_scripts/templates/Cpp/CMakeLists.txt b/_scripts/templates/Cpp/CMakeLists.txt index cade2b635e..485cd56ad2 100644 --- a/_scripts/templates/Cpp/CMakeLists.txt +++ b/_scripts/templates/Cpp/CMakeLists.txt @@ -1,4 +1,4 @@ -# Template generated code from trgen +# Generated from trgen cmake_minimum_required (VERSION 3.14) list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) @@ -7,12 +7,13 @@ set(CMAKE_CXX_STANDARD 17) set(ANTLR_EXECUTABLE "") set(Java_JAVA_EXECUTABLE "java") set(ANTLR4_TAG v4.11.1) - set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) - #SET(GCC_COVERAGE_COMPILE_FLAGS "-g -pg") #SET(GCC_COVERAGE_LINK_FLAGS "-g -pg") + +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /MT") + #SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS}") #SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}") @@ -32,11 +33,9 @@ include_directories(${ANTLR4_INCLUDE_DIRS}) include_directories(${ANTLR__OUTPUT_DIR}) add_executable(Test - Program.cpp + Test.cpp ErrorListener.cpp ErrorListener.h - CaseChangingCharStream.cpp - CaseChangingCharStream.h } >${ANTLR__CXX_OUTPUTS} ) @@ -59,11 +58,9 @@ include_directories(${ANTLR__OUTPUT_DIR}) include_directories(${ANTLR__OUTPUT_DIR}) add_executable(Test - Program.cpp + Test.cpp ErrorListener.cpp ErrorListener.h - CaseChangingCharStream.cpp - CaseChangingCharStream.h } >${ANTLR__CXX_OUTPUTS} ${ANTLR__CXX_OUTPUTS} @@ -71,4 +68,8 @@ add_executable(Test + +target_compile_options(Test PUBLIC /MT) + + target_link_libraries(Test antlr4_static Threads::Threads) diff --git a/_scripts/templates/Cpp/CaseChangingCharStream.cpp b/_scripts/templates/Cpp/CaseChangingCharStream.cpp deleted file mode 100644 index e5d1502e6e..0000000000 --- a/_scripts/templates/Cpp/CaseChangingCharStream.cpp +++ /dev/null @@ -1,87 +0,0 @@ -// Template generated code from trgen - -#include \ -#include \ -#include "ANTLRInputStream.h" -#include "CommonTokenStream.h" -#include "tree/ParseTree.h" -#include "tree/TerminalNode.h" -#include "tree/TerminalNodeImpl.h" -#include "misc/Interval.h" -#include "ConsoleErrorListener.h" -#include "CharStream.h" -#include "CaseChangingCharStream.h" -#include \ -#include \ -#include \ -#include \ -#include \ - -/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved. - * Use of this file is governed by the BSD 3-clause license that - * can be found in the LICENSE.txt file in the project root. - */ -antlr4::runtime::CaseChangingCharStream::CaseChangingCharStream(antlr4::CharStream* stream, bool upper) -{ - this->stream = stream; - this->upper = upper; -} - -size_t antlr4::runtime::CaseChangingCharStream::index() -{ - return stream->index(); -} - -size_t antlr4::runtime::CaseChangingCharStream::size() -{ - return stream->size(); -} - -std::string antlr4::runtime::CaseChangingCharStream::getSourceName() const -{ - return stream->getSourceName(); -} - -void antlr4::runtime::CaseChangingCharStream::consume() -{ - stream->consume(); -} - -std::string antlr4::runtime::CaseChangingCharStream::getText(const antlr4::misc::Interval& interval) -{ - return stream->getText(interval); -} - -size_t antlr4::runtime::CaseChangingCharStream::LA(ssize_t i) -{ - ssize_t c = stream->LA(i); - if (c \<= 0) - { - return c; - } - if (upper) - { - return (ssize_t)toupper(c); - } - return (ssize_t)tolower(c); -} - -ssize_t antlr4::runtime::CaseChangingCharStream::mark() -{ - return stream->mark(); -} - -void antlr4::runtime::CaseChangingCharStream::release(ssize_t marker) -{ - stream->release(marker); -} - -void antlr4::runtime::CaseChangingCharStream::seek(size_t index) -{ - stream->seek(index); -} - -std::string antlr4::runtime::CaseChangingCharStream::toString() const -{ - return stream->toString(); -} diff --git a/_scripts/templates/Cpp/CaseChangingCharStream.h b/_scripts/templates/Cpp/CaseChangingCharStream.h deleted file mode 100644 index f4e6d9d1cd..0000000000 --- a/_scripts/templates/Cpp/CaseChangingCharStream.h +++ /dev/null @@ -1,56 +0,0 @@ -// Template generated code from trgen - -#pragma once - -#include \ -#include \ -#include "ANTLRInputStream.h" -#include "CommonTokenStream.h" -#include "tree/ParseTree.h" -#include "tree/TerminalNode.h" -#include "tree/TerminalNodeImpl.h" -#include "misc/Interval.h" -#include "ConsoleErrorListener.h" -#include "CharStream.h" - -namespace antlr4 { -namespace runtime { - - /// \ - /// This class supports case-insensitive lexing by wrapping an existing - /// \ and forcing the lexer to see either upper or - /// lowercase characters. Grammar literals should then be either upper or - /// lower case such as 'BEGIN' or 'begin'. The text of the character - /// stream is unaffected. Example: input 'BeGiN' would match lexer rule - /// 'BEGIN' if constructor parameter upper=true but getText() would return - /// 'BeGiN'. - /// \ - class CaseChangingCharStream : public antlr4::CharStream - { - private: - antlr4::CharStream* stream; - private: - bool upper; - - /// \ - /// Constructs a new CaseChangingCharStream wrapping the given \ forcing - /// all characters to upper case or lower case. - /// \ - /// \The stream to wrap.\ - /// \If true force each symbol to upper - /// case, otherwise force to lower.\ - public: - CaseChangingCharStream(antlr4::CharStream* stream, bool upper); - virtual size_t index(); - virtual size_t size(); - virtual std::string getSourceName() const; - virtual void consume(); - virtual std::string getText(const antlr4::misc::Interval& interval); - virtual size_t LA(ssize_t i); - virtual ssize_t mark(); - virtual void release(ssize_t marker); - virtual void seek(size_t index); - virtual std::string toString() const; - }; -} -} \ No newline at end of file diff --git a/_scripts/templates/Cpp/ErrorListener.cpp b/_scripts/templates/Cpp/ErrorListener.cpp index 7e89c0c1db..6a183773c3 100644 --- a/_scripts/templates/Cpp/ErrorListener.cpp +++ b/_scripts/templates/Cpp/ErrorListener.cpp @@ -1,9 +1,24 @@ -// Template generated code from trgen +// Generated from trgen #include "ErrorListener.h" +ErrorListener::ErrorListener(bool quiet, bool tee, std::ostream* output) +{ + had_error = false; + _quiet = quiet; + _tee = tee; + _output = output; +} + void ErrorListener::syntaxError(antlr4::Recognizer* recognizer, antlr4::Token* offendingSymbol, size_t line, size_t col, const std::string& msg, std::exception_ptr e) { had_error = true; - std::cout \<\< "line " \<\< line \<\< ":" \<\< col \<\< " " \<\< msg \<\< std::endl; + if (_tee) + { + (*_output) \<\< "line " \<\< line \<\< ":" \<\< col \<\< " " \<\< msg \<\< std::endl; + } + if (!_quiet) + { + std::cerr \<\< "line " \<\< line \<\< ":" \<\< col \<\< " " \<\< msg \<\< std::endl; + } } diff --git a/_scripts/templates/Cpp/ErrorListener.h b/_scripts/templates/Cpp/ErrorListener.h index ae238b0322..961be7c268 100644 --- a/_scripts/templates/Cpp/ErrorListener.h +++ b/_scripts/templates/Cpp/ErrorListener.h @@ -1,4 +1,4 @@ -// Template generated code from trgen +// Generated from trgen #pragma once @@ -16,6 +16,10 @@ class ErrorListener : public antlr4::ConsoleErrorListener { public: bool had_error; + bool _quiet; + bool _tee; + std::ostream* _output; public: + ErrorListener(bool quiet, bool tee, std::ostream* output); void syntaxError(antlr4::Recognizer* recognizer, antlr4::Token* offendingSymbol, size_t line, size_t col, const std::string& msg, std::exception_ptr e) override; }; diff --git a/_scripts/templates/Cpp/Program.cpp b/_scripts/templates/Cpp/Program.cpp deleted file mode 100644 index d7bb7194db..0000000000 --- a/_scripts/templates/Cpp/Program.cpp +++ /dev/null @@ -1,121 +0,0 @@ -// Template generated code from trgen - -#include \ -#include \ -#include \ -#include \ -#include \ -#include "ANTLRInputStream.h" -#include "ErrorListener.h" -" -} > -#include "CaseChangingCharStream.h" - -std::string formatDuration(uint64_t duration) { - std::stringstream oss; - // duration is in microseconds units. - long tseconds = duration / 1000000; - long minutes = tseconds / 60; - long seconds = tseconds % 60; - long microseconds = duration % 1000000; - oss \<\< std::setfill('0') - \<\< minutes - \<\< ":" - \<\< std::setw(2) - \<\< seconds - \<\< "." - \<\< std::setw(6) - \<\< microseconds; - return oss.str(); -} - -int TryParse(std::vector\& args) -{ - bool show_tree = false; - bool show_tokens = false; - std::string * file_name = nullptr; - std::string * input = nullptr; - for (int i = 0; i \< args.size(); ++i) - { - if (args[i] == "-tokens") - { - show_tokens = true; - continue; - } - else if (args[i] == "-tree") - { - show_tree = true; - continue; - } - else if (args[i] == "-input") - input = & args[++i]; - else if (args[i] == "-file") - file_name = & args[++i]; - } - antlr4::CharStream* str = nullptr; - if (input == nullptr && file_name == nullptr) - { - str = new antlr4::ANTLRInputStream(std::cin); - } else if (input != nullptr) - { - str = new antlr4::ANTLRInputStream(*input); - } else if (file_name != nullptr) - { - std::fstream fs(*file_name); - str = new antlr4::ANTLRInputStream(fs); - } - - bool up = (new std::string("Upper"))->compare("Upper") == 0; - str = new antlr4::runtime::CaseChangingCharStream(str, up); - - antlr4::Lexer * lexer = new (str); - if (show_tokens) - { - for (int i = 0; ; ++i) - { - auto token = lexer->nextToken(); - std::cout \<\< token->toString() \<\< std::endl; - if (token->getType() == antlr4::IntStream::EOF) - break; - } - lexer->reset(); - } - auto tokens = new antlr4::CommonTokenStream(lexer); - auto * parser = new (tokens); - auto listener_lexer = new ErrorListener(); - auto listener_parser = new ErrorListener(); - lexer->removeErrorListeners(); - parser->removeErrorListeners(); - lexer->addErrorListener(listener_lexer); - parser->addErrorListener(listener_parser); - auto before = std::chrono::steady_clock::now(); - auto tree = parser->(); - auto after = std::chrono::steady_clock::now(); - auto duration = std::chrono::duration_cast\(after - before); - if (listener_parser->had_error || listener_lexer->had_error) - { - // Listener will have already printed the error(s) to stdout. - std::cerr \<\< "Parse failed." \<\< std::endl; - } - else - { - std::cerr \<\< "Parse succeeded." \<\< std::endl; - if (show_tree) - { - std::cout \<\< tree->toStringTree(parser, false) \<\< std::endl; - } - } - std::cerr \<\< "Time: " \<\< formatDuration(duration.count()) \<\< std::endl; - return listener_parser->had_error || listener_lexer->had_error ? 1 : 0; -} - -int main(int argc, const char * argv[]) -{ - std::vector \ args; - for (int i = 1; i \< argc; ++i) - { - args.push_back(argv[i]); - } - return TryParse(args); -} - diff --git a/_scripts/templates/Cpp/Test.cpp b/_scripts/templates/Cpp/Test.cpp new file mode 100644 index 0000000000..9e9e925d9d --- /dev/null +++ b/_scripts/templates/Cpp/Test.cpp @@ -0,0 +1,236 @@ +// Generated from trgen + +#include \ +#include \ +#include \ +#include \ +#include \ +#include \ +#include "ANTLRInputStream.h" +#include "ErrorListener.h" +" +} > + +std::string formatDuration(uint64_t duration) { + std::stringstream oss; + // duration is in microseconds units. + long tseconds = duration / 1000000; + long minutes = tseconds / 60; + long seconds = tseconds % 60; + long microseconds = duration % 1000000; + oss \<\< std::setfill('0') + \<\< minutes + \<\< ":" + \<\< std::setw(2) + \<\< seconds + \<\< "." + \<\< std::setw(6) + \<\< microseconds; + return oss.str(); +} + +std::string formatDurationSeconds(uint64_t duration) { + std::stringstream oss; + // duration is in microseconds units. + long tseconds = duration / 1000000; + long minutes = tseconds / 60; + long seconds = tseconds % 60; + long microseconds = duration % 1000000; + double s = minutes * 60.0 + seconds + (microseconds / 1000000.0); + oss \<\< s; + return oss.str(); +} + +bool tee = false; +bool show_tree = false; +bool show_tokens = false; +bool show_trace = false; +std::vector\ inputs; +std::vector\ is_fns; +int error_code = 0; +int string_instance = 0; +std::string prefix; +bool quiet = false; + +void DoParse(antlr4::CharStream* str, std::string input_name, int row_number) +{ + antlr4::Lexer* lexer = new (str); + if (show_tokens) + { + for (int i = 0; ; ++i) + { + auto token = lexer->nextToken(); + std::cerr \<\< token->toString() \<\< std::endl; + if (token->getType() == antlr4::IntStream::EOF) + break; + } + lexer->reset(); + } + auto tokens = new antlr4::CommonTokenStream(lexer); + auto* parser = new (tokens); + std::ostream* output = tee + ? new std::ofstream(input_name + ".errors") + : &std::cerr; + auto listener_lexer = new ErrorListener(quiet, tee, output); + auto listener_parser = new ErrorListener(quiet, tee, output); + lexer->removeErrorListeners(); + parser->removeErrorListeners(); + lexer->addErrorListener(listener_lexer); + parser->addErrorListener(listener_parser); + if (show_trace) + { + parser->setTrace(true); + // Missing ATN trace. + } + auto before = std::chrono::steady_clock::now(); + auto* tree = parser->(); + auto after = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast\(after - before); + std::string result; + if (listener_parser->had_error || listener_lexer->had_error) + { + result = "fail"; + error_code = 1; + } + else + { + result = "success"; + } + if (show_tree) + { + if (tee) + { + try { + auto fn = input_name + ".tree"; + auto out = new std::ofstream(fn); + (*out) \<\< tree->toStringTree(parser); + delete out; + } + catch (...) { + } + } + else + { + std::cerr \<\< tree->toStringTree(parser) \<\< std::endl; + } + } + if (!quiet) + { + std::cerr \<\< prefix \<\< "Cpp " \<\< row_number \<\< " " \<\< input_name \<\< " " \<\< result \<\< " " \<\< formatDurationSeconds(duration.count()) \<\< std::endl; + } + if (tee) + { + delete output; + } +} + + +void ParseStdin() +{ + antlr4::CharStream* str = nullptr; + str = new antlr4::ANTLRInputStream(std::cin); + DoParse(str, "stdin", 0); +} + +void ParseString(std::string input, int row_number) +{ + antlr4::CharStream* str = nullptr; + str = new antlr4::ANTLRInputStream(input); + DoParse(str, "string" + string_instance++, row_number); +} + +void ParseFilename(std::string input, int row_number) +{ + antlr4::CharStream* str = nullptr; + std::fstream fs(input); + str = new antlr4::ANTLRInputStream(fs); + DoParse(str, input, row_number); +} + +int TryParse(std::vector\& args) +{ + for (int i = 0; i \< args.size(); ++i) + { + if (args[i] == "-tokens") + { + show_tokens = true; + } + else if (args[i] == "-tree") + { + show_tree = true; + } + else if (args[i] == "-prefix") + { + prefix = args[++i] + " "; + } + else if (args[i] == "-input") + { + ++i; + inputs.push_back(args[i]); + is_fns.push_back(false); + } + else if (args[i] == "-tee") + { + tee = true; + } + else if (args[i] == "-x") + { + for (; ; ) + { + std::string line; + if (! std::getline(std::cin, line)) break; + std::string_view v = line; + v.remove_prefix(std::min(v.find_first_not_of(" "), v.size())); + if (line == "") + { + break; + } + inputs.push_back(line); + is_fns.push_back(true); + } + } + else if (args[i] == "-q") + { + quiet = true; + } + else if (args[i] == "-trace") + { + show_trace = true; + } + else + { + inputs.push_back(args[i]); + is_fns.push_back(true); + } + } + if (inputs.size() == 0) + { + ParseStdin(); + } + else + { + auto before = std::chrono::steady_clock::now(); + for (int f = 0; f \< inputs.size(); ++f) + { + if (is_fns[f]) + ParseFilename(inputs[f], f); + else + ParseString(inputs[f], f); + } + auto after = std::chrono::steady_clock::now(); + auto duration = std::chrono::duration_cast\(after - before); + if (! quiet) std::cerr \<\< "Total Time: " \<\< formatDurationSeconds(duration.count()) \<\< std::endl; + } + return error_code; +} + +int main(int argc, const char * argv[]) +{ + std::vector \ args; + for (int i = 1; i \< argc; ++i) + { + args.push_back(argv[i]); + } + return TryParse(args); +} + diff --git a/_scripts/templates/Cpp/build.ps1 b/_scripts/templates/Cpp/build.ps1 new file mode 100644 index 0000000000..bcf8b360e8 --- /dev/null +++ b/_scripts/templates/Cpp/build.ps1 @@ -0,0 +1,30 @@ +# Generated from trgen +function rmrf([string]$Path) { + try { + Remove-Item -Recurse -ErrorAction:Stop $Path + } catch [System.Management.Automation.ItemNotFoundException] { + # Ignore + $Error.Clear() + } +} + +if (Test-Path -Path transformGrammar.py -PathType Leaf) { + $(& python3 transformGrammar.py ) 2>&1 | Write-Host +} + +rmrf('build') +New-Item -Path 'build' -ItemType Directory +Set-Location 'build' + +$(& cmake .. ; $compile_exit_code = $LASTEXITCODE ) | Write-Host +if($compile_exit_code -ne 0){ + Write-Host "Failed first cmake call $compile_exit_code." + exit $compile_exit_code +} + +$(& cmake --build . --config Release$(MAKE) ; $compile_exit_code = $LASTEXITCODE ) | Write-Host +if($compile_exit_code -ne 0){ + Write-Host "Failed second cmake call $compile_exit_code." + exit $compile_exit_code +} +exit $compile_exit_code diff --git a/_scripts/templates/Cpp/build.sh b/_scripts/templates/Cpp/build.sh new file mode 100644 index 0000000000..581c68cde6 --- /dev/null +++ b/_scripts/templates/Cpp/build.sh @@ -0,0 +1,9 @@ +# Generated from trgen +set -e +if [ -f transformGrammar.py ]; then python3 transformGrammar.py ; fi +rm -rf build +mkdir build +cd build +cmake .. +cmake --build . --config Releasemake +exit 0 diff --git a/_scripts/templates/Cpp/clean.ps1 b/_scripts/templates/Cpp/clean.ps1 new file mode 100644 index 0000000000..1030e6bd05 --- /dev/null +++ b/_scripts/templates/Cpp/clean.ps1 @@ -0,0 +1,4 @@ +# Generated from trgen +$(& Remove-Item build -Recurse -Force ) 2>&1 | Out-Null +$(& Remove-Item /antlr4_runtime -Recurse -Force ) 2>&1 | Out-Null +exit 0 diff --git a/_scripts/templates/Cpp/clean.sh b/_scripts/templates/Cpp/clean.sh new file mode 100644 index 0000000000..cc6f97d43a --- /dev/null +++ b/_scripts/templates/Cpp/clean.sh @@ -0,0 +1,2 @@ +# Generated from trgen +rm -rf build /antlr4_runtime diff --git a/_scripts/templates/Cpp/cmake/ExternalAntlr4Cpp.cmake b/_scripts/templates/Cpp/cmake/ExternalAntlr4Cpp.cmake index 19d21fd35a..593d0eae0f 100644 --- a/_scripts/templates/Cpp/cmake/ExternalAntlr4Cpp.cmake +++ b/_scripts/templates/Cpp/cmake/ExternalAntlr4Cpp.cmake @@ -13,7 +13,7 @@ if(NOT DEFINED ANTLR4_TAG) endif() if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*") - set(ANTLR4_OUTPUT_DIR ${ANTLR4_ROOT}/runtime/Cpp/dist/$(Configuration)) + set(ANTLR4_OUTPUT_DIR ${ANTLR4_ROOT}/runtime/Cpp/dist) elseif(${CMAKE_GENERATOR} MATCHES "Xcode.*") set(ANTLR4_OUTPUT_DIR ${ANTLR4_ROOT}/runtime/Cpp/dist/$(CONFIGURATION)) else() diff --git a/_scripts/templates/Cpp/cmake/FindANTLR.cmake b/_scripts/templates/Cpp/cmake/FindANTLR.cmake index b07d58bda0..2d6e70e24d 100644 --- a/_scripts/templates/Cpp/cmake/FindANTLR.cmake +++ b/_scripts/templates/Cpp/cmake/FindANTLR.cmake @@ -14,7 +14,7 @@ if(ANTLR_EXECUTABLE AND Java_JAVA_EXECUTABLE) OUTPUT_STRIP_TRAILING_WHITESPACE) if(ANTLR_COMMAND_RESULT EQUAL 0) - string(REGEX MATCH "Version [0-9]+(\\\\.[0-9])*" ANTLR_VERSION ${ANTLR_COMMAND_OUTPUT}) + string(REGEX MATCH "Version [0-9]+(\\\\.[0-9]+)*" ANTLR_VERSION ${ANTLR_COMMAND_OUTPUT}) string(REPLACE "Version " "" ANTLR_VERSION ${ANTLR_VERSION}) else() message( diff --git a/_scripts/templates/Cpp/makefile b/_scripts/templates/Cpp/makefile index 4336925b83..fd63eb1289 100644 --- a/_scripts/templates/Cpp/makefile +++ b/_scripts/templates/Cpp/makefile @@ -1,11 +1,10 @@ -# Template generated code from trgen -default: setup - rm -rf build; mkdir build; cd build; cmake .. ; $(MAKE) -setup: - if [ -f transformGrammar.py ]; then python3 transformGrammar.py ; fi +# Generated from trgen +default: + bash build.sh run: - cd build; trwdog ./ $(RUNARGS) + @trwdog ./build/Release/ $(RUNARGS) clean: - rm -rf build /antlr4_runtime -test: + bash clean.sh +FORCE: ; +test: FORCE bash test.sh diff --git a/_scripts/templates/Cpp/test.ps1 b/_scripts/templates/Cpp/test.ps1 new file mode 100644 index 0000000000..009b87d6a5 --- /dev/null +++ b/_scripts/templates/Cpp/test.ps1 @@ -0,0 +1,149 @@ +# Generated from trgen + +$TestDirectory = "../../" +Write-Host "Test cases here: $TestDirectory" + +# People often specify a test file directory, but sometimes no +# tests are provided. Git won't check in an empty directory. +# Test if the test file directory does not exist, or it is just +# an empty directory. +if (!(Test-Path -Path "$TestDirectory")) { + Write-Host "No test cases provided." + exit 0 +} elseif (!(Test-Path "$TestDirectory/*")) { + Write-Host "No test cases provided." + exit 0 +} + +# Get a list of test files from the test directory. Do not include any +# .errors or .tree files. Pay close attention to remove only file names +# that end with the suffix .errors or .tree. +if (Test-Path -Path "tests.txt" -PathType Leaf) { + Remove-Item "tests.txt" +} +$files = New-Object System.Collections.Generic.List[string] +foreach ($item in Get-ChildItem $TestDirectory -Recurse) { + $file = $item.fullname + $ext = $item.Extension + if (Test-Path $file -PathType Container) { + continue + } elseif ($ext -eq ".errors") { + continue + } elseif ($ext -eq ".tree") { + continue + } else { + $(& triconv -f utf-8 $file ; $last = $LASTEXITCODE ) | Out-Null + if ($last -ne 0) + { + continue + } + $files.Add($item) + Write-Host "Test case: $item" + } +} +foreach ($file in $files) { + Add-Content "tests.txt" $file +} +if (-not(Test-Path -Path "tests.txt" -PathType Leaf)) { + Write-Host "No test cases provided." + exit 0 +} + +# Parse all input files. +get-content "tests.txt" | trwdog ./build/Release/ -q -x -tee -tree *> parse.txt +$status = $LASTEXITCODE + +# trwdog returns 255 if it cannot spawn the process. This could happen +# if the environment for running the program does not exist, or the +# program did not build. +if ( $status -eq 255 ) { + Write-Host "Test failed." + Get-Content $file | Write-Host + exit 1 +} + +# Any parse errors will be put in .errors files. But, if there's any +# output from the program in stdout or stderr, it's all bad news. +$size = (Get-Item -Path "parse.txt").Length +if ( $size -eq 0 ) { +} else { + Write-Host "Test failed." + Get-Content "parse.txt" | Write-Host + exit 1 +} + +# Check if any .errors/.tree files have changed. That's not good. +$message = git diff --exit-code --name-only $TestDirectory +$updated = $LASTEXITCODE +Write-Host $message + +# Check if any untracked .errors files are not empty. +$new_errors_txt = New-Object System.Collections.Generic.List[string] +$new_errors2_txt = git ls-files --exclude-standard -o --ignored $TestDirectory +$new_errors = $LASTEXITCODE + +# Gather up all untracked .errors file output. These are new errors +# and must be reported as a parse fail. +if ( ! [String]::IsNullOrWhiteSpace($new_errors2_txt) ) { + $new_errors3_txt = $new_errors2_txt.Split("\n\r\t ") +} else { + $new_errors3_txt = [System.Collections.Arraylist]@() +} +if (Test-Path -Path "new_errors.txt" -PathType Leaf) { + Remove-Item "new_errors.txt" +} +New-Item -Path . -Name "new_errors.txt" -ItemType "file" -Value "" | Out-Null +foreach ($s in $new_errors3_txt) { + if ( [String]::IsNullOrWhiteSpace($s) ) { + continue + } + $ext = $item.Extension + if (! $s.EndsWith(".errors")) { + continue + } + $file = $s + $size = (Get-Item -Path $file).Length + if ( $size -eq 0 ) { + } else { + $new_errors_txt.Add($item) + Add-Content -Path "new_errors.txt" -Value "$item" + ((Get-Content $file) -join "`n") + "`n" | Add-Content -Path "new_errors.txt" + } +} + +# If "git diff" reported an exit code of 129, it is because +# the directory containing the grammar is not in a repo. In this +# case, assume parse error code as the defacto result. +if ( $updated -eq 129 ) { + Write-Host "Grammar outside a git repository. Assuming parse exit code." + if ( $status -eq 0 ) { + Write-Host "Test succeeded." + } else { + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + } + $err = $status + exit 1 +} + +# "Git diff" reported a difference. Redo the "git diff" to print out all +# the differences. Also, output any untracked, non-zero length .errors files. +if ( $updated -eq 1 ) { + Write-Host "Difference in output." + git diff . | Write-Host + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + exit 1 +} + +# If there's non-zero length .errors flies that are new, report them +# as errors in the parse. +if ( $new_errors_txt.Count -gt 0 ) { + Write-Host "New errors in output." + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + exit 1 +} + +Write-Host "Test succeeded." +exit 0 diff --git a/_scripts/templates/Cpp/test.sh b/_scripts/templates/Cpp/test.sh index 6985680620..6f37e2e1b7 100644 --- a/_scripts/templates/Cpp/test.sh +++ b/_scripts/templates/Cpp/test.sh @@ -1,58 +1,149 @@ -# Template generated code from trgen -err=0 +# Generated from trgen + +# People often specify a test file directory, but sometimes no +# tests are provided. Git won't check in an empty directory. +# Test if the test file directory does not exist, or it is just +# an empty directory. +if [ ! -d ../ ] +then + echo "No test cases provided." + exit 0 +elif [ ! "$(ls -A ../)" ] +then + echo "No test cases provided." + exit 0 +fi + SAVEIFS=$IFS IFS=$(echo -en "\n\b") -parse_out_file=`mktemp --tmpdir=. parse_out.XXXXXXXXXX` -for g in `find ../ -type f | grep -v '.errors$' | grep -v '.tree$'` + +# Get a list of test files from the test directory. Do not include any +# .errors or .tree files. Pay close attention to remove only file names +# that end with the suffix .errors or .tree. +files2=`find ../ -type f | grep -v '.errors$' | grep -v '.tree$'` +files=() +for f in $files2 do - file="$g" - x1="${g##*.}" - if [ "$x1" != "errors" ] - then - echo "$file" - trwdog ./build/ -file "$file" -tree > "$parse_out_file" - status="$?" - if [ -f "$file".errors ] + triconv -f utf-8 $f > /dev/null 2>&1 + if [ "$?" = "0" ] then - if [ "$status" = "0" ] - then - echo Expected parse fail. - err=1 - break - else - echo Expected. - diff \<(tr -d "\r\n" \< "$file".errors) \<(tr -d "\r\n" \< "$parse_out_file") - status="$?" - if [ "$status" = "0" ] - then - echo Parse errors match succeeded. - else - echo Expected parse errors match. - err=1 - break - fi - fi - else # No .errors file - if [ "$status" != "0" ] - then - err=1 - break - fi - if [ -f "$file".tree ] - then - diff \<(tr -d "\r\n" \< "$file".tree) \<(tr -d "\r\n" \< "$parse_out_file") - status="$?" - if [ "$status" = "0" ] + files+=( $f ) + fi +done + +# Parse all input files. +echo "${files[*]}" | trwdog ./build/Release/ -q -x -tee -tree > parse.txt 2>&1 +status=$? + +# trwdog returns 255 if it cannot spawn the process. This could happen +# if the environment for running the program does not exist, or the +# program did not build. +if [ "$status" = "255" ] +then + echo "Test failed." + cat parse.txt + exit 1 +fi + +# Any parse errors will be put in .errors files. But, if there's any +# output from the program in stdout or stderr, it's all bad news. +if [ -s parse.txt ] +then + echo "Test failed." + cat parse.txt + exit 1 +fi + +# rm -rf `find ../ -type f -name '*.errors' -o -name '*.tree' -size 0` + +# For Unix environments, convert the newline in the .errors and .trees +# to Unix style. +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) machine=Linux;; + Darwin*) machine=Mac;; + CYGWIN*) machine=Cygwin;; + MINGW*) machine=MinGw;; + *) machine="UNKNOWN:${unameOut}" +esac +if [[ "$machine" == "MinGw" || "$machine" == "Msys" || "$machine" == "Cygwin" || "#machine" == "Linux" ]] +then + gen=`find ../ -type f -name '*.errors' -o -name '*.tree'` + if [ "$gen" != "" ] + then + dos2unix $gen + fi +fi + +old=`pwd` +cd ../ + +# Check if any files in the test files directory have changed. +git diff --exit-code --name-only . > $old/updated.txt 2>&1 +updated=$? + +# Check if any untracked .errors files. +git ls-files --exclude-standard -o --ignored > $old/new_errors2.txt 2>&1 +new_errors=$? + +# Gather up all untracked .errors file output. These are new errors +# and must be reported as a parse fail. +rm -f $old/new_errors.txt +touch $old/new_errors.txt +for f in `cat $old/new_errors2.txt` +do + ext=${f##*.} + ext=".$ext" + if [ "$ext" = ".errors" ] + then + if [ -s $f ] then - echo Parse tree match succeeded. - else - echo Expected parse tree match. - err=1 - break + echo $f >> $old/new_errors.txt + cat $f >> $old/new_errors.txt fi - fi fi - fi done -rm "$parse_out_file" -exit $err + +# If "git diff" reported an exit code of 129, it is because +# the directory containing the grammar is not in a repo. In this +# case, assume parse error code as the defacto result. +if [ "$updated" = "129" ] +then + echo "Grammar outside a git repository. Assuming parse exit code." + if [ "$status" = 0 ] + then + echo "Test succeeded." + else + cat $old/new_errors.txt + echo "Test failed." + fi + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit $status +fi + +# "Git diff" reported a difference. Redo the "git diff" to print out all +# the differences. Also, output any untracked, non-zero length .errors files. +if [ "$updated" = "1" ] +then + echo "Difference in output." + git diff . + cat $old/new_errors.txt + echo "Test failed." + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit 1 +fi + +# If there's non-zero length .errors flies that are new, report them +# as errors in the parse. +if [ -s $old/new_errors.txt ] +then + echo "New errors in output." + cat $old/new_errors.txt + echo "Test failed." + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit 1 +fi + +echo "Test succeeded." +rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt +exit 0 diff --git a/_scripts/templates/Dart/MyErrorListener.dart b/_scripts/templates/Dart/MyErrorListener.dart new file mode 100644 index 0000000000..13c4ffa5f2 --- /dev/null +++ b/_scripts/templates/Dart/MyErrorListener.dart @@ -0,0 +1,77 @@ +// Generated from trgen + +import 'package:antlr4/antlr4.dart'; +import 'dart:io'; +import 'dart:convert'; + +/// This is all the parsing support code essentially; most of it is error recovery stuff. */ +class MyErrorListener extends BaseErrorListener +{ + bool _quiet = false; + bool had_error = false; + bool _tee = false; + IOSink _output = stdout; + + MyErrorListener(bool quiet, bool tee, IOSink output) + { + had_error = false; + _quiet = quiet; + _tee = tee; + _output = output; + } + + @override + void syntaxError( + Recognizer recognizer, + Object? offendingSymbol, + int? line, + int charPositionInLine, + String msg, + RecognitionException? e, + ) { + had_error = true; + if (_tee) + { + _output.writeln('line $line:$charPositionInLine $msg'); + } + if (!_quiet) + { + stderr.writeln('line $line:$charPositionInLine $msg'); + } + } + + /* @override + void reportAmbiguity( + Parser recognizer, + DFA dfa, + int startIndex, + int stopIndex, + bool exact, + BitSet? ambigAlts, + ATNConfigSet configs, + ) {} + */ + +/* @override + void reportAttemptingFullContext( + Parser recognizer, + DFA dfa, + int startIndex, + int stopIndex, + BitSet? conflictingAlts, + ATNConfigSet configs, + ) {} + */ + +/* + @override + void reportContextSensitivity( + Parser recognizer, + DFA dfa, + int startIndex, + int stopIndex, + int prediction, + ATNConfigSet configs, + ) {} + */ +} diff --git a/_scripts/templates/Dart/Test.dart b/_scripts/templates/Dart/Test.dart new file mode 100644 index 0000000000..d15752efd4 --- /dev/null +++ b/_scripts/templates/Dart/Test.dart @@ -0,0 +1,199 @@ +// Generated from trgen + +import 'package:antlr4/antlr4.dart'; +'; +} > +import 'MyErrorListener.dart'; +import 'dart:io'; +import 'dart:convert'; + +var tee = false; +var show_tree = false; +var show_tokens = false; +var show_trace = false; +var inputs = List\.empty(growable: true); +var is_fns = List\.empty(growable: true); +var error_code = 0; +var string_instance = 0; +var prefix = ""; +var quiet = false; + +void main(List\ args) async { + for (int i = 0; i \< args.length; ++i) + { + if (args[i] == "-tokens") + { + show_tokens = true; + } + else if (args[i] == "-tree") + { + show_tree = true; + } + else if (args[i] == "-prefix") + { + prefix = args[++i] + " "; + } + else if (args[i] == "-input") + { + inputs.add(args[++i]); + is_fns.add(false); + } + else if (args[i] == "-tee") + { + tee = true; + } + else if (args[i] == "-x") + { + for (; ; ) + { + String? line = stdin.readLineSync(encoding: utf8); + line = line?.trim(); + if (line == null || line == "") + { + break; + } + inputs.add(line); + is_fns.add(true); + } + } + else if (args[i] == "-q") + { + quiet = true; + } + else if (args[i] == "-trace") + { + show_trace = true; + } + else + { + inputs.add(args[i]); + is_fns.add(true); + } + } + if (inputs.length == 0) + { + await ParseStdin(); + } + else + { + Stopwatch s = new Stopwatch(); + s.start(); + for (int f = 0; f \< inputs.length; ++f) + { + if (is_fns[f]) + await ParseFilename(inputs[f], f); + else + await ParseString(inputs[f], f); + } + s.stop(); + var et = s.elapsedMilliseconds / 1000.0; + if (!quiet) stderr.writeln("Total Time: " + et.toString()); + } + exit(error_code); +} + +Future\ ParseStdin() async +{ + final List\ bytes = \[]; + int byte = stdin.readByteSync(); + while (byte >= 0) { + bytes.add(byte); + byte = stdin.readByteSync(); + } + var input = utf8.decode(bytes); + var str = await InputStream.fromString(input); + await DoParse(str, "stdin", 0); +} + +Future\ ParseString(String input, int row_number) async +{ + var str = await InputStream.fromString(input); + await DoParse(str, "string" + string_instance.toString(), row_number); + string_instance++; +} + +Future\ ParseFilename(String input, int row_number) async +{ + var str = await InputStream.fromPath(input); + await DoParse(str, input, row_number); +} + +Future\ DoParse(CharStream str, String input_name, int row_number) async +{ + .checkVersion(); + }> + var lexer = (str); + if (show_tokens) + { + for (int i = 0; ; ++i) + { + var token = lexer.nextToken(); + stderr.writeln(token.toString()); + if (token.type == -1) + break; + } + lexer.reset(); + } + var tokens = CommonTokenStream(lexer); + var parser = (tokens); + IOSink output; + if (tee) + { + output = File(input_name + ".errors").openWrite(); + } + else + { + output = stderr; + } + var listener_lexer = new MyErrorListener(quiet, tee, output); + var listener_parser = new MyErrorListener(quiet, tee, output); + lexer.removeErrorListeners(); + parser.removeErrorListeners(); + lexer.addErrorListener(listener_lexer); + parser.addErrorListener(listener_parser); + if (show_trace) + { + parser.setTrace(true); + // Missing ATN trace. + } + Stopwatch s = new Stopwatch(); + s.start(); + var tree = parser.(); + s.stop(); + var et = s.elapsedMilliseconds / 1000.0; + var result = ""; + if (parser.numberOfSyntaxErrors > 0) + { + result = "fail"; + error_code = 1; + } + else + { + result = "success"; + } + if (show_tree) + { + if (tee) + { + try { + final fn = input_name + ".tree"; + final File file = File(fn); + await file.writeAsString(tree.toStringTree(parser: parser)); + } catch (e) { + } + } + else + { + stderr.writeln(tree.toStringTree(parser: parser)); + } + } + if (!quiet) + { + stderr.writeln(prefix + "Dart " + row_number.toString() + " " + input_name + " " + result + " " + et.toString()); + } + if (tee) + { + await output.flush(); + await output.close(); + } +} diff --git a/_scripts/templates/Dart/build.ps1 b/_scripts/templates/Dart/build.ps1 new file mode 100644 index 0000000000..d7e82469e3 --- /dev/null +++ b/_scripts/templates/Dart/build.ps1 @@ -0,0 +1,23 @@ +# Generated from trgen +if (Test-Path -Path transformGrammar.py -PathType Leaf) { + $(& python3 transformGrammar.py ) 2>&1 | Write-Host +} + + -encoding -Dlanguage=Dart } > ; $compile_exit_code = $LASTEXITCODE) | Write-Host +if($compile_exit_code -ne 0){ + exit $compile_exit_code +\} +}> +For ($i=0; $i -le 5; $i++) { + $(& dart pub get; $compile_exit_code = $LASTEXITCODE) | Write-Host + if($compile_exit_code -eq 0){ + Break + } + Write-Host "dart pub get failed. Trying again." +} +if($compile_exit_code -ne 0){ + exit $compile_exit_code +} +$(& dart compile exe Test.dart; $compile_exit_code = $LASTEXITCODE) | Write-Host +exit $compile_exit_code diff --git a/_scripts/templates/Dart/build.sh b/_scripts/templates/Dart/build.sh new file mode 100644 index 0000000000..721a25d1d0 --- /dev/null +++ b/_scripts/templates/Dart/build.sh @@ -0,0 +1,12 @@ +# Generated from trgen +set -e + +if [ -f transformGrammar.py ]; then python3 transformGrammar.py ; fi + +" -encoding -Dlanguage=Dart } > +} > + +dart pub get + +dart compile exe Test.dart diff --git a/_scripts/templates/Dart/clean.ps1 b/_scripts/templates/Dart/clean.ps1 new file mode 100644 index 0000000000..ef8a878a84 --- /dev/null +++ b/_scripts/templates/Dart/clean.ps1 @@ -0,0 +1,17 @@ +# Generated from trgen +$(& Remove-Item *.interp -Recurse -Force ) 2>&1 | Out-Null +$files = New-Object System.Collections.Generic.List[string] +" -depend -encoding -Dlanguage=Dart } > +foreach ($s in $f) { + $j = $s.Split(" ")[0] + $files.Add($j) +\} +foreach ($f in $files) +{ + $(& Remove-Item $f -Force ) 2>&1 | Out-Null +\} +} > +$(& Remove-Item pubspec.lock -Force ) 2>&1 | Out-Null +$(& Remove-Item Test.exe -Force ) 2>&1 | Out-Null +exit 0 diff --git a/_scripts/templates/Dart/clean.sh b/_scripts/templates/Dart/clean.sh new file mode 100644 index 0000000000..3e648f6905 --- /dev/null +++ b/_scripts/templates/Dart/clean.sh @@ -0,0 +1,12 @@ +# Generated from trgen +rm -f *.interp +files=() +" -depend -encoding -Dlanguage=Dart } > | awk '{print $1\}' | grep -v ':'` ) +} > +for i in ${files[*]} +do + rm -f $i +done +rm -f pubspec.lock +rm -f Test.exe diff --git a/_scripts/templates/Dart/cli.dart b/_scripts/templates/Dart/cli.dart deleted file mode 100644 index 30fe16fbb0..0000000000 --- a/_scripts/templates/Dart/cli.dart +++ /dev/null @@ -1,177 +0,0 @@ -// Template generated code from trgen - -import 'package:antlr4/antlr4.dart'; -'; -} > -import 'dart:io'; -import 'dart:convert'; - - - -/// This class supports case-insensitive lexing by wrapping an existing -/// {@link CharStream} and forcing the lexer to see either upper or -/// lowercase characters. Grammar literals should then be either upper or -/// lower case such as 'BEGIN' or 'begin'. The text of the character -/// stream is unaffected. Example: input 'BeGiN' would match lexer rule -/// 'BEGIN' if constructor parameter upper=true but getText() would return -/// 'BeGiN'. -class CaseChangingCharStream extends CharStream { - final CharStream stream; - final bool upper; - - /// Constructs a new CaseChangingCharStream wrapping the given [stream] forcing - /// all characters to upper case or lower case depending on [upper]. - CaseChangingCharStream(this.stream, this.upper); - - @override - int? LA(int i) { - int? c = stream.LA(i); - if (c == null || c \<= 0) { - return c; - } - String newCaseStr; - if (upper) { - newCaseStr = String.fromCharCode(c).toUpperCase(); - } else { - newCaseStr = String.fromCharCode(c).toLowerCase(); - } - // Skip changing case if length changes (e.g., ß -> SS). - if (newCaseStr.length != 1) { - return c; - } else { - return newCaseStr.codeUnitAt(0); - } - } - - @override - String get sourceName => stream.sourceName; - - @override - void consume() => stream.consume(); - - @override - String getText(Interval interval) => stream.getText(interval); - - @override - int get index => stream.index; - - @override - int mark() => stream.mark(); - - @override - void release(int marker) => stream.release(marker); - - @override - void seek(int index) => stream.seek(index); - - @override - int get size => stream.size; -} - - - -class MyErrorListener extends BaseErrorListener { - /// Provides a default instance of [MyErrorListener]. - static final INSTANCE = MyErrorListener(); - - /// {@inheritDoc} - /// - ///

- /// This implementation prints messages to {@link System//err} containing the - /// values of [line], [charPositionInLine], and [msg] using - /// the following format.

- /// - /// 
-  /// line line:charPositionInLine msg
-  ///
- @override - void syntaxError(recognizer, offendingSymbol, line, column, msg, e) { - stdout.writeln('line $line:$column $msg'); - } -} - - -void main(List\ args) async { - var show_tree = false; - var show_tokens = false; - var file_name = null; - var input = null; - var str = null; - for (int i = 0; i \< args.length; ++i) - { - if (args[i] == "-tokens") - { - show_tokens = true; - continue; - } - else if (args[i] == "-tree") - { - show_tree = true; - continue; - } - else if (args[i] == "-input") - input = args[++i]; - else if (args[i] == "-file") - file_name = args[++i]; - } - .checkVersion(); - }> - if (input == null && file_name == null) - { - final List\ bytes = \[]; - int byte = stdin.readByteSync(); - while (byte >= 0) { - bytes.add(byte); - byte = stdin.readByteSync(); - } - input = utf8.decode(bytes); - str = await InputStream.fromString(input); - } else if (input != null) - { - str = await InputStream.fromString(input); - } else if (file_name != null) - { - str = await InputStream.fromPath(file_name); - } - - str = CaseChangingCharStream(str, "" == "Upper"); - - var lexer = (str); - if (show_tokens) - { - for (int i = 0; ; ++i) - { - var token = lexer.nextToken(); - print(token.toString()); - if (token.type == -1) - break; - } - lexer.reset(); - } - var tokens = CommonTokenStream(lexer); - var parser = (tokens); - var listener_lexer = MyErrorListener(); - var listener_parser = MyErrorListener(); - lexer.addErrorListener(listener_lexer); - parser.addErrorListener(listener_parser); - Stopwatch s = new Stopwatch(); - s.start(); - var tree = parser.(); - s.stop(); - var et = s.elapsedMilliseconds / 1000.0; - stderr.writeln("Time: " + et.toString()); - if (parser.numberOfSyntaxErrors > 0) - { - // Listener will have already printed the error(s) to stdout. - stderr.writeln("Parse failed."); - } - else - { - stderr.writeln("Parse succeeded."); - if (show_tree) - { - print(tree.toStringTree(parser: parser)); - } - } - exit(parser.numberOfSyntaxErrors > 0 ? 1 : 0); -} diff --git a/_scripts/templates/Dart/makefile b/_scripts/templates/Dart/makefile index 6a218ec96c..83ef287e77 100644 --- a/_scripts/templates/Dart/makefile +++ b/_scripts/templates/Dart/makefile @@ -1,25 +1,10 @@ -# Template generated code from trgen -JAR = -CLASSPATH = $(JAR)\;:. -.SUFFIXES: .g4 .dart -ANTLRGRAMMARS ?= $(wildcard *.g4) -GENERATED = }> -SOURCES = $(GENERATED) \ - cli.dart -default: setup classes -setup: - if [ -f transformGrammar.py ]; then python3 transformGrammar.py ; fi -classes: $(SOURCES) - dart pub get - dart compile exe cli.dart +# Generated from trgen +default: + bash build.sh clean: - rm -f *.tokens *.interp - rm -f $(GENERATED) - rm -f pubspec.lock + bash clean.sh run: - trwdog dart.batdart run cli.dart $(RUNARGS) - : - java -jar $(JAR) -encoding -Dlanguage=Dart } > $\< -} > -test: + @trwdog Test.exe $(RUNARGS) +FORCE: ; +test: FORCE bash test.sh diff --git a/_scripts/templates/Dart/pubspec.yaml b/_scripts/templates/Dart/pubspec.yaml index 3ff10c531d..fba6a16be1 100644 --- a/_scripts/templates/Dart/pubspec.yaml +++ b/_scripts/templates/Dart/pubspec.yaml @@ -1,4 +1,4 @@ -# Template generated code from trgen +# Generated from trgen name: cli description: A sample command-line application. environment: diff --git a/_scripts/templates/Dart/test.ps1 b/_scripts/templates/Dart/test.ps1 new file mode 100644 index 0000000000..f6d20d39b9 --- /dev/null +++ b/_scripts/templates/Dart/test.ps1 @@ -0,0 +1,149 @@ +# Generated from trgen + +$TestDirectory = "../../" +Write-Host "Test cases here: $TestDirectory" + +# People often specify a test file directory, but sometimes no +# tests are provided. Git won't check in an empty directory. +# Test if the test file directory does not exist, or it is just +# an empty directory. +if (!(Test-Path -Path "$TestDirectory")) { + Write-Host "No test cases provided." + exit 0 +} elseif (!(Test-Path "$TestDirectory/*")) { + Write-Host "No test cases provided." + exit 0 +} + +# Get a list of test files from the test directory. Do not include any +# .errors or .tree files. Pay close attention to remove only file names +# that end with the suffix .errors or .tree. +if (Test-Path -Path "tests.txt" -PathType Leaf) { + Remove-Item "tests.txt" +} +$files = New-Object System.Collections.Generic.List[string] +foreach ($item in Get-ChildItem $TestDirectory -Recurse) { + $file = $item.fullname + $ext = $item.Extension + if (Test-Path $file -PathType Container) { + continue + } elseif ($ext -eq ".errors") { + continue + } elseif ($ext -eq ".tree") { + continue + } else { + $(& triconv -f utf-8 $file ; $last = $LASTEXITCODE ) | Out-Null + if ($last -ne 0) + { + continue + } + $files.Add($item) + Write-Host "Test case: $item" + } +} +foreach ($file in $files) { + Add-Content "tests.txt" $file +} +if (-not(Test-Path -Path "tests.txt" -PathType Leaf)) { + Write-Host "No test cases provided." + exit 0 +} + +# Parse all input files. +get-content "tests.txt" | trwdog ./Test.exe -q -x -tee -tree *> parse.txt +$status = $LASTEXITCODE + +# trwdog returns 255 if it cannot spawn the process. This could happen +# if the environment for running the program does not exist, or the +# program did not build. +if ( $status -eq 255 ) { + Write-Host "Test failed." + Get-Content $file | Write-Host + exit 1 +} + +# Any parse errors will be put in .errors files. But, if there's any +# output from the program in stdout or stderr, it's all bad news. +$size = (Get-Item -Path "parse.txt").Length +if ( $size -eq 0 ) { +} else { + Write-Host "Test failed." + Get-Content "parse.txt" | Write-Host + exit 1 +} + +# Check if any .errors/.tree files have changed. That's not good. +$message = git diff --exit-code --name-only $TestDirectory +$updated = $LASTEXITCODE +Write-Host $message + +# Check if any untracked .errors files are not empty. +$new_errors_txt = New-Object System.Collections.Generic.List[string] +$new_errors2_txt = git ls-files --exclude-standard -o --ignored $TestDirectory +$new_errors = $LASTEXITCODE + +# Gather up all untracked .errors file output. These are new errors +# and must be reported as a parse fail. +if ( ! [String]::IsNullOrWhiteSpace($new_errors2_txt) ) { + $new_errors3_txt = $new_errors2_txt.Split("\n\r\t ") +} else { + $new_errors3_txt = [System.Collections.Arraylist]@() +} +if (Test-Path -Path "new_errors.txt" -PathType Leaf) { + Remove-Item "new_errors.txt" +} +New-Item -Path . -Name "new_errors.txt" -ItemType "file" -Value "" | Out-Null +foreach ($s in $new_errors3_txt) { + if ( [String]::IsNullOrWhiteSpace($s) ) { + continue + } + $ext = $item.Extension + if (! $s.EndsWith(".errors")) { + continue + } + $file = $s + $size = (Get-Item -Path $file).Length + if ( $size -eq 0 ) { + } else { + $new_errors_txt.Add($item) + Add-Content -Path "new_errors.txt" -Value "$item" + ((Get-Content $file) -join "`n") + "`n" | Add-Content -Path "new_errors.txt" + } +} + +# If "git diff" reported an exit code of 129, it is because +# the directory containing the grammar is not in a repo. In this +# case, assume parse error code as the defacto result. +if ( $updated -eq 129 ) { + Write-Host "Grammar outside a git repository. Assuming parse exit code." + if ( $status -eq 0 ) { + Write-Host "Test succeeded." + } else { + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + } + $err = $status + exit 1 +} + +# "Git diff" reported a difference. Redo the "git diff" to print out all +# the differences. Also, output any untracked, non-zero length .errors files. +if ( $updated -eq 1 ) { + Write-Host "Difference in output." + git diff . | Write-Host + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + exit 1 +} + +# If there's non-zero length .errors flies that are new, report them +# as errors in the parse. +if ( $new_errors_txt.Count -gt 0 ) { + Write-Host "New errors in output." + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + exit 1 +} + +Write-Host "Test succeeded." +exit 0 diff --git a/_scripts/templates/Dart/test.sh b/_scripts/templates/Dart/test.sh index a6011cfeeb..6a2693bd47 100644 --- a/_scripts/templates/Dart/test.sh +++ b/_scripts/templates/Dart/test.sh @@ -1,58 +1,146 @@ -# Template generated code from trgen -err=0 +# Generated from trgen + +# People often specify a test file directory, but sometimes no +# tests are provided. Git won't check in an empty directory. +# Test if the test file directory does not exist, or it is just +# an empty directory. +if [ ! -d ../ ] +then + echo "No test cases provided." + exit 0 +elif [ ! "$(ls -A ../)" ] +then + echo "No test cases provided." + exit 0 +fi + SAVEIFS=$IFS IFS=$(echo -en "\n\b") -parse_out_file=`mktemp --tmpdir=. parse_out.XXXXXXXXXX` -for g in `find ../ -type f | grep -v '.errors$' | grep -v '.tree$'` + +files2=`find ../ -type f | grep -v '.errors$' | grep -v '.tree$'` +files=() +for f in $files2 do - file="$g" - x1="${g##*.}" - if [ "$x1" != "errors" ] - then - echo "$file" - trwdog ./cli.exe./cli -file "$file" -tree > "$parse_out_file" - status="$?" - if [ -f "$file".errors ] + triconv -f utf-8 $f > /dev/null 2>&1 + if [ "$?" = "0" ] then - if [ "$status" = "0" ] - then - echo Expected parse fail. - err=1 - break - else - echo Expected. - diff \<(tr -d "\r\n" \< "$file".errors) \<(tr -d "\r\n" \< "$parse_out_file") - status="$?" - if [ "$status" = "0" ] - then - echo Parse errors match succeeded. - else - echo Expected parse errors match. - err=1 - break - fi - fi - else # No .errors file - if [ "$status" != "0" ] - then - err=1 - break - fi - if [ -f "$file".tree ] - then - diff \<(tr -d "\r\n" \< "$file".tree) \<(tr -d "\r\n" \< "$parse_out_file") - status="$?" - if [ "$status" = "0" ] + files+=( $f ) + fi +done + +# Parse all input files. +echo "${files[*]}" | trwdog ./Test.exeTest -q -x -tee -tree > parse.txt 2>&1 +status=$? + +# trwdog returns 255 if it cannot spawn the process. This could happen +# if the environment for running the program does not exist, or the +# program did not build. +if [ "$status" = "255" ] +then + echo "Test failed." + cat parse.txt + exit 1 +fi + +# Any parse errors will be put in .errors files. But, if there's any +# output from the program in stdout or stderr, it's all bad news. +if [ -s parse.txt ] +then + echo "Test failed." + cat parse.txt + exit 1 +fi + +# rm -rf `find ../ -type f -name '*.errors' -o -name '*.tree' -size 0` + +# For Unix environments, convert the newline in the .errors and .trees +# to Unix style. +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) machine=Linux;; + Darwin*) machine=Mac;; + CYGWIN*) machine=Cygwin;; + MINGW*) machine=MinGw;; + *) machine="UNKNOWN:${unameOut}" +esac +if [[ "$machine" == "MinGw" || "$machine" == "Msys" || "$machine" == "Cygwin" || "#machine" == "Linux" ]] +then + gen=`find ../ -type f -name '*.errors' -o -name '*.tree'` + if [ "$gen" != "" ] + then + dos2unix $gen + fi +fi + +old=`pwd` +cd ../ + +# Check if any files in the test files directory have changed. +git diff --exit-code --name-only . > $old/updated.txt 2>&1 +updated=$? + +# Check if any untracked .errors files. +git ls-files --exclude-standard -o --ignored > $old/new_errors2.txt 2>&1 +new_errors=$? + +# Gather up all untracked .errors file output. These are new errors +# and must be reported as a parse fail. +rm -f $old/new_errors.txt +touch $old/new_errors.txt +for f in `cat $old/new_errors2.txt` +do + ext=${f##*.} + ext=".$ext" + if [ "$ext" = ".errors" ] + then + if [ -s $f ] then - echo Parse tree match succeeded. - else - echo Expected parse tree match. - err=1 - break + echo $f >> $old/new_errors.txt + cat $f >> $old/new_errors.txt fi - fi fi - fi done -rm "$parse_out_file" -exit $err + +# If "git diff" reported an exit code of 129, it is because +# the directory containing the grammar is not in a repo. In this +# case, assume parse error code as the defacto result. +if [ "$updated" = "129" ] +then + echo "Grammar outside a git repository. Assuming parse exit code." + if [ "$status" = 0 ] + then + echo "Test succeeded." + else + cat $old/new_errors.txt + echo "Test failed." + fi + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit $status +fi + +# "Git diff" reported a difference. Redo the "git diff" to print out all +# the differences. Also, output any untracked, non-zero length .errors files. +if [ "$updated" = "1" ] +then + echo "Difference in output." + git diff . + cat $old/new_errors.txt + echo "Test failed." + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit 1 +fi + +# If there's non-zero length .errors flies that are new, report them +# as errors in the parse. +if [ -s $old/new_errors.txt ] +then + echo "New errors in output." + cat $old/new_errors.txt + echo "Test failed." + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit 1 +fi + +echo "Test succeeded." +rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt +exit 0 diff --git a/_scripts/templates/Dart/tester.psm1 b/_scripts/templates/Dart/tester.psm1 deleted file mode 100644 index 32ae3cf026..0000000000 --- a/_scripts/templates/Dart/tester.psm1 +++ /dev/null @@ -1,77 +0,0 @@ -# Template generated code from trgen -function Build-Grammar { - -Dlanguage=Dart } > - if($LASTEXITCODE -ne 0){ - return @{ - Message = $g - Success = $false - \} - \} -}> - For ($i=0; $i -le 5; $i++) { - $g = dart pub get - if($LASTEXITCODE -eq 0){ - Break - } - Write-Host "dart pub get failed. Trying again." - } - if($LASTEXITCODE -ne 0){ - return @{ - Message = $g - Success = $false - } - } - $msg = dart compile exe cli.dart - return @{ - Message = $msg - Success = $LASTEXITCODE -eq 0 - } -} - -function Test-Case { - param ( - $InputFile, - $TokenFile, - $TreeFile, - $ErrorFile - ) - # Save input and output character encodings and switch to UTF-8. - $oldInputEncoding = [console]::InputEncoding - $oldOutputEncoding = [console]::OutputEncoding - $OutputEncoding = [console]::InputEncoding = [console]::OutputEncoding = New-Object System.Text.UTF8Encoding - - $parseOutFile = $InputFile + ".out" - $o = trwdog ./cli.exe -file $InputFile -tree | Out-File -LiteralPath "$parseOutFile" -Encoding UTF8 - - $parseOk = $LASTEXITCODE -eq 0 - $treeMatch = $true - if ($errorFile) { - # If we expected errors, then a failed parse was a successful test. - if (!$parseOk) { - Write-Host "Expected." - # Confirm that the errors we received are the ones we expected. - $expectedData = (Get-Content $errorFile -Encoding UTF8) -join "" -replace "\r\n","" - $actualData = (Get-Content $parseOutFile -Encoding UTF8) -join "" -replace "\r\n","" - $parseOk = ($actualData -eq $expectedData) - if ($parseOk) { - Write-Host "Error list match succeeded." - } else { - Write-Host "Expected error list match." -ForegroundColor Red - } - } - } else { - if ($parseOk -and (Test-Path $TreeFile)) { - # Confirm that the parse tree we received is the one we expected. - $expectedData = Get-Content $TreeFile -Encoding UTF8 - $actualData = Get-Content $parseOutFile -Encoding UTF8 - $treeMatch = ($actualData -eq $expectedData) - } - } - # Restore input and output character encodings. - [console]::InputEncoding = $oldInputEncoding - [console]::OutputEncoding = $oldOutputEncoding - - Remove-Item $parseOutFile - return $parseOk, $treeMatch -} \ No newline at end of file diff --git a/_scripts/templates/Go/Test.go b/_scripts/templates/Go/Test.go index 7e1b0df3e3..d38833cac4 100644 --- a/_scripts/templates/Go/Test.go +++ b/_scripts/templates/Go/Test.go @@ -1,4 +1,4 @@ -// Template generated code from trgen +// Generated from trgen package main import ( @@ -6,23 +6,42 @@ import ( "os" "io" "time" + "strconv" + "bufio" "github.com/antlr/antlr4/runtime/Go/antlr/v4" "example.com/myparser/" - - "example.com/myparser/antlr_resource" - ) type CustomErrorListener struct { errors int + quiet bool + tee bool + output *os.File } -func NewCustomErrorListener() *CustomErrorListener { - return new(CustomErrorListener) +func NewCustomErrorListener(q bool, t bool, o *os.File) *CustomErrorListener { + p := new(CustomErrorListener) + p.errors = 0 + p.quiet = q + p.tee = t + p.output = o + return p } -func (l *CustomErrorListener) SyntaxError(recognizer antlr.Recognizer, offendingSymbol interface{}, line, column int, msg string, e antlr.RecognitionException) { +func (l *CustomErrorListener) SyntaxError(recognizer antlr.Recognizer, offendingSymbol interface{}, line int, column int, msg string, e antlr.RecognitionException) { l.errors += 1 - fmt.Printf("line %d:%d %s", line, column, msg) + if l.tee { + l.output.WriteString("line ") + l.output.WriteString(strconv.Itoa(line)) + l.output.WriteString(":") + l.output.WriteString(strconv.Itoa(column)) + l.output.WriteString(" ") + l.output.WriteString(msg) + l.output.WriteString("\n") + } + if ! l.quiet { + fmt.Fprintf(os.Stderr, "line %d:%d %s", line, column, msg) + fmt.Fprintln(os.Stderr) + } } func (l *CustomErrorListener) ReportAmbiguity(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex int, exact bool, ambigAlts *antlr.BitSet, configs antlr.ATNConfigSet) { @@ -37,56 +56,109 @@ func (l *CustomErrorListener) ReportContextSensitivity(recognizer antlr.Parser, antlr.ConsoleErrorListenerINSTANCE.ReportContextSensitivity(recognizer, dfa, startIndex, stopIndex, prediction, configs) } +var inputs = make([]string, 0) +var is_fns = make([]bool, 0) +var error_code int = 0 +var show_tree = false +var show_tokens = false +var show_trace = false +var string_instance = 0 +var prefix = "" +var quiet = false +var tee = false func main() { - var show_tree = false - var show_tokens = false - var file_name = "" - var input = "" - var str antlr.CharStream = nil - for i := 0; i \< len(os.Args); i = i + 1 { + for i := 1; i \< len(os.Args); i = i + 1 { if os.Args[i] == "-tokens" { show_tokens = true - continue } else if os.Args[i] == "-tree" { show_tree = true - continue - } else if os.Args[i] == "-input" { + } else if os.Args[i] == "-prefix" { i = i + 1 - input = os.Args[i] - } else if os.Args[i] == "-file" { + prefix = os.Args[i] + " "; + } else if os.Args[i] == "-input" { i = i + 1 - file_name = os.Args[i] + inputs = append(inputs, os.Args[i]) + is_fns = append(is_fns, false) + } else if os.Args[i] == "-tee" { + tee = true + } else if os.Args[i] == "-x" { + scanner := bufio.NewScanner(os.Stdin) + for scanner.Scan() { + line := scanner.Text() + inputs = append(inputs, line) + is_fns = append(is_fns, true) + } + } else if os.Args[i] == "-q" { + quiet = true + } else if os.Args[i] == "-trace" { + show_trace = true + } else { + inputs = append(inputs, os.Args[i]) + is_fns = append(is_fns, true) } } - if input == "" && file_name == "" { - var b []byte = make([]byte, 1) - var st = "" - for { - _, err := os.Stdin.Read(b) - if err == io.EOF { - break + if len(inputs) == 0 { + ParseStdin() + } else { + start := time.Now() + for i := 0; i \< len(inputs); i = i + 1 { + if is_fns[i] { + ParseFilename(inputs[i], i) + } else { + ParseString(inputs[i], i) } - st = st + string(b) } - str = antlr.NewInputStream(st) - } else if input != "" { - str = antlr.NewInputStream(input) - } else if file_name != "" { - str, _ = antlr.NewFileStream(file_name); + elapsed := time.Since(start) + if ! quiet { + fmt.Fprintf(os.Stderr, "Total Time: %.3f s", elapsed.Seconds()) + fmt.Fprintln(os.Stderr) + } + } + if error_code != 0 { + os.Exit(1) + } else { + os.Exit(0) + } +} + +func ParseStdin() { + var b []byte = make([]byte, 1) + var st = "" + for { + _, err := os.Stdin.Read(b) + if err == io.EOF { + break + } + st = st + string(b) } - - str = antlr_resource.NewCaseChangingStream(str, "" == "Upper"); - + var str antlr.CharStream = nil + str = antlr.NewInputStream(st) + DoParse(str, "stdin", 0) +} + +func ParseString(input string, row_number int) { + str := antlr.NewInputStream(input) + DoParse(str, "string" + strconv.Itoa(string_instance), row_number) + string_instance = string_instance + 1 +} + +func ParseFilename(input string, row_number int) { + var str antlr.CharStream = nil + str, _ = antlr.NewFileStream(input) + DoParse(str, input, row_number) +} + +func DoParse(str antlr.CharStream, input_name string, row_number int) { var lexer = (str); if show_tokens { j := 0 for { t := lexer.NextToken() - fmt.Print(j) - fmt.Print(" ") + fmt.Fprintf(os.Stderr, "%d ", j) + fmt.Fprintf(os.Stderr, " ") // missing fmt.Println(t.String()) - fmt.Println(t.GetText()) + fmt.Fprintf(os.Stderr, "%d ", t.GetText()) if t.GetTokenType() == antlr.TokenEOF { break } @@ -98,32 +170,56 @@ func main() { var tokens = antlr.NewCommonTokenStream(lexer, 0) var parser = (tokens) - lexerErrors := &CustomErrorListener{} + var ( + output *os.File + ) + if tee { + output, _ = os.Create(input_name + ".errors") + } + + lexerErrors := NewCustomErrorListener(quiet, tee, output) lexer.RemoveErrorListeners() lexer.AddErrorListener(lexerErrors) - parserErrors := &CustomErrorListener{} + parserErrors := NewCustomErrorListener(quiet, tee, output) parser.RemoveErrorListeners() parser.AddErrorListener(parserErrors) + if show_trace { + //parser.SetTrace(true) + //antlr.ParserATNSimulatorTraceATNSim = true + } // mutated name--not lowercase. start := time.Now() var tree = parser.() elapsed := time.Since(start) - fmt.Fprintf(os.Stderr, "Time: %.3f s", elapsed.Seconds()) - fmt.Fprintln(os.Stderr) + var result = "" if parserErrors.errors > 0 || lexerErrors.errors > 0 { - fmt.Fprintln(os.Stderr, "Parse failed."); + result = "fail" + error_code = 1 } else { - fmt.Fprintln(os.Stderr, "Parse succeeded.") - if show_tree { - ss := tree.ToStringTree(parser.RuleNames, parser) - fmt.Println(ss) + result = "success" + } + if show_tree { + ss := tree.ToStringTree(parser.RuleNames, parser) + if tee { + f, _ := os.Create(input_name + ".tree") + f.Write([]byte(ss)) + f.Close() + } else { + fmt.Fprintf(os.Stderr, "%s", ss) } } - if parserErrors.errors > 0 || lexerErrors.errors > 0 { - os.Exit(1) - } else { - os.Exit(0) + if tee { + output.Close() + } + if ! quiet { + fmt.Fprintf(os.Stderr, "%s", prefix) + fmt.Fprintf(os.Stderr, "Go ") + fmt.Fprintf(os.Stderr, "%d ", row_number) + fmt.Fprintf(os.Stderr, "%s ", input_name) + fmt.Fprintf(os.Stderr, "%s ", result) + fmt.Fprintf(os.Stderr, "%.3f", elapsed.Seconds()) + fmt.Fprintln(os.Stderr) } } diff --git a/_scripts/templates/Go/antlr_resource/case_changing_stream.go b/_scripts/templates/Go/antlr_resource/case_changing_stream.go deleted file mode 100644 index ed53f76b47..0000000000 --- a/_scripts/templates/Go/antlr_resource/case_changing_stream.go +++ /dev/null @@ -1,39 +0,0 @@ -// Template generated code from trgen - -package antlr_resource - -import ( - "unicode" - - "github.com/antlr/antlr4/runtime/Go/antlr/v4" -) - -// CaseChangingStream wraps an existing CharStream, but upper cases, or -// lower cases the input before it is tokenized. -type CaseChangingStream struct { - antlr.CharStream - - upper bool -} - -// NewCaseChangingStream returns a new CaseChangingStream that forces -// all tokens read from the underlying stream to be either upper case -// or lower case based on the upper argument. -func NewCaseChangingStream(in antlr.CharStream, upper bool) *CaseChangingStream { - return &CaseChangingStream{in, upper} -} - -// LA gets the value of the symbol at offset from the current position -// from the underlying CharStream and converts it to either upper case -// or lower case. -func(is *CaseChangingStream) LA(offset int) int { - in := is.CharStream.LA(offset) - if in \< 0 { - // Such as antlr.TokenEOF which is -1 - return in - } - if is.upper { - return int(unicode.ToUpper(rune(in))) - } - return int(unicode.ToLower(rune(in))) -} \ No newline at end of file diff --git a/_scripts/templates/Go/build.ps1 b/_scripts/templates/Go/build.ps1 new file mode 100644 index 0000000000..4bac0964dc --- /dev/null +++ b/_scripts/templates/Go/build.ps1 @@ -0,0 +1,26 @@ +# Generated from trgen +$env:GO111MODULE = "on" +For ($i=0; $i -le 5; $i++) { + $(& go get github.com/antlr/antlr4/runtime/Go/antlr/v4 ; $compile_exit_code = $LASTEXITCODE) | Write-Host + if($compile_exit_code -eq 0){ + Break + } + Write-Host "go get failed. Trying again." +} +if($compile_exit_code -ne 0){ + exit $compile_exit_code +} + +if (Test-Path -Path transformGrammar.py -PathType Leaf) { + $(& python3 transformGrammar.py ) 2>&1 | Write-Host +} + + -encoding -Dlanguage=Go } > ; $compile_exit_code = $LASTEXITCODE) | Write-Host +if($compile_exit_code -ne 0){ + exit $compile_exit_code +\} +}> + +$(& go build Test.go; $compile_exit_code = $LASTEXITCODE) | Write-Host +exit $compile_exit_code diff --git a/_scripts/templates/Go/build.sh b/_scripts/templates/Go/build.sh new file mode 100644 index 0000000000..72b6bead54 --- /dev/null +++ b/_scripts/templates/Go/build.sh @@ -0,0 +1,13 @@ +# Generated from trgen +export GO111MODULE=on +for i in {1..5}; do go get github.com/antlr/antlr4/runtime/Go/antlr/v4; if [ "$?" = "0" ]; then break; fi; done; if [ "$?" != "0" ]; then exit 1; fi + +set -e + +if [ -f transformGrammar.py ]; then python3 transformGrammar.py ; fi + +" -encoding -Dlanguage=Go } > +} > + +go build Test.go diff --git a/_scripts/templates/Go/clean.ps1 b/_scripts/templates/Go/clean.ps1 new file mode 100644 index 0000000000..1aeee5017e --- /dev/null +++ b/_scripts/templates/Go/clean.ps1 @@ -0,0 +1,16 @@ +# Generated from trgen +$(& Remove-Item *.interp -Recurse -Force ) 2>&1 | Out-Null +$files = New-Object System.Collections.Generic.List[string] +" -depend -encoding -Dlanguage=Dart } > +foreach ($s in $f) { + $j = $s.Split(" ")[0] + $files.Add($j) +\} +foreach ($f in $files) +{ + $(& Remove-Item $f -Force ) 2>&1 | Out-Null +\} +} > +$(& Remove-Item ./Test.exeTest -Force ) 2>&1 | Out-Null +exit 0 diff --git a/_scripts/templates/Go/clean.sh b/_scripts/templates/Go/clean.sh new file mode 100644 index 0000000000..2456a0ac0e --- /dev/null +++ b/_scripts/templates/Go/clean.sh @@ -0,0 +1,11 @@ +# Generated from trgen +rm -f *.interp +files=() +" -depend -encoding -Dlanguage=Go } > | awk '{print $1\}' | grep -v ':'` ) +} > +for i in ${files[*]} +do + rm -f $i +done +rm -f ./Test.exeTest diff --git a/_scripts/templates/Go/makefile b/_scripts/templates/Go/makefile index 578151cace..ad39d0c5ea 100644 --- a/_scripts/templates/Go/makefile +++ b/_scripts/templates/Go/makefile @@ -1,27 +1,10 @@ -# Template generated code from trgen -JAR = -CLASSPATH = $(JAR)\;:. -.SUFFIXES: .g4 .go -ANTLRGRAMMARS ?= $(wildcard *.g4) -GENERATED = }> -SOURCES = $(GENERATED) Test.go -default: classes -classes: setup $(SOURCES) program -setup: - if [ -f transformGrammar.py ]; then python3 transformGrammar.py ; fi - -go env - -go version - export GO111MODULE=on; for i in {1..5}; do go get github.com/antlr/antlr4/runtime/Go/antlr/v4; if [ $$? = "0" ]; then break; fi; done; if [ $$? != "0" ]; then exit 1; fi -program: - export GO111MODULE=on; go build Test.go +# Generated from trgen +default: + bash build.sh clean: - rm -f *.tokens *.interp - rm -f $(GENERATED) + bash clean.sh run: - trwdog ./ $(RUNARGS) + @trwdog ./Test.exeTest $(RUNARGS) +FORCE: ; test: FORCE bash test.sh - : - java -jar $(JAR) -encoding -Dlanguage=Go } > $\< -} > -FORCE: ; diff --git a/_scripts/templates/Go/test.ps1 b/_scripts/templates/Go/test.ps1 new file mode 100644 index 0000000000..0b20ab8e15 --- /dev/null +++ b/_scripts/templates/Go/test.ps1 @@ -0,0 +1,149 @@ +# Generated from trgen + +$TestDirectory = "../../" +Write-Host "Test cases here: $TestDirectory" + +# People often specify a test file directory, but sometimes no +# tests are provided. Git won't check in an empty directory. +# Test if the test file directory does not exist, or it is just +# an empty directory. +if (!(Test-Path -Path "$TestDirectory")) { + Write-Host "No test cases provided." + exit 0 +} elseif (!(Test-Path "$TestDirectory/*")) { + Write-Host "No test cases provided." + exit 0 +} + +# Get a list of test files from the test directory. Do not include any +# .errors or .tree files. Pay close attention to remove only file names +# that end with the suffix .errors or .tree. +if (Test-Path -Path "tests.txt" -PathType Leaf) { + Remove-Item "tests.txt" +} +$files = New-Object System.Collections.Generic.List[string] +foreach ($item in Get-ChildItem $TestDirectory -Recurse) { + $file = $item.fullname + $ext = $item.Extension + if (Test-Path $file -PathType Container) { + continue + } elseif ($ext -eq ".errors") { + continue + } elseif ($ext -eq ".tree") { + continue + } else { + $(& triconv -f utf-8 $file ; $last = $LASTEXITCODE ) | Out-Null + if ($last -ne 0) + { + continue + } + $files.Add($item) + Write-Host "Test case: $item" + } +} +foreach ($file in $files) { + Add-Content "tests.txt" $file +} +if (-not(Test-Path -Path "tests.txt" -PathType Leaf)) { + Write-Host "No test cases provided." + exit 0 +} + +# Parse all input files. +get-content "tests.txt" | trwdog ./Test.exeTest -q -x -tee -tree *> parse.txt +$status = $LASTEXITCODE + +# trwdog returns 255 if it cannot spawn the process. This could happen +# if the environment for running the program does not exist, or the +# program did not build. +if ( $status -eq 255 ) { + Write-Host "Test failed." + Get-Content $file | Write-Host + exit 1 +} + +# Any parse errors will be put in .errors files. But, if there's any +# output from the program in stdout or stderr, it's all bad news. +$size = (Get-Item -Path "parse.txt").Length +if ( $size -eq 0 ) { +} else { + Write-Host "Test failed." + Get-Content "parse.txt" | Write-Host + exit 1 +} + +# Check if any .errors/.tree files have changed. That's not good. +$message = git diff --exit-code --name-only $TestDirectory +$updated = $LASTEXITCODE +Write-Host $message + +# Check if any untracked .errors files are not empty. +$new_errors_txt = New-Object System.Collections.Generic.List[string] +$new_errors2_txt = git ls-files --exclude-standard -o --ignored $TestDirectory +$new_errors = $LASTEXITCODE + +# Gather up all untracked .errors file output. These are new errors +# and must be reported as a parse fail. +if ( ! [String]::IsNullOrWhiteSpace($new_errors2_txt) ) { + $new_errors3_txt = $new_errors2_txt.Split("\n\r\t ") +} else { + $new_errors3_txt = [System.Collections.Arraylist]@() +} +if (Test-Path -Path "new_errors.txt" -PathType Leaf) { + Remove-Item "new_errors.txt" +} +New-Item -Path . -Name "new_errors.txt" -ItemType "file" -Value "" | Out-Null +foreach ($s in $new_errors3_txt) { + if ( [String]::IsNullOrWhiteSpace($s) ) { + continue + } + $ext = $item.Extension + if (! $s.EndsWith(".errors")) { + continue + } + $file = $s + $size = (Get-Item -Path $file).Length + if ( $size -eq 0 ) { + } else { + $new_errors_txt.Add($item) + Add-Content -Path "new_errors.txt" -Value "$item" + ((Get-Content $file) -join "`n") + "`n" | Add-Content -Path "new_errors.txt" + } +} + +# If "git diff" reported an exit code of 129, it is because +# the directory containing the grammar is not in a repo. In this +# case, assume parse error code as the defacto result. +if ( $updated -eq 129 ) { + Write-Host "Grammar outside a git repository. Assuming parse exit code." + if ( $status -eq 0 ) { + Write-Host "Test succeeded." + } else { + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + } + $err = $status + exit 1 +} + +# "Git diff" reported a difference. Redo the "git diff" to print out all +# the differences. Also, output any untracked, non-zero length .errors files. +if ( $updated -eq 1 ) { + Write-Host "Difference in output." + git diff . | Write-Host + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + exit 1 +} + +# If there's non-zero length .errors flies that are new, report them +# as errors in the parse. +if ( $new_errors_txt.Count -gt 0 ) { + Write-Host "New errors in output." + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + exit 1 +} + +Write-Host "Test succeeded." +exit 0 diff --git a/_scripts/templates/Go/test.sh b/_scripts/templates/Go/test.sh index 4be3bd4137..82e4d9e360 100644 --- a/_scripts/templates/Go/test.sh +++ b/_scripts/templates/Go/test.sh @@ -1,58 +1,149 @@ -# Template generated code from trgen -err=0 +# Generated from trgen + +# People often specify a test file directory, but sometimes no +# tests are provided. Git won't check in an empty directory. +# Test if the test file directory does not exist, or it is just +# an empty directory. +if [ ! -d ../ ] +then + echo "No test cases provided." + exit 0 +elif [ ! "$(ls -A ../)" ] +then + echo "No test cases provided." + exit 0 +fi + SAVEIFS=$IFS IFS=$(echo -en "\n\b") -parse_out_file=`mktemp --tmpdir=. parse_out.XXXXXXXXXX` -for g in `find ../ -type f | grep -v '.errors$' | grep -v '.tree$'` + +# Get a list of test files from the test directory. Do not include any +# .errors or .tree files. Pay close attention to remove only file names +# that end with the suffix .errors or .tree. +files2=`find ../ -type f | grep -v '.errors$' | grep -v '.tree$'` +files=() +for f in $files2 do - file="$g" - x1="${g##*.}" - if [ "$x1" != "errors" ] - then - echo "$file" - trwdog ./ -file "$file" -tree > "$parse_out_file" - status="$?" - if [ -f "$file".errors ] + triconv -f utf-8 $f > /dev/null 2>&1 + if [ "$?" = "0" ] then - if [ "$status" = "0" ] - then - echo Expected parse fail. - err=1 - break - else - echo Expected. - diff \<(tr -d "\r\n" \< "$file".errors) \<(tr -d "\r\n" \< "$parse_out_file") - status="$?" - if [ "$status" = "0" ] - then - echo Parse errors match succeeded. - else - echo Expected parse errors match. - err=1 - break - fi - fi - else # No .errors file - if [ "$status" != "0" ] - then - err=1 - break - fi - if [ -f "$file".tree ] - then - diff \<(tr -d "\r\n" \< "$file".tree) \<(tr -d "\r\n" \< "$parse_out_file") - status="$?" - if [ "$status" = "0" ] + files+=( $f ) + fi +done + +# Parse all input files. +echo "${files[*]}" | trwdog ./Test.exeTest -q -x -tee -tree > parse.txt 2>&1 +status=$? + +# trwdog returns 255 if it cannot spawn the process. This could happen +# if the environment for running the program does not exist, or the +# program did not build. +if [ "$status" = "255" ] +then + echo "Test failed." + cat parse.txt + exit 1 +fi + +# Any parse errors will be put in .errors files. But, if there's any +# output from the program in stdout or stderr, it's all bad news. +if [ -s parse.txt ] +then + echo "Test failed." + cat parse.txt + exit 1 +fi + +# rm -rf `find ../ -type f -name '*.errors' -o -name '*.tree' -size 0` + +# For Unix environments, convert the newline in the .errors and .trees +# to Unix style. +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) machine=Linux;; + Darwin*) machine=Mac;; + CYGWIN*) machine=Cygwin;; + MINGW*) machine=MinGw;; + *) machine="UNKNOWN:${unameOut}" +esac +if [[ "$machine" == "MinGw" || "$machine" == "Msys" || "$machine" == "Cygwin" || "#machine" == "Linux" ]] +then + gen=`find ../ -type f -name '*.errors' -o -name '*.tree'` + if [ "$gen" != "" ] + then + dos2unix $gen + fi +fi + +old=`pwd` +cd ../ + +# Check if any files in the test files directory have changed. +git diff --exit-code --name-only . > $old/updated.txt 2>&1 +updated=$? + +# Check if any untracked .errors files. +git ls-files --exclude-standard -o --ignored > $old/new_errors2.txt 2>&1 +new_errors=$? + +# Gather up all untracked .errors file output. These are new errors +# and must be reported as a parse fail. +rm -f $old/new_errors.txt +touch $old/new_errors.txt +for f in `cat $old/new_errors2.txt` +do + ext=${f##*.} + ext=".$ext" + if [ "$ext" = ".errors" ] + then + if [ -s $f ] then - echo Parse tree match succeeded. - else - echo Expected parse tree match. - err=1 - break + echo $f >> $old/new_errors.txt + cat $f >> $old/new_errors.txt fi - fi fi - fi done -rm "$parse_out_file" -exit $err + +# If "git diff" reported an exit code of 129, it is because +# the directory containing the grammar is not in a repo. In this +# case, assume parse error code as the defacto result. +if [ "$updated" = "129" ] +then + echo "Grammar outside a git repository. Assuming parse exit code." + if [ "$status" = 0 ] + then + echo "Test succeeded." + else + cat $old/new_errors.txt + echo "Test failed." + fi + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit $status +fi + +# "Git diff" reported a difference. Redo the "git diff" to print out all +# the differences. Also, output any untracked, non-zero length .errors files. +if [ "$updated" = "1" ] +then + echo "Difference in output." + git diff . + cat $old/new_errors.txt + echo "Test failed." + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit 1 +fi + +# If there's non-zero length .errors flies that are new, report them +# as errors in the parse. +if [ -s $old/new_errors.txt ] +then + echo "New errors in output." + cat $old/new_errors.txt + echo "Test failed." + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit 1 +fi + +echo "Test succeeded." +rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt +exit 0 diff --git a/_scripts/templates/Go/tester.psm1 b/_scripts/templates/Go/tester.psm1 deleted file mode 100644 index 12328ef20f..0000000000 --- a/_scripts/templates/Go/tester.psm1 +++ /dev/null @@ -1,83 +0,0 @@ -# Template generated code from trgen -function Build-Grammar { - -encoding -Dlanguage=Go } > - if($LASTEXITCODE -ne 0){ - return @{ - Message = $g - Success = $false - \} - \} -}> - # Output version of pwsh. - #Get-Host | Select-Object Version | Write-Host - # Output environmental variables. - #dir env: | Out-String | Write-Host - # Output go version - #go version | Write-Host - $env:GO111MODULE = "on" - For ($i=0; $i -le 5; $i++) { - $g = go get github.com/antlr/antlr4/runtime/Go/antlr/v4 - if($LASTEXITCODE -eq 0){ - Break - } - Write-Host "go get failed. Trying again." - } - if($LASTEXITCODE -ne 0){ - return @{ - Message = $g - Success = $false - } - } - $msg = go build Test.go - return @{ - Message = $msg - Success = $LASTEXITCODE -eq 0 - } -} - -function Test-Case { - param ( - $InputFile, - $TokenFile, - $TreeFile, - $ErrorFile - ) - # Save input and output character encodings and switch to UTF-8. - $oldInputEncoding = [console]::InputEncoding - $oldOutputEncoding = [console]::OutputEncoding - $OutputEncoding = [console]::InputEncoding = [console]::OutputEncoding = New-Object System.Text.UTF8Encoding - - $parseOutFile = $InputFile + ".out" - $o = trwdog ./Test -file $InputFile -tree | Out-File -LiteralPath "$parseOutFile" -Encoding UTF8 - $parseOk = $LASTEXITCODE -eq 0 - $treeMatch = $true - if ($errorFile) { - # If we expected errors, then a failed parse was a successful test. - if (!$parseOk) { - Write-Host "Expected." - # Confirm that the errors we received are the ones we expected. - $expectedData = (Get-Content $errorFile -Encoding UTF8) -join "" -replace "\r\n","" - $actualData = (Get-Content $parseOutFile -Encoding UTF8) -join "" -replace "\r\n","" - $parseOk = ($actualData -eq $expectedData) - if ($parseOk) { - Write-Host "Error list match succeeded." - } else { - Write-Host "Expected error list match." -ForegroundColor Red - } - } - } else { - if ($parseOk -and (Test-Path $TreeFile)) { - # Confirm that the parse tree we received is the one we expected. - $expectedData = Get-Content $TreeFile -Encoding UTF8 - $actualData = Get-Content $parseOutFile -Encoding UTF8 - $treeMatch = ($actualData -eq $expectedData) - } - } - # Restore input and output character encodings. - [console]::InputEncoding = $oldInputEncoding - [console]::OutputEncoding = $oldOutputEncoding - - Remove-Item $parseOutFile - return $parseOk, $treeMatch -} diff --git a/_scripts/templates/Java/CaseChangingCharStream.java b/_scripts/templates/Java/CaseChangingCharStream.java deleted file mode 100644 index 843d3056aa..0000000000 --- a/_scripts/templates/Java/CaseChangingCharStream.java +++ /dev/null @@ -1,84 +0,0 @@ -// Template generated code from trgen - -//package org.antlr.v4.runtime; - -import org.antlr.v4.runtime.*; -import org.antlr.v4.runtime.misc.*; - -/** - * This class supports case-insensitive lexing by wrapping an existing - * {@link CharStream} and forcing the lexer to see either upper or - * lowercase characters. Grammar literals should then be either upper or - * lower case such as 'BEGIN' or 'begin'. The text of the character - * stream is unaffected. Example: input 'BeGiN' would match lexer rule - * 'BEGIN' if constructor parameter upper=true but getText() would return - * 'BeGiN'. - */ -public class CaseChangingCharStream implements CharStream { - - final CharStream stream; - final boolean upper; - - /** - * Constructs a new CaseChangingCharStream wrapping the given {@link CharStream} forcing - * all characters to upper case or lower case. - * @param stream The stream to wrap. - * @param upper If true force each symbol to upper case, otherwise force to lower. - */ - public CaseChangingCharStream(CharStream stream, boolean upper) { - this.stream = stream; - this.upper = upper; - } - - @Override - public String getText(Interval interval) { - return stream.getText(interval); - } - - @Override - public void consume() { - stream.consume(); - } - - @Override - public int LA(int i) { - int c = stream.LA(i); - if (c \<= 0) { - return c; - } - if (upper) { - return Character.toUpperCase(c); - } - return Character.toLowerCase(c); - } - - @Override - public int mark() { - return stream.mark(); - } - - @Override - public void release(int marker) { - stream.release(marker); - } - - @Override - public int index() { - return stream.index(); - } - - @Override - public void seek(int index) { - stream.seek(index); - } - - @Override - public int size() { - return stream.size(); - } - - @Override - public String getSourceName() { - return stream.getSourceName(); - } -} \ No newline at end of file diff --git a/_scripts/templates/Java/ErrorListener.java b/_scripts/templates/Java/ErrorListener.java index edf46cb277..87ed8033d8 100644 --- a/_scripts/templates/Java/ErrorListener.java +++ b/_scripts/templates/Java/ErrorListener.java @@ -1,15 +1,26 @@ -// Template generated code from trgen +// Generated from trgen import org.antlr.v4.runtime.*; import java.nio.charset.StandardCharsets; import java.io.OutputStreamWriter; import java.io.PrintWriter; +import java.io.PrintStream; public class ErrorListener extends ConsoleErrorListener { public boolean had_error = false; - private static PrintWriter stdout_utf8 = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8), true); - + private static PrintWriter stderr_utf8 = new PrintWriter(new OutputStreamWriter(System.err, StandardCharsets.UTF_8), true); + private boolean quiet = false; + private boolean tee = false; + private PrintStream output = null; + + public ErrorListener(boolean q, boolean t, PrintStream o) + { + quiet = q; + tee = t; + output = o; + } + @Override public void syntaxError(Recognizer\ recognizer, Object offendingSymbol, @@ -19,6 +30,13 @@ public void syntaxError(Recognizer\ recognizer, RecognitionException e) { had_error = true; - stdout_utf8.println("line " + line + ":" + charPositionInLine + " " + msg); + if (tee) + { + output.println("line " + line + ":" + charPositionInLine + " " + msg); + } + if (! quiet) + { + stderr_utf8.println("line " + line + ":" + charPositionInLine + " " + msg); + } } } diff --git a/_scripts/templates/Java/Program.java b/_scripts/templates/Java/Program.java deleted file mode 100644 index 7752c34057..0000000000 --- a/_scripts/templates/Java/Program.java +++ /dev/null @@ -1,103 +0,0 @@ -// Template generated code from trgen - -import java.io.FileNotFoundException; -import java.io.IOException; -import org.antlr.v4.runtime.*; -import org.antlr.v4.runtime.tree.ParseTree; -import java.time.Instant; -import java.time.Duration; -import java.nio.charset.StandardCharsets; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; - -public class Program { - private static PrintWriter stdout_utf8 = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8), true); - private static PrintWriter stderr_utf8 = new PrintWriter(new OutputStreamWriter(System.err, StandardCharsets.UTF_8), true); - public static void main(String[] args) throws FileNotFoundException, IOException - { - boolean show_tree = false; - boolean show_tokens = false; - String file_name = null; - String input = null; - java.nio.charset.Charset charset = null; - for (int i = 0; i \< args.length; ++i) - { - if (args[i].equals("-tokens")) - { - show_tokens = true; - continue; - } - else if (args[i].equals("-tree")) - { - show_tree = true; - continue; - } - else if (args[i].equals("-input")) - input = args[++i]; - else if (args[i].equals("-file")) - file_name = args[++i]; - else if (args[i].equals("-encoding")) - { - charset = java.nio.charset.Charset.forName(args[++i]); - } - } - CharStream str = null; - if (input == null && file_name == null) - { - str = CharStreams.fromStream(System.in); - } else if (input != null) - { - str = CharStreams.fromString(input); - } else if (file_name != null) - { - if (charset == null) - str = CharStreams.fromFileName(file_name); - else - str = CharStreams.fromFileName(file_name, charset); - } - - str = new CaseChangingCharStream(str, "" == "Upper"); - - lexer = new (str); - if (show_tokens) - { - StringBuilder new_s = new StringBuilder(); - for (int i = 0; ; ++i) - { - var ro_token = lexer.nextToken(); - var token = (CommonToken)ro_token; - token.setTokenIndex(i); - new_s.append(token.toString()); - new_s.append(System.getProperty("line.separator")); - if (token.getType() == IntStream.EOF) - break; - } - System.out.println(new_s.toString()); - lexer.reset(); - } - var tokens = new CommonTokenStream(lexer); - parser = new (tokens); - ErrorListener lexer_listener = new ErrorListener(); - ErrorListener listener = new ErrorListener(); - parser.removeErrorListeners(); - lexer.removeErrorListeners(); - parser.addErrorListener(listener); - lexer.addErrorListener(lexer_listener); - Instant start = Instant.now(); - ParseTree tree = parser.(); - Instant finish = Instant.now(); - long timeElapsed = Duration.between(start, finish).toMillis(); - stderr_utf8.println("Time: " + (timeElapsed * 1.0) / 1000.0); - if (listener.had_error || lexer_listener.had_error) { - // Listener will have already printed the error(s) to stdout. - stderr_utf8.println("Parse failed."); - } else { - stderr_utf8.println("Parse succeeded."); - if (show_tree) - { - stdout_utf8.println(tree.toStringTree(parser)); - } - } - java.lang.System.exit(listener.had_error || lexer_listener.had_error ? 1 : 0); - } -} diff --git a/_scripts/templates/Java/Test.java b/_scripts/templates/Java/Test.java new file mode 100644 index 0000000000..99a96f2220 --- /dev/null +++ b/_scripts/templates/Java/Test.java @@ -0,0 +1,215 @@ +// Generated from trgen + +import java.io.FileNotFoundException; +import java.io.IOException; +import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.atn.*; +import org.antlr.v4.runtime.tree.ParseTree; +import java.time.Instant; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Scanner; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.OutputStreamWriter; +import java.io.FileOutputStream; +import java.nio.charset.StandardCharsets; +import java.io.File; + +public class Test { + + static boolean tee = false; + static boolean show_profile = false; + static boolean show_tree = false; + static boolean show_tokens = false; + static boolean show_trace = false; + static int error_code = 0; + static java.nio.charset.Charset charset = null; + static int string_instance = 0; + static String prefix = ""; + static boolean quiet = false; + + public static void main(String[] args) throws FileNotFoundException, IOException + { + List\ is_fns = new ArrayList\(); + List\ inputs = new ArrayList\(); + for (int i = 0; i \< args.length; ++i) + { + if (args[i].equals("-profile")) + { + show_profile = true; + } + else if (args[i].equals("-tokens")) + { + show_tokens = true; + } + else if (args[i].equals("-tree")) + { + show_tree = true; + } + else if (args[i].equals("-prefix")) + { + prefix = args[++i] + " "; + } + else if (args[i].equals("-input")) { + inputs.add(args[++i]); + is_fns.add(false); + } + else if (args[i].equals("-tee")) + { + tee = true; + } + else if (args[i].equals("-encoding")) + { + charset = java.nio.charset.Charset.forName(args[++i]); + } + else if (args[i].equals("-x")) + { + Scanner scanner = new Scanner(System.in); + for (; ; ) + { + if (!scanner.hasNext()) break; + String line = scanner.nextLine(); + if (line == null) break; + line = line.trim(); + if (line == "") break; + inputs.add(line); + is_fns.add(true); + } + } + else if (args[i].equals("-q")) + { + quiet = true; + } + else if (args[i].equals("-trace")) + { + show_trace = true; + continue; + } + else { + inputs.add(args[i]); + is_fns.add(true); + } + } + CharStream str = null; + if (inputs.size() == 0) + { + ParseStdin(); + } + else + { + Instant start = Instant.now(); + for (int f = 0; f \< inputs.size(); ++f) + { + if (is_fns.get(f)) + ParseFilename(inputs.get(f), f); + else + ParseString(inputs.get(f), f); + } + Instant finish = Instant.now(); + long timeElapsed = Duration.between(start, finish).toMillis(); + if (!quiet) System.err.println("Total Time: " + (timeElapsed * 1.0) / 1000.0); + } + java.lang.System.exit(error_code); + } + + static void ParseStdin()throws IOException { + CharStream str = CharStreams.fromStream(System.in); + DoParse(str, "stdin", 0); + } + + static void ParseString(String input, int row_number) throws IOException { + var str = CharStreams.fromString(input); + DoParse(str, "string" + string_instance++, row_number); + } + + static void ParseFilename(String input, int row_number) throws IOException + { + CharStream str = null; + if (charset == null) + str = CharStreams.fromFileName(input); + else + str = CharStreams.fromFileName(input, charset); + DoParse(str, input, row_number); + } + + static void DoParse(CharStream str, String input_name, int row_number) { + lexer = new (str); + if (show_tokens) + { + StringBuilder new_s = new StringBuilder(); + for (int i = 0; ; ++i) + { + var ro_token = lexer.nextToken(); + var token = (CommonToken)ro_token; + token.setTokenIndex(i); + new_s.append(token.toString()); + new_s.append(System.getProperty("line.separator")); + if (token.getType() == IntStream.EOF) + break; + } + System.err.println(new_s.toString()); + lexer.reset(); + } + var tokens = new CommonTokenStream(lexer); + parser = new (tokens); + PrintStream output = null; + try { + output = tee ? new PrintStream(new File(input_name + ".errors")) : System.out; + } catch (NullPointerException e) { + output = System.err; + } catch (FileNotFoundException e2) { + output = System.err; + } + ErrorListener listener_lexer = new ErrorListener(quiet, tee, output); + ErrorListener listener_parser = new ErrorListener(quiet, tee, output); + parser.removeErrorListeners(); + lexer.removeErrorListeners(); + parser.addErrorListener(listener_parser); + lexer.addErrorListener(listener_lexer); + if (show_trace) + { + parser.setTrace(true); +// ParserATNSimulator.trace_atn_sim = true; + } + Instant start = Instant.now(); + ParseTree tree = parser.(); + Instant finish = Instant.now(); + long timeElapsed = Duration.between(start, finish).toMillis(); + String result = ""; + if (listener_parser.had_error || listener_lexer.had_error) + { + result = "fail"; + error_code = 1; + } + else + result = "success"; + if (show_tree) + { + if (tee) + { + PrintWriter treef = null; + try { + treef = new PrintWriter(new OutputStreamWriter(new FileOutputStream(new File(input_name + ".tree")), StandardCharsets.UTF_8), true); + //treef = new PrintStream(new File(input_name + ".tree")); + } catch (NullPointerException e) { + treef = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8), true);; + } catch (FileNotFoundException e2) { + treef = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8), true); + } + treef.print(tree.toStringTree(parser)); + treef.close(); + } else + { + System.err.println(tree.toStringTree(parser)); + } + } + if (!quiet) + { + System.err.println(prefix + "Java " + row_number + " " + input_name + " " + result + " " + (timeElapsed * 1.0) / 1000.0); + } + if (tee) output.close(); + } +} diff --git a/_scripts/templates/Java/build.ps1 b/_scripts/templates/Java/build.ps1 new file mode 100644 index 0000000000..37d99a63a7 --- /dev/null +++ b/_scripts/templates/Java/build.ps1 @@ -0,0 +1,14 @@ +# Generated from trgen +if (Test-Path -Path transformGrammar.py -PathType Leaf) { + $(& python3 transformGrammar.py ) 2>&1 | Write-Host +} + + -encoding -Dlanguage=Java } > ; $compile_exit_code = $LASTEXITCODE) | Write-Host +if($compile_exit_code -ne 0){ + exit $compile_exit_code +\} +}> + +$(& javac -cp ";:." }> Test.java ErrorListener.java ; $compile_exit_code = $LASTEXITCODE) | Write-Host +exit $compile_exit_code diff --git a/_scripts/templates/Java/build.sh b/_scripts/templates/Java/build.sh new file mode 100644 index 0000000000..da523a682d --- /dev/null +++ b/_scripts/templates/Java/build.sh @@ -0,0 +1,12 @@ +# Generated from trgen +set -e + +" -encoding -Dlanguage=Java } > +}> + +JAR= +CLASSPATH="$JAR\;:." +javac -cp "$CLASSPATH" *.java + +exit 0 diff --git a/_scripts/templates/Java/clean.ps1 b/_scripts/templates/Java/clean.ps1 new file mode 100644 index 0000000000..2a037a3e9d --- /dev/null +++ b/_scripts/templates/Java/clean.ps1 @@ -0,0 +1,16 @@ +# Generated from trgen +$(& Remove-Item *.interp -Recurse -Force ) 2>&1 | Out-Null +$files = New-Object System.Collections.Generic.List[string] +" -depend -encoding -Dlanguage=Dart } > +foreach ($s in $f) { + $j = $s.Split(" ")[0] + $files.Add($j) +\} +foreach ($f in $files) +{ + $(& Remove-Item $f -Force ) 2>&1 | Out-Null +\} +} > +$(& Remove-Item *.class -Recurse -Force ) 2>&1 | Out-Null +exit 0 diff --git a/_scripts/templates/Java/clean.sh b/_scripts/templates/Java/clean.sh new file mode 100644 index 0000000000..8683434c94 --- /dev/null +++ b/_scripts/templates/Java/clean.sh @@ -0,0 +1,12 @@ +# Generated from trgen +rm -f *.interp +files=() +" -depend -encoding -Dlanguage=Java } > | awk '{print $1\}' | grep -v ':'` ) +} > +for i in ${files[*]} +do + rm -f $i +done +rm -f *.class +exit 0 diff --git a/_scripts/templates/Java/makefile b/_scripts/templates/Java/makefile index fc84c62adb..2e7cd0d4e6 100644 --- a/_scripts/templates/Java/makefile +++ b/_scripts/templates/Java/makefile @@ -1,28 +1,10 @@ -# Template generated code from trgen -JAR = -CLASSPATH = $(JAR)\;:. -.SUFFIXES: .g4 .java .class -.java.class: - javac -cp $(CLASSPATH) $*.java -ANTLRGRAMMARS ?= $(wildcard *.g4) -GENERATED = }> -SOURCES = $(GENERATED) \ - Program.java \ - - CaseChangingCharStream.java \ - - ErrorListener.java -default: classes -classes: $(GENERATED) $(SOURCES:.java=.class) +# Generated from trgen +default: + bash build.sh clean: - rm -f *.class - rm -f *.interp - rm -f *.tokens - rm -f $(GENERATED) + bash clean.sh run: - trwdog java -classpath $(CLASSPATH) Program $(RUNARGS) - : - java -jar $(JAR) -encoding } > $\< -} > -test: + @trwdog java -classpath $(CLASSPATH) Test $(RUNARGS) +FORCE: ; +test: FORCE bash test.sh diff --git a/_scripts/templates/Java/test.ps1 b/_scripts/templates/Java/test.ps1 new file mode 100644 index 0000000000..f7796695cd --- /dev/null +++ b/_scripts/templates/Java/test.ps1 @@ -0,0 +1,149 @@ +# Generated from trgen + +$TestDirectory = "../../" +Write-Host "Test cases here: $TestDirectory" + +# People often specify a test file directory, but sometimes no +# tests are provided. Git won't check in an empty directory. +# Test if the test file directory does not exist, or it is just +# an empty directory. +if (!(Test-Path -Path "$TestDirectory")) { + Write-Host "No test cases provided." + exit 0 +} elseif (!(Test-Path "$TestDirectory/*")) { + Write-Host "No test cases provided." + exit 0 +} + +# Get a list of test files from the test directory. Do not include any +# .errors or .tree files. Pay close attention to remove only file names +# that end with the suffix .errors or .tree. +if (Test-Path -Path "tests.txt" -PathType Leaf) { + Remove-Item "tests.txt" +} +$files = New-Object System.Collections.Generic.List[string] +foreach ($item in Get-ChildItem $TestDirectory -Recurse) { + $file = $item.fullname + $ext = $item.Extension + if (Test-Path $file -PathType Container) { + continue + } elseif ($ext -eq ".errors") { + continue + } elseif ($ext -eq ".tree") { + continue + } else { + $(& triconv -f utf-8 $file ; $last = $LASTEXITCODE ) | Out-Null + if ($last -ne 0) + { + continue + } + $files.Add($item) + Write-Host "Test case: $item" + } +} +foreach ($file in $files) { + Add-Content "tests.txt" $file +} +if (-not(Test-Path -Path "tests.txt" -PathType Leaf)) { + Write-Host "No test cases provided." + exit 0 +} + +# Parse all input files. +get-content "tests.txt" | trwdog java -cp ";:." Test -q -x -tee -tree *> parse.txt +$status = $LASTEXITCODE + +# trwdog returns 255 if it cannot spawn the process. This could happen +# if the environment for running the program does not exist, or the +# program did not build. +if ( $status -eq 255 ) { + Write-Host "Test failed." + Get-Content $file | Write-Host + exit 1 +} + +# Any parse errors will be put in .errors files. But, if there's any +# output from the program in stdout or stderr, it's all bad news. +$size = (Get-Item -Path "parse.txt").Length +if ( $size -eq 0 ) { +} else { + Write-Host "Test failed." + Get-Content "parse.txt" | Write-Host + exit 1 +} + +# Check if any .errors/.tree files have changed. That's not good. +$message = git diff --exit-code --name-only $TestDirectory +$updated = $LASTEXITCODE +Write-Host $message + +# Check if any untracked .errors files are not empty. +$new_errors_txt = New-Object System.Collections.Generic.List[string] +$new_errors2_txt = git ls-files --exclude-standard -o --ignored $TestDirectory +$new_errors = $LASTEXITCODE + +# Gather up all untracked .errors file output. These are new errors +# and must be reported as a parse fail. +if ( ! [String]::IsNullOrWhiteSpace($new_errors2_txt) ) { + $new_errors3_txt = $new_errors2_txt.Split("\n\r\t ") +} else { + $new_errors3_txt = [System.Collections.Arraylist]@() +} +if (Test-Path -Path "new_errors.txt" -PathType Leaf) { + Remove-Item "new_errors.txt" +} +New-Item -Path . -Name "new_errors.txt" -ItemType "file" -Value "" | Out-Null +foreach ($s in $new_errors3_txt) { + if ( [String]::IsNullOrWhiteSpace($s) ) { + continue + } + $ext = $item.Extension + if (! $s.EndsWith(".errors")) { + continue + } + $file = $s + $size = (Get-Item -Path $file).Length + if ( $size -eq 0 ) { + } else { + $new_errors_txt.Add($item) + Add-Content -Path "new_errors.txt" -Value "$item" + ((Get-Content $file) -join "`n") + "`n" | Add-Content -Path "new_errors.txt" + } +} + +# If "git diff" reported an exit code of 129, it is because +# the directory containing the grammar is not in a repo. In this +# case, assume parse error code as the defacto result. +if ( $updated -eq 129 ) { + Write-Host "Grammar outside a git repository. Assuming parse exit code." + if ( $status -eq 0 ) { + Write-Host "Test succeeded." + } else { + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + } + $err = $status + exit 1 +} + +# "Git diff" reported a difference. Redo the "git diff" to print out all +# the differences. Also, output any untracked, non-zero length .errors files. +if ( $updated -eq 1 ) { + Write-Host "Difference in output." + git diff . | Write-Host + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + exit 1 +} + +# If there's non-zero length .errors flies that are new, report them +# as errors in the parse. +if ( $new_errors_txt.Count -gt 0 ) { + Write-Host "New errors in output." + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + exit 1 +} + +Write-Host "Test succeeded." +exit 0 diff --git a/_scripts/templates/Java/test.sh b/_scripts/templates/Java/test.sh index 8004a400a7..bc15a390f1 100644 --- a/_scripts/templates/Java/test.sh +++ b/_scripts/templates/Java/test.sh @@ -1,60 +1,151 @@ -# Template generated code from trgen -JAR= -CLASSPATH=$JAR\;:. -err=0 +# Generated from trgen + +# People often specify a test file directory, but sometimes no +# tests are provided. Git won't check in an empty directory. +# Test if the test file directory does not exist, or it is just +# an empty directory. +if [ ! -d ../ ] +then + echo "No test cases provided." + exit 0 +elif [ ! "$(ls -A ../)" ] +then + echo "No test cases provided." + exit 0 +fi + SAVEIFS=$IFS IFS=$(echo -en "\n\b") -parse_out_file=`mktemp --tmpdir=. parse_out.XXXXXXXXXX` -for g in `find ../ -type f | grep -v '.errors$' | grep -v '.tree$'` + +# Get a list of test files from the test directory. Do not include any +# .errors or .tree files. Pay close attention to remove only file names +# that end with the suffix .errors or .tree. +files2=`find ../ -type f | grep -v '.errors$' | grep -v '.tree$'` +files=() +for f in $files2 do - file="$g" - x1="${g##*.}" - if [ "$x1" != "errors" ] - then - echo "$file" - trwdog java -classpath $CLASSPATH Program -file "$file" -tree > "$parse_out_file" - status="$?" - if [ -f "$file".errors ] + triconv -f utf-8 $f > /dev/null 2>&1 + if [ "$?" = "0" ] then - if [ "$status" = "0" ] - then - echo Expected parse fail. - err=1 - break - else - echo Expected. - diff \<(tr -d "\r\n" \< "$file".errors) \<(tr -d "\r\n" \< "$parse_out_file") - status="$?" - if [ "$status" = "0" ] - then - echo Parse errors match succeeded. - else - echo Expected parse errors match. - err=1 - break - fi - fi - else # No .errors file - if [ "$status" != "0" ] - then - err=1 - break - fi - if [ -f "$file".tree ] - then - diff \<(tr -d "\r\n" \< "$file".tree) \<(tr -d "\r\n" \< "$parse_out_file") - status="$?" - if [ "$status" = "0" ] + files+=( $f ) + fi +done + +# Parse all input files. +JAR="" +CLASSPATH="$JAR\;:." +echo "${files[*]}" | trwdog java -classpath "$CLASSPATH" Test -q -x -tee -tree > parse.txt 2>&1 +status=$? + +# trwdog returns 255 if it cannot spawn the process. This could happen +# if the environment for running the program does not exist, or the +# program did not build. +if [ "$status" = "255" ] +then + echo "Test failed." + cat parse.txt + exit 1 +fi + +# Any parse errors will be put in .errors files. But, if there's any +# output from the program in stdout or stderr, it's all bad news. +if [ -s parse.txt ] +then + echo "Test failed." + cat parse.txt + exit 1 +fi + +# rm -rf `find ../ -type f -name '*.errors' -o -name '*.tree' -size 0` + +# For Unix environments, convert the newline in the .errors and .trees +# to Unix style. +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) machine=Linux;; + Darwin*) machine=Mac;; + CYGWIN*) machine=Cygwin;; + MINGW*) machine=MinGw;; + *) machine="UNKNOWN:${unameOut}" +esac +if [[ "$machine" == "MinGw" || "$machine" == "Msys" || "$machine" == "Cygwin" || "#machine" == "Linux" ]] +then + gen=`find ../ -type f -name '*.errors' -o -name '*.tree'` + if [ "$gen" != "" ] + then + dos2unix $gen + fi +fi + +old=`pwd` +cd ../ + +# Check if any files in the test files directory have changed. +git diff --exit-code --name-only . > $old/updated.txt 2>&1 +updated=$? + +# Check if any untracked .errors files. +git ls-files --exclude-standard -o --ignored > $old/new_errors2.txt 2>&1 +new_errors=$? + +# Gather up all untracked .errors file output. These are new errors +# and must be reported as a parse fail. +rm -f $old/new_errors.txt +touch $old/new_errors.txt +for f in `cat $old/new_errors2.txt` +do + ext=${f##*.} + ext=".$ext" + if [ "$ext" = ".errors" ] + then + if [ -s $f ] then - echo Parse tree match succeeded. - else - echo Expected parse tree match. - err=1 - break + echo $f >> $old/new_errors.txt + cat $f >> $old/new_errors.txt fi - fi fi - fi done -rm "$parse_out_file" -exit $err + +# If "git diff" reported an exit code of 129, it is because +# the directory containing the grammar is not in a repo. In this +# case, assume parse error code as the defacto result. +if [ "$updated" = "129" ] +then + echo "Grammar outside a git repository. Assuming parse exit code." + if [ "$status" = 0 ] + then + echo "Test succeeded." + else + cat $old/new_errors.txt + echo "Test failed." + fi + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit $status +fi + +# "Git diff" reported a difference. Redo the "git diff" to print out all +# the differences. Also, output any untracked, non-zero length .errors files. +if [ "$updated" = "1" ] +then + echo "Difference in output." + git diff . + cat $old/new_errors.txt + echo "Test failed." + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit 1 +fi + +# If there's non-zero length .errors flies that are new, report them +# as errors in the parse. +if [ -s $old/new_errors.txt ] +then + echo "New errors in output." + cat $old/new_errors.txt + echo "Test failed." + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit 1 +fi + +echo "Test succeeded." +rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt +exit 0 diff --git a/_scripts/templates/Java/tester.psm1 b/_scripts/templates/Java/tester.psm1 deleted file mode 100644 index 54e4e89d24..0000000000 --- a/_scripts/templates/Java/tester.psm1 +++ /dev/null @@ -1,65 +0,0 @@ -# Template generated code from trgen -function Build-Grammar { - -Dlanguage=Java } > - if($LASTEXITCODE -ne 0){ - return @{ - Message = $g - Success = $false - \} - \} -}> - Write-Host "javac -cp \;:. }> Program.java ErrorListener.java" - $msg = javac -cp ";:." }> Program.java ErrorListener.java - return @{ - Message = $msg - Success = $LASTEXITCODE -eq 0 - } -} - -function Test-Case { - param ( - $InputFile, - $TokenFile, - $TreeFile, - $ErrorFile - ) - # Save input and output character encodings and switch to UTF-8. - $oldInputEncoding = [console]::InputEncoding - $oldOutputEncoding = [console]::OutputEncoding - $OutputEncoding = [console]::InputEncoding = [console]::OutputEncoding = New-Object System.Text.UTF8Encoding - - $parseOutFile = $InputFile + ".out" - $o = trwdog java -cp ";:." Program -file $InputFile -tree | Out-File -LiteralPath "$parseOutFile" -Encoding UTF8 - - $parseOk = $LASTEXITCODE -eq 0 - $treeMatch = $true - if ($errorFile) { - # If we expected errors, then a failed parse was a successful test. - if (!$parseOk) { - Write-Host "Expected." - # Confirm that the errors we received are the ones we expected. - $expectedData = (Get-Content $errorFile -Encoding UTF8) -join "" -replace "\r\n","" - $actualData = (Get-Content $parseOutFile -Encoding UTF8) -join "" -replace "\r\n","" - $parseOk = ($actualData -eq $expectedData) - if ($parseOk) { - Write-Host "Error list match succeeded." - } else { - Write-Host "Expected error list match." -ForegroundColor Red - } - } - } else { - if ($parseOk -and (Test-Path $TreeFile)) { - # Confirm that the parse tree we received is the one we expected. - $expectedData = Get-Content $TreeFile -Encoding UTF8 - $actualData = Get-Content $parseOutFile -Encoding UTF8 - $treeMatch = ($actualData -eq $expectedData) - } - } - # Restore input and output character encodings. - [console]::InputEncoding = $oldInputEncoding - [console]::OutputEncoding = $oldOutputEncoding - - Remove-Item $parseOutFile - return $parseOk, $treeMatch -} \ No newline at end of file diff --git a/_scripts/templates/JavaScript/CaseChangingStream.js b/_scripts/templates/JavaScript/CaseChangingStream.js deleted file mode 100644 index db5992d343..0000000000 --- a/_scripts/templates/JavaScript/CaseChangingStream.js +++ /dev/null @@ -1,63 +0,0 @@ -// Template generated code from trgen - -import antlr4 from 'antlr4'; - -export default class CaseChangingStream extends antlr4.InputStream { - - constructor(stream, upper) - { - super(stream); - this._stream = stream; - this._upper = upper; - } - - LA(offset) { - var c = this._stream.LA(offset); - if (c \<= 0) - { - return c; - } - return String.fromCodePoint(c)[this._upper ? "toUpperCase" : "toLowerCase"]().codePointAt(0); - } - - reset() { - return this._stream.reset(); - } - - consume() { - return this._stream.consume(); - } - - LT(offset) { - return this._stream.LT(offset); - } - - mark() { - return this._stream.mark(); - } - - release(marker) { - return this._stream.release(marker); - } - - seek(_index) { - return this._stream.seek(_index); - } - - getText(start, stop) { - return this._stream.getText(start, stop); - } - - toString() { - return this._stream.toString(); - } - - get index(){ - return this._index; - } - - get size(){ - return this._size; - } -} - diff --git a/_scripts/templates/JavaScript/Test.js b/_scripts/templates/JavaScript/Test.js new file mode 100644 index 0000000000..c3b5406112 --- /dev/null +++ b/_scripts/templates/JavaScript/Test.js @@ -0,0 +1,209 @@ +// Generated from trgen + +import antlr4 from 'antlr4'; + from './'; +} > +import strops from 'typescript-string-operations'; +import fs from 'fs-extra'; +import pkg from 'timer-node'; +import * as readline from 'node:readline'; + +const { Timer, Time, TimerOptions } = pkg; + +function getChar() { + let buffer = Buffer.alloc(1); + var xx = 0; + try { + xx = fs.readSync(0, buffer, 0, 1); + } catch (err) { + } + if (xx === 0) { + return ''; + } + return buffer.toString('utf8'); +} + + +class MyErrorListener extends antlr4.error.ErrorListener { + + constructor(quiet, tee, output) { + super(); + this._quiet = quiet; + this._tee = tee; + this._output = output; + this.had_error = false; + } + + syntaxError(recognizer, offendingSymbol, line, column, msg, err) { + this.had_error = true; + if (tee) { + fs.writeSync(this._output, `line ${line}:${column} ${msg}\n`); + } + if (! quiet) { + console.error(`line ${line}:${column} ${msg}`); + } + } +} + +var tee = false; +var show_profile = false; +var show_tree = false; +var show_tokens = false; +var show_trace = false; +var error_code = 0; +var quiet = false; +var encoding = 'utf8'; +var string_instance = 0; +var prefix = ''; +var inputs = []; +var is_fns = []; + +function splitLines(t) { return t.split(/\r\n|\r|\n/); } + +function main() { + for (let i = 2; i \< process.argv.length; ++i) + { + switch (process.argv[i]) { + case '-tokens': + show_tokens = true; + break; + case '-tree': + show_tree = true; + break; + case '-prefix': + prefix = process.argv[++i] + ' '; + break; + case '-input': + inputs.push(process.argv[++i]); + is_fns.push(false); + break; + case '-tee': + tee = true; + break; + case '-encoding': + encoding = process.argv[++i]; + break; + case '-x': + var sb = new strops.StringBuilder(); + var ch; + while ((ch = getChar()) != '') { + sb.Append(ch); + } + var input = sb.ToString(); + var sp = splitLines(input); + for (var ii of sp) { + if ( ii == '' ) continue; + inputs.push(ii); + is_fns.push(true); + } + break; + case '-q': + quiet = true; + break; + case '-trace': + show_trace = true; + break; + default: + inputs.push(process.argv[i]); + is_fns.push(true); + break; + } + } + if (inputs.length == 0) { + ParseStdin(); + } + else { + const timer = new Timer({ label: 'test-timer' }); + timer.start(); + for (var f = 0; f \< inputs.length; ++f) + { + if (is_fns[f]) + ParseFilename(inputs[f], f); + else + ParseString(inputs[f], f); + } + timer.stop(); + var t = timer.time().m * 60 + timer.time().s + timer.time().ms / 1000; + if (!quiet) console.error('Total Time: ' + t); + } + process.exitCode = error_code; +} + +function ParseStdin() { + var sb = new strops.StringBuilder(); + var ch; + while ((ch = getChar()) != '') { + sb.Append(ch); + } + var input = sb.ToString(); + var str = antlr4.CharStreams.fromString(input); + DoParse(str, "stdin", 0); +} + +function ParseString(input, row_number) { + var str = antlr4.CharStreams.fromString(input); + DoParse(str, "string" + string_instance++, row_number); +} + +function ParseFilename(input, row_number) { + var str = antlr4.CharStreams.fromPathSync(input, encoding); + DoParse(str, input, row_number); +} + +function DoParse(str, input_name, row_number) { + const lexer = new (str); + lexer.strictMode = false; + const tokens = new antlr4.CommonTokenStream(lexer); + const parser = new (tokens); + lexer.removeErrorListeners(); + parser.removeErrorListeners(); + var output = tee ? fs.openSync(input_name + ".errors", 'w') : 1; + var listener_parser = new MyErrorListener(quiet, tee, output); + var listener_lexer = new MyErrorListener(quiet, tee, output); + parser.addErrorListener(listener_parser); + lexer.addErrorListener(listener_lexer); + if (show_tokens) { + for (var i = 0; ; ++i) { + var ro_token = lexer.nextToken(); + var token = ro_token; + token.TokenIndex = i; + console.error(token.toString()); + if (token.type === antlr4.Token.EOF) + break; + } + lexer.reset(); + } + if (show_trace) { + parser._interp.trace_atn_sim = true; + antlr4.context.PredictionContext.trace_atn_sim = true; + } + const timer = new Timer({ label: 'test-timer2' }); + timer.start(); + const tree = parser.(); + timer.stop(); + var result = ""; + if (listener_parser.had_error || listener_lexer.had_error) { + result = 'fail'; + error_code = 1; + } + else { + result = 'success'; + } + var t = timer.time().m * 60 + timer.time().s + timer.time().ms / 1000; + if (show_tree) { + if (tee) { + fs.writeFileSync(input_name + ".tree", tree.toStringTree(parser.ruleNames)); + } else { + console.error(tree.toStringTree(parser.ruleNames)); + } + } + if (! quiet) { + console.error(prefix + 'JavaScript ' + row_number + ' ' + input_name + ' ' + result + ' ' + t); + } + if (tee) { + fs.closeSync(output); + } +} + + +main() diff --git a/_scripts/templates/JavaScript/build.ps1 b/_scripts/templates/JavaScript/build.ps1 new file mode 100644 index 0000000000..5187b4f879 --- /dev/null +++ b/_scripts/templates/JavaScript/build.ps1 @@ -0,0 +1,14 @@ +# Generated from trgen +if (Test-Path -Path transformGrammar.py -PathType Leaf) { + $(& python3 transformGrammar.py ) 2>&1 | Write-Host +} + + -encoding -Dlanguage=JavaScript } > ; $compile_exit_code = $LASTEXITCODE) | Write-Host +if($compile_exit_code -ne 0){ + exit $compile_exit_code +\} +}> + +$(& npm install ; $compile_exit_code = $LASTEXITCODE ) | Write-Host +exit $compile_exit_code diff --git a/_scripts/templates/JavaScript/build.sh b/_scripts/templates/JavaScript/build.sh new file mode 100644 index 0000000000..c08aeb6a64 --- /dev/null +++ b/_scripts/templates/JavaScript/build.sh @@ -0,0 +1,11 @@ +# Generated from trgen +set -e + +if [ -f transformGrammar.py ]; then python3 transformGrammar.py ; fi + +" -encoding -Dlanguage=JavaScript } > +} > + +npm install +exit 0 diff --git a/_scripts/templates/JavaScript/clean.ps1 b/_scripts/templates/JavaScript/clean.ps1 new file mode 100644 index 0000000000..2e12a7cdba --- /dev/null +++ b/_scripts/templates/JavaScript/clean.ps1 @@ -0,0 +1,17 @@ +# Generated from trgen +$(& Remove-Item *.interp -Recurse -Force ) 2>&1 | Out-Null +$files = New-Object System.Collections.Generic.List[string] +" -depend -encoding -Dlanguage=Dart } > +foreach ($s in $f) { + $j = $s.Split(" ")[0] + $files.Add($j) +\} +foreach ($f in $files) +{ + $(& Remove-Item $f -Force ) 2>&1 | Out-Null +\} +} > +$(& Remove-Item node_modules -Recurse -Force ) 2>&1 | Out-Null +$(& Remove-Item package-lock.json -Recurse -Force ) 2>&1 | Out-Null +exit 0 diff --git a/_scripts/templates/JavaScript/clean.sh b/_scripts/templates/JavaScript/clean.sh new file mode 100644 index 0000000000..b037cc7223 --- /dev/null +++ b/_scripts/templates/JavaScript/clean.sh @@ -0,0 +1,12 @@ +# Generated from trgen +rm -f *.interp +files=() +" -depend -encoding -Dlanguage=JavaScript } > | awk '{print $1\}' | grep -v ':'` ) +} > +for i in ${files[*]} +do + rm -f $i +done +rm -rf node_modules package-lock.json +exit 0 diff --git a/_scripts/templates/JavaScript/index.js b/_scripts/templates/JavaScript/index.js deleted file mode 100644 index 8d589a9f2d..0000000000 --- a/_scripts/templates/JavaScript/index.js +++ /dev/null @@ -1,107 +0,0 @@ -// Template generated code from trgen - -import antlr4 from 'antlr4'; - from './'; -} > -import CaseChangingStream from './CaseChangingStream.js'; -import strops from 'typescript-string-operations'; -import fs from 'fs-extra'; - -function getChar() { - let buffer = Buffer.alloc(1); - var xx = fs.readSync(0, buffer, 0, 1); - if (xx === 0) { - return ''; - } - return buffer.toString('utf8'); -} - -class MyErrorListener extends antlr4.error.ErrorListener { - syntaxError(recognizer, offendingSymbol, line, column, msg, err) { - num_errors++; - console.log(`line ${line}:${column} ${msg}`); - } -} - -var show_tokens = false; -var show_tree = false; -var input = null; -var file_name = null; -for (let i = 2; i \< process.argv.length; ++i) -{ - switch (process.argv[i]) { - case '-tokens': - var show_tokens = true; - break; - case '-tree': - var show_tree = true; - break; - case '-input': - var input = process.argv[++i]; - break; - case '-file': - var file_name = process.argv[++i]; - break; - default: - console.log('unknown '.concat(process.argv[i])); - } -} -var str = null; -if (input == null && file_name == null) -{ - var sb = new strops.StringBuilder(); - var ch; - while ((ch = getChar()) != '') - { - sb.Append(ch); - } - var input = sb.ToString(); - str = antlr4.CharStreams.fromString(input); -} else if (input != null) -{ - str = antlr4.CharStreams.fromString(input); -} else if (file_name != null) -{ - str = antlr4.CharStreams.fromPathSync(file_name, 'utf8'); -} -var num_errors = 0; - -str = new CaseChangingStream(str, "" === "Upper"); - -const lexer = new (str); -lexer.strictMode = false; -const tokens = new antlr4.CommonTokenStream(lexer); -const parser = new (tokens); -lexer.removeErrorListeners(); -parser.removeErrorListeners(); -parser.addErrorListener(new MyErrorListener()); -lexer.addErrorListener(new MyErrorListener()); -if (show_tokens) -{ - for (var i = 0; ; ++i) - { - var ro_token = lexer.nextToken(); - var token = ro_token; - token.TokenIndex = i; - console.log(token.toString()); - if (token.type === antlr4.Token.EOF) - break; - } - lexer.reset(); -} -const tree = parser.(); -if (num_errors > 0) -{ - // Listener will have already printed the error(s) to stdout. - console.error('Parse failed.'); - process.exitCode = 1; -} -else -{ - console.error('Parse succeeded.'); - process.exitCode = 0; - if (show_tree) - { - console.log(tree.toStringTree(parser.ruleNames)); - } -} diff --git a/_scripts/templates/JavaScript/makefile b/_scripts/templates/JavaScript/makefile index 760ec47ee6..9996fc71b3 100644 --- a/_scripts/templates/JavaScript/makefile +++ b/_scripts/templates/JavaScript/makefile @@ -1,25 +1,10 @@ -# Template generated code from trgen -JAR = -CLASSPATH = $(JAR)\;:. -.SUFFIXES: .g4 .js -ANTLRGRAMMARS ?= $(wildcard *.g4) -GENERATED = }> -SOURCES = $(GENERATED) index.js -default: setup classes -setup: - if [ -f transformGrammar.py ]; then python3 transformGrammar.py ; fi -classes: $(SOURCES) - npm install +# Generated from trgen +default: + bash build.sh clean: - rm -rf node_modules - rm -f package-lock.json - rm -f *.interp - rm -f *.tokens - rm -f $(GENERATED) + bash clean.sh run: - trwdog node index.js $(RUNARGS) - : - java -jar $(JAR) -encoding -Dlanguage=JavaScript } > $\< -} > -test: + @trwdog node Test.js $(RUNARGS) +FORCE: ; +test: FORCE bash test.sh diff --git a/_scripts/templates/JavaScript/package.json b/_scripts/templates/JavaScript/package.json index 75a798edea..93f9308517 100644 --- a/_scripts/templates/JavaScript/package.json +++ b/_scripts/templates/JavaScript/package.json @@ -2,7 +2,7 @@ "name": "i", "version": "1.0.0", "description": "", - "main": "index.js", + "main": "Test.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1" }, @@ -10,7 +10,8 @@ "license": "ISC", "dependencies": { "antlr4": "^4.11.0", - "fs-extra": "^11.0.0", + "fs-extra": "^10.1.0", + "timer-node": "^5.0.6", "typescript-string-operations": "^1.4.1" }, "type": "module" diff --git a/_scripts/templates/JavaScript/test.ps1 b/_scripts/templates/JavaScript/test.ps1 new file mode 100644 index 0000000000..d1010828ab --- /dev/null +++ b/_scripts/templates/JavaScript/test.ps1 @@ -0,0 +1,149 @@ +# Generated from trgen + +$TestDirectory = "../../" +Write-Host "Test cases here: $TestDirectory" + +# People often specify a test file directory, but sometimes no +# tests are provided. Git won't check in an empty directory. +# Test if the test file directory does not exist, or it is just +# an empty directory. +if (!(Test-Path -Path "$TestDirectory")) { + Write-Host "No test cases provided." + exit 0 +} elseif (!(Test-Path "$TestDirectory/*")) { + Write-Host "No test cases provided." + exit 0 +} + +# Get a list of test files from the test directory. Do not include any +# .errors or .tree files. Pay close attention to remove only file names +# that end with the suffix .errors or .tree. +if (Test-Path -Path "tests.txt" -PathType Leaf) { + Remove-Item "tests.txt" +} +$files = New-Object System.Collections.Generic.List[string] +foreach ($item in Get-ChildItem $TestDirectory -Recurse) { + $file = $item.fullname + $ext = $item.Extension + if (Test-Path $file -PathType Container) { + continue + } elseif ($ext -eq ".errors") { + continue + } elseif ($ext -eq ".tree") { + continue + } else { + $(& triconv -f utf-8 $file ; $last = $LASTEXITCODE ) | Out-Null + if ($last -ne 0) + { + continue + } + $files.Add($item) + Write-Host "Test case: $item" + } +} +foreach ($file in $files) { + Add-Content "tests.txt" $file +} +if (-not(Test-Path -Path "tests.txt" -PathType Leaf)) { + Write-Host "No test cases provided." + exit 0 +} + +# Parse all input files. +get-content "tests.txt" | trwdog node Test.js -q -x -tee -tree *> parse.txt +$status = $LASTEXITCODE + +# trwdog returns 255 if it cannot spawn the process. This could happen +# if the environment for running the program does not exist, or the +# program did not build. +if ( $status -eq 255 ) { + Write-Host "Test failed." + Get-Content $file | Write-Host + exit 1 +} + +# Any parse errors will be put in .errors files. But, if there's any +# output from the program in stdout or stderr, it's all bad news. +$size = (Get-Item -Path "parse.txt").Length +if ( $size -eq 0 ) { +} else { + Write-Host "Test failed." + Get-Content "parse.txt" | Write-Host + exit 1 +} + +# Check if any .errors/.tree files have changed. That's not good. +$message = git diff --exit-code --name-only $TestDirectory +$updated = $LASTEXITCODE +Write-Host $message + +# Check if any untracked .errors files are not empty. +$new_errors_txt = New-Object System.Collections.Generic.List[string] +$new_errors2_txt = git ls-files --exclude-standard -o --ignored $TestDirectory +$new_errors = $LASTEXITCODE + +# Gather up all untracked .errors file output. These are new errors +# and must be reported as a parse fail. +if ( ! [String]::IsNullOrWhiteSpace($new_errors2_txt) ) { + $new_errors3_txt = $new_errors2_txt.Split("\n\r\t ") +} else { + $new_errors3_txt = [System.Collections.Arraylist]@() +} +if (Test-Path -Path "new_errors.txt" -PathType Leaf) { + Remove-Item "new_errors.txt" +} +New-Item -Path . -Name "new_errors.txt" -ItemType "file" -Value "" | Out-Null +foreach ($s in $new_errors3_txt) { + if ( [String]::IsNullOrWhiteSpace($s) ) { + continue + } + $ext = $item.Extension + if (! $s.EndsWith(".errors")) { + continue + } + $file = $s + $size = (Get-Item -Path $file).Length + if ( $size -eq 0 ) { + } else { + $new_errors_txt.Add($item) + Add-Content -Path "new_errors.txt" -Value "$item" + ((Get-Content $file) -join "`n") + "`n" | Add-Content -Path "new_errors.txt" + } +} + +# If "git diff" reported an exit code of 129, it is because +# the directory containing the grammar is not in a repo. In this +# case, assume parse error code as the defacto result. +if ( $updated -eq 129 ) { + Write-Host "Grammar outside a git repository. Assuming parse exit code." + if ( $status -eq 0 ) { + Write-Host "Test succeeded." + } else { + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + } + $err = $status + exit 1 +} + +# "Git diff" reported a difference. Redo the "git diff" to print out all +# the differences. Also, output any untracked, non-zero length .errors files. +if ( $updated -eq 1 ) { + Write-Host "Difference in output." + git diff . | Write-Host + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + exit 1 +} + +# If there's non-zero length .errors flies that are new, report them +# as errors in the parse. +if ( $new_errors_txt.Count -gt 0 ) { + Write-Host "New errors in output." + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + exit 1 +} + +Write-Host "Test succeeded." +exit 0 diff --git a/_scripts/templates/JavaScript/test.sh b/_scripts/templates/JavaScript/test.sh index 60a3e596a3..6de752302f 100644 --- a/_scripts/templates/JavaScript/test.sh +++ b/_scripts/templates/JavaScript/test.sh @@ -1,60 +1,149 @@ -# Template generated code from trgen -err=0 +# Generated from trgen + +# People often specify a test file directory, but sometimes no +# tests are provided. Git won't check in an empty directory. +# Test if the test file directory does not exist, or it is just +# an empty directory. +if [ ! -d ../ ] +then + echo "No test cases provided." + exit 0 +elif [ ! "$(ls -A ../)" ] +then + echo "No test cases provided." + exit 0 +fi + SAVEIFS=$IFS IFS=$(echo -en "\n\b") -stderr_file=`mktemp --tmpdir=. stderr.XXXXXXXXXX` -parse_out_file=`mktemp --tmpdir=. parse_out.XXXXXXXXXX` -for g in `find ../ -type f | grep -v '.errors$' | grep -v '.tree$'` + +# Get a list of test files from the test directory. Do not include any +# .errors or .tree files. Pay close attention to remove only file names +# that end with the suffix .errors or .tree. +files2=`find ../ -type f | grep -v '.errors$' | grep -v '.tree$'` +files=() +for f in $files2 do - file="$g" - x1="${g##*.}" - if [ "$x1" != "errors" ] - then - echo "$file" - trwdog node index.js -file "$file" -tree > "$parse_out_file" 2> "$stderr_file" - status="$?" - head -55 "$stderr_file" - if [ -f "$file".errors ] + triconv -f utf-8 $f > /dev/null 2>&1 + if [ "$?" = "0" ] then - if [ "$status" = "0" ] - then - echo Expected parse fail. - err=1 - break - else - echo Expected. - diff \<(tr -d "\r\n" \< "$file".errors) \<(tr -d "\r\n" \< "$parse_out_file") - status="$?" - if [ "$status" = "0" ] - then - echo Parse errors match succeeded. - else - echo Expected parse errors match. - err=1 - break - fi - fi - else # No .errors file - if [ "$status" != "0" ] - then - err=1 - break - fi - if [ -f "$file".tree ] - then - diff \<(tr -d "\r\n" \< "$file".tree) \<(tr -d "\r\n" \< "$parse_out_file") - status="$?" - if [ "$status" = "0" ] + files+=( $f ) + fi +done + +# Parse all input files. +echo "${files[*]}" | trwdog node Test.js -q -x -tee -tree > parse.txt 2>&1 +status=$? + +# trwdog returns 255 if it cannot spawn the process. This could happen +# if the environment for running the program does not exist, or the +# program did not build. +if [ "$status" = "255" ] +then + echo "Test failed." + cat parse.txt + exit 1 +fi + +# Any parse errors will be put in .errors files. But, if there's any +# output from the program in stdout or stderr, it's all bad news. +if [ -s parse.txt ] +then + echo "Test failed." + cat parse.txt + exit 1 +fi + +# rm -rf `find ../ -type f -name '*.errors' -o -name '*.tree' -size 0` + +# For Unix environments, convert the newline in the .errors and .trees +# to Unix style. +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) machine=Linux;; + Darwin*) machine=Mac;; + CYGWIN*) machine=Cygwin;; + MINGW*) machine=MinGw;; + *) machine="UNKNOWN:${unameOut}" +esac +if [[ "$machine" == "MinGw" || "$machine" == "Msys" || "$machine" == "Cygwin" || "#machine" == "Linux" ]] +then + gen=`find ../ -type f -name '*.errors' -o -name '*.tree'` + if [ "$gen" != "" ] + then + dos2unix $gen + fi +fi + +old=`pwd` +cd ../ + +# Check if any files in the test files directory have changed. +git diff --exit-code --name-only . > $old/updated.txt 2>&1 +updated=$? + +# Check if any untracked .errors files. +git ls-files --exclude-standard -o --ignored > $old/new_errors2.txt 2>&1 +new_errors=$? + +# Gather up all untracked .errors file output. These are new errors +# and must be reported as a parse fail. +rm -f $old/new_errors.txt +touch $old/new_errors.txt +for f in `cat $old/new_errors2.txt` +do + ext=${f##*.} + ext=".$ext" + if [ "$ext" = ".errors" ] + then + if [ -s $f ] then - echo Parse tree match succeeded. - else - echo Expected parse tree match. - err=1 - break + echo $f >> $old/new_errors.txt + cat $f >> $old/new_errors.txt fi - fi fi - fi done -rm "$stderr_file" "$parse_out_file" -exit $err + +# If "git diff" reported an exit code of 129, it is because +# the directory containing the grammar is not in a repo. In this +# case, assume parse error code as the defacto result. +if [ "$updated" = "129" ] +then + echo "Grammar outside a git repository. Assuming parse exit code." + if [ "$status" = 0 ] + then + echo "Test succeeded." + else + cat $old/new_errors.txt + echo "Test failed." + fi + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit $status +fi + +# "Git diff" reported a difference. Redo the "git diff" to print out all +# the differences. Also, output any untracked, non-zero length .errors files. +if [ "$updated" = "1" ] +then + echo "Difference in output." + git diff . + cat $old/new_errors.txt + echo "Test failed." + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit 1 +fi + +# If there's non-zero length .errors flies that are new, report them +# as errors in the parse. +if [ -s $old/new_errors.txt ] +then + echo "New errors in output." + cat $old/new_errors.txt + echo "Test failed." + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit 1 +fi + +echo "Test succeeded." +rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt +exit 0 diff --git a/_scripts/templates/JavaScript/tester.psm1 b/_scripts/templates/JavaScript/tester.psm1 deleted file mode 100644 index 52506c0ee7..0000000000 --- a/_scripts/templates/JavaScript/tester.psm1 +++ /dev/null @@ -1,63 +0,0 @@ -# Template generated code from trgen -function Build-Grammar { - -Dlanguage=JavaScript } > - if($LASTEXITCODE -ne 0){ - return @{ - Message = $g - Success = $false - \} - \} -}> - $msg = npm install - return @{ - Message = $msg - Success = $LASTEXITCODE -eq 0 - } -} - -function Test-Case { - param ( - $InputFile, - $TokenFile, - $TreeFile, - $ErrorFile - ) - # Save input and output character encodings and switch to UTF-8. - $oldInputEncoding = [console]::InputEncoding - $oldOutputEncoding = [console]::OutputEncoding - $OutputEncoding = [console]::InputEncoding = [console]::OutputEncoding = New-Object System.Text.UTF8Encoding - - $parseOutFile = $InputFile + ".out" - $o = trwdog node index.js -file $InputFile -tree | Out-File -LiteralPath "$parseOutFile" -Encoding UTF8 - $parseOk = $LASTEXITCODE -eq 0 - $treeMatch = $true - if ($errorFile) { - # If we expected errors, then a failed parse was a successful test. - if (!$parseOk) { - Write-Host "Expected." - # Confirm that the errors we received are the ones we expected. - $expectedData = (Get-Content $errorFile -Encoding UTF8) -join "" -replace "\r\n","" - $actualData = (Get-Content $parseOutFile -Encoding UTF8) -join "" -replace "\r\n","" - $parseOk = ($actualData -eq $expectedData) - if ($parseOk) { - Write-Host "Error list match succeeded." - } else { - Write-Host "Expected error list match." -ForegroundColor Red - } - } - } else { - if ($parseOk -and (Test-Path $TreeFile)) { - # Confirm that the parse tree we received is the one we expected. - $expectedData = Get-Content $TreeFile -Encoding UTF8 - $actualData = Get-Content $parseOutFile -Encoding UTF8 - $treeMatch = ($actualData -eq $expectedData) - } - } - # Restore input and output character encodings. - [console]::InputEncoding = $oldInputEncoding - [console]::OutputEncoding = $oldOutputEncoding - - Remove-Item $parseOutFile - return $parseOk, $treeMatch -} \ No newline at end of file diff --git a/_scripts/templates/PHP/Test.php b/_scripts/templates/PHP/Test.php index a86605f752..e5e6b720e3 100644 --- a/_scripts/templates/PHP/Test.php +++ b/_scripts/templates/PHP/Test.php @@ -1,8 +1,12 @@ \ require_once 'vendor/autoload.php'; '; } > +require __DIR__ . '/vendor/autoload.php'; + +use SebastianBergmann\Timer\Timer; use Antlr\Antlr4\Runtime\CommonTokenStream; use Antlr\Antlr4\Runtime\Error\Listeners\DiagnosticErrorListener; //use Antlr\Antlr4\Runtime\Error\Listeners\ConsoleErrorListener; @@ -17,88 +21,207 @@ use Antlr\Antlr4\Runtime\Tree\TerminalNode; use Antlr\Antlr4\Runtime\Token; use Antlr\Antlr4\Runtime\CommonToken; + final class TreeShapeListener implements ParseTreeListener { - public function visitTerminal(TerminalNode $node) : void {} - public function visitErrorNode(ErrorNode $node) : void {} - public function exitEveryRule(ParserRuleContext $ctx) : void {} - public function enterEveryRule(ParserRuleContext $ctx) : void { - echo $ctx->getText(); - } + public function visitTerminal(TerminalNode $node) : void {} + public function visitErrorNode(ErrorNode $node) : void {} + public function exitEveryRule(ParserRuleContext $ctx) : void {} + + public function enterEveryRule(ParserRuleContext $ctx) : void { + echo $ctx->getText(); + } } + // TODO: should ConsoleErrorListener be final? class MyErrorListener extends BaseErrorListener /*extends ConsoleErrorListener*/ { - public bool $noError = true; - public function syntaxError( - Recognizer $recognizer, - ?object $offendingSymbol, - int $line, - int $charPositionInLine, - string $msg, - ?RecognitionException $e - ) : void { - \fwrite(\STDOUT, \sprintf("line %d:%d %s\n", $line, $charPositionInLine, $msg)); - //parent::syntaxError($recognizer,$offendingSymbol,$line,$charPositionInLine,$msg,$e); - $this->noError = false; - } + public bool $had_error; + public bool $quiet; + public bool $tee; + public mixed $output; + public function __construct($q, $t, $o) { + $this->output = $o; + $this->quiet = $q; + $this->tee = $t; + $this->had_error = false; + } + public function syntaxError( + Recognizer $recognizer, + ?object $offendingSymbol, + int $line, + int $charPositionInLine, + string $msg, + ?RecognitionException $e + ) : void { + $this->had_error = true; + if ($this->tee) { + fwrite($this->output, sprintf("line %d:%d %s\n", $line, $charPositionInLine, $msg)); + } + if (! $this->quiet) { + fwrite(STDERR, sprintf("line %d:%d %s\n", $line, $charPositionInLine, $msg)); + } + } } + +$tee = false; +$show_profile = false; $show_tree = false; $show_tokens = false; -$file_name = ''; -$input = ''; -$count = count($argv); -for ($i = 0; $i \< $count; $i++) { - if ($argv[$i] == "-tokens") { - $show_tokens = true; - continue; - } else if ($argv[$i] == "-tree") { - $show_tree = true; - continue; - } else if ($argv[$i] == "-input") { - $input = $argv[++$i]; - } else if ($argv[$i] == "-file") { - $file_name = $argv[++$i]; - } -} -if ($input == "" && $file_name == "") { - $in = fopen("php://stdin", "r"); - $s = ''; - while (!feof($in)) { - $s .= fread($in,65536); - } - $str = InputStream::fromString($s); - fclose($in); -} else if ($input != "") { - $str = InputStream::fromString($input); -} else if ($file_name != "") { - $str = InputStream::fromPath($file_name); -} - -$lexer = new ($str); -if ($show_tokens) { - for ($i=0; ; $i++) { - $token = $lexer->nextToken(); - $token->setTokenIndex($i); - print($token . PHP_EOL); - if ($token->getType() == Token::EOF){ - break; - } - } - $lexer->reset(); +$show_trace = false; +$inputs = array(); +$is_fns = array(); +$error_code = 0; +$string_instance = 0; +$prefix = ""; +$quiet = false; + +function main($argv) : void { + global $tee; + global $show_profile; + global $show_tree; + global $show_tokens; + global $show_trace; + global $inputs; + global $is_fns; + global $error_code; + global $prefix; + global $quiet; + for ($i = 1; $i \< count($argv); $i++) { + if ($argv[$i] == "-tokens") { + $show_tokens = true; + } else if ($argv[$i] == "-tree") { + $show_tree = true; + } else if ($argv[$i] == "-prefix") { + $prefix = $argv[++$i] . " "; + } else if ($argv[$i] == "-input") { + array_push($inputs, $argv[++$i]); + array_push($is_fns, false); + } else if ($argv[$i] == "-tee") { + $tee = true; + } else if ($argv[$i] == "-x") { + while($f = fgets(STDIN)){ + $f = trim($f); + array_push($inputs, $f); + array_push($is_fns, true); + } + } else if ($argv[$i] == "-q") { + $quiet = true; + } else if ($argv[$i] == "-trace") { + $show_trace = true; + } else { + array_push($inputs, $argv[$i]); + array_push($is_fns, true); + } + } + if (count($inputs) == 0) { + ParseStdin(); + } + else { + $timer = new Timer; + $timer->start(); + for ($f = 0; $f \< count($inputs); ++$f) + { + if ($is_fns[$f]) + ParseFilename($inputs[$f], $f); + else + ParseString($inputs[$f], $f); + } + $duration = $timer->stop(); + if (! $quiet) { + fwrite(STDERR, "Total Time: " . $duration->asSeconds() . "\n"); + } + } + exit($error_code); +} + +function ParseStdin() { + $in = fopen("php://stdin", "r"); + $s = ''; + while (!feof($in)) { + $s .= fread($in,65536); + } + $str = InputStream::fromString($s); + fclose($in); + DoParse($str, "stdin", 0); } -$lexerErrorListener = new MyErrorListener(); -$lexer->removeErrorListeners(); -$lexer->addErrorListener($lexerErrorListener); -$tokens = new CommonTokenStream($lexer); -$parser = new ($tokens); -$parserErrorListener = new MyErrorListener(); -$parser->removeErrorListeners(); -$parser->addErrorListener($parserErrorListener); -$tree = $parser->(); -if ($parserErrorListener->noError&&$lexerErrorListener->noError){ - if ($show_tree) { - print($tree->toStringTree($parser->getRuleNames()) . "\n"); - } - exit(0); + +function ParseString($input, $row_number) { + global $string_instance; + $str = InputStream::fromString($input); + DoParse($str, "string" . $string_instance++, $row_number); +} + +function ParseFilename($input, $row_number) { + $str = InputStream::fromPath($input); + DoParse($str, $input, $row_number); } -// Listener will have already printed the error(s) to stdout. -exit(1); + +function DoParse($str, $input_name, $row_number) { + global $tee; + global $show_tree; + global $show_tokens; + global $show_trace; + global $inputs; + global $is_fns; + global $error_code; + global $prefix; + global $quiet; + $lexer = new ($str); + if ($show_tokens) { + for ($i=0; ; $i++) { + $token = $lexer->nextToken(); + $token->setTokenIndex($i); + fwrite(STDERR, $token . PHP_EOL); + if ($token->getType() == Token::EOF){ + break; + } + } + $lexer->reset(); + } + + if ( $tee ) { + $output = fopen($input_name . ".errors", "w"); + } else { + $output = STDERR; + } + $lexerErrorListener = new MyErrorListener($quiet, $tee, $output); + $lexer->removeErrorListeners(); + $lexer->addErrorListener($lexerErrorListener); + $tokens = new CommonTokenStream($lexer); + $parser = new ($tokens); + $parserErrorListener = new MyErrorListener($quiet, $tee, $output); + $parser->removeErrorListeners(); + $parser->addErrorListener($parserErrorListener); + if ($show_trace) { + $parser->setTrace(true); + Antlr\Antlr4\Runtime\Atn\ParserATNSimulator::$traceAtnSimulation = true; + } + $timer2 = new Timer; + $timer2->start(); + $tree = $parser->(); + $duration = $timer2->stop(); + $result = ""; + if ($parserErrorListener->had_error || $lexerErrorListener->had_error) { + $result = "fail"; + $error_code = 1; + } + else { + $result = "success"; + } + if ($show_tree) { + if ($tee) { + $handle = fopen($input_name . ".tree", "w"); + fprintf($handle, "%s", $tree->toStringTree($parser->getRuleNames())); + fclose($handle); + } else { + fwrite(STDERR, $tree->toStringTree($parser->getRuleNames())); + } + } + if ( ! $quiet ) { + fwrite(STDERR, $prefix . "PHP " . $row_number . " " . $input_name . " " . $result . " " . $duration->asSeconds() . "\n"); + } + if ( $tee ) { + fclose($output); + } +} + +main($argv); diff --git a/_scripts/templates/PHP/build.ps1 b/_scripts/templates/PHP/build.ps1 new file mode 100644 index 0000000000..04173d0494 --- /dev/null +++ b/_scripts/templates/PHP/build.ps1 @@ -0,0 +1,14 @@ +# Generated from trgen +if (Test-Path -Path transformGrammar.py -PathType Leaf) { + $(& python3 transformGrammar.py ) 2>&1 | Write-Host +} + + -encoding -Dlanguage=PHP } > ; $compile_exit_code = $LASTEXITCODE) | Write-Host +if($compile_exit_code -ne 0){ + exit $compile_exit_code +\} +}> + +$(& composer install ; $compile_exit_code = $LASTEXITCODE ) | Write-Host +exit $compile_exit_code diff --git a/_scripts/templates/PHP/build.sh b/_scripts/templates/PHP/build.sh new file mode 100644 index 0000000000..cf6082b634 --- /dev/null +++ b/_scripts/templates/PHP/build.sh @@ -0,0 +1,12 @@ +# Generated from trgen +set -e + +if [ -f transformGrammar.py ]; then python3 transformGrammar.py ; fi + +" -encoding -Dlanguage=PHP } > +}> + +composer install + +exit 0 diff --git a/_scripts/templates/PHP/clean.ps1 b/_scripts/templates/PHP/clean.ps1 new file mode 100644 index 0000000000..0906002a78 --- /dev/null +++ b/_scripts/templates/PHP/clean.ps1 @@ -0,0 +1,17 @@ +# Generated from trgen +$(& Remove-Item *.interp -Recurse -Force ) 2>&1 | Out-Null +$files = New-Object System.Collections.Generic.List[string] +" -depend -encoding -Dlanguage=Dart } > +foreach ($s in $f) { + $j = $s.Split(" ")[0] + $files.Add($j) +\} +foreach ($f in $files) +{ + $(& Remove-Item $f -Force ) 2>&1 | Out-Null +\} +} > +$(& Remove-Item vendor -Recurse -Force ) 2>&1 | Out-Null +$(& Remove-Item composer.log -Recurse -Force ) 2>&1 | Out-Null +exit 0 diff --git a/_scripts/templates/PHP/clean.sh b/_scripts/templates/PHP/clean.sh new file mode 100644 index 0000000000..d7b638afc3 --- /dev/null +++ b/_scripts/templates/PHP/clean.sh @@ -0,0 +1,12 @@ +# Generated from trgen +rm -f *.interp +files=() +" -depend -encoding -Dlanguage=PHP } > | awk '{print $1\}' | grep -v ':'` ) +} > +for i in ${files[*]} +do + rm -f $i +done +rm -rf vendor composer.lock +exit 0 diff --git a/_scripts/templates/PHP/composer.json b/_scripts/templates/PHP/composer.json index ef71b9f10b..40c5a4d0fd 100644 --- a/_scripts/templates/PHP/composer.json +++ b/_scripts/templates/PHP/composer.json @@ -1,5 +1,9 @@ { - "require": { - "antlr/antlr4-php-runtime": "0.8.0" - } + "require": { + "antlr/antlr4-php-runtime": "0.8.0", + "phpunit/php-timer": "^5.0", + "php": "^8.0", + "ext-mbstring": "^8.0", + "psr/log": "^2.0 || ^3.0" + } } diff --git a/_scripts/templates/PHP/makefile b/_scripts/templates/PHP/makefile index 50c2ffe5a8..37e07cb868 100644 --- a/_scripts/templates/PHP/makefile +++ b/_scripts/templates/PHP/makefile @@ -1,23 +1,10 @@ -# Template generated code from trgen -JAR = -CLASSPATH = $(JAR)\;:. -.SUFFIXES: .g4 .php -ANTLRGRAMMARS ?= $(wildcard *.g4) -GENERATED = }> -SOURCES = $(GENERATED) Test.php -default: setup classes -setup: - if [ -f transformGrammar.py ]; then python3 transformGrammar.py ; fi - composer install -classes: $(SOURCES) +# Generated from trgen +default: + bash build.sh clean: - rm -f *.interp - rm -f *.tokens - rm -f $(GENERATED) + bash clean.sh run: - trwdog php Test.php $(RUNARGS) - : - java -jar $(JAR) -encoding -Dlanguage=PHP } > $\< -} > -test: + @trwdog php -d memory_limit=1G Test.php $(RUNARGS) +FORCE: ; +test: FORCE bash test.sh diff --git a/_scripts/templates/PHP/test.ps1 b/_scripts/templates/PHP/test.ps1 new file mode 100644 index 0000000000..7cf697695a --- /dev/null +++ b/_scripts/templates/PHP/test.ps1 @@ -0,0 +1,149 @@ +# Generated from trgen + +$TestDirectory = "../../" +Write-Host "Test cases here: $TestDirectory" + +# People often specify a test file directory, but sometimes no +# tests are provided. Git won't check in an empty directory. +# Test if the test file directory does not exist, or it is just +# an empty directory. +if (!(Test-Path -Path "$TestDirectory")) { + Write-Host "No test cases provided." + exit 0 +} elseif (!(Test-Path "$TestDirectory/*")) { + Write-Host "No test cases provided." + exit 0 +} + +# Get a list of test files from the test directory. Do not include any +# .errors or .tree files. Pay close attention to remove only file names +# that end with the suffix .errors or .tree. +if (Test-Path -Path "tests.txt" -PathType Leaf) { + Remove-Item "tests.txt" +} +$files = New-Object System.Collections.Generic.List[string] +foreach ($item in Get-ChildItem $TestDirectory -Recurse) { + $file = $item.fullname + $ext = $item.Extension + if (Test-Path $file -PathType Container) { + continue + } elseif ($ext -eq ".errors") { + continue + } elseif ($ext -eq ".tree") { + continue + } else { + $(& triconv -f utf-8 $file ; $last = $LASTEXITCODE ) | Out-Null + if ($last -ne 0) + { + continue + } + $files.Add($item) + Write-Host "Test case: $item" + } +} +foreach ($file in $files) { + Add-Content "tests.txt" $file +} +if (-not(Test-Path -Path "tests.txt" -PathType Leaf)) { + Write-Host "No test cases provided." + exit 0 +} + +# Parse all input files. +get-content "tests.txt" | trwdog php -d memory_limit=1G Test.php -q -x -tee -tree *> parse.txt +$status = $LASTEXITCODE + +# trwdog returns 255 if it cannot spawn the process. This could happen +# if the environment for running the program does not exist, or the +# program did not build. +if ( $status -eq 255 ) { + Write-Host "Test failed." + Get-Content $file | Write-Host + exit 1 +} + +# Any parse errors will be put in .errors files. But, if there's any +# output from the program in stdout or stderr, it's all bad news. +$size = (Get-Item -Path "parse.txt").Length +if ( $size -eq 0 ) { +} else { + Write-Host "Test failed." + Get-Content "parse.txt" | Write-Host + exit 1 +} + +# Check if any .errors/.tree files have changed. That's not good. +$message = git diff --exit-code --name-only $TestDirectory +$updated = $LASTEXITCODE +Write-Host $message + +# Check if any untracked .errors files are not empty. +$new_errors_txt = New-Object System.Collections.Generic.List[string] +$new_errors2_txt = git ls-files --exclude-standard -o --ignored $TestDirectory +$new_errors = $LASTEXITCODE + +# Gather up all untracked .errors file output. These are new errors +# and must be reported as a parse fail. +if ( ! [String]::IsNullOrWhiteSpace($new_errors2_txt) ) { + $new_errors3_txt = $new_errors2_txt.Split("\n\r\t ") +} else { + $new_errors3_txt = [System.Collections.Arraylist]@() +} +if (Test-Path -Path "new_errors.txt" -PathType Leaf) { + Remove-Item "new_errors.txt" +} +New-Item -Path . -Name "new_errors.txt" -ItemType "file" -Value "" | Out-Null +foreach ($s in $new_errors3_txt) { + if ( [String]::IsNullOrWhiteSpace($s) ) { + continue + } + $ext = $item.Extension + if (! $s.EndsWith(".errors")) { + continue + } + $file = $s + $size = (Get-Item -Path $file).Length + if ( $size -eq 0 ) { + } else { + $new_errors_txt.Add($item) + Add-Content -Path "new_errors.txt" -Value "$item" + ((Get-Content $file) -join "`n") + "`n" | Add-Content -Path "new_errors.txt" + } +} + +# If "git diff" reported an exit code of 129, it is because +# the directory containing the grammar is not in a repo. In this +# case, assume parse error code as the defacto result. +if ( $updated -eq 129 ) { + Write-Host "Grammar outside a git repository. Assuming parse exit code." + if ( $status -eq 0 ) { + Write-Host "Test succeeded." + } else { + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + } + $err = $status + exit 1 +} + +# "Git diff" reported a difference. Redo the "git diff" to print out all +# the differences. Also, output any untracked, non-zero length .errors files. +if ( $updated -eq 1 ) { + Write-Host "Difference in output." + git diff . | Write-Host + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + exit 1 +} + +# If there's non-zero length .errors flies that are new, report them +# as errors in the parse. +if ( $new_errors_txt.Count -gt 0 ) { + Write-Host "New errors in output." + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + exit 1 +} + +Write-Host "Test succeeded." +exit 0 diff --git a/_scripts/templates/PHP/test.sh b/_scripts/templates/PHP/test.sh index 92c98d3d5d..cc2a1b0d70 100644 --- a/_scripts/templates/PHP/test.sh +++ b/_scripts/templates/PHP/test.sh @@ -1,58 +1,149 @@ -# Template generated code from trgen -err=0 +# Generated from trgen + +# People often specify a test file directory, but sometimes no +# tests are provided. Git won't check in an empty directory. +# Test if the test file directory does not exist, or it is just +# an empty directory. +if [ ! -d ../ ] +then + echo "No test cases provided." + exit 0 +elif [ ! "$(ls -A ../)" ] +then + echo "No test cases provided." + exit 0 +fi + SAVEIFS=$IFS IFS=$(echo -en "\n\b") -parse_out_file=`mktemp --tmpdir=. parse_out.XXXXXXXXXX` -for g in `find ../ -type f | grep -v '.errors$' | grep -v '.tree$'` + +# Get a list of test files from the test directory. Do not include any +# .errors or .tree files. Pay close attention to remove only file names +# that end with the suffix .errors or .tree. +files2=`find ../ -type f | grep -v '.errors$' | grep -v '.tree$'` +files=() +for f in $files2 do - file="$g" - x1="${g##*.}" - if [ "$x1" != "errors" ] - then - echo "$file" - trwdog php Test.php -file "$file" -tree > "$parse_out_file" - status="$?" - if [ -f "$file".errors ] + triconv -f utf-8 $f > /dev/null 2>&1 + if [ "$?" = "0" ] then - if [ "$status" = "0" ] - then - echo Expected parse fail. - err=1 - break - else - echo Expected. - diff \<(tr -d "\r\n" \< "$file".errors) \<(tr -d "\r\n" \< "$parse_out_file") - status="$?" - if [ "$status" = "0" ] - then - echo Parse errors match succeeded. - else - echo Expected parse errors match. - err=1 - break - fi - fi - else # No .errors file - if [ "$status" != "0" ] - then - err=1 - break - fi - if [ -f "$file".tree ] - then - diff \<(tr -d "\r\n" \< "$file".tree) \<(tr -d "\r\n" \< "$parse_out_file") - status="$?" - if [ "$status" = "0" ] + files+=( $f ) + fi +done + +# Parse all input files. +echo "${files[*]}" | trwdog php -d memory_limit=1G Test.php -q -x -tee -tree > parse.txt 2>&1 +status=$? + +# trwdog returns 255 if it cannot spawn the process. This could happen +# if the environment for running the program does not exist, or the +# program did not build. +if [ "$status" = "255" ] +then + echo "Test failed." + cat parse.txt + exit 1 +fi + +# Any parse errors will be put in .errors files. But, if there's any +# output from the program in stdout or stderr, it's all bad news. +if [ -s parse.txt ] +then + echo "Test failed." + cat parse.txt + exit 1 +fi + +# rm -rf `find ../ -type f -name '*.errors' -o -name '*.tree' -size 0` + +# For Unix environments, convert the newline in the .errors and .trees +# to Unix style. +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) machine=Linux;; + Darwin*) machine=Mac;; + CYGWIN*) machine=Cygwin;; + MINGW*) machine=MinGw;; + *) machine="UNKNOWN:${unameOut}" +esac +if [[ "$machine" == "MinGw" || "$machine" == "Msys" || "$machine" == "Cygwin" || "#machine" == "Linux" ]] +then + gen=`find ../ -type f -name '*.errors' -o -name '*.tree'` + if [ "$gen" != "" ] + then + dos2unix $gen + fi +fi + +old=`pwd` +cd ../ + +# Check if any files in the test files directory have changed. +git diff --exit-code --name-only . > $old/updated.txt 2>&1 +updated=$? + +# Check if any untracked .errors files. +git ls-files --exclude-standard -o --ignored > $old/new_errors2.txt 2>&1 +new_errors=$? + +# Gather up all untracked .errors file output. These are new errors +# and must be reported as a parse fail. +rm -f $old/new_errors.txt +touch $old/new_errors.txt +for f in `cat $old/new_errors2.txt` +do + ext=${f##*.} + ext=".$ext" + if [ "$ext" = ".errors" ] + then + if [ -s $f ] then - echo Parse tree match succeeded. - else - echo Expected parse tree match. - err=1 - break + echo $f >> $old/new_errors.txt + cat $f >> $old/new_errors.txt fi - fi fi - fi done -rm "$parse_out_file" -exit $err + +# If "git diff" reported an exit code of 129, it is because +# the directory containing the grammar is not in a repo. In this +# case, assume parse error code as the defacto result. +if [ "$updated" = "129" ] +then + echo "Grammar outside a git repository. Assuming parse exit code." + if [ "$status" = 0 ] + then + echo "Test succeeded." + else + cat $old/new_errors.txt + echo "Test failed." + fi + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit $status +fi + +# "Git diff" reported a difference. Redo the "git diff" to print out all +# the differences. Also, output any untracked, non-zero length .errors files. +if [ "$updated" = "1" ] +then + echo "Difference in output." + git diff . + cat $old/new_errors.txt + echo "Test failed." + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit 1 +fi + +# If there's non-zero length .errors flies that are new, report them +# as errors in the parse. +if [ -s $old/new_errors.txt ] +then + echo "New errors in output." + cat $old/new_errors.txt + echo "Test failed." + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit 1 +fi + +echo "Test succeeded." +rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt +exit 0 diff --git a/_scripts/templates/PHP/tester.psm1 b/_scripts/templates/PHP/tester.psm1 deleted file mode 100644 index 692f62a5b6..0000000000 --- a/_scripts/templates/PHP/tester.psm1 +++ /dev/null @@ -1,63 +0,0 @@ -# Template generated code from trgen -function Build-Grammar { - -Dlanguage=PHP } > - if($LASTEXITCODE -ne 0){ - return @{ - Message = $g - Success = $false - \} - \} -}> - $msg = composer install - return @{ - Message = $msg - Success = $LASTEXITCODE -eq 0 - } -} - -function Test-Case { - param ( - $InputFile, - $TokenFile, - $TreeFile, - $ErrorFile - ) - # Save input and output character encodings and switch to UTF-8. - $oldInputEncoding = [console]::InputEncoding - $oldOutputEncoding = [console]::OutputEncoding - $OutputEncoding = [console]::InputEncoding = [console]::OutputEncoding = New-Object System.Text.UTF8Encoding - - $parseOutFile = $InputFile + ".out" - $o = trwdog php Test.php -file $InputFile-tree | Out-File -LiteralPath "$parseOutFile" -Encoding UTF8 - $parseOk = $LASTEXITCODE -eq 0 - $treeMatch = $true - if ($errorFile) { - # If we expected errors, then a failed parse was a successful test. - if (!$parseOk) { - Write-Host "Expected." - # Confirm that the errors we received are the ones we expected. - $expectedData = (Get-Content $errorFile -Encoding UTF8) -join "" -replace "\r\n","" - $actualData = (Get-Content $parseOutFile -Encoding UTF8) -join "" -replace "\r\n","" - $parseOk = ($actualData -eq $expectedData) - if ($parseOk) { - Write-Host "Error list match succeeded." - } else { - Write-Host "Expected error list match." -ForegroundColor Red - } - } - } else { - if ($parseOk -and (Test-Path $TreeFile)) { - # Confirm that the parse tree we received is the one we expected. - $expectedData = Get-Content $TreeFile -Encoding UTF8 - $actualData = Get-Content $parseOutFile -Encoding UTF8 - $treeMatch = ($actualData -eq $expectedData) - } - } - # Restore input and output character encodings. - [console]::InputEncoding = $oldInputEncoding - [console]::OutputEncoding = $oldOutputEncoding - - Remove-Item $parseOutFile - return $parseOk, $treeMatch -} \ No newline at end of file diff --git a/_scripts/templates/Python3/CaseChangingStream.py b/_scripts/templates/Python3/CaseChangingStream.py deleted file mode 100644 index e99fc4720f..0000000000 --- a/_scripts/templates/Python3/CaseChangingStream.py +++ /dev/null @@ -1,14 +0,0 @@ -# Template generated code from trgen -class CaseChangingStream(): - def __init__(self, stream, upper): - self._stream = stream - self._upper = upper - - def __getattr__(self, name): - return self._stream.__getattribute__(name) - - def LA(self, offset): - c = self._stream.LA(offset) - if c \<= 0: - return c - return ord(chr(c).upper() if self._upper else chr(c).lower()) \ No newline at end of file diff --git a/_scripts/templates/Python3/Program.py b/_scripts/templates/Python3/Program.py deleted file mode 100644 index 232452f159..0000000000 --- a/_scripts/templates/Python3/Program.py +++ /dev/null @@ -1,108 +0,0 @@ -# Template generated code from trgen - -import sys -from antlr4 import * -from antlr4.error.ErrorListener import ErrorListener -from readchar import readchar -from import ; -from import ; -from CaseChangingStream import *; -from datetime import datetime - -def getChar(): - xx = readchar() - if (xx == 0): - return ''; - return xx - -class MyErrorListener(ErrorListener): - __slots__ = 'num_errors' - - def __init__(self): - super().__init__() - self.num_errors = 0 - - def syntaxError(self, recognizer, offendingSymbol, line, column, msg, e): - self.num_errors = self.num_errors + 1 - print(f"line {line}:{column} {msg}"); - -def main(argv): - show_tokens = False - show_tree = False - input = None - file_name = None - i = 1 - encoding = "utf-8" - while i \< len(argv): - arg = argv[i] - if arg in ("-tokens"): - show_tokens = True - elif arg in ("-tree"): - show_tree = True - elif arg in ("-input"): - i = i + 1 - input = argv[i] - elif arg in ("-file"): - i = i + 1 - file_name = argv[i] - elif arg in ("-encoding"): - i = i + 1 - encoding = argv[i] - else: - print("unknown") - i = i + 1 - - if (input == None and file_name == None): - sb = "" - ch = getChar() - while (ch != ''): - sb = sb + ch - ch = getChar() - input = sb - str = InputStream(input); - elif (input != None): - str = InputStream(input); - elif (file_name != None): - str = FileStream(file_name, encoding); - - str = CaseChangingStream(str, "" == "Upper") - - lexer = (str); - lexer.removeErrorListeners() - l_listener = MyErrorListener() - lexer.addErrorListener(l_listener) - # lexer.strictMode = false - tokens = CommonTokenStream(lexer) - parser = (tokens) - parser.removeErrorListeners() - p_listener = MyErrorListener() - parser.addErrorListener(p_listener) - if (show_tokens): - i = 0 - while True: - ro_token = lexer.nextToken() - token = ro_token - # token.TokenIndex = i - i = i + 1 - print(token) - if (token.type == -1): - break - lexer.reset() - start_time = datetime.now() - tree = parser.() - end_time = datetime.now() - diff = end_time - start_time - diff_time = diff.total_seconds() - print(f'Time: {diff_time}', file=sys.stderr); - if p_listener.num_errors > 0 or l_listener.num_errors > 0: - # Listener will have already printed the error(s) to stdout. - print('Parse failed.', file=sys.stderr); - sys.exit(1) - else: - print('Parse succeeded.', file=sys.stderr); - if (show_tree): - print(tree.toStringTree(recog=parser)) - sys.exit(0) - -if __name__ == '__main__': - main(sys.argv) diff --git a/_scripts/templates/Python3/Test.py b/_scripts/templates/Python3/Test.py new file mode 100644 index 0000000000..0afd7a5ce9 --- /dev/null +++ b/_scripts/templates/Python3/Test.py @@ -0,0 +1,196 @@ +# Generated from trgen + +import sys +from antlr4 import * +from antlr4.error.ErrorListener import ErrorListener +from readchar import readchar +from import ; +from import ; +from datetime import datetime + +def getChar(): + xx = readchar() + if (xx == 0): + return ''; + return xx + +class MyErrorListener(ErrorListener): + + def __init__(self, q, t, o): + super().__init__() + self.had_error = False + self.quiet = q + self.tee = t; + self.output = o + + def syntaxError(self, recognizer, offendingSymbol, line, column, msg, e): + self.had_error = True + if ( self.tee ): + self.output.write(f"line {line}:{column} {msg}\n"); + if (not self.quiet): + print(f"line {line}:{column} {msg}", file=sys.stderr); + +tee = False +show_tokens = False +show_tree = False +show_trace = False +encoding = "utf-8" +error_code = 0 +string_instance = 0 +prefix = "" +quiet = False + +def main(argv): + global tee + global show_tokens + global show_tree + global show_trace + global encoding + global prefix + global quiet + global error_code + + inputs = [] + is_fns = [] + prefix = "" + i = 1 + while i \< len(argv): + arg = argv[i] + if arg in ("-tokens"): + show_tokens = True + elif arg in ("-tree"): + show_tree = True + elif arg in ("-prefix"): + i = i + 1 + prefix = argv[i] + " " + elif arg in ("-input"): + i = i + 1 + inputs.append(argv[i]) + is_fns.append(false) + elif arg in ("-encoding"): + i = i + 1 + encoding = argv[i] + elif arg in ("-tee"): + tee = True + elif arg in ("-x"): + while f := sys.stdin.readline(): + f = f.strip() + inputs.append(f) + is_fns.append(True) + elif arg in ("-q"): + quiet = True + elif arg in ("-trace"): + show_trace = True + else: + inputs.append(argv[i]) + is_fns.append(True) + i = i + 1 + if len(inputs) == 0: + ParseStdin() + else: + start_time = datetime.now() + for f in range(0, len(inputs)): + if is_fns[f]: + ParseFilename(inputs[f], f) + else: + ParseString(inputs[f], f) + end_time = datetime.now() + diff = end_time - start_time + diff_time = diff.total_seconds() + if (not quiet): + print(f'Total Time: {diff_time}', file=sys.stderr); + sys.exit(error_code) + +def ParseStdin(): + sb = "" + ch = getChar() + while (ch != ''): + sb = sb + ch + ch = getChar() + input = sb + str = InputStream(input); + DoParse(str, 'stdin', 0) + +def ParseString(input, row_number): + str = InputStream(input) + DoParse(str, 'string' + string_instance, row_number) + string_instance = string_instance + 1 + +def ParseFilename(input, row_number): + global encoding + str = FileStream(input, encoding) + DoParse(str, input, row_number) + +def DoParse(str, input_name, row_number): + global tee + global show_tokens + global show_tree + global show_trace + global encoding + global prefix + global quiet + global error_code + + lexer = (str) + lexer.removeErrorListeners() + if (tee): + output = open(input_name + ".errors", "w") + else: + output = sys.stderr + listener_lexer = MyErrorListener(quiet, tee, output) + lexer.addErrorListener(listener_lexer) + # lexer.strictMode = false + tokens = CommonTokenStream(lexer) + parser = (tokens) + parser.removeErrorListeners() + listener_parser = MyErrorListener(quiet, tee, output) + parser.addErrorListener(listener_parser) + if (show_tokens): + i = 0 + while True: + ro_token = lexer.nextToken() + token = ro_token + # token.TokenIndex = i + i = i + 1 + print(token, file=sys.stderr) + if (token.type == -1): + break + lexer.reset() + if (show_trace) : + parser.setTrace(False) + ParserATNSimulator.trace_atn_sim = True + PredictionContext._trace_atn_sim = True + start_time = datetime.now() + tree = parser.() + end_time = datetime.now() + diff = end_time - start_time + diff_time = diff.total_seconds() + result = '' + if listener_parser.had_error or listener_lexer.had_error: + result = 'fail' + error_code = 1 + else: + result = 'success' + if (show_tree): + if (tee): + f = open(input_name + '.tree', 'w', encoding='utf-8') + f.write(tree.toStringTree(recog=parser)) + f.close() + else: + print(tree.toStringTree(recog=parser), file=sys.stderr) + if (not quiet): + sys.stderr.write(prefix) + sys.stderr.write('Python3 ') + sys.stderr.write(f'{row_number}') + sys.stderr.write(' ') + sys.stderr.write(input_name) + sys.stderr.write(' ') + sys.stderr.write(result) + sys.stderr.write(' ') + sys.stderr.write(f'{diff_time}') + sys.stderr.write('\n') + if (tee): + output.close() + +if __name__ == '__main__': + main(sys.argv) diff --git a/_scripts/templates/Python3/build.ps1 b/_scripts/templates/Python3/build.ps1 new file mode 100644 index 0000000000..3d4d8cc1aa --- /dev/null +++ b/_scripts/templates/Python3/build.ps1 @@ -0,0 +1,14 @@ +# Generated from trgen +if (Test-Path -Path transformGrammar.py -PathType Leaf) { + $(& python3 transformGrammar.py ) 2>&1 | Write-Host +} + + -encoding -Dlanguage=Python3 } > ; $compile_exit_code = $LASTEXITCODE) | Write-Host +if($compile_exit_code -ne 0){ + exit $compile_exit_code +\} +}> + +$(& pip install -r requirements.txt ; $compile_exit_code = $LASTEXITCODE ) | Write-Host +exit $compile_exit_code diff --git a/_scripts/templates/Python3/build.sh b/_scripts/templates/Python3/build.sh new file mode 100644 index 0000000000..bfd66ab221 --- /dev/null +++ b/_scripts/templates/Python3/build.sh @@ -0,0 +1,12 @@ +# Generated from trgen +set -e + +if [ -f transformGrammar.py ]; then python3 transformGrammar.py ; fi + +" -encoding -Dlanguage=Python3 } > +}> + +pip install -r requirements.txt + +exit 0 diff --git a/_scripts/templates/Python3/clean.ps1 b/_scripts/templates/Python3/clean.ps1 new file mode 100644 index 0000000000..df77f42671 --- /dev/null +++ b/_scripts/templates/Python3/clean.ps1 @@ -0,0 +1,16 @@ +# Generated from trgen +$(& Remove-Item *.interp -Recurse -Force ) 2>&1 | Out-Null +$files = New-Object System.Collections.Generic.List[string] +" -depend -encoding -Dlanguage=Dart } > +foreach ($s in $f) { + $j = $s.Split(" ")[0] + $files.Add($j) +\} +foreach ($f in $files) +{ + $(& Remove-Item $f -Force ) 2>&1 | Out-Null +\} +} > +$(& Remove-Item __pycache__ -Recurse -Force ) 2>&1 | Out-Null +exit 0 diff --git a/_scripts/templates/Python3/clean.sh b/_scripts/templates/Python3/clean.sh new file mode 100644 index 0000000000..15a1b42f35 --- /dev/null +++ b/_scripts/templates/Python3/clean.sh @@ -0,0 +1,12 @@ +# Generated from trgen +rm -f *.interp +files=() +" -depend -encoding -Dlanguage=Python3 } > | awk '{print $1\}' | grep -v ':'` ) +} > +for i in ${files[*]} +do + rm -f $i +done +rm -rf __pycache__ +exit 0 diff --git a/_scripts/templates/Python3/makefile b/_scripts/templates/Python3/makefile index 5e5c3d783f..4794ab1ec8 100644 --- a/_scripts/templates/Python3/makefile +++ b/_scripts/templates/Python3/makefile @@ -1,26 +1,10 @@ -# Template generated code from trgen -JAR = -CLASSPATH = $(JAR)\;:. -.SUFFIXES: .g4 .py -ANTLRGRAMMARS ?= $(wildcard *.g4) -GENERATED = }> -SOURCES = $(GENERATED) \ - Program.py -default: setup classes -setup: - if [ -f transformGrammar.py ]; then python3 transformGrammar.py ; fi -classes: $(SOURCES) - pip install -r requirements.txt +# Generated from trgen +default: + bash build.sh clean: - rm -f *.tokens *.interp - rm -f *.interp - rm -f *.tokens - rm -f $(GENERATED) - rm -rf __pycache__ + bash clean.sh run: - trwdog python3 Program.py $(RUNARGS) - : - java -jar $(JAR) -encoding -Dlanguage=Python3 } > $\< -} > -test: + @trwdog python3 Test.py $(RUNARGS) +FORCE: ; +test: FORCE bash test.sh diff --git a/_scripts/templates/Python3/test.ps1 b/_scripts/templates/Python3/test.ps1 new file mode 100644 index 0000000000..011cead979 --- /dev/null +++ b/_scripts/templates/Python3/test.ps1 @@ -0,0 +1,149 @@ +# Generated from trgen + +$TestDirectory = "../../" +Write-Host "Test cases here: $TestDirectory" + +# People often specify a test file directory, but sometimes no +# tests are provided. Git won't check in an empty directory. +# Test if the test file directory does not exist, or it is just +# an empty directory. +if (!(Test-Path -Path "$TestDirectory")) { + Write-Host "No test cases provided." + exit 0 +} elseif (!(Test-Path "$TestDirectory/*")) { + Write-Host "No test cases provided." + exit 0 +} + +# Get a list of test files from the test directory. Do not include any +# .errors or .tree files. Pay close attention to remove only file names +# that end with the suffix .errors or .tree. +if (Test-Path -Path "tests.txt" -PathType Leaf) { + Remove-Item "tests.txt" +} +$files = New-Object System.Collections.Generic.List[string] +foreach ($item in Get-ChildItem $TestDirectory -Recurse) { + $file = $item.fullname + $ext = $item.Extension + if (Test-Path $file -PathType Container) { + continue + } elseif ($ext -eq ".errors") { + continue + } elseif ($ext -eq ".tree") { + continue + } else { + $(& triconv -f utf-8 $file ; $last = $LASTEXITCODE ) | Out-Null + if ($last -ne 0) + { + continue + } + $files.Add($item) + Write-Host "Test case: $item" + } +} +foreach ($file in $files) { + Add-Content "tests.txt" $file +} +if (-not(Test-Path -Path "tests.txt" -PathType Leaf)) { + Write-Host "No test cases provided." + exit 0 +} + +# Parse all input files. +get-content "tests.txt" | trwdog python3 Test.py -q -x -tee -tree *> parse.txt +$status = $LASTEXITCODE + +# trwdog returns 255 if it cannot spawn the process. This could happen +# if the environment for running the program does not exist, or the +# program did not build. +if ( $status -eq 255 ) { + Write-Host "Test failed." + Get-Content $file | Write-Host + exit 1 +} + +# Any parse errors will be put in .errors files. But, if there's any +# output from the program in stdout or stderr, it's all bad news. +$size = (Get-Item -Path "parse.txt").Length +if ( $size -eq 0 ) { +} else { + Write-Host "Test failed." + Get-Content "parse.txt" | Write-Host + exit 1 +} + +# Check if any .errors/.tree files have changed. That's not good. +$message = git diff --exit-code --name-only $TestDirectory +$updated = $LASTEXITCODE +Write-Host $message + +# Check if any untracked .errors files are not empty. +$new_errors_txt = New-Object System.Collections.Generic.List[string] +$new_errors2_txt = git ls-files --exclude-standard -o --ignored $TestDirectory +$new_errors = $LASTEXITCODE + +# Gather up all untracked .errors file output. These are new errors +# and must be reported as a parse fail. +if ( ! [String]::IsNullOrWhiteSpace($new_errors2_txt) ) { + $new_errors3_txt = $new_errors2_txt.Split("\n\r\t ") +} else { + $new_errors3_txt = [System.Collections.Arraylist]@() +} +if (Test-Path -Path "new_errors.txt" -PathType Leaf) { + Remove-Item "new_errors.txt" +} +New-Item -Path . -Name "new_errors.txt" -ItemType "file" -Value "" | Out-Null +foreach ($s in $new_errors3_txt) { + if ( [String]::IsNullOrWhiteSpace($s) ) { + continue + } + $ext = $item.Extension + if (! $s.EndsWith(".errors")) { + continue + } + $file = $s + $size = (Get-Item -Path $file).Length + if ( $size -eq 0 ) { + } else { + $new_errors_txt.Add($item) + Add-Content -Path "new_errors.txt" -Value "$item" + ((Get-Content $file) -join "`n") + "`n" | Add-Content -Path "new_errors.txt" + } +} + +# If "git diff" reported an exit code of 129, it is because +# the directory containing the grammar is not in a repo. In this +# case, assume parse error code as the defacto result. +if ( $updated -eq 129 ) { + Write-Host "Grammar outside a git repository. Assuming parse exit code." + if ( $status -eq 0 ) { + Write-Host "Test succeeded." + } else { + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + } + $err = $status + exit 1 +} + +# "Git diff" reported a difference. Redo the "git diff" to print out all +# the differences. Also, output any untracked, non-zero length .errors files. +if ( $updated -eq 1 ) { + Write-Host "Difference in output." + git diff . | Write-Host + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + exit 1 +} + +# If there's non-zero length .errors flies that are new, report them +# as errors in the parse. +if ( $new_errors_txt.Count -gt 0 ) { + Write-Host "New errors in output." + Get-Content "new_errors.txt" | Write-Host + Write-Host "Test failed." + exit 1 +} + +Write-Host "Test succeeded." +exit 0 diff --git a/_scripts/templates/Python3/test.sh b/_scripts/templates/Python3/test.sh index 3973663fef..c10e19183e 100644 --- a/_scripts/templates/Python3/test.sh +++ b/_scripts/templates/Python3/test.sh @@ -1,58 +1,149 @@ -# Template generated code from trgen -err=0 +# Generated from trgen + +# People often specify a test file directory, but sometimes no +# tests are provided. Git won't check in an empty directory. +# Test if the test file directory does not exist, or it is just +# an empty directory. +if [ ! -d ../ ] +then + echo "No test cases provided." + exit 0 +elif [ ! "$(ls -A ../)" ] +then + echo "No test cases provided." + exit 0 +fi + SAVEIFS=$IFS IFS=$(echo -en "\n\b") -parse_out_file=`mktemp --tmpdir=. parse_out.XXXXXXXXXX` -for g in `find ../ -type f | grep -v '.errors$' | grep -v '.tree$'` + +# Get a list of test files from the test directory. Do not include any +# .errors or .tree files. Pay close attention to remove only file names +# that end with the suffix .errors or .tree. +files2=`find ../ -type f | grep -v '.errors$' | grep -v '.tree$'` +files=() +for f in $files2 do - file="$g" - x1="${g##*.}" - if [ "$x1" != "errors" ] - then - echo "$file" - trwdog python3 Program.py -file "$file" -tree > "$parse_out_file" - status="$?" - if [ -f "$file".errors ] + triconv -f utf-8 $f > /dev/null 2>&1 + if [ "$?" = "0" ] then - if [ "$status" = "0" ] - then - echo Expected parse fail. - err=1 - break - else - echo Expected. - diff \<(tr -d "\r\n" \< "$file".errors) \<(tr -d "\r\n" \< "$parse_out_file") - status="$?" - if [ "$status" = "0" ] - then - echo Parse errors match succeeded. - else - echo Expected parse errors match. - err=1 - break - fi - fi - else # No .errors file - if [ "$status" != "0" ] - then - err=1 - break - fi - if [ -f "$file".tree ] - then - diff \<(tr -d "\r\n" \< "$file".tree) \<(tr -d "\r\n" \< "$parse_out_file") - status="$?" - if [ "$status" = "0" ] + files+=( $f ) + fi +done + +# Parse all input files. +echo "${files[*]}" | trwdog python3 Test.py -q -x -tee -tree > parse.txt 2>&1 +status=$? + +# trwdog returns 255 if it cannot spawn the process. This could happen +# if the environment for running the program does not exist, or the +# program did not build. +if [ "$status" = "255" ] +then + echo "Test failed." + cat parse.txt + exit 1 +fi + +# Any parse errors will be put in .errors files. But, if there's any +# output from the program in stdout or stderr, it's all bad news. +if [ -s parse.txt ] +then + echo "Test failed." + cat parse.txt + exit 1 +fi + +# rm -rf `find ../ -type f -name '*.errors' -o -name '*.tree' -size 0` + +# For Unix environments, convert the newline in the .errors and .trees +# to Unix style. +unameOut="$(uname -s)" +case "${unameOut}" in + Linux*) machine=Linux;; + Darwin*) machine=Mac;; + CYGWIN*) machine=Cygwin;; + MINGW*) machine=MinGw;; + *) machine="UNKNOWN:${unameOut}" +esac +if [[ "$machine" == "MinGw" || "$machine" == "Msys" || "$machine" == "Cygwin" || "#machine" == "Linux" ]] +then + gen=`find ../ -type f -name '*.errors' -o -name '*.tree'` + if [ "$gen" != "" ] + then + dos2unix $gen + fi +fi + +old=`pwd` +cd ../ + +# Check if any files in the test files directory have changed. +git diff --exit-code --name-only . > $old/updated.txt 2>&1 +updated=$? + +# Check if any untracked .errors files. +git ls-files --exclude-standard -o --ignored > $old/new_errors2.txt 2>&1 +new_errors=$? + +# Gather up all untracked .errors file output. These are new errors +# and must be reported as a parse fail. +rm -f $old/new_errors.txt +touch $old/new_errors.txt +for f in `cat $old/new_errors2.txt` +do + ext=${f##*.} + ext=".$ext" + if [ "$ext" = ".errors" ] + then + if [ -s $f ] then - echo Parse tree match succeeded. - else - echo Expected parse tree match. - err=1 - break + echo $f >> $old/new_errors.txt + cat $f >> $old/new_errors.txt fi - fi fi - fi done -rm "$parse_out_file" -exit $err + +# If "git diff" reported an exit code of 129, it is because +# the directory containing the grammar is not in a repo. In this +# case, assume parse error code as the defacto result. +if [ "$updated" = "129" ] +then + echo "Grammar outside a git repository. Assuming parse exit code." + if [ "$status" = 0 ] + then + echo "Test succeeded." + else + cat $old/new_errors.txt + echo "Test failed." + fi + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit $status +fi + +# "Git diff" reported a difference. Redo the "git diff" to print out all +# the differences. Also, output any untracked, non-zero length .errors files. +if [ "$updated" = "1" ] +then + echo "Difference in output." + git diff . + cat $old/new_errors.txt + echo "Test failed." + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit 1 +fi + +# If there's non-zero length .errors flies that are new, report them +# as errors in the parse. +if [ -s $old/new_errors.txt ] +then + echo "New errors in output." + cat $old/new_errors.txt + echo "Test failed." + rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt + exit 1 +fi + +echo "Test succeeded." +rm -f $old/updated.txt $old/new_errors2.txt $old/new_errors.txt +exit 0 diff --git a/_scripts/templates/Python3/tester.psm1 b/_scripts/templates/Python3/tester.psm1 deleted file mode 100644 index 1bdd29d1be..0000000000 --- a/_scripts/templates/Python3/tester.psm1 +++ /dev/null @@ -1,63 +0,0 @@ -# Template generated code from trgen -function Build-Grammar { - -Dlanguage=Python3 } > - if($LASTEXITCODE -ne 0){ - return @{ - Message = $g - Success = $false - \} - \} -}> - $msg = pip install -r requirements.txt - return @{ - Message = $msg - Success = $LASTEXITCODE -eq 0 - } -} - -function Test-Case { - param ( - $InputFile, - $TokenFile, - $TreeFile, - $ErrorFile - ) - # Save input and output character encodings and switch to UTF-8. - $oldInputEncoding = [console]::InputEncoding - $oldOutputEncoding = [console]::OutputEncoding - $OutputEncoding = [console]::InputEncoding = [console]::OutputEncoding = New-Object System.Text.UTF8Encoding - - $parseOutFile = $InputFile + ".out" - $o = trwdog python3 -X utf8 Program.py -file $InputFile -tree | Out-File -LiteralPath "$parseOutFile" -Encoding UTF8 - $parseOk = $LASTEXITCODE -eq 0 - $treeMatch = $true - if ($errorFile) { - # If we expected errors, then a failed parse was a successful test. - if (!$parseOk) { - Write-Host "Expected." - # Confirm that the errors we received are the ones we expected. - $expectedData = (Get-Content $errorFile -Encoding UTF8) -join "" -replace "\r\n","" - $actualData = (Get-Content $parseOutFile -Encoding UTF8) -join "" -replace "\r\n","" - $parseOk = ($actualData -eq $expectedData) - if ($parseOk) { - Write-Host "Error list match succeeded." - } else { - Write-Host "Expected error list match." -ForegroundColor Red - } - } - } else { - if ($parseOk -and (Test-Path $TreeFile)) { - # Confirm that the parse tree we received is the one we expected. - $expectedData = Get-Content $TreeFile -Encoding UTF8 - $actualData = Get-Content $parseOutFile -Encoding UTF8 - $treeMatch = ($actualData -eq $expectedData) - } - } - # Restore input and output character encodings. - [console]::InputEncoding = $oldInputEncoding - [console]::OutputEncoding = $oldOutputEncoding - - Remove-Item $parseOutFile - return $parseOk, $treeMatch -} \ No newline at end of file diff --git a/_scripts/templates/files b/_scripts/templates/files index 71080435c4..208c8b49ff 100644 --- a/_scripts/templates/files +++ b/_scripts/templates/files @@ -1,12 +1,19 @@ +./Antlr4cs/build.ps1 +./Antlr4cs/build.sh ./Antlr4cs/CaseChangingCharStream.cs +./Antlr4cs/clean.ps1 +./Antlr4cs/clean.sh +./Antlr4cs/ErrorListener.cs ./Antlr4cs/makefile -./Antlr4cs/Program.cs +./Antlr4cs/Test.cs ./Antlr4cs/Test.csproj +./Antlr4cs/test.ps1 ./Antlr4cs/test.sh -./Antlr4cs/tester.psm1 ./Arithmetic.g4 -./Cpp/CaseChangingCharStream.cpp -./Cpp/CaseChangingCharStream.h +./Cpp/build.ps1 +./Cpp/build.sh +./Cpp/clean.ps1 +./Cpp/clean.sh ./Cpp/cmake/antlr4-generator.cmake.in ./Cpp/cmake/antlr4-runtime.cmake.in ./Cpp/cmake/Antlr4Package.md @@ -17,50 +24,75 @@ ./Cpp/ErrorListener.cpp ./Cpp/ErrorListener.h ./Cpp/makefile -./Cpp/Program.cpp ./Cpp/readme.md +./Cpp/Test.cpp +./Cpp/test.ps1 ./Cpp/test.sh -./CSharp/CaseChangingCharStream.cs +./CSharp/build.ps1 +./CSharp/build.sh +./CSharp/clean.ps1 +./CSharp/clean.sh ./CSharp/Encodings.cs ./CSharp/ErrorListener.cs ./CSharp/makefile -./CSharp/Program.cs +./CSharp/Test.cs ./CSharp/Test.csproj +./CSharp/test.ps1 ./CSharp/test.sh -./CSharp/tester.psm1 ./Dart/analysis_options.yaml -./Dart/cli.dart +./Dart/build.ps1 +./Dart/build.sh +./Dart/clean.ps1 +./Dart/clean.sh ./Dart/makefile +./Dart/MyErrorListener.dart ./Dart/pubspec.yaml +./Dart/Test.dart +./Dart/test.ps1 ./Dart/test.sh -./Dart/tester.psm1 ./files -./Go/antlr_resource/case_changing_stream.go +./Go/build.ps1 +./Go/build.sh +./Go/clean.ps1 +./Go/clean.sh ./Go/go.mod ./Go/makefile ./Go/Test.go +./Go/test.ps1 ./Go/test.sh -./Go/tester.psm1 -./Java/CaseChangingCharStream.java +./Java/build.ps1 +./Java/build.sh +./Java/clean.ps1 +./Java/clean.sh ./Java/ErrorListener.java ./Java/makefile -./Java/Program.java +./Java/Test.java +./Java/test.ps1 ./Java/test.sh -./Java/tester.psm1 -./JavaScript/CaseChangingStream.js -./JavaScript/index.js +./JavaScript/build.ps1 +./JavaScript/build.sh +./JavaScript/clean.ps1 +./JavaScript/clean.sh ./JavaScript/makefile ./JavaScript/package.json +./JavaScript/Test.js +./JavaScript/test.ps1 ./JavaScript/test.sh -./JavaScript/tester.psm1 +./PHP/build.ps1 +./PHP/build.sh +./PHP/clean.ps1 +./PHP/clean.sh ./PHP/composer.json ./PHP/makefile ./PHP/Test.php +./PHP/test.ps1 ./PHP/test.sh -./PHP/tester.psm1 -./Python3/CaseChangingStream.py +./Python3/build.ps1 +./Python3/build.sh +./Python3/clean.ps1 +./Python3/clean.sh ./Python3/makefile -./Python3/Program.py ./Python3/requirements.txt +./Python3/test.ps1 +./Python3/Test.py ./Python3/test.sh -./Python3/tester.psm1 diff --git a/_scripts/test.ps1 b/_scripts/test.ps1 index 15768acf1f..4c1917542a 100644 --- a/_scripts/test.ps1 +++ b/_scripts/test.ps1 @@ -56,10 +56,6 @@ function Test-Grammar { Set-Location $Directory $failStage = [FailStage]::Success - $hasTest = Test-Path "./examples" - if ($hasTest) { - $testDir = Resolve-Path "./examples" - } $success = $true $start = Get-Date @@ -71,8 +67,9 @@ function Test-Grammar { $failStage = [FailStage]::CodeGeneration Write-Host "trgen failed" -ForegroundColor Red } - if (Test-Path 'Generated') { - Set-Location 'Generated' + $targetgenerated = "Generated-$Target" + if (Test-Path $targetgenerated) { + Set-Location $targetgenerated } else { $failStage = [FailStage]::CodeGeneration @@ -94,19 +91,16 @@ function Test-Grammar { # build # see _scripts/templates/*/tester.psm1 - Import-Module ./tester.psm1 - - $buildResult = Build-Grammar + ./build.ps1 + $buildResult = $LASTEXITCODE - if (!$buildResult.Success) { + if ($buildResult -ne 0) { $failStage = [FailStage]::Compile Write-Host "Build failed" -ForegroundColor Red - Write-Host $buildResult.Message } Write-Host "Build completed, time: $((Get-Date) - $start)" -ForegroundColor Yellow if ($failStage -ne [FailStage]::Success) { - Remove-Module tester Set-Location $cwd return @{ Success = $false @@ -117,18 +111,16 @@ function Test-Grammar { # test $start2 = Get-Date - Write-Host "Testing" - $failedList = @() - if ($hasTest) { - $failedList = Test-GrammarTestCases -TestDirectory $testDir - if ($failedList.Length -gt 0) { - $success = $false - $failStage = [FailStage]::Test - } + Write-Host "--- Testing files ---" + ./test.ps1 + $passed = $LASTEXITCODE -eq 0 + + if (! $passed) { + $success = $false + $failStage = [FailStage]::Test } Write-Host "Test completed, time: $((Get-Date) - $start2)" -ForegroundColor Yellow - Remove-Module tester Set-Location $cwd return @{ Success = $success @@ -137,59 +129,6 @@ function Test-Grammar { } } -function Test-GrammarTestCases { - param ( - $TestDirectory - ) - $failedList = @() - Write-Host "Test cases here: $TestDirectory" - foreach ($item in Get-ChildItem $TestDirectory -Recurse) { - - Write-Host "Test case: $item" - - $case = $item.fullname - $ext = $item.Extension - if (($ext -eq ".errors") -or ($ext -eq ".tree")) { - continue - } - if (Test-Path $case -PathType Container) { - continue - } - - $errorFile = "$case.errors" - $shouldFail = Test-Path $errorFile - if (!$shouldFail) { - $errorFile = "" - } - $treeFile = "$case.tree" - Write-Host "--- Testing file $item ---" - $parseOk, $treeMatch = Test-Case -InputFile $case -ErrorFile $errorFile -TreeFile $treeFile - - if (!$parseOk) { - if ($shouldFail) { - Write-Host "Test case should return error" - Write-Host "$TestDirectory test $case failed" -ForegroundColor Red - $failedList += $case - continue - } - else { - Write-Host "$TestDirectory test $case failed" -ForegroundColor Red - $failedList += $case - continue - } - } - if (Test-Path $treeFile) { - if ($treeMatch) { - Write-Host "Parse tree match succeeded" - } else { - Write-Host "$TestDirectory test $case parse tree match failed" -ForegroundColor Red - $failedList += $case - } - } - } - return $failedList -} - function Get-Grammars { param ( $Directory = "" diff --git a/antlr/antlr2/CSharp/LexerAdaptor.cs b/antlr/antlr2/CSharp/LexerAdaptor.cs index 56b5b58377..4cf13e394f 100644 --- a/antlr/antlr2/CSharp/LexerAdaptor.cs +++ b/antlr/antlr2/CSharp/LexerAdaptor.cs @@ -164,6 +164,6 @@ public override IToken Emit() public override void Reset() { CurrentRuleType = TokenConstants.InvalidType; - base.Reset(); + base.Reset(); } } diff --git a/antlr/antlr4/examples/three.g4.errors b/antlr/antlr4/examples/three.g4.errors index 5fa72aba80..b7b4ce2785 100644 --- a/antlr/antlr4/examples/three.g4.errors +++ b/antlr/antlr4/examples/three.g4.errors @@ -1,2 +1 @@ line 4:0 mismatched input '' expecting {'lexer', 'parser', 'grammar'} - diff --git a/cpp/examples/union_non_inheritance.cpp.errors b/cpp/examples/union_non_inheritance.cpp.errors index 9553ca257f..a9b7b9ab44 100644 --- a/cpp/examples/union_non_inheritance.cpp.errors +++ b/cpp/examples/union_non_inheritance.cpp.errors @@ -1,3 +1,3 @@ line 4:8 no viable alternative at input 'unionB:' line 6:1 missing '{' at ';' -line 6:2 missing '}' at '' \ No newline at end of file +line 6:2 missing '}' at '' diff --git a/cto/examples/invalid/datetime.cto.errors b/cto/examples/invalid/datetime.cto.errors index 631821dd9d..078b9abca6 100644 --- a/cto/examples/invalid/datetime.cto.errors +++ b/cto/examples/invalid/datetime.cto.errors @@ -1 +1 @@ -line 5:28 mismatched input '"2000-01-31T23:59:59.001+05:41"' expecting DATE_TIME_LITERAL \ No newline at end of file +line 5:28 mismatched input '"2000-01-31T23:59:59.001+05:41"' expecting DATE_TIME_LITERAL diff --git a/cto/examples/invalid/namespace.cto.errors b/cto/examples/invalid/namespace.cto.errors index f4c95ba605..052abf8aa2 100644 --- a/cto/examples/invalid/namespace.cto.errors +++ b/cto/examples/invalid/namespace.cto.errors @@ -1 +1 @@ -line 3:0 mismatched input 'enum' expecting 'namespace' \ No newline at end of file +line 3:0 mismatched input 'enum' expecting 'namespace' diff --git a/cto/examples/invalid/reference.cto.errors b/cto/examples/invalid/reference.cto.errors index 21e28f675b..80c2f0bf79 100644 --- a/cto/examples/invalid/reference.cto.errors +++ b/cto/examples/invalid/reference.cto.errors @@ -1 +1 @@ -line 5:23 extraneous input 'default' expecting {'}', ';', '--> ', 'o '} \ No newline at end of file +line 5:23 extraneous input 'default' expecting {'}', ';', '--> ', 'o '} diff --git a/dice/examples/invalid_dice_format.txt.errors b/dice/examples/invalid_dice_format.txt.errors index 31f211bffe..7e5dabb13b 100644 --- a/dice/examples/invalid_dice_format.txt.errors +++ b/dice/examples/invalid_dice_format.txt.errors @@ -1,2 +1,2 @@ line 1:2 token recognition error at: 'z' -line 1:3 no viable alternative at input '5d' \ No newline at end of file +line 1:3 no viable alternative at input '5d' diff --git a/dice/examples/invalid_dice_missing_side.txt.errors b/dice/examples/invalid_dice_missing_side.txt.errors index 958b456e7c..ea795c5857 100644 --- a/dice/examples/invalid_dice_missing_side.txt.errors +++ b/dice/examples/invalid_dice_missing_side.txt.errors @@ -1 +1 @@ -line 1:2 no viable alternative at input '1d' \ No newline at end of file +line 1:2 no viable alternative at input '1d' diff --git a/dice/examples/invalid_dice_suffix.txt.errors b/dice/examples/invalid_dice_suffix.txt.errors index 1e26d78b93..b775568f9e 100644 --- a/dice/examples/invalid_dice_suffix.txt.errors +++ b/dice/examples/invalid_dice_suffix.txt.errors @@ -1 +1 @@ -line 1:3 token recognition error at: 'y' \ No newline at end of file +line 1:3 token recognition error at: 'y' diff --git a/dice/examples/invalid_dice_whitespace_end.txt.errors b/dice/examples/invalid_dice_whitespace_end.txt.errors index 8cff9c9962..789e2059fb 100644 --- a/dice/examples/invalid_dice_whitespace_end.txt.errors +++ b/dice/examples/invalid_dice_whitespace_end.txt.errors @@ -1 +1 @@ -line 1:2 token recognition error at: ' ' \ No newline at end of file +line 1:2 token recognition error at: ' ' diff --git a/dice/examples/invalid_dice_whitespace_start.txt.errors b/dice/examples/invalid_dice_whitespace_start.txt.errors index 283df9e09c..21828db1d7 100644 --- a/dice/examples/invalid_dice_whitespace_start.txt.errors +++ b/dice/examples/invalid_dice_whitespace_start.txt.errors @@ -1 +1 @@ -line 1:1 token recognition error at: ' ' \ No newline at end of file +line 1:1 token recognition error at: ' ' diff --git a/java/java8/CodepointRangeScanner.java b/java/java8/Java/CodepointRangeScanner.java similarity index 100% rename from java/java8/CodepointRangeScanner.java rename to java/java8/Java/CodepointRangeScanner.java diff --git a/java/java8/Test.java b/java/java8/Java/Test.java.alternative similarity index 100% rename from java/java8/Test.java rename to java/java8/Java/Test.java.alternative diff --git a/java/java9/Test.java b/java/java9/Java/Test.java.alternative similarity index 100% rename from java/java9/Test.java rename to java/java9/Java/Test.java.alternative diff --git a/kirikiri-tjs/build.sh b/kirikiri-tjs/Java/build.sh.alternative similarity index 100% rename from kirikiri-tjs/build.sh rename to kirikiri-tjs/Java/build.sh.alternative diff --git a/kirikiri-tjs/clean.sh b/kirikiri-tjs/Java/clean.sh.alternative similarity index 100% rename from kirikiri-tjs/clean.sh rename to kirikiri-tjs/Java/clean.sh.alternative diff --git a/logo/logo/examples/logo_feature_butfail_2.txt.errors b/logo/logo/examples/logo_feature_butfail_2.txt.errors index fd301e3c9c..3d301c8047 100644 --- a/logo/logo/examples/logo_feature_butfail_2.txt.errors +++ b/logo/logo/examples/logo_feature_butfail_2.txt.errors @@ -1,3 +1,2 @@ line 3:12 token recognition error at: '�' line 3:13 token recognition error at: '�' - diff --git "a/molecule/examples/NiC2O4 \302\267 2H2O.txt" b/molecule/examples/NiC2O4-2H2O.txt similarity index 100% rename from "molecule/examples/NiC2O4 \302\267 2H2O.txt" rename to molecule/examples/NiC2O4-2H2O.txt diff --git a/rego/examples/e00002.stmt.errors b/rego/examples/e00002.stmt.errors index 3a5c5015d9..457bef9d74 100644 --- a/rego/examples/e00002.stmt.errors +++ b/rego/examples/e00002.stmt.errors @@ -1 +1 @@ -line 1:0 token recognition error at: '"foo' \ No newline at end of file +line 1:0 token recognition error at: '"foo' diff --git a/rego/examples/e00003.stmt.errors b/rego/examples/e00003.stmt.errors index d7b9a334bb..9a7d3b1ccf 100644 --- a/rego/examples/e00003.stmt.errors +++ b/rego/examples/e00003.stmt.errors @@ -1 +1 @@ -line 1:0 token recognition error at: '`foo' \ No newline at end of file +line 1:0 token recognition error at: '`foo' diff --git a/rego/examples/e00004.stmt.errors b/rego/examples/e00004.stmt.errors index b0f01744bb..e5cba26623 100644 --- a/rego/examples/e00004.stmt.errors +++ b/rego/examples/e00004.stmt.errors @@ -1,2 +1,2 @@ line 1:0 token recognition error at: ''' -line 1:9 token recognition error at: ''' \ No newline at end of file +line 1:9 token recognition error at: ''' diff --git a/rego/examples/e00011.stmt.errors b/rego/examples/e00011.stmt.errors index 2737cde2a8..686f45a2ad 100644 --- a/rego/examples/e00011.stmt.errors +++ b/rego/examples/e00011.stmt.errors @@ -1 +1 @@ -line 1:3 no viable alternative at input 'not' \ No newline at end of file +line 1:3 no viable alternative at input 'not' diff --git a/rego/examples/e00012.stmt.errors b/rego/examples/e00012.stmt.errors index faf3224c34..512a9dccbe 100644 --- a/rego/examples/e00012.stmt.errors +++ b/rego/examples/e00012.stmt.errors @@ -1 +1 @@ -line 1:7 missing Name at '' \ No newline at end of file +line 1:7 missing Name at '' diff --git a/rego/examples/e00013.stmt.errors b/rego/examples/e00013.stmt.errors index 0f01702d3f..a5293ab3d1 100644 --- a/rego/examples/e00013.stmt.errors +++ b/rego/examples/e00013.stmt.errors @@ -1 +1 @@ -line 1:6 missing Name at '' \ No newline at end of file +line 1:6 missing Name at '' diff --git a/rego/examples/e00014.stmt.errors b/rego/examples/e00014.stmt.errors index 8d122e9268..1d412aae06 100644 --- a/rego/examples/e00014.stmt.errors +++ b/rego/examples/e00014.stmt.errors @@ -1 +1 @@ -line 1:4 no viable alternative at input 'foo.' \ No newline at end of file +line 1:4 no viable alternative at input 'foo.' diff --git a/rego/examples/e00015.stmt.errors b/rego/examples/e00015.stmt.errors index e464006fe3..20ba1b9c5e 100644 --- a/rego/examples/e00015.stmt.errors +++ b/rego/examples/e00015.stmt.errors @@ -1,2 +1,2 @@ line 1:5 no viable alternative at input 'foo[].' -line 1:5 extraneous input '.' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} \ No newline at end of file +line 1:5 extraneous input '.' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} diff --git a/rego/examples/e00016.stmt.errors b/rego/examples/e00016.stmt.errors index 872b38bfc2..2ea4fd1e62 100644 --- a/rego/examples/e00016.stmt.errors +++ b/rego/examples/e00016.stmt.errors @@ -1 +1 @@ -line 1:30 extraneous input '' expecting {'}', ','} \ No newline at end of file +line 1:30 extraneous input '' expecting {'}', ','} diff --git a/rego/examples/e00017.stmt.errors b/rego/examples/e00017.stmt.errors index c497a9feff..82f51d0c39 100644 --- a/rego/examples/e00017.stmt.errors +++ b/rego/examples/e00017.stmt.errors @@ -1 +1 @@ -line 1:25 mismatched input '' expecting {String, Bool, 'null', 'not', 'set(', '[', '{', ArithOperator, UnsignedNumber, Name} \ No newline at end of file +line 1:25 mismatched input '' expecting {String, Bool, 'null', 'not', 'set(', '[', '{', ArithOperator, UnsignedNumber, Name} diff --git a/rego/examples/e00018.stmt.errors b/rego/examples/e00018.stmt.errors index 03ebd0e42a..bb0d5452c0 100644 --- a/rego/examples/e00018.stmt.errors +++ b/rego/examples/e00018.stmt.errors @@ -1 +1 @@ -line 1:24 mismatched input '' expecting {'[', ':', '.'} \ No newline at end of file +line 1:24 mismatched input '' expecting {'[', ':', '.'} diff --git a/rego/examples/e00019.stmt.errors b/rego/examples/e00019.stmt.errors index 6af17d122d..eabdcc6f2d 100644 --- a/rego/examples/e00019.stmt.errors +++ b/rego/examples/e00019.stmt.errors @@ -1 +1 @@ -line 1:20 no viable alternative at input ', ' \ No newline at end of file +line 1:20 no viable alternative at input ', ' diff --git a/rego/examples/e00020.stmt.errors b/rego/examples/e00020.stmt.errors index c6b7340f0f..477b6399a9 100644 --- a/rego/examples/e00020.stmt.errors +++ b/rego/examples/e00020.stmt.errors @@ -1,4 +1,4 @@ line 1:10 no viable alternative at input '{foo: bar baz' line 1:4 extraneous input ':' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} line 1:13 extraneous input ':' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} -line 1:17 extraneous input '}' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} \ No newline at end of file +line 1:17 extraneous input '}' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} diff --git a/rego/examples/e00021.stmt.errors b/rego/examples/e00021.stmt.errors index 34a330c5ae..ddd89409e6 100644 --- a/rego/examples/e00021.stmt.errors +++ b/rego/examples/e00021.stmt.errors @@ -3,4 +3,4 @@ line 1:8 extraneous input ',' expecting {, String, Bool, 'null', 'default', line 1:13 extraneous input ':' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} line 1:17 extraneous input ',' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} line 1:22 extraneous input ':' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} -line 1:29 extraneous input '}' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} \ No newline at end of file +line 1:29 extraneous input '}' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} diff --git a/rego/examples/e00022.stmt.errors b/rego/examples/e00022.stmt.errors index 712b51550c..e4e11fe8e6 100644 --- a/rego/examples/e00022.stmt.errors +++ b/rego/examples/e00022.stmt.errors @@ -1 +1 @@ -line 1:9 missing ']' at '' \ No newline at end of file +line 1:9 missing ']' at '' diff --git a/rego/examples/e00023.stmt.errors b/rego/examples/e00023.stmt.errors index d4914381a2..036f9e9c53 100644 --- a/rego/examples/e00023.stmt.errors +++ b/rego/examples/e00023.stmt.errors @@ -1 +1 @@ -line 1:11 mismatched input '' expecting {String, Bool, 'null', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} \ No newline at end of file +line 1:11 mismatched input '' expecting {String, Bool, 'null', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} diff --git a/rego/examples/e00024.stmt.errors b/rego/examples/e00024.stmt.errors index fb2bac7cd2..e0cf2ade57 100644 --- a/rego/examples/e00024.stmt.errors +++ b/rego/examples/e00024.stmt.errors @@ -1,2 +1,2 @@ line 1:5 no viable alternative at input '[foo bar' -line 1:8 extraneous input ']' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} \ No newline at end of file +line 1:8 extraneous input ']' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} diff --git a/rego/examples/e00025.stmt.errors b/rego/examples/e00025.stmt.errors index 4e8bf40bc9..1252acfb84 100644 --- a/rego/examples/e00025.stmt.errors +++ b/rego/examples/e00025.stmt.errors @@ -1,3 +1,3 @@ line 1:3 extraneous input ',' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} line 1:8 extraneous input ',' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} -line 1:13 extraneous input ']' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} \ No newline at end of file +line 1:13 extraneous input ']' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} diff --git a/rego/examples/e00026.stmt.errors b/rego/examples/e00026.stmt.errors index 684a5bace4..6c229316ad 100644 --- a/rego/examples/e00026.stmt.errors +++ b/rego/examples/e00026.stmt.errors @@ -1 +1 @@ -line 1:4 missing ')' at '' \ No newline at end of file +line 1:4 missing ')' at '' diff --git a/rego/examples/e00027.stmt.errors b/rego/examples/e00027.stmt.errors index 87c193de10..201d826a97 100644 --- a/rego/examples/e00027.stmt.errors +++ b/rego/examples/e00027.stmt.errors @@ -1 +1 @@ -line 1:9 missing '}' at '' \ No newline at end of file +line 1:9 missing '}' at '' diff --git a/rego/examples/e00028.stmt.errors b/rego/examples/e00028.stmt.errors index d4914381a2..036f9e9c53 100644 --- a/rego/examples/e00028.stmt.errors +++ b/rego/examples/e00028.stmt.errors @@ -1 +1 @@ -line 1:11 mismatched input '' expecting {String, Bool, 'null', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} \ No newline at end of file +line 1:11 mismatched input '' expecting {String, Bool, 'null', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} diff --git a/rego/examples/e00030.stmt.errors b/rego/examples/e00030.stmt.errors index 4c67cf230a..ee8e28953a 100644 --- a/rego/examples/e00030.stmt.errors +++ b/rego/examples/e00030.stmt.errors @@ -1,3 +1,3 @@ line 1:3 extraneous input ',' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} line 1:8 extraneous input ',' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} -line 1:13 extraneous input '}' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} \ No newline at end of file +line 1:13 extraneous input '}' expecting {, String, Bool, 'null', 'default', 'import', 'package', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} diff --git a/rego/examples/e00039.stmt.errors b/rego/examples/e00039.stmt.errors index 895f4f0708..7b70f70e6e 100644 --- a/rego/examples/e00039.stmt.errors +++ b/rego/examples/e00039.stmt.errors @@ -1 +1 @@ -line 1:15 no viable alternative at input '{ "foo" =' \ No newline at end of file +line 1:15 no viable alternative at input '{ "foo" =' diff --git a/rego/examples/e00040.stmt.errors b/rego/examples/e00040.stmt.errors index 361f8fafea..dbe5e83ce8 100644 --- a/rego/examples/e00040.stmt.errors +++ b/rego/examples/e00040.stmt.errors @@ -1 +1 @@ -line 1:27 no viable alternative at input '{ trim(x, ".", y) ' \ No newline at end of file +line 1:27 no viable alternative at input '{ trim(x, ".", y) ' diff --git a/rego/examples/e00041.stmt.errors b/rego/examples/e00041.stmt.errors index c56364c73a..e26e242866 100644 --- a/rego/examples/e00041.stmt.errors +++ b/rego/examples/e00041.stmt.errors @@ -1,2 +1 @@ line 1:17 mismatched input '}' expecting {String, Bool, 'null', 'not', 'set(', '[', '{', '(', ArithOperator, UnsignedNumber, Name} - diff --git a/vb6/examples/form1.vb.errors b/vb6/examples/form1.vb.errors index e02f5aa4d8..6da43387db 100644 --- a/vb6/examples/form1.vb.errors +++ b/vb6/examples/form1.vb.errors @@ -1,2 +1 @@ line 1:13 mismatched input 'Form1' expecting -