Fix for Issue 2988: Change individual to grouped parsing

.github/workflows/main.yml

@@ -121,8 +121,9 @@ jobs:
     - name: Install trgen
       shell: pwsh
       run: |
-        dotnet tool install -g trgen --version 0.16.3
-        dotnet tool install -g trwdog --version 0.16.3
+        dotnet tool install -g trgen --version 0.19.0
+        dotnet tool install -g triconv --version 0.19.0
+        dotnet tool install -g trwdog --version 0.19.0
         if ("${{ matrix.os }}" -eq "ubuntu-latest") {
             echo "$HOME/.dotnet/tools" >> $env:GITHUB_PATH
         }

.gitignore

@@ -27,3 +27,8 @@
 
 # Ignore downloaded temporaries create by _scripts/get-antlr.ps1
 antlr4-*-complete.jar
+
+# Ignore untracked .errors and .tree files. If it was
+# checked in, it is tracked, and cannot be ignored.
+/**/*.errors
+/**/*.tree

_scripts/readme.md

@@ -3,12 +3,11 @@
 This directory contains scripts for CI testing.
 
 * regtest.sh -- This is a Bash-based script to test any target. To
-run, cd to grammar-vs/, then type `bash regtest.sh <target>` where `<target>` is CSharp, Java, Cpp, Dart, Go, or one
-of the other targets. To test the target, you will need the NET SDK installed,
-access to the internet, and to toolchain for the target you want to test.
-For CSharp, it will download the Antlr4 tool and runtime. For the other
-targets, you will need to download the Antlr4 tool antlr-4.9.3-complete.jar and
-place it in /tmp.
+run, cd to grammar-vs/ or a grammar directory, then type `bash regtest.sh <target>`
+where `<target>` is Antlr4cs, Cpp, CSharp, Dart, Go, Java, PHP, or Python3.
+To test the target, you will need the NET SDK installed (required for the trgen
+driver generator), access to the internet, and to toolchain for the target
+you want to test. You will also need to set the Antlr4 tool.
 
 * test.ps1 -- this is a Powershell script for testing, similar to regtest.sh.
 

_scripts/really-run.ps1

@@ -14,8 +14,9 @@ $antlrPath = _scripts/get-antlr.ps1 "4.11.1"
 # Set up env as it is used in test script.
 echo "antlr_path=$antlrPath" >> $env:GITHUB_ENV
 
-dotnet tool install -g trgen --version 0.16.3
-dotnet tool install -g trwdog --version 0.16.3
+dotnet tool install -g trgen --version 0.19.0
+dotnet tool install -g triconv --version 0.19.0
+dotnet tool install -g trwdog --version 0.19.0
 
 # Call test script.
 $env:ANTLR_JAR_PATH="$antlrPath"

_scripts/regtest.sh

@@ -97,7 +97,7 @@ add() {
     fi
 }
 
-rm -rf `find . -name Generated -type d`
+rm -rf `find . -name 'Generated*' -type d`
 
 build()
 {
@@ -108,7 +108,7 @@ build()
     echo Building $testname
     echo ""
     # Assert(cwd is valid)
-    pushd "$x/Generated"
+    pushd "$x/Generated-$target"
     if [[ $? != "0" ]]
     then
         echo "$1 is not a valid directory"
@@ -136,14 +136,14 @@ test()
     echo Testing $testname
     echo ""
     # Assert(cwd is valid)
-    pushd "$x/Generated"
+    pushd "$x/Generated-$target"
     if [[ $? != "0" ]]
     then
         echo "$1 is not a valid directory"
         exit 1
     fi
     date1=$(date +"%s")
-    make test
+    bash test.sh
     status="$?"
     date2=$(date +"%s")
     DIFF=$(($date2-$date1))
@@ -166,9 +166,10 @@ setupdeps()
     if [ $? != "0" ]
     then
         echo "Setting up trgen and antlr jar."
-        dotnet tool install -g trgen --version 0.16.3
-        dotnet tool install -g trxml2 --version 0.16.3
-        dotnet tool install -g trwdog --version 0.16.3
+        dotnet tool install -g trgen --version 0.19.0
+        dotnet tool install -g triconv --version 0.19.0
+        dotnet tool install -g trxml2 --version 0.19.0
+        dotnet tool install -g trwdog --version 0.19.0
 	case "${unameOut}" in
 		Linux*)     curl 'https://repo1.maven.org/maven2/org/antlr/antlr4/4.11.1/antlr4-4.11.1-complete.jar' -o /tmp/antlr4-4.11.1-complete.jar;;
 		Darwin*)    curl 'https://repo1.maven.org/maven2/org/antlr/antlr4/4.11.1/antlr4-4.11.1-complete.jar' -o /tmp/antlr4-4.11.1-complete.jar;;
@@ -185,7 +186,7 @@ part1()
 {
     date
     # 1) Generate driver source code from poms.
-    rm -rf `find . -name Generated -type d`
+    rm -rf `find . -name Generated-$target -type d`
     echo "Generating drivers."
     if [[ "$invert" == "" ]]
     then
@@ -208,7 +209,7 @@ part2()
     esac
     echo prefix $prefix
     echo bft $build_file_type
-    build_files=`find $prefix -type f -name $build_file_type | grep Generated`
+    build_files=`find $prefix -type f -name $build_file_type | grep Generated-$target`
     echo bf $build_files
     for build_file in $build_files
     do
@@ -229,7 +230,7 @@ part3()
     # 3) Test generated parser on examples.
     echo "Parsing."
     date
-    build_files=`find $prefix -type f -name $build_file_type | grep Generated`
+    build_files=`find $prefix -type f -name $build_file_type | grep Generated-$target`
     echo bf $build_files
     for build_file in $build_files
     do

_scripts/skip-go.txt

@@ -3,6 +3,7 @@ antlr/antlr2
 antlr/antlr3
 antlr/antlr4
 apex
+asm/asm8086
 asn/asn_3gpp
 clif
 cobol85
@@ -11,6 +12,7 @@ csharp
 dart2
 databank
 edif300
+fasta
 fortran77
 gdscript
 glsl
@@ -73,6 +75,7 @@ sql/hive/v2
 sql/hive/v3
 sql/plsql
 sql/postgresql
+sql/tsql
 stringtemplate
 swift/swift2
 swift/swift3

_scripts/skip-javascript.txt

@@ -35,6 +35,7 @@ r
 rego
 rexx
 rust
+scss
 sql/hive/v2
 sql/hive/v3
 sql/mariadb

_scripts/skip-python3.txt

@@ -33,6 +33,7 @@ javascript/ecmascript
 javascript/jsx
 javascript/typescript
 joss
+kirikiri-tjs
 kotlin/kotlin
 kotlin/kotlin-formal
 kuka
@@ -84,6 +85,8 @@ sparql
 sql/hive/v2
 sql/hive/v3
 sql/mariadb
+sql/mysql
+sql/plsql
 sql/postgresql
 sql/sqlite
 sql/tsql

_scripts/templates/Antlr4cs/CaseChangingCharStream.cs

@@ -1,4 +1,4 @@
-// Template generated code from trgen <version>
+// Generated from trgen <version>
 
 /* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
  * Use of this file is governed by the BSD 3-clause license that

_scripts/templates/Antlr4cs/ErrorListener.cs

@@ -1,35 +1,39 @@
+// Generated from trgen <version>
+
+using Antlr4.Runtime;
+using Antlr4.Runtime.Misc;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
 <if(has_name_space)>namespace <name_space>
 {<endif>
-    public class ErrorListener<Symbol> : IAntlrErrorListener<Symbol>
+    public class ErrorListener\<S> : IAntlrErrorListener\<S>
     {
-        /// <summary>
-        /// Provides a default instance of
-        /// <see cref="ErrorListener{Symbol}"/>
-        /// .
-        /// </summary>
-        public static readonly ErrorListener<Symbol> Instance = new ErrorListener<Symbol>();
+        public bool had_error;
+        bool _quiet;
+        bool _tee;
+        TextWriter _out;
+
+        public ErrorListener(bool quiet, bool tee, TextWriter @out)
+        {
+            _quiet = quiet;
+            _tee = tee;
+            _out = @out;
+        }
 
-        /// <summary>
-        /// <inheritDoc/>
-        /// <p>
-        /// This implementation prints messages to
-        /// <see cref="System.Console.Out"/>
-        /// containing the
-        /// values of
-        /// <paramref name="line"/>
-        /// ,
-        /// <paramref name="charPositionInLine"/>
-        /// , and
-        /// <paramref name="msg"/>
-        /// using
-        /// the following format.</p>
-        /// <pre>
-        /// line <em>line</em>:<em>charPositionInLine</em> <em>msg</em>
-        /// </pre>
-        /// </summary>
-        public virtual void SyntaxError(IRecognizer recognizer, Symbol offendingSymbol, int line, int charPositionInLine, string msg, RecognitionException e)
+        public virtual void SyntaxError(IRecognizer recognizer, S offendingSymbol, int line, int col, string msg, RecognitionException e)
         {
-            System.Console.Out.WriteLine("line " + line + ":" + charPositionInLine + " " + msg);
+            had_error = true;
+            if (_tee)
+            {
+                _out.WriteLine("line " + line + ":" + col + " " + msg);
+            }
+            if (!_quiet)
+            {
+                System.Console.Error.WriteLine("line " + line + ":" + col + " " + msg);
+            }
         }
     }
-<if(has_name_space)>}<endif>
+<if(has_name_space)>}<endif>