Enable testing of grammars with Antlr4ng (#4019)

* Add in Antlr4ng for grammars that work.

* Fix typo.

* Add in templates for Antlr4ng.

* Fix pshl build script for Antlr4ng.

* Fixes for building and testing the Antlr4ng target.

* Add node always. Required for Antlr4ng.

.github/workflows/main.yml

@@ -124,12 +124,10 @@ jobs:
       run: |
          pip install antlr4-tools
     - name: Install JavaScript
-      if: ${{ matrix.language == 'JavaScript' }}
       uses: actions/[email protected]
       with:
-        node-version: '16.13.0'
+        node-version: '21.7.1'
     - name: Test JavaScript
-      if: ${{ matrix.language == 'JavaScript' }}
       run: |
         node --version
     - name: Update paths

_scripts/templates/Antlr4ng/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "i",
+  "version": "1.0.0",
+  "description": "",
+  "main": "Test.js",
+  "scripts": {
+    "build": "tsc -p tsconfig.json"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "antlr4ng": "3.0.4",
+    "antlr4ng-cli": "2.0.0",
+    "buffer": "^6.0.3",
+    "fs-extra": "^11.1.1",
+    "timer-node": "^5.0.6",
+    "typescript-string-operations": "^1.5.0"
+  },
+  "type": "module",
+  "devDependencies": {
+    "@types/node": "^20.2.5"
+  }
+}

_scripts/templates/Antlr4ng/st.Test.ts

@@ -0,0 +1,225 @@
+// Generated from trgen <version>
+
+import { ATNSimulator } from 'antlr4ng';
+import { CharStream } from 'antlr4ng';
+import { CommonTokenStream } from 'antlr4ng';
+import { ConsoleErrorListener } from 'antlr4ng';
+import { BaseErrorListener } from 'antlr4ng';
+//import { InputStream } from 'antlr4ng';
+import { Recognizer } from 'antlr4ng';
+import { RecognitionException } from 'antlr4ng';
+import { Token } from 'antlr4ng';
+import { readFileSync } from 'fs';
+import { writeFileSync } from 'fs';
+import { openSync } from 'fs';
+import { readSync } from 'fs';
+import { writeSync } from 'fs';
+import { closeSync } from 'fs';
+import { readFile } from 'fs/promises'
+
+<tool_grammar_tuples: {x | import { <x.GrammarAutomName> \} from './<x.GrammarAutomName>.js';
+} >
+import { StringBuilder, emptyString, joinString, formatString, isNullOrWhiteSpace } from 'typescript-string-operations';
+import { Timer, Time, TimerOptions } from 'timer-node';
+
+
+function getChar() {
+    let buffer = Buffer.alloc(1);
+    var xx = 0;
+    try {
+        xx = readSync(0, buffer, 0, 1, null);
+    } catch (err) {
+    }
+    if (xx === 0) {
+        return '';
+    }
+    return buffer.toString('utf8');
+}
+
+
+class MyErrorListener\<T  extends ATNSimulator> extends ConsoleErrorListener {
+    _quiet: boolean;
+    _tee: boolean;
+    _output: any;
+    had_error: boolean;
+
+    constructor(quiet: boolean, tee: boolean, output: any) {
+        super();
+        this._quiet = quiet;
+        this._tee = tee;
+        this._output = output;
+        this.had_error = false;
+    }
+
+    syntaxError\<T extends ATNSimulator>(recognizer: Recognizer\<T> | null, offendingSymbol: unknown, line: number, column: number, msg: string | null, e: RecognitionException | null): void {
+        this.had_error = true;
+        if (this._tee) {
+            writeSync(this._output, `line ${line}:${column} ${msg}\n`);
+        }
+        if (!this._quiet) {
+            console.error(`line ${line}:${column} ${msg}`);
+        }
+    }
+}
+
+var tee = false;
+var show_profile = false;
+var show_tree = false;
+var show_tokens = false;
+var show_trace = false;
+var error_code = 0;
+var quiet = false;
+var enc = 'utf8';
+var string_instance = 0;
+var prefix = '';
+var inputs: string[] = [];
+var is_fns: boolean[] = [];
+
+function splitLines(t: string) { return t.split(/\r\n|\r|\n/); }
+
+function main() {
+    for (let i = 2; i \<process.argv.length; ++i)
+    {
+        switch (process.argv[i]) {
+            case '-tokens':
+                show_tokens = true;
+                break;
+            case '-tree':
+                show_tree = true;
+                break;
+            case '-prefix':
+                prefix = process.argv[++i] + ' ';
+                break;
+            case '-input':
+                inputs.push(process.argv[++i]);
+                is_fns.push(false);
+                break;
+            case '-tee':
+                tee = true;
+                break;
+            case '-encoding':
+                enc = process.argv[++i];
+                break;
+            case '-x':
+                var sb = new StringBuilder();
+                var ch;
+                while ((ch = getChar()) != '') {
+                    sb.Append(ch);
+                }
+                var input = sb.ToString();
+                var sp = splitLines(input);
+                for (var ii of sp) {
+                    if (ii == '') continue;
+                    inputs.push(ii);
+                    is_fns.push(true);
+                }
+                break;
+            case '-q':
+                quiet = true;
+                break;
+            case '-trace':
+                show_trace = true;
+                break;
+            default:
+                inputs.push(process.argv[i]);
+                is_fns.push(true);
+                break;
+        }
+    }
+    if (inputs.length == 0) {
+        ParseStdin();
+    }
+    else {
+        const timer = new Timer({ label: 'test-timer' });
+        timer.start();
+        for (var f = 0; f \<inputs.length; ++f)
+        {
+            if (is_fns[f])
+                ParseFilename(inputs[f], f);
+            else
+                ParseString(inputs[f], f);
+        }
+        timer.stop();
+        var t = timer.time().m * 60 + timer.time().s + timer.time().ms / 1000;
+        if (!quiet) console.error('Total Time: ' + t);
+    }
+    process.exitCode = error_code;
+}
+
+function ParseStdin() {
+    var sb = new StringBuilder();
+    var ch;
+    while ((ch = getChar()) != '') {
+        sb.Append(ch);
+    }
+    var input = sb.ToString();
+    var str = CharStream.fromString(input);
+    DoParse(str, "stdin", 0);
+}
+
+function ParseString(input: string, row_number: number) {
+    var str = CharStream.fromString(input);
+    DoParse(str, "string" + string_instance++, row_number);
+}
+
+function ParseFilename(input: string, row_number: number) {
+    var buffer = readFileSync(input, { encoding: enc as BufferEncoding });
+    var str = CharStream.fromString(buffer);
+    DoParse(str, input, row_number);
+}
+
+function DoParse(str: CharStream, input_name: string, row_number: number) {
+    const lexer = new <lexer_name>(str);
+    const tokens = new CommonTokenStream(lexer);
+    const parser = new <parser_name>(tokens);
+    lexer.removeErrorListeners();
+    parser.removeErrorListeners();
+    var output = tee ? openSync(input_name + ".errors", 'w') : 1;
+    var listener_parser = new MyErrorListener(quiet, tee, output);
+    var listener_lexer = new MyErrorListener(quiet, tee, output);
+    parser.addErrorListener(listener_parser);
+    lexer.addErrorListener(listener_lexer);
+    if (show_tokens) {
+        for (var i = 0; ; ++i) {
+            var ro_token = lexer.nextToken();
+            var token = ro_token;
+            token.tokenIndex = i;
+            console.error(token.toString());
+            if (token.type === Token.EOF)
+                break;
+        }
+//        lexer.reset();
+    }
+    if (show_trace) {
+//       parser._interp.trace_atn_sim = true;
+    }
+    const timer = new Timer({ label: 'test-timer2' });
+    timer.start();
+    const tree = parser.<start_symbol>();
+    timer.stop();
+    var result = "";
+    if (listener_parser.had_error || listener_lexer.had_error) {
+        result = 'fail';
+        error_code = 1;
+    }
+    else {
+        result = 'success';
+    }
+    var t = timer.time().m * 60 + timer.time().s + timer.time().ms / 1000;
+    if (show_tree) {
+        if (tee) {
+            writeFileSync(input_name + ".tree", tree.toStringTree(parser.ruleNames, parser));
+        } else {
+            console.error(tree.toStringTree(parser.ruleNames, parser));
+        }
+    }
+    if (!quiet) {
+        console.error(prefix + 'TypeScript ' + row_number + ' ' + input_name + ' ' + result + ' ' + t);
+    }
+    if (tee) {
+        closeSync(output);
+    }
+}
+
+
+main()

_scripts/templates/Antlr4ng/st.build.ps1

@@ -0,0 +1,27 @@
+# Generated from trgen <version>
+
+if (Test-Path -Path transformGrammar.py -PathType Leaf) {
+    $(& python3 transformGrammar.py ) 2>&1 | Write-Host
+}
+
+$(& npm install -g typescript ts-node ; $compile_exit_code = $LASTEXITCODE ) | Write-Host
+if($compile_exit_code -ne 0){
+    exit $compile_exit_code
+\}
+
+$(& npm install ; $compile_exit_code = $LASTEXITCODE ) | Write-Host
+if($compile_exit_code -ne 0){
+    exit $compile_exit_code
+\}
+
+$jarFile = Get-ChildItem ./node_modules/antlr4ng-cli/*.jar
+<tool_grammar_tuples:{x |
+$(& java -jar $jarFile.FullName <x.GrammarFileName> -encoding <antlr_encoding> -Dlanguage=TypeScript <x.AntlrArgs> <antlr_tool_args:{y | <y> } > ; $compile_exit_code = $LASTEXITCODE) | Write-Host
+if($compile_exit_code -ne 0){
+    exit $compile_exit_code
+\}
+}>
+
+$(& tsc -p tsconfig.json --pretty ; $compile_exit_code = $LASTEXITCODE ) | Write-Host
+
+exit $compile_exit_code

_scripts/templates/Antlr4ng/st.build.sh

@@ -0,0 +1,20 @@
+# Generated from trgen <version>
+set -e
+rm -rf node_modules package-lock.json
+npm install -g typescript ts-node
+npm install
+
+if [ -f transformGrammar.py ]; then python3 transformGrammar.py ; fi
+
+# Because there is no integrated build script for Dart targets, we need
+# to manually look at the version in package.json and extract the
+# version number. We can then use this with antlr4 to generate the
+# parser and lexer.
+version=`grep antlr4 package.json | awk '{print $2}' | tr -d '"' | tr -d ',' | tr -d '\r' | tr -d '\n'`
+
+<tool_grammar_tuples:{x |
+java -jar ./node_modules/antlr4ng-cli/*.jar -encoding <antlr_encoding> -Dlanguage=TypeScript <x.AntlrArgs> <antlr_tool_args:{y | <y> } > <x.GrammarFileName>
+} >
+
+tsc -p tsconfig.json --pretty
+exit 0

_scripts/templates/Antlr4ng/st.clean.ps1

@@ -0,0 +1,17 @@
+# Generated from trgen <version>
+$(& Remove-Item *.interp -Recurse -Force ) 2>&1 | Out-Null
+$files = New-Object System.Collections.Generic.List[string]
+<tool_grammar_tuples:{x |
+$f = java -jar "<antlr_tool_path>" -depend -encoding <antlr_encoding> -Dlanguage=TypeScript <x.AntlrArgs> <antlr_tool_args:{y | <y> } > <x.GrammarFileName>
+foreach ($s in $f) {
+    $j = $s.Split(" ")[0]
+    $files.Add($j)
+\}
+foreach ($f in $files)
+{
+    $(& Remove-Item $f -Force ) 2>&1 | Out-Null
+\}
+} >
+$(& Remove-Item node_modules -Recurse -Force ) 2>&1 | Out-Null
+$(& Remove-Item package-lock.json -Recurse -Force ) 2>&1 | Out-Null
+exit 0

_scripts/templates/Antlr4ng/st.clean.sh

@@ -0,0 +1,12 @@
+# Generated from trgen <version>
+rm -f *.interp
+files=()
+<tool_grammar_tuples:{x |
+files+=( `java -jar "<antlr_tool_path>" -depend -encoding <antlr_encoding> -Dlanguage=TypeScript <x.AntlrArgs> <antlr_tool_args:{y | <y> } > <x.GrammarFileName> | awk '{print $1\}' | grep -v ':'` )
+} >
+for i in ${files[*]}
+do
+    rm -f $i
+done
+rm -rf node_modules package-lock.json
+exit 0

_scripts/templates/Antlr4ng/st.makefile

@@ -0,0 +1,8 @@
+# Generated from trgen <version>
+build: FORCE
+	bash build.sh
+clean: FORCE
+	bash clean.sh
+FORCE: ;
+test: FORCE
+	bash test.sh