Merge pull request #1546 from dodona-edu/fix/large-file-invalid-argument

Fix invalid argument error for large files since tree-sitter 0.21

Author: rien

.github/workflows/ci-cd.yml

@@ -126,7 +126,7 @@ jobs:
       - name: Build 🔧
         run: |
           cd cli/
-          npm run build --force
+          npm run build -- --force
 
   cli-lint:
     name: "CLI 💻️ - lint 📏"
@@ -203,7 +203,7 @@ jobs:
         run: |
           cd core/
           npm run build
-          npm run test -v --serial
+          npm run test -- -v --serial
 
   core-lint:
     name: "Core ❤️ - lint 📏"
@@ -282,7 +282,7 @@ jobs:
         run: |
           cd lib/
           npm run build
-          npm run test -v --serial
+          npm run test -- -v --serial
 
   parsers-build:
     name: "Parsers 👓 - build 🔧"
@@ -587,7 +587,7 @@ jobs:
       - name: Build CLI 🔧
         run: |
           cd cli/
-          npm run build --force
+          npm run build -- --force
 
       - name: Build web 🔧
         run: |

lib/src/lib/tokenizer/codeTokenizer.ts

@@ -27,7 +27,7 @@ export class CodeTokenizer extends Tokenizer {
    * @param text The text string to parse
    */
   public tokenize(text: string): string {
-    const tree = this.parser.parse(text);
+    const tree = this.parser.parse(text, undefined, { bufferSize: Math.max(32 * 1024, text.length * 2) });
     return tree.rootNode.toString();
   }
 
@@ -39,7 +39,7 @@ export class CodeTokenizer extends Tokenizer {
    * @param text The text string to parse
    */
   public *generateTokens(text: string): IterableIterator<Token> {
-    const tree = this.parser.parse(text);
+    const tree = this.parser.parse(text, undefined, { bufferSize: Math.max(32 * 1024, text.length * 2) });
     yield* this.tokenizeNode(tree.rootNode);
   }
 

lib/src/test/tokenizer.test.ts

@@ -76,3 +76,14 @@ test("should be able to use external tree-sitter parsers (tree-sitter-json)", as
   const { tokens } = tokenizer.tokenizeFile(file);
   t.truthy(tokens);
 });
+
+test("should be able to parse larger files", async t => {
+  const file = new File("long.js", "var test = 1;\n".repeat(10000));
+  const language = await (new LanguagePicker().findLanguage("javascript"));
+
+  const tokenizer = await language.createTokenizer();
+  t.truthy(tokenizer);
+
+  const { tokens } = tokenizer.tokenizeFile(file);
+  t.truthy(tokens);
+});