Skip to content

Commit 846bc57

Browse files
pomberpomber
committed
Refactor tokenizeLine
1 parent 67e00f0 commit 846bc57

File tree

7 files changed

+297
-110
lines changed

7 files changed

+297
-110
lines changed

‎lib/dist/browser.esm.mjs

+27-27
Original file line numberDiff line numberDiff line change
@@ -2412,40 +2412,40 @@ function tokenize(code, grammar, colors) {
24122412
let stack = null;
24132413
const lines = code.split(/\r?\n|\r/g);
24142414
return lines.map((line) => {
2415-
const { tokens, ruleStack } = grammar.tokenizeLine2(line, stack);
2416-
const newTokens = [];
2417-
let tokenEnd = line.length;
2418-
for (let i = tokens.length - 2; i >= 0; i = i - 2) {
2419-
const tokenStart = tokens[i];
2420-
const metadata = tokens[i + 1];
2421-
newTokens.unshift({
2422-
content: line.slice(tokenStart, tokenEnd),
2423-
style: getStyle(metadata, colors),
2424-
});
2425-
tokenEnd = tokenStart;
2426-
}
2427-
stack = ruleStack;
2428-
return newTokens;
2415+
const { rawTokens, nextStack } = tokenizeLine(grammar, stack, line);
2416+
stack = nextStack;
2417+
return rawTokens.map(({ content, metadata }) => ({
2418+
content,
2419+
style: getStyle(metadata, colors),
2420+
}));
24292421
});
24302422
}
2423+
function tokenizeLine(grammar, stack, line) {
2424+
const { tokens, ruleStack } = grammar.tokenizeLine2(line, stack);
2425+
const newTokens = [];
2426+
let tokenEnd = line.length;
2427+
for (let i = tokens.length - 2; i >= 0; i = i - 2) {
2428+
const tokenStart = tokens[i];
2429+
const metadata = tokens[i + 1];
2430+
newTokens.unshift({
2431+
content: line.slice(tokenStart, tokenEnd),
2432+
metadata,
2433+
});
2434+
tokenEnd = tokenStart;
2435+
}
2436+
return { rawTokens: newTokens, nextStack: ruleStack };
2437+
}
24312438
function tokenizeWithScopes(code, grammar, colors) {
24322439
let stack = null;
24332440
const lines = code.split(/\r?\n|\r/g);
24342441
return lines.map((line) => {
2435-
const { tokens, ruleStack } = grammar.tokenizeLine2(line, stack);
2436-
const newTokens = [];
2437-
let tokenEnd = line.length;
2438-
for (let i = tokens.length - 2; i >= 0; i = i - 2) {
2439-
const tokenStart = tokens[i];
2440-
const metadata = tokens[i + 1];
2441-
newTokens.unshift({
2442-
content: line.slice(tokenStart, tokenEnd),
2443-
style: getStyle(metadata, colors),
2444-
});
2445-
tokenEnd = tokenStart;
2446-
}
2442+
const { rawTokens, nextStack } = tokenizeLine(grammar, stack, line);
2443+
const newTokens = rawTokens.map(({ content, metadata }) => ({
2444+
content,
2445+
style: getStyle(metadata, colors),
2446+
}));
24472447
const tokensWithScopes = addScopesToLine(line, stack, grammar, newTokens);
2448-
stack = ruleStack;
2448+
stack = nextStack;
24492449
return tokensWithScopes;
24502450
});
24512451
}

‎lib/dist/index.cjs.js

+27-27
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎lib/dist/index.esm.mjs

+27-27
Original file line numberDiff line numberDiff line change
@@ -2760,40 +2760,40 @@ function tokenize(code, grammar, colors) {
27602760
let stack = null;
27612761
const lines = code.split(/\r?\n|\r/g);
27622762
return lines.map((line) => {
2763-
const { tokens, ruleStack } = grammar.tokenizeLine2(line, stack);
2764-
const newTokens = [];
2765-
let tokenEnd = line.length;
2766-
for (let i = tokens.length - 2; i >= 0; i = i - 2) {
2767-
const tokenStart = tokens[i];
2768-
const metadata = tokens[i + 1];
2769-
newTokens.unshift({
2770-
content: line.slice(tokenStart, tokenEnd),
2771-
style: getStyle(metadata, colors),
2772-
});
2773-
tokenEnd = tokenStart;
2774-
}
2775-
stack = ruleStack;
2776-
return newTokens;
2763+
const { rawTokens, nextStack } = tokenizeLine(grammar, stack, line);
2764+
stack = nextStack;
2765+
return rawTokens.map(({ content, metadata }) => ({
2766+
content,
2767+
style: getStyle(metadata, colors),
2768+
}));
27772769
});
27782770
}
2771+
function tokenizeLine(grammar, stack, line) {
2772+
const { tokens, ruleStack } = grammar.tokenizeLine2(line, stack);
2773+
const newTokens = [];
2774+
let tokenEnd = line.length;
2775+
for (let i = tokens.length - 2; i >= 0; i = i - 2) {
2776+
const tokenStart = tokens[i];
2777+
const metadata = tokens[i + 1];
2778+
newTokens.unshift({
2779+
content: line.slice(tokenStart, tokenEnd),
2780+
metadata,
2781+
});
2782+
tokenEnd = tokenStart;
2783+
}
2784+
return { rawTokens: newTokens, nextStack: ruleStack };
2785+
}
27792786
function tokenizeWithScopes(code, grammar, colors) {
27802787
let stack = null;
27812788
const lines = code.split(/\r?\n|\r/g);
27822789
return lines.map((line) => {
2783-
const { tokens, ruleStack } = grammar.tokenizeLine2(line, stack);
2784-
const newTokens = [];
2785-
let tokenEnd = line.length;
2786-
for (let i = tokens.length - 2; i >= 0; i = i - 2) {
2787-
const tokenStart = tokens[i];
2788-
const metadata = tokens[i + 1];
2789-
newTokens.unshift({
2790-
content: line.slice(tokenStart, tokenEnd),
2791-
style: getStyle(metadata, colors),
2792-
});
2793-
tokenEnd = tokenStart;
2794-
}
2790+
const { rawTokens, nextStack } = tokenizeLine(grammar, stack, line);
2791+
const newTokens = rawTokens.map(({ content, metadata }) => ({
2792+
content,
2793+
style: getStyle(metadata, colors),
2794+
}));
27952795
const tokensWithScopes = addScopesToLine(line, stack, grammar, newTokens);
2796-
stack = ruleStack;
2796+
stack = nextStack;
27972797
return tokensWithScopes;
27982798
});
27992799
}

‎lib/src/tokenizer.ts

+31-28
Original file line numberDiff line numberDiff line change
@@ -24,23 +24,33 @@ export function tokenize(code: string, grammar: IGrammar, colors: string[]) {
2424
let stack: StackElement | null = null;
2525
const lines = code.split(/\r?\n|\r/g);
2626
return lines.map((line) => {
27-
const { tokens, ruleStack } = grammar.tokenizeLine2(line, stack);
28-
const newTokens: Token[] = [];
29-
let tokenEnd = line.length;
30-
for (let i = tokens.length - 2; i >= 0; i = i - 2) {
31-
const tokenStart = tokens[i];
32-
const metadata = tokens[i + 1];
33-
newTokens.unshift({
34-
content: line.slice(tokenStart, tokenEnd),
35-
style: getStyle(metadata, colors),
36-
});
37-
tokenEnd = tokenStart;
38-
}
39-
stack = ruleStack;
40-
return newTokens;
27+
const { rawTokens, nextStack } = tokenizeLine(grammar, stack, line);
28+
stack = nextStack;
29+
return rawTokens.map(({ content, metadata }) => ({
30+
content,
31+
style: getStyle(metadata, colors),
32+
}));
4133
});
4234
}
4335

36+
type RawToken = { content: string; metadata: number };
37+
38+
function tokenizeLine(grammar: IGrammar, stack: StackElement, line: string) {
39+
const { tokens, ruleStack } = grammar.tokenizeLine2(line, stack);
40+
const newTokens: RawToken[] = [];
41+
let tokenEnd = line.length;
42+
for (let i = tokens.length - 2; i >= 0; i = i - 2) {
43+
const tokenStart = tokens[i];
44+
const metadata = tokens[i + 1];
45+
newTokens.unshift({
46+
content: line.slice(tokenStart, tokenEnd),
47+
metadata,
48+
});
49+
tokenEnd = tokenStart;
50+
}
51+
return { rawTokens: newTokens, nextStack: ruleStack };
52+
}
53+
4454
export function tokenizeWithScopes(
4555
code: string,
4656
grammar: IGrammar,
@@ -50,21 +60,14 @@ export function tokenizeWithScopes(
5060
const lines = code.split(/\r?\n|\r/g);
5161

5262
return lines.map((line) => {
53-
const { tokens, ruleStack } = grammar.tokenizeLine2(line, stack);
54-
55-
const newTokens: Token[] = [];
56-
let tokenEnd = line.length;
57-
for (let i = tokens.length - 2; i >= 0; i = i - 2) {
58-
const tokenStart = tokens[i];
59-
const metadata = tokens[i + 1];
60-
newTokens.unshift({
61-
content: line.slice(tokenStart, tokenEnd),
62-
style: getStyle(metadata, colors),
63-
} as Token);
64-
tokenEnd = tokenStart;
65-
}
63+
const { rawTokens, nextStack } = tokenizeLine(grammar, stack, line);
64+
const newTokens = rawTokens.map(({ content, metadata }) => ({
65+
content,
66+
style: getStyle(metadata, colors),
67+
}));
6668
const tokensWithScopes = addScopesToLine(line, stack, grammar, newTokens);
67-
stack = ruleStack;
69+
70+
stack = nextStack;
6871
return tokensWithScopes;
6972
});
7073
}

0 commit comments

Comments
 (0)