Skip to content

Commit 2433479

Browse files
pomberpomber
committed
Add highlight with scopes
1 parent ae57aba commit 2433479

File tree

10 files changed

+337
-82
lines changed

10 files changed

+337
-82
lines changed

‎.changeset/shaggy-cats-tie.md

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@code-hike/lighter": patch
3+
---
4+
5+
Add highlight with scopes

‎lib/dist/browser.esm.mjs

+56-18
Original file line numberDiff line numberDiff line change
@@ -2081,23 +2081,43 @@ function tokenize(code, grammar, colors) {
20812081
return newTokens;
20822082
});
20832083
}
2084-
// export function tokenizeWithScopes(code: string, grammar: IGrammar) {
2085-
// let stack: StackElement | null = null;
2086-
// const lines = code.split(/\r?\n|\r/g);
2087-
// return lines.map((line) => {
2088-
// const { tokens, ruleStack } = grammar.tokenizeLine(line, stack);
2089-
// const newTokens: { content: string; scopes: string }[] = [];
2090-
// for (let i = 0; i < tokens.length; i++) {
2091-
// const { startIndex, endIndex, scopes } = tokens[i];
2092-
// newTokens.push({
2093-
// content: line.slice(startIndex, endIndex),
2094-
// scopes: scopes.join(" "),
2095-
// });
2096-
// }
2097-
// stack = ruleStack;
2098-
// return newTokens;
2099-
// });
2100-
// }
2084+
function tokenizeWithScopes(code, grammar, colors) {
2085+
let stack = null;
2086+
const lines = code.split(/\r?\n|\r/g);
2087+
return lines.map((line) => {
2088+
const { tokens, ruleStack } = grammar.tokenizeLine2(line, stack);
2089+
const newTokens = [];
2090+
let tokenEnd = line.length;
2091+
for (let i = tokens.length - 2; i >= 0; i = i - 2) {
2092+
const tokenStart = tokens[i];
2093+
const metadata = tokens[i + 1];
2094+
newTokens.unshift({
2095+
content: line.slice(tokenStart, tokenEnd),
2096+
style: getStyle(metadata, colors),
2097+
});
2098+
tokenEnd = tokenStart;
2099+
}
2100+
const tokensWithScopes = addScopesToLine(line, stack, grammar, newTokens);
2101+
stack = ruleStack;
2102+
return tokensWithScopes;
2103+
});
2104+
}
2105+
function addScopesToLine(line, stack, grammar, styledTokens) {
2106+
const { tokens } = grammar.tokenizeLine(line, stack);
2107+
const newTokens = [];
2108+
for (let i = 0; i < tokens.length; i++) {
2109+
const { startIndex, endIndex, scopes } = tokens[i];
2110+
let count = 0;
2111+
const styledToken = styledTokens.find((t) => {
2112+
count += t.content.length;
2113+
if (startIndex < count) {
2114+
return true;
2115+
}
2116+
});
2117+
newTokens.push(Object.assign(Object.assign({}, styledToken), { content: line.slice(startIndex, endIndex), scopes: scopes.reverse() }));
2118+
}
2119+
return newTokens;
2120+
}
21012121
function getStyle(metadata, colors) {
21022122
const fg = (metadata & FOREGROUND_MASK) >>> FOREGROUND_OFFSET;
21032123
// const bg = (metadata & BACKGROUND_MASK) >>> BACKGROUND_OFFSET;
@@ -2153,6 +2173,11 @@ function highlightTokens(code, grammar, theme) {
21532173
registry.setTheme(theme);
21542174
const colorMap = registry.getColorMap();
21552175
return tokenize(code, grammar, colorMap);
2176+
}
2177+
function highlightTokensWithScopes(code, grammar, theme) {
2178+
registry.setTheme(theme);
2179+
const colorMap = registry.getColorMap();
2180+
return tokenizeWithScopes(code, grammar, colorMap);
21562181
}
21572182

21582183
function parseRelativeRanges(relativeRange, lineNumber) {
@@ -2612,6 +2637,19 @@ function splitAnnotations(annotations) {
26122637
};
26132638
}
26142639

2640+
async function highlightWithScopes(code, alias, themeOrThemeName = "dark-plus") {
2641+
const { langId, grammarsPromise } = loadGrammars(alias);
2642+
const theme = await loadTheme(themeOrThemeName);
2643+
if (!theme) {
2644+
throw new UnknownThemeError(themeOrThemeName);
2645+
}
2646+
const grammar = await grammarsPromise;
2647+
return {
2648+
lines: highlightTokensWithScopes(code, grammar, theme),
2649+
lang: langId,
2650+
colors: getThemeColors(theme),
2651+
};
2652+
}
26152653
async function highlight(code, alias, themeOrThemeName = "dark-plus") {
26162654
const { langId, grammarsPromise } = loadGrammars(alias);
26172655
const theme = await loadTheme(themeOrThemeName);
@@ -2655,4 +2693,4 @@ class UnknownThemeError extends Error {
26552693
}
26562694
}
26572695

2658-
export { UnknownLanguageError, UnknownThemeError, annotatedHighlight, extractAnnotations, highlight };
2696+
export { UnknownLanguageError, UnknownThemeError, annotatedHighlight, extractAnnotations, highlight, highlightWithScopes };

‎lib/dist/index.cjs.js

+56-17
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎lib/dist/index.d.ts

+37-1
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ type Token = {
9292
fontWeight?: "bold";
9393
textDecoration?: "underline" | "line-through";
9494
};
95+
scopes?: string[];
9596
};
9697
type TokenGroup = {
9798
annotationName: string;
@@ -119,6 +120,41 @@ declare class UnknownLanguageError extends Error {
119120
constructor(alias: string);
120121
}
121122

123+
declare function highlightWithScopes(code: string, alias: LanguageAlias, themeOrThemeName?: Theme): Promise<{
124+
lines: Token[][];
125+
lang: LanguageName;
126+
colors: {
127+
background: string;
128+
foreground: string;
129+
lineNumberForeground: string;
130+
selectionBackground: string;
131+
editorBackground: string;
132+
editorGroupHeaderBackground: string;
133+
activeTabBackground: string;
134+
activeTabForeground: string;
135+
tabBorder: string;
136+
activeTabBorder: string;
137+
inactiveTabBackground: string;
138+
inactiveTabForeground: string;
139+
diffInsertedTextBackground: string;
140+
diffInsertedLineBackground: string;
141+
diffRemovedTextBackground: string;
142+
diffRemovedLineBackground: string;
143+
iconForeground: string;
144+
sideBarBackground: string;
145+
sideBarForeground: string;
146+
sideBarBorder: string;
147+
listSelectionBackground: string;
148+
listSelectionForeground: string;
149+
listHoverBackground: string;
150+
listHoverForeground: string;
151+
tabsBorder: string;
152+
activeTabTopBorder: string;
153+
hoverTabBackground: string;
154+
hoverTabForeground: string;
155+
colorScheme: "dark" | "light";
156+
};
157+
}>;
122158
declare function highlight(code: string, alias: LanguageAlias, themeOrThemeName?: Theme): Promise<{
123159
lines: Token[][];
124160
lang: LanguageName;
@@ -198,4 +234,4 @@ declare class UnknownThemeError extends Error {
198234
constructor(theme: string);
199235
}
200236

201-
export { Annotation, LanguageAlias, Line, LineGroup, Lines, RawTheme, StringTheme, Theme, ThemeColors, Token, TokenGroup, Tokens, UnknownLanguageError, UnknownThemeError, annotatedHighlight, extractAnnotations, highlight };
237+
export { Annotation, LanguageAlias, Line, LineGroup, Lines, RawTheme, StringTheme, Theme, ThemeColors, Token, TokenGroup, Tokens, UnknownLanguageError, UnknownThemeError, annotatedHighlight, extractAnnotations, highlight, highlightWithScopes };

‎lib/dist/index.esm.mjs

+56-18
Original file line numberDiff line numberDiff line change
@@ -2427,23 +2427,43 @@ function tokenize(code, grammar, colors) {
24272427
return newTokens;
24282428
});
24292429
}
2430-
// export function tokenizeWithScopes(code: string, grammar: IGrammar) {
2431-
// let stack: StackElement | null = null;
2432-
// const lines = code.split(/\r?\n|\r/g);
2433-
// return lines.map((line) => {
2434-
// const { tokens, ruleStack } = grammar.tokenizeLine(line, stack);
2435-
// const newTokens: { content: string; scopes: string }[] = [];
2436-
// for (let i = 0; i < tokens.length; i++) {
2437-
// const { startIndex, endIndex, scopes } = tokens[i];
2438-
// newTokens.push({
2439-
// content: line.slice(startIndex, endIndex),
2440-
// scopes: scopes.join(" "),
2441-
// });
2442-
// }
2443-
// stack = ruleStack;
2444-
// return newTokens;
2445-
// });
2446-
// }
2430+
function tokenizeWithScopes(code, grammar, colors) {
2431+
let stack = null;
2432+
const lines = code.split(/\r?\n|\r/g);
2433+
return lines.map((line) => {
2434+
const { tokens, ruleStack } = grammar.tokenizeLine2(line, stack);
2435+
const newTokens = [];
2436+
let tokenEnd = line.length;
2437+
for (let i = tokens.length - 2; i >= 0; i = i - 2) {
2438+
const tokenStart = tokens[i];
2439+
const metadata = tokens[i + 1];
2440+
newTokens.unshift({
2441+
content: line.slice(tokenStart, tokenEnd),
2442+
style: getStyle(metadata, colors),
2443+
});
2444+
tokenEnd = tokenStart;
2445+
}
2446+
const tokensWithScopes = addScopesToLine(line, stack, grammar, newTokens);
2447+
stack = ruleStack;
2448+
return tokensWithScopes;
2449+
});
2450+
}
2451+
function addScopesToLine(line, stack, grammar, styledTokens) {
2452+
const { tokens } = grammar.tokenizeLine(line, stack);
2453+
const newTokens = [];
2454+
for (let i = 0; i < tokens.length; i++) {
2455+
const { startIndex, endIndex, scopes } = tokens[i];
2456+
let count = 0;
2457+
const styledToken = styledTokens.find((t) => {
2458+
count += t.content.length;
2459+
if (startIndex < count) {
2460+
return true;
2461+
}
2462+
});
2463+
newTokens.push(Object.assign(Object.assign({}, styledToken), { content: line.slice(startIndex, endIndex), scopes: scopes.reverse() }));
2464+
}
2465+
return newTokens;
2466+
}
24472467
function getStyle(metadata, colors) {
24482468
const fg = (metadata & FOREGROUND_MASK) >>> FOREGROUND_OFFSET;
24492469
// const bg = (metadata & BACKGROUND_MASK) >>> BACKGROUND_OFFSET;
@@ -2499,6 +2519,11 @@ function highlightTokens(code, grammar, theme) {
24992519
registry.setTheme(theme);
25002520
const colorMap = registry.getColorMap();
25012521
return tokenize(code, grammar, colorMap);
2522+
}
2523+
function highlightTokensWithScopes(code, grammar, theme) {
2524+
registry.setTheme(theme);
2525+
const colorMap = registry.getColorMap();
2526+
return tokenizeWithScopes(code, grammar, colorMap);
25022527
}
25032528

25042529
function parseRelativeRanges(relativeRange, lineNumber) {
@@ -2958,6 +2983,19 @@ function splitAnnotations(annotations) {
29582983
};
29592984
}
29602985

2986+
async function highlightWithScopes(code, alias, themeOrThemeName = "dark-plus") {
2987+
const { langId, grammarsPromise } = loadGrammars(alias);
2988+
const theme = await loadTheme(themeOrThemeName);
2989+
if (!theme) {
2990+
throw new UnknownThemeError(themeOrThemeName);
2991+
}
2992+
const grammar = await grammarsPromise;
2993+
return {
2994+
lines: highlightTokensWithScopes(code, grammar, theme),
2995+
lang: langId,
2996+
colors: getThemeColors(theme),
2997+
};
2998+
}
29612999
async function highlight(code, alias, themeOrThemeName = "dark-plus") {
29623000
const { langId, grammarsPromise } = loadGrammars(alias);
29633001
const theme = await loadTheme(themeOrThemeName);
@@ -3001,4 +3039,4 @@ class UnknownThemeError extends Error {
30013039
}
30023040
}
30033041

3004-
export { UnknownLanguageError, UnknownThemeError, annotatedHighlight, extractAnnotations, highlight };
3042+
export { UnknownLanguageError, UnknownThemeError, annotatedHighlight, extractAnnotations, highlight, highlightWithScopes };

‎lib/src/annotations.ts

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ export type Token = {
1111
fontWeight?: "bold";
1212
textDecoration?: "underline" | "line-through";
1313
};
14+
scopes?: string[];
1415
};
1516

1617
export type TokenGroup = {

‎lib/src/highlighter.ts

+11-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import onig from "vscode-oniguruma/release/onig.wasm";
99
import { aliasToLangData } from "./language";
1010
import { loadGrammarByScope } from "./grammars";
1111
import { LanguageAlias } from "./language-data";
12-
import { tokenize } from "./tokenizer";
12+
import { tokenizeWithScopes, tokenize } from "./tokenizer";
1313
import { FinalTheme } from "./theme";
1414

1515
let registry: Registry | null = null;
@@ -56,3 +56,13 @@ export function highlightTokens(
5656
const colorMap = registry.getColorMap();
5757
return tokenize(code, grammar, colorMap);
5858
}
59+
60+
export function highlightTokensWithScopes(
61+
code: string,
62+
grammar: IGrammar,
63+
theme: FinalTheme
64+
) {
65+
registry.setTheme(theme);
66+
const colorMap = registry.getColorMap();
67+
return tokenizeWithScopes(code, grammar, colorMap);
68+
}

0 commit comments

Comments
 (0)