Merge pull request #37 from code-hike/regex-range

Regex range

Author: pomber

.changeset/many-phones-wonder.md

@@ -0,0 +1,5 @@
+---
+"@code-hike/lighter": patch
+---
+
+Add regex annotations

lib/dist/browser.esm.mjs

@@ -1,3 +1,16 @@
+type LineNumber = number;
+type ColumnNumber = number;
+type MultiLineRange = {
+    fromLineNumber: LineNumber;
+    toLineNumber: LineNumber;
+};
+type InlineRange = {
+    lineNumber: LineNumber;
+    fromColumn: ColumnNumber;
+    toColumn: ColumnNumber;
+};
+type CodeRange = MultiLineRange | InlineRange;
+
 type RawTheme = {
     name?: string;
     type?: string;
@@ -30,19 +43,6 @@ type NamesTuple = typeof LANG_NAMES;
 type LanguageAlias = NamesTuple[number];
 type LanguageName = "abap" | "actionscript-3" | "ada" | "apache" | "apex" | "apl" | "applescript" | "ara" | "asm" | "astro" | "awk" | "ballerina" | "bat" | "beancount" | "berry" | "bibtex" | "bicep" | "blade" | "c" | "cadence" | "clarity" | "clojure" | "cmake" | "cobol" | "codeql" | "coffee" | "cpp" | "crystal" | "csharp" | "css" | "cue" | "cypher" | "d" | "dart" | "dax" | "diff" | "docker" | "dream-maker" | "elixir" | "elm" | "erb" | "erlang" | "fish" | "fsharp" | "gdresource" | "gdscript" | "gdshader" | "gherkin" | "git-commit" | "git-rebase" | "glimmer-js" | "glimmer-ts" | "glsl" | "gnuplot" | "go" | "graphql" | "groovy" | "hack" | "haml" | "handlebars" | "haskell" | "hcl" | "hjson" | "hlsl" | "html" | "http" | "imba" | "ini" | "java" | "javascript" | "jinja-html" | "jison" | "json" | "json5" | "jsonc" | "jsonl" | "jsonnet" | "jssm" | "jsx" | "julia" | "kotlin" | "kusto" | "latex" | "less" | "liquid" | "lisp" | "logo" | "lua" | "make" | "markdown" | "marko" | "matlab" | "mdx" | "mermaid" | "narrat" | "nextflow" | "nginx" | "nim" | "nix" | "objective-c" | "objective-cpp" | "ocaml" | "pascal" | "perl" | "php" | "plsql" | "postcss" | "powerquery" | "powershell" | "prisma" | "prolog" | "proto" | "pug" | "puppet" | "purescript" | "python" | "r" | "raku" | "razor" | "reg" | "rel" | "riscv" | "rst" | "ruby" | "rust" | "sas" | "sass" | "scala" | "scheme" | "scss" | "shaderlab" | "shellscript" | "shellsession" | "smalltalk" | "solidity" | "sparql" | "sql" | "ssh-config" | "stata" | "stylus" | "svelte" | "swift" | "system-verilog" | "tasl" | "tcl" | "tex" | "toml" | "tsx" | "turtle" | "twig" | "txt" | "typescript" | "v" | "vb" | "verilog" | "vhdl" | "viml" | "vue-html" | "vue" | "vyper" | "wasm" | "wenyan" | "wgsl" | "wolfram" | "xml" | "xsl" | "yaml" | "zenscript";
 
-type LineNumber = number;
-type ColumnNumber = number;
-type MultiLineRange = {
-    fromLineNumber: LineNumber;
-    toLineNumber: LineNumber;
-};
-type InlineRange = {
-    lineNumber: LineNumber;
-    fromColumn: ColumnNumber;
-    toColumn: ColumnNumber;
-};
-type CodeRange = MultiLineRange | InlineRange;
-
 type Annotation = {
     name: string;
     query?: string;
@@ -121,7 +121,11 @@ declare function highlightSync(code: string, lang: LanguageAlias, themeOrThemeNa
 declare function highlightSync(code: string, lang: LanguageAlias, themeOrThemeName: Theme, config: AnnotatedConfig): AnnotatedLighterResult;
 declare function extractAnnotations(code: string, lang: LanguageAlias, annotationExtractor?: AnnotationExtractor): Promise<{
     code: string;
-    annotations: Annotation[];
+    annotations: {
+        ranges: CodeRange[];
+        name: string;
+        query?: string;
+    }[];
 }>;
 declare function getThemeColors(themeOrThemeName: Theme): Promise<{
     colorScheme: string;

lib/dist/index.cjs.js

@@ -3,6 +3,7 @@ import { Token } from "./annotations";
 import { highlightText, highlightTokens } from "./highlighter";
 import { CodeRange, parseRelativeRanges } from "./range";
 import { FinalTheme } from "./theme";
+import { blockRegexToRange, inlineRegexToRange } from "./regex-range";
 
 const PUNCTUATION = "#001";
 const COMMENT = "#010";
@@ -38,6 +39,8 @@ export type AnnotationData = {
   query?: string;
 };
 
+type RawAnnotation = AnnotationData & { lineNumber: number };
+
 export type AnnotationExtractor =
   | string[]
   | ((comment: string) => null | AnnotationData);
@@ -52,7 +55,7 @@ export function extractCommentsFromCode(
     ? highlightText(code)
     : highlightTokens(code, grammar, commentsTheme);
 
-  const allAnnotations: Annotation[] = [];
+  const allAnnotations: RawAnnotation[] = [];
 
   let lineNumber = 1;
   const newCode = lines
@@ -86,14 +89,24 @@ export function extractCommentsFromCode(
     .filter((line) => line !== null)
     .join(`\n`);
 
-  return { newCode, annotations: allAnnotations };
+  const annotations = allAnnotations
+    .map(({ rangeString, lineNumber, ...rest }) => ({
+      ...rest,
+      ranges: parseRangeString(rangeString, lineNumber, newCode),
+    }))
+    .filter((a) => a.ranges.length > 0);
+
+  return { newCode, annotations };
 }
 
 function getAnnotationsFromLine(
   tokens: Token[],
   annotationExtractor: AnnotationExtractor,
   lineNumber: number
-) {
+): {
+  annotations: RawAnnotation[];
+  lineWithoutComments: Token[] | null;
+} {
   // if no punctuation return empty
   if (!tokens.some((token) => token.style.color === PUNCTUATION)) {
     return { annotations: [], lineWithoutComments: tokens };
@@ -104,7 +117,8 @@ function getAnnotationsFromLine(
     tokens: Token[];
     name: string;
     query?: string;
-    ranges: CodeRange[];
+    rangeString: string;
+    lineNumber: number;
   }[] = [];
   let i = 0;
   while (i < tokens.length) {
@@ -144,7 +158,8 @@ function getAnnotationsFromLine(
       tokens: commentTokens,
       name,
       query,
-      ranges: parseRelativeRanges(rangeString, lineNumber),
+      rangeString,
+      lineNumber,
     });
 
     i += 2;
@@ -165,12 +180,26 @@ function getAnnotationsFromLine(
     annotations: comments.map((a) => ({
       name: a.name,
       query: a.query,
-      ranges: a.ranges,
+      lineNumber: a.lineNumber,
+      rangeString: a.rangeString,
     })),
     lineWithoutComments: newLine,
   };
 }
 
+function parseRangeString(
+  rangeString: string,
+  lineNumber: number,
+  code: string
+) {
+  if (rangeString && rangeString.startsWith("(/")) {
+    return blockRegexToRange(code, rangeString, lineNumber);
+  } else if (rangeString && rangeString.startsWith("[/")) {
+    return inlineRegexToRange(code, rangeString, lineNumber);
+  }
+  return parseRelativeRanges(rangeString, lineNumber);
+}
+
 function getAnnotationDataFromNames(content: string, names: string[]) {
   const regex = /\s*([\w-]+)?(\([^\)]*\)|\[[^\]]*\])?(.*)$/;
   const match = content.match(regex);

lib/dist/index.d.ts

@@ -0,0 +1,18 @@
+import { test, expect } from "vitest";
+import { extractor } from "./extractor";
+
+test("extraction", () => {
+  expectExtraction("!foo", ["foo", , ""]);
+  expectExtraction("!foo(1)", ["foo", "(1)", ""]);
+  expectExtraction("!foo[1]", ["foo", "[1]", ""]);
+  expectExtraction("!foo[   ] q q", ["foo", "[   ]", "q q"]);
+  expectExtraction("!foo(/bar/)", ["foo", "(/bar/)", ""]);
+  expectExtraction("!foo(/bar baz/g) 1 2", ["foo", "(/bar baz/g)", "1 2"]);
+  expectExtraction("!Focus(/y/) bar", ["Focus", "(/y/)", "bar"]);
+});
+
+function expectExtraction(comment: string, expected: [string, string, string]) {
+  const result = extractor(comment);
+  const [name, rangeString, query] = expected;
+  expect(result).toEqual({ name, rangeString, query });
+}

lib/dist/index.esm.mjs

@@ -0,0 +1,19 @@
+export function extractor(comment: string, prefix: string = "!") {
+  // const regex = /\s*(!?[\w-]+)?(\([^\)]*\)|\[[^\]]*\])?(.*)$/;
+
+  const regex = new RegExp(
+    `\\s*(${prefix}?[\\w-]+)?(\\([^\\)]*\\)|\\[[^\\]]*\\])?(.*)$`
+  );
+  const match = comment.match(regex);
+  const name = match[1];
+  const rangeString = match[2];
+  const query = match[3]?.trim();
+  if (!name || !name.startsWith(prefix)) {
+    return null;
+  }
+  return {
+    name: name.slice(prefix.length),
+    rangeString,
+    query,
+  };
+}

lib/dist/worker.esm.mjs

@@ -0,0 +1,123 @@
+import { expect, test } from "vitest";
+import { blockRegexToRange, inlineRegexToRange } from "./regex-range";
+
+test("block regex range", () => {
+  const code = `
+function x() {
+  console.log(1)
+  console.log(2)
+  if (true) {
+    console.log(3)
+  }
+}
+
+function y() {
+  console.log(1)
+}
+`.trim();
+  const result = blockRegexToRange(code, "(/^ {2}.*(?:\n {2}.*)*/gm)", 1);
+  expect(result).toMatchInlineSnapshot(`
+    [
+      {
+        "fromLineNumber": 2,
+        "toLineNumber": 6,
+      },
+      {
+        "fromLineNumber": 10,
+        "toLineNumber": 10,
+      },
+    ]
+  `);
+});
+
+test("block regex range repeated line", () => {
+  const code = `
+const foo = 1;
+const bar = 2
+foo = foo + bar;
+`.trim();
+  const result = blockRegexToRange(code, "(/foo/g)", 1);
+  expect(result).toMatchInlineSnapshot(`
+    [
+      {
+        "fromLineNumber": 1,
+        "toLineNumber": 1,
+      },
+      {
+        "fromLineNumber": 3,
+        "toLineNumber": 3,
+      },
+    ]
+  `);
+});
+
+test("inline regex range", () => {
+  const code = `
+const foo = 1;
+const bar = 2
+foo = foo + bar;
+`.trim();
+  const result = inlineRegexToRange(code, "[/foo/g]", 1);
+
+  expect(result).toMatchInlineSnapshot(`
+    [
+      {
+        "fromColumn": 7,
+        "lineNumber": 1,
+        "toColumn": 9,
+      },
+    ]
+  `);
+});
+
+test("inline regex range with capture group", () => {
+  const code = `
+function C() {
+  return <div className="bg-red-500">
+    <span className="text-blue-500"><a className="x">Hello</a></span>
+  </div>
+}
+`.trim();
+  const result = inlineRegexToRange(code, '[/className="(.*?)"/gm]', 1);
+
+  expect(result).toMatchInlineSnapshot(`
+    [
+      {
+        "fromColumn": 26,
+        "lineNumber": 2,
+        "toColumn": 35,
+      },
+      {
+        "fromColumn": 22,
+        "lineNumber": 3,
+        "toColumn": 34,
+      },
+      {
+        "fromColumn": 51,
+        "lineNumber": 3,
+        "toColumn": 51,
+      },
+    ]
+  `);
+});
+
+test("inline regex range with capture group not g", () => {
+  const code = `
+function C() {
+  return <div className="">
+    <span className="text-blue-500"><a className="x">Hello</a></span>
+  </div>
+}
+`.trim();
+  const result = inlineRegexToRange(code, '[/className="(.*?)"/]', 3);
+
+  expect(result).toMatchInlineSnapshot(`
+    [
+      {
+        "fromColumn": 22,
+        "lineNumber": 3,
+        "toColumn": 34,
+      },
+    ]
+  `);
+});