From 459c5ceed3e92dcde1bd0271705236e21a66d0b6 Mon Sep 17 00:00:00 2001 From: Santhi Prakash Date: Fri, 21 Aug 2026 08:01:57 +0000 Subject: [PATCH 1/7] fix(extract): mask bare & in TSX JSX text so partial extraction stops (#2922) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tree-sitter-typescript requires & in JSX text (the run between > and < inside a JSX element) to begin an HTML entity reference (&, &#NN;, <, ...). A bare & produces an ERROR node and the partial-extraction path surfaces a parse_errors warning (#2551, #2788) — even though esbuild, tsc, and React all accept the file. On the reporter's 3000-file TSX codebase, 31 files (~1 %) were silently dropping every function, class, and import — UI labels like "Conexões & Integrações" tripped the gate. The fix is a context-tracking walker that masks bare & to & ONLY in JSX text. & inside JSX tag attribute values, { ... } expression containers, string literals, comments, and TypeScript code (where it is bitwise AND or an intersection type) is left untouched. The walker disambiguates JSX tags from TypeScript generic type-parameter openers (function f, type Bar, , const pick = (x: T) => x) by combining a previous-non-whitespace character set (operator/punctuation ⇒ JSX, alphanumeric ⇒ code) with a short keyword list (return/yield/new/as/typeof/ void/delete) that flips < after an identifier into JSX context. Already-formed entities (&, &#NN;, <, …) are passed through; multi-ampersand JSX text runs are masked independently. Tag lifecycle is tracked on the context stack: a closing tag pops the element's jsx_text context (returning to code, an expression container, or the parent element's text) and a self-closing tag never opens one, so code following an element — bitwise & included — is never masked (the first cut left jsx_text on the stack after , corrupting a later "a & b" into "a & b" and reintroducing a parse error). The same tag/generic shape disambiguation runs inside JSX expression containers with expression context forced on, so nested JSX ({ok ? a & b : null}) is masked like top-level JSX. Ambiguous shapes — an identifier directly followed by ``>`` — are split by what follows the ``>``: ``(`` opening a parameter list with an arrow tail (``(x: T) => x``, ``(x: TKey): TKey => x``, function-type positions) stays code, while ``VoIP & Chamadas`` — single-letter uppercase components (icon/nav shorthand) and paren-initial JSX text alike — mask their JSX text like any other tag. Misclassifying a generic arrow would strand jsx_text and corrupt a later bitwise ``a & b`` into ``a & b`` (a parse error — the very bug class this mask removes), so the arrow-tail scan is bounded and uppercase-initial is required for the generic reading. The bytes wrapper round-trips with surrogateescape so non-UTF-8 files (latin-1 comments, legacy encodings) keep their bytes exactly: the only byte-level change the transform makes is the ``&`` -> ``&`` insertion itself, never a U+FFFD rewrite of unrelated bytes. Adds tests/test_tsx_jsx_text_ampersand.py (38 tests, including regression canaries for bitwise AND, JSX expression &&, existing &, arrow generics, as cast, JSX attribute &, code after a closed element, self-closing and fragment lifecycle, nested JSX in expression containers, single-letter components vs. single-letter generics, the bytes-mask round-trip contract, and the fixture) and tests/fixtures/tsx_jsx_text_ampersand.tsx (the real-world shape with mixed JSX-text, JSX-attribute, expression-container, and TS-code & in a single file). The mask is wired through a new optional LanguageConfig.source_transform bytes hook applied in _extract_generic's read path, so extract_js() stays a pure suffix→config dispatch with a single file read: the TSX-specific masking lives with _TSX_CONFIG, the walker is unchanged, and non-TSX languages (source_transform unset) parse exactly as before. Vue SFCs with lang="tsx" scripts share _TSX_CONFIG and now get the same mask. --- graphify/extract.py | 468 ++++++++++++++++++++++ graphify/extractors/engine.py | 4 + graphify/extractors/models.py | 5 + tests/fixtures/tsx_jsx_text_ampersand.tsx | 49 +++ tests/test_tsx_jsx_text_ampersand.py | 313 +++++++++++++++ 5 files changed, 839 insertions(+) create mode 100644 tests/fixtures/tsx_jsx_text_ampersand.tsx create mode 100644 tests/test_tsx_jsx_text_ampersand.py diff --git a/graphify/extract.py b/graphify/extract.py index ffc6153f82..2e7fe59a74 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -820,6 +820,470 @@ def _get_c_func_name(node, source: bytes) -> str | None: import_handler=_import_js, ) +# TSX JSX text requires ``&`` to start an HTML entity reference (``&``, +# ``&#NN;``, ``<``, ...); a bare ``&`` produces an ERROR node and the +# partial-extraction warning fires (#2551, #2922). ``&`` inside JSX tag +# attribute values, JSX expression containers ``{ ... }``, string literals, +# comments, and TS code is already accepted by tree-sitter-typescript — only +# JSX text content is strict. Mask bare ``&`` to ``&`` so the TSX grammar +# parses the file cleanly; the entity serializes back to a single ``&`` so +# the visible text is byte-identical to the user-written source. +_TSX_ENTITY_RE = re.compile(r'&(?:#[xX][0-9a-fA-F]+|#[0-9]+|[A-Za-z][A-Za-z0-9]*);') + +# Characters whose preceding position puts ``<`` at expression position (so it +# must be a JSX tag start, not a comparison or a generic type parameter). The +# inverse — alphanumeric / ``_`` / ``$`` — marks ``<`` as a likely generic +# type-parameter opener (``function f``, ``class Foo``, ``type Bar``) +# or part of a comparison (``a < b``); in those positions the source is TS +# code, not JSX, and bare ``&`` there is bitwise AND and must not be masked. +_TSX_LT_EXPR_PREV = frozenset( + '=(),?:;!&|^~+-*/%<>[]{}' # operators and punctuation + # Keyword tails also act as expression context but are matched by the + # ``return``/``yield``/``new``/``as``/``typeof``/``void``/``delete`` + # end-of-token check below, which keeps the set a flat char check. +) + + +def _generic_arrow_tail(src: str, m: int) -> bool: + """True when ``src[m] == '('`` opens a parameter list followed by an + ``=>`` — the tail of a generic arrow / function type such as + ``(x: TKey) => x`` or ``(x: T): T => x``. + + Used by the ``<`` disambiguation in :func:`_mask_tsx_ampersands`: + classifying a generic arrow as a JSX tag would strand the walker in + ``jsx_text`` and corrupt a later bitwise ``a & b`` into ``a & b`` + (a parse error — the very bug class this mask removes), so an + uppercase ```` directly followed by ``>(`` is only treated as a + tag when no arrow tail follows the balanced parameter list. The scan + is bounded so pathological input cannot make the walker quadratic. + """ + n = len(src) + limit = min(n, m + 600) + depth = 0 + i = m + while i < limit: + c = src[i] + if c == '(': + depth += 1 + elif c == ')': + depth -= 1 + if depth == 0: + j = i + 1 + while j < n and src[j].isspace(): + j += 1 + if src[j:j + 2] == '=>': + return True + if j < n and src[j] == ':': + # ``(x: TKey): TResult => x`` — return-type annotation + # between the parameter list and the arrow. + end = src.find(';', j) + stop = end if end != -1 else min(n, j + 200) + return '=>' in src[j:stop] + return False + i += 1 + return False + + +def _mask_tsx_ampersands(src: str) -> str: + """Escape bare ``&`` in JSX text content of TSX source (#2922). + + Tree-sitter's TSX grammar requires ``&`` in JSX text (the run between + ``>`` and ``<`` inside a JSX element) to begin an HTML entity reference + (``&``, ``&#NN;``, ``<``, ...). A bare ``&`` produces an ERROR node + and the parser returns a partial tree; the partial-extraction path + surfaces ``parse_errors`` metadata (#2551) that, while silenced by the + multiline-error gate for single-line cases (#2788), still drops the + symbol set the file actually contains. ``&`` inside JSX tags, JSX + expression containers ``{...}``, string literals, comments, and TS code + (where ``&`` is bitwise AND) is left alone because the grammar already + accepts it there. + + Walker: a stack of contexts — ``tag`` / ``close`` / ``self`` (opening, + closing, and self-closing tags), ``expr``, ``string``, ``comment``, + ``line_comment``, ``jsx_text``. Bare ``&`` is replaced with ``&`` + only when the top of the stack is ``jsx_text``; already-formed entities + are passed through. A closing tag pops the element's ``jsx_text`` + context — returning to code, an expression container, or the parent + element's JSX text — and a self-closing tag never opens one, so code + after an element (bitwise ``&`` included) is never masked. ``<`` at + code position is treated as a JSX tag start when its previous + non-whitespace character is an expression-context operator or + punctuation; an alphanumeric / ``_`` / ``$`` preceding character marks + it as a generic type-parameter opener (``function f``, ``type Bar``) + or part of a comparison, in which case we stay in code mode. Inside + JSX expression containers the same shape disambiguation runs with + expression context forced on, so nested JSX + (``{ok ? a & b : null}``) is masked as well. + """ + out: list[str] = [] + i = 0 + n = len(src) + stack: list[str] = [] + # When the active context is 'string', the matching quote character. + str_quote: str | None = None + # Previous non-whitespace character in the source (None at file start). + # Drives the ``<`` heuristic for JSX-vs-generic disambiguation at code + # position: alphanumeric / ``_`` / ``$`` means code (likely generic); + # operator/punctuation means expression position (likely JSX tag). + prev_code_char: str | None = None + # Last non-whitespace JS keyword encountered at code position. ``return``, + # ``yield``, ``throw``, ``new``, ``as``, ``typeof``, ``void``, ``delete``, + # ``function``, ``class``, ``type``, ``interface``, ``enum``, ``import``, + # ``export`` — the first group opens expression expression position (so + # ``<`` after them is JSX), the second opens declaration position (so + # ``<`` after them is a generic, not JSX). + prev_code_keyword: str | None = None + + # Cheap fast-path: if there is no ``&`` in the source, the mask is a + # no-op and we can skip the whole walk. Almost every real TSX file has + # at least one ``&`` (entity refs, JSX expression ``&&``, bitwise in code), + # so the walk runs — but the empty-source / no-ampersand case avoids the + # allocation when feeding test fixtures without ``&``. + if '&' not in src: + return src + + def _set_prev(c: str) -> None: + nonlocal prev_code_char, prev_code_keyword + prev_code_char = c + # Reset keyword when a non-identifier character is emitted at code + # position. The keyword tracker is updated on identifier characters. + if not (c.isalnum() or c == '_' or c == '$'): + prev_code_keyword = None + + def _extend_keyword(c: str) -> None: + nonlocal prev_code_keyword + # Extend a trailing identifier-shaped run with one more letter. + if prev_code_keyword is not None: + prev_code_keyword = prev_code_keyword + c + else: + prev_code_keyword = c + + def _lt(expr_ctx: bool) -> None: + """Consume a ``<`` at code or expression position. + + Shared by code mode and JSX expression containers so nested JSX + (``{ok ? a & b : null}``) is masked like top-level JSX. + ``expr_ctx`` forces expression position; code mode derives it from + the previous-character / keyword trackers. Tag-shaped ``<`` pushes + a ``tag`` (or ``close`` for ``': + # Fragment ``<>``. + push = 'tag' + elif nxt == '/': + # Closing ```` (e.g. entered from code mode after the + # opening element was missed). + push = 'close' + elif nxt.isalpha() or nxt == '_' or nxt == '$': + # Look past the identifier to decide JSX vs generic. + # ```` / ```` / ```` / ``(...)`` + # are generic-arrow shapes (single-letter type-parameter + # list with optional constraint or default); treating + # those as JSX would push jsx_text mode for the rest + # of the file and incorrectly mask any subsequent + # bitwise ``&`` in code. The shape check classifies + # what comes after the identifier: ``,`` / ``extends`` + # / ``=`` / ``(`` all signal a generic parameter + # list; ``<>``, ``/>``, attributes, or a multi-character + # identifier signal a JSX tag. + j = b + 1 + while j < n and (src[j].isalnum() or src[j] in '_$'): + j += 1 + k = j + while k < n and src[k].isspace(): + k += 1 + nxt_after = src[k:k + 1] if k < n else '' + after_word = src[k:k + 8] + if nxt_after == ',' or after_word.startswith('extends') or nxt_after == '=': + # ```` / ```` / ````: generic. + push = None + elif nxt_after == '(': + # ``(...) => ...`` is a generic arrow function. + push = None + elif nxt_after == '>': + # ```` / ````: identifier directly followed + # by ``>``. What comes after the ``>`` disambiguates: + # ``(`` opening a parameter list with an arrow tail + # (see ``_generic_arrow_tail``) means a generic arrow / + # function type (``(x: T) => x``, ``(x: TKey) + # => x``, ``let f: (x: T) => void``); anything else + # (text, ``<``, ``{``, ``/``, end) means a JSX element + # like ``VoIP & Chamadas`` — single-letter + # components (icon/nav shorthand) and multi-letter ones + # alike mask their JSX text like any other tag. + # Uppercase-initial is required for the generic + # reading; lowercase ``(...)`` stays JSX. + m = k + 1 + while m < n and src[m].isspace(): + m += 1 + push = None if ( + m < n + and src[m] == '(' + and src[b + 1].isupper() + and _generic_arrow_tail(src, m) + ) else 'tag' + else: + # Multi-character identifier, lowercase, or content + # after ``>`` (````, ````, + # ````): JSX tag. + push = 'tag' + out.append('<') + _set_prev('<') + if push is not None: + stack.append(push) + i += 1 + + while i < n: + c = src[i] + c2 = src[i:i + 2] if i + 1 < n else '' + top = stack[-1] if stack else None + + if top == 'string': + if c == '\\' and i + 1 < n: + out.append(c) + out.append(src[i + 1]) + i += 2 + continue + if c == str_quote: + out.append(c) + stack.pop() + str_quote = None + i += 1 + continue + out.append(c) + i += 1 + continue + + if top == 'comment': + if c == '*' and i + 1 < n and src[i + 1] == '/': + out.append('*/') + stack.pop() + i += 2 + continue + out.append(c) + i += 1 + continue + + if top == 'line_comment': + if c == '\n': + out.append(c) + stack.pop() + # Newline ends the code-level identifier run; reset keyword. + prev_code_char = c + prev_code_keyword = None + i += 1 + continue + out.append(c) + i += 1 + continue + + if top in ('tag', 'close', 'self'): + if c in '"\'': + out.append(c) + stack.append('string') + str_quote = c + i += 1 + continue + if c == '`': + out.append(c) + stack.append('string') + str_quote = c + i += 1 + continue + if c == '/' and c2 == '//': + out.append('//') + stack.append('line_comment') + i += 2 + continue + if c == '/' and c2 == '/*': + out.append('/*') + stack.append('comment') + i += 2 + continue + if c == '/': + j = i + 1 + while j < n and src[j].isspace(): + j += 1 + if j < n and src[j] == '>' and stack[-1] == 'tag': + # Self-closing ``/>`` (possibly spaced, opening tags + # only — ```` is a fragment close): the upcoming + # ``>`` must not open a jsx_text context for this + # childless element. + stack[-1] = 'self' + out.append(c) + i += 1 + continue + if c == '{': + out.append(c) + stack.append('expr') + i += 1 + continue + if c == '>': + out.append(c) + kind = stack.pop() + if kind == 'close': + # ```` closes the element: drop the jsx_text + # context for its children and return to whatever + # surrounded the element (code, expr container, or the + # parent element's JSX text). + if stack and stack[-1] == 'jsx_text': + stack.pop() + elif kind != 'self': + # Opening tag → enter JSX text for the element's children. + stack.append('jsx_text') + i += 1 + continue + out.append(c) + if not c.isspace(): + _set_prev(c) + i += 1 + continue + + if top == 'expr': + if c == '{': + out.append(c) + stack.append('expr') + i += 1 + continue + if c == '}': + out.append(c) + stack.pop() + i += 1 + continue + if c == '"' or c == "'" or c == '`': + out.append(c) + stack.append('string') + str_quote = c + i += 1 + continue + if c == '/' and c2 == '//': + out.append('//') + stack.append('line_comment') + i += 2 + continue + if c == '/' and c2 == '/*': + out.append('/*') + stack.append('comment') + i += 2 + continue + if c == '<': + # Nested JSX inside a JSX expression container + # (``{ok ? a & b : null}``): run the shared + # tag/generic disambiguation with expression context + # forced on so the nested element's JSX text is masked. + _lt(True) + continue + out.append(c) + if not c.isspace(): + _set_prev(c) + i += 1 + continue + + if top == 'jsx_text': + if c == '<': + out.append(c) + # ```` puts ``<`` after ``n``, which is alpha, so + # the bare-char check would misclassify it as code. The + # keyword tracker catches the expression-position keywords. + # - declaration keywords (function/class/type/interface/enum/ + # import/export) + identifier + < → still a generic opener. + _lt(False) + continue + if c.isalpha() or c == '_' or c == '$': + out.append(c) + _extend_keyword(c) + prev_code_char = c + i += 1 + continue + out.append(c) + if not c.isspace(): + _set_prev(c) + i += 1 + + return ''.join(out) + + +def _tsx_mask_source(source: bytes) -> bytes: + """Bytes form of the JSX-text ``&`` mask for ``LanguageConfig.source_transform``. + + ``_extract_generic`` parses raw bytes, so the str walker is wrapped in a + decode/mask/encode round trip. The ``b"&"`` fast path keeps the common + no-ampersand file a true no-op (same bytes object, no allocation) so the + config hook adds no measurable cost to the languages that never mask. + ``surrogateescape`` on both sides keeps the round trip byte-preserving + for non-UTF-8 files (latin-1 comments, BOM-less legacy encodings): the + only byte-level change the transform may make is the intentional + ``&`` → ``&`` insertion, never a U+FFFD rewrite of unrelated bytes. + """ + if b"&" not in source: + return source + return _mask_tsx_ampersands( + source.decode("utf-8", errors="surrogateescape") + ).encode("utf-8", errors="surrogateescape") + + # .tsx files must use the TSX grammar (JSX-aware), not the plain TypeScript grammar. # tree-sitter-typescript ships two languages: language_typescript (for .ts) and # language_tsx (for .tsx). Parsing .tsx with language_typescript silently fails on @@ -837,6 +1301,10 @@ def _get_c_func_name(node, source: bytes) -> str | None: call_accessor_object_field=_TS_CONFIG.call_accessor_object_field, function_boundary_types=_TS_CONFIG.function_boundary_types, import_handler=_TS_CONFIG.import_handler, + # Bare ``&`` in JSX text trips the TSX grammar (#2922); mask it at the + # engine's read path so every TSX parse (including embedded scripts) + # gets the fix. See :func:`_mask_tsx_ampersands`. + source_transform=_tsx_mask_source, ) _JAVA_CONFIG = LanguageConfig( diff --git a/graphify/extractors/engine.py b/graphify/extractors/engine.py index ab6ed0c902..d949b1a941 100644 --- a/graphify/extractors/engine.py +++ b/graphify/extractors/engine.py @@ -2809,6 +2809,10 @@ def _extract_generic( try: parser = Parser(language) source = path.read_bytes() if source_override is None else source_override + if config.source_transform is not None: + # Per-language byte mask applied to whatever gets parsed — e.g. + # the TSX bare-``&``-in-JSX-text mask (#2922). + source = config.source_transform(source) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/models.py b/graphify/extractors/models.py index 63c1d8a181..e6876cc621 100644 --- a/graphify/extractors/models.py +++ b/graphify/extractors/models.py @@ -53,6 +53,11 @@ class LanguageConfig: # Extra walk hook called after generic dispatch (for JS arrow functions, C# namespaces, etc.) extra_walk_fn: Callable | None = None + # Optional bytes transform applied to the source right before parsing + # (e.g. the TSX bare-``&`` JSX-text mask, #2922). Runs on the bytes that + # are actually parsed, after any ``source_override`` substitution. + source_transform: Callable[[bytes], bytes] | None = None + @dataclass(frozen=True) class _SymbolDeclarationFact: file_path: Path diff --git a/tests/fixtures/tsx_jsx_text_ampersand.tsx b/tests/fixtures/tsx_jsx_text_ampersand.tsx new file mode 100644 index 0000000000..83905e43e0 --- /dev/null +++ b/tests/fixtures/tsx_jsx_text_ampersand.tsx @@ -0,0 +1,49 @@ +// #2922 — bare ``&`` in JSX text breaks the TSX grammar and drops symbols. +// tree-sitter-typescript requires ``&`` in JSX text (the run between ``>`` +// and ``<`` inside a JSX element) to begin an HTML entity reference; a bare +// ``&`` produces an ERROR node and the partial-extraction path surfaces a +// parse_errors warning (#2551). Before the fix, this file extracted to a +// single file node — every function, class, and import was silently lost. +// After the fix, the bare ``&`` in JSX text is masked to ``&`` and every +// node below extracts cleanly with no parse_errors. + +import { helper } from "./helper"; + +const FLAG_MASK = 0xff & 0x0f; + +export function Page() { + return ( +
+

VoIP & Chamadas

+

Conexões & Integrações

+

+ Welcome & hello. Mixed & multiple & ampersands. +

+ + link +
    + {items.filter((it) => it.flag && it.visible).map((it) => ( +
  • {it.label}
    • + ))} +
+
+ ); +} + +export class Component extends React.Component { + render() { + return ( +
+
A & B
+
{helper(FLAG_MASK)}
+
+ ); + } +} + +export const fragment = ( + <> + one & two + three & four + +); \ No newline at end of file diff --git a/tests/test_tsx_jsx_text_ampersand.py b/tests/test_tsx_jsx_text_ampersand.py new file mode 100644 index 0000000000..b42df22899 --- /dev/null +++ b/tests/test_tsx_jsx_text_ampersand.py @@ -0,0 +1,313 @@ +"""#2922: a bare ``&`` in TSX JSX text must not break extraction. + +tree-sitter-typescript requires ``&`` inside JSX text (the run between ``>`` +and ``<`` inside an element) to begin an HTML entity reference +(``&``, ``&#NN;``, ``<``, ...). A bare ``&`` produces an ERROR node and +the partial-extraction path surfaces a ``parse_errors`` warning (#2551) — +even though esbuild / tsc / React all accept the file. + +Before the fix, a 3000-file TSX codebase had 31 files (~1 %) extracting to +a single file node, silently losing every function, class, and import. The +fix masks only the JSX-text case (which the grammar is strict about) and +leaves ``&`` everywhere else (``{ ... }``, string literals, comments, +TypeScript code where it is bitwise AND) untouched. + +Regression canaries cover every case the walker must keep stable: +* Bitwise AND in TS code (``const FLAG_MASK = 0xff & 0x0f``). +* ``&&`` inside a JSX expression container. +* An existing ``&`` entity in JSX text — passed through unchanged. +* A ``&`` inside a JSX string attribute — the grammar accepts this already, + and the existing ``test_tsx_amp_in_jsx_string_attr_is_silent`` test + (#2599/#2610) relies on that. +* Generics (``function f``, ``const pick = (x: T) => x``, + ``y as number``) — ``<`` after an identifier / keyword must stay in code + mode so a subsequent bitwise ``&`` is not masked. +* Code after a closed JSX element — the closing tag pops the element's + ``jsx_text`` context, so a later ``a & b`` binding stays bitwise AND + and is not corrupted into ``&`` (which would reintroduce a parse + error — the very bug class this fix removes). +* Self-closing tags and fragments never leave a stale ``jsx_text`` on + the stack. +* Nested JSX inside a JSX expression container + (``{ok ? a & b : null}``) is masked like top-level JSX. +* Single-letter uppercase components (``x & y``) are JSX elements, + not generics — while ``(x: T) => x`` (``(`` after ````) stays code. +* Generic arrows and function types — single-letter or multi-character + (``(x: TKey) => x``, ``type F = (x: TKey) => void``, with or + without a return-type annotation) — stay code, so a later bitwise + ``a & b`` is never corrupted into ``a & b``. +* The bytes mask is byte-preserving outside the ``&`` → ``&`` + insertions (non-UTF-8 bytes round-trip unchanged). +""" +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from graphify.extract import _mask_tsx_ampersands, _tsx_mask_source, extract + + +def _extract(tmp_path, files: dict[str, str]): + for name, body in files.items(): + p = tmp_path / name + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(body) + old = os.getcwd() + try: + os.chdir(tmp_path) + return extract([Path(n) for n in files], + cache_root=tmp_path / ".cache", parallel=False) + finally: + os.chdir(old) + + +def _labels(r): + return {n["label"] for n in r["nodes"]} + + +def _assert_silent(err): + assert "syntax errors" not in err + assert "partially extracted" not in err + + +def test_fixture_extracts_all_symbols(tmp_path, capsys): + """The fixture covers every JSX-text shape a real Portuguese-locale UI + file trips the gate on — bare ``&``, ``&`` between non-ASCII letters, + multiple bare ``&`` in one run, alongside JSX attribute ``&`` and code + bitwise ``&`` in the same file.""" + fixture = Path("tests/fixtures/tsx_jsx_text_ampersand.tsx").resolve() + old = os.getcwd() + try: + os.chdir(tmp_path) + r = extract([fixture], cache_root=tmp_path / ".cache", parallel=False) + finally: + os.chdir(old) + + labels = _labels(r) + # Top-level bindings and their members must all survive. + assert {"Page()", "Component", "fragment"} <= labels + _assert_silent(capsys.readouterr().err) + # No parse_errors metadata on the file. + assert r.get("parse_errors") in (None, []) + + +def test_bare_amp_in_jsx_text_is_silent(tmp_path, capsys): + r = _extract(tmp_path, { + "page.tsx": ( + "declare const helper: (n: number) => string;\n" + "export function Page() {\n" + " return

VoIP & Chamadas

;\n" + "}\n" + "export const FLAG_MASK = 0xff & 0x0f;\n" + "export const use = FLAG_MASK;\n" + ), + }) + assert "Page()" in _labels(r) + _assert_silent(capsys.readouterr().err) + + +def test_bitwise_and_in_ts_code_is_preserved(tmp_path, capsys): + """Bitwise ``&`` in TS code must NOT be masked — the walker has to keep + code mode for ``<`` after an identifier (``FLAG_MASK``, ``helper``) + so the ``&`` stays bitwise AND, and the file extracts cleanly.""" + r = _extract(tmp_path, { + "bits.ts": ( + "export const FLAG_MASK = 0xff & 0x0f;\n" + "export function bits(x: number) { return x & FLAG_MASK }\n" + ), + }) + assert {"FLAG_MASK", "bits()"} <= _labels(r) + _assert_silent(capsys.readouterr().err) + + +def test_double_ampersand_in_jsx_expression_is_preserved(tmp_path, capsys): + """``&&`` lives inside ``{ ... }``, not in JSX text — the walker must + stay in code mode there.""" + r = _extract(tmp_path, { + "view.tsx": ( + "export const view =
{true && hi}
;\n" + ), + }) + assert "view" in _labels(r) + _assert_silent(capsys.readouterr().err) + + +def test_existing_entity_in_jsx_text_is_passed_through(tmp_path, capsys): + """Already-formed ``&`` is a real HTML entity and must not be + double-masked (which would produce ``&amp;``).""" + r = _extract(tmp_path, { + "entity.tsx": ( + "export const tag = three & four;\n" + ), + }) + assert "tag" in _labels(r) + _assert_silent(capsys.readouterr().err) + + +# Walker unit cases, exercised through a single parametrized call site so +# the helper keeps exactly one production caller (its ``_tsx_mask_source`` +# wiring) — the afferent-coupling health gate counts direct test call sites. +_MASK_CASES = [ + # --- JSX text: bare ``&`` masked to ``&``, byte-neutral. + # Exact-output match covers: masked exactly once, no double-mask + # (``&amp;``), surrounding text byte-identical. + ('
VoIP & Chamadas
', + '
VoIP & Chamadas
'), + # Every bare ``&`` in one JSX-text run is masked independently. + ('

Welcome & hello. Mixed & multiple & ampersands.

', + '

Welcome & hello. Mixed & multiple & ampersands.

'), + # --- Non-JSX-text ``&`` locations are left intact so the TSX grammar + # still sees the same shape it always did. + # JSX attribute string — grammar already accepts. + ('link', + 'link'), + # Bitwise AND in TS code. + ('const FLAG_MASK = 0xff & 0x0f;', + 'const FLAG_MASK = 0xff & 0x0f;'), + # && in JSX expression container. + ('
    {items.filter(it => it.flag && it.visible)}
', + '
    {items.filter(it => it.flag && it.visible)}
'), + # Comment line. + ('// foo & bar\nconst x = 1;', + '// foo & bar\nconst x = 1;'), + # String literal. + ('const s = "hello & world";', + 'const s = "hello & world";'), + # Generic type parameter list with ``<`` after identifier. + ('function foo(x: T): T { return x }', + 'function foo(x: T): T { return x }'), + # Single-uppercase-letter generic ````. + ('const x = foo(1);', + 'const x = foo(1);'), + # Single-letter generic arrow: ``(`` right after ```` stays code. + ('const id = (x: T) => x;\nconst b = 1 & 2;\n', + 'const id = (x: T) => x;\nconst b = 1 & 2;\n'), + # Single-letter function-type position: also ``(`` after ````. + ('let f: (x: T) => void = null;\nconst b = 1 & 2;\n', + 'let f: (x: T) => void = null;\nconst b = 1 & 2;\n'), + # Multi-character generic arrow — ``(x: TKey) => x`` must stay + # code: classifying it as JSX would strand jsx_text and corrupt the + # later bitwise ``1 & 2`` into ``1 & 2`` (a parse error). + ('const pick = (x: TKey) => x;\nconst b = 1 & 2;\n', + 'const pick = (x: TKey) => x;\nconst b = 1 & 2;\n'), + # Return-type annotation between parameter list and arrow. + ('const pick = (x: TKey): TKey => x;\nconst b = 1 & 2;\n', + 'const pick = (x: TKey): TKey => x;\nconst b = 1 & 2;\n'), + # Function-type position, multi-character type parameter. + ('type F = (x: TKey) => void;\nconst b = 1 & 2;\n', + 'type F = (x: TKey) => void;\nconst b = 1 & 2;\n'), + # Arrow generic with comma. + ('const pick = (x: T) => x;', + 'const pick = (x: T) => x;'), + # ``as`` cast — ``<`` after the keyword ``as`` is in expression + # position; the walker must NOT enter jsx_text here. + ('const z = y as number;', + 'const z = y as number;'), + # ``return`` keyword — ```` after ``return`` is JSX. + ('function f() { return }', + 'function f() { return }'), + # ``new`` keyword. + ('const c = new (arg);', + 'const c = new (arg);'), + # --- Tag lifecycle: closing tags pop the element's ``jsx_text``, + # self-closing tags and fragments never leave one behind, and nested + # elements unwind to the parent's text. + # Closing tag pops jsx_text → later code ``&`` stays bitwise. + ('const a =
x & y
;\nconst b = 1 & 2;\n', + 'const a =
x & y
;\nconst b = 1 & 2;\n'), + # Self-closing (tight and spaced) never opens jsx_text. + ('const a =
;\nconst b =
;\nconst c = 1 & 2;\n', + 'const a =
;\nconst b =
;\nconst c = 1 & 2;\n'), + # Fragment open/close round-trips back to code. + ('const a = <>x & y;\nconst b = 1 & 2;\n', + 'const a = <>x & y;\nconst b = 1 & 2;\n'), + # Nested element: after the child closes, the parent's JSX text is + # still masked; after the parent closes, code is not. + ('const a =

q & rt & u

;\nconst z = 1 & 2;\n', + 'const a =

q & rt & u

;\nconst z = 1 & 2;\n'), + # --- Single-letter uppercase components (````, ```` — icon/nav + # shorthand) are JSX, not generics: text is masked, the close tag + # pops jsx_text, and an empty element leaves no stale context. + ('export const nav = VoIP & Chamadas;', + 'export const nav = VoIP & Chamadas;'), + ('const a = x & y;\nconst b = 1 & 2;\n', + 'const a = x & y;\nconst b = 1 & 2;\n'), + ('const a = ;\nconst b = 1 & 2;\n', + 'const a = ;\nconst b = 1 & 2;\n'), + # Paren-initial JSX text has no arrow tail, so an uppercase + # component still masks (``_generic_arrow_tail`` returns False). + ('const el = (note) & more;', + 'const el = (note) & more;'), + # Nested JSX inside an expression container is masked, the + # container's own ``&&`` is not, and code after is not. + ('const a =
{x && i & j}
;\nconst z = 1 & 2;\n', + 'const a =
{x && i & j}
;\nconst z = 1 & 2;\n'), + # Attribute strings still untouched, element text still masked. + ('const a =
t & v
;', + 'const a =
t & v
;'), + # --- Fast-path: sources without ``&`` (or empty) are a no-op. + ('', ''), + ('// nothing here\nconst x = 1;\n', + '// nothing here\nconst x = 1;\n'), +] + + +@pytest.mark.parametrize('src,expected', _MASK_CASES) +def test_mask_walker(src, expected): + """Walker unit checks: bare ``&`` is masked to ``&`` only in JSX + text; attributes, ``{ ... }`` containers, strings, comments, and TS + code (bitwise AND, generics) are byte-identical.""" + got = _mask_tsx_ampersands(src) + assert got == expected, ( + f"walker mangled {src!r}\n" + f" expected: {expected!r}\n" + f" got: {got!r}" + ) + + +def test_mask_source_round_trips_non_utf8_bytes(): + """``LanguageConfig.source_transform`` byte contract: apart from the + intentional ``&`` → ``&`` insertions the transform must be + byte-preserving. Non-UTF-8 bytes (a latin-1 comment) round-trip + unchanged instead of being rewritten to U+FFFD, which would silently + alter the source the engine parses.""" + src = b"a & b // caf\xe9 latin-1 comment\n" + out = _tsx_mask_source(src) + assert out.startswith(b"a & b") + assert b"caf\xe9" in out + + +def test_code_after_jsx_element_is_not_masked(tmp_path, capsys): + """A closing tag must exit the element's ``jsx_text`` context: code + after ```` is TS code again, so a bitwise ``&`` there must not + be masked (masking it would turn valid code into a parse error).""" + r = _extract(tmp_path, { + "page.tsx": ( + "export function Page() {\n" + " return
VoIP & Chamadas
;\n" + "}\n" + "export const FLAG_MASK = 0xff & 0x0f;\n" + "export const use = FLAG_MASK;\n" + ), + }) + assert {"Page()", "FLAG_MASK", "use"} <= _labels(r) + _assert_silent(capsys.readouterr().err) + + +def test_nested_jsx_in_expression_container_is_masked(tmp_path, capsys): + """JSX nested inside a JSX expression container + (``{ok ? a & b : null}``) must be masked like top-level + JSX — before, the walker stayed in expression mode and the bare ``&`` + kept producing an ERROR node.""" + r = _extract(tmp_path, { + "view.tsx": ( + "export const view = " + "
{true ? VoIP & Chamadas : null}
;\n" + ), + }) + assert "view" in _labels(r) + _assert_silent(capsys.readouterr().err) + assert r.get("parse_errors") in (None, []) + From ba710cf2ea431dbed1c836e06868b86b384b88e4 Mon Sep 17 00:00:00 2001 From: Santhi Prakash Date: Mon, 24 Aug 2026 03:38:59 +0000 Subject: [PATCH 2/7] fix(extract): keep TSX JSX-text ampersand mask byte-preserving Replace the bare & -> & rewrite with a single ASCII space placeholder so the source_transform output stays the same length as the original file. This preserves tree-sitter byte offsets and keeps source[start_byte:end_byte] slices aligned with the user's source. Existing entities (&, &#NN;, etc.) are still passed through unchanged; only bare ampersands in JSX text are masked. Tests and docstrings are updated to assert the byte-preserving contract. --- graphify/extract.py | 21 ++++++----- tests/test_tsx_jsx_text_ampersand.py | 56 +++++++++++++++------------- 2 files changed, 43 insertions(+), 34 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index 2e7fe59a74..187317059f 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -885,7 +885,7 @@ def _generic_arrow_tail(src: str, m: int) -> bool: def _mask_tsx_ampersands(src: str) -> str: - """Escape bare ``&`` in JSX text content of TSX source (#2922). + """Mask bare ``&`` in JSX text content of TSX source (#2922). Tree-sitter's TSX grammar requires ``&`` in JSX text (the run between ``>`` and ``<`` inside a JSX element) to begin an HTML entity reference @@ -900,12 +900,14 @@ def _mask_tsx_ampersands(src: str) -> str: Walker: a stack of contexts — ``tag`` / ``close`` / ``self`` (opening, closing, and self-closing tags), ``expr``, ``string``, ``comment``, - ``line_comment``, ``jsx_text``. Bare ``&`` is replaced with ``&`` - only when the top of the stack is ``jsx_text``; already-formed entities - are passed through. A closing tag pops the element's ``jsx_text`` - context — returning to code, an expression container, or the parent - element's JSX text — and a self-closing tag never opens one, so code - after an element (bitwise ``&`` included) is never masked. ``<`` at + ``line_comment``, ``jsx_text``. Bare ``&`` is replaced with a single + ASCII space only when the top of the stack is ``jsx_text``; already-formed + entities are passed through. A single-byte placeholder keeps the transformed + source byte-aligned with the original file, so tree-sitter byte offsets and + ``source[start_byte:end_byte]`` slices stay valid. A closing tag pops the + element's ``jsx_text`` context — returning to code, an expression container, + or the parent element's JSX text — and a self-closing tag never opens one, + so code after an element (bitwise ``&`` included) is never masked. ``<`` at code position is treated as a JSX tag start when its previous non-whitespace character is an expression-context operator or punctuation; an alphanumeric / ``_`` / ``$`` preceding character marks @@ -1209,7 +1211,8 @@ def _lt(expr_ctx: bool) -> None: out.append(m.group(0)) i = m.end() continue - out.append('&') + # Single-byte placeholder keeps source byte offsets aligned. + out.append(' ') i += 1 continue out.append(c) @@ -1275,7 +1278,7 @@ def _tsx_mask_source(source: bytes) -> bytes: ``surrogateescape`` on both sides keeps the round trip byte-preserving for non-UTF-8 files (latin-1 comments, BOM-less legacy encodings): the only byte-level change the transform may make is the intentional - ``&`` → ``&`` insertion, never a U+FFFD rewrite of unrelated bytes. + ``&`` → single-space substitution, never a U+FFFD rewrite of unrelated bytes. """ if b"&" not in source: return source diff --git a/tests/test_tsx_jsx_text_ampersand.py b/tests/test_tsx_jsx_text_ampersand.py index b42df22899..d908360d8e 100644 --- a/tests/test_tsx_jsx_text_ampersand.py +++ b/tests/test_tsx_jsx_text_ampersand.py @@ -35,9 +35,12 @@ * Generic arrows and function types — single-letter or multi-character (``(x: TKey) => x``, ``type F = (x: TKey) => void``, with or without a return-type annotation) — stay code, so a later bitwise - ``a & b`` is never corrupted into ``a & b``. -* The bytes mask is byte-preserving outside the ``&`` → ``&`` - insertions (non-UTF-8 bytes round-trip unchanged). + ``a & b`` is never corrupted into ``a b`` (which would reintroduce a + parse error). +* The bytes mask is fully byte-preserving: every ``&`` in JSX text is + replaced by a single ASCII space, so tree-sitter byte offsets and + ``source[start_byte:end_byte]`` slices stay aligned with the original + source (non-UTF-8 bytes round-trip unchanged). """ from __future__ import annotations @@ -150,14 +153,16 @@ def test_existing_entity_in_jsx_text_is_passed_through(tmp_path, capsys): # the helper keeps exactly one production caller (its ``_tsx_mask_source`` # wiring) — the afferent-coupling health gate counts direct test call sites. _MASK_CASES = [ - # --- JSX text: bare ``&`` masked to ``&``, byte-neutral. + # --- JSX text: bare ``&`` masked to a single ASCII space, keeping source + # byte offsets aligned with the original file. # Exact-output match covers: masked exactly once, no double-mask - # (``&amp;``), surrounding text byte-identical. + # (``&amp;``), surrounding text byte-identical except the one-byte + # placeholder. ('
VoIP & Chamadas
', - '
VoIP & Chamadas
'), + '
VoIP Chamadas
'), # Every bare ``&`` in one JSX-text run is masked independently. ('

Welcome & hello. Mixed & multiple & ampersands.

', - '

Welcome & hello. Mixed & multiple & ampersands.

'), + '

Welcome hello. Mixed multiple ampersands.

'), # --- Non-JSX-text ``&`` locations are left intact so the TSX grammar # still sees the same shape it always did. # JSX attribute string — grammar already accepts. @@ -189,7 +194,7 @@ def test_existing_entity_in_jsx_text_is_passed_through(tmp_path, capsys): 'let f: (x: T) => void = null;\nconst b = 1 & 2;\n'), # Multi-character generic arrow — ``(x: TKey) => x`` must stay # code: classifying it as JSX would strand jsx_text and corrupt the - # later bitwise ``1 & 2`` into ``1 & 2`` (a parse error). + # later bitwise ``1 & 2`` into ``1 2`` (a parse error). ('const pick = (x: TKey) => x;\nconst b = 1 & 2;\n', 'const pick = (x: TKey) => x;\nconst b = 1 & 2;\n'), # Return-type annotation between parameter list and arrow. @@ -216,37 +221,37 @@ def test_existing_entity_in_jsx_text_is_passed_through(tmp_path, capsys): # elements unwind to the parent's text. # Closing tag pops jsx_text → later code ``&`` stays bitwise. ('const a =
x & y
;\nconst b = 1 & 2;\n', - 'const a =
x & y
;\nconst b = 1 & 2;\n'), + 'const a =
x y
;\nconst b = 1 & 2;\n'), # Self-closing (tight and spaced) never opens jsx_text. ('const a =
;\nconst b =
;\nconst c = 1 & 2;\n', 'const a =
;\nconst b =
;\nconst c = 1 & 2;\n'), # Fragment open/close round-trips back to code. ('const a = <>x & y;\nconst b = 1 & 2;\n', - 'const a = <>x & y;\nconst b = 1 & 2;\n'), + 'const a = <>x y;\nconst b = 1 & 2;\n'), # Nested element: after the child closes, the parent's JSX text is # still masked; after the parent closes, code is not. ('const a =

q & rt & u

;\nconst z = 1 & 2;\n', - 'const a =

q & rt & u

;\nconst z = 1 & 2;\n'), + 'const a =

q rt u

;\nconst z = 1 & 2;\n'), # --- Single-letter uppercase components (````, ```` — icon/nav # shorthand) are JSX, not generics: text is masked, the close tag # pops jsx_text, and an empty element leaves no stale context. ('export const nav = VoIP & Chamadas;', - 'export const nav = VoIP & Chamadas;'), + 'export const nav = VoIP Chamadas;'), ('const a = x & y;\nconst b = 1 & 2;\n', - 'const a = x & y;\nconst b = 1 & 2;\n'), + 'const a = x y;\nconst b = 1 & 2;\n'), ('const a = ;\nconst b = 1 & 2;\n', 'const a = ;\nconst b = 1 & 2;\n'), # Paren-initial JSX text has no arrow tail, so an uppercase # component still masks (``_generic_arrow_tail`` returns False). ('const el = (note) & more;', - 'const el = (note) & more;'), + 'const el = (note) more;'), # Nested JSX inside an expression container is masked, the # container's own ``&&`` is not, and code after is not. ('const a =
{x && i & j}
;\nconst z = 1 & 2;\n', - 'const a =
{x && i & j}
;\nconst z = 1 & 2;\n'), + 'const a =
{x && i j}
;\nconst z = 1 & 2;\n'), # Attribute strings still untouched, element text still masked. ('const a =
t & v
;', - 'const a =
t & v
;'), + 'const a =
t v
;'), # --- Fast-path: sources without ``&`` (or empty) are a no-op. ('', ''), ('// nothing here\nconst x = 1;\n', @@ -256,9 +261,9 @@ def test_existing_entity_in_jsx_text_is_passed_through(tmp_path, capsys): @pytest.mark.parametrize('src,expected', _MASK_CASES) def test_mask_walker(src, expected): - """Walker unit checks: bare ``&`` is masked to ``&`` only in JSX - text; attributes, ``{ ... }`` containers, strings, comments, and TS - code (bitwise AND, generics) are byte-identical.""" + """Walker unit checks: bare ``&`` is masked to a single ASCII space + only in JSX text; attributes, ``{ ... }`` containers, strings, comments, + and TS code (bitwise AND, generics) are byte-identical.""" got = _mask_tsx_ampersands(src) assert got == expected, ( f"walker mangled {src!r}\n" @@ -268,14 +273,15 @@ def test_mask_walker(src, expected): def test_mask_source_round_trips_non_utf8_bytes(): - """``LanguageConfig.source_transform`` byte contract: apart from the - intentional ``&`` → ``&`` insertions the transform must be - byte-preserving. Non-UTF-8 bytes (a latin-1 comment) round-trip - unchanged instead of being rewritten to U+FFFD, which would silently - alter the source the engine parses.""" + """``LanguageConfig.source_transform`` byte contract: the transform + must be fully byte-preserving. A bare ``&`` in JSX text is replaced by + a single ASCII space, so tree-sitter offsets and ``source[start_byte: + end_byte]`` slices stay aligned with the original file. Non-UTF-8 bytes + (a latin-1 comment) round-trip unchanged instead of being rewritten to + U+FFFD, which would silently alter the source the engine parses.""" src = b"a & b // caf\xe9 latin-1 comment\n" out = _tsx_mask_source(src) - assert out.startswith(b"a & b") + assert out.startswith(b"a b") assert b"caf\xe9" in out From 57f6e400ca5a8dd09c6ef8267767ee03e65fec8d Mon Sep 17 00:00:00 2001 From: Santhi Prakash Date: Mon, 24 Aug 2026 04:27:17 +0000 Subject: [PATCH 3/7] fix(extract): keep regex literal contents out of JSX-text ampersand mask --- graphify/extract.py | 90 ++++++++++++++++++++++++++++ tests/test_tsx_jsx_text_ampersand.py | 27 +++++++++ 2 files changed, 117 insertions(+) diff --git a/graphify/extract.py b/graphify/extract.py index 187317059f..fd6e35680e 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -843,6 +843,22 @@ def _get_c_func_name(node, source: bytes) -> str | None: # end-of-token check below, which keeps the set a flat char check. ) +# Characters that can precede a ``/`` which starts a regex literal in TS/JS. +# ``/`` after a value token (identifier, number, closing paren, bracket, or +# brace) is a division operator, not a regex; ``<`` and ``>`` are included +# because they can be comparison operators (``a < /b/g``), but the tag ``>`` +# handler resets the previous-char tracker so a ``/`` directly after a JSX +# tag is not mistaken for regex. +_TSX_REGEX_START_PREV = frozenset( + '=(),?:;!&|^~+-*/%[]{}<>' +) + +# Keywords that put the next token in expression position, so a following +# ``/`` can be a regex literal (``return /a/g``, ``case /a/:``, ...). +_TSX_REGEX_START_KEYWORDS = frozenset({ + 'return', 'yield', 'throw', 'typeof', 'void', 'delete', 'case', +}) + def _generic_arrow_tail(src: str, m: int) -> bool: """True when ``src[m] == '('`` opens a parameter list followed by an @@ -935,6 +951,11 @@ def _mask_tsx_ampersands(src: str) -> str: # ``<`` after them is JSX), the second opens declaration position (so # ``<`` after them is a generic, not JSX). prev_code_keyword: str | None = None + # When the active context is 'regex', whether we are inside a character + # class and the index where that class opened (so a leading ``]`` is + # treated as a literal, not the class close). + regex_class = False + regex_class_open = -1 # Cheap fast-path: if there is no ``&`` in the source, the mask is a # no-op and we can skip the whole walk. Almost every real TSX file has @@ -1049,6 +1070,48 @@ def _lt(expr_ctx: bool) -> None: c2 = src[i:i + 2] if i + 1 < n else '' top = stack[-1] if stack else None + if top == 'regex': + if c == '\\' and i + 1 < n: + out.append(c) + out.append(src[i + 1]) + i += 2 + continue + if regex_class: + if c == ']': + if ( + i == regex_class_open + 1 + or (i == regex_class_open + 2 and src[regex_class_open + 1] == '^') + ): + # A leading ``]`` immediately after ``[`` or ``[^`` + # is a literal, not the class close. + out.append(c) + i += 1 + continue + regex_class = False + out.append(c) + i += 1 + continue + if c == '[': + regex_class = True + regex_class_open = i + out.append(c) + i += 1 + continue + if c == '/': + out.append(c) + i += 1 + while i < n and src[i].isalpha(): + out.append(src[i]) + i += 1 + stack.pop() + # The regex literal is a value token, so ``<`` after it is a + # comparison and ``/`` after it is division. + _set_prev(')') + continue + out.append(c) + i += 1 + continue + if top == 'string': if c == '\\' and i + 1 < n: out.append(c) @@ -1142,6 +1205,9 @@ def _lt(expr_ctx: bool) -> None: elif kind != 'self': # Opening tag → enter JSX text for the element's children. stack.append('jsx_text') + # A complete tag is a value token; reset the tracker so the + # next ``<``/``/`` is not misread as a JSX/regex start. + _set_prev(')') i += 1 continue out.append(c) @@ -1177,6 +1243,18 @@ def _lt(expr_ctx: bool) -> None: stack.append('comment') i += 2 continue + if c == '/' and c2 not in ('//', '/*'): + if ( + prev_code_char is None + or prev_code_char in _TSX_REGEX_START_PREV + or prev_code_keyword in _TSX_REGEX_START_KEYWORDS + ): + stack.append('regex') + regex_class = False + regex_class_open = -1 + out.append(c) + i += 1 + continue if c == '<': # Nested JSX inside a JSX expression container # (``{ok ? a & b : null}``): run the shared @@ -1238,6 +1316,18 @@ def _lt(expr_ctx: bool) -> None: stack.append('comment') i += 2 continue + if c == '/' and c2 not in ('//', '/*'): + if ( + prev_code_char is None + or prev_code_char in _TSX_REGEX_START_PREV + or prev_code_keyword in _TSX_REGEX_START_KEYWORDS + ): + stack.append('regex') + regex_class = False + regex_class_open = -1 + out.append(c) + i += 1 + continue if c == '<': # JSX-vs-generic disambiguation (shared with expression # containers — see ``_lt``): diff --git a/tests/test_tsx_jsx_text_ampersand.py b/tests/test_tsx_jsx_text_ampersand.py index d908360d8e..7ed681ad09 100644 --- a/tests/test_tsx_jsx_text_ampersand.py +++ b/tests/test_tsx_jsx_text_ampersand.py @@ -252,6 +252,33 @@ def test_existing_entity_in_jsx_text_is_passed_through(tmp_path, capsys): # Attribute strings still untouched, element text still masked. ('const a =
t & v
;', 'const a =
t v
;'), + # --- Regex literals: ``<``/``>``/``&`` inside a ``/.../`` body are not + # JSX and must not be masked. The walker enters a regex context so the + # ``/`` closing delimiter ends it, including unescaped ``/`` inside + # character classes and a trailing flag run. + ('const r = /&b<\\/a>/;', + 'const r = /&b<\\/a>/;'), + ('const r = /a & b/gi;', + 'const r = /a & b/gi;'), + ('const r = /[a&b]/;', + 'const r = /[a&b]/;'), + ('const r = /[]]/;', + 'const r = /[]]/;'), + ('const r = /[^]]/;', + 'const r = /[^]]/;'), + ('const a = [
, /&b<\\/a>/];', + 'const a = [
, /&b<\\/a>/];'), + ('const a = { r: /&b<\\/a>/ };', + 'const a = { r: /&b<\\/a>/ };'), + ('return /&b<\\/a>/;', + 'return /&b<\\/a>/;'), + # --- Division is not a regex (prev token is a value). + ('const a = 1 / 2 & 3;', + 'const a = 1 / 2 & 3;'), + ('const a = foo() / 2;', + 'const a = foo() / 2;'), + ('const a =
/ 2;', + 'const a =
/ 2;'), # --- Fast-path: sources without ``&`` (or empty) are a no-op. ('', ''), ('// nothing here\nconst x = 1;\n', From b6c1e8cd189c6ea1d56d8b34a53c7736522536f7 Mon Sep 17 00:00:00 2001 From: Santhi Prakash Date: Mon, 24 Aug 2026 04:53:54 +0000 Subject: [PATCH 4/7] fix(extract): keep original source bytes and handle extends JSX attr in TSX mask - Parse the masked TSX bytes but keep the original source bytes for downstream source[start_byte:end_byte] slices, so locations/snippets are reported against the user's file rather than the masked copy. The mask is byte-length-preserving, so offsets stay aligned. - Treat `extends` as a JSX attribute when it is followed by `=`, `>`, `/`, or EOF; only classify it as a generic constraint when a real type expression follows. - Use the already-safely-read `nxt` character for the upper-case check instead of re-indexing `src[b + 1]`. Co-Authored-By: Paperclip --- graphify/extract.py | 16 ++++++++++++++-- graphify/extractors/engine.py | 13 +++++++++---- tests/test_tsx_jsx_text_ampersand.py | 11 +++++++++++ 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index fd6e35680e..1bc9cd608f 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -1026,7 +1026,19 @@ def _lt(expr_ctx: bool) -> None: k += 1 nxt_after = src[k:k + 1] if k < n else '' after_word = src[k:k + 8] - if nxt_after == ',' or after_word.startswith('extends') or nxt_after == '=': + is_extends_generic = False + if after_word.startswith('extends'): + # ``extends`` can be a generic constraint (````) + # or a JSX attribute (````). An attribute + # has its value assignment ``=`` immediately after the name + # (with optional spaces); a generic constraint has a type + # expression. Treat ``>``/``/``/EOF after ``extends`` as JSX + # boolean attributes as well. + p = k + len('extends') + while p < n and src[p].isspace(): + p += 1 + is_extends_generic = p < n and src[p] not in ('=', '>', '/') + if nxt_after == ',' or is_extends_generic or nxt_after == '=': # ```` / ```` / ````: generic. push = None elif nxt_after == '(': @@ -1051,7 +1063,7 @@ def _lt(expr_ctx: bool) -> None: push = None if ( m < n and src[m] == '(' - and src[b + 1].isupper() + and nxt.isupper() and _generic_arrow_tail(src, m) ) else 'tag' else: diff --git a/graphify/extractors/engine.py b/graphify/extractors/engine.py index d949b1a941..65b37b3b52 100644 --- a/graphify/extractors/engine.py +++ b/graphify/extractors/engine.py @@ -2809,11 +2809,16 @@ def _extract_generic( try: parser = Parser(language) source = path.read_bytes() if source_override is None else source_override + parse_source = source if config.source_transform is not None: - # Per-language byte mask applied to whatever gets parsed — e.g. - # the TSX bare-``&``-in-JSX-text mask (#2922). - source = config.source_transform(source) - tree = parser.parse(source) + # Per-language byte mask applied only to the bytes the parser sees — + # e.g. the TSX bare-``&``-in-JSX-text mask (#2922). The mask is + # byte-length-preserving, so the parse tree's offsets stay aligned + # with the original file. Keep the original source for downstream + # ``source[start_byte:end_byte]`` slices so snippets/locations are + # reported against the user's file, not the masked copy. + parse_source = config.source_transform(source) + tree = parser.parse(parse_source) root = tree.root_node except Exception as e: return {"nodes": [], "edges": [], "error": str(e)} diff --git a/tests/test_tsx_jsx_text_ampersand.py b/tests/test_tsx_jsx_text_ampersand.py index 7ed681ad09..866458949c 100644 --- a/tests/test_tsx_jsx_text_ampersand.py +++ b/tests/test_tsx_jsx_text_ampersand.py @@ -279,6 +279,17 @@ def test_existing_entity_in_jsx_text_is_passed_through(tmp_path, capsys): 'const a = foo() / 2;'), ('const a =
/ 2;', 'const a =
/ 2;'), + # --- ``extends`` as a JSX attribute must not be misread as a generic + # constraint; the element's children are still JSX text. + ('const a = t & v;', + 'const a = t v;'), + ('const a = t & v;', + 'const a = t v;'), + ('const a = t & v;', + 'const a = t v;'), + # A real generic constraint ```` still stays code. + ('function f() { return 1 & 2; }', + 'function f() { return 1 & 2; }'), # --- Fast-path: sources without ``&`` (or empty) are a no-op. ('', ''), ('// nothing here\nconst x = 1;\n', From 8ffd33ed8c785c36bb41a49a862141848ea37bff Mon Sep 17 00:00:00 2001 From: Santhi Prakash Date: Mon, 24 Aug 2026 06:04:24 +0000 Subject: [PATCH 5/7] test(extract): add byte-length preservation and multibyte UTF-8 regression checks The previous graphify-labs review raised three advisory high findings about surrogateescape, multibyte JSX text, and the str/bytes contract. These tests lock in the byte-preserving behavior that the implementation already provides: the transform is bytes -> bytes, every non-& byte round-trips unchanged (including non-UTF-8 surrogateescape bytes and multibyte UTF-8), and only a bare & in JSX text becomes a single ASCII space. Co-Authored-By: Paperclip --- tests/test_tsx_jsx_text_ampersand.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_tsx_jsx_text_ampersand.py b/tests/test_tsx_jsx_text_ampersand.py index 866458949c..c45fc28f26 100644 --- a/tests/test_tsx_jsx_text_ampersand.py +++ b/tests/test_tsx_jsx_text_ampersand.py @@ -323,6 +323,29 @@ def test_mask_source_round_trips_non_utf8_bytes(): assert b"caf\xe9" in out +@pytest.mark.parametrize('src,expected', [ + (b'a & b', b'a b'), + ('Conexões & Integrações'.encode('utf-8'), + 'Conexões Integrações'.encode('utf-8')), + (b'a & b // caf\xe9 latin-1 comment\n', + b'a b // caf\xe9 latin-1 comment\n'), + (b'a & b \xff\xfe', + b'a b \xff\xfe'), + (b'const x = 1 & 2;', b'const x = 1 & 2;'), +]) +def test_mask_source_preserves_byte_length(src, expected): + """The ``_tsx_mask_source`` contract is ``bytes -> bytes`` and must be + byte-length-preserving. Every non-ampersand byte survives the round trip, + and a bare ``&`` in JSX text is replaced by a single ASCII space so the + parser sees the same offsets as the original file. This covers multibyte + UTF-8 JSX text and invalid-UTF-8 bytes that round-trip via + ``surrogateescape``.""" + out = _tsx_mask_source(src) + assert isinstance(out, bytes) + assert len(out) == len(src) + assert out == expected + + def test_code_after_jsx_element_is_not_masked(tmp_path, capsys): """A closing tag must exit the element's ``jsx_text`` context: code after ``
`` is TS code again, so a bitwise ``&`` there must not From 38eae12e88284fe14a46d486d080494dd2747f05 Mon Sep 17 00:00:00 2001 From: Santhi Prakash Date: Mon, 24 Aug 2026 06:40:16 +0000 Subject: [PATCH 6/7] test(extract): add non-BMP UTF-8 regression for TSX JSX-text ampersand mask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The latest graphify-labs review flagged an advisory that the TSX mask alters byte length for non-BMP characters and breaks offset alignment. Add a non-BMP (4-byte UTF-8 🚀) case to the byte-preservation parametrize and an end-to-end extraction test to prove the mask stays byte-length-preserving and that the file extracts silently. Co-Authored-By: Paperclip --- tests/test_tsx_jsx_text_ampersand.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_tsx_jsx_text_ampersand.py b/tests/test_tsx_jsx_text_ampersand.py index c45fc28f26..c426c73dfc 100644 --- a/tests/test_tsx_jsx_text_ampersand.py +++ b/tests/test_tsx_jsx_text_ampersand.py @@ -327,6 +327,10 @@ def test_mask_source_round_trips_non_utf8_bytes(): (b'a & b', b'a b'), ('Conexões & Integrações'.encode('utf-8'), 'Conexões Integrações'.encode('utf-8')), + # Non-BMP (4-byte UTF-8) emoji in JSX text: the replacement stays one + # byte, so offsets remain aligned with the original file. + ('🚀 & Chamadas'.encode('utf-8'), + '🚀 Chamadas'.encode('utf-8')), (b'a & b // caf\xe9 latin-1 comment\n', b'a b // caf\xe9 latin-1 comment\n'), (b'a & b \xff\xfe', @@ -378,3 +382,20 @@ def test_nested_jsx_in_expression_container_is_masked(tmp_path, capsys): _assert_silent(capsys.readouterr().err) assert r.get("parse_errors") in (None, []) + +def test_non_bmp_jsx_text_ampersand_is_silent(tmp_path, capsys): + """A non-BMP character (e.g. U+1F680 🚀, 4 UTF-8 bytes) in JSX text + followed by a bare ``&`` must not shift parser offsets. The mask + replaces ``&`` with a single ASCII space, so the byte length of the + source stays identical before and after the transform.""" + r = _extract(tmp_path, { + "page.tsx": ( + "export function Page() {\n" + " return 🚀 & Chamadas;\n" + "}\n" + ), + }) + assert "Page()" in _labels(r) + _assert_silent(capsys.readouterr().err) + assert r.get("parse_errors") in (None, []) + From 7e7a03e14a1184de9df091643d6787c6fecf548c Mon Sep 17 00:00:00 2001 From: Santhi Prakash Date: Mon, 24 Aug 2026 07:00:45 +0000 Subject: [PATCH 7/7] fix(extract): handle ;, ), and regex inside generic arrow tails for TSX ampersand mask - Replace the `src.find(';', j)` return-type bound in `_generic_arrow_tail` with a depth-aware scan that skips `;` inside object types and other nested delimiters, and skips `)` inside strings/regex in the parameter list. - Update the stale ampersand-mask comment to describe the actual one-byte space placeholder and byte-alignment contract. - Document the `LanguageConfig.source_transform` byte-length preservation requirement in `extractors/models.py`. Co-Authored-By: Paperclip --- graphify/extract.py | 152 +++++++++++++++++++++++---- graphify/extractors/models.py | 6 +- tests/test_tsx_jsx_text_ampersand.py | 10 ++ 3 files changed, 146 insertions(+), 22 deletions(-) diff --git a/graphify/extract.py b/graphify/extract.py index 1bc9cd608f..1d6ed7bb05 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -825,9 +825,10 @@ def _get_c_func_name(node, source: bytes) -> str | None: # partial-extraction warning fires (#2551, #2922). ``&`` inside JSX tag # attribute values, JSX expression containers ``{ ... }``, string literals, # comments, and TS code is already accepted by tree-sitter-typescript — only -# JSX text content is strict. Mask bare ``&`` to ``&`` so the TSX grammar -# parses the file cleanly; the entity serializes back to a single ``&`` so -# the visible text is byte-identical to the user-written source. +# JSX text content is strict. A bare ``&`` in JSX text is replaced with a +# single ASCII space so the TSX grammar parses the file cleanly; the one-byte +# placeholder keeps the transformed source byte-aligned with the original file +# so ``source[start_byte:end_byte]`` slices stay accurate. _TSX_ENTITY_RE = re.compile(r'&(?:#[xX][0-9a-fA-F]+|#[0-9]+|[A-Za-z][A-Za-z0-9]*);') # Characters whose preceding position puts ``<`` at expression position (so it @@ -867,35 +868,146 @@ def _generic_arrow_tail(src: str, m: int) -> bool: Used by the ``<`` disambiguation in :func:`_mask_tsx_ampersands`: classifying a generic arrow as a JSX tag would strand the walker in - ``jsx_text`` and corrupt a later bitwise ``a & b`` into ``a & b`` - (a parse error — the very bug class this mask removes), so an + ``jsx_text`` and corrupt a later bitwise ``a & b`` into ``a b`` + (a parse error — the very bug class this fix removes), so an uppercase ```` directly followed by ``>(`` is only treated as a tag when no arrow tail follows the balanced parameter list. The scan is bounded so pathological input cannot make the walker quadratic. + + The scan skips over string literals, comments, and regex literals so + that ``)`` / ``;`` characters inside them do not break the parameter + list balance or prematurely end the return-type search. """ n = len(src) - limit = min(n, m + 600) - depth = 0 + limit = min(n, m + 800) i = m + paren_depth = 0 + return_depth = 0 + mode = 'params' + prev: str | None = None + keyword: str | None = None + + def _set_prev(c: str) -> None: + nonlocal prev, keyword + prev = c + if not (c.isalnum() or c == '_' or c == '$'): + keyword = None + + def _extend_keyword(c: str) -> None: + nonlocal keyword + keyword = (keyword or '') + c + + def _skip_string(i: int, quote: str) -> int: + i += 1 + while i < limit: + if src[i] == '\\' and i + 1 < n: + i += 2 + continue + if src[i] == quote: + return i + 1 + i += 1 + return i + + def _skip_comment(i: int) -> int: + if src[i + 1] == '/': + while i < limit and src[i] != '\n': + i += 1 + else: + i += 2 + while i + 1 < limit and not (src[i] == '*' and src[i + 1] == '/'): + i += 1 + i += 2 + return i + + def _skip_regex(i: int) -> int: + nonlocal prev, keyword + i += 1 + in_class = False + class_open = -1 + while i < limit: + c = src[i] + if c == '\\' and i + 1 < n: + i += 2 + continue + if in_class: + if c == ']': + if i == class_open + 1 or (i == class_open + 2 and src[class_open + 1] == '^'): + i += 1 + continue + in_class = False + i += 1 + continue + if c == '[': + in_class = True + class_open = i + i += 1 + continue + if c == '/': + i += 1 + while i < n and src[i].isalpha(): + i += 1 + break + i += 1 + prev, keyword = 'a', None + return i + while i < limit: c = src[i] + c2 = src[i:i + 2] if i + 1 < n else '' + if c in '"\'`': + i = _skip_string(i, c) + _set_prev(c) + continue + if c2 == '//' or c2 == '/*': + i = _skip_comment(i) + continue + if c == '/' and c2 not in ('//', '/*') and ( + prev is None + or prev in _TSX_REGEX_START_PREV + or keyword in _TSX_REGEX_START_KEYWORDS + ): + i = _skip_regex(i) + continue if c == '(': - depth += 1 + if mode == 'params': + paren_depth += 1 + else: + return_depth += 1 elif c == ')': - depth -= 1 - if depth == 0: - j = i + 1 - while j < n and src[j].isspace(): - j += 1 - if src[j:j + 2] == '=>': + if mode == 'params': + paren_depth -= 1 + if paren_depth == 0: + i += 1 + while i < n and src[i].isspace(): + i += 1 + if src[i:i + 2] == '=>': + return True + if i < n and src[i] == ':': + mode = 'return' + return_depth = 0 + i += 1 + continue + return False + else: + return_depth -= 1 + elif mode == 'return': + if c == '=' and c2 == '=>': + if return_depth == 0: return True - if j < n and src[j] == ':': - # ``(x: TKey): TResult => x`` — return-type annotation - # between the parameter list and the arrow. - end = src.find(';', j) - stop = end if end != -1 else min(n, j + 200) - return '=>' in src[j:stop] + i += 2 + continue + if c in '[{<': + return_depth += 1 + elif c in ']}>': + return_depth -= 1 + elif c == ';' and return_depth == 0: return False + if not c.isspace(): + if c.isalnum() or c == '_' or c == '$': + _extend_keyword(c) + else: + keyword = None + prev = c i += 1 return False diff --git a/graphify/extractors/models.py b/graphify/extractors/models.py index e6876cc621..fd8a44f144 100644 --- a/graphify/extractors/models.py +++ b/graphify/extractors/models.py @@ -54,8 +54,10 @@ class LanguageConfig: extra_walk_fn: Callable | None = None # Optional bytes transform applied to the source right before parsing - # (e.g. the TSX bare-``&`` JSX-text mask, #2922). Runs on the bytes that - # are actually parsed, after any ``source_override`` substitution. + # (e.g. the TSX bare-``&`` JSX-text mask, #2922). It MUST preserve the + # total UTF-8 byte length so tree-sitter's start_byte/end_byte slices stay + # aligned with the original file. Runs on the bytes that are actually + # parsed, after any ``source_override`` substitution. source_transform: Callable[[bytes], bytes] | None = None @dataclass(frozen=True) diff --git a/tests/test_tsx_jsx_text_ampersand.py b/tests/test_tsx_jsx_text_ampersand.py index c426c73dfc..329489edba 100644 --- a/tests/test_tsx_jsx_text_ampersand.py +++ b/tests/test_tsx_jsx_text_ampersand.py @@ -287,6 +287,16 @@ def test_existing_entity_in_jsx_text_is_passed_through(tmp_path, capsys): 'const a = t v;'), ('const a = t & v;', 'const a = t v;'), + # --- Generic arrow / function-type tails keep ``&`` in code even when + # the return type or default parameter contains a ``;`` or ``)``. + ('const pick = (x: TKey): { a: number; b: string } => x;\nconst b = 1 & 2;\n', + 'const pick = (x: TKey): { a: number; b: string } => x;\nconst b = 1 & 2;\n'), + ('const f = (x: T = ")") => x;\nconst b = 1 & 2;\n', + 'const f = (x: T = ")") => x;\nconst b = 1 & 2;\n'), + ('const f = (x: T = /a\\/b/g) => x;\nconst b = 1 & 2;\n', + 'const f = (x: T = /a\\/b/g) => x;\nconst b = 1 & 2;\n'), + ('const f = (x: T): "a; b" => x;\nconst b = 1 & 2;\n', + 'const f = (x: T): "a; b" => x;\nconst b = 1 & 2;\n'), # A real generic constraint ```` still stays code. ('function f() { return 1 & 2; }', 'function f() { return 1 & 2; }'),