diff --git a/graphify/extract.py b/graphify/extract.py index 9afb543198..e57e749e15 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -1958,13 +1958,111 @@ def _augment_cpp_string_tests(path: Path, result: dict) -> dict: return result +# ── C++/CLI normalization (#2876) ──────────────────────────────────────────── +# tree-sitter-cpp implements none of `ref class`, `Type^`, `Type%`, `gcnew` or +# `[assembly:…]`. The ERROR lands on the type header, which dissolves the whole +# class body — a 141-method .NET interop wrapper yielded 12 junk symbols. These +# rewrites map each spelling onto the nearest standard C++ one. + +# Only files carrying one of these engage the rewrite, so plain C/C++/CUDA is +# parsed byte-for-byte as before. `^`/`%` alone are not markers: they are the +# ordinary XOR and modulo operators. +_CPP_CLI_MARKER_RE = re.compile( + rb"\b(?:ref|value)\s+(?:class|struct)\b" + rb"|\binterface\s+class\b" + rb"|\bgcnew\b" + rb"|\[\s*(?:assembly|module)\s*:" +) + +# `public ref class Foo` / `value struct Bar` / `interface class Baz` → the +# access specifier goes too: it is not legal at namespace scope, and leaving it +# behind is what made recovery invent a stray `public` node. +_CPP_CLI_CLASS_RE = re.compile( + rb"(?:\b(?:public|private|protected)\s+)?\b(?:ref|value)\s+(?=(?:class|struct)\b)" + rb"|(?:\b(?:public|private|protected)\s+)?\binterface\s+(?=class\b)" +) +# Handle (`String^ s`) and tracking-reference (`int% n`) suffixes. +# +# `^` and `%` are also XOR and modulo, and `String^ s` is lexically identical to +# `a^ b` — attachment to the preceding token does not separate them, because +# `a% b` and `hash^ mask` are attached too. Rewriting on attachment alone +# corrupted those into `a b` / `hash mask`. So the rewrite is restricted to +# the two positions where an operator reading is impossible or implausible. +# +# 1. Followed by a token that cannot begin an operand: `f(String^, int)`, +# `List`, `(String^)x`, `Object^;`. `a^,` is not valid C++, so +# there is no arithmetic to lose here. `*` and `&` are deliberately NOT in +# the set — `a^*p` and `a^&b` are valid XOR expressions, and `String^*` is +# rare enough not to be worth trading for them. +_CPP_CLI_SUFFIX_UNAMBIGUOUS_RE = re.compile(rb"(?<=[A-Za-z0-9_>])[\^%](?=\s*[,)\]>;])") +# 2. A type-shaped left side followed by a declarator: `System::String^ s`, +# `List^ items`, `DataTable^ t`, `int% n`. Qualified names, a closing +# generic bracket, .NET's PascalCase convention and the primitive value +# types are all type positions; requiring one keeps lowercase operands like +# `count% 2` and `hash^ mask` as arithmetic. A capitalized name must also +# carry a lowercase letter, so SCREAMING_CASE constants stay arithmetic +# too (`MASK^ value`); a lone capital is exempt for generic parameters +# (`T^ x`). The type is captured and re-emitted so the substitution stays +# byte-length preserving. +_CPP_CLI_SUFFIX_DECL_RE = re.compile( + rb"(\b[A-Za-z_][A-Za-z0-9_]*(?:::[A-Za-z_][A-Za-z0-9_]*)+" # System::String + rb"|\b[A-Z][A-Za-z0-9_]*[a-z][A-Za-z0-9_]*" # String, DataTable + rb"|\b[A-Z](?=[\^%])" # T + rb"|\b(?:bool|char|wchar_t|short|int|long|float|double|unsigned|signed)" + rb"|>)" # List^ + rb"[\^%](?=\s+[A-Za-z_])" +) +# `[assembly:AssemblyVersion("1.0")]` and friends. +_CPP_CLI_ATTR_RE = re.compile(rb"\[\s*(?:assembly|module)\s*:[^\[\]]*\]", re.S) + + +def _blank_keeping_newlines(m: "re.Match[bytes]") -> bytes: + """Replace a match with spaces, but keep its line breaks. + + Byte length alone is not enough. Both the class-header and the attribute + pattern can span lines — ``[assembly:AssemblyVersion(\\n "1.0"\\n)]`` is + ordinary formatting — and blanking a newline merges two source lines, which + shifts the reported line number of every symbol below it. Preserving CR and + LF in place keeps line and column stable as well as offset. + """ + return re.sub(rb"[^\r\n]", b" ", m.group(0)) + + +def _normalize_cpp_cli(source: bytes) -> bytes | None: + """Rewrite C++/CLI spellings to standard C++ ones, or None if not C++/CLI. + + The rewrite is **byte-length preserving** — dropped tokens are overwritten + with spaces, never deleted, and ``gcnew`` becomes ``new`` plus padding — and + line breaks inside a removed token are kept, so every offset, line and + column still points at the same place in the file on disk and reported + source locations stay accurate (#2876). + """ + if not _CPP_CLI_MARKER_RE.search(source): + return None + out = _CPP_CLI_CLASS_RE.sub(_blank_keeping_newlines, source) + out = re.sub(rb"\bgcnew\b", b"new ", out) + out = _CPP_CLI_SUFFIX_UNAMBIGUOUS_RE.sub(b" ", out) + out = _CPP_CLI_SUFFIX_DECL_RE.sub(rb"\1 ", out) + return _CPP_CLI_ATTR_RE.sub(_blank_keeping_newlines, out) + + def extract_cpp(path: Path) -> dict: """Extract functions, classes, and includes from a .cpp/.cc/.cxx/.hpp file. + C++/CLI sources are normalized to standard C++ first (#2876); see + :func:`_normalize_cpp_cli`. + Recovers doctest/Catch2 ``TEST_CASE("name")`` test cases that tree-sitter-cpp drops as ERROR nodes (issue #2594), mirroring the Spock fallback for Groovy. """ - result = _extract_generic(path, _CPP_CONFIG) + try: + source = path.read_bytes() + except OSError: + # Let _extract_generic report the read failure in its usual shape. + return _augment_cpp_string_tests(path, _extract_generic(path, _CPP_CONFIG)) + result = _extract_generic( + path, _CPP_CONFIG, source_override=_normalize_cpp_cli(source) or source + ) return _augment_cpp_string_tests(path, result) diff --git a/graphify/extractors/engine.py b/graphify/extractors/engine.py index 77c4672bdb..3bf3da124d 100644 --- a/graphify/extractors/engine.py +++ b/graphify/extractors/engine.py @@ -3983,8 +3983,23 @@ def _emit_java_parent_type(type_node, rel: str, at_line: int) -> None: and any(c.type == "function_declarator" for c in d.children)) for d in decls ) - if not is_method: - type_node = node.child_by_field_name("type") + type_node = node.child_by_field_name("type") + # A nested type (`class Inner { … };` inside a class body) is a + # field_declaration whose `type` field IS the class_specifier, so + # returning from this branch used to drop Inner and everything it + # declares — silently, with no parse error (#2876). Walk it as a + # class instead: the engine's existing nested-type handling gives + # it a `contains` edge from the enclosing type. The declarator loop + # below still runs, since `class Inner { } inst;` declares a member + # alongside the type. + is_nested_type = ( + type_node is not None + and type_node.type in config.class_types + and type_node.child_by_field_name("body") is not None + ) + if is_nested_type: + walk(type_node, parent_class_nid) + if not is_method and not is_nested_type: if type_node is not None: line = node.start_point[0] + 1 refs: list[tuple[str, str]] = [] diff --git a/tests/test_cpp_nested_and_cli.py b/tests/test_cpp_nested_and_cli.py new file mode 100644 index 0000000000..6b7d4647f2 --- /dev/null +++ b/tests/test_cpp_nested_and_cli.py @@ -0,0 +1,251 @@ +"""C++ nested types and C++/CLI keep their symbols (#2876).""" +from pathlib import Path + +import pytest + +from graphify.extract import _normalize_cpp_cli, extract_cpp + +pytest.importorskip("tree_sitter_cpp") + + +def _labels(path: Path) -> list[str]: + return [n["label"] for n in extract_cpp(path)["nodes"]] + + +def test_nested_cpp_class_is_extracted(tmp_path): + # A nested type is a field_declaration whose `type` field IS the + # class_specifier; the member-variable branch used to consume it and return + # before the walk could descend, dropping Inner with no parse error. + p = tmp_path / "nested.h" + p.write_text( + "namespace N {\n" + " class Outer\n" + " {\n" + " public:\n" + " class Inner\n" + " {\n" + " public:\n" + " static void Method() { }\n" + " };\n" + " };\n" + "}\n" + ) + result = extract_cpp(p) + assert result.get("parse_errors") is None + assert [n["label"] for n in result["nodes"]] == [ + "nested.h", "Outer", "Inner", ".Method()", + ] + # Inner is contained by Outer, not by the file (#2040). + outer = next(n["id"] for n in result["nodes"] if n["label"] == "Outer") + inner = next(n["id"] for n in result["nodes"] if n["label"] == "Inner") + assert any( + e["source"] == outer and e["target"] == inner and e["relation"] == "contains" + for e in result["edges"] + ) + + +def test_nested_type_declared_with_an_instance(tmp_path): + """`class Inner { } inst;` declares both a type and a member.""" + p = tmp_path / "both.h" + p.write_text( + "class Outer\n" + "{\n" + "public:\n" + " class Inner { int x; } inst;\n" + "};\n" + ) + labels = _labels(p) + assert "Inner" in labels + assert "inst" in labels + + +def test_cpp_cli_class_body_survives(tmp_path): + p = tmp_path / "cli.h" + p.write_text( + "namespace N {\n" + " public ref class Wrapper\n" + " {\n" + " public:\n" + " static void Init() { }\n" + ' static System::String^ Name() { return gcnew System::String(""); }\n' + " };\n" + "}\n" + ) + result = extract_cpp(p) + assert result.get("parse_errors") is None + labels = [n["label"] for n in result["nodes"]] + assert "Wrapper" in labels + assert ".Init()" in labels + assert ".Name()" in labels + # recovery no longer invents a `Wrapper()` free function or a `public` node + assert "Wrapper()" not in labels + assert "public" not in labels + + +def test_cli_normalization_preserves_byte_offsets(tmp_path): + src = ( + '[assembly:AssemblyVersion("1.0")];\n' + "public ref struct S { void F(System::Object^ o, int% n) { gcnew S(); } };\n" + ).encode() + out = _normalize_cpp_cli(src) + assert out is not None + assert len(out) == len(src) + # every line still starts at the same offset + assert [i for i, b in enumerate(src) if b == 0x0A] == [ + i for i, b in enumerate(out) if b == 0x0A + ] + assert b"ref struct" not in out + assert b"gcnew" not in out + assert b"assembly" not in out + + +def test_plain_cpp_is_not_rewritten(tmp_path): + src = b"int f(int a, int b) { return (a ^ b) % 7; }\n" + assert _normalize_cpp_cli(src) is None + + +def test_operators_survive_in_a_cli_file(tmp_path): + """The `^`/`%` rewrite only touches the suffix spelling, not the operators.""" + src = b"ref class C { int f(int a, int b) { return (a ^ b) % 7; } };\n" + out = _normalize_cpp_cli(src) + assert out is not None + assert b"(a ^ b) % 7" in out + + +def test_multiline_cli_attribute_keeps_line_numbers(tmp_path): + """A removed token that spans lines must keep its line breaks. + + `[assembly:AssemblyVersion(\n "1.0"\n)]` is ordinary formatting. Blanking + its newlines preserved byte length but merged source lines, so every symbol + below it reported a line number that was too low. + """ + p = tmp_path / "cli.h" + p.write_text( + "[assembly:AssemblyVersion(\n" + ' "1.0.0.0"\n' + ")];\n" + "namespace N {\n" + " public ref class Wrapper\n" + " {\n" + " public:\n" + " static void Init() { }\n" + " };\n" + "}\n" + ) + nodes = {n["label"]: n["source_location"] for n in extract_cpp(p)["nodes"]} + assert nodes["Wrapper"] == "L5" + assert nodes[".Init()"] == "L8" + + +def test_cli_normalization_preserves_line_breaks(tmp_path): + src = ( + "[assembly:AssemblyVersion(\n" + ' "1.0.0.0"\n' + ")];\n" + "namespace N {\n" + " public\n" # access specifier split from the keyword + " ref class W { };\n" + "}\n" + ).encode() + out = _normalize_cpp_cli(src) + assert out is not None + assert len(out) == len(src) + assert [i for i, b in enumerate(src) if b == 0x0A] == [ + i for i, b in enumerate(out) if b == 0x0A + ] + assert b"ref class" not in out + assert b"assembly" not in out + + +def test_attached_arithmetic_is_not_mistaken_for_a_handle(tmp_path): + """`a% b` and `hash^ mask` are modulo and XOR, not CLI type suffixes. + + Attachment to the preceding token does not separate the two readings — + `String^ s` and `a^ b` are lexically identical — so rewriting on + attachment alone turned arithmetic into `a b` and broke the statement. + """ + p = tmp_path / "ops.h" + p.write_text( + "namespace N {\n" + " public ref class Hasher\n" + " {\n" + " public:\n" + ' static System::String^ Name() { return gcnew System::String(""); }\n' + " static int Mix(int a, int b) { return a% b; }\n" + " static int Fold(int hash, int mask) { return hash^ mask; }\n" + " static void Track(System::Object^ o, int% n) { }\n" + " };\n" + "}\n" + ) + result = extract_cpp(p) + assert result.get("parse_errors") is None + labels = [n["label"] for n in result["nodes"]] + for expected in ("Hasher", ".Name()", ".Mix()", ".Fold()", ".Track()"): + assert expected in labels + + # The operator characters themselves survive in the parsed source. + out = _normalize_cpp_cli(p.read_bytes()) + assert b"return a% b;" in out + assert b"return hash^ mask;" in out + + +def test_screaming_case_constants_stay_arithmetic(tmp_path): + """`MASK^ value` is XOR against a constant, not a `MASK^` handle. + + SCREAMING_CASE is the macro/constant convention and never a .NET type + name, so a capitalized left side must also carry a lowercase letter. A + lone capital stays a type position for generic parameters (`T^ x`). + """ + p = tmp_path / "consts.h" + p.write_text( + '''namespace N { + public ref class Hasher + { + public: + static int Fold(int value) { return MASK^ value; } + static int Trim(int value) { return LIMIT% value; } + static System::Object^ Box(T^ item) { return gcnew System::Object(); } + }; +} +''' + ) + out = _normalize_cpp_cli(p.read_bytes()) + assert b"return MASK^ value;" in out + assert b"return LIMIT% value;" in out + assert b"System::Object Box(T item)" in out + + result = extract_cpp(p) + assert result.get("parse_errors") is None + assert ".Fold()" in _labels(p) + + +@pytest.mark.parametrize("expr", [ + b"int m = a% b;", + b"int x = hash^ mask;", + b"int c = count% 2;", + b"int d = (a ^ b) % 7;", + b"int e = a^*p;", + b"int f = a^&b;", + b"x %= y;", +]) +def test_arithmetic_forms_survive(expr): + src = b"ref class C { void f() { " + expr + b" } };" + out = _normalize_cpp_cli(src) + assert out is not None + assert expr in out, f"{expr!r} was rewritten" + + +@pytest.mark.parametrize("decl,rewritten", [ + (b"System::String^ s;", b"System::String s;"), + (b"List^ items;", b"List items;"), + (b"DataTable^ t;", b"DataTable t;"), + (b"int% n;", b"int n;"), + (b"void F(String^, int);", b"void F(String , int);"), + (b"array^ a;", b"array a;"), +]) +def test_cli_type_suffixes_are_still_rewritten(decl, rewritten): + src = b"ref class C { " + decl + b" };" + out = _normalize_cpp_cli(src) + assert out is not None + assert len(out) == len(src) + assert rewritten in out