refactor: ensure consistency, migrate, sanitise

migrate from aggregating test blocks in pure .rs file testing, to a proper project structure to manage dependancies properly(tokio,...)

Author: x0rw

exts/rust-code-runner/__init__.py

@@ -1,26 +1,49 @@
 from . import rust_examples_aggregate 
 from . import rustc 
 import os
+from pathlib import Path
 
 def setup(app):
 
-    app.output_rust_file = "build/rust-code-blocks/generated.rs"
 
-    # create build dir 
-    if not os.path.exists("build/rust-code-blocks"):
-        os.makedirs("build/rust-code-blocks")
-    if os.path.isfile(app.output_rust_file):
-        with open(app.output_rust_file, 'w'):
-            pass  
+    # Define output directory
+    app.output_rust = "build/rust-code-blocks/"
+
+    # Ensure the src directory exists
+    base_dir = Path(app.output_rust)
+    src_dir = base_dir / "src"
+    src_dir.mkdir(parents=True, exist_ok=True)
+
+
+    # Write Cargo.toml with required dependencies
+    cargo_toml = base_dir / "Cargo.toml"
+    cargo_toml.write_text(
+        """[package]
+name = "sc_generated_tests"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+tokio = { version = "1", features = ["macros", "rt-multi-thread"] }
+""",
+        encoding="utf-8",
+    )
+
+
+    print(f"Setup complete in '{base_dir.resolve()}'")
 
     # we hook into 'source-read' because data is mutable at this point and easier to parse 
     # and it also makes this extension indepandant from `needs`.
-    #
-    app.connect('source-read', rust_examples_aggregate.preprocess_rst_for_rust_code)
-
-    if app.config.test_rust_blocks:
+    if not app.config.test_rust_blocks:
+        # empty lib.rs on every run (incremental build is not supported)
+        with open(app.output_rust + "src/lib.rs", "w", encoding="utf-8"):
+            pass
+        app.connect('source-read', rust_examples_aggregate.preprocess_rst_for_rust_code)
+    else:
         app.connect('build-finished', rustc.check_rust_test_errors)
+
     return {
         'version': '0.1',
         'parallel_read_safe': False,
+        'parallel_write_safe': False,
     }

exts/rust-code-runner/rust_examples_aggregate.py

@@ -64,23 +64,51 @@ def replacer(match):
     modified_text = code_block_re.sub(replacer, source_text)
     return modified_text
 
+import re
+
+def sanitize_code_blocks(code_blocks):
+    """
+    Removes unwanted attributes from each Rust code block:
+      - `#[macro_export]` (to avoid exported-macro conflicts)
+      - `#[tokio::main]`     (to keep compilation as a library/test)
+    """
+    patterns = [
+        r'\s*#\s*\[macro_export\]',
+        r'\s*#\s*\[tokio::main\]'
+    ]
+    sanitized = []
+    for block in code_blocks:
+        lines = block.splitlines()
+        cleaned = [
+            line for line in lines
+            if not any(re.match(pat, line) for pat in patterns)
+        ]
+        sanitized.append("\n".join(cleaned))
+    return sanitized
 
 def preprocess_rst_for_rust_code(app, docname, source):
 
     original_content = source[0]
     code_blocks = extract_code_blocks(original_content)
+    code_blocks = sanitize_code_blocks(code_blocks) 
     modified_content = remove_hidden_blocks_from_document(original_content)
     source[0] = modified_content
 
     # print(f"Original content length: {len(original_content)}")
     # print(f"Extracted {len(code_blocks)} code blocks")
 
     safe_docname = docname.replace("/", "_").replace("-", "_")
-    with open(app.output_rust_file, "a", encoding="utf-8") as f:
-        for i, block in enumerate(code_blocks, start=1):
-            f.write(f"// ==== Code Block {i} ====\n")
-            f.write("#[test]\n")
-            f.write(f"fn test_block_{safe_docname}_{i}() {{\n")
-            for line in block.splitlines():
-                f.write(f"    {line}\n") 
-            f.write("}\n\n")
+    try:
+        with open(app.output_rust + "src/lib.rs", "a", encoding="utf-8") as f:
+            for i, block in enumerate(code_blocks, start=1):
+                f.write(f"// ==== Code Block {i} ====\n")
+                f.write(f"mod code_block_{i}_{safe_docname} {{\n")
+                f.write("    #[test]\n")
+                f.write(f"    fn test_block_{safe_docname}_{i}() {{\n")
+                for line in block.splitlines():
+                    f.write(f"        {line}\n")  # extra indent for the module
+                f.write("    }\n")  # close fn
+                f.write("}\n\n")    # close mod
+    except Exception as e:
+        print("Error writing file:", e)
+

exts/rust-code-runner/rustc.py

@@ -1,4 +1,6 @@
 import json
+import sys 
+import os 
 import subprocess
 
 def print_code_snippet(file_path, line_num, context=3):
@@ -31,52 +33,51 @@ def print_code_snippet(file_path, line_num, context=3):
         print(f"Could not read file {file_path}: {e}")
 
 
-def parse_rustc_json(stderr: str, file_path):
-    """
-    Parses rustc's JSON output and prints only the first error with a single snippet.
-
-    Args:
-        stderr (str): JSON-formatted stderr output from rustc.
-        file_path: Path to the Rust file.
+import json
 
-    Returns:
-        None
+def parse_cargo_errors(output: str, output_rust):
+    """
+    Parses Cargo’s JSON output and prints only the first compiler error it finds.
+    Ignores warnings and notes entirely.
     """
-    for line in stderr.splitlines():
+    for line in output.splitlines():
         line = line.strip()
         if not line:
             continue
 
         try:
-            diagnostic = json.loads(line)
+            rec = json.loads(line)
         except json.JSONDecodeError:
             continue
 
-        if diagnostic.get("$message_type") != "diagnostic":
+        # Only look at compiler messages
+        if rec.get("reason") != "compiler-message":
             continue
 
-        if diagnostic.get("level") != "error":
+        msg = rec["message"]
+        # Skip anything that isn't an error
+        if msg.get("level") != "error":
             continue
 
-        message = diagnostic.get("message", "")
-        spans = diagnostic.get("spans", [])
+        text = msg.get("message", "")
+        spans = msg.get("spans", [])
 
-        # Prefer the primary span in the current file
+        # Print the high-level error first
+        print(f"\nerror: {text}")
+
+        # Then try to show its primary location
         for span in spans:
-            if span.get("is_primary") and span["file_name"] == file_path: 
-                line_num = span["line_start"]
+            if span.get("is_primary"):
+                file = span.get("file_name")
+                line_start = span.get("line_start")
                 label = span.get("label", "")
-                print(f"error: line {line_num}: {message}")
-                if label:
-                    print(f"--> {label}")
-                print("=" * 25)
-                snippet = print_code_snippet(file_path, line_num, context=8)
-                print(snippet)
-                print("=" * 25)
-                return  # we return because we only print the first error--in json format there can be multiple error messages(primary and non primary) for 1 error-- if you want to see them comment this line.
-
-        # fallback: print the error message if no span in the current file
-        print(f"error: {message}")
+                print(f"  --> {file}:{line_start} {label}".rstrip(), file= sys.stderr)
+                # and a snippet
+                snippet = print_code_snippet(output_rust + file, line_start, context=5)
+                print("\n" + snippet, file = sys.stderr)
+                break
+
+        # Stop after the first error
         return
 
 def check_rust_test_errors(app, exception):
@@ -86,31 +87,31 @@ def check_rust_test_errors(app, exception):
     This function is connected to the Sphinx build lifecycle and is executed after the build finishes.
     It invokes `rustc` in test mode on the generated Rust file and reports any compilation or test-related
     errors.
-
-    Args:
-        app: The Sphinx application object. Must have an `output_rust_file` attribute containing
-             the path to the generated Rust source file.
-        exception: Exception raised during the build process, or None if the build completed successfully.
     """
-    rs_path = app.output_rust_file
+    rs_path = app.output_rust
+    cargo_toml_path = os.path.join(rs_path, "Cargo.toml")
     # Run the Rust compiler in test mode with JSON error output format.
     # capturing stdout and stderr as text.
     result = subprocess.run(
-        ["rustc", "--test", "--edition=2024", "--error-format=json", "--emit=metadata", rs_path],
-        # --emit=metadata or else rustc will produce a binary ./generated
+        [
+            "cargo",
+            "test",
+            "--message-format=json",
+            "--manifest-path",
+            cargo_toml_path
+        ],
         capture_output=True,
-        text=True
+        text=True,
     )
 
     if result.returncode != 0:
-        print("--- rustc Errors/Warnings ---")
-        parse_rustc_json(result.stderr, app.output_rust_file)
+        print("\033[31m--- Cargo test errors ---\033[0m")
+        parse_cargo_errors(result.stdout, app.output_rust)  # parse stdout JSON lines
         # print("--- rustc Output ---")
         # print(result.stdout)
-
     else:
-        print("--- rustc Output  ---")
-        print(result.stdout)
-        if result.stderr: 
-            print("\n\n--- rustc Warnings---")
-            print(result.stderr)
+        print("\033[1;32mAll tests succeeded\033[0m") # ANSI magic
+        # print(result.stdout)
+        # if result.stderr:
+            # print("\n\n--- rustc Warnings ---")
+            # print(result.stderr)