diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 31ea1171d..a2f9ab93e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,6 +15,7 @@ name: Run Bazel Tests on: pull_request: types: [opened, reopened, synchronize] + workflow_call: jobs: code: runs-on: ubuntu-latest @@ -39,3 +40,19 @@ jobs: run: | bazel run //:ide_support bazel test //src/... + + - name: Prepare bundled consumer report + if: always() + # Creating tests-report directory + # Follow Symlinks via '-L' to copy correctly + # Copy everything inside the 'test-reports' folder + run: | + mkdir -p tests-report + rsync -amL --include='*/' --include='test.xml' --include='test.log' --exclude='*' bazel-testlogs/ tests-report/ + + - name: Upload bundled consumer report + if: always() + uses: actions/upload-artifact@v4 + with: + name: tests-report + path: tests-report diff --git a/.github/workflows/test_and_docs.yml b/.github/workflows/test_and_docs.yml index 542bc6da9..c52438950 100644 --- a/.github/workflows/test_and_docs.yml +++ b/.github/workflows/test_and_docs.yml @@ -38,13 +38,13 @@ jobs: bazel-docs-verify-target: "//:docs_check" # This is the user configurable part of the workflow - consumer-tests: - uses: ./.github/workflows/consumer_test.yml + unit-tests: + uses: ./.github/workflows/test.yml secrets: inherit docs-build: # Waits for consumer-tests but run only when docs verification succeeded - needs: [docs-verify, consumer-tests] + needs: [docs-verify, unit-tests] if: ${{ always() && needs.docs-verify.result == 'success' }} uses: eclipse-score/cicd-workflows/.github/workflows/docs.yml@main permissions: diff --git a/docs/internals/extensions/source_code_linker.md b/docs/internals/extensions/source_code_linker.md index 7ef81e599..51ba76904 100644 --- a/docs/internals/extensions/source_code_linker.md +++ b/docs/internals/extensions/source_code_linker.md @@ -64,7 +64,7 @@ These tags are extracted and matched to Sphinx needs via the `source_code_link` ### ✅ TestLink: Test Result Integration -TestLink scans test result XMLs from Bazel and converts each test case with metadata into Sphinx external needs, allowing links from tests to requirements. +TestLink scans test result XMLs from Bazel (bazel-testlogs) or in the folder 'tests-report' and converts each test case with metadata into Sphinx external needs, allowing links from tests to requirements. This depends on the `attribute_plugin` in our tooling repository, find it [here](https://github.com/eclipse-score/tooling/tree/main/python_basics/score_pytest) #### Test Tagging Options diff --git a/src/extensions/score_source_code_linker/tests/test_xml_parser.py b/src/extensions/score_source_code_linker/tests/test_xml_parser.py index c234e08bf..adc0d6b94 100644 --- a/src/extensions/score_source_code_linker/tests/test_xml_parser.py +++ b/src/extensions/score_source_code_linker/tests/test_xml_parser.py @@ -17,6 +17,7 @@ """ import xml.etree.ElementTree as ET +from collections.abc import Callable from pathlib import Path from typing import Any @@ -30,67 +31,73 @@ # Unsure if I should make these last a session or not +def _write_test_xml( + path: Path, + name: str, + result: str = "", + props: dict[str, str] | None = None, + file: str = "", + line: int = 0, +): + """Helper to create the XML structure for a test case.""" + ts = ET.Element("testsuites") + suite = ET.SubElement(ts, "testsuite") + + # Create testcase with attributes + tc_attrs = {"name": name} + if file: + tc_attrs["file"] = file + if line: + tc_attrs["line"] = str(line) + tc = ET.SubElement(suite, "testcase", tc_attrs) + + # Add failure/skipped status + if result == "failed": + ET.SubElement(tc, "failure", {"message": "failmsg"}) + elif result == "skipped": + ET.SubElement(tc, "skipped", {"message": "skipmsg"}) + + # Add properties if provided + if props: + props_el = ET.SubElement(tc, "properties") + for k, v in props.items(): + ET.SubElement(props_el, "property", {"name": k, "value": v}) + + # Save to file + ET.ElementTree(ts).write(path, encoding="utf-8", xml_declaration=True) + + @pytest.fixture -def tmp_xml_dirs(tmp_path: Path) -> tuple[Path, Path, Path]: - root: Path = tmp_path / "bazel-testlogs" - dir1: Path = root / "with_props" - dir2: Path = root / "no_props" - dir1.mkdir(parents=True) - dir2.mkdir(parents=True) - - def write(file_path: Path, testcases: list[ET.Element]): - ts = ET.Element("testsuites") - suite = ET.SubElement(ts, "testsuite") - for tc in testcases: - suite.append(tc) - tree = ET.ElementTree(ts) - tree.write(file_path, encoding="utf-8", xml_declaration=True) - - def make_tc( - name: str, - result: str = "", - props: dict[str, str] | None = None, - file: str = "", - line: int = 0, - ): - tc = ET.Element("testcase", {"name": name}) - if file: - tc.set("file", file) - if line: - tc.set("line", str(line)) - if result == "failed": - ET.SubElement(tc, "failure", {"message": "failmsg"}) - elif result == "skipped": - ET.SubElement(tc, "skipped", {"message": "skipmsg"}) - if props: - props_el = ET.SubElement(tc, "properties") - for k, v in props.items(): - ET.SubElement(props_el, "property", {"name": k, "value": v}) - return tc - - # File with properties - tc1 = make_tc( - "tc_with_props", - result="failed", - props={ - "PartiallyVerifies": "REQ1", - "FullyVerifies": "", - "TestType": "type", - "DerivationTechnique": "tech", - "Description": "desc", - }, - file="path1", - line=10, - ) - write(dir1 / "test.xml", [tc1]) - - # File without properties - # HINT: Once the assertions in xml_parser are back and active, this should allow us - # to catch that the tests Need to be changed too. - tc2 = make_tc("tc_no_props", file="path2", line=20) - write(dir2 / "test.xml", [tc2]) - - return root, dir1, dir2 +def tmp_xml_dirs(tmp_path: Path) -> Callable[..., tuple[Path, Path, Path]]: + def _tmp_xml_dirs(test_folder: str = "bazel-testlogs") -> tuple[Path, Path, Path]: + root = tmp_path / test_folder + dir1, dir2 = root / "with_props", root / "no_props" + + for d in (dir1, dir2): + d.mkdir(parents=True, exist_ok=True) + + # File with properties + _write_test_xml( + dir1 / "test.xml", + name="tc_with_props", + result="failed", + file="path1", + line=10, + props={ + "PartiallyVerifies": "REQ1", + "FullyVerifies": "", + "TestType": "type", + "DerivationTechnique": "tech", + "Description": "desc", + }, + ) + + # File without properties + _write_test_xml(dir2 / "test.xml", name="tc_no_props", file="path2", line=20) + + return root, dir1, dir2 + + return _tmp_xml_dirs @add_test_properties( @@ -98,17 +105,62 @@ def make_tc( test_type="requirements-based", derivation_technique="requirements-analysis", ) -def test_find_xml_files(tmp_xml_dirs: tuple[Path, Path, Path]): - """Ensure xml files are found as expected""" +def test_find_xml_files(tmp_xml_dirs: Callable[..., tuple[Path, Path, Path]]): + """Ensure xml files are found as expected if bazel-testlogs is used""" root: Path dir1: Path dir2: Path - root, dir1, dir2 = tmp_xml_dirs + root, dir1, dir2 = tmp_xml_dirs() found = xml_parser.find_xml_files(root) expected: set[Path] = {dir1 / "test.xml", dir2 / "test.xml"} assert set(found) == expected +def test_find_xml_folder(tmp_xml_dirs: Callable[..., tuple[Path, Path, Path]]): + """Ensure xml files are found as expected if bazel-testlogs is used""" + root: Path + root, _, _ = tmp_xml_dirs() + found = xml_parser.find_test_folder(base_path=root.parent) + assert found is not None + assert found == root + + +def test_find_xml_folder_test_reports( + tmp_xml_dirs: Callable[..., tuple[Path, Path, Path]], +): + # root is the 'tests-report' folder inside tmp_path + root, _, _ = tmp_xml_dirs(test_folder="tests-report") + # We pass the PARENT of 'tests-report' as the workspace root + found = xml_parser.find_test_folder(base_path=root.parent) + assert found is not None + assert found == root + + +def test_find_xml_files_test_reports( + tmp_xml_dirs: Callable[..., tuple[Path, Path, Path]], +): + """Ensure xml files are found as expected if tests-report is used""" + root: Path + dir1: Path + dir2: Path + root, dir1, dir2 = tmp_xml_dirs(test_folder="tests-report") + found = xml_parser.find_xml_files(dir=root) + assert found is not None + expected: set[Path] = {root / dir1 / "test.xml", root / dir2 / "test.xml"} + assert set(found) == expected + + +def test_early_return(tmp_path: Path): + """ + Ensure that if tests-report & bazel-testlogs is not found, + we return None for early return inside extension + """ + # Move the test execution context to a 100% empty folder + + found = xml_parser.find_test_folder(tmp_path) + assert found is None + + @add_test_properties( partially_verifies=["tool_req__docs_test_link_testcase"], test_type="requirements-based", @@ -152,12 +204,12 @@ def test_parse_properties(): test_type="requirements-based", derivation_technique="requirements-analysis", ) -def test_read_test_xml_file(tmp_xml_dirs: tuple[Path, Path, Path]): +def test_read_test_xml_file(tmp_xml_dirs: Callable[..., tuple[Path, Path, Path]]): """Ensure a whole pre-defined xml file is parsed correctly""" _: Path dir1: Path dir2: Path - _, dir1, dir2 = tmp_xml_dirs + _, dir1, dir2 = tmp_xml_dirs() needs1, no_props1 = xml_parser.read_test_xml_file(dir1 / "test.xml") assert isinstance(needs1, list) and len(needs1) == 1 diff --git a/src/extensions/score_source_code_linker/xml_parser.py b/src/extensions/score_source_code_linker/xml_parser.py index 53c18b23e..022168d26 100644 --- a/src/extensions/score_source_code_linker/xml_parser.py +++ b/src/extensions/score_source_code_linker/xml_parser.py @@ -171,16 +171,28 @@ def find_xml_files(dir: Path) -> list[Path]: return xml_paths +def find_test_folder(base_path: Path | None = None) -> Path | None: + ws_root = base_path if base_path is not None else find_ws_root() + assert ws_root is not None + if os.path.isdir(ws_root / "tests-report"): + return ws_root / "tests-report" + if os.path.isdir(ws_root / "bazel-testlogs"): + return ws_root / "bazel-testlogs" + logger.info("could not find tests-report or bazel-testlogs to parse testcases") + return None + + def run_xml_parser(app: Sphinx, env: BuildEnvironment): """ This is the 'main' function for parsing test.xml's and building testcase needs. It gets called from the source_code_linker __init__ """ - ws_root = find_ws_root() - assert ws_root is not None - bazel_testlogs = ws_root / "bazel-testlogs" - xml_file_paths = find_xml_files(bazel_testlogs) + testlogs_dir = find_test_folder() + # early return + if testlogs_dir is None: + return + xml_file_paths = find_xml_files(testlogs_dir) test_case_needs = build_test_needs_from_files(app, env, xml_file_paths) # Saving the test case needs for cache store_data_of_test_case_json(