Skip to content

Commit 6ecbbd8

Browse files
Titas-GhoshTitas-Ghosh
committed
test: add phase-2 cross-runtime conformance mapping baseline
1 parent 5d46e60 commit 6ecbbd8

File tree

3 files changed

+275
-0
lines changed

3 files changed

+275
-0
lines changed

‎tests/protocol_fixtures/PROTOCOL_FIXTURES.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,16 @@ This directory contains the phase-1 protocol conformance baseline for Python.
44

55
- `schema.phase1.json`: fixture document shape and supported case targets.
66
- `python_phase1_cases.json`: initial baseline cases (report-only mode metadata).
7+
- `cross_runtime_matrix.phase2.json`: phase-2 cross-runtime mapping matrix in report-only mode.
78

89
Phase-1 scope:
910

1011
- No runtime behavior changes.
1112
- Python-only execution through `tests/test_protocol_conformance.py`.
1213
- Fixture format is language-neutral to enable future cross-binding runners.
14+
15+
Phase-2 scope (mapping only):
16+
17+
- No runtime behavior changes.
18+
- Adds a cross-runtime matrix to track per-case audit status and classification.
19+
- Keeps CI non-blocking for non-Python runtimes by marking them as `not_audited` until adapters are added.

‎tests/protocol_fixtures/cross_runtime_matrix.phase2.json‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
{
2+
"schema_version": "1.0",
3+
"phase": "2",
4+
"mode": "report_only",
5+
"source_fixture": "python_phase1_cases.json",
6+
"runtimes": [
7+
"python",
8+
"cpp",
9+
"matlab",
10+
"octave",
11+
"verilog"
12+
],
13+
"classifications": [
14+
"required",
15+
"implementation_defined",
16+
"known_deviation"
17+
],
18+
"statuses": [
19+
"observed_pass",
20+
"observed_fail",
21+
"not_audited"
22+
],
23+
"cases": [
24+
{
25+
"id": "parse_params/simple_types_and_whitespace",
26+
"target": "parse_params",
27+
"runtime_results": {
28+
"python": {
29+
"status": "observed_pass",
30+
"classification": "required",
31+
"note": "Phase-1 baseline execution."
32+
},
33+
"cpp": {
34+
"status": "not_audited",
35+
"classification": "required",
36+
"note": "Audit planned in phase 2."
37+
},
38+
"matlab": {
39+
"status": "not_audited",
40+
"classification": "required",
41+
"note": "Audit planned in phase 2."
42+
},
43+
"octave": {
44+
"status": "not_audited",
45+
"classification": "required",
46+
"note": "Audit planned in phase 2."
47+
},
48+
"verilog": {
49+
"status": "not_audited",
50+
"classification": "implementation_defined",
51+
"note": "May require binding-specific interpretation."
52+
}
53+
}
54+
},
55+
{
56+
"id": "parse_params/embedded_equals_not_split",
57+
"target": "parse_params",
58+
"runtime_results": {
59+
"python": {
60+
"status": "observed_pass",
61+
"classification": "required",
62+
"note": "Phase-1 baseline execution."
63+
},
64+
"cpp": {
65+
"status": "not_audited",
66+
"classification": "required",
67+
"note": "Audit planned in phase 2."
68+
},
69+
"matlab": {
70+
"status": "not_audited",
71+
"classification": "required",
72+
"note": "Audit planned in phase 2."
73+
},
74+
"octave": {
75+
"status": "not_audited",
76+
"classification": "required",
77+
"note": "Audit planned in phase 2."
78+
},
79+
"verilog": {
80+
"status": "not_audited",
81+
"classification": "implementation_defined",
82+
"note": "May require binding-specific interpretation."
83+
}
84+
}
85+
},
86+
{
87+
"id": "initval/valid_list_sets_simtime",
88+
"target": "initval",
89+
"runtime_results": {
90+
"python": {
91+
"status": "observed_pass",
92+
"classification": "required",
93+
"note": "Phase-1 baseline execution."
94+
},
95+
"cpp": {
96+
"status": "not_audited",
97+
"classification": "required",
98+
"note": "Audit planned in phase 2."
99+
},
100+
"matlab": {
101+
"status": "not_audited",
102+
"classification": "required",
103+
"note": "Audit planned in phase 2."
104+
},
105+
"octave": {
106+
"status": "not_audited",
107+
"classification": "required",
108+
"note": "Audit planned in phase 2."
109+
},
110+
"verilog": {
111+
"status": "not_audited",
112+
"classification": "implementation_defined",
113+
"note": "May require binding-specific interpretation."
114+
}
115+
}
116+
},
117+
{
118+
"id": "initval/invalid_input_returns_empty_and_preserves_simtime",
119+
"target": "initval",
120+
"runtime_results": {
121+
"python": {
122+
"status": "observed_pass",
123+
"classification": "required",
124+
"note": "Phase-1 baseline execution."
125+
},
126+
"cpp": {
127+
"status": "not_audited",
128+
"classification": "required",
129+
"note": "Audit planned in phase 2."
130+
},
131+
"matlab": {
132+
"status": "not_audited",
133+
"classification": "required",
134+
"note": "Audit planned in phase 2."
135+
},
136+
"octave": {
137+
"status": "not_audited",
138+
"classification": "required",
139+
"note": "Audit planned in phase 2."
140+
},
141+
"verilog": {
142+
"status": "not_audited",
143+
"classification": "implementation_defined",
144+
"note": "May require binding-specific interpretation."
145+
}
146+
}
147+
},
148+
{
149+
"id": "write_zmq/list_payload_prepends_timestamp_without_mutation",
150+
"target": "write_zmq",
151+
"runtime_results": {
152+
"python": {
153+
"status": "observed_pass",
154+
"classification": "required",
155+
"note": "Phase-1 baseline execution."
156+
},
157+
"cpp": {
158+
"status": "not_audited",
159+
"classification": "required",
160+
"note": "Audit planned in phase 2."
161+
},
162+
"matlab": {
163+
"status": "not_audited",
164+
"classification": "required",
165+
"note": "Audit planned in phase 2."
166+
},
167+
"octave": {
168+
"status": "not_audited",
169+
"classification": "required",
170+
"note": "Audit planned in phase 2."
171+
},
172+
"verilog": {
173+
"status": "not_audited",
174+
"classification": "implementation_defined",
175+
"note": "May require binding-specific interpretation."
176+
}
177+
}
178+
},
179+
{
180+
"id": "write_zmq/non_list_payload_forwarded_as_is",
181+
"target": "write_zmq",
182+
"runtime_results": {
183+
"python": {
184+
"status": "observed_pass",
185+
"classification": "required",
186+
"note": "Phase-1 baseline execution."
187+
},
188+
"cpp": {
189+
"status": "not_audited",
190+
"classification": "required",
191+
"note": "Audit planned in phase 2."
192+
},
193+
"matlab": {
194+
"status": "not_audited",
195+
"classification": "required",
196+
"note": "Audit planned in phase 2."
197+
},
198+
"octave": {
199+
"status": "not_audited",
200+
"classification": "required",
201+
"note": "Audit planned in phase 2."
202+
},
203+
"verilog": {
204+
"status": "not_audited",
205+
"classification": "implementation_defined",
206+
"note": "May require binding-specific interpretation."
207+
}
208+
}
209+
}
210+
]
211+
}

‎tests/test_protocol_conformance_phase2.py‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
import json
2+
from pathlib import Path
3+
4+
5+
FIXTURE_DIR = Path(__file__).parent / "protocol_fixtures"
6+
PHASE1_CASES_PATH = FIXTURE_DIR / "python_phase1_cases.json"
7+
PHASE2_MATRIX_PATH = FIXTURE_DIR / "cross_runtime_matrix.phase2.json"
8+
9+
EXPECTED_RUNTIMES = {"python", "cpp", "matlab", "octave", "verilog"}
10+
EXPECTED_CLASSIFICATIONS = {"required", "implementation_defined", "known_deviation"}
11+
EXPECTED_STATUSES = {"observed_pass", "observed_fail", "not_audited"}
12+
13+
14+
def _load_json(path):
15+
with path.open("r", encoding="utf-8") as f:
16+
return json.load(f)
17+
18+
19+
def _phase1_cases():
20+
doc = _load_json(PHASE1_CASES_PATH)
21+
return {case["id"]: case for case in doc["cases"]}
22+
23+
24+
def _phase2_matrix():
25+
return _load_json(PHASE2_MATRIX_PATH)
26+
27+
28+
def test_phase2_matrix_metadata_and_enums():
29+
doc = _phase2_matrix()
30+
assert doc["phase"] == "2"
31+
assert doc["mode"] == "report_only"
32+
assert doc["source_fixture"] == "python_phase1_cases.json"
33+
assert set(doc["runtimes"]) == EXPECTED_RUNTIMES
34+
assert set(doc["classifications"]) == EXPECTED_CLASSIFICATIONS
35+
assert set(doc["statuses"]) == EXPECTED_STATUSES
36+
37+
38+
def test_phase2_matrix_covers_all_phase1_cases():
39+
phase1 = _phase1_cases()
40+
matrix_cases = _phase2_matrix()["cases"]
41+
matrix_ids = {case["id"] for case in matrix_cases}
42+
assert matrix_ids == set(phase1.keys())
43+
44+
45+
def test_phase2_matrix_rows_have_consistent_shape():
46+
phase1 = _phase1_cases()
47+
for row in _phase2_matrix()["cases"]:
48+
assert row["id"] in phase1
49+
assert row["target"] == phase1[row["id"]]["target"]
50+
assert set(row["runtime_results"].keys()) == EXPECTED_RUNTIMES
51+
52+
for runtime, result in row["runtime_results"].items():
53+
assert result["status"] in EXPECTED_STATUSES
54+
assert result["classification"] in EXPECTED_CLASSIFICATIONS
55+
assert isinstance(result["note"], str) and result["note"].strip()
56+
if runtime == "python":
57+
assert result["status"] == "observed_pass"

0 commit comments

Comments
 (0)