Skip to content

Commit 2a325c2

Browse files
committed
Prototype markdown table import into Github issues
1 parent 3958527 commit 2a325c2

File tree

11 files changed

+636
-3
lines changed

11 files changed

+636
-3
lines changed

Makefile

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
PYTHON_FILES = \
2+
scripts/markdown_to_github_issue.py \
3+
scripts/tests/*.py \
4+
5+
all: check test
6+
7+
check: check_python
8+
9+
check_python: check_ruff check_mypy
10+
11+
check_ruff:
12+
ruff check $(PYTHON_FILES)
13+
14+
check_mypy:
15+
mypy $(PYTHON_FILES)
16+
17+
test: test_python
18+
19+
test_python:
20+
pytest scripts/tests/*.py
21+
22+
.PHONY: all check check_python check_mypy test test_python

pyproject.toml

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ dependencies = [
1212
"sphinx-autobuild>=2024.10.3",
1313
"sphinx-needs>=5.1.0",
1414
"sphinx-rtd-theme>=3.0.2",
15+
"markdown>=3.9",
16+
"beautifulsoup4[xml]>=4.13.5",
17+
"lxml>=6.0.1",
18+
"pygithub>=2.8.1",
1519
]
1620

1721
[tool.uv.workspace]
@@ -30,6 +34,7 @@ lint.select = [
3034
"TID", # Some good import practices
3135
"C4", # Catch incorrect use of comprehensions, dict, list, etc
3236
]
37+
3338
# Remove or reduce ignores to catch more issues
3439
lint.ignore = [
3540
"E501", # Ignore long lines, bc we use them frequently while composing .rst templates
@@ -44,6 +49,8 @@ allow-dict-calls-with-keyword-arguments = true
4449

4550
[dependency-groups]
4651
dev = [
52+
"mypy>=1.17.1",
53+
"pytest>=8.4.2",
4754
"ruff>=0.12.3",
55+
"types-markdown>=3.9.0.20250906",
4856
]
49-

scripts/README.md

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
### `auto-pr-helper.py`
1+
# Scripts for automating processes around the coding guidelines
2+
3+
## `auto-pr-helper.py`
24

35
This script is a utility for automating the generation of guidelines. It takes a GitHub issue's JSON data from standard input, parses its body (which is expected to follow a specific issue template), and converts it into a formatted reStructuredText (`.rst`) guideline.
46

@@ -17,9 +19,18 @@ cat path/to/your_issue.json | uv run scripts/auto-pr-helper.py
1719
```
1820

1921
#### 2. Fetching from the GitHub API directly
22+
2023
You can fetch the data for a live issue directly from the GitHub API using curl and pipe it to the script. This is useful for getting the most up-to-date content.
2124

2225
```bash
2326
curl https://api.github.com/repos/rustfoundation/safety-critical-rust-coding-guidelines/issues/156 | uv run ./scripts/auto-pr-helper.py
2427
```
25-
```
28+
29+
## `markdown_to_github_issue.py`
30+
31+
### How to use
32+
33+
You need to create a personal access token for the target repository as [described here](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-fine-grained-personal-access-token).
34+
Make sure the "Issues" permission is granted as "read and write" for the token.
35+
36+
Pass the token to the tool via the `--auth-token` command line parameter.

scripts/__init__.py

Whitespace-only changes.
Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
#!/usr/bin/env python3
2+
3+
from __future__ import annotations
4+
5+
import argparse
6+
from dataclasses import dataclass
7+
from pathlib import Path
8+
from typing import Final
9+
10+
import bs4
11+
import markdown
12+
from github import Auth, Github
13+
14+
EXPECTED_HEADINGS: Final[list[str]] = [
15+
"Guideline",
16+
"Guideline Name",
17+
"MISRA C:2025 Status",
18+
"Decidability",
19+
"Scope",
20+
"Rationale",
21+
"Applicability",
22+
"Adjusted Category",
23+
]
24+
25+
26+
@dataclass(eq=True)
27+
class MISRA_Rule:
28+
title: str
29+
section: str | None = None
30+
status: str | None = None
31+
decidability: str | None = None
32+
scope: str | None = None
33+
rationale: str | None = None
34+
applicability: str | None = None
35+
category: str | None = None
36+
37+
@classmethod
38+
def from_cols(cls, cols: list[str | None]) -> MISRA_Rule | None:
39+
assert len(cols) == len(EXPECTED_HEADINGS), (
40+
f"Expected {len(EXPECTED_HEADINGS)}, got {len(cols)}"
41+
)
42+
# Cannot create rule without a title
43+
title = cols[EXPECTED_HEADINGS.index("Guideline Name")]
44+
if title is None:
45+
return None
46+
return MISRA_Rule(
47+
title=title,
48+
section=cols[EXPECTED_HEADINGS.index("Guideline")],
49+
status=cols[EXPECTED_HEADINGS.index("MISRA C:2025 Status")],
50+
decidability=cols[EXPECTED_HEADINGS.index("Decidability")],
51+
scope=cols[EXPECTED_HEADINGS.index("Scope")],
52+
rationale=cols[EXPECTED_HEADINGS.index("Rationale")],
53+
applicability=cols[EXPECTED_HEADINGS.index("Applicability")],
54+
category=cols[EXPECTED_HEADINGS.index("Adjusted Category")],
55+
)
56+
57+
@property
58+
def issue_body(self) -> str:
59+
# FIXME(senier): Properly layout (we could even use .github/ISSUE_TEMPLATE/CODING-GUILDELINE.yml to validate the format)
60+
# FIXME(senier): Transform into dedicated coding guidline object and do layouting there
61+
return str(self)
62+
63+
64+
def convert_md(file: Path) -> list[MISRA_Rule] | None:
65+
result = None
66+
67+
with file.open() as f:
68+
html = markdown.markdown(f.read(), extensions=["tables"], output_format="xhtml")
69+
soup = bs4.BeautifulSoup(html, features="lxml")
70+
71+
table = soup.find("table")
72+
if table is None or not isinstance(table, bs4.Tag):
73+
return None
74+
75+
headings = table.find("thead")
76+
if headings is None or not isinstance(headings, bs4.Tag):
77+
return None
78+
79+
values = [h.text for h in headings.find_all("th")]
80+
if values != EXPECTED_HEADINGS:
81+
return None
82+
83+
body = table.find("tbody")
84+
if body is None or not isinstance(body, bs4.Tag):
85+
return None
86+
87+
for row in body.find_all("tr"):
88+
if row is None or not isinstance(row, bs4.Tag):
89+
continue
90+
91+
cols = [r.text or None for r in row.find_all("td")]
92+
assert len(cols) == 0 or len(cols) == len(EXPECTED_HEADINGS), f"{cols}"
93+
94+
# skip empty rows
95+
if all(c is None for c in cols):
96+
continue
97+
98+
if result is None:
99+
result = []
100+
rule = MISRA_Rule.from_cols(cols)
101+
if rule is not None:
102+
result.append(rule)
103+
return result
104+
105+
106+
def create_issues(repo_name: str, token: str, rules: list[MISRA_Rule]):
107+
auth = Auth.Token(token=token)
108+
github = Github(auth=auth)
109+
repo = github.get_repo(repo_name)
110+
111+
for rule in rules:
112+
if rule.title is None:
113+
continue
114+
repo.create_issue(title=rule.title, body=rule.issue_body)
115+
116+
117+
def import_rules(file: Path, repo: str, token: str) -> int | str:
118+
md = convert_md(file)
119+
if md is None:
120+
return "No rules found"
121+
create_issues(repo_name=repo, token=token, rules=md)
122+
return 1
123+
124+
125+
def main() -> int | str:
126+
parser = argparse.ArgumentParser()
127+
parser.add_argument(
128+
"-m",
129+
"--markdown",
130+
type=Path,
131+
required=True,
132+
help="Markdown file to extract rules from",
133+
)
134+
parser.add_argument(
135+
"-r",
136+
"--repository",
137+
type=str,
138+
required=True,
139+
help="Github repository to import rules to (format: account/repository)",
140+
)
141+
parser.add_argument(
142+
"-a",
143+
"--auth-token",
144+
type=str,
145+
required=True,
146+
help="Github authentication token",
147+
)
148+
args = parser.parse_args()
149+
return import_rules(file=args.markdown, repo=args.repository, token=args.auth_token)
150+
151+
152+
if __name__ == "__main__":
153+
main()

scripts/tests/__init__.py

Whitespace-only changes.

scripts/tests/data/empty_table.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
| Guideline | Guideline Name | MISRA C:2025 Status | Decidability | Scope | Rationale | Applicability | Adjusted Category |
2+
| --- | --- | --- | --- | --- | --- | --- | --- |
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
| Guideline | Guideline Name | MISRA C:2025 Status | Decidability | Scope | Rationale | Applicability | Adjusted Category |
2+
| --- | --- | --- | --- | --- | --- | --- | --- |
3+
| D.1.2 | The use of language extensions should be minimized | Advisory | | | IDB | Yes Yes | Required |
4+
| R.1.3 | There shall be no occurrence of undefined or critical unspecified behaviour | Required | Undecidable | System | UB, IDB | Yes Yes | Required |
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
| Guideline | Guideline Name | MISRA C:2025 Status | Decidability | Scope | Rationale | Applicability | Adjusted Category |
2+
| --- | --- | --- | --- | --- | --- | --- | --- |
3+
| D.1.2 | The use of language extensions should be minimized | Advisory | | | IDB | Yes Yes | Required |
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
from pathlib import Path
2+
3+
import pytest
4+
5+
from .. import markdown_to_github_issue as mtgi # noqa: TID252
6+
7+
DATA_DIR = Path("scripts/tests/data")
8+
9+
10+
@pytest.mark.parametrize(
11+
["file", "expected"],
12+
[
13+
("empty_table", None),
14+
(
15+
"table_with_single_line",
16+
[
17+
mtgi.MISRA_Rule(
18+
section="D.1.2",
19+
title="The use of language extensions should be minimized",
20+
status="Advisory",
21+
rationale="IDB",
22+
applicability="Yes Yes",
23+
category="Required",
24+
),
25+
],
26+
),
27+
(
28+
"table_with_multiple_lines",
29+
[
30+
mtgi.MISRA_Rule(
31+
section="D.1.2",
32+
title="The use of language extensions should be minimized",
33+
status="Advisory",
34+
rationale="IDB",
35+
applicability="Yes Yes",
36+
category="Required",
37+
),
38+
mtgi.MISRA_Rule(
39+
section="R.1.3",
40+
title="There shall be no occurrence of undefined or critical unspecified behaviour",
41+
status="Required",
42+
decidability="Undecidable",
43+
scope="System",
44+
rationale="UB, IDB",
45+
applicability="Yes Yes",
46+
category="Required",
47+
),
48+
],
49+
),
50+
],
51+
)
52+
def test_foo(file: str, expected: list[mtgi.MISRA_Rule] | None) -> None:
53+
assert mtgi.convert_md(DATA_DIR / f"{file}.md") == expected

0 commit comments

Comments
 (0)