Skip to content

Commit b5619d6

Browse files
BjwebbBjwebb
authored
Merge pull request #397 from OpenDataServices/cove-1366-org-file-ordering-no-gha-change
sort_xml: Handle extensions in the schema xsd correctly when sorting
2 parents 84d83cb + 1adfb33 commit b5619d6

8 files changed

+909
-8
lines changed

‎CHANGELOG.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
66

77
## [Unreleased]
88

9+
## [0.17.2] - 2022-06-15
10+
11+
### Fixed
12+
13+
- Handle extensions in the schema xsd correctly when sorting https://github.com/OpenDataServices/cove/issues/1366
14+
915
## [0.17.1] - 2021-07-21
1016

1117
### Fixed

‎examples/iati/iati-organisations-schema.xsd

Lines changed: 781 additions & 0 deletions
Large diffs are not rendered by default.

‎flatten-tool

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
#!/usr/bin/env python
22
import flattentool.cli
3+
34
flattentool.cli.main()

‎flattentool/sort_xml.py

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,23 @@ def get_schema_element(self, tag_name, name_attribute):
7474
return schema_element
7575
return schema_element
7676

77+
def handle_complexType(self, complexType):
78+
type_elements = []
79+
if complexType is not None:
80+
extension = complexType.find(
81+
"xsd:complexContent/xsd:extension", namespaces=namespaces
82+
)
83+
if extension:
84+
base = extension.attrib.get("base")
85+
complexType = self.get_schema_element("complexType", base)
86+
type_elements = self.handle_complexType(complexType)
87+
else:
88+
type_elements = []
89+
type_elements += complexType.findall(
90+
"xsd:choice/xsd:element", namespaces=namespaces
91+
) + complexType.findall("xsd:sequence/xsd:element", namespaces=namespaces)
92+
return type_elements
93+
7794
def element_loop(self, element, path):
7895
"""
7996
Return information about the children of the supplied element.
@@ -82,12 +99,7 @@ def element_loop(self, element, path):
8299
type_elements = []
83100
if "type" in a:
84101
complexType = self.get_schema_element("complexType", a["type"])
85-
if complexType is not None:
86-
type_elements = complexType.findall(
87-
"xsd:choice/xsd:element", namespaces=namespaces
88-
) + complexType.findall(
89-
"xsd:sequence/xsd:element", namespaces=namespaces
90-
)
102+
type_elements += self.handle_complexType(complexType)
91103

92104
children = (
93105
element.findall(

‎flattentool/tests/fixtures/iati-org-with-documents.xml

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
<?xml version='1.0' encoding='utf-8'?>
2+
<iati-organisations version="@generated-datetime">
3+
<!--XML generated by flatten-tool-->
4+
<iati-organisation default-currency="USD" last-updated-datetime="2022-05-24T07:35:54.240000+00:00" xml:lang="en">
5+
<organisation-identifier>XE-EXAMPLE-ORG</organisation-identifier>
6+
<name>
7+
<narrative>Example Org</narrative>
8+
</name>
9+
<reporting-org ref="XE-EXAMPLE-ORG" type="40">
10+
<narrative>Example Org</narrative>
11+
</reporting-org>
12+
<total-budget status="2">
13+
<period-start iso-date="2016-01-01"/>
14+
<period-end iso-date="2016-12-31"/>
15+
<value currency="USD" value-date="2016-01-01">100000</value>
16+
</total-budget>
17+
<total-budget status="2">
18+
<period-start iso-date="2017-01-01"/>
19+
<period-end iso-date="2017-12-31"/>
20+
<value currency="USD" value-date="2017-01-01">200000</value>
21+
</total-budget>
22+
<total-budget status="1">
23+
<period-start iso-date="2018-01-01"/>
24+
<period-end iso-date="2018-12-31"/>
25+
<value currency="USD" value-date="2018-01-01">300000</value>
26+
</total-budget>
27+
<total-budget status="1">
28+
<period-start iso-date="2019-01-01"/>
29+
<period-end iso-date="2019-12-31"/>
30+
<value currency="USD" value-date="2019-01-01">400000</value>
31+
</total-budget>
32+
<total-budget status="1">
33+
<period-start iso-date="2020-01-01"/>
34+
<period-end iso-date="2020-12-31"/>
35+
<value currency="USD" value-date="2020-01-01">300000</value>
36+
</total-budget>
37+
<total-budget status="1">
38+
<period-start iso-date="2021-01-01"/>
39+
<period-end iso-date="2021-12-31"/>
40+
<value currency="USD" value-date="2021-01-01">200000</value>
41+
</total-budget>
42+
<total-expenditure>
43+
<period-start iso-date="2015-01-01"/>
44+
<period-end iso-date="2015-12-31"/>
45+
<value currency="USD" value-date="2015-01-01">100000</value>
46+
</total-expenditure>
47+
<total-expenditure>
48+
<period-start iso-date="2016-01-01"/>
49+
<period-end iso-date="2016-12-31"/>
50+
<value currency="USD" value-date="2016-01-01">100000</value>
51+
</total-expenditure>
52+
<total-expenditure>
53+
<period-start iso-date="2017-01-01"/>
54+
<period-end iso-date="2017-12-31"/>
55+
<value currency="USD" value-date="2017-01-01">100000</value>
56+
</total-expenditure>
57+
<document-link format="application/pdf" url="http://example.org/document.pdf">
58+
<title>
59+
<narrative xml:lang="en">Example Org document</narrative>
60+
</title>
61+
<category code="B02"/>
62+
<language code="en"/>
63+
<document-date iso-date="2018-01-01"/>
64+
<recipient-country code="SS"/>
65+
</document-link>
66+
</iati-organisation>
67+
</iati-organisations>

‎flattentool/tests/fixtures/xlsx/IATI CoVE #organisation #broken-docs #template #public #demo.xlsx

14.8 KB
Binary file not shown.

‎flattentool/tests/test_unflatten.py

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,14 @@ def test_unflatten_xml_comment(tmpdir, dirname):
120120

121121

122122
@pytest.mark.parametrize("input_format", ["xlsx", "ods"])
123-
def test_unflatten_org_xml_xlsx(tmpdir, input_format):
123+
def test_unflatten_org_xml_minimal(tmpdir, input_format):
124+
schema_path = "examples/iati"
125+
schemas = [
126+
"iati-activities-schema.xsd",
127+
"iati-organisations-schema.xsd",
128+
"iati-common.xsd",
129+
]
130+
schema_filepaths = ["{}/{}".format(schema_path, schema) for schema in schemas]
124131
unflatten(
125132
input_name="flattentool/tests/fixtures/{}/iati-org.{}".format(
126133
input_format, input_format
@@ -130,13 +137,40 @@ def test_unflatten_org_xml_xlsx(tmpdir, input_format):
130137
id_name="organisation-identifier",
131138
xml=True,
132139
metatab_name="Meta",
140+
xml_schemas=schema_filepaths,
133141
)
134142
assert (
135143
open("flattentool/tests/fixtures/iati-org.xml").read()
136144
== tmpdir.join("output.xml").read()
137145
)
138146

139147

148+
@pytest.mark.parametrize("input_format", ["xlsx"])
149+
def test_unflatten_org_xml_with_documents(tmpdir, input_format):
150+
schema_path = "examples/iati"
151+
schemas = [
152+
"iati-activities-schema.xsd",
153+
"iati-organisations-schema.xsd",
154+
"iati-common.xsd",
155+
]
156+
schema_filepaths = ["{}/{}".format(schema_path, schema) for schema in schemas]
157+
unflatten(
158+
input_name="flattentool/tests/fixtures/{}/IATI CoVE #organisation #broken-docs #template #public #demo.{}".format(
159+
input_format, input_format
160+
),
161+
output_name=tmpdir.join("output.xml").strpath,
162+
input_format=input_format,
163+
id_name="organisation-identifier",
164+
xml=True,
165+
metatab_name="Meta",
166+
xml_schemas=schema_filepaths,
167+
)
168+
assert (
169+
open("flattentool/tests/fixtures/iati-org-with-documents.xml").read()
170+
== tmpdir.join("output.xml").read()
171+
)
172+
173+
140174
@pytest.mark.parametrize("input_format", ["xlsx", "ods"])
141175
def test_unflatten_empty_column_header(tmpdir, input_format):
142176
unflatten(

‎setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ def run(self):
4343

4444
setup(
4545
name="flattentool",
46-
version="0.17.1",
46+
version="0.17.2",
4747
author="Open Data Services",
4848
author_email="[email protected]",
4949
packages=["flattentool"],

0 commit comments

Comments
 (0)