Skip to content

Commit 39692b2

Browse files
bennatibennati
committed
Parse setup.py file with AST instead of using regex.
Signed-off-by: Bennati, Stefano <[email protected]>
1 parent 98fe24c commit 39692b2

File tree

5 files changed

+235
-21
lines changed

5 files changed

+235
-21
lines changed

‎src/python_inspector/setup_py_live_eval.py‎

Lines changed: 62 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,23 +10,26 @@
1010
"""Generate requirements from `setup.py` and `requirements-devel.txt`."""
1111

1212
import os
13-
import re
1413
import sys
14+
import ast
1515

1616
try:
1717
import configparser
1818
except ImportError: # pragma: no cover
1919
import ConfigParser as configparser
2020

2121
import mock
22+
import setuptools
23+
import distutils.core
2224
from commoncode.command import pushd
2325
from packvers.requirements import Requirement
2426

2527

2628
def minver_error(pkg_name):
2729
"""Report error about missing minimum version constraint and exit."""
2830
print(
29-
'ERROR: specify minimal version of "{0}" using ' '">=" or "=="'.format(pkg_name),
31+
'ERROR: specify minimal version of "{0}" using '
32+
'">=" or "=="'.format(pkg_name),
3033
file=sys.stderr,
3134
)
3235
sys.exit(1)
@@ -55,18 +58,65 @@ def iter_requirements(level, extras, setup_file):
5558
with pushd(os.path.dirname(setup_file)):
5659
with open(setup_file) as sf:
5760
file_contents = sf.read()
58-
setup_provider = re.findall(r"from ([a-z._]+) import setup", file_contents)
59-
if len(setup_provider) == 1:
60-
setup_provider = setup_provider[0]
61+
node = ast.parse(file_contents)
62+
asnames = {}
63+
imports = []
64+
for elem in ast.walk(node):
65+
# save the asnames to parse aliases later
66+
if isinstance(elem, ast.Import):
67+
for n in elem.names:
68+
asnames[(n.asname if n.asname is not None else n.name)] = n.name
69+
for elem in ast.walk(node):
70+
# for function imports, e.g. from setuptools import setup; setup()
71+
if isinstance(elem, ast.ImportFrom) and "setup" in [
72+
e.name for e in elem.names
73+
]:
74+
imports.append(elem.module)
75+
# for module imports, e.g. import setuptools; setuptools.setup(...)
76+
elif (
77+
isinstance(elem, ast.Expr)
78+
and isinstance(elem.value, ast.Call)
79+
and isinstance(elem.value.func, ast.Attribute)
80+
and isinstance(elem.value.func.value, ast.Name)
81+
and elem.value.func.attr == "setup"
82+
):
83+
name = elem.value.func.value.id
84+
if name in asnames.keys():
85+
name = asnames[name]
86+
imports.append(name)
87+
# for module imports, e.g. import disttools.core; disttools.core.setup(...)
88+
elif (
89+
isinstance(elem, ast.Expr)
90+
and isinstance(elem.value, ast.Call)
91+
and isinstance(elem.value.func, ast.Attribute)
92+
and isinstance(elem.value.func.value, ast.Attribute)
93+
and elem.value.func.attr == "setup"
94+
):
95+
name = (
96+
str(elem.value.func.value.value.id)
97+
+ "."
98+
+ str(elem.value.func.value.attr)
99+
)
100+
if name in asnames.keys():
101+
name = asnames[name]
102+
imports.append(name)
103+
setup_providers = [
104+
i for i in imports if i in ["distutils.core", "setuptools"]
105+
]
106+
if len(setup_providers) == 0:
107+
print(
108+
f"Warning: unable to recognize setup provider in {setup_file}: "
109+
"defaulting to 'distutils.core'."
110+
)
111+
setup_provider = "distutils.core"
112+
elif len(setup_providers) == 1:
113+
setup_provider = setup_providers[0]
61114
else:
62-
setup_provider = ""
63-
if not ((setup_provider == "distutils.core") or (setup_provider == "setuptools")):
64115
print(
65-
f"Warning: unable to recognize 'import {setup_provider}' in {setup_file}: "
116+
f"Warning: ambiguous setup provider in {setup_file}: candidates are {setup_providers}"
66117
"defaulting to 'distutils.core'."
67118
)
68119
setup_provider = "distutils.core"
69-
exec(f"import {setup_provider}")
70120
with mock.patch.object(eval(setup_provider), "setup") as mock_setup:
71121
sys.path.append(os.path.dirname(setup_file))
72122
g = {"__file__": setup_file, "__name__": "__main__"}
@@ -145,7 +195,9 @@ def iter_requirements(level, extras, setup_file):
145195
result[pkg.name] = "{0}=={1}".format(build_pkg_name(pkg), specs["~="])
146196
else:
147197
ver, _ = os.path.splitext(specs["~="])
148-
result[pkg.name] = "{0}>={1},=={2}.*".format(build_pkg_name(pkg), specs["~="], ver)
198+
result[pkg.name] = "{0}>={1},=={2}.*".format(
199+
build_pkg_name(pkg), specs["~="], ver
200+
)
149201

150202
else:
151203
if level == "min":

‎tests/data/setup-distutils-asnames.txt‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
"""
2+
Copyright 2018 Matthew Aynalem
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
"""
16+
import distutils.core as dts, os
17+
from setuptools import find_packages
18+
19+
dts.setup(
20+
name='packer.py',
21+
version='0.3.0',
22+
author='Matthew Aynalem',
23+
author_email='[email protected]',
24+
packages=['packerpy'],
25+
url='https://github.com/mayn/packer.py',
26+
license='Apache License 2.0',
27+
description='packer.py - python library to run hashicorp packer CLI commands',
28+
keywords="hashicorp packer",
29+
install_requires=[
30+
],
31+
classifiers=[
32+
'License :: OSI Approved :: Apache Software License',
33+
'Programming Language :: Python :: 2',
34+
'Programming Language :: Python :: 2.7',
35+
'Programming Language :: Python :: 3',
36+
'Programming Language :: Python :: 3.4',
37+
'Programming Language :: Python :: 3.5',
38+
'Programming Language :: Python :: 3.6',
39+
],
40+
)

‎tests/data/setup-distutils-qualifiedfct.txt‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
"""
2+
Copyright 2018 Matthew Aynalem
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
"""
16+
import distutils.core
17+
from setuptools import find_packages
18+
19+
distutils.core.setup(
20+
name='packer.py',
21+
version='0.3.0',
22+
author='Matthew Aynalem',
23+
author_email='[email protected]',
24+
packages=['packerpy'],
25+
url='https://github.com/mayn/packer.py',
26+
license='Apache License 2.0',
27+
description='packer.py - python library to run hashicorp packer CLI commands',
28+
keywords="hashicorp packer",
29+
install_requires=[
30+
],
31+
classifiers=[
32+
'License :: OSI Approved :: Apache Software License',
33+
'Programming Language :: Python :: 2',
34+
'Programming Language :: Python :: 2.7',
35+
'Programming Language :: Python :: 3',
36+
'Programming Language :: Python :: 3.4',
37+
'Programming Language :: Python :: 3.5',
38+
'Programming Language :: Python :: 3.6',
39+
],
40+
)

‎tests/data/setup-qualifiedfct.txt‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# -*- coding: utf-8 -*-
2+
#
3+
# This file is part of Requirements-Builder
4+
# Copyright (C) 2015, 2016, 2017, 2018, 2019, 2020 CERN.
5+
#
6+
# Requirements-Builder is free software; you can redistribute it and/or
7+
# modify it under the terms of the Revised BSD License; see LICENSE
8+
# file for more details.
9+
#
10+
"""Build requirements files from setup.py requirements."""
11+
12+
import os
13+
14+
import setuptools
15+
16+
# Get the version string. Cannot be done with import!
17+
g = {}
18+
19+
install_requires = [
20+
'click>=6.1.0',
21+
'mock>=1.3.0',
22+
]
23+
24+
tests_require = [
25+
'check-manifest>=0.25',
26+
'coverage>=4.0',
27+
'isort>=4.0.0',
28+
'pydocstyle>=1.0.0',
29+
'pytest-cache>=1.0',
30+
'pytest-cov>=2.0.0',
31+
'pytest-pep8>=1.0.6',
32+
'pytest>=2.8.0',
33+
]
34+
35+
extras_require = {
36+
'docs': [
37+
'Sphinx>=2.4',
38+
],
39+
'tests': tests_require,
40+
}
41+
42+
extras_require['all'] = extras_require['tests'] + extras_require['docs']
43+
44+
setup_requires = ['pytest-runner>=2.6.2', ]
45+
46+
setuptools.setup(
47+
name='requirements-builder',
48+
version="0.1.0",
49+
description=__doc__,
50+
long_description='\n\n',
51+
author="Invenio Collaboration",
52+
author_email='[email protected]',
53+
url='https://github.com/inveniosoftware/requirements-builder',
54+
entry_points={
55+
'console_scripts':
56+
["requirements-builder = requirements_builder.cli:cli"]
57+
},
58+
packages=['requirements_builder', ],
59+
include_package_data=True,
60+
extras_require=extras_require,
61+
install_requires=install_requires,
62+
setup_requires=setup_requires,
63+
tests_require=tests_require,
64+
license='BSD',
65+
zip_safe=False,
66+
keywords='requirements-builder',
67+
classifiers=[
68+
'Intended Audience :: Developers',
69+
'License :: OSI Approved :: BSD License',
70+
'Natural Language :: English',
71+
'Programming Language :: Python :: 3',
72+
'Programming Language :: Python :: 3.5',
73+
'Programming Language :: Python :: 3.6',
74+
'Programming Language :: Python :: 3.7',
75+
'Programming Language :: Python :: 3.8',
76+
],
77+
)

‎tests/test_setup_py_live_eval.py‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
# file for more details.
99
#
1010
"""Tests for `requirements-builder` module."""
11+
import pytest
1112

1213
from os.path import abspath
1314
from os.path import dirname
@@ -16,29 +17,33 @@
1617
from python_inspector.setup_py_live_eval import iter_requirements
1718

1819
REQ = abspath(join(dirname(__file__), "./data/requirements.devel.txt"))
19-
SETUP = abspath(join(dirname(__file__), "./data/setup.txt"))
20-
SETUP_DISTUTILS = abspath(join(dirname(__file__), "./data/setup-distutils.txt"))
2120

2221

23-
def test_iter_requirements_with_setup_py():
22+
@pytest.mark.parametrize("setup_py", [abspath(join(dirname(__file__), "./data/setup.txt")),
23+
abspath(join(dirname(__file__), "./data/setup-qualifiedfct.txt"))])
24+
def test_iter_requirements_with_setup_py(setup_py):
2425
"""Test requirements-builder."""
2526
# Min
26-
assert list(iter_requirements("min", [], SETUP)) == ["click==6.1.0", "mock==1.3.0"]
27+
assert list(iter_requirements("min", [], setup_py)) == ["click==6.1.0", "mock==1.3.0"]
2728

2829
# PyPI
29-
assert list(iter_requirements("pypi", [], SETUP)) == ["click>=6.1.0", "mock>=1.3.0"]
30+
assert list(iter_requirements("pypi", [], setup_py)) == ["click>=6.1.0", "mock>=1.3.0"]
3031

3132
# Dev
32-
assert list(iter_requirements("dev", [], SETUP)) == ["click>=6.1.0", "mock>=1.3.0"]
33+
assert list(iter_requirements("dev", [], setup_py)) == ["click>=6.1.0", "mock>=1.3.0"]
3334

3435

35-
def test_iter_requirements_with_setup_py_distutils():
36-
"""Test against setup.py files which import distutils"""
36+
@pytest.mark.parametrize("setup_py", [abspath(join(dirname(__file__), "./data/setup-distutils.txt")),
37+
abspath(join(dirname(__file__),
38+
"./data/setup-distutils-qualifiedfct.txt")),
39+
abspath(join(dirname(__file__), "./data/setup-distutils-asnames.txt"))])
40+
def test_iter_requirements_with_setup_py_noreqs(setup_py):
41+
"""Test against setup.py files which import setup in different ways"""
3742
# Min
38-
assert list(iter_requirements("min", [], SETUP_DISTUTILS)) == []
43+
assert list(iter_requirements("min", [], setup_py)) == []
3944

4045
# PyPI
41-
assert list(iter_requirements("pypi", [], SETUP_DISTUTILS)) == []
46+
assert list(iter_requirements("pypi", [], setup_py)) == []
4247

4348
# Dev
44-
assert list(iter_requirements("dev", [], SETUP_DISTUTILS)) == []
49+
assert list(iter_requirements("dev", [], setup_py)) == []

0 commit comments

Comments
 (0)