AVRO-2576: Fix Premature Import pycodestyle (#664)

* AVRO-2445: Remove SimpleJSON License and Rat Excludes

* AVRO-2576: Fix Premature Import pycodestyle

* AVRO-2576: Run pycodestyle from setup for py2

* AVRO-2576: Fix and Ignore Lint Failures

* AVRO-2576: Update Setuptools

* AVRO-2576: Use Local pycodestyle If We Can

* AVRO-2576: Run isort from setuptools

* AVRO-2576: Include isort at setuptools time

* AVRO-2576: Rat Exclude Python Setup Files

LICENSE.txt

@@ -295,31 +295,6 @@ Copyright (C) 2006 Toni Ronkko
 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 | OTHER DEALINGS IN THE SOFTWARE.
 
-----------------------------------------------------------------------
-License for simplejson used in the python implementation:
-
-Source from: https://github.com/simplejson/simplejson
-
-Copyright (c) 2006 Bob Ippolito
-
-| Permission is hereby granted, free of charge, to any person obtaining a copy of
-| this software and associated documentation files (the "Software"), to deal in
-| the Software without restriction, including without limitation the rights to
-| use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-| of the Software, and to permit persons to whom the Software is furnished to do
-| so, subject to the following conditions:
-|
-| The above copyright notice and this permission notice shall be included in all
-| copies or substantial portions of the Software.
-|
-| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-| SOFTWARE.
-
 ----------------------------------------------------------------------
 License for ivy-2.2.0.jar used in the python implementation:
 

lang/py/.gitignore

@@ -1,4 +1,5 @@
+*.egg-info/
+.eggs/
 build/
 lib/
 userlogs/
-*.egg-info/

lang/py/build.sh

@@ -28,8 +28,7 @@ main() {
   for target; do
     case "$target" in
       lint)
-        ./setup.py isort
-        pycodestyle .
+        ./setup.py isort lint
         ;;
       test)
         ant test

lang/py/setup.cfg

@@ -19,7 +19,7 @@ line_length = 150
 known_third_party=zope
 
 [pycodestyle]
+exclude = .eggs,build
+ignore = E101,E111,E114,E121,E122,E124,E125,E126,E127,E128,E129,E201,E202,E203,E222,E226,E225,E231,E241,E251,E261,E262,E265,E266,E301,E302,E303,E305,E306,E402,E501,E701,E703,E704,E711,E722,W191,W291,W292,W293,W391,W503,W504,W601
 max-line-length = 150
 statistics = True
-exclude = build
-ignore = E101,E111,E114,E121,E122,E124,E125,E126,E127,E128,E129,E201,E202,E203,E222,E226,E225,E231,E241,E251,E261,E262,E265,E266,E301,E302,E303,E305,E306,E402,E501,E701,E703,E704,E711,W191,W291,W292,W293,W391,W503,W601

lang/py/setup.py

@@ -15,7 +15,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+
+import distutils.errors
+import glob
 import os
+import subprocess
+
 import setuptools
 
 
@@ -29,12 +35,42 @@ def _get_version():
     return verfile.read().rstrip().replace("-", "+")
 
 
+class LintCommand(setuptools.Command):
+    """Run pycodestyle on all your modules"""
+    description = __doc__
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        # setuptools does not seem to make pycodestyle available
+        # in the pythonpath, so we do it ourselves.
+        try:
+            env = {'PYTHONPATH': next(glob.iglob('.eggs/pycodestyle-*.egg'))}
+        except StopIteration:
+            env = None  # pycodestyle is already installed
+        p = subprocess.Popen(['python', '-m', 'pycodestyle', '.'], close_fds=True, env=env)
+        if p.wait():
+            raise distutils.errors.DistutilsError("pycodestyle exited with a nonzero exit code.")
+
+
 setuptools.setup(
   name = 'avro',
   version = _get_version(),
   packages = ['avro'],
   package_dir = {'': 'src'},
   scripts = ["./scripts/avro"],
+  setup_requires = [
+    'isort',
+    'pycodestyle',
+  ],
+  cmdclass={
+      "lint": LintCommand,
+  },
 
   #include_package_data=True,
   package_data={'avro': ['LICENSE', 'NOTICE']},

lang/py/src/avro/protocol.py

@@ -24,6 +24,7 @@
   from md5 import md5
 
 import json
+
 from avro import schema
 
 #

lang/py/src/avro/tether/tether_task_runner.py

@@ -96,7 +96,7 @@ def invoke(self, message, request):
 def HTTPHandlerGen(runner):
   """
   This is a class factory for the HTTPHandler. We need
-  a factory b\c we need a reference to the runner
+  a factory because we need a reference to the runner
 
   Parameters
   -----------------------------------------------------------------

lang/py3/.gitignore

@@ -1,3 +1,4 @@
+.eggs/
 avro/HandshakeRequest.avsc
 avro/HandshakeResponse.avsc
 avro/VERSION.txt

lang/py3/setup.cfg

@@ -42,6 +42,9 @@ package_dir =
     avro = avro
 include_package_data = true
 packages = avro
+setup_requires =
+  isort
+  pycodestyle
 install_requires =
 zip_safe = true
 scripts =
@@ -68,6 +71,7 @@ line_length = 150
 known_third_party=zope
 
 [pycodestyle]
+exclude = .eggs,build
+ignore = E111,E114,E121,E122,E124,E127,E128,E129,E201,E202,E203,E221,E225,E231,E241,E261,E301,E302,E303,E305,E402,E701,E703,E722,W503,W504
 max-line-length = 150
 statistics = True
-ignore = E111,E114,E121,E122,E124,E127,E128,E129,E201,E202,E203,E221,E225,E231,E241,E261,E301,E302,E303,E305,E402,E701,E703,W503

lang/py3/setup.py

@@ -26,16 +26,16 @@
 https://pypi.org/project/avro-python3/
 """
 
-import distutils
 import distutils.command.clean
 import distutils.dir_util
+import distutils.errors
 import distutils.file_util
 import distutils.log
 import fnmatch
+import glob
 import os
 import subprocess
 
-import pycodestyle
 import setuptools
 
 _HERE = os.path.dirname(os.path.abspath(__file__))
@@ -145,8 +145,14 @@ def finalize_options(self):
         pass
 
     def run(self):
+        # setuptools does not seem to make pycodestyle available
+        # in the pythonpath, so we do it ourselves.
         try:
-            subprocess.run(['pycodestyle', '.'], check=True)
+            env = {'PYTHONPATH': next(glob.iglob('.eggs/pycodestyle-*.egg'))}
+        except StopIteration:
+            env = None  # pycodestyle is already installed
+        try:
+            subprocess.run(['python3', '-m', 'pycodestyle', '.'], env=env)
         except subprocess.CalledProcessError:
             raise distutils.errors.DistutilsError("pycodestyle exited with a nonzero exit code.")
 

pom.xml

@@ -301,7 +301,6 @@
                 <exclude>lang/csharp/build/**</exclude>
                 <exclude>lang/perl/pm_to_blib</exclude>
                 <exclude>lang/perl/blib/**/.exists</exclude>
-                <exclude>lang/py/build/**</exclude>
                 <exclude>lang/ruby/Gemfile.lock</exclude>
                 <exclude>lang/ruby/avro.gemspec</exclude>
                 <exclude>lang/ruby/.gem/**</exclude>
@@ -350,27 +349,33 @@
                 <exclude>lang/csharp/Avro.sln</exclude> <!-- visual studio -->
                 <!-- build-related files -->
                 <exclude>**/README.md</exclude>
-                <exclude>BUILD.md</exclude>
-                <exclude>.travis.yml</exclude>
                 <exclude>**/VERSION.txt</exclude>
                 <exclude>**/dependency-reduced-pom.xml</exclude>
+                <exclude>.travis.yml</exclude>
+                <exclude>BUILD.md</exclude>
+                <exclude>lang/c/src/avro-c.pc.in</exclude>
+                <exclude>lang/csharp/**/bin/Release/**/Avro.xml</exclude>
+                <exclude>lang/csharp/TestResult.xml</exclude>
+                <exclude>lang/csharp/src/apache/*/obj/**</exclude>
+                <exclude>lang/java/archetypes/avro-service-archetype/src/test/integration/projects/basic/goal.txt</exclude>
+                <exclude>lang/java/mapred/userlogs/**</exclude>
+                <exclude>lang/java/tools/userlogs/**</exclude>
+                <exclude>lang/js/coverage/**</exclude>
+                <exclude>lang/js/test/mocha.opts</exclude>
                 <exclude>lang/perl/.shipit</exclude>
-                <exclude>lang/perl/inc/Module/Install/*.pm</exclude>
-                <exclude>lang/perl/inc/Module/Install.pm</exclude>
-                <exclude>lang/perl/Makefile*</exclude>
                 <exclude>lang/perl/META.yml</exclude>
                 <exclude>lang/perl/MYMETA.yml</exclude>
+                <exclude>lang/perl/Makefile*</exclude>
+                <exclude>lang/perl/inc/Module/Install.pm</exclude>
+                <exclude>lang/perl/inc/Module/Install/*.pm</exclude>
+                <exclude>lang/py/.eggs/**</exclude>
+                <exclude>lang/py/build/**</exclude>
+                <exclude>lang/py/dist/**</exclude>
                 <exclude>lang/py/userlogs/**</exclude>
-                <exclude>lang/c/src/avro-c.pc.in</exclude>
+                <exclude>lang/py3/.eggs/**</exclude>
+                <exclude>lang/py3/build/**</exclude>
+                <exclude>lang/py3/dist/**</exclude>
                 <exclude>lang/ruby/Manifest</exclude>
-                <exclude>lang/java/tools/userlogs/**</exclude>
-                <exclude>lang/java/mapred/userlogs/**</exclude>
-                <exclude>lang/java/archetypes/avro-service-archetype/src/test/integration/projects/basic/goal.txt</exclude>
-                <exclude>lang/js/test/mocha.opts</exclude>
-                <exclude>lang/csharp/TestResult.xml</exclude>
-                <exclude>lang/csharp/src/apache/*/obj/**</exclude>
-                <exclude>lang/csharp/**/bin/Release/**/Avro.xml</exclude>
-                <exclude>lang/js/coverage/**</exclude>
                 <!-- text documentation files -->
                 <exclude>CHANGES.txt</exclude>
                 <exclude>DIST_README.txt</exclude>
@@ -380,7 +385,6 @@
                 <exclude>.github/PULL_REQUEST_TEMPLATE.md</exclude>
                 <exclude>lang/java/archetypes/avro-service-archetype/src/test/integration/projects/basic/archetype.properties</exclude> <!-- used to generate user projects -->
                 <!-- files and directories covered by LICENSE.txt -->
-                <exclude>lang/py/lib/simplejson/**</exclude>
                 <exclude>lang/c/jansson/**</exclude>
                 <exclude>lang/c/src/avro/msinttypes.h</exclude>
                 <exclude>lang/c/src/avro/msstdint.h</exclude>

share/docker/Dockerfile

@@ -59,7 +59,6 @@ RUN apt-get -qq update && \
     g++ \
     gcc \
     git \
-    isort \
     libboost-all-dev \
     libfontconfig1-dev \
     libfreetype6-dev \
@@ -75,14 +74,11 @@ RUN apt-get -qq update && \
     php${PHP5_VERSION}-gmp \
     php${PHP7_VERSION} \
     php${PHP7_VERSION}-gmp \
-    pycodestyle \
     python \
-    python-isort \
     python-pip \
     python-setuptools \
     python-snappy \
     python-wheel \
-    python3-isort \
     python3-pip \
     python3-setuptools \
     python3-snappy \
@@ -109,10 +105,12 @@ RUN curl -L https://cpanmin.us | perl - --self-upgrade && \
 RUN wget -O /usr/local/bin/phpunit https://phar.phpunit.de/phpunit-5.7.phar && chmod +x /usr/local/bin/phpunit
 
 # Install Python2 packages
-RUN pip install zstandard
+RUN python2 -m pip install --upgrade pip setuptools \
+ && python2 -m pip install zstandard
 
 # Install Python3 packages
-RUN pip3 install zstandard
+RUN python3 -m pip install --upgrade pip setuptools \
+ && python3 -m pip install zstandard
 
 # Install Ruby modules
 COPY lang/ruby/Gemfile /tmp