Merge pull request lisa-lab#1251 from euphoris/python3

improve compatibility for python 3

.travis.sh

@@ -10,5 +10,6 @@ else
     # present. We can't download the original as it is too big to
     # download each time. If present run: python make_dataset.py
     (cd pylearn2/scripts/tutorials/grbm_smd && wget http://www.iro.umontreal.ca/~lisa/datasets/cifar10_preprocessed_train.pkl)
+    if [ $TRAVIS_PYTHON_VERSION = '3.4' ]; then python pylearn2/devtools/convert_pkl.py pylearn2/scripts/tutorials/grbm_smd/cifar10_preprocessed_train.pkl; fi
     THEANO_FLAGS="$FLAGS",blas.ldflags="-lblas -lgfortran",-warn.ignore_bug_before=all,on_opt_error=raise,on_shape_error=raise theano-nose -v $PART
 fi

.travis.yml

@@ -10,8 +10,9 @@ before_install:
   - export PATH=/home/travis/miniconda/bin:$PATH
   - conda update --yes conda
 install:
-  - conda create --yes -q -n py26 python=2.6 cython=0.2 pillow numpy=1.6 numpydoc scipy=0.11 pytables=3.0 numexpr=2.2.2 nose=1.1 pyyaml sphinx argparse pyflakes pip
-  - source activate py26
+  - if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then conda create --yes -q -n pyenv python=2.6 cython=0.2 pillow numpy=1.6 numpydoc scipy=0.11 pytables=3.0 numexpr=2.2.2 nose=1.1 pyyaml sphinx pyflakes argparse pip; fi
+  - if [[ $TRAVIS_PYTHON_VERSION == '3.4' ]]; then conda create --yes -q -n pyenv python=3.4 cython=0.21.1 pillow numpy=1.9.1 numpydoc scipy=0.14.0 pytables=3.1.1 numexpr=2.3.1 nose=1.3.4 pyyaml sphinx pyflakes pip; fi
+  - source activate pyenv
   - pip install -q git+git://git.assembla.com/jobman.git
   - pip install -q --no-deps git+git://github.com/Theano/Theano.git
   - pip install -q nose-exclude

pylearn2/classifier.py

@@ -9,12 +9,12 @@
 warnings.warn("pylearn2.classifier has been deprecated and will be removed "
         "from the library on or after Aug 24, 2014.")
 
-from deprecated.classifier import Block
-from deprecated.classifier import CumulativeProbabilitiesLayer
-from deprecated.classifier import LogisticRegressionLayer
-from deprecated.classifier import Model
-from deprecated.classifier import VectorSpace
-from deprecated.classifier import numpy
-from deprecated.classifier import sharedX
-from deprecated.classifier import tensor
-from deprecated.classifier import theano
+from .deprecated.classifier import Block
+from .deprecated.classifier import CumulativeProbabilitiesLayer
+from .deprecated.classifier import LogisticRegressionLayer
+from .deprecated.classifier import Model
+from .deprecated.classifier import VectorSpace
+from .deprecated.classifier import numpy
+from .deprecated.classifier import sharedX
+from .deprecated.classifier import tensor
+from .deprecated.classifier import theano

pylearn2/costs/dbm.py

@@ -9,11 +9,13 @@
 __license__ = "3-clause BSD"
 __maintainer__ = "LISA Lab"
 
+import collections
 import numpy as np
 import logging
+import operator
 import warnings
 
-from theano.compat.six.moves import xrange
+from theano.compat.six.moves import reduce, xrange
 from theano import config
 from theano.sandbox.rng_mrg import MRG_RandomStreams
 RandomStreams = MRG_RandomStreams
@@ -369,7 +371,7 @@ def _get_negative_phase(self, model, X, Y=None):
         layer_to_chains = model.make_layer_to_state(self.num_chains)
 
         def recurse_check(l):
-            if isinstance(l, (list, tuple)):
+            if isinstance(l, (list, tuple, collections.ValuesView)):
                 for elem in l:
                     recurse_check(elem)
             else:
@@ -800,7 +802,7 @@ def expr(self, model, data, ** kwargs):
         if len(layer_costs) == 0:
             return T.as_tensor_variable(0.)
         else:
-            total_cost = reduce(lambda x, y: x + y, layer_costs)
+            total_cost = reduce(operator.add, layer_costs)
         total_cost.name = 'MF_L1_ActCost'
 
         assert total_cost.ndim == 0
@@ -1023,7 +1025,7 @@ def expr(self, model, data, ** kwargs):
             rval.name = '0_weight_decay'
             return rval
         else:
-            total_cost = reduce(lambda x, y: x + y, layer_costs)
+            total_cost = reduce(operator.add, layer_costs)
         total_cost.name = 'DBM_WeightDecay'
 
         assert total_cost.ndim == 0

pylearn2/costs/mlp/__init__.py

@@ -4,7 +4,10 @@
 __authors__ = 'Vincent Archambault-Bouffard, Ian Goodfellow'
 __copyright__ = "Copyright 2013, Universite de Montreal"
 
+import operator
+
 from theano import tensor as T
+from theano.compat.six.moves import reduce
 
 from pylearn2.costs.cost import Cost, DefaultDataSpecsMixin, NullDataSpecsMixin
 from pylearn2.utils import safe_izip
@@ -100,7 +103,7 @@ def wrapped_layer_cost(layer, coef):
             rval.name = '0_weight_decay'
             return rval
         else:
-            total_cost = reduce(lambda x, y: x + y, layer_costs)
+            total_cost = reduce(operator.add, layer_costs)
         total_cost.name = 'MLP_WeightDecay'
 
         assert total_cost.ndim == 0
@@ -159,7 +162,7 @@ def expr(self, model, data, ** kwargs):
             rval.name = '0_l1_penalty'
             return rval
         else:
-            total_cost = reduce(lambda x, y: x + y, layer_costs)
+            total_cost = reduce(operator.add, layer_costs)
         total_cost.name = 'MLP_L1Penalty'
 
         assert total_cost.ndim == 0

pylearn2/dataset_get/dataset-get.py

@@ -24,6 +24,8 @@
 import tarfile
 import subprocess
 
+from theano.compat.six.moves import input
+
 logger = logging.getLogger(__name__)
 
 
@@ -817,7 +819,7 @@ def upgrade_packages(packages_to_upgrade, hook=None ):
             readable_size = packages_sources[this_package].readable_size
             logger.info("{0} ({1})".format(this_package, readable_size))
 
-        r=raw_input("Proceed? [yes/N] ")
+        r = input("Proceed? [yes/N] ")
         if r=='y' or r=='yes':
             for  this_package in packages_really_to_upgrade:
                 install_upgrade( packages_sources[this_package], upgrade=True, progress_hook=hook )
@@ -872,7 +874,7 @@ def install_packages( packages_to_install, force_install=False, hook=None ):
             readable_size = packages_sources[this_package].readable_size
             logger.info("{0} ({1})".format(this_package, readable_size))
 
-        r=raw_input("Proceed? [yes/N] ")
+        r = input("Proceed? [yes/N] ")
         if r=='y' or r=='yes':
             for  this_package in packages_really_to_install:
                 install_upgrade( packages_sources[this_package], upgrade=False, progress_hook=hook )
@@ -913,12 +915,12 @@ def install_packages_from_file( packages_to_install ):
             packages.append(corename(this_package))
         logger.info(' '.join(packages))
 
-        r=raw_input("Proceed? [yes/N] ")
+        r = input("Proceed? [yes/N] ")
         if r=='y' or r=='yes':
             for  this_package in packages_really_to_install:
                 #install_upgrade( this_package, upgrade=False, progress_hook=hook )
                 if os.path.exists(dataset_data_path+corename(this_package)):
-                    r=raw_input("[in] '%s' already installed, overwrite? [yes/N] " % corename(this_package))
+                    r = input("[in] '%s' already installed, overwrite? [yes/N] " % corename(this_package))
 
                     if r!='y' and r!='yes':
                         logger.info("[in] skipping package "
@@ -995,7 +997,7 @@ def remove_packages( packages_to_remove ):
             packages.append(this_package)
         logger.info(' '.join(packages))
 
-        r=raw_input("Proceed? [yes/N] ")
+        r = input("Proceed? [yes/N] ")
         if r=='y' or r=='yes':
             for  this_package in packages_really_to_remove:
                 remove_package( installed_packages_list[this_package], dataset_data_path )

pylearn2/dataset_get/helper-scripts/make-archive.py

@@ -9,7 +9,9 @@
 
 
 import logging
-import os,re,sys,tarfile
+import os, sys, tarfile
+
+from theano.compat.six.moves import input
 
 logger = logging.getLogger(__name__)
 
@@ -68,7 +70,7 @@ def checks(path):
 def create_archive( source, archive_name ):
 
     if os.path.exists(archive_name):
-        r= raw_input("'%s' exists, overwrite? [yes/N] " % archive_name)
+        r = input("'%s' exists, overwrite? [yes/N] " % archive_name)
         if (r!="y") and (r!="yes"):
             logger.info("taking '{0}' for no, so there.".format(r))
             #bail out

pylearn2/datasets/norb_small.py

@@ -7,6 +7,8 @@
 np = numpy
 import os
 
+from theano.compat.six.moves import reduce
+
 from pylearn2.datasets import dense_design_matrix
 from pylearn2.datasets import retina
 from pylearn2.datasets.cache import datasetCache

pylearn2/datasets/tests/test_transformer_dataset.py

@@ -0,0 +1,27 @@
+"""
+Unit tests for ../transformer_dataset.py
+"""
+
+import os
+
+import pylearn2
+from pylearn2.blocks import Block
+from pylearn2.datasets.csv_dataset import CSVDataset
+from pylearn2.datasets.transformer_dataset import TransformerDataset
+
+
+def test_transformer_iterator():
+    """
+    Tests whether TransformerIterator is iterable
+    """
+
+    test_path = os.path.join(pylearn2.__path__[0],
+                             'datasets', 'tests', 'test.csv')
+    raw = CSVDataset(path=test_path, expect_headers=False)
+    block = Block()
+    dataset = TransformerDataset(raw, block)
+    iterator = dataset.iterator('shuffled_sequential', 3)
+    try:
+        iter(iterator)
+    except TypeError:
+        assert False, "TransformerIterator isn't iterable"

pylearn2/datasets/transformer_dataset.py

@@ -9,6 +9,8 @@
 __maintainer__ = "LISA Lab"
 __email__ = "pylearn-dev@googlegroups"
 
+from theano.compat.six import Iterator
+
 from pylearn2.datasets.dataset import Dataset
 from pylearn2.space import CompositeSpace
 from pylearn2.utils.data_specs import is_flat_specs
@@ -183,7 +185,7 @@ def get_num_examples(self):
         return self.raw.get_num_examples()
 
 
-class TransformerIterator(object):
+class TransformerIterator(Iterator):
 
     """
     .. todo::
@@ -211,7 +213,7 @@ def __iter__(self):
         """
         return self
 
-    def next(self):
+    def __next__(self):
         """
         .. todo::
 

pylearn2/devtools/convert_pkl.py

@@ -0,0 +1,18 @@
+"""
+Convert a .pkl from Python 2 to Python 3
+"""
+
+import pickle
+import sys
+
+
+if __name__ == "__main__":
+    source_pkl = sys.argv[1]
+    try:
+        target_pkl = sys.argv[2]
+    except IndexError:
+        target_pkl = source_pkl
+    with open(source_pkl, 'rb') as f:
+        obj = pickle.load(f, encoding='latin-1')
+    with open(target_pkl, 'wb') as f:
+        pickle.dump(obj, f)

pylearn2/devtools/run_pyflakes.py

@@ -16,15 +16,19 @@
 __maintainer__ = "LISA Lab"
 __email__ = "pylearn-dev@googlegroups"
 
+import sys
 import logging
+
+from theano.compat import six
+
 from pylearn2.devtools.list_files import list_files
 from pylearn2.utils.shell import run_shell_command
 
 
 logger = logging.getLogger(__name__)
 
 
-def run_pyflakes(no_warnings = False):
+def run_pyflakes(no_warnings=False):
     """
     Return a description of all errors pyflakes finds in Pylearn2.
 
@@ -47,15 +51,16 @@ def run_pyflakes(no_warnings = False):
 
     for filepath in files:
         output, rc = run_shell_command('pyflakes ' + filepath)
-        if 'pyflakes: not found' in output:
+        output = output.decode(sys.getdefaultencoding())
+        if u'pyflakes: not found' in output:
             # The return code alone does not make it possible to detect
             # if pyflakes is present or not. When pyflakes is not present,
             # the return code seems to always be 127, but 127 can also be
             # the result of finding 127 warnings in a file.
             # Therefore, we examine the output instead.
             raise RuntimeError("Couldn't run 'pyflakes " + filepath + "'. "
-                    "Error code returned:" + str(rc)\
-                    + " Output was: " + output)
+                               "Error code returned:" + str(rc) +
+                               " Output was: " + output)
 
         output = _filter(output, no_warnings)
 
@@ -64,6 +69,7 @@ def run_pyflakes(no_warnings = False):
 
     return rval
 
+
 def _filter(output, no_warnings):
     """
     .. todo::
@@ -86,33 +92,31 @@ def _filter(output, no_warnings):
     """
     lines = output.split('\n')
 
-    lines = [ line for line in lines if line != '' ]
-
+    lines = [line for line in lines
+             if line != '' and line.find("undefined name 'long'") == -1]
 
     if no_warnings:
 
-        lines = [ line for line in lines if
-                line.find("is assigned to but never used") == -1 ]
-
-        lines = [ line for line in lines if
-                line.find('imported but unused') == -1 ]
+        lines = [line for line in lines if
+                 line.find("is assigned to but never used") == -1]
 
-        lines = [ line for line in lines if
-                line.find('redefinition of unused ') == -1 ]
+        lines = [line for line in lines if
+                 line.find('imported but unused') == -1]
 
+        lines = [line for line in lines if
+                 line.find('redefinition of unused ') == -1]
 
     if len(lines) == 0:
         return None
     return '\n'.join(lines)
 
 if __name__ == '__main__':
-    import sys
     if len(sys.argv) > 1:
         no_warnings = bool(sys.argv[1])
     else:
         no_warnings = False
 
-    d = run_pyflakes( no_warnings = no_warnings)
+    d = run_pyflakes(no_warnings=no_warnings)
 
     for key in d:
         logger.info('{0}:'.format(key))

pylearn2/devtools/tests/docscrape.py

@@ -8,7 +8,6 @@
 from nose.plugins.skip import SkipTest
 import re
 import sys
-import types
 
 from theano.compat import six
 
@@ -132,7 +131,7 @@ def __getitem__(self,key):
         return self._parsed_data[key]
 
     def __setitem__(self,key,val):
-        if not self._parsed_data.has_key(key):
+        if key not in self._parsed_data:
             raise ValueError("Unknown section %s" % key)
         else:
             self._parsed_data[key] = val
@@ -779,7 +778,9 @@ def docstring_errors(filename, global_dict=None):
     if '__doc__' not in global_dict:
         global_dict['__doc__'] = None
     try:
-        execfile(filename, global_dict)
+        with open(filename) as f:
+            code = compile(f.read(), filename, 'exec')
+            exec(code, global_dict)
     except SystemExit:
         pass
     except SkipTest:

pylearn2/devtools/tests/test_format.py

@@ -734,7 +734,7 @@ def verify_format_docstrings():
             continue
         try:
             format_infractions.extend(docstring_errors(path))
-        except StandardError as e:
+        except Exception as e:
             format_infractions.append(["%s failed to run so format cannot "
                                        "be checked. Error message:\n %s" %
                                        (rel_path, e)])