From ce525ee091dec860f0b09f885ed0c37a92f8063c Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Mon, 9 Oct 2023 11:57:27 -0700 Subject: [PATCH 01/27] Bump version to 1.3.dev0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eb8a692..15f7bad 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ setup( name='remoteobjects', - version='1.2.2', + version='1.3.dev0', description='an Object RESTational Model', author='SAY Media Ltd.', author_email='python@saymedia.com', From 6c3c25db93e4156e4244d8bcfb497699e40c5a75 Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Fri, 6 Oct 2023 11:43:25 -0700 Subject: [PATCH 02/27] Fix a couple doctests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I still don’t run doctest though since python -m doctest fails on python3 (No module named 'http.client'; 'http' is not a package ) and there are doctest errors in other files. --- remoteobjects/fields.py | 8 +++++++- remoteobjects/listobject.py | 7 +++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/remoteobjects/fields.py b/remoteobjects/fields.py index 69af15a..2613feb 100644 --- a/remoteobjects/fields.py +++ b/remoteobjects/fields.py @@ -439,11 +439,17 @@ class Link(AcceptsStringCls, Property): For example: + >>> from remoteobjects import RemoteObject + >>> from remoteobjects.fields import Link + >>> class Event(RemoteObject): pass + ... >>> class Item(RemoteObject): ... feed = Link(Event) ... >>> i = Item.get('http://example.com/item/') - >>> f = i.feed # f's URL: http://example.com/item/feed + >>> f = i.feed + >>> f._location + 'http://example.com/item/feed' Override the `__get__` method of a `Link` subclass to customize how the URLs to linked objects are constructed. diff --git a/remoteobjects/listobject.py b/remoteobjects/listobject.py index bcf1582..96985b9 100644 --- a/remoteobjects/listobject.py +++ b/remoteobjects/listobject.py @@ -92,6 +92,10 @@ class PageOf(PromiseObject.__metaclass__): new `PageObject` classes that contain objects of a specified other class, like so: + >>> from remoteobjects import RemoteObject + >>> from remoteobjects.listobject import PageObject, PageOf + >>> class Entry(RemoteObject): pass + ... >>> PageOfEntry = PageOf(Entry) This is equivalent to defining ``PageOfEntry`` yourself: @@ -173,6 +177,9 @@ class PageObject(SequenceProxy, PromiseObject): `PageOf`, with the class reference you would use to construct an `Object` field. That is, these declarations are equivalent: + >>> from remoteobjects import fields, RemoteObject + >>> from remoteobjects.listobject import PageObject, PageOf + >>> class Entry(RemoteObject): pass >>> PageOfEntry = PageOf(Entry) >>> class PageOfEntry(PageObject): From 25cb372915dffe20dcb49d775188817529d7788c Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Wed, 4 Oct 2023 14:57:17 -0700 Subject: [PATCH 03/27] Fix urlencode parameter type (errors found by mypy) error: Argument 1 to "urlencode" has incompatible type "list[str]"; expected "Union[Mapping[Any, Any], Mapping[Any, Sequence[Any]], Sequence[tuple[Any, Any]], Sequence[tuple[Any, Sequence[Any]]]]" [arg-type] --- examples/twitter.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/twitter.py b/examples/twitter.py index fdb66ac..6165f12 100644 --- a/examples/twitter.py +++ b/examples/twitter.py @@ -77,7 +77,7 @@ def get_user(cls, http=None, **kwargs): url += '/%s.json' % quote_plus(kwargs['id']) else: url += '.json' - query = urlencode(filter(lambda x: x in ('screen_name', 'user_id'), kwargs)) + query = urlencode([(key, value) for key, value in kwargs.iteritems() if key in ('screen_name', 'user_id')]) url = urlunsplit((None, None, url, query, None)) return cls.get(urljoin(Twitter.endpoint, url), http=http) @@ -143,14 +143,14 @@ def __getitem__(self, key): @classmethod def get_messages(cls, http=None, **kwargs): url = '/direct_messages.json' - query = urlencode(filter(lambda x: x in ('since_id', 'page'), kwargs)) + query = urlencode([(key, value) for key, value in kwargs.iteritems() if key in ('since_id', 'page')]) url = urlunsplit((None, None, url, query, None)) return cls.get(urljoin(Twitter.endpoint, url), http=http) @classmethod def get_sent_messages(cls, http=None, **kwargs): url = '/direct_messages/sent.json' - query = urlencode(filter(lambda x: x in ('since_id', 'page'), kwargs)) + query = urlencode([(key, value) for key, value in kwargs.iteritems() if key in ('since_id', 'page')]) url = urlunsplit((None, None, url, query, None)) return cls.get(urljoin(Twitter.endpoint, url), http=http) @@ -177,7 +177,7 @@ def get_related(cls, relation, http=None, **kwargs): url += '/%s.json' % quote_plus(kwargs['id']) else: url += '.json' - query = urlencode(filter(lambda x: x in ('screen_name', 'user_id', 'page'), kwargs)) + query = urlencode([(key, value) for key, value in kwargs.iteritems() if key in ('screen_name', 'user_id', 'page')]) url = urlunsplit((None, None, url, query, None)) return cls.get(urljoin(Twitter.endpoint, url), http=http) @@ -195,7 +195,7 @@ def public(cls, http=None): @classmethod def friends(cls, http=None, **kwargs): - query = urlencode(filter(lambda x: x in ('since_id', 'max_id', 'count', 'page'), kwargs)) + query = urlencode([(key, value) for key, value in kwargs.iteritems() if key in ('since_id', 'max_id', 'count', 'page')]) url = urlunsplit((None, None, '/statuses/friends_timeline.json', query, None)) return cls.get(urljoin(Twitter.endpoint, url), http=http) @@ -206,13 +206,13 @@ def user(cls, http=None, **kwargs): url += '/%s.json' % quote_plus(kwargs['id']) else: url += '.json' - query = urlencode(filter(lambda x: x in ('screen_name', 'user_id', 'since_id', 'max_id', 'page'), kwargs)) + query = urlencode([(key, value) for key, value in kwargs.iteritems() if key in ('screen_name', 'user_id', 'since_id', 'max_id', 'page')]) url = urlunsplit((None, None, url, query, None)) return cls.get(urljoin(Twitter.endpoint, url), http=http) @classmethod def mentions(cls, http=None, **kwargs): - query = urlencode(filter(lambda x: x in ('since_id', 'max_id', 'page'), kwargs)) + query = urlencode([(key, value) for key, value in kwargs.iteritems() if key in ('since_id', 'max_id', 'page')]) url = urlunsplit((None, None, '/statuses/mentions.json', query, None)) return cls.get(urljoin(Twitter.endpoint, url), http=http) From 01c49bdd89b7c3b090c4254b446764687d8ef822 Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Mon, 9 Oct 2023 15:39:02 -0700 Subject: [PATCH 04/27] Upgrade to python 2.7 I want to call unittest.assertIsInstance --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 15f7bad..01f1542 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,7 @@ packages=['remoteobjects'], provides=['remoteobjects'], + python_requires='>=2.7, <3.0', requires=['simplejson(>=2.0.0)', 'httplib2(>=0.5.0)', 'dateutil(>=2.1)'], install_requires=['simplejson>=2.0.0', 'httplib2>=0.5.0', From cf5c5a009231635fd3032f3f4d7c01732120add7 Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Mon, 2 Oct 2023 14:24:19 -0700 Subject: [PATCH 05/27] Remove the obsolete `requires` keyword in setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Distutils had a `requires` keyword argument, but “`requires` is superseded by `install_requires` and should not be used anymore.” https://docs.python.org/3/distutils/setupscript.html#relationships-between-distributions-and-packages https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-requires --- setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.py b/setup.py index 01f1542..27aecff 100644 --- a/setup.py +++ b/setup.py @@ -62,8 +62,6 @@ packages=['remoteobjects'], provides=['remoteobjects'], python_requires='>=2.7, <3.0', - requires=['simplejson(>=2.0.0)', 'httplib2(>=0.5.0)', - 'dateutil(>=2.1)'], install_requires=['simplejson>=2.0.0', 'httplib2>=0.5.0', 'python-dateutil>=2.1'], ) From 9fbba40d68cdd8ca686cdd11283d0eca2ab04c38 Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Mon, 2 Oct 2023 14:18:14 -0700 Subject: [PATCH 06/27] Use exact versions in requirements.txt Add the test requirements to extra_require in setup.py, and use pip freeze to list the exact versions of each dependency in requirements.txt so that tests are reproducible. Use python2 to pip freeze since the last version of pyparsing in python2 is 2.4.7 whereas it is 3.1.1 in python3 --- requirements-test.txt | 4 ++-- requirements.txt | 8 +++++--- setup.py | 6 ++++++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index 10d6203..da3f569 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,2 +1,2 @@ -nose -mox +mox==0.5.3 +nose==1.3.7 diff --git a/requirements.txt b/requirements.txt index 45927a6..0087195 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ -simplejson -httplib2>=0.5.0 -python-dateutil +httplib2==0.22.0 +pyparsing==2.4.7 +python-dateutil==2.8.2 +simplejson==3.19.2 +six==1.16.0 diff --git a/setup.py b/setup.py index 27aecff..c4ef55d 100644 --- a/setup.py +++ b/setup.py @@ -64,4 +64,10 @@ python_requires='>=2.7, <3.0', install_requires=['simplejson>=2.0.0', 'httplib2>=0.5.0', 'python-dateutil>=2.1'], + extras_require={ + 'test': [ + 'nose', + 'mox', + ] + }, ) From de779012434262f8787d9889664c5bfe6766abab Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Tue, 3 Oct 2023 00:15:15 -0700 Subject: [PATCH 07/27] Delete unused tests/requirements.txt This duplicates requirements-test.txt --- tests/requirements.txt | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 tests/requirements.txt diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index 9308e8d..0000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ --r ../requirements.txt -nose -mox From 83a1df8d4eafceacbc57e4745ef3e7829029d6a0 Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Wed, 4 Oct 2023 15:30:35 -0700 Subject: [PATCH 08/27] Add github workflow tests.yml --- .github/workflows/tests.yml | 42 +++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..b8bfa32 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,42 @@ +name: Tests + +on: [push, pull_request] + +permissions: + contents: read + +jobs: + build: + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + include: + - python-version: "2.7" + os: "ubuntu-20.04" + - python-version: "2.7" + os: "ubuntu-22.04" + steps: + - uses: actions/checkout@v4.1.0 + - name: Set up Python ${{ matrix.python-version }} + # since actions/python-versions removed 2.7 from versions-manifest.json + # and also deleted all the python 2.7 binary artifacts, + # we have to apt-get install python2 + # https://github.com/actions/setup-python/issues/672 + run: | + set -eux + sudo apt-get update + sudo apt-get install -y python2 python3-virtualenv + virtualenv -p python2 "${{ runner.temp }}/venv" + # Fix for error in ubuntu-20.04 pip (fixed in ubuntu-22.04) + # can't find '__main__' module in '/usr/share/python-wheels/pep517-0.8.2-py2.py3-none-any.whl/pep517/_in_process.py + # https://github.com/pypa/pip/issues/7874#issuecomment-605520503 + "${{ runner.temp }}/venv/bin/pip" install --force-reinstall --upgrade pip + echo "${{ runner.temp }}/venv/bin" >> $GITHUB_PATH + - name: Install prod dependencies + run: python -m pip install -r requirements.txt + - name: Install test dependencies + run: python -m pip install -r requirements-test.txt + - name: Run tests + run: nosetests From e08d099e136cb3a9c602981678a5cccd85594e2d Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Wed, 4 Oct 2023 15:32:11 -0700 Subject: [PATCH 09/27] Add test that requirements.txt is complete Add test that requirements.txt contains all the setup.py install_requires, and test that requirements.txt contains all the recursive dependencies. --- .github/workflows/tests.yml | 8 +++ tests/test_requirements_txt.py | 111 +++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 tests/test_requirements_txt.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b8bfa32..5acba15 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,7 +36,15 @@ jobs: echo "${{ runner.temp }}/venv/bin" >> $GITHUB_PATH - name: Install prod dependencies run: python -m pip install -r requirements.txt + - name: Check that all recursive dependencies are in requirements.txt + run: python -m unittest tests.test_requirements_txt + env: + TEST_REQUIREMENTS: prod - name: Install test dependencies run: python -m pip install -r requirements-test.txt - name: Run tests run: nosetests + - name: Check that all recursive dependencies are in requirements.txt and requirements-test.txt + run: python -m unittest tests.test_requirements_txt + env: + TEST_REQUIREMENTS: test diff --git a/tests/test_requirements_txt.py b/tests/test_requirements_txt.py new file mode 100644 index 0000000..f8c74e3 --- /dev/null +++ b/tests/test_requirements_txt.py @@ -0,0 +1,111 @@ +from __future__ import print_function +import os +import subprocess +import sys +import unittest + + +class TestRequirementsTxt(unittest.TestCase): + def check_pip_freeze_empty(self, flags): + # Run the pip freeze command and capture its output + args = [sys.executable, '-m', 'pip', 'freeze', '--exclude-editable'] + args.extend(flags) + with open(os.devnull, 'w') as fnull: + # python 3.3: stder=subprocess.DEVNULL + result = subprocess.check_output(args, stderr=fnull) + lines = result.decode('utf-8').splitlines() + + # raises ValueError if not present + added_index = lines.index( + '## The following requirements were added by pip freeze:') + added_dependencies = [ + line for line in lines[added_index + 1:] + if len(line) > 0 + # workaround: as of ubuntu 20.04 (fixed in 22.04), + # pip freeze within virtualenv gave extraneous line + # that cannot be installed + # https://bugs.launchpad.net/ubuntu/+source/python-pip/+bug/1635463 + and line != 'pkg-resources==0.0.0' + ] + if len(added_dependencies) > 0: + self.fail( + 'Error: {} were missing recursive dependencies:\n{}'.format( + ' '.join(flags), '\n'.join(added_dependencies))) + + @unittest.skipUnless( + os.environ.get('TEST_REQUIREMENTS') == 'prod', + 'requirements test must be explicitly enabled with TEST_REQUIREMENTS' + ) + def test_requirements_txt_is_complete(self): + '''Ensure that requirements.txt has all recursive dependencies''' + self.check_pip_freeze_empty(['-r', 'requirements.txt']) + + @unittest.skipUnless( + os.environ.get('TEST_REQUIREMENTS') == 'test', + 'requirements test must be explicitly enabled with TEST_REQUIREMENTS' + ) + def test_requirements_test_txt_is_complete(self): + '''Ensure that requirements-test.txt has all recursive dependencies''' + self.check_pip_freeze_empty( + ['-r', 'requirements.txt', '-r', 'requirements-test.txt']) + + def check_pip_install_empty(self, pkg, freeze_flags): + args = [sys.executable, '-m', 'pip', 'install', '--dry-run', pkg] + live_args = [arg for arg in args if arg != '--dry-run'] + freeze_args = [sys.executable, '-m', 'pip', 'freeze', + '--exclude-editable'] + freeze_args.extend(freeze_args) + # TODO: use the --report= arg and parse json instead + with open(os.devnull, 'w') as fnull: + # python 3.3: stder=subprocess.DEVNULL + result = subprocess.check_output(args, stderr=fnull) + lines = result.decode('utf-8').splitlines() + WOULD_INSTALL = 'Would install ' + would_install_lines = [line for line in lines + if line.startswith(WOULD_INSTALL)] + self.assertGreater( + len(would_install_lines), 0, + 'Could not find Would install line after {}'.format( + ' '.join(args))) + would_install_str = would_install_lines[0][len(WOULD_INSTALL):] + would_install_packages = would_install_str.split(' ') + would_install_packages = [ + p for p in would_install_packages + if not p.startswith('remoteobjects-') + ] + self.assertEqual( + would_install_packages, [], + 'requirements missing {}; re-run {} && {}'.format( + ' '.join(would_install_packages), + ' '.join(live_args), + ' '.join(freeze_args), + ) + ) + + # skip on python2 since --dry-run was added in pip 22.2, + # but pip stopped supporting python2 in pip 21 + # https://pip.pypa.io/en/stable/news/#v22-2 + # https://pip.pypa.io/en/stable/news/#v21-0 + @unittest.skipUnless( + sys.version_info[0] >= 3, + 'setup.py prod requirements test requires pip 21' + ) + def test_setup_dependencies_are_installed(self): + '''Ensure that setup.py install_requires are all installed''' + self.check_pip_install_empty('.', ['-r', 'requirements.txt']) + + # skip on python2 since --dry-run was added in pip 22.2, + # but pip stopped supporting python2 in pip 21 + # https://pip.pypa.io/en/stable/news/#v22-2 + # https://pip.pypa.io/en/stable/news/#v21-0 + @unittest.skipUnless( + sys.version_info[0] >= 3 and + os.environ.get('TEST_REQUIREMENTS') == 'test', + 'setup.py test requirements test requires pip 21 and ' + 'must be explicitly enabled with TEST_REQUIREMENTS' + ) + def test_setup_test_dependencies_are_installed(self): + '''Ensure that setup.py extra_require[test] are all installed''' + self.check_pip_install_empty( + '.[test]', + ['-r', 'requirements.txt', '-r', 'requirements-test.txt']) From b3bba78287b244c81c7b7b27e6fb611f7fad86a0 Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Tue, 3 Oct 2023 00:28:03 -0700 Subject: [PATCH 10/27] Remove nose; use python -m unittest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit “Nose has been in maintenance mode for the past several years and will likely cease without a new person/team to take over maintainership. New projects should consider using Nose2, py.test, or just plain unittest/unittest2.” https://nose.readthedocs.io/en/latest/ We can run `python -m unittest discover` instead of nose as of python 2.7 / 3.2 https://docs.python.org/3/library/unittest.html#unittest-test-discovery Use functools.wraps (python 2.5) instead of nose.tools.make_decorator --- .github/workflows/tests.yml | 2 +- requirements-test.txt | 1 - setup.py | 1 - tests/utils.py | 5 ++--- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5acba15..a94e260 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,7 +43,7 @@ jobs: - name: Install test dependencies run: python -m pip install -r requirements-test.txt - name: Run tests - run: nosetests + run: python -m unittest discover - name: Check that all recursive dependencies are in requirements.txt and requirements-test.txt run: python -m unittest tests.test_requirements_txt env: diff --git a/requirements-test.txt b/requirements-test.txt index da3f569..4659e53 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,2 +1 @@ mox==0.5.3 -nose==1.3.7 diff --git a/setup.py b/setup.py index c4ef55d..783d59f 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,6 @@ 'python-dateutil>=2.1'], extras_require={ 'test': [ - 'nose', 'mox', ] }, diff --git a/tests/utils.py b/tests/utils.py index 888a5f7..dc4a65a 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -27,17 +27,16 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +from functools import wraps import httplib2 import logging import os import mox -import nose -import nose.tools def todo(fn): - @nose.tools.make_decorator(fn) + @wraps(fn) def test_reverse(*args, **kwargs): try: fn(*args, **kwargs) From 4b48095c21a78cbf7e4cc36b8cdf4694b9c259d1 Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Wed, 4 Oct 2023 15:47:46 -0700 Subject: [PATCH 11/27] Fix flake8 F401 module imported but unused Fix errors given by flake8 --select=F401 --- doc/exts/document_init_methods.py | 2 -- examples/couchdb.py | 2 +- examples/giantbomb.py | 2 -- examples/netflix.py | 4 +--- remoteobjects/dataobject.py | 1 - remoteobjects/fields.py | 2 -- remoteobjects/http.py | 3 +-- remoteobjects/json.py | 3 +-- remoteobjects/listobject.py | 7 +------ remoteobjects/promise.py | 2 -- tests/performance/benchmark_decoding.py | 2 -- tests/test_dataobject.py | 1 - tests/test_http.py | 3 --- tests/test_listobject.py | 3 +-- tests/test_promise.py | 2 +- tests/utils.py | 1 - 16 files changed, 7 insertions(+), 33 deletions(-) diff --git a/doc/exts/document_init_methods.py b/doc/exts/document_init_methods.py index 7b4db46..dffbe63 100644 --- a/doc/exts/document_init_methods.py +++ b/doc/exts/document_init_methods.py @@ -34,8 +34,6 @@ """ -import logging - def document_init_methods(app, what, name, obj, skip, options): if not skip: return diff --git a/examples/couchdb.py b/examples/couchdb.py index c7f1421..01233fc 100755 --- a/examples/couchdb.py +++ b/examples/couchdb.py @@ -44,7 +44,7 @@ from optparse import OptionParser import simplejson as json import sys -from urlparse import urljoin, urlparse +from urlparse import urljoin from remoteobjects import RemoteObject, fields, ListObject diff --git a/examples/giantbomb.py b/examples/giantbomb.py index 6b39bec..35c7f3f 100644 --- a/examples/giantbomb.py +++ b/examples/giantbomb.py @@ -41,10 +41,8 @@ from cgi import parse_qs -from datetime import datetime from optparse import OptionParser import sys -import time from urllib import urlencode from urlparse import urljoin, urlparse, urlunparse diff --git a/examples/netflix.py b/examples/netflix.py index 68351e6..b5c790a 100644 --- a/examples/netflix.py +++ b/examples/netflix.py @@ -43,14 +43,12 @@ import cgi from optparse import OptionParser import sys -from urllib import urlencode import urlparse from xml.etree import ElementTree -import httplib2 from oauth.oauth import OAuthConsumer, OAuthRequest, OAuthSignatureMethod_HMAC_SHA1 -from remoteobjects import RemoteObject, fields, PageObject +from remoteobjects import RemoteObject, fields class Flixject(RemoteObject): diff --git a/remoteobjects/dataobject.py b/remoteobjects/dataobject.py index 6c9d764..715c7d1 100644 --- a/remoteobjects/dataobject.py +++ b/remoteobjects/dataobject.py @@ -41,7 +41,6 @@ from copy import deepcopy -import logging import remoteobjects.fields diff --git a/remoteobjects/fields.py b/remoteobjects/fields.py index 2613feb..9efb0d5 100644 --- a/remoteobjects/fields.py +++ b/remoteobjects/fields.py @@ -40,8 +40,6 @@ from datetime import datetime, tzinfo, timedelta import dateutil.parser -import logging -import time import urlparse import remoteobjects.dataobject diff --git a/remoteobjects/http.py b/remoteobjects/http.py index 6bab3d4..a85c0c1 100644 --- a/remoteobjects/http.py +++ b/remoteobjects/http.py @@ -34,8 +34,7 @@ import httplib import logging -from remoteobjects.dataobject import DataObject, DataObjectMetaclass -from remoteobjects import fields +from remoteobjects.dataobject import DataObject userAgent = httplib2.Http() diff --git a/remoteobjects/json.py b/remoteobjects/json.py index 7ef953f..550a29f 100644 --- a/remoteobjects/json.py +++ b/remoteobjects/json.py @@ -28,9 +28,8 @@ # POSSIBILITY OF SUCH DAMAGE. from simplejson import JSONDecoder -from simplejson.decoder import FLAGS, BACKSLASH, STRINGCHUNK, DEFAULT_ENCODING +from simplejson.decoder import BACKSLASH, STRINGCHUNK, DEFAULT_ENCODING from simplejson.scanner import py_make_scanner -import re # Truly heinous... we are going to the trouble of reproducing this diff --git a/remoteobjects/listobject.py b/remoteobjects/listobject.py index 96985b9..f9a64dd 100644 --- a/remoteobjects/listobject.py +++ b/remoteobjects/listobject.py @@ -27,15 +27,10 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from urlparse import urljoin, urlparse, urlunparse -import cgi -import inspect import sys -import urllib import remoteobjects.fields as fields -from remoteobjects.dataobject import find_by_name -from remoteobjects.promise import PromiseObject, PromiseError +from remoteobjects.promise import PromiseObject class SequenceProxy(object): diff --git a/remoteobjects/promise.py b/remoteobjects/promise.py index 8ff03ea..361ce74 100644 --- a/remoteobjects/promise.py +++ b/remoteobjects/promise.py @@ -31,11 +31,9 @@ import urllib import cgi -import httplib import httplib2 import remoteobjects.http -from remoteobjects.fields import Property class PromiseError(Exception): diff --git a/tests/performance/benchmark_decoding.py b/tests/performance/benchmark_decoding.py index c3b520b..2cbf17e 100755 --- a/tests/performance/benchmark_decoding.py +++ b/tests/performance/benchmark_decoding.py @@ -38,8 +38,6 @@ import optparse import time -import remoteobjects -import mox from tests import utils diff --git a/tests/test_dataobject.py b/tests/test_dataobject.py index 5b51d2f..10a4ef6 100644 --- a/tests/test_dataobject.py +++ b/tests/test_dataobject.py @@ -28,7 +28,6 @@ # POSSIBILITY OF SUCH DAMAGE. from datetime import datetime, timedelta, tzinfo -import logging import pickle import sys import unittest diff --git a/tests/test_http.py b/tests/test_http.py index 32e8f33..61fc0d4 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -27,9 +27,6 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from datetime import datetime -import logging -import sys import unittest import mox diff --git a/tests/test_listobject.py b/tests/test_listobject.py index 1f25d32..72f7959 100644 --- a/tests/test_listobject.py +++ b/tests/test_listobject.py @@ -32,8 +32,7 @@ import httplib2 import mox -from remoteobjects import fields, http, promise, listobject -from tests import test_dataobject, test_http +from remoteobjects import listobject from tests import utils diff --git a/tests/test_promise.py b/tests/test_promise.py index 765f83b..f99364f 100644 --- a/tests/test_promise.py +++ b/tests/test_promise.py @@ -32,7 +32,7 @@ import httplib2 import mox -from remoteobjects import fields, http, promise +from remoteobjects import fields, promise from tests import test_dataobject, test_http from tests import utils diff --git a/tests/utils.py b/tests/utils.py index dc4a65a..0366644 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -30,7 +30,6 @@ from functools import wraps import httplib2 import logging -import os import mox From 8ebdf7902a0e870832ba1af4c7db10e364f0ab7e Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Wed, 4 Oct 2023 09:52:35 -0700 Subject: [PATCH 12/27] Fix flake8 F821 undefined name errmsg Fix NameError that were caught by flake8 --select=F821, or from running mypy. --- remoteobjects/json.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/remoteobjects/json.py b/remoteobjects/json.py index 550a29f..bf471c2 100644 --- a/remoteobjects/json.py +++ b/remoteobjects/json.py @@ -29,8 +29,20 @@ from simplejson import JSONDecoder from simplejson.decoder import BACKSLASH, STRINGCHUNK, DEFAULT_ENCODING +try: + # simplejson >=3.12 + from simplejson.errors import errmsg +except ImportError: + try: + # simplejson >=3.1.0, <3.12, before this commit: + # https://github.com/simplejson/simplejson/commit/0d36c5cd16055d55e6eceaf252f072a9339e0746 + from simplejson.scanner import errmsg + except ImportError: + # simplejson >=1.1,<3.1.0, before this commit: + # https://github.com/simplejson/simplejson/commit/104b40fcf6aa39d9ba7b240c3c528d1f85e86ef2 + from simplejson.decoder import errmsg from simplejson.scanner import py_make_scanner - +import sys # Truly heinous... we are going to the trouble of reproducing this # entire routine, because we need to supply an errors="replace" From b5c89e8c101163e736f528feb7bfd242405df157 Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Mon, 27 Sep 2021 12:41:48 -0700 Subject: [PATCH 13/27] Fix flake8 F632 use ==/!= to compare constant literals Use == and != to compare strings, not the is operator, since other implementations are not guaranteed to return True. In python3 this fixes SyntaxWarning: "is not" with a literal. Did you mean "!="? --- remoteobjects/promise.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/remoteobjects/promise.py b/remoteobjects/promise.py index 361ce74..a613a53 100644 --- a/remoteobjects/promise.py +++ b/remoteobjects/promise.py @@ -182,12 +182,12 @@ def options(self, http=None, **kwargs): return resp def __setattr__(self, name, value): - if name is not '_delivered' and not self._delivered and name in self.fields: + if name != '_delivered' and not self._delivered and name in self.fields: self.deliver() return super(PromiseObject, self).__setattr__(name, value) def __delattr__(self, name): - if name is not '_delivered' and not self._delivered and name in self.fields: + if name != '_delivered' and not self._delivered and name in self.fields: self.deliver() return super(PromiseObject, self).__delattr__(name) From 59fe4a127e9a169bc577bee99b437785908ce51f Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Wed, 4 Oct 2023 16:18:13 -0700 Subject: [PATCH 14/27] Fix pycodestyle E722 do not use bare 'except' --- tests/performance/benchmark_decoding.py | 2 +- tests/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/performance/benchmark_decoding.py b/tests/performance/benchmark_decoding.py index 2cbf17e..cfe0b06 100755 --- a/tests/performance/benchmark_decoding.py +++ b/tests/performance/benchmark_decoding.py @@ -76,7 +76,7 @@ def test_decoding(object_class, json, count): try: fd = open(args[0]) json = fd.read() - except: + except Exception: parser.error("Unable to read file: '%s'" % args[1]) finally: fd.close() diff --git a/tests/utils.py b/tests/utils.py index 0366644..c993a45 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -39,7 +39,7 @@ def todo(fn): def test_reverse(*args, **kwargs): try: fn(*args, **kwargs) - except: + except Exception: pass else: raise AssertionError('test %s unexpectedly succeeded' % fn.__name__) From 94aa4aff027c8beb1a7569369b13ec3f0324b448 Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Wed, 4 Oct 2023 17:07:58 -0700 Subject: [PATCH 15/27] Fix pycodestyle E721 do not compare types When overriding __eq__, call isinstance instead of type(self) == type(other), and return NotImplemented instead of returning False so that python will try the reflected comparison. This allows a different class to consider itself equal to this class. --- remoteobjects/dataobject.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/remoteobjects/dataobject.py b/remoteobjects/dataobject.py index 715c7d1..a34ab2b 100644 --- a/remoteobjects/dataobject.py +++ b/remoteobjects/dataobject.py @@ -152,8 +152,8 @@ def __eq__(self, other): same data in all their fields, the objects are equivalent. """ - if type(self) != type(other): - return False + if not isinstance(other, type(self)): + return NotImplemented for k, v in self.fields.iteritems(): if isinstance(v, remoteobjects.fields.Field): if getattr(self, k) != getattr(other, k): From ad110490ea225665145541382406e7156d89724c Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Tue, 17 Oct 2023 03:34:35 -0700 Subject: [PATCH 16/27] Fix pycodestyle E401,W291,W391,W293 python -m autopep8 --in-place --recursive . E401 multiple imports on one line W291 trailing whitespace W391 blank line at end of file Additionally, fix this one manually W293 blank line contains whitespace --- doc/conf.py | 3 ++- remoteobjects/fields.py | 2 +- remoteobjects/http.py | 2 +- remoteobjects/json.py | 3 ++- tests/__init__.py | 1 - tests/test_dataobject.py | 2 +- tests/test_listobject.py | 2 +- 7 files changed, 8 insertions(+), 7 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index f01a717..2e958b9 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -11,7 +11,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import sys +import os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the diff --git a/remoteobjects/fields.py b/remoteobjects/fields.py index 9efb0d5..db97395 100644 --- a/remoteobjects/fields.py +++ b/remoteobjects/fields.py @@ -231,7 +231,7 @@ def __get__(self, obj, cls): return self # Since it's a constant, always return the same value. return self.value - + def __set__(self, obj, value): # If it's the correct value, do nothing. Else, raise an exception. if value != self.value: diff --git a/remoteobjects/http.py b/remoteobjects/http.py index a85c0c1..f0e7d93 100644 --- a/remoteobjects/http.py +++ b/remoteobjects/http.py @@ -126,7 +126,7 @@ class RequestError(httplib.HTTPException): client's request. This exception corresponds to the HTTP status code 400. - + """ pass diff --git a/remoteobjects/json.py b/remoteobjects/json.py index bf471c2..ee582a6 100644 --- a/remoteobjects/json.py +++ b/remoteobjects/json.py @@ -53,7 +53,8 @@ def forgiving_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=ST Unescapes all valid JSON string escape sequences and raises ValueError on attempt to decode an invalid string. If strict is False then literal control characters are allowed in the string. - + + Returns a tuple of the decoded string and the index of the character in s after the end quote.""" if encoding is None: diff --git a/tests/__init__.py b/tests/__init__.py index 121920c..ecd4135 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -26,4 +26,3 @@ # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. - diff --git a/tests/test_dataobject.py b/tests/test_dataobject.py index 10a4ef6..6380d5c 100644 --- a/tests/test_dataobject.py +++ b/tests/test_dataobject.py @@ -535,7 +535,7 @@ class Timely(dataobject.DataObject): when = datetime(year=2008, month=12, day=31, hour=4, minute=0, second=1, tzinfo=fields.Datetime.utc) self.assertEquals(t.when, when, 'Datetime data decoded into the expected datetime') - self.assert_(t.when.tzinfo is fields.Datetime.utc, + self.assert_(t.when.tzinfo is fields.Datetime.utc, 'Datetime data decoded with utc timezone info') when = datetime(year=2010, month=2, day=11, hour=4, minute=37, second=44) diff --git a/tests/test_listobject.py b/tests/test_listobject.py index 72f7959..fd06763 100644 --- a/tests/test_listobject.py +++ b/tests/test_listobject.py @@ -85,4 +85,4 @@ class Toybox(self.cls): b = Toybox.get('http://example.com/whahay', http=h) self.assertEqual(b[7], 7) - mox.Verify(h) + mox.Verify(h) From 9c535ab88a8aad1ff16970026c95be9f41c49634 Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Tue, 17 Oct 2023 04:32:32 -0700 Subject: [PATCH 17/27] Run flake8 from github workflow Use flake8 version 3.9.2 which can run from python 2.7 to 3.11 (but not python 3.12). Use a .flake8 file that ignores pep8 whitespace issues --- .flake8 | 36 ++++++++++++++++++++++++++++++++++++ .github/workflows/tests.yml | 5 +++++ requirements-test.txt | 13 +++++++++++++ setup.py | 1 + 4 files changed, 55 insertions(+) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..f30b319 --- /dev/null +++ b/.flake8 @@ -0,0 +1,36 @@ +[flake8] +exclude = + .git, + .direnv, +# flake8 has extend-ignore, but autopep8 does not support it +ignore = + # python3 -m flake8 --help (3.9.2, with no .flake8 config) + # gives these defaults: + # E704 multiple statements on one line (def) + E704, + # E24 multiple spaces after ‘,’ or tab after ‘,’ + E24, + # E226 missing whitespace around arithmetic operator + E226, + # E126 continuation line over-indented for hanging indent + E126, + # W503 line break before binary operator + W503, + # W504 line break after binary operator + W504, + # E121 continuation line under-indented for hanging indent + E121, + # E123 closing bracket does not match indentation of opening bracket’s line + E123, + + # too many changes for now + # E501 line too long + E501, + # fixing style is too many changes for now + # https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes + # E1 Indentation + E1, + # E2 Whitespace + E2, + # E3 Blank line + E3, diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a94e260..ddd7680 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,3 +48,8 @@ jobs: run: python -m unittest tests.test_requirements_txt env: TEST_REQUIREMENTS: test + - name: Run flake8 + # need to upgrade flake8 to 6.1.0 for python 3.12 + # https://flake8.pycqa.org/en/latest/release-notes/6.1.0.html + if: matrix.python-version != '3.12' + run: python -m flake8 diff --git a/requirements-test.txt b/requirements-test.txt index 4659e53..ff47e60 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1 +1,14 @@ +configparser==4.0.2 +contextlib2==0.6.0.post1 +enum34==1.1.10 +flake8==3.9.2 +functools32==3.2.3.post2 +importlib-metadata==2.1.3 +mccabe==0.6.1 mox==0.5.3 +pathlib2==2.3.7.post1 +pycodestyle==2.7.0 +pyflakes==2.3.1 +scandir==1.10.0 +typing==3.10.0.0 +zipp==1.2.0 diff --git a/setup.py b/setup.py index 783d59f..c00b980 100644 --- a/setup.py +++ b/setup.py @@ -66,6 +66,7 @@ 'python-dateutil>=2.1'], extras_require={ 'test': [ + 'flake8~=3.9', 'mox', ] }, From 76114e7af4e764081a0d9b68f1d5e884b1240740 Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Mon, 27 Sep 2021 17:34:49 -0700 Subject: [PATCH 18/27] Switch from mox to mock Mox (https://code.google.com/archive/p/pymox/wikis/MoxDocumentation.wiki) and mox3 (https://opendev.org/openstack/mox3/) are deprecated; mock (https://mock.readthedocs.io/en/latest/) is the recommended replacement. We can use mock for now and then switch to unittest.mock when we drop python<3.3 support --- requirements-test.txt | 3 ++- setup.py | 6 +++++- tests/test_dataobject.py | 4 ++-- tests/test_http.py | 34 ++++++++++++++++------------------ tests/test_listobject.py | 10 ++++------ tests/test_promise.py | 17 ++++++++--------- tests/utils.py | 9 ++++----- 7 files changed, 41 insertions(+), 42 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index ff47e60..5f230eb 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -2,10 +2,11 @@ configparser==4.0.2 contextlib2==0.6.0.post1 enum34==1.1.10 flake8==3.9.2 +funcsigs==1.0.2 functools32==3.2.3.post2 importlib-metadata==2.1.3 mccabe==0.6.1 -mox==0.5.3 +mock==3.0.5 pathlib2==2.3.7.post1 pycodestyle==2.7.0 pyflakes==2.3.1 diff --git a/setup.py b/setup.py index c00b980..daa4529 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,11 @@ extras_require={ 'test': [ 'flake8~=3.9', - 'mox', + # according to https://mock.readthedocs.io/en/latest/ + # version 3.0.5 is the last version supporting Python <= 3.5. + # Once we migrate to python 3.3+, we can switch from mock to + # unittest.mock + 'mock~=3.0.5', ] }, ) diff --git a/tests/test_dataobject.py b/tests/test_dataobject.py index 6380d5c..39aebb6 100644 --- a/tests/test_dataobject.py +++ b/tests/test_dataobject.py @@ -32,7 +32,7 @@ import sys import unittest -import mox +import mock from remoteobjects import fields, dataobject from tests import utils @@ -365,7 +365,7 @@ class BasicMost(self.cls): # Simulate a special module for this BasicMost, so pickle can find # the class for it. - pickletest_module = mox.MockAnything() + pickletest_module = mock.Mock() pickletest_module.BasicMost = BasicMost # Note this pseudomodule has no file, so coverage doesn't get a mock # method by mistake. diff --git a/tests/test_http.py b/tests/test_http.py index 61fc0d4..517a883 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -29,8 +29,6 @@ import unittest -import mox - from remoteobjects import fields, http from tests import test_dataobject from tests import utils @@ -63,7 +61,7 @@ class BasicMost(self.cls): headers={"x-test": "boo"}) self.assertEquals(b.name, 'Fred') self.assertEquals(b.value, 7) - mox.Verify(h) + h.request.assert_called_once_with(**request) def test_get_bad_encoding(self): @@ -82,7 +80,7 @@ class BasicMost(self.cls): self.assertEquals(b.name, u"Fred\ufffd") # Bad characters are replaced with the unicode Replacement Character 0xFFFD. self.assertEquals(b.value, u"image by \ufffdrew Example") - mox.Verify(h) + h.request.assert_called_once_with(**request) def test_post(self): @@ -101,7 +99,7 @@ class ContainerMost(self.cls): h = utils.mock_http(request, content) c = ContainerMost.get('http://example.com/asfdasf', http=h) self.assertEquals(c.name, 'CBS') - mox.Verify(h) + h.request.assert_called_once_with(**request) b = BasicMost(name='Fred Friendly', value=True) @@ -116,7 +114,7 @@ class ContainerMost(self.cls): location='http://example.com/fred') h = utils.mock_http(request, response) c.post(b, http=h) - mox.Verify(h) + h.request.assert_called_once_with(**request) self.assertEquals(b._location, 'http://example.com/fred') self.assertEquals(b._etag, 'xyz') @@ -138,7 +136,7 @@ class BasicMost(self.cls): h = utils.mock_http(request, content) b = BasicMost.get('http://example.com/bwuh', http=h) self.assertEquals(b.name, 'Molly') - mox.Verify(h) + h.request.assert_called_once_with(**request) headers = { 'accept': 'application/json', @@ -149,7 +147,7 @@ class BasicMost(self.cls): response = dict(content=content, etag='xyz') h = utils.mock_http(request, response) b.put(http=h) - mox.Verify(h) + h.request.assert_called_once_with(**request) self.assertEquals(b._etag, 'xyz') @@ -171,7 +169,7 @@ class BasicMost(self.cls): h = utils.mock_http(request, content) b = BasicMost.get('http://example.com/bwuh', http=h) self.assertEquals(b.name, 'Molly') - mox.Verify(h) + h.request.assert_called_once_with(**request) headers = { 'accept': 'application/json', @@ -182,7 +180,7 @@ class BasicMost(self.cls): response = dict(content="", status=204) h = utils.mock_http(request, response) b.put(http=h) - mox.Verify(h) + h.request.assert_called_once_with(**request) self.assertEquals(b.name, 'Molly') @@ -200,7 +198,7 @@ class BasicMost(self.cls): h = utils.mock_http(request, content) b = BasicMost.get('http://example.com/bwuh', http=h) self.assertEquals(b.value, 80) - mox.Verify(h) + h.request.assert_called_once_with(**request) b.value = 'superluminal' @@ -216,7 +214,7 @@ class BasicMost(self.cls): response = dict(status=412) h = utils.mock_http(request, response) self.assertRaises(BasicMost.PreconditionFailed, lambda: b.put(http=h)) - mox.Verify(h) + h.request.assert_called_once_with(**request) def test_delete(self): @@ -235,7 +233,7 @@ class BasicMost(self.cls): h = utils.mock_http(request, content) b = BasicMost.get('http://example.com/bwuh', http=h) self.assertEquals(b.value, 80) - mox.Verify(h) + h.request.assert_called_once_with(**request) headers = { 'accept': 'application/json', @@ -245,7 +243,7 @@ class BasicMost(self.cls): response = dict(status=204) h = utils.mock_http(request, response) b.delete(http=h) - mox.Verify(h) + h.request.assert_called_once_with(**request) self.failIf(b._location is not None) self.failIf(hasattr(b, '_etag')) @@ -269,7 +267,7 @@ class BasicMost(self.cls): h = utils.mock_http(request, response) self.assertRaises(BasicMost.PreconditionFailed, lambda: b.delete(http=h)) - mox.Verify(h) + h.request.assert_called_once_with(**request) def test_not_found(self): self.assert_(self.cls.NotFound) @@ -286,7 +284,7 @@ class Huh(self.cls): response = {'content': '', 'status': 404} http = utils.mock_http(request, response) self.assertRaises(Huh.NotFound, lambda: Huh.get('http://example.com/bwuh', http=http).name) - mox.Verify(http) + http.request.assert_called_once_with(**request) @utils.todo def test_not_found_discrete(self): @@ -324,9 +322,9 @@ def try_that(http): 'headers': {'accept': 'application/json'}, } response = dict(status=404) - http = utils.MockedHttp(request, response) + http = utils.mock_http(request, response) self.assertRaises(What.NotFound, lambda: try_that(http)) - mox.Verify(http) + http.request.assert_called_once_with(**request) if __name__ == '__main__': diff --git a/tests/test_listobject.py b/tests/test_listobject.py index fd06763..6a83380 100644 --- a/tests/test_listobject.py +++ b/tests/test_listobject.py @@ -30,7 +30,7 @@ import unittest import httplib2 -import mox +import mock from remoteobjects import listobject from tests import utils @@ -45,8 +45,7 @@ def test_slice_filter(self): class Toybox(self.cls): pass - h = mox.MockObject(httplib2.Http) - mox.Replay(h) + h = mock.NonCallableMock(spec_set=httplib2.Http) b = Toybox.get('http://example.com/foo', http=h) self.assertEquals(b._location, 'http://example.com/foo') @@ -68,7 +67,7 @@ class Toybox(self.cls): self.assertEquals(j._location, 'http://example.com/foo?limit=10') # Nobody did any HTTP, right? - mox.Verify(h) + self.assertEqual([], h.method_calls) def test_index(self): @@ -80,9 +79,8 @@ class Toybox(self.cls): request = dict(uri=url, headers=headers) content = """{"entries":[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}""" h = utils.mock_http(request, content) - mox.Replay(h) b = Toybox.get('http://example.com/whahay', http=h) self.assertEqual(b[7], 7) - mox.Verify(h) + h.request.assert_called_once_with(**request) diff --git a/tests/test_promise.py b/tests/test_promise.py index f99364f..c0972a3 100644 --- a/tests/test_promise.py +++ b/tests/test_promise.py @@ -30,7 +30,7 @@ import unittest import httplib2 -import mox +import mock from remoteobjects import fields, promise from tests import test_dataobject, test_http @@ -56,30 +56,29 @@ def test_basic(self): class Tiny(self.cls): name = fields.Field() - h = mox.MockObject(httplib2.Http) - mox.Replay(h) + h = mock.NonCallableMock(spec_set=httplib2.Http) url = 'http://example.com/whahay' t = Tiny.get(url, http=h) # Check that we didn't do anything. - mox.Verify(h) + self.assertEqual([], h.method_calls) headers = {"accept": "application/json"} request = dict(uri=url, headers=headers) content = """{"name": "Mollifred"}""" h = utils.mock_http(request, content) - t._http = h # inject, oops + t._http = h + self.assertEquals(t.name, 'Mollifred') - mox.Verify(h) + h.request.assert_called_once_with(**request) def test_filter(self): class Toy(self.cls): name = fields.Field() - h = mox.MockObject(httplib2.Http) - mox.Replay(h) + h = mock.NonCallableMock(spec_set=httplib2.Http) b = Toy.get('http://example.com/foo', http=h) self.assertEquals(b._location, 'http://example.com/foo') @@ -95,7 +94,7 @@ class Toy(self.cls): self.assertEquals(y._location, 'http://example.com/foo?awesome=no') # Nobody did any HTTP, right? - mox.Verify(h) + self.assertEqual([], h.method_calls) def test_awesome(self): diff --git a/tests/utils.py b/tests/utils.py index c993a45..5b665e0 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -31,7 +31,7 @@ import httplib2 import logging -import mox +import mock def todo(fn): @@ -47,7 +47,7 @@ def test_reverse(*args, **kwargs): def mock_http(req, resp_or_content): - mock = mox.MockObject(httplib2.Http) + m = mock.Mock(spec_set=httplib2.Http) if not isinstance(req, dict): req = dict(uri=req) @@ -81,9 +81,8 @@ def make_response(response, url): return httplib2.Response(response_info), content resp, content = make_response(resp_or_content, req['uri']) - mock.request(**req).AndReturn((resp, content)) - mox.Replay(mock) - return mock + m.request.return_value = (resp, content) + return m def log(): From 416747a5a8cb37535592684210eb94fbfb0f4165 Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Mon, 27 Sep 2021 19:58:59 -0700 Subject: [PATCH 19/27] Add test that filter takes both str and unicode --- tests/test_promise.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_promise.py b/tests/test_promise.py index c0972a3..46c09f7 100644 --- a/tests/test_promise.py +++ b/tests/test_promise.py @@ -96,6 +96,22 @@ class Toy(self.cls): # Nobody did any HTTP, right? self.assertEqual([], h.method_calls) + def test_filter_mix_str_unicode(self): + """On python2, test that filter accepts both unicode and str""" + class Toy(self.cls): + name = fields.Field() + + h = mock.NonCallableMock(spec_set=httplib2.Http) + + b = Toy.get('http://example.com/foo', http=h) + self.assertEquals(b._location, 'http://example.com/foo') + + y = b.filter(a='a', b=u'b') + self.assertEquals(y._location, 'http://example.com/foo?a=a&b=b') + y = b.filter(**{'a': 'a', u'b': u'b'}) + self.assertEquals(y._location, 'http://example.com/foo?a=a&b=b') + + def test_awesome(self): class Toy(self.cls): From ca1001b9e360fdd2bc4b1b006aaa56260554fd55 Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Sun, 9 Jul 2023 20:09:40 -0700 Subject: [PATCH 20/27] Fix deprecated TestCase methods These old methods were considered deprecated as of python 2.7, logged DeprecationWarnings in python 3.2, and were removed in python 3.12. assert_ -> assertTrue failIf -> assertFalse assertEquals -> assertEqual self\.assertTrue\(isinstance\((.*)\)(, .*)?\) -> self.assertIsInstance($1$2) --- tests/test_dataobject.py | 208 +++++++++++++++++++-------------------- tests/test_http.py | 34 +++---- tests/test_listobject.py | 18 ++-- tests/test_promise.py | 32 +++--- 4 files changed, 146 insertions(+), 146 deletions(-) diff --git a/tests/test_dataobject.py b/tests/test_dataobject.py index 39aebb6..536b8b9 100644 --- a/tests/test_dataobject.py +++ b/tests/test_dataobject.py @@ -49,27 +49,27 @@ class BasicMost(self.cls): value = fields.Field() b = BasicMost.from_dict({ 'name': 'foo', 'value': '4' }) - self.assert_(b, 'from_dict() returned something True') - self.assertEquals(b.name, 'foo', 'from_dict() result has correct name') - self.assertEquals(b.value, '4', 'from_dict() result has correct value') + self.assertTrue(b, 'from_dict() returned something True') + self.assertEqual(b.name, 'foo', 'from_dict() result has correct name') + self.assertEqual(b.value, '4', 'from_dict() result has correct value') b = BasicMost(name='bar', value='47').to_dict() - self.assert_(b, 'to_dict() returned something True') - self.assertEquals({ 'name': 'bar', 'value': '47' }, b, 'Basic dict has proper contents') + self.assertTrue(b, 'to_dict() returned something True') + self.assertEqual({ 'name': 'bar', 'value': '47' }, b, 'Basic dict has proper contents') - self.assertEquals(BasicMost.__name__, 'BasicMost', + self.assertEqual(BasicMost.__name__, 'BasicMost', "metaclass magic didn't break our class's name") bm = BasicMost(name='fred', value=2) bm.api_data = {"name": "fred", "value": 2} bm_dict = bm.to_dict() - self.assertEquals({ 'name': 'fred', 'value': 2 }, bm_dict, 'First go-round has proper contents') + self.assertEqual({ 'name': 'fred', 'value': 2 }, bm_dict, 'First go-round has proper contents') bm.name = 'tom' bm_dict = bm.to_dict() - self.assertEquals({ 'name': 'tom', 'value': 2 }, bm_dict, 'Setting name to another string works') + self.assertEqual({ 'name': 'tom', 'value': 2 }, bm_dict, 'Setting name to another string works') bm.name = None bm_dict = bm.to_dict() - self.assertEquals({ 'value': 2 }, bm_dict, 'Setting name to None works, and name is omitted in the dict') + self.assertEqual({ 'value': 2 }, bm_dict, 'Setting name to None works, and name is omitted in the dict') def test_descriptorwise(self): @@ -79,10 +79,10 @@ class BasicMost(self.cls): b = BasicMost() b.name = 'hi' - self.assertEquals(b.name, 'hi') + self.assertEqual(b.name, 'hi') del b.name - self.assert_(b.name is None) + self.assertTrue(b.name is None) def test_types(self): @@ -96,10 +96,10 @@ class WithTypes(self.cls): 'value': 4, 'when': '2008-12-31T04:00:01Z', }) - self.assert_(w, 'from_dict returned something True') - self.assertEquals(w.name, 'foo', 'Typething got the right name') - self.assertEquals(w.value, 4, 'Typething got the right value') - self.assertEquals(w.when, datetime(2008, 12, 31, 4, 0, 1, + self.assertTrue(w, 'from_dict returned something True') + self.assertEqual(w.name, 'foo', 'Typething got the right name') + self.assertEqual(w.value, 4, 'Typething got the right value') + self.assertEqual(w.when, datetime(2008, 12, 31, 4, 0, 1, tzinfo=fields.Datetime.utc), 'Typething got something like the right when') @@ -109,7 +109,7 @@ class WithTypes(self.cls): 'when': '2012-08-17T14:49:50-05:00' }) - self.assertEquals(w.when, datetime(2012, 8, 17, 19, 49, 50, + self.assertEqual(w.when, datetime(2012, 8, 17, 19, 49, 50, tzinfo=fields.Datetime.utc), 'Non-UTC timezone was parsed and converted to UTC') @@ -138,8 +138,8 @@ class WithTypes(self.cls): self.fail('No TypeError parsing malformatted timestamp') w = WithTypes(name='hi', value=99, when=datetime(2009, 2, 3, 10, 44, 0, tzinfo=None)).to_dict() - self.assert_(w, 'to_dict() returned something True') - self.assertEquals(w, { 'name': 'hi', 'value': 99, 'when': '2009-02-03T10:44:00Z' }, + self.assertTrue(w, 'to_dict() returned something True') + self.assertEqual(w, { 'name': 'hi', 'value': 99, 'when': '2009-02-03T10:44:00Z' }, 'Typething dict has proper contents') def test_must_ignore(self): @@ -154,34 +154,34 @@ class BasicMost(self.cls): 'secret': 'codes', }) - self.assert_(b) - self.assert_(b.name) + self.assertTrue(b) + self.assertTrue(b.name) self.assertRaises(AttributeError, lambda: b.secret) d = b.to_dict() - self.assert_('name' in d) - self.assert_('secret' in d) - self.assertEquals(d['secret'], 'codes') + self.assertTrue('name' in d) + self.assertTrue('secret' in d) + self.assertEqual(d['secret'], 'codes') d['blah'] = 'meh' d = b.to_dict() - self.assert_('blah' not in d) + self.assertTrue('blah' not in d) x = BasicMost.from_dict({ 'name': 'foo', 'value': '4', }) self.assertNotEqual(id(b), id(x)) - self.assert_(x) - self.assert_(x.name) + self.assertTrue(x) + self.assertTrue(x.name) x.update_from_dict({ 'secret': 'codes' }) self.assertRaises(AttributeError, lambda: x.secret) d = x.to_dict() - self.assert_('name' not in d) - self.assert_('secret' in d) - self.assertEquals(d['secret'], 'codes') + self.assertTrue('name' not in d) + self.assertTrue('secret' in d) + self.assertEqual(d['secret'], 'codes') def test_spooky_action(self): """Tests that an instance's content can't be changed through the data @@ -202,23 +202,23 @@ class BasicMost(self.cls): x = BasicMost.from_dict(initial) initial['name'] = 'bar' - self.assertEquals(x.name, 'bar', + self.assertEqual(x.name, 'bar', "Changing initial data does change instance's " "internal data") initial['secret']['code'] = 'steak' d = x.to_dict() - self.assertEquals(d['secret']['code'], 'steak', + self.assertEqual(d['secret']['code'], 'steak', "Changing deep hidden initial data *does* change instance's " "original data for export") d['name'] = 'baz' - self.assertEquals(x.name, 'bar', + self.assertEqual(x.name, 'bar', "Changing shallow exported data doesn't change instance's " "internal data retroactively") d['secret']['code'] = 'walt sent me' - self.assertEquals(x.to_dict()['secret']['code'], 'steak', + self.assertEqual(x.to_dict()['secret']['code'], 'steak', "Changing deep exported data doesn't change instance's " "internal data retroactively") @@ -241,7 +241,7 @@ class WithTypes(self.cls): }) self.assertRaises(TypeError, lambda: testobj.when) - self.assert_(testobj.bleh, 'Accessing properly formatted subobject raises no exceptions') + self.assertTrue(testobj.bleh, 'Accessing properly formatted subobject raises no exceptions') testobj = WithTypes.from_dict({ 'name': 'foo', @@ -250,7 +250,7 @@ class WithTypes(self.cls): 'bleh': True, }) - self.assert_(testobj.when, 'Accessing properly formatted datetime attribute raises no exceptions') + self.assertTrue(testobj.when, 'Accessing properly formatted datetime attribute raises no exceptions') self.assertRaises(TypeError, lambda: testobj.bleh) def test_complex(self): @@ -271,23 +271,23 @@ class Parentish(self.cls): ], }) - self.assert_(p, 'from_dict() returned something True for a parent') - self.assertEquals(p.name, 'the parent', 'parent has correct name') - self.assert_(p.children, 'parent has some children') - self.assert_(isinstance(p.children, list), 'children set is a Python list') - self.assertEquals(len(p.children), 3, 'parent has 3 children') + self.assertTrue(p, 'from_dict() returned something True for a parent') + self.assertEqual(p.name, 'the parent', 'parent has correct name') + self.assertTrue(p.children, 'parent has some children') + self.assertIsInstance(p.children, list, 'children set is a Python list') + self.assertEqual(len(p.children), 3, 'parent has 3 children') f, b, w = p.children - self.assert_(isinstance(f, Childer), "parent's first child is a Childer") - self.assert_(isinstance(b, Childer), "parent's twoth child is a Childer") - self.assert_(isinstance(w, Childer), "parent's third child is a Childer") - self.assertEquals(f.name, 'fredina', "parent's first child is named fredina") - self.assertEquals(b.name, 'billzebub', "parent's twoth child is named billzebub") - self.assertEquals(w.name, 'wurfledurf', "parent's third child is named wurfledurf") + self.assertIsInstance(f, Childer, "parent's first child is a Childer") + self.assertIsInstance(b, Childer, "parent's twoth child is a Childer") + self.assertIsInstance(w, Childer, "parent's third child is a Childer") + self.assertEqual(f.name, 'fredina', "parent's first child is named fredina") + self.assertEqual(b.name, 'billzebub', "parent's twoth child is named billzebub") + self.assertEqual(w.name, 'wurfledurf', "parent's third child is named wurfledurf") childs = Childer(name='jeff'), Childer(name='lisa'), Childer(name='conway') p = Parentish(name='molly', children=childs).to_dict() - self.assert_(p, 'to_dict() returned something True') - self.assertEquals(p, { + self.assertTrue(p, 'to_dict() returned something True') + self.assertEqual(p, { 'name': 'molly', 'children': [ { 'name': 'jeff' }, @@ -301,7 +301,7 @@ class Parentish(self.cls): 'children': None }) - self.assertEquals(p.children, None) + self.assertEqual(p.children, None) def test_complex_dict(self): @@ -314,7 +314,7 @@ class Thing(self.cls): 'attributes': None, }) - self.assertEquals(t.attributes, None) + self.assertEqual(t.attributes, None) def test_self_reference(self): @@ -328,10 +328,10 @@ class Reflexive(self.cls): 'themselves': [ {}, {}, {} ], }) - self.assert_(r) - self.assert_(isinstance(r, Reflexive)) - self.assert_(isinstance(r.itself, Reflexive)) - self.assert_(isinstance(r.themselves[0], Reflexive)) + self.assertTrue(r) + self.assertIsInstance(r, Reflexive) + self.assertIsInstance(r.itself, Reflexive) + self.assertIsInstance(r.themselves[0], Reflexive) def test_post_reference(self): @@ -348,15 +348,15 @@ class NotRelated(extra_dataobject.OtherRelated): r = Referencive.from_dict({ 'related': {}, 'other': {} }) - self.assert_(isinstance(r, Referencive)) - self.assert_(isinstance(r.related, Related)) # not extra_dataobject.Related - self.assert_(isinstance(r.other, extra_dataobject.OtherRelated)) # not NotRelated + self.assertIsInstance(r, Referencive) + self.assertIsInstance(r.related, Related) # not extra_dataobject.Related + self.assertIsInstance(r.other, extra_dataobject.OtherRelated) # not NotRelated r = extra_dataobject.Referencive.from_dict({ 'related': {}, 'other': {} }) - self.assert_(isinstance(r, extra_dataobject.Referencive)) - self.assert_(isinstance(r.related, Related)) # not extra_dataobject.Related - self.assert_(isinstance(r.other, extra_dataobject.OtherRelated)) # not NotRelated + self.assertIsInstance(r, extra_dataobject.Referencive) + self.assertIsInstance(r.related, Related) # not extra_dataobject.Related + self.assertIsInstance(r.other, extra_dataobject.OtherRelated) # not NotRelated def set_up_pickling_class(self): class BasicMost(self.cls): @@ -382,16 +382,16 @@ def test_pickling(self): obj = BasicMost(name='fred', value=7) pickled_obj = pickle.dumps(obj) - self.assert_(pickled_obj) + self.assertTrue(pickled_obj) unpickled_obj = pickle.loads(pickled_obj) - self.assertEquals(unpickled_obj, obj) + self.assertEqual(unpickled_obj, obj) obj = BasicMost.from_dict({'name': 'fred', 'value': 7}) cloned_obj = pickle.loads(pickle.dumps(obj)) - self.assert_(cloned_obj) - self.assert_(hasattr(cloned_obj, 'api_data'), "unpickled instance has api_data too") - self.assertEquals(cloned_obj.api_data, obj.api_data, + self.assertTrue(cloned_obj) + self.assertTrue(hasattr(cloned_obj, 'api_data'), "unpickled instance has api_data too") + self.assertEqual(cloned_obj.api_data, obj.api_data, "unpickled instance kept original's api_data") def test_field_override(self): @@ -403,9 +403,9 @@ class Parent(dataobject.DataObject): class Child(Parent): ted = fields.Datetime() - self.assert_('fred' in Child.fields, 'Child class inherited the fred field') - self.assert_('ted' in Child.fields, 'Child class has a ted field (from somewhere') - self.assert_(isinstance(Child.fields['ted'], fields.Datetime), + self.assertTrue('fred' in Child.fields, 'Child class inherited the fred field') + self.assertTrue('ted' in Child.fields, 'Child class has a ted field (from somewhere') + self.assertIsInstance(Child.fields['ted'], fields.Datetime, 'Child class has overridden ted field, yay') def test_field_api_name(self): @@ -421,16 +421,16 @@ class WeirdNames(dataobject.DataObject): 'plugh': 'http://en.wikipedia.org/wiki/Xyzzy#Poor_password_choice', }) - self.assertEquals(w.normal, 'asfdasf', 'normal value carried through') - self.assertEquals(w.fooBarBaz, 'wurfledurf', 'fbb value carried through') - self.assertEquals(w.xyzzy, 'http://en.wikipedia.org/wiki/Xyzzy#Poor_password_choice', + self.assertEqual(w.normal, 'asfdasf', 'normal value carried through') + self.assertEqual(w.fooBarBaz, 'wurfledurf', 'fbb value carried through') + self.assertEqual(w.xyzzy, 'http://en.wikipedia.org/wiki/Xyzzy#Poor_password_choice', 'xyzzy value carried through') w = WeirdNames(normal='gloing', fooBarBaz='grumdabble', xyzzy='slartibartfast') d = w.to_dict() - self.assert_(d, 'api_named to_dict() returned something True') - self.assertEquals(d, { + self.assertTrue(d, 'api_named to_dict() returned something True') + self.assertEqual(d, { 'normal': 'gloing', 'foo-bar-baz': 'grumdabble', 'plugh': 'slartibartfast', @@ -442,7 +442,7 @@ def test_field_default(self): cheezCalled = False def cheezburgh(obj): - self.assert_(isinstance(obj, WithDefaults)) + self.assertIsInstance(obj, WithDefaults) global cheezCalled cheezCalled = True return 'CHEEZBURGH' @@ -458,21 +458,21 @@ class WithDefaults(dataobject.DataObject): 'itsUsuallySomething': 'omg hi', }) - self.assertEquals(w.plain, 'awesome') - self.assertEquals(w.itsAlwaysSomething, 'haptics') - self.assertEquals(w.itsUsuallySomething, 'omg hi') - self.failIf(cheezCalled) + self.assertEqual(w.plain, 'awesome') + self.assertEqual(w.itsAlwaysSomething, 'haptics') + self.assertEqual(w.itsUsuallySomething, 'omg hi') + self.assertFalse(cheezCalled) for x in (WithDefaults.from_dict({}), WithDefaults()): - self.assert_(x.plain is None) - self.assertEquals(x.itsAlwaysSomething, 7) - self.assertEquals(x.itsUsuallySomething, 'CHEEZBURGH') - self.assert_(cheezCalled) + self.assertTrue(x.plain is None) + self.assertEqual(x.itsAlwaysSomething, 7) + self.assertEqual(x.itsUsuallySomething, 'CHEEZBURGH') + self.assertTrue(cheezCalled) d = WithDefaults().to_dict() - self.assert_('plain' not in d) - self.assertEquals(d['itsAlwaysSomething'], 7) - self.assertEquals(d['itsUsuallySomething'], 'CHEEZBURGH') + self.assertTrue('plain' not in d) + self.assertEqual(d['itsAlwaysSomething'], 7) + self.assertEqual(d['itsUsuallySomething'], 'CHEEZBURGH') def test_field_constant(self): @@ -482,10 +482,10 @@ class WithConstant(dataobject.DataObject): alwaysTheSame = fields.Constant(noninconstant) d = WithConstant().to_dict() - self.assertEquals(d['alwaysTheSame'], noninconstant) + self.assertEqual(d['alwaysTheSame'], noninconstant) x = WithConstant() - self.assertEquals(x.alwaysTheSame, noninconstant) + self.assertEqual(x.alwaysTheSame, noninconstant) try: x.alwaysTheSame = 'snarf' @@ -496,7 +496,7 @@ class WithConstant(dataobject.DataObject): x.alwaysTheSame = noninconstant # Just to make sure - self.assertEquals(x.alwaysTheSame, noninconstant) + self.assertEqual(x.alwaysTheSame, noninconstant) def test_field_link(self): @@ -509,7 +509,7 @@ class WithLink(dataobject.DataObject): x = WithLink() x.link = Frob() # Links don't serialize... for now anyways. - self.assertEquals(x.to_dict(), {}) + self.assertEqual(x.to_dict(), {}) def test_forwards_link(self): class Foo(dataobject.DataObject): @@ -519,7 +519,7 @@ class Bar(dataobject.DataObject): thing = fields.Field() # The string class name should be converted to the class - self.assertEquals(Foo.__dict__["link"].cls, Bar) + self.assertEqual(Foo.__dict__["link"].cls, Bar) def test_field_datetime(self): @@ -530,24 +530,24 @@ class Timely(dataobject.DataObject): 'when': '2008-12-31T04:00:01Z', }) - self.assert_(isinstance(t, Timely), 'Datetime class decoded properly') - self.assert_(isinstance(t.when, datetime), 'Datetime data decoded into a datetime') + self.assertIsInstance(t, Timely, 'Datetime class decoded properly') + self.assertIsInstance(t.when, datetime, 'Datetime data decoded into a datetime') when = datetime(year=2008, month=12, day=31, hour=4, minute=0, second=1, tzinfo=fields.Datetime.utc) - self.assertEquals(t.when, when, 'Datetime data decoded into the expected datetime') - self.assert_(t.when.tzinfo is fields.Datetime.utc, + self.assertEqual(t.when, when, 'Datetime data decoded into the expected datetime') + self.assertTrue(t.when.tzinfo is fields.Datetime.utc, 'Datetime data decoded with utc timezone info') when = datetime(year=2010, month=2, day=11, hour=4, minute=37, second=44) t_data = Timely(when=when).to_dict() - self.assert_(isinstance(t_data, dict), 'Datetime dict encoded properly') - self.assertEquals(t_data['when'], '2010-02-11T04:37:44Z', 'Datetime dict encoded with expected timestamp') + self.assertIsInstance(t_data, dict, 'Datetime dict encoded properly') + self.assertEqual(t_data['when'], '2010-02-11T04:37:44Z', 'Datetime dict encoded with expected timestamp') when = datetime(year=2010, month=2, day=11, hour=4, minute=37, second=44, tzinfo=fields.Datetime.utc) t_data = Timely(when=when).to_dict() - self.assert_(isinstance(t_data, dict), 'Datetime dict with UTC tzinfo encoded properly') - self.assertEquals(t_data['when'], '2010-02-11T04:37:44Z', 'Datetime dict encoded with expected timestamp') + self.assertIsInstance(t_data, dict, 'Datetime dict with UTC tzinfo encoded properly') + self.assertEqual(t_data['when'], '2010-02-11T04:37:44Z', 'Datetime dict encoded with expected timestamp') class EST(tzinfo): @@ -563,18 +563,18 @@ def dst(self, dt): when = datetime(year=2010, month=2, day=10, hour=23, minute=37, second=44, tzinfo=EST()) t_data = Timely(when=when).to_dict() - self.assert_(isinstance(t_data, dict), 'Datetime dict with non-UTC tzinfo encoded properly') - self.assertEquals(t_data['when'], '2010-02-11T04:37:44Z', 'Datetime dict encoded with expected timestamp') + self.assertIsInstance(t_data, dict, 'Datetime dict with non-UTC tzinfo encoded properly') + self.assertEqual(t_data['when'], '2010-02-11T04:37:44Z', 'Datetime dict encoded with expected timestamp') t = Timely.from_dict({ 'when': None, }) - self.assert_(isinstance(t, Timely), 'Datetime with None data decoded properly') - self.assert_(t.when is None, 'Datetime with None data decoded to None timestamp') + self.assertIsInstance(t, Timely, 'Datetime with None data decoded properly') + self.assertTrue(t.when is None, 'Datetime with None data decoded to None timestamp') t = Timely.from_dict({}) - self.assert_(isinstance(t, Timely), 'Datetime with missing data decoded properly') - self.assert_(t.when is None, 'Datetime with missing data decoded to None timestamp') + self.assertIsInstance(t, Timely, 'Datetime with missing data decoded properly') + self.assertTrue(t.when is None, 'Datetime with missing data decoded to None timestamp') if __name__ == '__main__': diff --git a/tests/test_http.py b/tests/test_http.py index 517a883..3eafe3e 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -59,8 +59,8 @@ class BasicMost(self.cls): h = utils.mock_http(request, content) b = BasicMost.get('http://example.com/ohhai', http=h, headers={"x-test": "boo"}) - self.assertEquals(b.name, 'Fred') - self.assertEquals(b.value, 7) + self.assertEqual(b.name, 'Fred') + self.assertEqual(b.value, 7) h.request.assert_called_once_with(**request) def test_get_bad_encoding(self): @@ -77,9 +77,9 @@ class BasicMost(self.cls): h = utils.mock_http(request, content) b = BasicMost.get('http://example.com/ohhai', http=h) - self.assertEquals(b.name, u"Fred\ufffd") + self.assertEqual(b.name, u"Fred\ufffd") # Bad characters are replaced with the unicode Replacement Character 0xFFFD. - self.assertEquals(b.value, u"image by \ufffdrew Example") + self.assertEqual(b.value, u"image by \ufffdrew Example") h.request.assert_called_once_with(**request) def test_post(self): @@ -98,7 +98,7 @@ class ContainerMost(self.cls): content = """{"name": "CBS"}""" h = utils.mock_http(request, content) c = ContainerMost.get('http://example.com/asfdasf', http=h) - self.assertEquals(c.name, 'CBS') + self.assertEqual(c.name, 'CBS') h.request.assert_called_once_with(**request) b = BasicMost(name='Fred Friendly', value=True) @@ -116,8 +116,8 @@ class ContainerMost(self.cls): c.post(b, http=h) h.request.assert_called_once_with(**request) - self.assertEquals(b._location, 'http://example.com/fred') - self.assertEquals(b._etag, 'xyz') + self.assertEqual(b._location, 'http://example.com/fred') + self.assertEqual(b._etag, 'xyz') def test_put(self): @@ -135,7 +135,7 @@ class BasicMost(self.cls): content = """{"name": "Molly", "value": 80}""" h = utils.mock_http(request, content) b = BasicMost.get('http://example.com/bwuh', http=h) - self.assertEquals(b.name, 'Molly') + self.assertEqual(b.name, 'Molly') h.request.assert_called_once_with(**request) headers = { @@ -149,7 +149,7 @@ class BasicMost(self.cls): b.put(http=h) h.request.assert_called_once_with(**request) - self.assertEquals(b._etag, 'xyz') + self.assertEqual(b._etag, 'xyz') def test_put_no_content(self): """ @@ -168,7 +168,7 @@ class BasicMost(self.cls): content = """{"name": "Molly", "value": 80}""" h = utils.mock_http(request, content) b = BasicMost.get('http://example.com/bwuh', http=h) - self.assertEquals(b.name, 'Molly') + self.assertEqual(b.name, 'Molly') h.request.assert_called_once_with(**request) headers = { @@ -182,7 +182,7 @@ class BasicMost(self.cls): b.put(http=h) h.request.assert_called_once_with(**request) - self.assertEquals(b.name, 'Molly') + self.assertEqual(b.name, 'Molly') def test_put_failure(self): @@ -197,7 +197,7 @@ class BasicMost(self.cls): content = """{"name": "Molly", "value": 80}""" h = utils.mock_http(request, content) b = BasicMost.get('http://example.com/bwuh', http=h) - self.assertEquals(b.value, 80) + self.assertEqual(b.value, 80) h.request.assert_called_once_with(**request) b.value = 'superluminal' @@ -232,7 +232,7 @@ class BasicMost(self.cls): content = """{"name": "Molly", "value": 80}""" h = utils.mock_http(request, content) b = BasicMost.get('http://example.com/bwuh', http=h) - self.assertEquals(b.value, 80) + self.assertEqual(b.value, 80) h.request.assert_called_once_with(**request) headers = { @@ -245,8 +245,8 @@ class BasicMost(self.cls): b.delete(http=h) h.request.assert_called_once_with(**request) - self.failIf(b._location is not None) - self.failIf(hasattr(b, '_etag')) + self.assertFalse(b._location is not None) + self.assertFalse(hasattr(b, '_etag')) def test_delete_failure(self): @@ -270,12 +270,12 @@ class BasicMost(self.cls): h.request.assert_called_once_with(**request) def test_not_found(self): - self.assert_(self.cls.NotFound) + self.assertTrue(self.cls.NotFound) class Huh(self.cls): name = fields.Field() - self.assert_(Huh.NotFound) + self.assertTrue(Huh.NotFound) request = { 'uri': 'http://example.com/bwuh', diff --git a/tests/test_listobject.py b/tests/test_listobject.py index 6a83380..bdbeab0 100644 --- a/tests/test_listobject.py +++ b/tests/test_listobject.py @@ -48,23 +48,23 @@ class Toybox(self.cls): h = mock.NonCallableMock(spec_set=httplib2.Http) b = Toybox.get('http://example.com/foo', http=h) - self.assertEquals(b._location, 'http://example.com/foo') + self.assertEqual(b._location, 'http://example.com/foo') j = b[0:10] - self.assert_(isinstance(j, Toybox)) - self.assertEquals(j._location, 'http://example.com/foo?limit=10&offset=0') + self.assertIsInstance(j, Toybox) + self.assertEqual(j._location, 'http://example.com/foo?limit=10&offset=0') j = b[300:370] - self.assert_(isinstance(j, Toybox)) - self.assertEquals(j._location, 'http://example.com/foo?limit=70&offset=300') + self.assertIsInstance(j, Toybox) + self.assertEqual(j._location, 'http://example.com/foo?limit=70&offset=300') j = b[1:] - self.assert_(isinstance(j, Toybox)) - self.assertEquals(j._location, 'http://example.com/foo?offset=1') + self.assertIsInstance(j, Toybox) + self.assertEqual(j._location, 'http://example.com/foo?offset=1') j = b[:10] - self.assert_(isinstance(j, Toybox)) - self.assertEquals(j._location, 'http://example.com/foo?limit=10') + self.assertIsInstance(j, Toybox) + self.assertEqual(j._location, 'http://example.com/foo?limit=10') # Nobody did any HTTP, right? self.assertEqual([], h.method_calls) diff --git a/tests/test_promise.py b/tests/test_promise.py index 46c09f7..818f367 100644 --- a/tests/test_promise.py +++ b/tests/test_promise.py @@ -70,7 +70,7 @@ class Tiny(self.cls): h = utils.mock_http(request, content) t._http = h - self.assertEquals(t.name, 'Mollifred') + self.assertEqual(t.name, 'Mollifred') h.request.assert_called_once_with(**request) def test_filter(self): @@ -81,17 +81,17 @@ class Toy(self.cls): h = mock.NonCallableMock(spec_set=httplib2.Http) b = Toy.get('http://example.com/foo', http=h) - self.assertEquals(b._location, 'http://example.com/foo') + self.assertEqual(b._location, 'http://example.com/foo') x = b.filter(limit=10, offset=7) - self.assert_(x is not b) - self.assertEquals(b._location, 'http://example.com/foo') - self.assertEquals(x._location, 'http://example.com/foo?limit=10&offset=7') + self.assertTrue(x is not b) + self.assertEqual(b._location, 'http://example.com/foo') + self.assertEqual(x._location, 'http://example.com/foo?limit=10&offset=7') y = b.filter(awesome='yes') - self.assertEquals(y._location, 'http://example.com/foo?awesome=yes') + self.assertEqual(y._location, 'http://example.com/foo?awesome=yes') y = y.filter(awesome='no') - self.assertEquals(y._location, 'http://example.com/foo?awesome=no') + self.assertEqual(y._location, 'http://example.com/foo?awesome=no') # Nobody did any HTTP, right? self.assertEqual([], h.method_calls) @@ -104,12 +104,12 @@ class Toy(self.cls): h = mock.NonCallableMock(spec_set=httplib2.Http) b = Toy.get('http://example.com/foo', http=h) - self.assertEquals(b._location, 'http://example.com/foo') + self.assertEqual(b._location, 'http://example.com/foo') y = b.filter(a='a', b=u'b') - self.assertEquals(y._location, 'http://example.com/foo?a=a&b=b') + self.assertEqual(y._location, 'http://example.com/foo?a=a&b=b') y = b.filter(**{'a': 'a', u'b': u'b'}) - self.assertEquals(y._location, 'http://example.com/foo?a=a&b=b') + self.assertEqual(y._location, 'http://example.com/foo?a=a&b=b') def test_awesome(self): @@ -122,8 +122,8 @@ class Room(self.cls): r = Room.get('http://example.com/bwuh/') b = r.toybox - self.assert_(isinstance(b, Toy)) - self.assertEquals(b._location, 'http://example.com/bwuh/toybox') + self.assertIsInstance(b, Toy) + self.assertEqual(b._location, 'http://example.com/bwuh/toybox') def test_set_before_delivery(self): @@ -142,9 +142,9 @@ class Toy(self.cls): t.names = ["New name"] d = t.to_dict() # this delivers the object - # self.assertEquals(t.foo, "something") - self.assertEquals(d['names'][0], "New name") - self.assertEquals(t.names[0], "New name") + # self.assertEqual(t.foo, "something") + self.assertEqual(d['names'][0], "New name") + self.assertEqual(t.names[0], "New name") h = utils.mock_http(request, content) # test case where we update_from_dict explictly after setting attributes @@ -152,4 +152,4 @@ class Toy(self.cls): t.foo = "local change" t.update_from_dict({"names": ["local update"]}) - self.assertEquals(t.foo, None) + self.assertEqual(t.foo, None) From f9bee5c63028066c338adf01775c849093c10c01 Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Mon, 27 Sep 2021 19:06:42 -0700 Subject: [PATCH 21/27] Call urlparse.parse_qs instead of cgi.parse_qs cgi.parse_qs has been deprecated since 2.6 (2008) (https://docs.python.org/2.6/whatsnew/2.6.html, https://bugs.python.org/issue600362) and was removed in python 3.0. 2to3 and derived fixers (modernize, futurize) do not fix cgi.parse_qs; you need to move it to urlparse first. --- examples/giantbomb.py | 3 +-- examples/netflix.py | 4 ++-- remoteobjects/promise.py | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/examples/giantbomb.py b/examples/giantbomb.py index 35c7f3f..f2b8993 100644 --- a/examples/giantbomb.py +++ b/examples/giantbomb.py @@ -40,11 +40,10 @@ __author__ = 'Mark Paschal' -from cgi import parse_qs from optparse import OptionParser import sys from urllib import urlencode -from urlparse import urljoin, urlparse, urlunparse +from urlparse import parse_qs, urljoin, urlparse, urlunparse from remoteobjects import RemoteObject, fields diff --git a/examples/netflix.py b/examples/netflix.py index b5c790a..c4dbd42 100644 --- a/examples/netflix.py +++ b/examples/netflix.py @@ -40,10 +40,10 @@ __author__ = 'Mark Paschal' -import cgi from optparse import OptionParser import sys import urlparse +from urlparse import parse_qs from xml.etree import ElementTree from oauth.oauth import OAuthConsumer, OAuthRequest, OAuthSignatureMethod_HMAC_SHA1 @@ -67,7 +67,7 @@ def get_request(self, headers=None, **kwargs): # OAuthRequest will strip our query parameters, so add them back in. parts = list(urlparse.urlparse(self._location)) - queryargs = cgi.parse_qs(parts[4], keep_blank_values=True) + queryargs = parse_qs(parts[4], keep_blank_values=True) for key, value in queryargs.iteritems(): orq.set_parameter(key, value[0]) diff --git a/remoteobjects/promise.py b/remoteobjects/promise.py index a613a53..2e1cf24 100644 --- a/remoteobjects/promise.py +++ b/remoteobjects/promise.py @@ -28,8 +28,8 @@ # POSSIBILITY OF SUCH DAMAGE. import urlparse +from urlparse import parse_qs import urllib -import cgi import httplib2 @@ -246,7 +246,7 @@ def filter(self, **kwargs): """ parts = list(urlparse.urlparse(self._location)) - queryargs = cgi.parse_qs(parts[4], keep_blank_values=True) + queryargs = parse_qs(parts[4], keep_blank_values=True) queryargs = dict([(k, v[0]) for k, v in queryargs.iteritems()]) queryargs.update(kwargs) parts[4] = urllib.urlencode(queryargs) From d53fea13ceb5c13743aac0c5975b1255f13729f4 Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Mon, 9 Oct 2023 15:39:02 -0700 Subject: [PATCH 22/27] Add tests of PageOf and ListOf --- tests/test_listobject.py | 54 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/tests/test_listobject.py b/tests/test_listobject.py index bdbeab0..3c9bc7c 100644 --- a/tests/test_listobject.py +++ b/tests/test_listobject.py @@ -32,7 +32,7 @@ import httplib2 import mock -from remoteobjects import listobject +from remoteobjects import fields, listobject, promise from tests import utils @@ -84,3 +84,55 @@ class Toybox(self.cls): self.assertEqual(b[7], 7) h.request.assert_called_once_with(**request) + + +class TestPageOf(unittest.TestCase): + + def test_basemodule(self): + # When creating PageOf(myclass), it should use PageObject as superclass + self.assertEqual(listobject.PageOf._basemodule, listobject.PageObject) + + def test_to_dict(self): + class MyObj(promise.PromiseObject): + myfield = fields.Field() + MyObjPage = listobject.PageOf(MyObj) + obj = MyObjPage(entries=[MyObj(myfield="myval")]) + # When creating PageOf(myclass), it should use PageObject as superclass + self.assertIsInstance(obj, listobject.PageObject) + self.assertEqual({"entries": [{"myfield": "myval"}]}, obj.to_dict()) + + def test_from_dict(self): + class MyObj(promise.PromiseObject): + myfield = fields.Field() + MyObjPage = listobject.PageOf(MyObj) + actual = MyObjPage.from_dict({"entries": [{"myfield": "myval"}]}) + self.assertIsInstance(actual, listobject.PageObject) + self.assertEqual("myval", actual.entries[0].myfield) + expected = MyObjPage(entries=[MyObj(myfield="myval")]) + self.assertEqual(actual, expected) + + +class TestListOf(unittest.TestCase): + + def test_basemodule(self): + # When creating ListOf(myclass), it should use ListObject as superclass + self.assertEqual(listobject.ListOf._basemodule, listobject.ListObject) + + def test_to_dict(self): + class MyObj(promise.PromiseObject): + myfield = fields.Field() + MyObjList = listobject.ListOf(MyObj) + obj = MyObjList(entries=[MyObj(myfield="myval")]) + # When creating ListOf(myclass), it should use ListObject as superclass + self.assertIsInstance(obj, listobject.ListObject) + self.assertEqual([{"myfield": "myval"}], obj.to_dict()) + + def test_from_dict(self): + class MyObj(promise.PromiseObject): + myfield = fields.Field() + MyObjList = listobject.ListOf(MyObj) + actual = MyObjList.from_dict([{"myfield": "myval"}]) + expected = MyObjList(entries=[MyObj(myfield="myval")]) + self.assertEqual("myval", actual.entries[0].myfield) + self.assertIsInstance(actual, listobject.ListObject) + self.assertEqual(actual, expected) From 3245056d7814de994f220ef4c003a87437ce7305 Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Sat, 25 Sep 2021 20:44:25 -0700 Subject: [PATCH 23/27] Add support for python3 Add support for python 3 while retaining python 2.7 compatibility. Most of the changes were automatically done by modernize and futurize. I manually undid some of the unnecessary list(d.items()) from 2to3 --fix=dict. And I had to add b'' binary literals in some places. The more complex changes to make it compatible with python 3 are in the following commits. Add matrix to test on python 3. Add recursive dependencies to requirements.txt and requirements-test.txt. --- .github/workflows/tests.yml | 11 ++++- doc/readme_from_docstring.py | 2 +- examples/couchdb.py | 21 ++++----- examples/giantbomb.py | 24 +++++----- examples/netflix.py | 24 +++++----- examples/twitter.py | 48 +++++++++++--------- remoteobjects/dataobject.py | 17 ++++--- remoteobjects/fields.py | 8 ++-- remoteobjects/http.py | 59 +++++++++++++------------ remoteobjects/json.py | 5 ++- remoteobjects/listobject.py | 13 ++---- remoteobjects/promise.py | 15 +++---- requirements-test.txt | 4 +- requirements.txt | 14 ++++++ setup.py | 10 +++-- tests/performance/benchmark_decoding.py | 10 +++-- tests/test_http.py | 2 +- 17 files changed, 158 insertions(+), 129 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ddd7680..9e6057b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,6 +12,8 @@ jobs: strategy: fail-fast: false matrix: + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.8", "pypy-3.9"] + os: [ubuntu-20.04, ubuntu-22.04, macOS-latest, windows-latest] include: - python-version: "2.7" os: "ubuntu-20.04" @@ -19,7 +21,14 @@ jobs: os: "ubuntu-22.04" steps: - uses: actions/checkout@v4.1.0 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python ${{ matrix.python-version }} (github action) + if: matrix.python-version != '2.7' + uses: actions/setup-python@v4.7.0 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Set up Python ${{ matrix.python-version }} (apt-get) + if: matrix.python-version == '2.7' # since actions/python-versions removed 2.7 from versions-manifest.json # and also deleted all the python 2.7 binary artifacts, # we have to apt-get install python2 diff --git a/doc/readme_from_docstring.py b/doc/readme_from_docstring.py index 6b57524..3d4086e 100755 --- a/doc/readme_from_docstring.py +++ b/doc/readme_from_docstring.py @@ -31,7 +31,7 @@ import remoteobjects -readme = file('README.rst', 'w') +readme = open('README.rst', 'w') readme.write(remoteobjects.__doc__.strip()) readme.write("\n") readme.close() diff --git a/examples/couchdb.py b/examples/couchdb.py index 01233fc..6e2c0ba 100755 --- a/examples/couchdb.py +++ b/examples/couchdb.py @@ -34,17 +34,18 @@ A generic example CouchDB client, implemented using remoteobjects. """ +from __future__ import print_function __version__ = '1.0' __date__ = '24 August 2009' __author__ = 'Mark Paschal' -import httplib from optparse import OptionParser import simplejson as json import sys -from urlparse import urljoin +from six.moves import http_client +from six.moves.urllib.parse import urljoin from remoteobjects import RemoteObject, fields, ListObject @@ -53,10 +54,10 @@ class CouchObject(RemoteObject): # CouchDB omits Locations for Created responses, so don't expect them. location_headers = dict(RemoteObject.location_headers) - location_headers[httplib.CREATED] = None + location_headers[http_client.CREATED] = None def update_from_response(self, url, response, content): - if response.status == httplib.CREATED: + if response.status == http_client.CREATED: # CouchDB CREATED responses don't contain full content, so only # unpack the ID and revision. data = json.loads(content) @@ -103,7 +104,7 @@ class ViewResult(CouchObject, ListObject): entries = fields.List(fields.Object(ListItem), api_name='rows') def filter(self, **kwargs): - for k, v in kwargs.iteritems(): + for k, v in kwargs.items(): if isinstance(v, list) or isinstance(v, dict) or isinstance(v, bool): kwargs[k] = json.dumps(v) return super(ViewResult, self).filter(**kwargs) @@ -127,16 +128,16 @@ def create_db(url): db.deliver() except Database.NotFound: db.put() - print "Database %s created" % url + print("Database %s created" % url) else: - print "Database %s already exists" % url + print("Database %s already exists" % url) def create_view(dburl): viewset = Viewset.get(urljoin(dburl, '_design/profiles')) try: viewset.deliver() - print "Retrieved existing 'profiles' views" + print("Retrieved existing 'profiles' views") except Viewset.NotFound: # Start with an empty set of views. viewset.views = {} @@ -150,7 +151,7 @@ def create_view(dburl): viewset.views['url'] = View(mapfn=profiles_url_code) viewset.put() - print "Updated 'profiles' views for %s" % dburl + print("Updated 'profiles' views for %s" % dburl) def main(argv=None): @@ -164,7 +165,7 @@ def main(argv=None): db = opts.database if db is None: - print >>sys.stderr, "Option --database is required" + print("Option --database is required", file=sys.stderr) return 1 # Create the database, if necessary. diff --git a/examples/giantbomb.py b/examples/giantbomb.py index f2b8993..ca82423 100644 --- a/examples/giantbomb.py +++ b/examples/giantbomb.py @@ -34,6 +34,7 @@ An example Giant Bomb API client, implemented using remoteobjects. """ +from __future__ import print_function __version__ = '1.0' __date__ = '24 August 2009' @@ -42,8 +43,9 @@ from optparse import OptionParser import sys -from urllib import urlencode -from urlparse import parse_qs, urljoin, urlparse, urlunparse +from six.moves.urllib.parse import ( + urlencode, parse_qs, urljoin, urlparse, urlunparse +) from remoteobjects import RemoteObject, fields @@ -66,9 +68,9 @@ def filter(self, **kwargs): url = self._location parts = list(urlparse(url)) query = parse_qs(parts[4]) - query = dict([(k, v[0]) for k, v in query.iteritems()]) + query = dict([(k, v[0]) for k, v in query.items()]) - for k, v in kwargs.iteritems(): + for k, v in kwargs.items(): if v is None and k in query: del query[k] else: @@ -141,7 +143,7 @@ def main(argv=None): opts, args = parser.parse_args() if opts.key is None: - print >>sys.stderr, "Option --key is required" + print("Option --key is required", file=sys.stderr) return 1 query = ' '.join(args) @@ -152,16 +154,16 @@ def main(argv=None): search = search.filter(query=query) if len(search.results) == 0: - print "No results for %r" % query + print("No results for %r" % query) elif len(search.results) == 1: (game,) = search.results - print "## %s ##" % game.name - print - print game.summary + print("## %s ##" % game.name) + print() + print(game.summary) else: - print "## Search results for %r ##" % query + print("## Search results for %r ##" % query) for game in search.results: - print game.name + print(game.name) return 0 diff --git a/examples/netflix.py b/examples/netflix.py index c4dbd42..b713cb8 100644 --- a/examples/netflix.py +++ b/examples/netflix.py @@ -34,6 +34,7 @@ An example Netflix API client, implemented using remoteobjects. """ +from __future__ import print_function __version__ = '1.0' __date__ = '25 August 2009' @@ -42,8 +43,7 @@ from optparse import OptionParser import sys -import urlparse -from urlparse import parse_qs +from six.moves.urllib.parse import parse_qs, urlparse from xml.etree import ElementTree from oauth.oauth import OAuthConsumer, OAuthRequest, OAuthSignatureMethod_HMAC_SHA1 @@ -66,9 +66,9 @@ def get_request(self, headers=None, **kwargs): http_url=request['uri']) # OAuthRequest will strip our query parameters, so add them back in. - parts = list(urlparse.urlparse(self._location)) + parts = list(urlparse(self._location)) queryargs = parse_qs(parts[4], keep_blank_values=True) - for key, value in queryargs.iteritems(): + for key, value in queryargs.items(): orq.set_parameter(key, value[0]) # Sign the request. @@ -84,7 +84,7 @@ def get_request(self, headers=None, **kwargs): return request def update_from_tree(self, tree): - data = dict((k, v(tree)) for k, v in self.decoder_ring.items()) + data = dict((k, v(tree)) for k, v in list(self.decoder_ring.items())) self.update_from_dict(data) return self @@ -133,18 +133,18 @@ def do_search(opts, args): search.deliver() if len(search.results) == 0: - print "No results for %r" % query + print("No results for %r" % query) elif len(search.results) == 1: result = search.results[0] - print "## %s ##" % result.title + print("## %s ##" % result.title) else: - print "## Results for %r ##" % query - print + print("## Results for %r ##" % query) + print() for title in search.results: if title is None: - print "(oops, none)" + print("(oops, none)") else: - print title.title + print(title.title) return 0 @@ -164,7 +164,7 @@ def main(argv=None): opts, args = parser.parse_args() if opts.key is None or opts.secret is None: - print >>sys.stderr, "Options --key and --secret are required" + print("Options --key and --secret are required", file=sys.stderr) return 1 Flixject.api_token = (opts.key, opts.secret) diff --git a/examples/twitter.py b/examples/twitter.py index 6165f12..e7d829d 100644 --- a/examples/twitter.py +++ b/examples/twitter.py @@ -34,17 +34,21 @@ A Twitter API client, implemented using remoteobjects. """ +from __future__ import print_function __version__ = '1.1' __date__ = '17 April 2009' __author__ = 'Brad Choate' -import httplib from optparse import OptionParser import sys -from urllib import urlencode, quote_plus -from urlparse import urljoin, urlunsplit +from six import text_type +from six.moves import input +from six.moves import http_client # python3: http.client +from six.moves.urllib.parse import ( + urlencode, quote_plus, urljoin, urlunsplit +) from httplib2 import Http @@ -77,7 +81,7 @@ def get_user(cls, http=None, **kwargs): url += '/%s.json' % quote_plus(kwargs['id']) else: url += '.json' - query = urlencode([(key, value) for key, value in kwargs.iteritems() if key in ('screen_name', 'user_id')]) + query = urlencode([(key, value) for key, value in kwargs.items() if key in ('screen_name', 'user_id')]) url = urlunsplit((None, None, url, query, None)) return cls.get(urljoin(Twitter.endpoint, url), http=http) @@ -143,14 +147,14 @@ def __getitem__(self, key): @classmethod def get_messages(cls, http=None, **kwargs): url = '/direct_messages.json' - query = urlencode([(key, value) for key, value in kwargs.iteritems() if key in ('since_id', 'page')]) + query = urlencode([(key, value) for key, value in kwargs.items() if key in ('since_id', 'page')]) url = urlunsplit((None, None, url, query, None)) return cls.get(urljoin(Twitter.endpoint, url), http=http) @classmethod def get_sent_messages(cls, http=None, **kwargs): url = '/direct_messages/sent.json' - query = urlencode([(key, value) for key, value in kwargs.iteritems() if key in ('since_id', 'page')]) + query = urlencode([(key, value) for key, value in kwargs.items() if key in ('since_id', 'page')]) url = urlunsplit((None, None, url, query, None)) return cls.get(urljoin(Twitter.endpoint, url), http=http) @@ -177,7 +181,7 @@ def get_related(cls, relation, http=None, **kwargs): url += '/%s.json' % quote_plus(kwargs['id']) else: url += '.json' - query = urlencode([(key, value) for key, value in kwargs.iteritems() if key in ('screen_name', 'user_id', 'page')]) + query = urlencode([(key, value) for key, value in kwargs.items() if key in ('screen_name', 'user_id', 'page')]) url = urlunsplit((None, None, url, query, None)) return cls.get(urljoin(Twitter.endpoint, url), http=http) @@ -195,7 +199,7 @@ def public(cls, http=None): @classmethod def friends(cls, http=None, **kwargs): - query = urlencode([(key, value) for key, value in kwargs.iteritems() if key in ('since_id', 'max_id', 'count', 'page')]) + query = urlencode([(key, value) for key, value in kwargs.items() if key in ('since_id', 'max_id', 'count', 'page')]) url = urlunsplit((None, None, '/statuses/friends_timeline.json', query, None)) return cls.get(urljoin(Twitter.endpoint, url), http=http) @@ -206,13 +210,13 @@ def user(cls, http=None, **kwargs): url += '/%s.json' % quote_plus(kwargs['id']) else: url += '.json' - query = urlencode([(key, value) for key, value in kwargs.iteritems() if key in ('screen_name', 'user_id', 'since_id', 'max_id', 'page')]) + query = urlencode([(key, value) for key, value in kwargs.items() if key in ('screen_name', 'user_id', 'since_id', 'max_id', 'page')]) url = urlunsplit((None, None, url, query, None)) return cls.get(urljoin(Twitter.endpoint, url), http=http) @classmethod def mentions(cls, http=None, **kwargs): - query = urlencode([(key, value) for key, value in kwargs.iteritems() if key in ('since_id', 'max_id', 'page')]) + query = urlencode([(key, value) for key, value in kwargs.items() if key in ('since_id', 'max_id', 'page')]) url = urlunsplit((None, None, '/statuses/mentions.json', query, None)) return cls.get(urljoin(Twitter.endpoint, url), http=http) @@ -258,21 +262,21 @@ def direct_messages_sent(self, **kwargs): def show_public(twitter): - print "## Public timeline ##" + print("## Public timeline ##") for tweet in twitter.public_timeline(): - print unicode(tweet) + print(text_type(tweet)) def show_dms(twitter): - print "## Direct messages sent to me ##" + print("## Direct messages sent to me ##") for dm in twitter.direct_messages_received(): - print unicode(dm) + print(text_type(dm)) def show_friends(twitter): - print "## Tweets from my friends ##" + print("## Tweets from my friends ##") for tweet in twitter.friends_timeline(): - print unicode(tweet) + print(text_type(tweet)) def main(argv=None): @@ -296,18 +300,18 @@ def main(argv=None): # We'll use regular HTTP authentication, so ask for a password and add # it in the regular httplib2 way. if opts.username is not None: - password = raw_input("Password (will echo): ") + password = input("Password (will echo): ") twitter.add_credentials(opts.username, password) try: - print + print() opts.action(twitter) - print - except httplib.HTTPException, exc: + print() + except http_client.HTTPException as exc: # The API could be down, or the credentials on an auth-only request # could be wrong, so show the error to the end user. - print >>sys.stderr, "Error making request: %s: %s" \ - % (type(exc).__name__, str(exc)) + print("Error making request: %s: %s" + % (type(exc).__name__, text_type(exc)), file=sys.stderr) return 1 return 0 diff --git a/remoteobjects/dataobject.py b/remoteobjects/dataobject.py index a34ab2b..1d7fb65 100644 --- a/remoteobjects/dataobject.py +++ b/remoteobjects/dataobject.py @@ -41,6 +41,7 @@ from copy import deepcopy +from six import with_metaclass import remoteobjects.fields @@ -118,7 +119,7 @@ def add_to_class(cls, name, value): setattr(cls, name, value) -class DataObject(object): +class DataObject(with_metaclass(DataObjectMetaclass)): """An object that can be decoded from or encoded as a dictionary. @@ -138,8 +139,6 @@ class DataObject(object): """ - __metaclass__ = DataObjectMetaclass - def __init__(self, **kwargs): """Initializes a new `DataObject` with the given field values.""" self.api_data = {} @@ -154,7 +153,7 @@ def __eq__(self, other): """ if not isinstance(other, type(self)): return NotImplemented - for k, v in self.fields.iteritems(): + for k, v in self.fields.items(): if isinstance(v, remoteobjects.fields.Field): if getattr(self, k) != getattr(other, k): return False @@ -171,7 +170,7 @@ def __ne__(self, other): @classmethod def statefields(cls): - return cls.fields.keys() + ['api_data'] + return list(cls.fields.keys()) + ['api_data'] def __getstate__(self): return dict((k, self.__dict__[k]) for k in self.statefields() @@ -181,7 +180,7 @@ def get(self, attr, *args): return getattr(self, attr, *args) def __iter__(self): - for key in self.fields.keys(): + for key in list(self.fields.keys()): yield key def to_dict(self): @@ -190,7 +189,7 @@ def to_dict(self): data = deepcopy(self.api_data) # Now replace the data with what's actually in our object - for field_name, field in self.fields.iteritems(): + for field_name, field in self.fields.items(): value = getattr(self, field.attrname, None) if value is not None: data[field.api_name] = field.encode(value) @@ -199,7 +198,7 @@ def to_dict(self): # Now delete any fields that ended up being None # since we should exclude them in the resulting dict. - for k in data.keys(): + for k in list(data.keys()): if data[k] is None: del data[k] @@ -228,7 +227,7 @@ def update_from_dict(self, data): if not isinstance(data, dict): raise TypeError # Clear any local instance field data - for k in self.fields.iterkeys(): + for k in self.fields.keys(): if k in self.__dict__: del self.__dict__[k] self.api_data = data diff --git a/remoteobjects/fields.py b/remoteobjects/fields.py index db97395..519ae6f 100644 --- a/remoteobjects/fields.py +++ b/remoteobjects/fields.py @@ -40,7 +40,7 @@ from datetime import datetime, tzinfo, timedelta import dateutil.parser -import urlparse +from six.moves.urllib.parse import urljoin import remoteobjects.dataobject @@ -307,13 +307,13 @@ def decode(self, value): if callable(self.default): return self.default() return self.default or None - return dict((k, self.fld.decode(v)) for k, v in value.iteritems()) + return dict((k, self.fld.decode(v)) for k, v in value.items()) def encode(self, value): """Encodes a `DataObject` attribute (a dictionary with decoded `DataObject` attribute values for values) into a dictionary value (a dictionary with encoded dictionary values for values).""" - return dict((k, self.fld.encode(v)) for k, v in value.iteritems()) + return dict((k, self.fld.encode(v)) for k, v in value.items()) class AcceptsStringCls(object): @@ -486,5 +486,5 @@ def __get__(self, instance, owner): """ if instance._location is None: raise AttributeError('Cannot find URL of %s relative to URL-less %s' % (self.cls.__name__, owner.__name__)) - newurl = urlparse.urljoin(instance._location, self.api_name) + newurl = urljoin(instance._location, self.api_name) return self.cls.get(newurl) diff --git a/remoteobjects/http.py b/remoteobjects/http.py index f0e7d93..a57f690 100644 --- a/remoteobjects/http.py +++ b/remoteobjects/http.py @@ -31,7 +31,8 @@ from remoteobjects.json import ForgivingDecoder import httplib2 -import httplib +from six import PY3, text_type +from six.moves import http_client # python3: http.client import logging from remoteobjects.dataobject import DataObject @@ -47,7 +48,7 @@ def omit_nulls(data): if not hasattr(data, '__dict__'): return str(data) data = dict(data.__dict__) - for key in data.keys(): + for key in list(data.keys()): if data[key] is None: del data[key] return data @@ -59,36 +60,36 @@ class HttpObject(DataObject): JSON API.""" response_has_content = { - httplib.OK: True, - httplib.ACCEPTED: False, - httplib.CREATED: True, - httplib.NO_CONTENT: False, - httplib.MOVED_PERMANENTLY: True, - httplib.FOUND: True, - httplib.NOT_MODIFIED: True, + http_client.OK: True, + http_client.ACCEPTED: False, + http_client.CREATED: True, + http_client.NO_CONTENT: False, + http_client.MOVED_PERMANENTLY: True, + http_client.FOUND: True, + http_client.NOT_MODIFIED: True, } location_headers = { - httplib.OK: 'Content-Location', - httplib.CREATED: 'Location', - httplib.MOVED_PERMANENTLY: 'Location', - httplib.FOUND: 'Location', + http_client.OK: 'Content-Location', + http_client.CREATED: 'Location', + http_client.MOVED_PERMANENTLY: 'Location', + http_client.FOUND: 'Location', } location_header_required = { - httplib.CREATED: True, - httplib.MOVED_PERMANENTLY: True, - httplib.FOUND: True, + http_client.CREATED: True, + http_client.MOVED_PERMANENTLY: True, + http_client.FOUND: True, } content_types = ('application/json',) - class NotFound(httplib.HTTPException): + class NotFound(http_client.HTTPException): """An HTTPException thrown when the server reports that the requested resource was not found.""" pass - class Unauthorized(httplib.HTTPException): + class Unauthorized(http_client.HTTPException): """An HTTPException thrown when the server reports that the requested resource is not available through an unauthenticated request. @@ -99,7 +100,7 @@ class Unauthorized(httplib.HTTPException): """ pass - class Forbidden(httplib.HTTPException): + class Forbidden(http_client.HTTPException): """An HTTPException thrown when the server reports that the client, as authenticated, is not authorized to request the requested resource. @@ -110,7 +111,7 @@ class Forbidden(httplib.HTTPException): """ pass - class PreconditionFailed(httplib.HTTPException): + class PreconditionFailed(http_client.HTTPException): """An HTTPException thrown when the server reports that some of the conditions in a conditional request were not true. @@ -121,7 +122,7 @@ class PreconditionFailed(httplib.HTTPException): """ pass - class RequestError(httplib.HTTPException): + class RequestError(http_client.HTTPException): """An HTTPException thrown when the server reports an error in the client's request. @@ -130,7 +131,7 @@ class RequestError(httplib.HTTPException): """ pass - class ServerError(httplib.HTTPException): + class ServerError(http_client.HTTPException): """An HTTPException thrown when the server reports an unexpected error. This exception corresponds to the HTTP status code 500. @@ -138,7 +139,7 @@ class ServerError(httplib.HTTPException): """ pass - class BadResponse(httplib.HTTPException): + class BadResponse(http_client.HTTPException): """An HTTPException thrown when the client receives some other non-success HTTP response.""" pass @@ -186,17 +187,17 @@ def raise_for_response(cls, url, response, content): """ # Turn exceptional httplib2 responses into exceptions. classname = cls.__name__ - if response.status == httplib.NOT_FOUND: + if response.status == http_client.NOT_FOUND: raise cls.NotFound('No such %s %s' % (classname, url)) - if response.status == httplib.UNAUTHORIZED: + if response.status == http_client.UNAUTHORIZED: raise cls.Unauthorized('Not authorized to fetch %s %s' % (classname, url)) - if response.status == httplib.FORBIDDEN: + if response.status == http_client.FORBIDDEN: raise cls.Forbidden('Forbidden from fetching %s %s' % (classname, url)) - if response.status == httplib.PRECONDITION_FAILED: + if response.status == http_client.PRECONDITION_FAILED: raise cls.PreconditionFailed('Precondition failed for %s request to %s' % (classname, url)) - if response.status in (httplib.INTERNAL_SERVER_ERROR, httplib.BAD_REQUEST): - if response.status == httplib.BAD_REQUEST: + if response.status in (http_client.INTERNAL_SERVER_ERROR, http_client.BAD_REQUEST): + if response.status == http_client.BAD_REQUEST: err_cls = cls.RequestError else: err_cls = cls.ServerError diff --git a/remoteobjects/json.py b/remoteobjects/json.py index ee582a6..f1d2568 100644 --- a/remoteobjects/json.py +++ b/remoteobjects/json.py @@ -42,6 +42,7 @@ # https://github.com/simplejson/simplejson/commit/104b40fcf6aa39d9ba7b240c3c528d1f85e86ef2 from simplejson.decoder import errmsg from simplejson.scanner import py_make_scanner +from six import unichr, text_type import sys # Truly heinous... we are going to the trouble of reproducing this @@ -71,8 +72,8 @@ def forgiving_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=ST content, terminator = chunk.groups() # Content is contains zero or more unescaped string characters if content: - if not isinstance(content, unicode): - content = unicode(content, encoding, errors="replace") + if not isinstance(content, text_type): + content = text_type(content, encoding, errors="replace") _append(content) # Terminator is the end of string, a literal control character, # or a backslash denoting that an escape sequence follows diff --git a/remoteobjects/listobject.py b/remoteobjects/listobject.py index f9a64dd..ec8a0f0 100644 --- a/remoteobjects/listobject.py +++ b/remoteobjects/listobject.py @@ -28,6 +28,7 @@ # POSSIBILITY OF SUCH DAMAGE. import sys +from six import with_metaclass import remoteobjects.fields as fields from remoteobjects.promise import PromiseObject @@ -78,7 +79,7 @@ def __new__(cls, name, bases, attr): return type.__new__(cls, name, bases, attr) -class PageOf(PromiseObject.__metaclass__): +class PageOf(with_metaclass(OfOf, type(PromiseObject))): """Metaclass defining a `PageObject` containing a set of some other class's instances. @@ -102,8 +103,6 @@ class PageOf(PromiseObject.__metaclass__): """ - __metaclass__ = OfOf - _modulename = 'remoteobjects.listobject._pages' def __new__(cls, name, bases=None, attr=None): @@ -150,7 +149,7 @@ def __new__(cls, name, bases=None, attr=None): return newcls -class PageObject(SequenceProxy, PromiseObject): +class PageObject(with_metaclass(PageOf, SequenceProxy, PromiseObject)): """A `RemoteObject` representing a set of other `RemoteObject` instances. @@ -186,8 +185,6 @@ class PageObject(SequenceProxy, PromiseObject): """ - __metaclass__ = PageOf - entries = fields.List(fields.Field()) def __getitem__(self, key): @@ -217,9 +214,7 @@ class ListOf(PageOf): _modulename = 'remoteobjects.listobject._lists' -class ListObject(PageObject): - - __metaclass__ = ListOf +class ListObject(with_metaclass(ListOf, PageObject)): def update_from_dict(self, data): super(ListObject, self).update_from_dict({ 'entries': data }) diff --git a/remoteobjects/promise.py b/remoteobjects/promise.py index 2e1cf24..4696cbd 100644 --- a/remoteobjects/promise.py +++ b/remoteobjects/promise.py @@ -27,11 +27,8 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -import urlparse -from urlparse import parse_qs -import urllib - import httplib2 +from six.moves.urllib.parse import parse_qs, urlencode, urlparse, urlunparse import remoteobjects.http @@ -218,7 +215,7 @@ def update_from_dict(self, data): raise TypeError("Cannot update %r from non-dictionary data source %r" % (self, data)) # Clear any local instance field data - for k in self.fields.iterkeys(): + for k in self.fields.keys(): if k in self.__dict__: del self.__dict__[k] # Update directly to avoid triggering delivery. @@ -245,11 +242,11 @@ def filter(self, **kwargs): you require. """ - parts = list(urlparse.urlparse(self._location)) + parts = list(urlparse(self._location)) queryargs = parse_qs(parts[4], keep_blank_values=True) - queryargs = dict([(k, v[0]) for k, v in queryargs.iteritems()]) + queryargs = dict([(k, v[0]) for k, v in queryargs.items()]) queryargs.update(kwargs) - parts[4] = urllib.urlencode(queryargs) - newurl = urlparse.urlunparse(parts) + parts[4] = urlencode(queryargs) + newurl = urlunparse(parts) return self.get(newurl, http=self._http) diff --git a/requirements-test.txt b/requirements-test.txt index 5f230eb..2a2609d 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -3,7 +3,7 @@ contextlib2==0.6.0.post1 enum34==1.1.10 flake8==3.9.2 funcsigs==1.0.2 -functools32==3.2.3.post2 +functools32==3.2.3.post2; python_version < '3.0' importlib-metadata==2.1.3 mccabe==0.6.1 mock==3.0.5 @@ -11,5 +11,5 @@ pathlib2==2.3.7.post1 pycodestyle==2.7.0 pyflakes==2.3.1 scandir==1.10.0 -typing==3.10.0.0 +typing==3.10.0.0; python_version < '3.0' zipp==1.2.0 diff --git a/requirements.txt b/requirements.txt index 0087195..d8e5fcc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,17 @@ pyparsing==2.4.7 python-dateutil==2.8.2 simplejson==3.19.2 six==1.16.0 + +cffi==1.15.1; platform_python_implementation == 'PyPy' +greenlet==0.4.13; platform_python_implementation == 'PyPy' +hpy==0.0.4.dev179+g9b5d200; platform_python_implementation == 'PyPy' and python_version < '3.9' +readline==6.2.4.1; platform_python_implementation == 'PyPy' + +argcomplete==3.1.2; python_version == '3.9' and sys_platform == 'win32' +click==8.1.7; python_version == '3.9' and sys_platform == 'win32' +colorama==0.4.6; python_version == '3.9' and sys_platform == 'win32' +packaging==23.2; python_version == '3.9' and sys_platform == 'win32' +pipx==1.2.0; python_version == '3.9' and sys_platform == 'win32' +userpath==1.9.1; python_version == '3.9' and sys_platform == 'win32' + +certifi==2023.7.22; python_version == '3.11' and sys_platform == 'darwin' diff --git a/setup.py b/setup.py index daa4529..0bad657 100644 --- a/setup.py +++ b/setup.py @@ -61,9 +61,13 @@ packages=['remoteobjects'], provides=['remoteobjects'], - python_requires='>=2.7, <3.0', - install_requires=['simplejson>=2.0.0', 'httplib2>=0.5.0', - 'python-dateutil>=2.1'], + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*', + install_requires=[ + 'simplejson>=2.0.0', + 'httplib2>=0.5.0', + 'python-dateutil>=2.1', + 'six~=1.16.0', + ], extras_require={ 'test': [ 'flake8~=3.9', diff --git a/tests/performance/benchmark_decoding.py b/tests/performance/benchmark_decoding.py index cfe0b06..b413b9c 100755 --- a/tests/performance/benchmark_decoding.py +++ b/tests/performance/benchmark_decoding.py @@ -35,8 +35,10 @@ the second argument. The decode process is run as many times as you specify (via the -n flag). The raw times to decode the JSON data will be dumped to stdout. """ +from __future__ import print_function import optparse +from six.moves import range import time from tests import utils @@ -53,7 +55,7 @@ def test_decoding(object_class, json, count): o = object_class.get('http://example.com/ohhai', http=h) o.deliver() - for _ in xrange(count): + for _ in range(count): h = utils.mock_http(request, json) t = time.time() @@ -84,13 +86,13 @@ def test_decoding(object_class, json, count): module_name, _, class_name = args[1].rpartition('.') try: module = __import__(module_name) - except ImportError, e: + except ImportError as e: parser.error(e.message) try: RemoteObject = getattr(module, class_name) - except AttributeError, e: + except AttributeError as e: parser.error(e.message) for t in test_decoding(RemoteObject, json, options.num_runs): - print t + print(t) diff --git a/tests/test_http.py b/tests/test_http.py index 3eafe3e..cc6317d 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -73,7 +73,7 @@ class BasicMost(self.cls): 'uri': 'http://example.com/ohhai', 'headers': {'accept': 'application/json'}, } - content = """{"name": "Fred\xf1", "value": "image by \xefrew Example"}""" + content = b"""{"name": "Fred\xf1", "value": "image by \xefrew Example"}""" h = utils.mock_http(request, content) b = BasicMost.get('http://example.com/ohhai', http=h) From 51bf21a634c0068b32a11437d3f172612be8c46d Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Mon, 27 Sep 2021 18:39:47 -0700 Subject: [PATCH 24/27] python3: fix pickle error in unit test python3 pickle checks that the __qualname__ does not contain , so we have to set a fake value in this test AttributeError: Can't pickle local object 'TestDataObjects.set_up_pickling_class..BasicMost' --- tests/test_dataobject.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_dataobject.py b/tests/test_dataobject.py index 536b8b9..aab4b16 100644 --- a/tests/test_dataobject.py +++ b/tests/test_dataobject.py @@ -371,6 +371,8 @@ class BasicMost(self.cls): # method by mistake. pickletest_module.__file__ = None BasicMost.__module__ = 'remoteobjects._pickletest' + # In python3, we have to set __qualname__ too + BasicMost.__qualname__ = BasicMost.__name__ sys.modules['remoteobjects._pickletest'] = pickletest_module return BasicMost From 577d63832ee90cb2df7defac927ce0ffd4a945cb Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Wed, 29 Sep 2021 01:41:45 -0700 Subject: [PATCH 25/27] python3: decode string with replace before json.loads In python2, we subclassed JSONDecoder to forgive byte sequencess that are not valid UTF-8. But in python3 we can't do that because the bytes are decoded to str prior to JSONDecoder. This change makes us decode the whole bytes ourselves before calling json.loads. --- remoteobjects/http.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/remoteobjects/http.py b/remoteobjects/http.py index a57f690..43400c8 100644 --- a/remoteobjects/http.py +++ b/remoteobjects/http.py @@ -263,10 +263,22 @@ def update_from_response(self, url, response, content): self.raise_for_response(url, response, content) if self.response_has_content.get(response.status): - try: + if PY3: + # Both simplejson and the built-in json module decode the bytes + # before passing it to the JSONDecoder in python3, + # so we might as well do the same thing + # instead of passing bytes to a modified JSONDecoder + # https://github.com/simplejson/simplejson/blob/v3.17.5/simplejson/decoder.py#L368-L369 + # https://github.com/python/cpython/blob/v3.9.7/Lib/json/__init__.py#L341 + if not isinstance(content, text_type): + # use a forgiving decode + content = content.decode('utf-8', errors='replace') data = json.loads(content) - except UnicodeDecodeError: - data = json.loads(content, cls=ForgivingDecoder) + else: + try: + data = json.loads(content) + except UnicodeDecodeError: + data = json.loads(content, cls=ForgivingDecoder) self.update_from_dict(data) From 2a2ce877f73cf947e85ee9b03cf514377eea0ea0 Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Mon, 27 Sep 2021 19:23:43 -0700 Subject: [PATCH 26/27] python3 fix: make ListObject parameter order match unit test ListObject used to set 'offset' and then 'limit', but the unit test assumes that the order is limit=...&offset=.... Since dict and kwargs are guaranteed to preserve insertion order in python3.6+ (https://www.python.org/dev/peps/pep-0468/), this change makes them insert in the same order as the test expects Fixes AssertionError: 'http://example.com/foo?offset=0&limit=10' != 'http://example.com/foo?limit=10&offset=0' --- remoteobjects/listobject.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/remoteobjects/listobject.py b/remoteobjects/listobject.py index ec8a0f0..c41fca6 100644 --- a/remoteobjects/listobject.py +++ b/remoteobjects/listobject.py @@ -193,9 +193,9 @@ def __getitem__(self, key): if isinstance(key, slice): args = dict() if key.start is not None: - args['offset'] = key.start if key.stop is not None: args['limit'] = key.stop - key.start + args['offset'] = key.start elif key.stop is not None: args['limit'] = key.stop return self.filter(**args) From 532fba7f9b3ac0e482d8542a84e87dee190876a2 Mon Sep 17 00:00:00 2001 From: Yonathan Randolph Date: Mon, 27 Sep 2021 19:01:15 -0700 Subject: [PATCH 27/27] Add test that lone surrogates are not detected or fixed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upgrade simplejson to 3.3.0 since prior to that, decoding json with lone surrogates would raise “JSONDecodeError: Unpaired high surrogate” --- remoteobjects/json.py | 13 +++++-------- setup.py | 2 +- tests/test_http.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/remoteobjects/json.py b/remoteobjects/json.py index f1d2568..111cb45 100644 --- a/remoteobjects/json.py +++ b/remoteobjects/json.py @@ -33,14 +33,11 @@ # simplejson >=3.12 from simplejson.errors import errmsg except ImportError: - try: - # simplejson >=3.1.0, <3.12, before this commit: - # https://github.com/simplejson/simplejson/commit/0d36c5cd16055d55e6eceaf252f072a9339e0746 - from simplejson.scanner import errmsg - except ImportError: - # simplejson >=1.1,<3.1.0, before this commit: - # https://github.com/simplejson/simplejson/commit/104b40fcf6aa39d9ba7b240c3c528d1f85e86ef2 - from simplejson.decoder import errmsg + # simplejson >=3.1.0, <3.12, since this commit: + # https://github.com/simplejson/simplejson/commit/104b40fcf6aa39d9ba7b240c3c528d1f85e86ef2 + # and before this commit + # https://github.com/simplejson/simplejson/commit/0d36c5cd16055d55e6eceaf252f072a9339e0746 + from simplejson.scanner import errmsg from simplejson.scanner import py_make_scanner from six import unichr, text_type import sys diff --git a/setup.py b/setup.py index 0bad657..c9660d0 100644 --- a/setup.py +++ b/setup.py @@ -63,7 +63,7 @@ provides=['remoteobjects'], python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*', install_requires=[ - 'simplejson>=2.0.0', + 'simplejson>=3.3.0', 'httplib2>=0.5.0', 'python-dateutil>=2.1', 'six~=1.16.0', diff --git a/tests/test_http.py b/tests/test_http.py index cc6317d..13e0e81 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -30,6 +30,7 @@ import unittest from remoteobjects import fields, http +from six import PY2 from tests import test_dataobject from tests import utils @@ -81,6 +82,40 @@ class BasicMost(self.cls): # Bad characters are replaced with the unicode Replacement Character 0xFFFD. self.assertEqual(b.value, u"image by \ufffdrew Example") h.request.assert_called_once_with(**request) + h.reset_mock() + + # since simplejson 3.3.0, lone surrogates are passed through + # https://github.com/simplejson/simplejson/commit/35816bfe2d0ddeb5ddcc68239683cbb35b7e3ff2 + content = """{"name": "lone surrogate \\ud800", "value": "\\udc00 lone surrogate"}""" + h = utils.mock_http(request, content) + b = BasicMost.get('http://example.com/ohhai', http=h) + # Lone surrogates are passed through as lone surrogates in the python unicode value + self.assertEqual(b.name, u"lone surrogate \ud800") + self.assertEqual(b.value, u"\udc00 lone surrogate") + h.request.assert_called_once_with(**request) + + content = u"""{"name": "100 \u20AC", "value": "13000 \u00A5"}""".encode('utf-8') + h = utils.mock_http(request, content) + b = BasicMost.get('http://example.com/ohhai', http=h) + # JSON containing non-ascii UTF-8 should be decoded to unicode strings + self.assertEqual(b.name, u"100 \u20AC") + self.assertEqual(b.value, u"13000 \u00A5") + h.request.assert_called_once_with(**request) + + content = b"""{"name": "lone surrogate \xed\xa0\x80", "value": "\xed\xb0\x80 lone surrogate"}""" + h = utils.mock_http(request, content) + b = BasicMost.get('http://example.com/ohhai', http=h) + # Lone surrogates are passed through as lone surrogates in the python unicode value + if PY2: + # in python2, our JSONDecoder does not detect naked lone surrogates + self.assertEqual(b.name, u"lone surrogate \ud800") + self.assertEqual(b.value, u"\udc00 lone surrogate") + else: + # in python3, bytes.decode replaces lone surrogates with replacement char + self.assertEqual(b.name, u"lone surrogate \ufffd\ufffd\ufffd") + self.assertEqual(b.value, u"\ufffd\ufffd\ufffd lone surrogate") + + h.request.assert_called_once_with(**request) def test_post(self):