SG-25029 Add Python 3.9 coverage to Python API Azure Pipelines tests (#259)

* Map base 64 encode method
* Fix configparser compatibility
* Add tests config parser, string encoding and splitting urls
* Fix assertEquals warnings

Author: 000paradox000

.gitignore

@@ -30,3 +30,5 @@ htmlcov
 build
 dist
 shotgun_api3.egg-info
+/%1
+

azure-pipelines-templates/run-tests.yml

@@ -48,6 +48,8 @@ jobs:
         python.version: '2.7'
       Python37:
         python.version: '3.7'
+      Python39:
+        python.version: '3.9'
 
   # These are the steps that will be executed inside each job.
   steps:

shotgun_api3/lib/six.py

@@ -36,6 +36,7 @@
 PY2 = sys.version_info[0] == 2
 PY3 = sys.version_info[0] == 3
 PY34 = sys.version_info[0:2] >= (3, 4)
+PY38 = sys.version_info[0:2] >= (3, 8)
 
 if PY3:
     string_types = str,

shotgun_api3/shotgun.py

@@ -35,7 +35,6 @@
 from .lib.six import BytesIO               # used for attachment upload
 from .lib.six.moves import map
 
-import base64
 from .lib.six.moves import http_cookiejar  # used for attachment upload
 import datetime
 import logging
@@ -57,6 +56,12 @@
 # to be exposed as part of the API.
 from .lib.six.moves.xmlrpc_client import Error, ProtocolError, ResponseError  # noqa
 
+if six.PY3:
+    from base64 import encodebytes as base64encode
+else:
+    from base64 import encodestring as base64encode
+
+
 LOG = logging.getLogger("shotgun_api3")
 """
 Logging instance for shotgun_api3
@@ -651,18 +656,20 @@ def __init__(self,
         # if the service contains user information strip it out
         # copied from the xmlrpclib which turned the user:password into
         # and auth header
-        # Do NOT urlsplit(self.base_url) here, as it contains the lower case version
-        # of the base_url argument. Doing so would base64-encode the lowercase
-        # version of the credentials.
-        auth, self.config.server = urllib.parse.splituser(urllib.parse.urlsplit(base_url).netloc)
+
+        # Do NOT self._split_url(self.base_url) here, as it contains the lower
+        # case version of the base_url argument. Doing so would base64encode
+        # the lowercase version of the credentials.
+        auth, self.config.server = self._split_url(base_url)
         if auth:
-            auth = base64.encodestring(six.ensure_binary(urllib.parse.unquote(auth))).decode("utf-8")
+            auth = base64encode(six.ensure_binary(
+                urllib.parse.unquote(auth))).decode("utf-8")
             self.config.authorization = "Basic " + auth.strip()
 
         # foo:[email protected]:3456
         if http_proxy:
-            # check if we're using authentication. Start from the end since there might be
-            # @ in the user's password.
+            # check if we're using authentication. Start from the end since
+            # there might be @ in the user's password.
             p = http_proxy.rsplit("@", 1)
             if len(p) > 1:
                 self.config.proxy_user, self.config.proxy_pass = \
@@ -710,6 +717,33 @@ def __init__(self,
             self.config.user_password = None
             self.config.auth_token = None
 
+    def _split_url(self, base_url):
+        """
+        Extract the hostname:port and username/password/token from base_url
+        sent when connect to the API.
+
+        In python 3.8 `urllib.parse.splituser` was deprecated warning devs to
+        use `urllib.parse.urlparse`.
+        """
+        if six.PY38:
+            auth = None
+            results = urllib.parse.urlparse(base_url)
+            server = results.hostname
+            if results.port:
+                server = "{}:{}".format(server, results.port)
+
+            if results.username:
+                auth = results.username
+
+                if results.password:
+                    auth = "{}:{}".format(auth, results.password)
+
+        else:
+            auth, server = urllib.parse.splituser(
+                urllib.parse.urlsplit(base_url).netloc)
+
+        return auth, server
+
     # ========================================================================
     # API Functions
 

tests/base.py

@@ -2,8 +2,6 @@
 import os
 import re
 import unittest
-from shotgun_api3.lib.six.moves.configparser import ConfigParser
-
 
 from . import mock
 
@@ -12,6 +10,12 @@
 from shotgun_api3.shotgun import ServerCapabilities
 from shotgun_api3.lib import six
 
+if six.PY2:
+    from shotgun_api3.lib.six.moves.configparser import SafeConfigParser as ConfigParser
+else:
+    from shotgun_api3.lib.six.moves.configparser import ConfigParser
+
+
 try:
     # Attempt to import skip from unittest.  Since this was added in Python 2.7
     # in the case that we're running on Python 2.6 we'll need a decorator to

tests/test_api.py

@@ -1661,7 +1661,7 @@ def test_zero_is_not_none(self):
 
         # Should be filtered out
         result = self.sg.find('Asset', [['id', 'is', self.asset['id']], [num_field, 'is_not', None]], [num_field])
-        self.assertEquals([], result)
+        self.assertEqual([], result)
 
         # Set it to zero
         self.sg.update('Asset', self.asset['id'], {num_field: 0})
@@ -1681,17 +1681,17 @@ def test_include_archived_projects(self):
         if self.sg.server_caps.version > (5, 3, 13):
             # Ticket #25082
             result = self.sg.find_one('Shot', [['id', 'is', self.shot['id']]])
-            self.assertEquals(self.shot['id'], result['id'])
+            self.assertEqual(self.shot['id'], result['id'])
 
             # archive project
             self.sg.update('Project', self.project['id'], {'archived': True})
 
             # setting defaults to True, so we should get result
             result = self.sg.find_one('Shot', [['id', 'is', self.shot['id']]])
-            self.assertEquals(self.shot['id'], result['id'])
+            self.assertEqual(self.shot['id'], result['id'])
 
             result = self.sg.find_one('Shot', [['id', 'is', self.shot['id']]], include_archived_projects=False)
-            self.assertEquals(None, result)
+            self.assertEqual(None, result)
 
             # unarchive project
             self.sg.update('Project', self.project['id'], {'archived': False})
@@ -2810,7 +2810,7 @@ def test_import_httplib(self):
         # right one.)
         httplib2_compat_version = httplib2.Http.__module__.split(".")[-1]
         if six.PY2:
-            self.assertEquals(httplib2_compat_version, "python2")
+            self.assertEqual(httplib2_compat_version, "python2")
         elif six.PY3:
             self.assertTrue(httplib2_compat_version, "python3")
 

tests/test_client.py

@@ -12,11 +12,11 @@
 CRUD functions. These tests always use a mock http connection so not not
 need a live server to run against."""
 
-import base64
 import datetime
-from shotgun_api3.lib.six.moves import urllib
 import os
 import re
+
+from shotgun_api3.lib.six.moves import urllib
 from shotgun_api3.lib import six
 try:
     import simplejson as json
@@ -38,8 +38,14 @@
 from . import base
 
 
+if six.PY3:
+    from base64 import encodebytes as base64encode
+else:
+    from base64 import encodestring as base64encode
+
+
 def b64encode(val):
-    return base64.encodestring(six.ensure_binary(val)).decode("utf-8")
+    return base64encode(six.ensure_binary(val)).decode("utf-8")
 
 
 class TestShotgunClient(base.MockTestBase):
@@ -164,6 +170,61 @@ def test_url(self):
         expected = "Basic " + b64encode(urllib.parse.unquote(login_password)).strip()
         self.assertEqual(expected, sg.config.authorization)
 
+    def test_b64encode(self):
+        """Parse value using the proper encoder."""
+        login = "thelogin"
+        password = "%thepassw0r#$"
+        login_password = "%s:%s" % (login, password)
+        expected = 'dGhlbG9naW46JXRoZXBhc3N3MHIjJA=='
+        result = b64encode(urllib.parse.unquote(login_password)).strip()
+        self.assertEqual(expected, result)
+
+    def test_read_config(self):
+        """Validate that config values are properly coerced."""
+        this_dir = os.path.dirname(os.path.realpath(__file__))
+        config_path = os.path.join(this_dir, "test_config_file")
+        config = base.ConfigParser()
+        config.read(config_path)
+        result = config.get("SERVER_INFO", "api_key")
+        expected = "%abce"
+
+        self.assertEqual(expected, result)
+
+    def test_split_url(self):
+        """Validate that url parts are properly extracted."""
+
+        sg = api.Shotgun("https://ci.shotgunstudio.com",
+                         "foo", "bar", connect=False)
+
+
+        base_url = "https://ci.shotgunstudio.com"
+        expected_server = "ci.shotgunstudio.com"
+        expected_auth = None
+        auth, server = sg._split_url(base_url)
+        self.assertEqual(auth, expected_auth)
+        self.assertEqual(server, expected_server)
+
+        base_url = "https://ci.shotgunstudio.com:9500"
+        expected_server = "ci.shotgunstudio.com:9500"
+        expected_auth = None
+        auth, server = sg._split_url(base_url)
+        self.assertEqual(auth, expected_auth)
+        self.assertEqual(server, expected_server)
+
+        base_url = "https://x:[email protected]:9500"
+        expected_server = "ci.shotgunstudio.com:9500"
+        expected_auth = "x:y"
+        auth, server = sg._split_url(base_url)
+        self.assertEqual(auth, expected_auth)
+        self.assertEqual(server, expected_server)
+
+        base_url = "https://[email protected]:9500"
+        expected_server = "ci.shotgunstudio.com:9500"
+        expected_auth = "12345XYZ"
+        auth, server = sg._split_url(base_url)
+        self.assertEqual(auth, expected_auth)
+        self.assertEqual(server, expected_server)
+
     def test_authorization(self):
         """Authorization passed to server"""
         login = self.human_user['login']

tests/test_config_file

@@ -0,0 +1,7 @@
+[SERVER_INFO]
+server_url     : https://url
+script_name    : xyz
+api_key        : %%abce
+
+[TEST_DATA]
+project_name   : hjkl