Exposed progress block size and auto token refresh

Author: annatisch

CHANGES.txt

@@ -1,5 +1,11 @@
+v0.5.1, 2015-07-09	- Added auto-refresh for unattended tokens
+					- Exposed configuration of block size to vary upload and download callback frequency
+					- Progress callbacks now take 3 arguments, the percent commplete (float), the bytes transferred (float),
+					  and the total bytes to be transferred (float).
+
 v0.5.0, 2015-07-01	- Removed 3 instance minimum for pool creation.
 					- Added progress callbacks from file upload and download.
+
 v0.4.0, 2015-05-11	- Added optional auth config validation flag.
 					- Added JobSubmission.settings attribute
 					- Fixed error in SessionExpiredException.

LICENSE.txt

@@ -1,5 +1,5 @@
 
-The Azure Batch Apps Python Client ver. 0.5.0 (azure-batch-apps-python)
+The Azure Batch Apps Python Client ver. 0.5.1 (azure-batch-apps-python)
 
 Copyright (c) Microsoft Corporation
 All rights reserved. 

README.rst

@@ -45,6 +45,9 @@ Release History
 
 For full summary of changes, see CHANGES.txt
 
+* 2015-07-09	- 0.5.1
+	- Added auto-refresh for unattended tokens
+	- Exposed configuration of block size to vary upload and download callback frequency
 * 2015-07-01	- 0.5.0
 	- Added progress callbacks for file upload and download
 	- Removed 3 instance minimum for pool creation
@@ -56,8 +59,8 @@ For full summary of changes, see CHANGES.txt
 * 2014-11-26	- 0.2.0	
 	- Changed file upload format
 	- Changed Authentication config format
-	- Changed terminology regarding application → jobtype
-	- Changed terminology regarding service principal → unattended account
+	- Changed terminology: application to jobtype
+	- Changed terminology: service principal to unattended account
 	- Added FileCollection.index method
 	- Added better handling for missing auth values in Configuration
 * 2014-11-03	- 0.1.1 

batchapps/__init__.py

@@ -46,5 +46,5 @@
            "config"]
 
 
-__version__ = "0.5.0"
+__version__ = "0.5.1"
 __author__ = 'Microsoft Corp. <[email protected]>'

batchapps/api.py

@@ -437,7 +437,8 @@ def get_output(self,
                    job_id=None,
                    otype='output',
                    url=None,
-                   callback=None):
+                   callback=None,
+                   block=1024):
         """
         Gets the content of the job output or its thumbnail.
         Either ``url``, or both ``job_id`` and ``otype`` must be set.
@@ -461,8 +462,10 @@ def get_output(self,
               supplied, ``job_id`` and ``otype`` will not be used.
               The default is None.
             - callback (func): A function to be called to report download progress.
-              The function takes a single parameter, the percent downloaded as a
-              float.
+              The function takes three parameters: the percent downloaded (float), the 
+              bytes downloaded (float), and the total bytes to be downloaded (float).
+            - block (int): The amount of data downloaded in each block - determines 
+              the frequency with which the callback is called. Default is 1024.
 
         :Returns:
             - :class:`.Response` with the GET response, however this is not
@@ -474,14 +477,15 @@ def get_output(self,
         """
         self._log.debug(
             "get_output, download_dir={dd}, size={sz}, fname={fn}, "
-            "overwrite={ow}, job_id={ji}, url={ur}, otype={ot}".format(
+            "overwrite={ow}, job_id={ji}, url={ur}, otype={ot}, block={bl}".format(
                 dd=download_dir,
                 sz=size,
                 fn=fname,
                 ow=overwrite,
                 ji=job_id,
                 ur=url,
-                ot=otype))
+                ot=otype,
+                bl=block))
 
         if not url and job_id:
             if otype.lower() not in ['output', 'preview']:
@@ -510,6 +514,7 @@ def get_output(self,
                                             size,
                                             overwrite,
                                             f_name=fname,
+                                            block_size=block,
                                             callback=callback)
 
         except RestCallException as exp:
@@ -621,7 +626,8 @@ def get_output_file(self,
                         job_id=None,
                         fname=None,
                         url=None,
-                        callback=None):
+                        callback=None,
+                        block=1024):
         """
         Gets the content of a file created in a job.
         Either ``url``, or both ``job_id`` and ``fname`` must be set.
@@ -642,8 +648,10 @@ def get_output_file(self,
             - url (str): The URL directly to the file to be downloaded.
               The default is None.
             - callback (func): A function to be called to report download progress.
-              The function takes a single parameter, the percent downloaded as a
-              float.
+              The function takes three parameters: the percent downloaded (float), the 
+              bytes downloaded (float), and the total bytes to be downloaded (float).
+            - block (int): The amount of data downloaded in each block - determines 
+              the frequency with which the callback is called. Default is 1024.
 
         :Returns:
             - :class:`.Response` with the GET response, however this is not
@@ -655,12 +663,13 @@ def get_output_file(self,
         """
         self._log.debug("get_output_file, download_dir={dd}, size={sz}, "
                         "overwrite={ow}, job_id={ji}, fname={fn}, "
-                        "url={ur}".format(dd=download_dir,
+                        "url={ur}, block={bl}".format(dd=download_dir,
                                           sz=size,
                                           ow=overwrite,
                                           ji=job_id,
                                           fn=fname,
-                                          ur=url))
+                                          ur=url,
+                                          bl=block))
 
         name = fname if fname else None
 
@@ -683,6 +692,7 @@ def get_output_file(self,
                                             download_dir,
                                             size, overwrite,
                                             f_name=name,
+                                            block_size=block,
                                             callback=callback)
 
         except RestCallException as exp:
@@ -1013,7 +1023,8 @@ def query_missing_files(self, files):
         return Response(True, resp['files'])
 
 
-    def get_file(self, userfile, size, download_dir, overwrite=False, callback=None):
+    def get_file(self, userfile, size, download_dir, overwrite=False,
+                 callback=None, block=1024):
         """Gets the content of a file previously uploaded by the user.
 
         :Args:
@@ -1029,8 +1040,10 @@ def get_file(self, userfile, size, download_dir, overwrite=False, callback=None)
             - overwrite (bool): Whether to overwrite a destination file if it
               already exists. The default is ``False``.
             - callback (func): A function to be called to report download progress.
-              The function takes a single parameter, the percent downloaded as a
-              float.
+              The function takes three parameters: the percent downloaded (float), the 
+              bytes downloaded (float), and the total bytes to be downloaded (float).
+            - block (int): The amount of data downloaded in each block - determines 
+              the frequency with which the callback is called. Default is 1024.
 
         :Returns:
             - :class:`.Response` with the GET response, however this is not
@@ -1044,10 +1057,11 @@ def get_file(self, userfile, size, download_dir, overwrite=False, callback=None)
                                                      None))
 
         self._log.debug("get_file, file={0}, size={1}, "
-                        "download_dir={2}, overwrite={3}".format(userfile,
-                                                                 size,
-                                                                 download_dir,
-                                                                 overwrite))
+                        "download_dir={2}, overwrite={3}, block={4}".format(userfile,
+                                                                            size,
+                                                                            download_dir,
+                                                                            overwrite,
+                                                                            block))
         url = userfile.url
         self._log.debug("Get file url: {0}".format(url))
         try:
@@ -1057,6 +1071,7 @@ def get_file(self, userfile, size, download_dir, overwrite=False, callback=None)
                                             download_dir,
                                             size,
                                             overwrite=overwrite,
+                                            block_size=block,
                                             callback=callback)
 
         except RestCallException as exp:
@@ -1098,7 +1113,7 @@ def props_file(self, userfile):
         else:
             return Response(True, head_resp)
 
-    def send_file(self, userfile, callback=None):
+    def send_file(self, userfile, callback=None, block=1024):
         """Uploads a user file for use in a job.
 
         :Args:
@@ -1108,8 +1123,10 @@ def send_file(self, userfile, callback=None):
 
         :Kwargs:
             - callback (func): A function to be called to report upload progress.
-              The function takes a single parameter, the percent uploaded as a
-              float.
+              The function takes three parameters: the percent uploaded (float), the 
+              bytes uploaded (float), and the total bytes to be uploaded (float).
+            - block (int): The amount of data uploaded in each block - determines 
+              the frequency with which the callback is called. Default is 1024.
 
         :Returns:
             - :class:`.Response` with the PUT response, however this is not
@@ -1140,6 +1157,7 @@ def send_file(self, userfile, callback=None):
                                         self.headers,
                                         userfile,
                                         params,
+                                        block_size=block,
                                         callback=callback)
 
         except (RestCallException, FileMissingException) as exp:

batchapps/credentials.py

@@ -409,6 +409,7 @@ def __init__(self, config, client_id, token=None):
 
         self._log = logging.getLogger('batch_apps')
         self._id = str(client_id)
+        self._cfg = config
 
         if not hasattr(config, "aad_config"):
             raise InvalidConfigException(
@@ -418,12 +419,12 @@ def __init__(self, config, client_id, token=None):
             self._log.debug("No token supplied, attempting to "
                             "retrieve previous session.")
 
-            self.cfg = config.aad_config(unattended=False)
+            self.auth = config.aad_config(unattended=False)
             token = self.get_stored_auth()
 
         else:
             unattended = 'refresh_token' not in token
-            self.cfg = config.aad_config(unattended=unattended)
+            self.auth = config.aad_config(unattended=unattended)
 
         self.token = token
 
@@ -444,7 +445,7 @@ def get_session(self):
             - :class:`.AuthenticationException` if the token is invalid
               or expired.
         """
-        resource = _https(self.cfg['resource']) #DEP
+        resource = _https(self.auth['resource']) #DEP
         countdown = float(self.token['expires_at']) - time.time()
 
         self.token['expires_in'] = countdown
@@ -453,8 +454,8 @@ def get_session(self):
         try:
             if 'refresh_token' in self.token:
 
-                refresh = _https(self.cfg['root'], self.cfg['tenant'],
-                         self.cfg['token_uri'])
+                refresh = _https(self.auth['root'], self.auth['tenant'],
+                         self.auth['token_uri'])
 
                 new_session = requests_oauthlib.OAuth2Session(
                     self._id,
@@ -481,6 +482,26 @@ def get_session(self):
                 new_session.verify = CA_CERT
             return new_session
 
+    def refresh_session(self):
+        """Refresh unattended token
+        
+        :Returns:
+            - An authenticated :class:`requests_oauthlib.OAuth2Session` if
+              refreshed, else `None`.
+
+        """
+        unattended = 'refresh_token' not in self.token
+        if unattended:
+            try:
+                creds = AzureOAuth.get_unattended_session(self._cfg)
+                self.token = creds.token
+                return self.get_session()
+
+            except (AuthenticationException, InvalidConfigException) as exp:
+                self._log.warning("Failed to refresh unattended token: {0}".format(exp))
+
+        return None
+
     def get_stored_auth(self):
         """Retrieve a previously stored access token for refreshing