Raised default block size to 4096

Author: annatisch

batchapps/api.py

@@ -438,7 +438,7 @@ def get_output(self,
                    otype='output',
                    url=None,
                    callback=None,
-                   block=1024):
+                   block=4096):
         """
         Gets the content of the job output or its thumbnail.
         Either ``url``, or both ``job_id`` and ``otype`` must be set.
@@ -462,10 +462,10 @@ def get_output(self,
               supplied, ``job_id`` and ``otype`` will not be used.
               The default is None.
             - callback (func): A function to be called to report download progress.
-              The function takes three parameters: the percent downloaded (float), the 
+              The function must take three arguments: the percent downloaded (float), the 
               bytes downloaded (float), and the total bytes to be downloaded (float).
             - block (int): The amount of data downloaded in each block - determines 
-              the frequency with which the callback is called. Default is 1024.
+              the frequency with which the callback is called. Default is 4096.
 
         :Returns:
             - :class:`.Response` with the GET response, however this is not
@@ -627,7 +627,7 @@ def get_output_file(self,
                         fname=None,
                         url=None,
                         callback=None,
-                        block=1024):
+                        block=4096):
         """
         Gets the content of a file created in a job.
         Either ``url``, or both ``job_id`` and ``fname`` must be set.
@@ -648,10 +648,10 @@ def get_output_file(self,
             - url (str): The URL directly to the file to be downloaded.
               The default is None.
             - callback (func): A function to be called to report download progress.
-              The function takes three parameters: the percent downloaded (float), the 
+              The function must take three arguments: the percent downloaded (float), the 
               bytes downloaded (float), and the total bytes to be downloaded (float).
             - block (int): The amount of data downloaded in each block - determines 
-              the frequency with which the callback is called. Default is 1024.
+              the frequency with which the callback is called. Default is 4096.
 
         :Returns:
             - :class:`.Response` with the GET response, however this is not
@@ -1024,7 +1024,7 @@ def query_missing_files(self, files):
 
 
     def get_file(self, userfile, size, download_dir, overwrite=False,
-                 callback=None, block=1024):
+                 callback=None, block=4096):
         """Gets the content of a file previously uploaded by the user.
 
         :Args:
@@ -1040,10 +1040,10 @@ def get_file(self, userfile, size, download_dir, overwrite=False,
             - overwrite (bool): Whether to overwrite a destination file if it
               already exists. The default is ``False``.
             - callback (func): A function to be called to report download progress.
-              The function takes three parameters: the percent downloaded (float), the 
+              The function must take three arguments: the percent downloaded (float), the 
               bytes downloaded (float), and the total bytes to be downloaded (float).
             - block (int): The amount of data downloaded in each block - determines 
-              the frequency with which the callback is called. Default is 1024.
+              the frequency with which the callback is called. Default is 4096.
 
         :Returns:
             - :class:`.Response` with the GET response, however this is not
@@ -1113,7 +1113,7 @@ def props_file(self, userfile):
         else:
             return Response(True, head_resp)
 
-    def send_file(self, userfile, callback=None, block=1024):
+    def send_file(self, userfile, callback=None, block=4096):
         """Uploads a user file for use in a job.
 
         :Args:
@@ -1123,10 +1123,10 @@ def send_file(self, userfile, callback=None, block=1024):
 
         :Kwargs:
             - callback (func): A function to be called to report upload progress.
-              The function takes three parameters: the percent uploaded (float), the 
+              The function must take three arguments: the percent uploaded (float), the 
               bytes uploaded (float), and the total bytes to be uploaded (float).
             - block (int): The amount of data uploaded in each block - determines 
-              the frequency with which the callback is called. Default is 1024.
+              the frequency with which the callback is called. Default is 4096.
 
         :Returns:
             - :class:`.Response` with the PUT response, however this is not

batchapps/files.py

@@ -177,7 +177,7 @@ def __delitem__(self, filekey):
             self._collection = [a for a in self._collection
                                 if a.name != filekey]
 
-    def _upload_forced(self, userfile, callback=None, block=1024):
+    def _upload_forced(self, userfile, callback=None, block=4096):
         """Upload a single file in the collection, ignoring overwrite.
         Only used internally in :func:upload by the parallel subprocesses
 
@@ -187,18 +187,18 @@ def _upload_forced(self, userfile, callback=None, block=1024):
 
         :Kwargs:
             - callback (func): A function to be called to report upload progress.
-              The function takes three parameters: the percent uploaded (float), the 
+              The function must take three arguments: the percent uploaded (float), the 
               bytes uploaded (float), and the total bytes to be uploaded (float).
             - block (int): The amount of data uploaded in each block - determines 
-              the frequency with which the callback is called. Default is 1024.
+              the frequency with which the callback is called. Default is 4096.
 
         :Returns:
             - A tuple containing the result of the :func:`UserFile.upload` call,
               and the original userfile:
               ``(bool success, userfile, string result)``
         """
         self._log.debug("About to upload file: {0}".format(userfile))
-        resp = userfile.upload(force=True, callback=callback, block=1024)
+        resp = userfile.upload(force=True, callback=callback, block=4096)
 
         # TODO: Need to fix hanging when we try to return the result
         # object rather than just a string.
@@ -370,7 +370,7 @@ def remove(self, userfile):
             raise TypeError("File to remove must be userfile object, "
                             "filename string, userfile index int or slice")
 
-    def upload(self, force=False, threads=None, callback=None, block=1024):
+    def upload(self, force=False, threads=None, callback=None, block=4096):
         """Upload all files in a set, optionally in parallel
 
         :Kwargs:
@@ -380,10 +380,10 @@ def upload(self, force=False, threads=None, callback=None, block=1024):
             - threads (int): Maximum number of parallel uploads, default is 1
               (i.e. not parallel). Max threads is 10.
             - callback (func): A function to be called to report upload progress.
-              The function takes three parameters: the percent uploaded (float), the 
+              The function must take three arguments: the percent uploaded (float), the 
               bytes uploaded (float), and the total bytes to be uploaded (float).
             - block (int): The amount of data uploaded in each block - determines 
-              the frequency with which the callback is called. Default is 1024.
+              the frequency with which the callback is called. Default is 4096.
 
         :Returns:
             - A list of tuples containing any files that failed to upload and
@@ -787,18 +787,18 @@ def create_submit_specifier(self):
         self._log.debug("File specification: {0}".format(file_spec))
         return file_spec
 
-    def upload(self, force=False, callback=None, block=1024):
+    def upload(self, force=False, callback=None, block=4096):
         """Upload file.
 
         :Kwargs:
             - force (bool): If ``True``, uploads regardless of whether the
               file has already been uploaded. Otherwise checks if file has
               been uploaded, and if so, skips. The default is ``False``.
             - callback (func): A function to be called to report upload progress.
-              The function takes three parameters: the percent uploaded (float), the 
+              The function must take three arguments: the percent uploaded (float), the 
               bytes uploaded (float), and the total bytes to be uploaded (float).
             - block (int): The amount of data uploaded in each block - determines 
-              the frequency with which the callback is called. Default is 1024.
+              the frequency with which the callback is called. Default is 4096.
 
         :Returns:
             - Client :class:`.Response` object if upload was attempted.
@@ -846,7 +846,7 @@ def is_uploaded(self):
         else:
             raise resp.result
 
-    def download(self, download_dir, callback=None, block=1024):
+    def download(self, download_dir, callback=None, block=4096):
         """Download file.
         
         :Args:
@@ -855,10 +855,10 @@ def download(self, download_dir, callback=None, block=1024):
 
         :Kwargs:
             - callback (func): A function to be called to report download progress.
-              The function takes three parameters: the percent downloaded (float), the 
+              The function must take three arguments: the percent downloaded (float), the 
               bytes downloaded (float), and the total bytes to be downloaded (float).
             - block (int): The amount of data downloaded in each block - determines 
-              the frequency with which the callback is called. Default is 1024.
+              the frequency with which the callback is called. Default is 4096.
 
         :Raises:
             - :class:`.RestCallException` if an error occurred during the

batchapps/job.py

@@ -555,7 +555,7 @@ def _format_submission(self, sub):
         self._log.debug("Extracted job submission data: {0}".format(formatted))
         return formatted
 
-    def _get_final_output(self, download_dir, overwrite, callback=None, block=1024):
+    def _get_final_output(self, download_dir, overwrite, callback=None, block=4096):
         """Internal method to download jobs final output.
 
         :Args:
@@ -566,10 +566,10 @@ def _get_final_output(self, download_dir, overwrite, callback=None, block=1024):
 
         :Kwargs:
             - callback (func): A function to be called to report download progress.
-              The function takes three parameters: the percent downloaded (float), the 
+              The function must take three arguments: the percent downloaded (float), the 
               bytes downloaded (float), and the total bytes to be downloaded (float).
             - block (int): The amount of data downloaded in each block - determines 
-              the frequency with which the callback is called. Default is 1024.
+              the frequency with which the callback is called. Default is 4096.
 
         :Returns:
             - :class:`.Response` object returned directly from
@@ -599,7 +599,7 @@ def _get_final_output(self, download_dir, overwrite, callback=None, block=1024):
                                     callback=callback,
                                     block=block)
 
-    def _get_final_preview(self, download_dir, filename, overwrite, callback=None, block=1024):
+    def _get_final_preview(self, download_dir, filename, overwrite, callback=None, block=4096):
         """Internal method to download jobs final thumbnail.
         We don't bother to check file size for download feedback, as it's
         assumed to be very small.
@@ -613,10 +613,10 @@ def _get_final_preview(self, download_dir, filename, overwrite, callback=None, b
 
         :Kwargs:
             - callback (func): A function to be called to report download progress.
-              The function takes three parameters: the percent downloaded (float), the 
+              The function must take three arguments: the percent downloaded (float), the 
               bytes downloaded (float), and the total bytes to be downloaded (float).
             - block (int): The amount of data downloaded in each block - determines 
-              the frequency with which the callback is called. Default is 1024.
+              the frequency with which the callback is called. Default is 4096.
 
         :Returns:
             - :class:`.Response` object returned directly from
@@ -630,7 +630,7 @@ def _get_final_preview(self, download_dir, filename, overwrite, callback=None, b
                                     callback=callback,
                                     block=block)
 
-    def _get_intermediate_output(self, output, download_dir, overwrite, callback=None, block=1024):
+    def _get_intermediate_output(self, output, download_dir, overwrite, callback=None, block=4096):
         """Internal method to download any file from the job output.
 
         :Args:
@@ -643,10 +643,10 @@ def _get_intermediate_output(self, output, download_dir, overwrite, callback=Non
 
         :Kwargs:
             - callback (func): A function to be called to report download progress.
-              The function takes three parameters: the percent downloaded (float), the 
+              The function must take three arguments: the percent downloaded (float), the 
               bytes downloaded (float), and the total bytes to be downloaded (float).
             - block (int): The amount of data downloaded in each block - determines 
-              the frequency with which the callback is called. Default is 1024.
+              the frequency with which the callback is called. Default is 4096.
         
 
         :Returns:
@@ -705,7 +705,7 @@ def get_tasks(self):
         else:
             raise resp.result
 
-    def get_output(self, download_dir, output=None, overwrite=False, callback=None, block=1024):
+    def get_output(self, download_dir, output=None, overwrite=False, callback=None, block=4096):
         """
         Download a job output file.
         This could be the jobs final output, or if specified, any intermediate
@@ -721,10 +721,10 @@ def get_output(self, download_dir, output=None, overwrite=False, callback=None,
               will be downloaded, otherwise the job's final output will be
               downloaded.
             - callback (func): A function to be called to report download progress.
-              The function takes three parameters: the percent downloaded (float), the 
+              The function must take three arguments: the percent downloaded (float), the 
               bytes downloaded (float), and the total bytes to be downloaded (float).
             - block (int): The amount of data downloaded in each block - determines 
-              the frequency with which the callback is called. Default is 1024.
+              the frequency with which the callback is called. Default is 4096.
 
         :Returns:
             - The full path to the downloaded file (str).
@@ -987,7 +987,7 @@ def __init__(self, client, job_id, **props):
                 'type': _output.get('kind')
                 })
 
-    def _get_file(self, output, download_dir, overwrite, callback=None, block=1024):
+    def _get_file(self, output, download_dir, overwrite, callback=None, block=4096):
         """Internal method to download a task output.
 
         :Args:
@@ -1000,10 +1000,10 @@ def _get_file(self, output, download_dir, overwrite, callback=None, block=1024):
 
         :Kwargs:
             - callback (func): A function to be called to report download progress.
-              The function takes three parameters: the percent downloaded (float), the 
+              The function must take three arguments: the percent downloaded (float), the 
               bytes downloaded (float), and the total bytes to be downloaded (float).
             - block (int): The amount of data downloaded in each block - determines 
-              the frequency with which the callback is called. Default is 1024.
+              the frequency with which the callback is called. Default is 4096.
 
         :Returns:
             - :class:`.Response` object returned directly from
@@ -1044,7 +1044,7 @@ def get_thumbnail(self, download_dir=None, filename=None, overwrite=True, callba
             - overwrite (bool): Whether an existing file will be overwritten.
               The default is ``True``.
             - callback (func): A function to be called to report download progress.
-              The function takes three parameters: the percent downloaded (float), the 
+              The function must take three arguments: the percent downloaded (float), the 
               bytes downloaded (float), and the total bytes to be downloaded (float).
 
         :Returns:
@@ -1109,7 +1109,7 @@ def list_outputs(self):
         else:
             raise resp.result
 
-    def get_output(self, output, download_dir, overwrite=False, callback=None, block=1024):
+    def get_output(self, output, download_dir, overwrite=False, callback=None, block=4096):
         """Download a task output file.
 
         :Args:
@@ -1122,10 +1122,10 @@ def get_output(self, output, download_dir, overwrite=False, callback=None, block
             - overwrite (bool): Whether to overwrite an existing file.
               The default is ``False``.
             - callback (func): A function to be called to report download progress.
-              The function takes three parameters: the percent downloaded (float), the 
+              The function must take three arguments: the percent downloaded (float), the 
               bytes downloaded (float), and the total bytes to be downloaded (float).
             - block (int): The amount of data downloaded in each block - determines 
-              the frequency with which the callback is called. Default is 1024.
+              the frequency with which the callback is called. Default is 4096.
 
         :Returns:
             - The full path to the downloaded file (str).

batchapps/job_manager.py

@@ -182,7 +182,7 @@ def create_job(self, name, **jobdetails):
         """
         return JobSubmission(self._client, str(name), **jobdetails)
 
-    def submit(self, submitjob, upload_threads=None, callback=None, block=1024):
+    def submit(self, submitjob, upload_threads=None, callback=None, block=4096):
         """Submit a job, and upload all its assets
 
         :Args:
@@ -192,10 +192,10 @@ def submit(self, submitjob, upload_threads=None, callback=None, block=1024):
             - upload_threads (int): Maximum number of concurrent asset uploads.
               The default is 1.
             - callback (func): A function to be called to report upload progress.
-              The function takes three parameters: the percent uploaded (float), the 
+              The function must take three arguments: the percent uploaded (float), the 
               bytes uploaded (float), and the total bytes to be uploaded (float).
             - block (int): The amount of data uploaded in each block - determines 
-              the frequency with which the callback is called. Default is 1024.
+              the frequency with which the callback is called. Default is 4096.
 
         :Returns:
             - A job submission response dictionary in the format: