diff --git a/README.md b/README.md index 3722da8..594d25d 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ To upload from stdin: $ cat file | glacier-cmd upload Test --description "Some description" --stdin --name /path/BetterName -IMPORTANT NOTE: If you're uploading from stdin, and you don't specify a +IMPORTANT NOTE: If you're uploading from stdin or a pipe, and you don't specify a --partsize option, your upload will be limited to 1.3Tb, and the progress report will come out every 128Mb. For more details, run: @@ -156,7 +156,8 @@ its file name, its description, or limit the search by region and vault. If that is not enough you should use `getarchive` and specify the archive ID of the archive you want to retrieve: - $ TODO: example here + $ glacier-cmd download Test eBLl4DbMbZ4YMA7fD9cNacf2z1kGxpYxBqTV4qFVsuzgjuNlKSkWm2rFpw6Gq-bFT6Vt9cUZ1lGqSbtZjtbeh0jYn9tJC-MczQyA3tP6bezYUeN8dGGvqNqT3la79wjRRair1am1JA --outfile filename + Read 1 GB of 10 GB (10%). Rate 3.05 MB/s, average 2.71 MB/s, ETA 1:00:00. To remove uploaded archive use `rmarchive`. You can currently delete only by archive id (notice the use of `--` when the archive ID starts with a dash): @@ -221,80 +222,102 @@ Usage description(help): usage: glacier-cmd [-h] [-c FILE] [--logtostdout] [--aws-access-key AWS_ACCESS_KEY] [--aws-secret-key AWS_SECRET_KEY] [--region REGION] - [--bookkeeping] + [--account-id ACCOUNT_ID] [--bookkeeping] [--no-bookkeeping] [--bookkeeping-domain-name BOOKKEEPING_DOMAIN_NAME] [--logfile LOGFILE] [--loglevel {-1,DEBUG,0,INFO,1,WARNING,2,ERROR,3,CRITICAL}] - [--output {print,csv,json}] - - - {mkvault,lsvault,describevault,rmvault,upload,listmultiparts,abortmultipart,inventory,getarchive,download,rmarchive,search,listjobs,describejob,treehash} - ... + [--output {csv,json,print}] + [--sdb-access-key SDB_ACCESS_KEY] + [--sdb-secret-key SDB_SECRET_KEY] [--sdb-region SDB_REGION] + {mkvault,lsvault,describevault,rmvault,upload,listmultiparts,abortmultipart,inventory,getarchive,download,rmarchive,search,listjobs,describejob,treehash,sns} + ... Command line interface for Amazon Glacier optional arguments: - -h, --help show this help message and exit - -c FILE, --conf FILE Name of the file to log messages to. (default: - ~/.glacier-cmd) - --logtostdout Send log messages to stdout instead of the config - file. (default: False) + -h, --help show this help message and exit + -c FILE, --conf FILE Name of the file to log messages to. (default: + ~/.glacier-cmd) + --logtostdout Send log messages to stdout instead of the config + file. (default: False) Subcommands: - {mkvault,lsvault,describevault,rmvault,upload,listmultiparts,abortmultipart,inventory,getarchive,download,rmarchive,search,listjobs,describejob,treehash} - For subcommand help, use: glacier-cmd -h - mkvault Create a new vault. - lsvault List available vaults. - describevault Describe a vault. - rmvault Remove a vault. - upload Upload an archive to Amazon Glacier. - listmultiparts List all active multipart uploads. - abortmultipart Abort a multipart upload. - inventory List inventory of a vault, if available. If not - available, creates inventory retrieval job if none - running already. - getarchive Requests to make an archive available for download. - download Download a file by archive id. - rmarchive Remove archive from Amazon Glacier. - search Search Amazon SimpleDB database for available archives - (requires bookkeeping to be enabled). - listjobs List active jobs in a vault. - describejob Describe a job. - treehash Calculate the tree-hash (Amazon style sha256-hash) of - a file. + {mkvault,lsvault,describevault,rmvault,upload,listmultiparts,abortmultipart,inventory,getarchive,download,rmarchive,search,listjobs,describejob,treehash,sns} + For subcommand help, use: glacier-cmd -h + mkvault Create a new vault. + lsvault List available vaults. + describevault Describe a vault. + rmvault Remove a vault. + upload Upload an archive to Amazon Glacier. + listmultiparts List all active multipart uploads. + abortmultipart Abort a multipart upload. + inventory List inventory of a vault, if available. If not + available, creates inventory retrieval job if none + running already. + getarchive Requests to make an archive available for download. + download Download a file by archive id. + rmarchive Remove archive from Amazon Glacier. + search Search Amazon SimpleDB database for available archives + (requires bookkeeping to be enabled). + listjobs List active jobs in a vault. + describejob Describe a job. + treehash Calculate the tree-hash (Amazon style sha256-hash) of + a file. + sns Subcommands related to SNS aws: - --aws-access-key AWS_ACCESS_KEY - Your aws access key (Required if you have not created - a ~/.glacier-cmd or /etc/glacier-cmd.conf config file) - (default: AKIAIP5VPUSCSJQ6BSSQ) - --aws-secret-key AWS_SECRET_KEY - Your aws secret key (Required if you have not created - a ~/.glacier-cmd or /etc/glacier-cmd.conf config file) - (default: WDgq6ZZn7Y4Lkt5LxPuionw2pTLbonwdFZz1BGtS) + --aws-access-key AWS_ACCESS_KEY + Your aws access key (Required if you have not created + a ~/.glacier-cmd or /etc/glacier-cmd.conf config file) + (default: AKIAINJIQK32YOKKYIPA) + --aws-secret-key AWS_SECRET_KEY + Your aws secret key (Required if you have not created + a ~/.glacier-cmd or /etc/glacier-cmd.conf config file) + (default: Tl1NT/8b5sRxr0Dzz9ySUv50hoJM64hGa8QpiL5k) glacier: - --region REGION Region where you want to store your archives (Required - if you have not created a ~/.glacier-cmd or /etc - /glacier-cmd.conf config file) (default: us-east-1) - --bookkeeping Should we keep book of all created archives. This - requires a Amazon SimpleDB account and its bookkeeping - domain name set (default: True) - --bookkeeping-domain-name BOOKKEEPING_DOMAIN_NAME - Amazon SimpleDB domain name for bookkeeping. (default: - squirrel) - --no-bookkeeping If present, overrides either CLI or configuration file - options provided for bookkeeping either beforehand or - afterwards - --logfile LOGFILE File to write log messages to. (default: /home/wouter - /.glacier-cmd.log) - --loglevel {-1,DEBUG,0,INFO,1,WARNING,2,ERROR,3,CRITICAL} - Set the lowest level of messages you want to log. - (default: DEBUG) - --output {print,csv,json} - Set how to return results: print to the screen, or as - csv resp. json string. (default: print) + --region REGION Region where you want to store your archives (Required + if you have not created a ~/.glacier-cmd or /etc + /glacier-cmd.conf config file) (default: us-east-1) + --account-id ACCOUNT_ID + AWS account ID of the account that owns the vault + (default: -) + --bookkeeping Should we keep book of all created archives. This + requires a Amazon SimpleDB account and its bookkeeping + domain name set (default: False) + --no-bookkeeping Explicitly disables bookkeeping, regardless of other + configuration or command line options. (default: + False) + --bookkeeping-domain-name BOOKKEEPING_DOMAIN_NAME + Amazon SimpleDB domain name for bookkeeping. (default: + amazon-glacier) + --logfile LOGFILE File to write log messages to. (default: /home/gburca + /.glacier-cmd.log) + --loglevel {-1,DEBUG,0,INFO,1,WARNING,2,ERROR,3,CRITICAL} + Set the lowest level of messages you want to log. + (default: WARNING) + --output {csv,json,print} + Set how to return results: print to the screen, or as + csv resp. json string. NOTE: to receive full output + use csv or json. `print` removes lines longer than 138 + chars (default: print) + + sdb: + --sdb-access-key SDB_ACCESS_KEY + aws access key to be used with bookkeeping (Required + if you have not created a ~/.glacier-cmd or /etc + /glacier-cmd.conf config file) (default: + AKIAINJIQK32YOKKYIPA) + --sdb-secret-key SDB_SECRET_KEY + aws secret key to be used with bookkeeping (Required + if you have not created a ~/.glacier-cmd or /etc + /glacier-cmd.conf config file) (default: + Tl1NT/8b5sRxr0Dzz9ySUv50hoJM64hGa8QpiL5k) + --sdb-region SDB_REGION + Region where you want to store bookkeeping (Required + if you have not created a ~/.glacier-cmd or /etc + /glacier-cmd.conf config file) (default: us-east-1) SimpleDB bookkeeping (custom) domain name ----------------------------------------- @@ -306,7 +329,7 @@ Short Notification Service (SNS) is Amazon's technology that allows you to be no If you run `glacier-cmd sns sync` without specifing anything in your configuration file, it will automatically subscribe all your vaults to `aws-glacier-notifications` topic. - $ glacier.py sns sync + $ glacier-cmd sns sync +------------+-------------------------------------------------+ | Vault Name | Request Id | +------------+-------------------------------------------------+ diff --git a/glacier/GlacierWrapper.py b/glacier/GlacierWrapper.py index a2cb57b..0f6995a 100755 --- a/glacier/GlacierWrapper.py +++ b/glacier/GlacierWrapper.py @@ -12,6 +12,7 @@ import re import logging import os.path +import stat import time import sys import traceback @@ -219,14 +220,17 @@ def glacier_connect_wrap(*args, **kwargs): Connecting to Amazon Glacier with aws_access_key %s aws_secret_key %s - region %s\ + region %s + account_id %s\ """, self.aws_access_key, self.aws_secret_key, - self.region) + self.region, + self.account_id) self.glacierconn = GlacierConnection(self.aws_access_key, self.aws_secret_key, - region_name=self.region) + region_name=self.region, + account_id=self.account_id) except boto.exception.AWSConnectionError as e: raise ConnectionException( "Cannot connect to Amazon Glacier.", @@ -989,7 +993,13 @@ def upload(self, vault_name, file_name, description, region, total_size = 0 reader = None mmapped_file = None - if not stdin: + + if stdin: + is_pipe = False + else: + is_pipe = stat.S_ISFIFO(os.stat(file_name).st_mode) + + if not stdin and not is_pipe: if not file_name: raise InputException( "No file name given for upload.", @@ -1005,6 +1015,15 @@ def upload(self, vault_name, file_name, description, region, cause=e, code='FileError') + elif is_pipe: + try: + reader = open(file_name, 'rb') + total_size = 0 + except IOError: + raise InputException( + "Could not access pipe: %s."% file_name, + cause = e, code = 'FileError') + elif select.select([sys.stdin,],[],[],0.0)[0]: reader = sys.stdin total_size = 0 @@ -1073,9 +1092,9 @@ def upload(self, vault_name, file_name, description, region, start, stop = (int(p) for p in part['RangeInBytes'].split('-')) stop += 1 if not start == current_position: - if stdin: + if stdin or is_pipe: raise InputException( - 'Cannot verify non-sequential upload data from stdin.', + 'Cannot verify non-sequential upload data from stdin or pipe.', code='ResumeError') if reader: reader.seek(start) @@ -1199,7 +1218,9 @@ def upload(self, vault_name, file_name, description, region, self.logger.debug(msg) writer.close() - if not stdin: + if is_pipe: + reader.close() + elif not stdin: f.close() current_time = time.time() overall_rate = int(writer.uploaded_size/(current_time - start_time)) @@ -1619,7 +1640,7 @@ def inventory(self, vault_name, refresh): # in progress job. job_list = self.list_jobs(vault_name) inventory_done = False - for job in job_list: + for job in sorted(job_list, key=lambda x: x['CompletionDate'], reverse=True): if job['Action'] == "InventoryRetrieval": # As soon as a finished inventory job is found, we're done. @@ -1938,7 +1959,7 @@ def sns_unsubscribe(self, protocol, endpoint, topic, sns_options): return unsubscribed - def __init__(self, aws_access_key, aws_secret_key, region, + def __init__(self, aws_access_key, aws_secret_key, region, account_id='-', bookkeeping=False, no_bookkeeping=None, bookkeeping_domain_name=None, sdb_access_key=None, sdb_secret_key=None, sdb_region=None, logfile=None, loglevel='WARNING', logtostdout=True): @@ -1951,6 +1972,8 @@ def __init__(self, aws_access_key, aws_secret_key, region, :type aws_secret_key: str :param region: name of your default region, see :ref:`regions`. :type region: str + :param account_id: AWS account ID + :type account_id: str :param bookkeeping: whether to enable bookkeeping, see :reg:`bookkeeping`. :type bookkeeping: boolean :param bookkeeping_domain_name: your Amazon SimpleDB domain name where the bookkeeping information will be stored. @@ -1979,6 +2002,7 @@ def __init__(self, aws_access_key, aws_secret_key, region, self.bookkeeping_domain_name = bookkeeping_domain_name self.region = region + self.account_id = account_id self.sdb_access_key = sdb_access_key if sdb_access_key else aws_access_key self.sdb_secret_key = sdb_secret_key if sdb_secret_key else aws_secret_key @@ -1997,6 +2021,7 @@ def __init__(self, aws_access_key, aws_secret_key, region, nobookkeeping=%s, bookkeeping_domain_name=%s, region=%s, + account_id=%s, sdb_access_key=%s, sdb_secret_key=%s, sdb_region=%s, @@ -2005,6 +2030,6 @@ def __init__(self, aws_access_key, aws_secret_key, region, logging to stdout %s.""", aws_access_key, aws_secret_key, bookkeeping, no_bookkeeping, - bookkeeping_domain_name, region, + bookkeeping_domain_name, region, account_id, sdb_access_key, sdb_secret_key, sdb_region, logfile, loglevel, logtostdout) diff --git a/glacier/constants.py b/glacier/constants.py new file mode 100644 index 0000000..c7a446a --- /dev/null +++ b/glacier/constants.py @@ -0,0 +1,66 @@ + +# constants definition +CHUNK_SIZE = 1024 +DEFAULT_PART_SIZE = 128 # in MB, power of 2. +# After every failed block upload we sleep (SLEEP_TIME * retries) seconds. +# The more retries we've made for one particular block, the longer we sleep +# before re-attempting to re-upload that block. +SLEEP_TIME = 300 +# How many retries we should make to upload a particular block. We will not +# give up unless we've made at LEAST this many attempts to upload a block. +BLOCK_RETRIES = 10 +# How many retries we should allow for the whole upload. We will not give up +# unless we've made at LEAST this many attempts to upload the archive. +TOTAL_RETRIES = 100 +# For large files, the limits above could be surpassed. We also set a per-Gb +# criteria that allows more errors for larger uploads. +MAX_TOTAL_RETRY_PER_GB = 2 +TABLE_OUTPUT_FORMAT = ["csv", "json", "print"] +SYSTEM_WIDE_CONFIG_FILENAME = "/etc/glacier-cmd.conf" +USER_CONFIG_FILENAME = "~/.glacier-cmd" +HELP_MESSAGE_CONFIG = u"(Required if you have not created a ~/.glacier-cmd or /etc/glacier-cmd.conf config file)" +ERRORCODE = {'InternalError': 127, # Library internal error. + 'UndefinedErrorCode': 126, # Undefined code. + 'NoResults': 125, # Operation yielded no results. + 'GlacierConnectionError': 1, # Can not connect to Glacier. + 'SdbConnectionError': 2, # Can not connect to SimpleDB. + 'CommandError': 3, # Command line is invalid. + 'VaultNameError': 4, # Invalid vault name. + 'DescriptionError': 5, # Invalid archive description. + 'IdError': 6, # Invalid upload/archive/job ID given. + 'RegionError': 7, # Invalid region given. + 'FileError': 8, # Error related to reading/writing a file. + 'ResumeError': 9, # Problem resuming a multipart upload. + 'NotReady': 10, # Requested download is not ready yet. + 'BookkeepingError': 11, # Bookkeeping not available. + 'SdbCommunicationError': 12, # Problem reading/writing SimpleDB data. + 'ResourceNotFoundException': 13, # Glacier can not find the requested resource. + 'InvalidParameterValueException': 14, # Parameter not accepted. + 'DownloadError': 15, # Downloading an archive failed. + 'SNSConnectionError': 126, # Can not connect to SNS + 'SNSConfigurationError': 127, # Problem with configuration file + 'SNSParameterError':128, # Problem with arguments passed to SNS + } +VAULT_NAME_ALLOWED_CHARACTERS = "[a-zA-Z\.\-\_0-9]+" +ID_ALLOWED_CHARACTERS = "[a-zA-Z\-\_0-9]+" +MAX_VAULT_NAME_LENGTH = 255 +MAX_VAULT_DESCRIPTION_LENGTH = 1024 +MAX_PARTS = 10000 +AVAILABLE_REGIONS = ('us-east-1', 'us-west-2', 'us-west-1', + 'eu-west-1', 'eu-central-1', 'sa-east-1', + 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2') +AVAILABLE_REGIONS_MESSAGE = """\ + Invalid region. Available regions for Amazon Glacier are: + us-east-1 (US - Virginia) + us-west-1 (US - N. California) + us-west-2 (US - Oregon) + eu-west-1 (EU - Ireland) + eu-central-1 (EU - Frankfurt) + sa-east-1 (South America - Sao Paulo) + ap-northeast-1 (Asia-Pacific - Tokyo) + ap-southeast-1 (Asia Pacific (Singapore) + ap-southeast-2 (Asia-Pacific - Sydney)\ + """ + + +UPLOAD_DATA_ERROR_MSG = "Received data does not match uploaded data; please check your uploadid and try again." diff --git a/glacier/glacier.py b/glacier/glacier.py index 5bd5abe..8611483 100755 --- a/glacier/glacier.py +++ b/glacier/glacier.py @@ -16,12 +16,12 @@ import csv import json -from prettytable import PrettyTable +import glacierexception +import constants +from prettytable import PrettyTable from GlacierWrapper import GlacierWrapper - from functools import wraps -from glacierexception import * def output_headers(headers, output): """ @@ -31,26 +31,33 @@ def output_headers(headers, output): :type headers: dict """ rows = [(k, headers[k]) for k in headers.keys()] + + if output not in constants.TABLE_OUTPUT_FORMAT: + raise ValueError("Output format must be {}, got" + ": {}".format(constants.TABLE_OUTPUT_FORMAT, + output)) + if output == 'print': table = PrettyTable(["Header", "Value"]) for row in rows: - if len(str(row[1])) < 100: + if len(str(row[1])) <= 138: table.add_row(row) - + print table - + if output == 'csv': csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) for row in rows: csvwriter.writerow(row) - + if output == 'json': print json.dumps(headers) def output_table(results, output, keys=None, sort_key=None): """ Prettyprints results. Expects a list of identical dicts. - Use the dict keys as headers unless keys is given; one line for each item. + Use the dict keys as headers unless keys is given; + one line for each item. Expected format of data is a list of dicts: [{'key1':'data1.1', 'key2':'data1.2', ... }, @@ -62,6 +69,10 @@ def output_table(results, output, keys=None, sort_key=None): sort_key: the key to use for sorting the table. """ + if output not in constants.TABLE_OUTPUT_FORMAT: + raise ValueError("Output format must be {}, " + "got {}".format(constants.TABLE_OUTPUT_FORMAT, + output)) if output == 'print': if len(results) == 0: print 'No output!' @@ -74,16 +85,16 @@ def output_table(results, output, keys=None, sort_key=None): if sort_key: table.sortby = keys[sort_key] if keys else sort_key - + print table - + if output == 'csv': csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) keys = results[0].keys() csvwriter.writerow(keys) for row in results: csvwriter.writerow([row[k] for k in keys]) - + if output == 'json': print json.dumps(results) @@ -96,31 +107,39 @@ def output_msg(msg, output, success=True): :param success: whether the operation was a success or not. :type success: boolean """ - if output == 'print': - print msg - - if output == 'csv': - csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) - csvwriter.writerow(msg) - - if output == 'json': - print json.dumps(msg) - + + if output not in constants.TABLE_OUTPUT_FORMAT: + raise ValueError("Output format must be {}, " + "got {}".format(constants.TABLE_OUTPUT_FORMAT, + output)) + + if msg is not None: + if output == 'print': + print msg + + if output == 'csv': + csvwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) + csvwriter.writerow(msg) + + if output == 'json': + print json.dumps(msg) + if not success: sys.exit(125) def size_fmt(num, decimals = 1): """ - Formats file sizes in human readable format. Anything bigger than TB - is returned is TB. Number of decimals is optional, defaults to 1. + Formats file sizes in human readable format. Anything bigger than + TB is returned is TB. Number of decimals is optional, + defaults to 1. """ fmt = "%%3.%sf %%s"% decimals for x in ['bytes','KB','MB','GB']: if num < 1024.0: return fmt % (num, x) - + num /= 1024.0 - + return fmt % (num, 'TB') def default_glacier_wrapper(args, **kwargs): @@ -131,6 +150,7 @@ def default_glacier_wrapper(args, **kwargs): return GlacierWrapper(args.aws_access_key, args.aws_secret_key, args.region, + args.account_id, bookkeeping=args.bookkeeping, no_bookkeeping=args.no_bookkeeping, bookkeeping_domain_name=args.bookkeeping_domain_name, @@ -154,10 +174,10 @@ def handle_errors(fn): def wrapper(*args, **kwargs): try: return fn(*args, **kwargs) - except GlacierException as e: + except glacierexception.GlacierException as e: - # We are only interested in the error message in case it is a - # self-caused exception. + # We are only interested in the error message in case + # it is a self-caused exception. e.write(indentation='|| ', stack=False, message=True) sys.exit(e.exitcode) @@ -206,7 +226,7 @@ def describevault(args): 'SizeInBytes': "Size", 'VaultARN': "ARN", 'CreationDate': "Created"} - output_headers(response, args.output) + output_table([response], args.output, keys=headers) @handle_errors def listmultiparts(args): @@ -216,7 +236,8 @@ def listmultiparts(args): glacier = default_glacier_wrapper(args) response = glacier.listmultiparts(args.vault) if not response: - output_msg('No active multipart uploads.', args.output, success=False) + output_msg('No active multipart uploads.', args.output, + success=False) else: output_table(response, args.output) @@ -264,7 +285,8 @@ def download(args): """ glacier = default_glacier_wrapper(args) response = glacier.download(args.vault, args.archive, args.partsize, - out_file_name=args.outfile, overwrite=args.overwrite) + out_file_name=args.outfile, + overwrite=args.overwrite) if args.outfile: output_msg(response, args.output, success=True) @@ -278,9 +300,9 @@ def upload(args): # This is /path/to/vol001|vol002|vol003 if args.bacula: if len(args.filename) > 1: - raise InputException( - 'Bacula-style file name input can accept only one file name argument.') - + raise glacierexception.InputException("Bacula-style file name input can "\ + "accept only one file name argument.") + fileset = args.filename[0].split('|') if len(fileset) > 1: dirname = os.path.dirname(fileset[0]) @@ -297,7 +319,7 @@ def upload(args): # be read over stdin. if args.filename: for f in args.filename: - + # In case the shell does not expand wildcards, if any, do this here. if f[0] == '~': f = os.path.expanduser(f) @@ -305,32 +327,41 @@ def upload(args): globbed = glob.glob(f) if globbed: for g in globbed: - response = glacier.upload(args.vault, g, args.description, args.region, args.stdin, - args.name, args.partsize, args.uploadid, args.resume) + response = glacier.upload(args.vault, g, + args.description, + args.region, args.stdin, + args.name, args.partsize, + args.uploadid, + args.resume) results.append({"Uploaded file": g, "Created archive with ID": response[0], - "Archive SHA256 tree hash": response[1]}) + "Archive SHA256 tree hash": response[1], + "Description": args.description}) else: - raise InputException( - "File name given for upload can not be found: %s."% f, + raise glacierexception.InputException( + "File name given for upload can not "\ + "be found: {}.".format(f), code='CommandError') - + elif args.stdin: # No file name; using stdin. - response = glacier.upload(args.vault, None, args.description, args.region, args.stdin, - args.name, args.partsize, args.uploadid, args.resume) + response = glacier.upload(args.vault, None, args.description, + args.region, args.stdin, + args.name, args.partsize, + args.uploadid, args.resume) results = [{"Created archive with ID": response[0], - "Archive SHA256 tree hash": response[1]}] + "Archive SHA256 tree hash": response[1], + "Description": args.description}] else: - raise InputException( + raise glacierexception.InputException( '''No input given. Either give a file name or file names -on the command line, or use the --stdin switch and pipe -in the data over stdin.''', + on the command line, or use the --stdin switch and pipe + in the data over stdin.''', cause='No file name and no stdin pipe.', code='CommandError') - + output_table(results, args.output) if len(results) > 1 \ else output_headers(results[0], args.output) @@ -374,26 +405,30 @@ def inventory(args): if sys.stdout.isatty() and output == 'print': print 'Checking inventory, please wait.\r', sys.stdout.flush() - + job, inventory = glacier.inventory(args.vault, args.refresh) if inventory: if sys.stdout.isatty() and output == 'print': - print "Inventory of vault: %s" % (inventory["VaultARN"],) - print "Inventory Date: %s\n" % (inventory['InventoryDate'],) + print "Inventory of vault: {}".format(inventory["VaultARN"]) + print "Inventory Date: {}\n".format(['InventoryDate']) print "Content:" - + headers = {'ArchiveDescription': 'Archive Description', 'CreationDate': 'Uploaded', 'Size': 'Size', 'ArchiveId': 'Archive ID', 'SHA256TreeHash': 'SHA256 tree hash'} - output_table(inventory['ArchiveList'], args.output, keys=headers) + output_table(inventory['ArchiveList'], + args.output, + keys=headers) if sys.stdout.isatty() and output == 'print': size = 0 for item in inventory['ArchiveList']: size += int(item['Size']) - print 'This vault contains %s items, total size %s.'% (len(inventory['ArchiveList']), size_fmt(size)) + print "This vault contains {} items, total size "\ + "{}.".format(len(inventory['ArchiveList']), + size_fmt(size)) else: result = {'Status':'Inventory retrieval in progress.', @@ -411,10 +446,11 @@ def treehash(args): for f in args.filename: if f: - # In case the shell does not expand wildcards, if any, do this here. + # In case the shell does not expand wildcards, + # if any, do this here. if f[0] == '~': f = os.path.expanduser(f) - + globbed = glob.glob(f) if globbed: for g in globbed: @@ -422,24 +458,26 @@ def treehash(args): {'File name': g, 'SHA256 tree hash': glacier.get_tree_hash(g)}) else: - raise InputException( - 'No file name given.', - code='CommandError') + raise glacierexception.InputException('No file name given.', + code='CommandError') output_table(hash_results, args.output) def snssync(args): """ - If monitored_vaults is specified in configuration file, subscribe vaults - specificed in it to notifications, otherwiser subscribe all vault. + If monitored_vaults is specified in configuration file, subscribe + vaults specificed in it to notifications, otherwiser + subscribe all vault. """ glacier = default_glacier_wrapper(args) - response = glacier.sns_sync(sns_options=args.sns_options, output=args.output) + response = glacier.sns_sync(sns_options=args.sns_options, + output=args.output) output_table(response, args.output) def snssubscribe(args): """ - Subscribe individual vaults to notifications by method specified by user. + Subscribe individual vaults to notifications by method + specified by user. """ protocol = args.protocol endpoint = args.endpoint @@ -447,7 +485,9 @@ def snssubscribe(args): topic = args.topic glacier = default_glacier_wrapper(args) - response = glacier.sns_subscribe(protocol, endpoint, topic, vault_names=vault_names, sns_options=args.sns_options) + response = glacier.sns_subscribe(protocol, endpoint, topic, + vault_names=vault_names, + sns_options=args.sns_options) output_table(response, args.output) def snslistsubscriptions(args): @@ -459,7 +499,9 @@ def snslistsubscriptions(args): topic = args.topic glacier = default_glacier_wrapper(args) - response = glacier.sns_list_subscriptions(protocol, endpoint, topic, sns_options=args.sns_options) + response = glacier.sns_list_subscriptions(protocol, endpoint, + topic, + sns_options=args.sns_options) output_table(response, args.output) def snslisttopics(args): @@ -469,16 +511,37 @@ def snslisttopics(args): def snsunsubscribe(args): """ - Unsubscribe individual vaults from notifications for specified protocol, - endpoint and vault. + Unsubscribe individual vaults from notifications for + specified protocol, endpoint and vault. """ protocol = args.protocol endpoint = args.endpoint topic = args.topic glacier = default_glacier_wrapper(args) - response = glacier.sns_unsubscribe(protocol, endpoint, topic, sns_options=args.sns_options) - output_table(response, args.output) + response = glacier.sns_unsubscribe(protocol, endpoint, + topic, + sns_options=args.sns_options) + output_table(response, args.output) + +class CustomArgParseFormatter(argparse.ArgumentDefaultsHelpFormatter, + argparse.RawDescriptionHelpFormatter): + def _get_help_string(self, action): + """ + This method is identical to the base one, except that if the argument + ends in '-key', the default value is suppressed so that we don't print + out sensitive passwords (from the config file). + """ + help = action.help + if '%(default)' not in action.help: + if action.default is not argparse.SUPPRESS: + defaulting_nargs = [argparse.OPTIONAL, argparse.ZERO_OR_MORE] + if action.option_strings or action.nargs in defaulting_nargs: + if action.option_strings[0].endswith('-key'): + pass + else: + help += ' (default: %(default)s)' + return help def main(): program_description = u""" @@ -487,25 +550,32 @@ def main(): # Config parser conf_parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - add_help=False) - - conf_parser.add_argument("-c", "--conf", default="~/.glacier-cmd", - help="Name of the file to log messages to.", metavar="FILE") - conf_parser.add_argument('--logtostdout', action='store_true', - help='Send log messages to stdout instead of the config file.') + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + add_help=False) + + conf_parser.add_argument("-c", + "--conf", + default="~/.glacier-cmd", + help="Name of the file to log messages to.", + metavar="FILE") + conf_parser.add_argument('--logtostdout', + action='store_true', + help="Send log messages "\ + "to stdout instead of "\ + "the config file.") args, remaining_argv = conf_parser.parse_known_args() - # Here we parse config from files in home folder or in current folder - # We use separate topics for aws and glacier specific configs + # Here we parse config from files in home folder or in current + # folder We use separate topics for aws and glacier + # specific configs aws = glacier = sdb = {} config = ConfigParser.SafeConfigParser() sns = {'topics_present':False, 'topic':'aws-glacier-notifications'} - configs_read = config.read(['/etc/glacier-cmd.conf', - os.path.expanduser('~/.glacier-cmd'), + configs_read = config.read([constants.SYSTEM_WIDE_CONFIG_FILENAME, + os.path.expanduser(constants.USER_CONFIG_FILENAME), args.conf]) if configs_read: try: @@ -519,7 +589,7 @@ def main(): try: sdb = dict(config.items("sdb")) for key,value in sdb.items(): - sdb["sdb_%s"%key]=value + sdb["sdb_{}".format(key)]=value del sdb[key] except ConfigParser.NoSectionError: pass @@ -536,7 +606,7 @@ def main(): 'options':dict(config.items(topic)) } sns_topics += [s] - + if sns_topics: sns['topics'] = sns_topics elif any(topic for topic in config.sections() if topic == "SNS"): @@ -569,23 +639,24 @@ def main(): # Main configuration parser parser = argparse.ArgumentParser(parents=[conf_parser], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - description=program_description) + formatter_class=CustomArgParseFormatter, + description=program_description) subparsers = parser.add_subparsers(title='Subcommands', help=u"For subcommand help, use: glacier-cmd -h") # Amazon Web Services settings group = parser.add_argument_group('aws') - help_msg_config = u"(Required if you have not created a \ - ~/.glacier-cmd or /etc/glacier-cmd.conf config file)" + group.add_argument('--aws-access-key', required=a_required("aws-access-key"), default=a_default("aws-access-key"), - help="Your aws access key " + help_msg_config) + help="Your aws access key "\ + "{}".format(constants.HELP_MESSAGE_CONFIG)) group.add_argument('--aws-secret-key', required=a_required("aws-secret-key"), default=a_default("aws-secret-key"), - help="Your aws secret key " + help_msg_config) + help="Your aws secret key "\ + "{}".format(constants.HELP_MESSAGE_CONFIG)) # Glacier settings group = parser.add_argument_group('glacier') @@ -593,25 +664,33 @@ def main(): required=required("region"), default=default("region"), help="Region where you want to store \ - your archives " + help_msg_config) + your archives "\ + "{}".format(constants.HELP_MESSAGE_CONFIG)) + group.add_argument('--account-id', + required=False, + default=default("account-id") if default("account-id") else '-', + help="AWS account ID of the account that owns the vault") bookkeeping = True if default('bookkeeping') == 'True' else False group.add_argument('--bookkeeping', required=False, default=bookkeeping, action="store_true", - help="Should we keep book of all created archives.\ - This requires a Amazon SimpleDB account and its \ - bookkeeping domain name set") + help="Should we keep book of all created "\ + "archives. This requires a Amazon "\ + "SimpleDB account and its "\ + "bookkeeping domain name set") group.add_argument('--no-bookkeeping', required=False, default=False, action="store_true", - help="Explicitly disables bookkeeping, regardless of other\ - configuration or command line options.") + help="Explicitly disables bookkeeping, "\ + "regardless of other configuration "\ + "or command line options.") group.add_argument('--bookkeeping-domain-name', required=False, default=default("bookkeeping-domain-name"), - help="Amazon SimpleDB domain name for bookkeeping.") + help="Amazon SimpleDB domain name " + "for bookkeeping.") group.add_argument('--logfile', required=False, default=os.path.expanduser('~/.glacier-cmd.log'), @@ -625,24 +704,32 @@ def main(): group.add_argument('--output', required=False, default=default('output') if default('output') else 'print', - choices=['print', 'csv', 'json'], - help='Set how to return results: print to the screen, or as csv resp. json string.') + choices=constants.TABLE_OUTPUT_FORMAT, + help="Set how to return results: print to "\ + "the screen, or as csv resp. json string. "\ + "NOTE: to receive full output use csv or "\ + "json. `print` removes lines "\ + "longer than 138 chars") # SimpleDB settings group = parser.add_argument_group('sdb') group.add_argument('--sdb-access-key', required=False, - default=s_default("sdb-access-key") or a_default("aws-access-key"), - help="aws access key to be used with bookkeeping" + help_msg_config) + default=(s_default("sdb-access-key") or + a_default("aws-access-key")), + help="aws access key to be used with \ + bookkeeping {}".format(constants.HELP_MESSAGE_CONFIG)) group.add_argument('--sdb-secret-key', required=False, - default=s_default("sdb-secret-key") or a_default("aws-secret-key"), - help="aws secret key to be used with bookkeeping" + help_msg_config) + default=(s_default("sdb-secret-key") or + a_default("aws-secret-key")), + help="aws secret key to be used with "\ + "bookkeeping {}".format(constants.HELP_MESSAGE_CONFIG)) group.add_argument('--sdb-region', required=False, default=s_default("sdb-region") or default("region"), - help="Region where you want to store \ - your bookkeeping " + help_msg_config) + help="Region where you want to store "\ + "bookkeeping {}".format(constants.HELP_MESSAGE_CONFIG)) # glacier-cmd mkvault parser_mkvault = subparsers.add_parser("mkvault", @@ -651,7 +738,7 @@ def main(): help='The vault to be created.') parser_mkvault.set_defaults(func=mkvault) - # glacier-cmd lsvault + # glacier-cmd lsvault parser_lsvault = subparsers.add_parser("lsvault", help="List available vaults.") parser_lsvault.set_defaults(func=lsvault) @@ -677,19 +764,18 @@ def main(): help='Upload an archive to Amazon Glacier.') parser_upload.add_argument('vault', help='The vault the archive is to be stored in.') -## group = parser_upload.add_mutually_exclusive_group(required=True) parser_upload.add_argument('filename', nargs='*', default=None, help='''\ The name(s) of the local file(s) to be uploaded. Wildcards are accepted. Can not be used if --stdin is used.''') parser_upload.add_argument('--stdin', action='store_true', help='''\ -Read data from stdin, instead of local file. +Read data from stdin, instead of local file. Can not be used if is given.''') parser_upload.add_argument('--name', default=None, help='''\ -Use the given name as the filename for bookkeeping -purposes. To be used in conjunction with --stdin or +Use the given name as the filename for bookkeeping +purposes. To be used in conjunction with --stdin or when the file being uploaded is a temporary file.''') parser_upload.add_argument('--partsize', type=int, default=-1, help='''\ @@ -716,7 +802,7 @@ def main(): If not given, the smallest possible part size will be used when uploading a file, and 128 MB -when uploading from stdin.''') +when uploading from stdin or from a FIFO pipe.''') parser_upload.add_argument('--description', default=None, help='''\ Description of the file to be uploaded. Use quotes @@ -758,13 +844,14 @@ def main(): # glacier-cmd inventory [--refresh] parser_inventory = subparsers.add_parser('inventory', - help='List inventory of a vault, if available. If not available, \ - creates inventory retrieval job if none running already.') + help="List inventory of a vault, if available. If not "\ + "available, creates inventory retrieval job if none "\ + "running already.") parser_inventory.add_argument('vault', help='The vault to list the inventory of.') parser_inventory.add_argument('--refresh', action='store_true', - help='Create an inventory retrieval job, even if inventory is \ - available or with another retrieval job running.') + help="Create an inventory retrieval job, even if inventory is "\ + "available or with another retrieval job running.") parser_inventory.set_defaults(func=inventory) # glacier-cmd getarchive @@ -860,47 +947,61 @@ def main(): # glacier-cmd hash parser_describejob = subparsers.add_parser('treehash', - help='Calculate the tree-hash (Amazon style sha256-hash) of a file.') + help="Calculate the tree-hash (Amazon style sha256-hash) "\ + "of a file.") parser_describejob.add_argument('filename', nargs='*', help='The filename to calculate the treehash of.') parser_describejob.set_defaults(func=treehash) - # SNS related commands are located in their own subparser - parser_sns = subparsers.add_parser('sns', + # SNS related commands are located in their own subparser + parser_sns = subparsers.add_parser('sns', help="Subcommands related to SNS") - sns_subparsers = parser_sns.add_subparsers(title="Subcommands related to SNS") + sns_subparsers = parser_sns.add_subparsers(title="Subcommands "\ + "related to SNS") # glacier-cmd sns syncs sns_parser_sync = sns_subparsers.add_parser('sync', - help="Go through configuration file and either subscribe all vaults to default topic or, if sections are present, create separate topics and subscribe specified vaults to that topic.") + help="Go through configuration file and either "\ + "subscribe all vaults to default topic or, "\ + "if sections are present, create separate "\ + "topics and subscribe specified vaults to that topic.") sns_parser_sync.set_defaults(func=snssync, sns_options=sns) # glacier-cmd sns subscribe protocol endpoint topic [--vault] sns_parser_subscribe = sns_subparsers.add_parser('subscribe', help="Subscribe to topic.") sns_parser_subscribe.add_argument("protocol", - help="Protocol used for notifications. Can be email, http, https or sms.") + help="Protocol used for notifications. Can be email, "\ + "http, https or sms.") sns_parser_subscribe.add_argument("endpoint", - help="Valid applicable endpoint - email address, URL or phone number.") - sns_parser_subscribe.add_argument("topic", - help="Topic for which notifications will be sent to specified protocol and endpoint.") + help="Valid applicable endpoint - email address, "\ + "URL or phone number.") + sns_parser_subscribe.add_argument("topic", + help="Topic for which notifications will be sent "\ + "to specified protocol and endpoint.") sns_parser_subscribe.add_argument("--vault", - help="Optional vault names, seperated by comma, for this a new topic will be created and subscribed to.") - sns_parser_subscribe.set_defaults(func=snssubscribe, sns_options={ "options":sns, }) + help="Optional vault names, seperated by comma, "\ + "for this a new topic will be created and subscribed to.") + sns_parser_subscribe.set_defaults(func=snssubscribe, + sns_options={ "options":sns, }) # glacier-cmd sns unsubscribe [--protocol ] [--endpoint ] [--topic ] sns_parser_unsubscribe = sns_subparsers.add_parser('unsubscribe', help="Unsubscribe from a specified topic.") sns_parser_unsubscribe.add_argument("--protocol", - help="Protocol used for notifications. Can be email, http, https or sms.") + help="Protocol used for notifications. Can be email, "\ + "http, https or sms.") sns_parser_unsubscribe.add_argument("--endpoint", - help="Valid applicable endpoint - email address, URL or phone number.") + help="Valid applicable endpoint - email address, "\ + "URL or phone number.") sns_parser_unsubscribe.add_argument("--topic", - help="Topic for which notifications will be sent to specified protocol and endpoint.") - sns_parser_unsubscribe.set_defaults(func=snsunsubscribe, sns_options=sns) + help="Topic for which notifications will be sent to "\ + "specified protocol and endpoint.") + sns_parser_unsubscribe.set_defaults(func=snsunsubscribe, + sns_options=sns) # glacier-cmd sns lssub [--protocol ] [--endpoint ] [--topic ] - sns_parser_listsubs = sns_subparsers.add_parser('lssub', + sns_parser_listsubs = sns_subparsers.add_parser('lssub', help="List subscriptions. Other arguments are ANDed together.") sns_parser_listsubs.add_argument("--protocol", help="Show only subscriptions on a specified protocol.") @@ -908,23 +1009,25 @@ def main(): help="Show only subscriptions to a specified endpoint.") sns_parser_listsubs.add_argument("--topic", help="Show only subscriptions for a specified topic.") - sns_parser_listsubs.set_defaults(func=snslistsubscriptions, sns_options=sns) + sns_parser_listsubs.set_defaults(func=snslistsubscriptions, + sns_options=sns) # glacier-cmd sns lstopic sns_parser_listtopics = sns_subparsers.add_parser('lstopic', help="List all topics.") - sns_parser_listtopics.set_defaults(func=snslisttopics, sns_options=sns) - + sns_parser_listtopics.set_defaults(func=snslisttopics, + sns_options=sns) + - # TODO args.logtostdout becomes false when parsing the remaining_argv - # so here we bridge this. An ugly hack but it works. + # TODO args.logtostdout becomes false when parsing the + # remaining_argv so here we bridge this. An ugly hack but it works. logtostdout = args.logtostdout # Process the remaining arguments. args = parser.parse_args(remaining_argv) - + args.logtostdout = logtostdout - + # Run the subcommand. args.func(args) diff --git a/glacier/glaciercorecalls.py b/glacier/glaciercorecalls.py index 9b73802..1cf1f9a 100755 --- a/glacier/glaciercorecalls.py +++ b/glacier/glaciercorecalls.py @@ -1,18 +1,18 @@ #!/usr/bin/env python # encoding: utf-8 """ -.. module:: botocorecalls - :platform: Unix, Windows - :synopsis: boto calls to access Amazon Glacier. - +.. module:: botocorecalls + :platform: Unix, Windows + :synopsis: boto calls to access Amazon Glacier. + This depends on the boto library, use version 2.6.0 or newer. - - writer = GlacierWriter(glacierconn, GLACIER_VAULT) - writer.write(block of data) - writer.close() - # Get the id of the newly created archive - archive_id = writer.get_archive_id()from boto.connection import AWSAuthConnection + + writer = GlacierWriter(glacierconn, GLACIER_VAULT) + writer.write(block of data) + writer.close() + # Get the id of the newly created archive + archive_id = writer.get_archive_id()from boto.connection import AWSAuthConnection """ import urllib @@ -25,12 +25,13 @@ import boto.glacier.layer1 from glacierexception import * +from boto.glacier.exceptions import UnexpectedHTTPResponseError # Placeholder, effectively renaming the class. class GlacierConnection(boto.glacier.layer1.Layer1): pass - + def chunk_hashes(data): """ @@ -73,7 +74,20 @@ class GlacierWriter(object): Archive. The data is written using the multi-part upload API. """ DEFAULT_PART_SIZE = 128 # in MB, power of 2. - + # After every failed block upload we sleep (SLEEP_TIME * retries) seconds. + # The more retries we've made for one particular block, the longer we sleep + # before re-attempting to re-upload that block. + SLEEP_TIME = 300 + # How many retries we should make to upload a particular block. We will not + # give up unless we've made at LEAST this many attempts to upload a block. + BLOCK_RETRIES = 10 + # How many retries we should allow for the whole upload. We will not give up + # unless we've made at LEAST this many attempts to upload the archive. + TOTAL_RETRIES = 100 + # For large files, the limits above could be surpassed. We also set a per-Gb + # criteria that allows more errors for larger uploads. + MAX_TOTAL_RETRY_PER_GB = 2 + def __init__(self, connection, vault_name, description=None, part_size_in_bytes=DEFAULT_PART_SIZE*1024*1024, uploadid=None, logger=None): @@ -83,6 +97,7 @@ def __init__(self, connection, vault_name, self.connection = connection ## self.location = None self.logger = logger + self.total_retries = 0 if uploadid: self.uploadid = uploadid @@ -98,7 +113,7 @@ def __init__(self, connection, vault_name, ## self.upload_url = response.getheader("location") def write(self, data): - + if self.closed: raise CommunicationError( "Tried to write to a GlacierWriter that is already closed.", @@ -108,7 +123,7 @@ def write(self, data): raise InputException ( 'Block of data provided must be equal to or smaller than the set block size.', code='InternalError') - + part_tree_hash = tree_hash(chunk_hashes(data)) self.tree_hashes.append(part_tree_hash) headers = { @@ -121,58 +136,66 @@ def write(self, data): "x-amz-content-sha256": hashlib.sha256(data).hexdigest() } - response = self.connection.upload_part(self.vault_name, - self.uploadid, - hashlib.sha256(data).hexdigest(), - bytes_to_hex(part_tree_hash), - (self.uploaded_size, self.uploaded_size+len(data)-1), - data) - response.read() - -## retries = 0 -## while True: -## response = self.connection.make_request( -## "PUT", -## self.upload_url, -## headers, -## data) -## -## # Success. -## if response.status == 204: -## break -## -## # Time-out recieved: sleep for 5 minutes and try again. -## # Do not try more than five times; after that it's over. -## elif response.status == 408: -## if retries >= 5: -## resp = json.loads(response.read()) -## raise ResonseException( -## resp['message'], -## cause='Timeout', -## code=resp['code']) -## -## if self.logger: -## logger.warning(resp['message']) -## logger.warning('sleeping 300 seconds (5 minutes) before retrying.') -## -## retries += 1 -## time.sleep(300) -## -## else: -## raise ResponseException( -## "Multipart upload part expected response status 204 (got %s):\n%s"\ -## % (response.status, response.read()), -## cause=resp['message'], -## code=resp['code']) - -## response.read() + # How many times we tried uploading this block + retries = 0 + + while True: + try: + response = self.connection.upload_part(self.vault_name, + self.uploadid, + hashlib.sha256(data).hexdigest(), + bytes_to_hex(part_tree_hash), + (self.uploaded_size, self.uploaded_size+len(data)-1), + data) + response.read() + break + + except Exception as e: + if '408' in e.message or e.code == "ServiceUnavailableException" or isinstance(e, UnexpectedHTTPResponseError): + uploaded_gb = self.uploaded_size / (1024 * 1024 * 1024) + if retries >= self.BLOCK_RETRIES and retries > math.log10(uploaded_gb) * 10: + if self.logger: + self.logger.warning('Retries exhausted for this block.') + raise e + + if uploaded_gb > 0: + retry_per_gb = self.total_retries / uploaded_gb + else: + retry_per_gb = 0 + if self.total_retries >= self.TOTAL_RETRIES and retry_per_gb > self.MAX_TOTAL_RETRY_PER_GB: + if self.logger: + self.logger.warning('Total retries exhausted.') + raise e + + retries += 1 + self.total_retries += 1 + + if self.logger: + self.logger.warning(e.message) + if sys.version_info < (2, 7, 0): + self.logger.warning('Total uploaded size = %d, block hash = %s' % (self.uploaded_size, bytes_to_hex(part_tree_hash))) + else: + # Commify large numbers + self.logger.warning('Total uploaded size = {:,d}, block hash = {:}'.format(self.uploaded_size, bytes_to_hex(part_tree_hash))) + + self.logger.warning('Retries (this block, total) = %d/%d, %d/%d' % (retries, self.BLOCK_RETRIES, self.total_retries, self.TOTAL_RETRIES)) + self.logger.warning('Check the AWS status at: http://status.aws.amazon.com/') + self.logger.warning('Sleeping %d seconds (%.1f minutes) before retrying this block.' % (self.SLEEP_TIME, self.SLEEP_TIME / 60.0)) + + time.sleep(self.SLEEP_TIME * retries) + + else: + self.logger.warning(e.message) + self.logger.warning('Not re-trying on this error') + raise e + self.uploaded_size += len(data) def close(self): - + if self.closed: return - + # Complete the multiplart glacier upload response = self.connection.complete_multipart_upload(self.vault_name, self.uploadid,