diff --git a/.travis.yml b/.travis.yml index 53d30ac0cbe..36e2ffe1622 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ install: - pip install --upgrade coveralls - python setup.py develop script: - - flake8 tools/c7n_mailer + - flake8 tools - make sphinx && py.test -v -n auto --cov=c7n tests tools/c7n_* after_success: coveralls diff --git a/tools/c7n_salactus/c7n_salactus/cli.py b/tools/c7n_salactus/c7n_salactus/cli.py index 9aa24fa3e71..f088a548b8b 100644 --- a/tools/c7n_salactus/c7n_salactus/cli.py +++ b/tools/c7n_salactus/c7n_salactus/cli.py @@ -35,7 +35,7 @@ def debug(f): def _f(*args, **kw): try: f(*args, **kw) - except (SystemExit, KeyboardInterrupt) as e: + except (SystemExit, KeyboardInterrupt): raise except: import traceback, sys, pdb diff --git a/tools/c7n_salactus/c7n_salactus/db.py b/tools/c7n_salactus/c7n_salactus/db.py index 1258b813ab6..3e1029e518b 100644 --- a/tools/c7n_salactus/c7n_salactus/db.py +++ b/tools/c7n_salactus/c7n_salactus/db.py @@ -20,8 +20,8 @@ def accounts(self, accounts=()): for k in self.data['bucket-size'].keys(): a, b = k.split(':') accounts.setdefault(a, []).append(k) - return [Account(a, [Bucket(b, self.data) for b in buckets]) - for a, buckets in accounts.items()] + return [Account(aa, [Bucket(bb, self.data) for bb in buckets]) + for aa, buckets in accounts.items()] def buckets(self, accounts=()): if accounts: diff --git a/tools/c7n_salactus/c7n_salactus/worker.py b/tools/c7n_salactus/c7n_salactus/worker.py index a5967c63a70..56e8fa37359 100644 --- a/tools/c7n_salactus/c7n_salactus/worker.py +++ b/tools/c7n_salactus/c7n_salactus/worker.py @@ -70,7 +70,7 @@ # Minimum size of the bucket before partitioning PARTITION_BUCKET_SIZE_THRESHOLD = 100000 -#PARTITION_BUCKET_SIZE_THRESHOLD = 20000 +# PARTITION_BUCKET_SIZE_THRESHOLD = 20000 # Page size for keys found during partition PARTITION_KEYSET_THRESHOLD = 500 @@ -83,7 +83,7 @@ ('NextKeyMarker', 'NextVersionIdMarker')), False: ('Contents', 'list_objects_v2', ('NextContinuationToken',)) - } +} connection = redis.Redis(host=REDIS_HOST) # Increase timeouts to assist with non local regions, also @@ -210,7 +210,7 @@ def bucket_key_count(client, bucket): hour=0, minute=0, second=0, microsecond=0) - timedelta(1), EndTime=datetime.now().replace( hour=0, minute=0, second=0, microsecond=0), - Period=60*60*24, + Period=60 * 60 * 24, Statistics=['Minimum']) response = client.get_metric_statistics(**params) if not response['Datapoints']: @@ -298,7 +298,6 @@ class CharSet(object): ascii_lower_digits = set(string.ascii_lowercase + string.digits) ascii_alphanum = set(string.ascii_letters + string.digits) - punctuation = set(string.punctuation) @classmethod @@ -478,7 +477,7 @@ def detect_partition_strategy(account_info, bucket, delimiters=('/', '-'), prefi process_bucket_iterator, [account_info, bucket], prefixes) -@job('bucket-partition', timeout=3600*12, connection=connection) +@job('bucket-partition', timeout=3600 * 12, connection=connection) def process_bucket_partitions( account_info, bucket, prefix_set=('',), partition='/', strategy=None, limit=4): @@ -547,8 +546,8 @@ def statm(prefix): if len(prefix_queue) > PARTITION_QUEUE_THRESHOLD: log.info("Partition add friends, %s", statm(prefix)) for s_prefix_set in chunks( - prefix_queue[PARTITION_QUEUE_THRESHOLD-1:], - PARTITION_QUEUE_THRESHOLD-1): + prefix_queue[PARTITION_QUEUE_THRESHOLD - 1:], + PARTITION_QUEUE_THRESHOLD - 1): for s in list(s_prefix_set): if strategy.is_depth_exceeded(prefix): @@ -562,13 +561,13 @@ def statm(prefix): account_info, bucket, prefix_set=s_prefix_set, partition=partition, strategy=strategy, limit=limit) - prefix_queue = prefix_queue[:PARTITION_QUEUE_THRESHOLD-1] + prefix_queue = prefix_queue[:PARTITION_QUEUE_THRESHOLD - 1] if keyset: invoke(process_keyset, account_info, bucket, {contents_key: keyset}) -@job('bucket-page-iterator', timeout=3600*24, connection=connection) +@job('bucket-page-iterator', timeout=3600 * 24, connection=connection) def process_bucket_iterator(account_info, bucket, prefix="", delimiter="", **continuation): """Bucket pagination @@ -595,7 +594,7 @@ def process_bucket_iterator(account_info, bucket, invoke(process_keyset, account_info, bucket, page) -@job('bucket-keyset-scan', timeout=3600*12, connection=connection) +@job('bucket-keyset-scan', timeout=3600 * 12, connection=connection) def process_keyset(account_info, bucket, key_set): session = get_session(account_info) s3 = session.client('s3', region_name=bucket['region'], config=s3config) @@ -603,8 +602,7 @@ def process_keyset(account_info, bucket, key_set): remediation_count = 0 denied_count = 0 contents_key, _, _ = BUCKET_OBJ_DESC[bucket['versioned']] - processor = (bucket['versioned'] and processor.process_version - or processor.process_key) + processor = (bucket['versioned'] and processor.process_version or processor.process_key) connection.hincrby( 'keys-scanned', bucket_id(account_info, bucket['name']), len(key_set.get(contents_key, []))) diff --git a/tools/c7n_salactus/setup.py b/tools/c7n_salactus/setup.py index bd3d63da95d..842055a13bf 100644 --- a/tools/c7n_salactus/setup.py +++ b/tools/c7n_salactus/setup.py @@ -19,8 +19,8 @@ version='0.1.4', description="Cloud Custodian - Salactus S3", classifiers=[ - "Topic :: System :: Systems Administration", - "Topic :: System :: Distributed Computing" + "Topic :: System :: Systems Administration", + "Topic :: System :: Distributed Computing" ], url="https://github.com/capitalone/cloud-custodian", license="Apache-2.0", @@ -30,4 +30,3 @@ 'c7n-salactus = c7n_salactus.cli:cli']}, install_requires=["c7n", "click", "rq", "redis"], ) - diff --git a/tools/c7n_sentry/c7n_sentry/c7nsentry.py b/tools/c7n_sentry/c7n_sentry/c7nsentry.py index 9216bcf2d45..b1915d32ca9 100644 --- a/tools/c7n_sentry/c7n_sentry/c7nsentry.py +++ b/tools/c7n_sentry/c7n_sentry/c7nsentry.py @@ -101,7 +101,7 @@ def process_log_event(event, context): # Grab the actual error log payload serialized = event['awslogs'].pop('data') data = json.loads(zlib.decompress( - base64.b64decode(serialized), 16+zlib.MAX_WBITS)) + base64.b64decode(serialized), 16 + zlib.MAX_WBITS)) msg = get_sentry_message(config, data) if msg is None: return @@ -151,7 +151,7 @@ def process_log_group(config): event_count = 0 log.debug("Querying log events with %s", params) for p in paginator.paginate(**params): - #log.debug("Searched streams\n %s", ", ".join( + # log.debug("Searched streams\n %s", ", ".join( # [s['logStreamName'] for s in p['searchedLogStreams']])) for e in p['events']: event_count += 1 @@ -180,7 +180,7 @@ def send_sentry_message(sentry_dsn, msg): auth_header_keys = [ ('sentry_timestamp', time.time()), ('sentry_client', client), - ('sentry_version', '7'), # try 7? + ('sentry_version', '7'), # try 7? ('sentry_key', key), ('sentry_secret', secret)] auth_header = "Sentry %s" % ', '.join( @@ -250,7 +250,7 @@ def get_sentry_message(config, data, log_client=None, is_lambda=True): sentry_msg = { 'event_id': uuid.uuid4().hex, 'timestamp': datetime.fromtimestamp( - data['logEvents'][0]['timestamp']/1000).isoformat(), + data['logEvents'][0]['timestamp'] / 1000).isoformat(), 'user': { 'id': config['account_id'], 'username': config['account_name']}, @@ -271,7 +271,7 @@ def get_sentry_message(config, data, log_client=None, is_lambda=True): sentry_msg['breadcrumbs'] = [ {'category': 'policy', 'message': e['message'], - 'timestamp': e['timestamp']/1000} for e in breadcrumbs] + 'timestamp': e['timestamp'] / 1000} for e in breadcrumbs] return sentry_msg @@ -393,7 +393,7 @@ def process_account(a): log.info("creating account project %s", a['name']) spost(endpoint + "teams/%s/%s/projects/" % ( options.sentry_org, team_name), - json={'name': a['name']}) + json={'name': a['name']}) bagger = partial( Bag, @@ -429,7 +429,6 @@ def process_account(a): return [process_account(a) for a in accounts] - with ThreadPoolExecutor(max_workers=3) as w: futures = {} for a in accounts: @@ -453,7 +452,9 @@ def deploy(options): def deploy_one(region_name, account, policy, sentry_dsn): from c7n.mu import LambdaManager - session_factory = lambda: boto3.Session(region_name=region_name) + + def session_factory(): + return boto3.Session(region_name=region_name) log_group_name = '/aws/lambda/custodian-{}'.format(policy['name']) arn = 'arn:aws:logs:{}:{}:log-group:{}:*'.format( region_name, account['account_id'], log_group_name) @@ -484,7 +485,7 @@ def setup_parser(): common_parser(cmd_orgreplay) cmd_orgreplay.set_defaults(command=orgreplay) cmd_orgreplay.add_argument('--profile') - #cmd_orgreplay.add_argument('--role') + # cmd_orgreplay.add_argument('--role') cmd_orgreplay.add_argument('--start') cmd_orgreplay.add_argument('--end') cmd_orgreplay.add_argument('--sentry-org', default="c7n") diff --git a/tools/c7n_sentry/setup.py b/tools/c7n_sentry/setup.py index a760960317c..69a5c0a95cb 100644 --- a/tools/c7n_sentry/setup.py +++ b/tools/c7n_sentry/setup.py @@ -19,8 +19,8 @@ version='0.1', description="Cloud Custodian - Sentry", classifiers=[ - "Topic :: System :: Systems Administration", - "Topic :: System :: Distributed Computing" + "Topic :: System :: Systems Administration", + "Topic :: System :: Distributed Computing" ], url="https://github.com/capitalone/cloud-custodian", license="Apache-2.0", @@ -30,4 +30,3 @@ 'c7n-sentry = c7n_sentry.c7nsentry:main']}, install_requires=["c7n"], ) - diff --git a/tools/c7n_sentry/test_sentry.py b/tools/c7n_sentry/test_sentry.py index b0a16641631..046d2776316 100644 --- a/tools/c7n_sentry/test_sentry.py +++ b/tools/c7n_sentry/test_sentry.py @@ -31,7 +31,7 @@ def test_get_sentry_message(self): project='custodian', account_name='c7n-test', account_id='9111411911411', - ) + ) sentry_msg = c7nsentry.get_sentry_message( config, {'logGroup': '/cloud-custodian/', 'logStream': 'night-policy', 'logEvents': [{ @@ -45,6 +45,7 @@ def test_preserve_full_message(self): "FinalDBSnapshotIdentifier is not a valid identifier", error['value']) + msg = """2016-07-07 19:14:24,160 - ERROR - custodian.output - Error while executing policy\nTraceback (most recent call last):\n File \"/usr/local/custodian/lib/python2.7/site-packages/c7n/policy.py\", line 191, in poll\n resources = self.resource_manager.resources()\n File \"/usr/local/custodian/lib/python2.7/site-packages/c7n/query.py\", line 141, in resources\n resources = self.augment(resources)\n File \"/usr/local/custodian/lib/python2.7/site-packages/c7n/resources/s3.py\", line 95, in augment\n results = filter(None, results)\n File \"/usr/local/custodian/lib/python2.7/site-packages/concurrent/futures/_base.py\", line 581, in result_iterator\n yield future.result()\n File \"/usr/local/custodian/lib/python2.7/site-packages/concurrent/futures/_base.py\", line 405, in result\n return self.__get_result()\n File \"/usr/local/custodian/lib/python2.7/site-packages/concurrent/futures/thread.py\", line 55, in run\n result = self.fn(*self.args, **self.kwargs)\n File \"/usr/local/custodian/lib/python2.7/site-packages/c7n/resources/s3.py\", line 126, in assemble_bucket\n v = method(Bucket=b['Name'])\n File \"/usr/local/custodian/lib/python2.7/site-packages/botocore/client.py\", line 258, in _api_call\n return self._make_api_call(operation_name, kwargs)\n File \"/usr/local/custodian/lib/python2.7/site-packages/botocore/client.py\", line 537, in _make_api_call\n operation_model, request_dict)\n File \"/usr/local/custodian/lib/python2.7/site-packages/botocore/endpoint.py\", line 117, in make_request\n return self._send_request(request_dict, operation_model)\n File \"/usr/local/custodian/lib/python2.7/site-packages/botocore/endpoint.py\", line 146, in _send_request\n success_response, exception):\n File \"/usr/local/custodian/lib/python2.7/site-packages/botocore/endpoint.py\", line 219, in _needs_retry\n caught_exception=caught_exception)\n File \"/usr/local/custodian/lib/python2.7/site-packages/botocore/hooks.py\", line 227, in emit\n return self._emit(event_name, kwargs)\n File \"/usr/local/custodian/lib/python2.7/site-packages/botocore/hooks.py\", line 210, in _emit\n response = handler(**kwargs)\n File \"/usr/local/custodian/lib/python2.7/site-packages/botocore/retryhandler.py\", line 183, in __call__\n if self._checker(attempts, response, caught_exception):\n File \"/usr/local/custodian/lib/python2.7/site-packages/botocore/retryhandler.py\", line 251, in __call__\n caught_exception)\n File \"/usr/local/custodian/lib/python2.7/site-packages/botocore/retryhandler.py\", line 274, in _should_retry\n return self._checker(attempt_number, response, caught_exception)\n File \"/usr/local/custodian/lib/python2.7/site-packages/botocore/retryhandler.py\", line 314, in __call__\n caught_exception)\n File \"/usr/local/custodian/lib/python2.7/site-packages/botocore/retryhandler.py\", line 223, in __call__\n attempt_number, caught_exception)\n File \"/usr/local/custodian/lib/python2.7/site-packages/botocore/retryhandler.py\", line 356, in _check_caught_exception\n raise caught_exception\nSSLError: EOF occurred in violation of protocol (_ssl.c:765)""" diff --git a/tools/c7n_sphinxext/c7n_sphinxext/c7n_schema.py b/tools/c7n_sphinxext/c7n_sphinxext/c7n_schema.py index b573f25166c..dfb68a95b0f 100644 --- a/tools/c7n_sphinxext/c7n_sphinxext/c7n_schema.py +++ b/tools/c7n_sphinxext/c7n_sphinxext/c7n_schema.py @@ -63,14 +63,20 @@ def run(self): try: module = importlib.import_module(module_name) except ImportError: - raise SphinxError("Unable to generate reference docs for %s, couldn't import module '%s'" % (model_name, module_name)) + raise SphinxError( + "Unable to generate reference docs for %s, couldn't import module '%s'" % + (model_name, module_name)) model = getattr(module, model_name, None) if model is None: - raise SphinxError("Unable to generate reference docs for %s, no model '%s' in %s" % (model_name, model_name, module_name)) + raise SphinxError( + "Unable to generate reference docs for %s, no model '%s' in %s" % + (model_name, model_name, module_name)) if not hasattr(model, 'schema'): - raise SphinxError("Unable to generate reference docs for %s, model '%s' does not have a 'schema' attribute" % (model_name, model_name)) + raise SphinxError( + "Unable to generate reference docs for %s, model '%s' does not\ + have a 'schema' attribute" % (model_name, model_name)) schema = reformat_schema(model) diff --git a/tools/c7n_sphinxext/setup.py b/tools/c7n_sphinxext/setup.py index 19e4b5140a9..7892967f4b6 100644 --- a/tools/c7n_sphinxext/setup.py +++ b/tools/c7n_sphinxext/setup.py @@ -19,8 +19,8 @@ version='1.0', description="Cloud Custodian - Sphinx Extensions", classifiers=[ - "Topic :: System :: Systems Administration", - "Topic :: System :: Distributed Computing" + "Topic :: System :: Systems Administration", + "Topic :: System :: Distributed Computing" ], url="https://github.com/capitalone/cloud-custodian", license="Apache-2.0", diff --git a/tools/c7n_traildb/traildb.py b/tools/c7n_traildb/traildb.py index 2590607d18e..9237bd9606a 100644 --- a/tools/c7n_traildb/traildb.py +++ b/tools/c7n_traildb/traildb.py @@ -93,6 +93,7 @@ def _init(self): # response text, # request text, # user text, + def insert(self, records): self.cursor.executemany( "insert into events values (?, ?, ?, ?, ?, ?, ?, ?, ?)", @@ -111,9 +112,9 @@ def reduce_records(x, y): return y -#STOP = 42 +# STOP = 42 # -#def store_records(output, q): +# def store_records(output, q): # db = TrailDB(output) # while True: # results = q.get() @@ -171,13 +172,13 @@ def process_records(records, r.get('requestID', ''), r.get('sourceIPAddress', ''), uid, -# TODO make this optional, for now omit for size -# json.dumps(r['requestParameters']), -# json.dumps(r['responseElements']), -# json.dumps(r['userIdentity']), + # TODO make this optional, for now omit for size + # json.dumps(r['requestParameters']), + # json.dumps(r['responseElements']), + # json.dumps(r['userIdentity']), r.get('errorCode', None), r.get('errorMessage', None) - )) + )) if data_dir: if not user_records: return @@ -202,7 +203,7 @@ def process_bucket( # PyPy has some memory leaks.... :-( pool = Pool(maxtasksperchild=10) t = time.time() - object_count = object_size = idx = 0 + object_count = object_size = 0 log.info("Processing:%d cloud-trail %s" % ( cpu_count(), @@ -223,7 +224,7 @@ def process_bucket( trail_bucket=bucket_name) db = TrailDB(output) - bsize = math.ceil(1000/float(cpu_count())) + bsize = math.ceil(1000 / float(cpu_count())) for page in paginator.paginate(Bucket=bucket_name, Prefix=prefix): objects = page.get('Contents', ()) object_count += len(objects) @@ -236,7 +237,7 @@ def process_bucket( results = map(object_processor, chunks(objects, bsize)) st = time.time() - log.info("Loaded page time:%0.2fs", st-pt) + log.info("Loaded page time:%0.2fs", st - pt) for r in results: for fpath in r: @@ -248,10 +249,10 @@ def process_bucket( l = t t = time.time() - log.info("Stored page time:%0.2fs", t-st) + log.info("Stored page time:%0.2fs", t - st) log.info( "Processed paged time:%0.2f size:%s count:%s" % ( - t-l, object_size, object_count)) + t - l, object_size, object_count)) if objects: log.info('Last Page Key: %s', objects[-1]['Key']) @@ -295,7 +296,6 @@ def main(): parser = setup_parser() options = parser.parse_args() - if options.tmpdir and not os.path.exists(options.tmpdir): os.makedirs(options.tmpdir) prefix = get_bucket_path(options) @@ -309,7 +309,7 @@ def main(): options.source, options.not_source, options.tmpdir - ) + ) if __name__ == '__main__': diff --git a/tools/c7n_traildb/trailts.py b/tools/c7n_traildb/trailts.py index 8c30adb0883..dc653a029f8 100644 --- a/tools/c7n_traildb/trailts.py +++ b/tools/c7n_traildb/trailts.py @@ -85,7 +85,7 @@ def process_traildb(db, influx, account_name, region, since=None): qt = time.time() log.debug("query account:%s region:%s services time:%0.2f incremental:%s", - account_name, region, time.time()-qt, since) + account_name, region, time.time() - qt, since) record_count = 0 for b in ['console', 'program']: @@ -98,7 +98,7 @@ def process_traildb(db, influx, account_name, region, since=None): log.debug( "query account:%s region:%s bucket:%s field:%s points:%d time:%0.2f", - account_name, region, b, f, len(results), time.time()-qt) + account_name, region, b, f, len(results), time.time() - qt) measurements = [] for p in results: if f == 'user_id': @@ -126,7 +126,7 @@ def process_traildb(db, influx, account_name, region, since=None): record_count += len(measurements) log.debug( "post account:%s region:%s bucket:%s field:%s points:%d time:%0.2f", - account_name, region, b, f, len(measurements), time.time()-pt) + account_name, region, b, f, len(measurements), time.time() - pt) return record_count @@ -145,7 +145,7 @@ def query_by( rdb.text('call_count > 3')) if field == 'error_code': - query = query.where(t.c.error_code != None) + query = query.where(t.c.error_code is not None) query = query.group_by(t.c.event_source) @@ -168,7 +168,7 @@ def query_by( elif error: query = query.where( rdb.and_( - t.c.error_code != None, + t.c.error_code is not None, rdb.or_( t.c.error_code != 'ThrottlingException', t.c.error_code != 'Client.RequestLimitExceeded'))) @@ -213,11 +213,11 @@ def index_account(config, region, account, day, incremental): return raise s3.download_file(bucket, key, fh.name) - log.debug("downloaded %s in %0.2f", key, time.time()-st) + log.debug("downloaded %s in %0.2f", key, time.time() - st) t = time.time() subprocess.check_call(["lbzip2", "-d", fh.name]) - log.debug("decompressed %s in %0.2f", fh.name, time.time()-t) + log.debug("decompressed %s in %0.2f", fh.name, time.time() - t) t = time.time() since = incremental and day or None @@ -225,15 +225,15 @@ def index_account(config, region, account, day, incremental): record_count = process_traildb( rdb.create_engine("sqlite:////%s" % fh.name[:-4]), influx, name, region, since) - log.debug("indexed %s in %0.2f", fh.name, time.time()-t) + log.debug("indexed %s in %0.2f", fh.name, time.time() - t) os.remove(fh.name[:-4]) log.debug("account:%s day:%s region:%s records:%d complete:%0.2f", name, day.strftime("%Y-%m-%d"), region, record_count, - time.time()-st) + time.time() - st) - return {'time': time.time()-st, 'records': record_count, 'region': region, - 'account': name, 'day': day.strftime("%Y-%m-%d"), + return {'time': time.time() - st, 'records': record_count, 'region': region, + 'account': name, 'day': day.strftime("%Y-%m-%d"), 'db-date': key_info['LastModified']} @@ -254,7 +254,7 @@ def get_date_range(start, end): days = [] n_start = start.replace(hour=0, minute=0, second=0, microsecond=0) - for n in range(1, (end-n_start).days): + for n in range(1, (end - n_start).days): days.append(n_start + datetime.timedelta(n)) days.insert(0, start) if n_start != end: @@ -398,5 +398,6 @@ def index(config, start, end, incremental=False, concurrency=5, accounts=None, "records:%(records)s time:%(time)0.2f db-date:%(db-date)s" ) % result) + if __name__ == '__main__': trailts(auto_envvar_prefix='TRAIL') diff --git a/tools/dev/license-headers.py b/tools/dev/license-headers.py index 90ed349e807..95ef4132b04 100644 --- a/tools/dev/license-headers.py +++ b/tools/dev/license-headers.py @@ -57,12 +57,13 @@ def update_headers(src_tree): fh.write( '%s%s%s' % (header, suffix, contents)) + def main(): srctree = os.path.dirname(inspect.getabsfile(c7n)) update_headers(srctree) update_headers(os.path.abspath('tests')) update_headers(os.path.abspath('ftests')) - + if __name__ == '__main__': main() diff --git a/tools/ops/logsetup.py b/tools/ops/logsetup.py index 4def3e3c8e7..f7c39238d91 100644 --- a/tools/ops/logsetup.py +++ b/tools/ops/logsetup.py @@ -104,4 +104,3 @@ def main(): if __name__ == '__main__': main() - diff --git a/tools/ops/mugc.py b/tools/ops/mugc.py index 1f805b444aa..8c7a69dd5bc 100644 --- a/tools/ops/mugc.py +++ b/tools/ops/mugc.py @@ -122,7 +122,5 @@ def main(): resources_gc_prefix(options, policies) - - if __name__ == '__main__': main() diff --git a/tools/ops/policyrename.py b/tools/ops/policyrename.py index 8da538f1fa8..9c113fd82bf 100644 --- a/tools/ops/policyrename.py +++ b/tools/ops/policyrename.py @@ -80,7 +80,7 @@ def s3_rename(output_dir, old, new, sse_kms_key_id): if page.get('Contents') is None: raise ArgumentError('Key {} does not exist in bucket {}'.format( - old, bucket)) + old, bucket)) # Loop through the old objects copying and deleting for obj in page.get('Contents'): diff --git a/tools/utils/ec2recovery.py b/tools/utils/ec2recovery.py deleted file mode 100644 index 98a49c6ebc6..00000000000 --- a/tools/utils/ec2recovery.py +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright 2016 Capital One Services, LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from datetime import datetime -from boto3.session import Session -from botocore.exceptions import ClientError -import argparse -import sys -import os - - -def configure_subparser(subparser): - subparser.add_argument('--dryrun', action='store_true', - default=False, help='Set DryRun option') - subparser.add_argument('--accesskey', help='AWS Access Key') - subparser.add_argument('--secretkey', help='AWS Access Key Secret') - subparser.add_argument('--region', default='us-east-1', - help='AWS Region') - subparser.add_argument('--profile', help='AWS CLI Profile Name') - subparser.add_argument('--instanceid', help='EC2 Instance ID') - - -def generate_argparser(): - parser = argparse.ArgumentParser() - subs = parser.add_subparsers() - lists = subs.add_parser('list', - description='List snapshot for a ' - 'given ec2 instance') - lists.set_defaults(which='list') - configure_subparser(lists) - - restore = subs.add_parser('restore', - description='Restore EBS volume(s) from ' - 'snapshot(s)') - restore.set_defaults(which='restore') - configure_subparser(restore) - restore.add_argument('--snapshots', - help='Comma-delimited string of snapshot ' - 'Ids to restore volumes from') - - rebuild = subs.add_parser('rebuild', - description='Rebuild an EC2 ' - 'instance from snapshot(s)') - rebuild.set_defaults(which='rebuild') - configure_subparser(rebuild) - rebuild.add_argument('--ami', required=True, - help='AMI ID to restore from ' - '(required when performing full restore') - rebuild.add_argument('--snapshot', action='append', required=True, - help='Comma-delimited string of' - ' snapshot IDs to restore') - rebuild.add_argument('--sg', action='append', required=True, - help='Comma-delimited string of security' - ' groups to which the new instance will' - ' be associated with') - rebuild.add_argument('--subnet', required=True, - help='Subnet ID to associated to instance') - rebuild.add_argument('--role', default='', - help='IAM Role name to associate to instance') - rebuild.add_argument('--keypair', default='', - help='Name of SSH KeyPair to associate to instance') - rebuild.add_argument('--type', help='EC2 instance type (size)') - rebuild.add_argument('--userdata', default='', - help='Path to userdata script to run on new instance') - return parser - - -def validate_snapshots(dryrun, client, snapshots): - if not snapshots: - print "[Error] No snapshot(s) specified." - sys.exit(-1) - results = [] - try: - c = client.describe_snapshots( - DryRun=dryrun, - SnapshotIds=snapshots.split(','))['Snapshots'] - for s in c: - for tag in s['Tags']: - if tag['Key'] == 'Encrypted': - if tag['Value']: - continue - if 'DeviceName' in tag['Key']: - volume = { - "DeviceName": tag['Value'], - "Ebs": { - "SnapshotId": s['SnapshotId'] - } - } - results.append(volume) - return results - except ClientError as e: - print "[Error]: %s" % e.response['Error']['Message'] - sys.exit(-1) - - -def list_snapshots(session, instance): - snapshots = session.client('ec2').describe_snapshots( - Filters=[{ - 'Name': 'description', - 'Values': ["Automated,Backup,%s,*" % ( - instance)]}])['Snapshots'] - snapshot_map = {} - for snapshot in snapshots: - start_date = snapshot['StartTime'].date() - snapshot_id = snapshot['SnapshotId'] - volume_id = snapshot['VolumeId'] - if not start_date in snapshot_map: - snapshot_map[start_date] = [] - for tag in snapshot['Tags']: - if 'DeviceName' in tag['Key']: - snapshot_map[start_date].append([ - volume_id, tag['Value'], - snapshot_id]) - for snapshot in snapshot_map: - print "\n\tSnapshot Date: %s" % snapshot - for value in snapshot_map[snapshot]: - print "\t - VolumeId: %s (%s), SnapshotId: %s" % ( - value[0], value[1], value[2]) - - -def read_user_date(userdata): - if not os.path.isfile(userdata): - print "Cannot find file '%s'" % userdata - return "" - with open (userdata, "r") as input: - return input.readlines() - - -def excessive_volume_waiter(r, volumeid, status): - """ - Using this method to wait for a volume to detatch as the existing - waiters are continuously timing out - """ - import time - while r.Volume(volumeid).state != status: - time.sleep(15) - - -def validate_snapshot(snapshot): - results = {} - for tag in snapshot.tags: - if tag['Key'] == 'DeviceName': - results['DeviceName'] = tag['Value'] - if tag['Key'] == 'Name' or tag['Key'] == 'VolumeId': - results['VolumeId'] = tag['Value'] - if tag['Key'] == 'AvailabilityZone': - results['AvailabilityZone'] = tag['Value'] - return results - - -def restore_volume(dryrun, session, instanceid, snapshots): - r = session.resource('ec2') - instance = r.Instance(instanceid) - if instance.state['Name'] == 'running': - instance.stop() - session.client('ec2').get_waiter('instance_stopped').wait( - DryRun=dryrun, InstanceIds=[instanceid]) - for snapshot in snapshots: - snap = r.Snapshot(snapshot) - tags = [t for t in snap.tags if 'aws:' not in t['Key']] - values = validate_snapshot(snap) - if not values: - continue - devname = values['DeviceName'] - volid = values['VolumeId'] - vol = r.Volume(volid) - if not 'AvailabilityZone' in values: - az = vol.availability_zone - tags.append({'Key': 'AvailabilityZone', 'Value': az}) - else: - az = values['AvailabilityZone'] - - waiter = session.client('ec2').get_waiter('volume_available') - for d in instance.block_device_mappings: - if d['DeviceName'] != devname: - continue - instance.detach_volume( - DryRun=dryrun, VolumeId=d['Ebs']['VolumeId'], Force=True) - excessive_volume_waiter(r, d['Ebs']['VolumeId'], 'available') - - newvol = r.create_volume( - DryRun=dryrun, SnapshotId=snapshot, AvailabilityZone=az) - waiter.wait(DryRun=dryrun, VolumeIds=[newvol.volume_id]) - - waiter = session.client('ec2').get_waiter('volume_in_use') - r.Volume(newvol.volume_id).attach_to_instance( - DryRun=dryrun, InstanceId=instanceid, Device=devname) - waiter.wait(DryRun=dryrun, VolumeIds=[newvol.volume_id]) - - if tags: - r.Volume(newvol.volume_id).create_tags( - DryRun=dryrun,Tags=tags) - instance.start() - waiter = session.client('ec2').get_waiter('instance_running') - waiter.wait(DryRun=dryrun, InstanceIds=[instanceid]) - - -def copy_instance_tags(session, oldid, newid): - try: - r = session.resource('ec2') - oldec2 = r.Instance(oldid) - tags = [t for t in oldec2.tags if 'aws:' not in t['Key']] - tags.append({ - 'Key': 'RecoveryDate', - 'Value': datetime.now().strftime('%m/%d/%Y')}) - r.Instance(newid).create_tags(Tags=tags) - except: - return - - -def copy_tags(session, oldid, newid): - try: - c = session.client('ec2') - tags = c.describe_tags( - Filters=[{ - 'Name': 'resource-id', - 'Values': [oldid]}])['Tags'] - tags = [t for t in oval.tags if 'aws:' not in t['Key']] - tags.append({ - 'Key': 'RecoveryDate', - 'Value': datetime.now().strftime('%m/%d/%Y')}) - c.create_tags( - Resources=[newid], - Tags=tags) - except: - return False - - -def rebuild_instance(dryrun, session, ami, instanceid, keypair, sgs, type, - subnet, snapshots, role, userdata): - - try: - c = session.client('ec2') - instance = c.run_instances( - DryRun=dryrun, - ImageId=ami, - MinCount=1, - MaxCount=1, - KeyName=keypair, - SecurityGroupIds=sgs, - InstanceType=type, - SubnetId=subnet, - IamInstanceProfile={'Name': role}, - UserData=userdata)['Instances'][0] - copy_instance_tags(session, instanceid, instance['InstanceId']) - - if instanceid: - copy_tags(session, instanceid, instance['InstanceId']) - else: - copy_tags(session, snapshots[0], instance['InstanceId']) - - c.get_waiter('instance_running').wait( - DryRun=dryrun, InstanceIds=[instance['InstanceId']]) - - c.stop_instances( - DryRun=dryrun, InstanceIds=[instance['InstanceId']]) - c.get_waiter('instance_stopped').wait( - DryRun=dryrun, InstanceIds=[instance['InstanceId']]) - - print "New instance created. Instance Id: %s (%s)" % ( - instance['InstanceId'], instance['PrivateIpAddress']) - print "Restoring volumes..." - restore_volume(dryrun, session, instance['InstanceId'], snapshots) - except ClientError as e: - print e.response['Error']['Message'] - - -def main(): - parser = generate_argparser() - args = parser.parse_args() - if args.profile: - session = Session(profile_name=args.profile, region_name=args.region) - elif args.accesskey and args.secretkey: - session = boto3.Session( - session = Session( - aws_access_key_id=args.accesskey, - aws_secret_access_key=args.secretkey, - region_name=args.region) - else: - return - - if args.which == 'list': - list_snapshots(session, args.instanceid) - elif args.which == 'restore': - restore_volume(args.dryrun, session, args.instanceid, args.snapshots) - elif args.which == 'rebuild': - rebuild_instance(args.dryrun, session, args.ami, args.instanceid, - args.keypair, args.sg, args.type, args.subnet, - args.snapshot, args.role, args.userdata) - - -if __name__ == '__main__': - main() diff --git a/tools/utils/s3stat.py b/tools/utils/s3stat.py index bb28a2f7e0a..f0c2ce63418 100644 --- a/tools/utils/s3stat.py +++ b/tools/utils/s3stat.py @@ -13,66 +13,63 @@ # limitations under the License. from datetime import datetime, timedelta -import json -import logging -import os -from botocore.exceptions import ClientError import boto3 +import json +import logging def bucket_info(c, bucket): result = {'Bucket': bucket} response = c.get_metric_statistics( - Namespace='AWS/S3', - MetricName='NumberOfObjects', - Dimensions=[ - {'Name': 'BucketName', + Namespace='AWS/S3', + MetricName='NumberOfObjects', + Dimensions=[ + {'Name': 'BucketName', 'Value': bucket}, - {'Name': 'StorageType', - 'Value': 'AllStorageTypes'} - ], - StartTime=datetime.now().replace( - hour=0, minute=0, second=0, microsecond=0) - timedelta(1), - EndTime=datetime.now().replace( - hour=0, minute=0, second=0, microsecond=0), - Period=60*24*24, - Statistics=['Average']) + {'Name': 'StorageType', + 'Value': 'AllStorageTypes'} + ], + StartTime=datetime.now().replace( + hour=0, minute=0, second=0, microsecond=0) - timedelta(1), + EndTime=datetime.now().replace( + hour=0, minute=0, second=0, microsecond=0), + Period=60 * 24 * 24, + Statistics=['Average']) if not response['Datapoints']: - result['ObjectCount'] = 0 + result['ObjectCount'] = 0 else: - result['ObjectCount'] = response['Datapoints'][0]['Average'] + result['ObjectCount'] = response['Datapoints'][0]['Average'] response = c.get_metric_statistics( - Namespace='AWS/S3', - MetricName='BucketSizeBytes', - Dimensions=[ - {'Name': 'BucketName', + Namespace='AWS/S3', + MetricName='BucketSizeBytes', + Dimensions=[ + {'Name': 'BucketName', 'Value': bucket}, - {'Name': 'StorageType', - 'Value': 'StandardStorage'}, - ], - StartTime=datetime.now().replace( - hour=0, minute=0, second=0, microsecond=0) - timedelta(10), - EndTime=datetime.now().replace( - hour=0, minute=0, second=0, microsecond=0), - Period=60*24*24, - Statistics=['Average']) + {'Name': 'StorageType', + 'Value': 'StandardStorage'}, + ], + StartTime=datetime.now().replace( + hour=0, minute=0, second=0, microsecond=0) - timedelta(10), + EndTime=datetime.now().replace( + hour=0, minute=0, second=0, microsecond=0), + Period=60 * 24 * 24, + Statistics=['Average']) if not response['Datapoints']: - result['Size'] = 0 - result['SizeGB'] = 0 + result['Size'] = 0 + result['SizeGB'] = 0 else: - result['Size'] = response['Datapoints'][0]['Average'] - result['SizeGB'] = result['Size'] / (1024.0 * 1024 * 1024) + result['Size'] = response['Datapoints'][0]['Average'] + result['SizeGB'] = result['Size'] / (1024.0 * 1024 * 1024) return result def main(): logging.basicConfig(level=logging.INFO) - bucket = os.environ.get('BUCKET') results = {'buckets':[]} size_count = obj_count = 0.0 s = boto3.Session() @@ -91,7 +88,7 @@ def main(): # special case per https://goo.gl/iXdpnl elif bucket_region == "EU": bucket_region = "eu-west-1" - except ClientError as e: + except: # We don't have permission to the bucket, try us-east-1 bucket_region = "us-east-1" @@ -111,6 +108,6 @@ def main(): results['TotalSizeGB'] = size_count print(json.dumps(results, indent=2)) -if __name__ == '__main__': - main() +if __name__ == '__main__': + main()