diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..2b0cc318 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,10 @@ +sudo: false +language: python +python: + - 3.3 + - 3.4 + - 3.5 +install: + - pip install flake8 +script: + - flake8 --ignore E501 homu diff --git a/homu/git_helper.py b/homu/git_helper.py index 78b49786..d0b4bf08 100755 --- a/homu/git_helper.py +++ b/homu/git_helper.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 import sys -import subprocess import os SSH_KEY_FILE = os.path.join(os.path.dirname(__file__), '../cache/key') + def main(): args = ['ssh', '-i', SSH_KEY_FILE] + sys.argv[1:] os.execvp('ssh', args) diff --git a/homu/main.py b/homu/main.py index 5bb00fe7..8f252f8a 100644 --- a/homu/main.py +++ b/homu/main.py @@ -29,6 +29,7 @@ INTERRUPTED_BY_HOMU_FMT = 'Interrupted by Homu ({})' INTERRUPTED_BY_HOMU_RE = re.compile(r'Interrupted by Homu \((.+?)\)') + @contextmanager def buildbot_sess(repo_cfg): sess = requests.Session() @@ -43,10 +44,13 @@ def buildbot_sess(repo_cfg): sess.get(repo_cfg['buildbot']['url'] + '/logout', allow_redirects=False) db_query_lock = Lock() + + def db_query(db, *args): with db_query_lock: db.execute(*args) + class PullReqState: num = 0 priority = 0 @@ -214,17 +218,20 @@ def fake_merged(self, repo_cfg): if self.get_repo().merge(self.base_ref, self.head_sha, msg): self.rebased = True + def sha_cmp(short, full): return len(short) >= 4 and short == full[:len(short)] + def sha_or_blank(sha): return sha if re.match(r'^[0-9a-f]+$', sha) else '' + def parse_commands(body, username, repo_cfg, state, my_username, db, *, realtime=False, sha=''): try_only = False if username not in repo_cfg['reviewers'] and username != my_username: if username == state.delegate: - pass # Allow users who have been delegated review powers + pass # Allow users who have been delegated review powers elif username in repo_cfg.get('try_users', []): try_only = True else: @@ -238,11 +245,12 @@ def parse_commands(body, username, repo_cfg, state, my_username, db, *, realtime if word == 'r+' or word.startswith('r='): if try_only: - if realtime: state.add_comment(':key: Insufficient privileges') + if realtime: + state.add_comment(':key: Insufficient privileges') continue - if not sha and i+1 < len(words): - cur_sha = sha_or_blank(words[i+1]) + if not sha and i + 1 < len(words): + cur_sha = sha_or_blank(words[i + 1]) else: cur_sha = sha @@ -277,7 +285,8 @@ def parse_commands(body, username, repo_cfg, state, my_username, db, *, realtime elif word == 'r-': if try_only: - if realtime: state.add_comment(':key: Insufficient privileges') + if realtime: + state.add_comment(':key: Insufficient privileges') continue state.approved_by = '' @@ -285,20 +294,24 @@ def parse_commands(body, username, repo_cfg, state, my_username, db, *, realtime state.save() elif word.startswith('p='): - try: state.priority = int(word[len('p='):]) - except ValueError: pass + try: + state.priority = int(word[len('p='):]) + except ValueError: + pass state.save() elif word.startswith('delegate='): if try_only: - if realtime: state.add_comment(':key: Insufficient privileges') + if realtime: + state.add_comment(':key: Insufficient privileges') continue state.delegate = word[len('delegate='):] state.save() - if realtime: state.add_comment(':v: @{} can now approve this pull request'.format(state.delegate)) + if realtime: + state.add_comment(':v: @{} can now approve this pull request'.format(state.delegate)) elif word == 'delegate-': state.delegate = '' @@ -306,13 +319,15 @@ def parse_commands(body, username, repo_cfg, state, my_username, db, *, realtime elif word == 'delegate+': if try_only: - if realtime: state.add_comment(':key: Insufficient privileges') + if realtime: + state.add_comment(':key: Insufficient privileges') continue state.delegate = state.get_repo().pull_request(state.num).user.login state.save() - if realtime: state.add_comment(':v: @{} can now approve this pull request'.format(state.delegate)) + if realtime: + state.add_comment(':v: @{} can now approve this pull request'.format(state.delegate)) elif word == 'retry' and realtime: state.set_status('') @@ -343,7 +358,8 @@ def parse_commands(body, username, repo_cfg, state, my_username, db, *, realtime mat = re.search('(?s)
(.*?)
', res.text) if mat: err = mat.group(1).strip() - if not err: err = 'Unknown error' + if not err: + err = 'Unknown error' else: err = '' @@ -366,6 +382,7 @@ def parse_commands(body, username, repo_cfg, state, my_username, db, *, realtime return state_changed + def create_merge(state, repo_cfg, branch, git_cfg): base_sha = state.get_repo().ref('heads/' + state.base_ref).object.sha @@ -453,9 +470,11 @@ def create_merge(state, repo_cfg, branch, git_cfg): force=True, ) - try: merge_commit = state.get_repo().merge(branch, state.head_sha, merge_msg) + try: + merge_commit = state.get_repo().merge(branch, state.head_sha, merge_msg) except github3.models.GitHubError as e: - if e.code != 409: raise + if e.code != 409: + raise else: return merge_commit.sha if merge_commit else '' @@ -466,6 +485,7 @@ def create_merge(state, repo_cfg, branch, git_cfg): return '' + def start_build(state, repo_cfgs, buildbot_slots, logger, db, git_cfg): if buildbot_slots[0]: return True @@ -539,6 +559,7 @@ def start_build(state, repo_cfgs, buildbot_slots, logger, db, git_cfg): return True + def start_rebuild(state, repo_cfgs): repo_cfg = repo_cfgs[state.repo_label] @@ -603,12 +624,14 @@ def start_rebuild(state, repo_cfgs): return True + def start_build_or_rebuild(state, repo_cfgs, *args): if start_rebuild(state, repo_cfgs): return True return start_build(state, repo_cfgs, *args) + def process_queue(states, repos, repo_cfgs, logger, buildbot_slots, db, git_cfg): for repo_label, repo in repos.items(): repo_states = sorted(states[repo_label].values()) @@ -634,6 +657,7 @@ def process_queue(states, repos, repo_cfgs, logger, buildbot_slots, db, git_cfg) if start_build(state, repo_cfgs, buildbot_slots, logger, db, git_cfg): return + def fetch_mergeability(mergeable_que): re_pull_num = re.compile('(?i)merge (?:of|pull request) #([0-9]+)') @@ -650,8 +674,10 @@ def fetch_mergeability(mergeable_que): if cause: mat = re_pull_num.search(cause['title']) - if mat: issue_or_commit = '#' + mat.group(1) - else: issue_or_commit = cause['sha'][:7] + if mat: + issue_or_commit = '#' + mat.group(1) + else: + issue_or_commit = cause['sha'][:7] else: issue_or_commit = '' @@ -667,6 +693,7 @@ def fetch_mergeability(mergeable_que): finally: mergeable_que.task_done() + def synchronize(repo_label, repo_cfg, logger, gh, states, repos, db, mergeable_que, my_username, repo_labels): logger.info('Synchronizing {}...'.format(repo_label)) @@ -727,15 +754,17 @@ def synchronize(repo_label, repo_cfg, logger, gh, states, repos, db, mergeable_q logger.info('Done synchronizing {}!'.format(repo_label)) + def arguments(): - parser = argparse.ArgumentParser(description = - 'A bot that integrates with GitHub and ' - 'your favorite continuous integration service') + parser = argparse.ArgumentParser( + description='A bot that integrates with GitHub and your favorite ' + 'continuous integration service') parser.add_argument('-v', '--verbose', action='store_true', help='Enable more verbose logging') return parser.parse_args() + def main(): args = arguments() @@ -752,7 +781,8 @@ def main(): gh = github3.login(token=cfg['github']['access_token']) user = gh.user() - try: user_email = [x for x in gh.iter_emails() if x['primary']][0]['email'] + try: + user_email = [x for x in gh.iter_emails() if x['primary']][0]['email'] except IndexError: raise RuntimeError('Primary email not set, or "user" scope not granted') @@ -858,8 +888,10 @@ def main(): for repo_label, num, builder, res, url, merge_sha in db.fetchall(): try: state = states[repo_label][num] - if builder not in state.build_res: raise KeyError - if state.merge_sha != merge_sha: raise KeyError + if builder not in state.build_res: + raise KeyError + if state.merge_sha != merge_sha: + raise KeyError except KeyError: db_query(db, 'DELETE FROM build_res WHERE repo = ? AND num = ? AND builder = ?', [repo_label, num, builder]) continue @@ -871,7 +903,8 @@ def main(): db_query(db, 'SELECT repo, num, mergeable FROM mergeable') for repo_label, num, mergeable in db.fetchall(): - try: state = states[repo_label][num] + try: + state = states[repo_label][num] except KeyError: db_query(db, 'DELETE FROM mergeable WHERE repo = ? AND num = ?', [repo_label, num]) continue @@ -879,6 +912,7 @@ def main(): state.mergeable = bool(mergeable) if mergeable is not None else None queue_handler_lock = Lock() + def queue_handler(): with queue_handler_lock: return process_queue(states, repos, repo_cfgs, logger, buildbot_slots, db, git_cfg) diff --git a/homu/server.py b/homu/server.py index 4e768cb7..7beaad48 100644 --- a/homu/server.py +++ b/homu/server.py @@ -12,11 +12,15 @@ import hashlib from threading import Thread -import bottle; bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 * 10 +import bottle +bottle.BaseRequest.MEMFILE_MAX = 1024 * 1024 * 10 -class G: pass + +class G: + pass g = G() + def find_state(sha): for repo_label, repo_states in g.states.items(): for state in repo_states.values(): @@ -25,6 +29,7 @@ def find_state(sha): raise ValueError('Invalid SHA') + def get_repo(repo_label, repo_cfg): repo = g.repos[repo_label] if not repo: @@ -34,10 +39,12 @@ def get_repo(repo_label, repo_cfg): assert repo.name == repo_cfg['name'] return repo + @get('/') def index(): return g.tpls['index'].render(repos=sorted(g.repos)) + @get('/queue/') def queue(repo_label): logger = g.logger.getChild('queue') @@ -71,15 +78,16 @@ def queue(repo_label): }) return g.tpls['queue'].render( - repo_label = repo_label, - states = rows, - oauth_client_id = g.cfg['github']['app_client_id'], - total = len(pull_states), - approved = len([x for x in pull_states if x.approved_by]), - rolled_up = len([x for x in pull_states if x.rollup]), - failed = len([x for x in pull_states if x.status == 'failure' or x.status == 'error']), + repo_label=repo_label, + states=rows, + oauth_client_id=g.cfg['github']['app_client_id'], + total=len(pull_states), + approved=len([x for x in pull_states if x.approved_by]), + rolled_up=len([x for x in pull_states if x.rollup]), + failed=len([x for x in pull_states if x.status == 'failure' or x.status == 'error']), ) + @get('/callback') def callback(): logger = g.logger.getChild('callback') @@ -112,14 +120,17 @@ def callback(): else: abort(400, 'Invalid command') + def rollup(user_gh, state, repo_label, repo_cfg, repo): user_repo = user_gh.repository(user_gh.user().login, repo.name) base_repo = user_gh.repository(repo.owner.login, repo.name) nums = state.get('nums', []) if nums: - try: rollup_states = [g.states[repo_label][num] for num in nums] - except KeyError as e: return 'Invalid PR number: {}'.format(e.args[0]) + try: + rollup_states = [g.states[repo_label][num] for num in nums] + except KeyError as e: + return 'Invalid PR number: {}'.format(e.args[0]) else: rollup_states = [x for x in g.states[repo_label].values() if x.rollup] rollup_states = [x for x in rollup_states if x.approved_by] @@ -154,9 +165,11 @@ def rollup(user_gh, state, repo_label, repo_cfg, repo): state.body, ) - try: user_repo.merge(repo_cfg.get('branch', {}).get('rollup', 'rollup'), state.head_sha, merge_msg) + try: + user_repo.merge(repo_cfg.get('branch', {}).get('rollup', 'rollup'), state.head_sha, merge_msg) except github3.models.GitHubError as e: - if e.code != 409: raise + if e.code != 409: + raise failures.append(state.num) else: @@ -180,6 +193,7 @@ def rollup(user_gh, state, repo_label, repo_cfg, repo): else: redirect(pull.html_url) + @post('/github') def github(): logger = g.logger.getChild('github') @@ -348,7 +362,8 @@ def github(): g.queue_handler() elif event_type == 'status': - try: state, repo_label = find_state(info['sha']) + try: + state, repo_label = find_state(info['sha']) except ValueError: return 'OK' @@ -369,10 +384,11 @@ def github(): return 'OK' + def report_build_res(succ, url, builder, state, logger, repo_cfg): lazy_debug(logger, lambda: 'build result {}: builder = {}, succ = {}, current build_res = {}' - .format(state, builder, succ, state.build_res_summary())) + .format(state, builder, succ, state.build_res_summary())) state.set_build_res(builder, succ, url) @@ -412,6 +428,7 @@ def report_build_res(succ, url, builder, state, logger, repo_cfg): g.queue_handler() + @post('/buildbot') def buildbot(): logger = g.logger.getChild('buildbot') @@ -425,11 +442,14 @@ def buildbot(): info = row['payload']['build'] props = dict(x[:2] for x in info['properties']) - if 'retry' in info['text']: continue + if 'retry' in info['text']: + continue - if not props['revision']: continue + if not props['revision']: + continue - try: state, repo_label = find_state(props['revision']) + try: + state, repo_label = find_state(props['revision']) except ValueError: lazy_debug(logger, lambda: 'Invalid commit ID from Buildbot: {}'.format(props['revision'])) @@ -496,10 +516,13 @@ def buildbot(): info = row['payload']['build'] props = dict(x[:2] for x in info['properties']) - if not props['revision']: continue + if not props['revision']: + continue - try: state, repo_label = find_state(props['revision']) - except ValueError: pass + try: + state, repo_label = find_state(props['revision']) + except ValueError: + pass else: if info['builderName'] in state.build_res: repo_cfg = g.repo_cfgs[repo_label] @@ -522,6 +545,7 @@ def buildbot(): return 'OK' + @post('/travis') def travis(): logger = g.logger.getChild('travis') @@ -530,7 +554,8 @@ def travis(): lazy_debug(logger, lambda: 'info: {}'.format(utils.remove_url_keys_from_json(info))) - try: state, repo_label = find_state(info['commit']) + try: + state, repo_label = find_state(info['commit']) except ValueError: lazy_debug(logger, lambda: 'Invalid commit ID from Travis: {}'.format(info['commit'])) return 'OK' @@ -550,7 +575,7 @@ def travis(): # fabricating travis notifications to try to trick Homu, but, # I imagine that this will most often occur because a repo is # misconfigured. - logger.warn('authorization failed for {}, maybe the repo has the wrong travis token? ' \ + logger.warn('authorization failed for {}, maybe the repo has the wrong travis token? ' 'header = {}, computed = {}' .format(state, auth_header, code)) abort(400, 'Authorization failed') @@ -561,6 +586,7 @@ def travis(): return 'OK' + def synch(user_gh, state, repo_label, repo_cfg, repo): if not repo.is_collaborator(user_gh.user().login): abort(400, 'You are not a collaborator') @@ -569,6 +595,7 @@ def synch(user_gh, state, repo_label, repo_cfg, repo): return 'Synchronizing {}...'.format(repo_label) + @post('/admin') def admin(): if request.json['secret'] != g.cfg['web']['secret']: @@ -615,10 +642,11 @@ def admin(): return 'Unrecognized command' + def start(cfg, states, queue_handler, repo_cfgs, repos, logger, buildbot_slots, my_username, db, repo_labels, mergeable_que, gh): env = jinja2.Environment( - loader = jinja2.FileSystemLoader(pkg_resources.resource_filename(__name__, 'html')), - autoescape = True, + loader=jinja2.FileSystemLoader(pkg_resources.resource_filename(__name__, 'html')), + autoescape=True, ) tpls = {} tpls['index'] = env.get_template('index.html') diff --git a/homu/utils.py b/homu/utils.py index ac0085ca..3cee8dbb 100644 --- a/homu/utils.py +++ b/homu/utils.py @@ -2,13 +2,14 @@ import github3 import logging import subprocess -import sys + def github_set_ref(repo, ref, sha, *, force=False, auto_create=True): url = repo._build_url('git', 'refs', ref, base_url=repo._api) data = {'sha': sha, 'force': force} - try: js = repo._json(repo._patch(url, data=json.dumps(data)), 200) + try: + js = repo._json(repo._patch(url, data=json.dumps(data)), 200) except github3.models.GitHubError as e: if e.code == 422 and auto_create: return repo.create_ref('refs/' + ref, sha) @@ -17,16 +18,19 @@ def github_set_ref(repo, ref, sha, *, force=False, auto_create=True): return github3.git.Reference(js, repo) if js else None + class Status(github3.repos.status.Status): def __init__(self, info): super(Status, self).__init__(info) self.context = info.get('context') + def github_iter_statuses(repo, sha): url = repo._build_url('statuses', sha, base_url=repo._api) return repo._iter(-1, url, Status) + def github_create_status(repo, sha, state, target_url='', description='', *, context=''): data = {'state': state, 'target_url': target_url, @@ -35,6 +39,7 @@ def github_create_status(repo, sha, state, target_url='', description='', *, js = repo._json(repo._post(url, data=data), 201) return Status(js) if js else None + def remove_url_keys_from_json(json): if isinstance(json, dict): return {key: remove_url_keys_from_json(value) @@ -45,15 +50,19 @@ def remove_url_keys_from_json(json): else: return json + def lazy_debug(logger, f): if logger.isEnabledFor(logging.DEBUG): logger.debug(f()) + def logged_call(args): - try: subprocess.check_call(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - except subprocess.CalledProcessError as e: + try: + subprocess.check_call(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: print('* Failed to execute command: {}'.format(args)) raise + def silent_call(args): return subprocess.call(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)