diff --git a/check_kernel_commits.py b/check_kernel_commits.py index 4cd8198..2240dc9 100644 --- a/check_kernel_commits.py +++ b/check_kernel_commits.py @@ -8,19 +8,18 @@ import textwrap from typing import Optional - -def run_git(repo, args): - """Run a git command in the given repository and return its output as a string.""" - result = subprocess.run(["git", "-C", repo] + args, text=True, capture_output=True, check=False) - if result.returncode != 0: - raise RuntimeError(f"Git command failed: {' '.join(args)}\n{result.stderr}") - return result.stdout +from ciq_helpers import ( + CIQ_find_fixes_in_mainline, + CIQ_get_commit_body, + CIQ_hash_exists_in_ref, + CIQ_run_git, +) def ref_exists(repo, ref): """Return True if the given ref exists in the repository, False otherwise.""" try: - run_git(repo, ["rev-parse", "--verify", "--quiet", ref]) + CIQ_run_git(repo, ["rev-parse", "--verify", "--quiet", ref]) return True except RuntimeError: return False @@ -28,18 +27,13 @@ def ref_exists(repo, ref): def get_pr_commits(repo, pr_branch, base_branch): """Get a list of commit SHAs that are in the PR branch but not in the base branch.""" - output = run_git(repo, ["rev-list", f"{base_branch}..{pr_branch}"]) + output = CIQ_run_git(repo, ["rev-list", f"{base_branch}..{pr_branch}"]) return output.strip().splitlines() -def get_commit_message(repo, sha): - """Get the commit message for a given commit SHA.""" - return run_git(repo, ["log", "-n", "1", "--format=%B", sha]) - - def get_short_hash_and_subject(repo, sha): """Get the abbreviated commit hash and subject for a given commit SHA.""" - output = run_git(repo, ["log", "-n", "1", "--format=%h%x00%s", sha]).strip() + output = CIQ_run_git(repo, ["log", "-n", "1", "--format=%h%x00%s", sha]).strip() short_hash, subject = output.split("\x00", 1) return short_hash, subject @@ -48,61 +42,8 @@ def hash_exists_in_mainline(repo, upstream_ref, hash_): """ Return True if hash_ is reachable from upstream_ref (i.e., is an ancestor of it). """ - try: - run_git(repo, ["merge-base", "--is-ancestor", hash_, upstream_ref]) - return True - except RuntimeError: - return False - - -def find_fixes_in_mainline(repo, pr_branch, upstream_ref, hash_): - """ - Return unique commits in upstream_ref that have Fixes: in their message, case-insensitive. - Start from 12 chars and work down to 6, but do not include duplicates if already found at a longer length. - Returns a list of tuples: (full_hash, display_string) - """ - results = [] - # Get all commits with 'Fixes:' in the message - output = run_git(repo, ["log", upstream_ref, "--grep", "Fixes:", "-i", "--format=%H %h %s (%an)%x0a%B%x00"]).strip() - if not output: - return [] - # Each commit is separated by a NUL character and a newline - commits = output.split("\x00\x0a") - # Prepare hash prefixes from 12 down to 6 - hash_prefixes = [hash_[:index] for index in range(12, 5, -1)] - for commit in commits: - if not commit.strip(): - continue - # The first line is the summary, the rest is the body - lines = commit.splitlines() - if not lines: - continue - header = lines[0] - full_hash = header.split()[0] - # Search for Fixes: lines in the commit message - for line in lines[1:]: - m = re.match(r"^\s*Fixes:\s*([0-9a-fA-F]{6,40})", line, re.IGNORECASE) - if m: - for prefix in hash_prefixes: - if m.group(1).lower().startswith(prefix.lower()): - if not commit_exists_in_branch(repo, pr_branch, full_hash): - results.append((full_hash, " ".join(header.split()[1:]))) - break - else: - continue - return results - - -def commit_exists_in_branch(repo, pr_branch, upstream_hash_): - """ - Return True if upstream_hash_ has been backported and it exists in the - pr branch - """ - output = run_git(repo, ["log", pr_branch, "--grep", "commit " + upstream_hash_]) - if not output: - return False - return True + return CIQ_hash_exists_in_ref(repo, upstream_ref, hash_) def wrap_paragraph(text, width=80, initial_indent="", subsequent_indent=""): @@ -176,7 +117,7 @@ def main(): if os.path.exists(vulns_repo): # Repository exists, update it with git pull try: - run_git(vulns_repo, ["pull"]) + CIQ_run_git(vulns_repo, ["pull"]) except RuntimeError as e: print(f"WARNING: Failed to update vulns repo: {e}") print("Continuing with existing repository...") @@ -222,7 +163,7 @@ def main(): for sha in reversed(pr_commits): # oldest first short_hash, subject = get_short_hash_and_subject(args.repo, sha) pr_commit_desc = f"{short_hash} ({subject})" - msg = get_commit_message(args.repo, sha) + msg = CIQ_get_commit_body(args.repo, sha) upstream_hashes = re.findall(r"^commit\s+([0-9a-fA-F]{40})", msg, re.MULTILINE) for uhash in upstream_hashes: short_uhash = uhash[:12] @@ -248,7 +189,7 @@ def main(): ) out_lines.append("") # blank line continue - fixes = find_fixes_in_mainline(args.repo, args.pr_branch, upstream_ref, uhash) + fixes = CIQ_find_fixes_in_mainline(args.repo, args.pr_branch, upstream_ref, uhash) if fixes: any_findings = True diff --git a/ciq-cherry-pick.py b/ciq-cherry-pick.py index 9dded99..03b582a 100644 --- a/ciq-cherry-pick.py +++ b/ciq-cherry-pick.py @@ -1,19 +1,185 @@ import argparse +import logging import os +import re import subprocess +import traceback import git -from ciq_helpers import CIQ_cherry_pick_commit_standardization, CIQ_original_commit_author_to_tag_string - -# from ciq_helpers import * +from ciq_helpers import ( + CIQ_cherry_pick_commit_standardization, + CIQ_commit_exists_in_current_branch, + CIQ_find_fixes_in_mainline_current_branch, + CIQ_fixes_references, + CIQ_get_full_hash, + CIQ_original_commit_author_to_tag_string, + CIQ_raise_or_warn, + CIQ_reset_HEAD, + CIQ_run_git, +) MERGE_MSG = git.Repo(os.getcwd()).git_dir + "/MERGE_MSG" +MERGE_MSG_BAK = f"{MERGE_MSG}.bak" + + +def check_fixes(sha, ignore_fixes_check): + """ + Checks if commit has "Fixes:" references and if so, it checks if the + commit(s) that it tries to fix are part of the current branch + """ + + fixes = CIQ_fixes_references(repo_path=os.getcwd(), sha=sha) + if len(fixes) == 0: + logging.warning("The commit you try to cherry pick has no Fixes: reference; review it carefully") + return + + not_present_fixes = [] + for fix in fixes: + if not CIQ_commit_exists_in_current_branch(os.getcwd(), fix): + not_present_fixes.append(fix) + + err = f"The commit you want to cherry pick has the following Fixes: references that are not part of the tree {not_present_fixes}" + CIQ_raise_or_warn(cond=not not_present_fixes, error_msg=err, warn=ignore_fixes_check) + + +def manage_commit_message(full_sha, ciq_tags, jira_ticket, commit_successful): + """ + It standardize the commit message by including the ciq_tags, original + author and the original commit full sha. + + Original message location: MERGE_MSG + Makes a copy of the original message in MERGE_MSG_BAK + + The new standardized commit message is written to MERGE_MSG + """ + + subprocess.run(["cp", MERGE_MSG, MERGE_MSG_BAK], check=True) + + # Make sure it's a deep copy because ciq_tags may be used for other cherry-picks + new_tags = [tag for tag in ciq_tags] + + author = CIQ_original_commit_author_to_tag_string(repo_path=os.getcwd(), sha=full_sha) + if author is None: + raise RuntimeError(f"Could not find author of commit {full_sha}") + + new_tags.append(author) + try: + with open(MERGE_MSG, "r") as file: + original_msg = file.readlines() + except IOError as e: + raise RuntimeError(f"Failed to read commit message from {MERGE_MSG}: {e}") from e + + optional_msg = "" if commit_successful else "upstream-diff |" + new_msg = CIQ_cherry_pick_commit_standardization( + original_msg, full_sha, jira=jira_ticket, tags=new_tags, optional_msg=optional_msg + ) + + print(f"Cherry Pick New Message for {full_sha}") + print(f"\n Original Message located here: {MERGE_MSG_BAK}") + + try: + with open(MERGE_MSG, "w") as file: + file.writelines(new_msg) + except IOError as e: + raise RuntimeError(f"Failed to write commit message to {MERGE_MSG}: {e}") from e + + +def cherry_pick(sha, ciq_tags, jira_ticket, ignore_fixes_check): + """ + Cherry picks a commit and it adds the ciq standardized format + In case of error (cherry pick conflict): + - MERGE_MSG.bak contains the original commit message + - MERGE_MSG contains the standardized commit message + - Conflict has to be solved manually + In case runtime errors that are not cherry pick conflicts, the cherry + pick changes are reverted. (git reset --hard HEAD) + + In case of success: + - the commit is cherry picked + - MERGE_MSG.bak is deleted + - You can still see MERGE_MSG for the original message + """ + + # Expand the provided SHA1 to the full SHA1 in case it's either abbreviated or an expression + try: + full_sha = CIQ_get_full_hash(repo=os.getcwd(), short_hash=sha) + except RuntimeError as e: + raise RuntimeError(f"Invalid commit SHA {sha}: {e}") from e + + check_fixes(sha=full_sha, ignore_fixes_check=ignore_fixes_check) + + # Commit message is in MERGE_MSG + commit_successful = True + try: + CIQ_run_git(repo_path=os.getcwd(), args=["cherry-pick", "-nsx", full_sha]) + except RuntimeError: + commit_successful = False + + try: + manage_commit_message( + full_sha=full_sha, ciq_tags=ciq_tags, jira_ticket=jira_ticket, commit_successful=commit_successful + ) + except RuntimeError as e: + CIQ_reset_HEAD(repo=os.getcwd()) + raise RuntimeError(f"Could not create proper commit message: {e}") from e + + if not commit_successful: + error_str = ( + f"[FAILED] git cherry-pick -nsx {full_sha}\n" + "Manually resolve conflict and add explanation under `upstream-diff` tag in commit message\n" + ) + raise RuntimeError(error_str) + + CIQ_run_git(repo_path=os.getcwd(), args=["commit", "-F", MERGE_MSG]) + + +def cherry_pick_fixes(sha, ciq_tags, jira_ticket, upstream_ref, ignore_fixes_check): + """ + It checks upstream_ref for commits that have this reference: + Fixes: . If any, these will also be cherry picked with the ciq + tag = cve-bf. If the tag was cve-pre, it stays the same. + """ + fixes_in_mainline = CIQ_find_fixes_in_mainline_current_branch(os.getcwd(), upstream_ref, sha) + + # Replace cve with cve-bf + # Leave cve-pre and cve-bf as they are + bf_ciq_tags = [re.sub(r"^cve ", "cve-bf ", s) for s in ciq_tags] + for full_hash, display_str in fixes_in_mainline: + print(f"Extra cherry picking {display_str}") + full_cherry_pick( + sha=full_hash, + ciq_tags=bf_ciq_tags, + jira_ticket=jira_ticket, + upstream_ref=upstream_ref, + ignore_fixes_check=ignore_fixes_check, + ) + + +def full_cherry_pick(sha, ciq_tags, jira_ticket, upstream_ref, ignore_fixes_check): + """ + It cherry picks a commit from upstream-ref along with its Fixes: references. + If cherry-pick or cherry_pick_fixes fail, the exception is propagated + If one of the cherry picks fails, an exception is returned and the previous + successful cherry picks are left as they are. + """ + # Cherry pick the commit + cherry_pick(sha=sha, ciq_tags=ciq_tags, jira_ticket=jira_ticket, ignore_fixes_check=ignore_fixes_check) + + # Cherry pick the fixed-by dependencies + cherry_pick_fixes( + sha=sha, + ciq_tags=ciq_tags, + jira_ticket=jira_ticket, + upstream_ref=upstream_ref, + ignore_fixes_check=ignore_fixes_check, + ) + if __name__ == "__main__": print("CIQ custom cherry picker") parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument("--sha", help="Target SHA1 to cherry-pick") + parser.add_argument("--sha", help="Target SHA1 to cherry-pick", required=True) parser.add_argument("--ticket", help="Ticket associated to cherry-pick work, comma separated list is supported.") parser.add_argument( "--ciq-tag", @@ -23,51 +189,34 @@ " cve-pre CVE-1974-0001 - A pre-condition or dependency needed for the CVE\n" "Multiple tags are separated with a comma. ex: cve CVE-1974-0001, cve CVE-1974-0002\n", ) + parser.add_argument( + "--upstream-ref", + default="origin/kernel-mainline", + help="Reference to upstream mainline branch (default: origin/kernel-mainline)", + ) + parser.add_argument( + "--ignore-fixes-check", + action="store_true", + help="If the commit(s) this commit is trying to fix are not part of the tree, do not exit", + ) + args = parser.parse_args() - # Expand the provided SHA1 to the full SHA1 in case it's either abbreviated or an expression - git_sha_res = subprocess.run(["git", "show", "--pretty=%H", "-s", args.sha], stdout=subprocess.PIPE) - if git_sha_res.returncode != 0: - print(f"[FAILED] git show --pretty=%H -s {args.sha}") - print("Subprocess Call:") - print(git_sha_res) - print("") - else: - args.sha = git_sha_res.stdout.decode("utf-8").strip() + logging.basicConfig(level=logging.INFO) tags = [] if args.ciq_tag is not None: tags = args.ciq_tag.split(",") - author = CIQ_original_commit_author_to_tag_string(repo_path=os.getcwd(), sha=args.sha) - if author is None: + try: + full_cherry_pick( + sha=args.sha, + ciq_tags=tags, + jira_ticket=args.ticket, + upstream_ref=args.upstream_ref, + ignore_fixes_check=args.ignore_fixes_check, + ) + except Exception as e: + print(f"full_cherry_pick failed {e}") + traceback.print_exc() exit(1) - - git_res = subprocess.run(["git", "cherry-pick", "-nsx", args.sha]) - if git_res.returncode != 0: - print(f"[FAILED] git cherry-pick -nsx {args.sha}") - print(" Manually resolve conflict and include `upstream-diff` tag in commit message") - print("Subprocess Call:") - print(git_res) - print("") - - print(os.getcwd()) - subprocess.run(["cp", MERGE_MSG, f"{MERGE_MSG}.bak"]) - - tags.append(author) - - with open(MERGE_MSG, "r") as file: - original_msg = file.readlines() - - new_msg = CIQ_cherry_pick_commit_standardization(original_msg, args.sha, jira=args.ticket, tags=tags) - - print(f"Cherry Pick New Message for {args.sha}") - for line in new_msg: - print(line.strip("\n")) - print(f"\n Original Message located here: {MERGE_MSG}.bak") - - with open(MERGE_MSG, "w") as file: - file.writelines(new_msg) - - if git_res.returncode == 0: - subprocess.run(["git", "commit", "-F", MERGE_MSG]) diff --git a/ciq_helpers.py b/ciq_helpers.py index e97f11d..4b58eb6 100644 --- a/ciq_helpers.py +++ b/ciq_helpers.py @@ -3,6 +3,7 @@ # CIQ Kernel Tools function library +import logging import os import re import subprocess @@ -169,6 +170,153 @@ def CIQ_original_commit_author_to_tag_string(repo_path, sha): return "commit-author " + git_auth_res.stdout.decode("utf-8").replace('"', "").strip() +def CIQ_run_git(repo_path, args): + """ + Run a git command in the given repository and return its output as a string. + """ + result = subprocess.run(["git", "-C", repo_path] + args, text=True, capture_output=True, check=False) + if result.returncode != 0: + raise RuntimeError(f"Git command failed: {' '.join(args)}\n{result.stderr}") + + return result.stdout + + +def CIQ_get_commit_body(repo_path, sha): + return CIQ_run_git(repo_path, ["show", "-s", sha, "--format=%B"]) + + +def CIQ_extract_fixes_references_from_commit_body_lines(lines): + fixes = [] + for line in lines: + m = re.match(r"^\s*Fixes:\s*([0-9a-fA-F]{6,40})", line, re.IGNORECASE) + if not m: + continue + + fixes.append(m.group(1)) + + return fixes + + +def CIQ_fixes_references(repo_path, sha): + """ + If commit message of sha contains lines like + Fixes: , this returns a list of , otherwise an empty list + """ + + commit_body = CIQ_get_commit_body(repo_path, sha) + return CIQ_extract_fixes_references_from_commit_body_lines(lines=commit_body.splitlines()) + + +def CIQ_get_full_hash(repo, short_hash): + return CIQ_run_git(repo, ["show", "-s", "--pretty=%H", short_hash]).strip() + + +def CIQ_get_current_branch(repo): + return CIQ_run_git(repo, ["branch", "--show-current"]).strip() + + +def CIQ_hash_exists_in_ref(repo, pr_ref, hash_): + """ + Return True if hash_ is reachable from pr_ref + """ + + try: + CIQ_run_git(repo, ["merge-base", "--is-ancestor", hash_, pr_ref]) + return True + except RuntimeError: + return False + + +def CIQ_commit_exists_in_branch(repo, pr_branch, upstream_hash_): + """ + Return True if upstream_hash_ has been backported and it exists in the pr branch + """ + + # First check if the commit has been backported by CIQ + output = CIQ_run_git(repo, ["log", pr_branch, "--grep", "^commit " + upstream_hash_]) + if output: + return True + + # If it was not backported by CIQ, maybe it came from upstream as it is + return CIQ_hash_exists_in_ref(repo, pr_branch, upstream_hash_) + + +def CIQ_commit_exists_in_current_branch(repo, upstream_hash_): + """ + Return True if upstream_hash_ has been backported and it exists in the current branch + """ + + current_branch = CIQ_get_current_branch(repo) + full_upstream_hash = CIQ_get_full_hash(repo, upstream_hash_) + + return CIQ_commit_exists_in_branch(repo, current_branch, full_upstream_hash) + + +def CIQ_find_fixes_in_mainline(repo, pr_branch, upstream_ref, hash_): + """ + Return unique commits in upstream_ref that have Fixes: in their message, case-insensitive, + if they have not been committed in the pr_branch. + Start from 12 chars and work down to 6, but do not include duplicates if already found at a longer length. + Returns a list of tuples: (full_hash, display_string) + """ + results = [] + + # Prepare hash prefixes from 12 down to 6 + hash_prefixes = [hash_[:index] for index in range(12, 5, -1)] + + # Get all commits with 'Fixes:' in the message + output = CIQ_run_git( + repo, + [ + "log", + upstream_ref, + "--grep", + "Fixes:", + "-i", + "--format=%H %h %s (%an)%x0a%B%x00", + ], + ).strip() + if not output: + return [] + + # Each commit is separated by a NUL character and a newline + commits = output.split("\x00\x0a") + for commit in commits: + if not commit.strip(): + continue + + lines = commit.splitlines() + # The first line is the summary, the rest is the body + header = lines[0] + full_hash, display_string = (lambda h: (h[0], " ".join(h[1:])))(header.split()) + fixes = CIQ_extract_fixes_references_from_commit_body_lines(lines=lines[1:]) + for fix in fixes: + for prefix in hash_prefixes: + if fix.lower().startswith(prefix.lower()): + if not CIQ_commit_exists_in_branch(repo, pr_branch, full_hash): + results.append((full_hash, display_string)) + break + + return results + + +def CIQ_find_fixes_in_mainline_current_branch(repo, upstream_ref, hash_): + current_branch = CIQ_get_current_branch(repo) + + return CIQ_find_fixes_in_mainline(repo, current_branch, upstream_ref, hash_) + + +def CIQ_reset_HEAD(repo): + return CIQ_run_git(repo=repo, args=["reset", "--hard", "HEAD"]) + + +def CIQ_raise_or_warn(cond, error_msg, warn): + if not warn: + raise RuntimeError(error_msg) + + logging.warning(error_msg) + + def repo_init(repo): """Initialize a git repo object.