From c2faf5d25794857ec2606c7341c43921c1935e52 Mon Sep 17 00:00:00 2001 From: Siddharth Date: Fri, 7 Feb 2025 18:28:47 +0530 Subject: [PATCH 1/2] refactor: fixed broken swe-bench docker image generation. - created new images with updated composio sdk - fixes: 1155 --- python/swe/agent/benchmark.py | 4 +- python/swe/dockerfiles/build.py | 158 ++++++++++++++++--------- python/swe/dockerfiles/create_index.py | 40 +++++-- python/tox.ini | 14 +-- 4 files changed, 141 insertions(+), 75 deletions(-) diff --git a/python/swe/agent/benchmark.py b/python/swe/agent/benchmark.py index 285ba8736da..cd35a985486 100644 --- a/python/swe/agent/benchmark.py +++ b/python/swe/agent/benchmark.py @@ -7,7 +7,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from typing import List -from langchain_aws import BedrockChat +from langchain_aws import ChatBedrock from langchain_core.messages import HumanMessage from langchain_openai import ChatOpenAI from langgraph.errors import GraphRecursionError @@ -41,7 +41,7 @@ def retry_with_exponential_backoff(func, *args, **kwargs): def get_llm_response(system_prompt: str, human_prompt: str) -> str: try: if MODEL == "claude": - client = BedrockChat( + client = ChatBedrock( credentials_profile_name="default", model_id="anthropic.claude-3-5-sonnet-20240620-v1:0", region_name="us-west-2", diff --git a/python/swe/dockerfiles/build.py b/python/swe/dockerfiles/build.py index 9cdc225d4a7..d9e60a0c8ca 100644 --- a/python/swe/dockerfiles/build.py +++ b/python/swe/dockerfiles/build.py @@ -1,4 +1,7 @@ +import signal import subprocess +import sys +import threading import typing as t from concurrent.futures import ThreadPoolExecutor from pathlib import Path @@ -9,7 +12,89 @@ logs = Path.cwd() / "logs" logs.mkdir(exist_ok=True) -errors = [] +# Lock for synchronizing access to the success log +success_lock = threading.Lock() +# Global set to keep track of successfully pushed images +successful_builds = set() + + +# Load successful builds from logs/success.log into the global set +def load_successful_builds() -> None: + success_file = logs / "success.log" + if success_file.exists(): + with open(success_file, "r", encoding="utf-8") as f: + for line in f: + tag = line.strip() + if tag: + successful_builds.add(tag) + + +# Record a successful build by writing to the log file and updating the global set +def record_success(tag: str) -> None: + with success_lock: + successful_builds.add(tag) + with open(logs / "success.log", "a", encoding="utf-8") as f: + f.write(tag + "\n") + + +# Insert new global variables and functions for graceful stop and resume support +stop_requested = False + + +def handle_stop(signum, frame): + global stop_requested + print("Received stop signal. Gracefully stopping new builds...") + stop_requested = True + + +def _base(generated: Path, multi: bool = False) -> None: + base = generated / "base" + with ThreadPoolExecutor() as executor: + futures = [] + for file in base.iterdir(): + if stop_requested: + print("Graceful stop activated. Halting base builds.") + break + try: + _, tag_part = file.name.split(".", maxsplit=1) + except ValueError: + print(f"Skipping invalid file name format: {file.name}") + continue + full_tag = f"composio/swe:{tag_part}" + if full_tag in successful_builds: + print(f"Skipping build for {full_tag} as it has been already pushed.") + continue + futures.append(executor.submit(_build, file, tag_part, multi)) + [fut.result() for fut in futures] + + +def _swes(generated: Path, multi: bool = False) -> None: + with ThreadPoolExecutor() as executor: + futures = [] + for child in generated.iterdir(): + if child.name == "base": + continue + if child.is_file(): + continue + repo = child.name.replace("__", "-") + for version in child.iterdir(): + if stop_requested: + print("Graceful stop activated. Halting SWES builds.") + break + tag_part = f"{repo}-{version.name.replace('.', '-') }" + full_tag = f"composio/swe:{tag_part}" + if full_tag in successful_builds: + print( + f"Skipping build for {full_tag} as it has been already pushed." + ) + continue + futures.append( + executor.submit(_build, version / "Dockerfile", tag_part, multi) + ) + if stop_requested: + break + [fut.result() for fut in futures] + ARCHS = ("linux/arm64", "linux/amd64") @@ -50,81 +135,44 @@ def _build(file: Path, tag: str, multi: bool, *flags: str) -> None: if process.returncode == 0: print(f"Finished build for {tag}") + record_success(tag) else: print(f"Error building {tag} - {logs / log}") - errors.append(f"Error building {tag} - {logs / log}") - - -def _base(generated: Path, multi: bool = False) -> None: - base = generated / "base" - with ThreadPoolExecutor() as executor: - futures = [] - for file in base.iterdir(): - _, tag = file.name.split(".", maxsplit=1) - futures.append(executor.submit(_build, file, tag, multi)) - _ = [fut.result() for fut in futures] - - -def _swes(generated: Path, multi: bool = False) -> None: - with ThreadPoolExecutor() as executor: - futures = [] - for child in generated.iterdir(): - if child.name == "base": - continue - - if child.is_file(): - continue - - repo = child.name.replace("__", "-") - for version in child.iterdir(): - tag = f"{repo}-{version.name.replace('.', '-')}" - futures.append( - executor.submit( - _build, - version / "Dockerfile", - tag, - multi, - ) - ) - - _ = [fut.result() for fut in futures] def _pyenv(file: t.Optional[Path] = None, multi: bool = False) -> None: print("Print building pyenv base") file = file or Path(__file__).parent / "templates" / "Dockerfile.pyenv" + full_tag = "composio/swe:pyenv" + if full_tag in successful_builds: + print(f"Skipping build for {full_tag} as it has already been pushed.") + return _build(file=file, tag="pyenv", multi=multi) @click.command(name="build") -@click.argument( - "generated", - type=str, - default="./generated", -) +@click.argument("generated", type=str, default="./generated") @click.option( - "--multi", - is_flag=True, - help="Use this flag to build multi-plaform images", + "--multi", is_flag=True, help="Use this flag to build multi-plaform images" ) def build(generated: Path, multi: bool = False) -> None: """Build docker images for SWEKIT.""" + load_successful_builds() + signal.signal(signal.SIGINT, handle_stop) + signal.signal(signal.SIGTERM, handle_stop) + _pyenv(multi=multi) - if len(errors) > 0: - print("==== Errors ====") - print("\n".join(errors)) - return + print("==== Successful Builds (after pyenv) ====") + print("\n".join(successful_builds)) generated = Path(generated or Path.cwd() / "generated").resolve() _base(generated=generated, multi=multi) - if len(errors) > 0: - print("==== Errors ====") - print("\n".join(errors)) - return + print("==== Successful Builds (after base) ====") + print("\n".join(successful_builds)) _swes(generated=generated, multi=multi) - print("==== Errors ====") - print("\n".join(errors)) + print("==== Final Successful Builds ====") + print("\n".join(successful_builds)) if __name__ == "__main__": diff --git a/python/swe/dockerfiles/create_index.py b/python/swe/dockerfiles/create_index.py index c283478b7dd..d3b58914c90 100644 --- a/python/swe/dockerfiles/create_index.py +++ b/python/swe/dockerfiles/create_index.py @@ -5,13 +5,14 @@ from pathlib import Path import click -from swebench import get_eval_refs +from swebench.harness.constants import SWEbenchInstance +from swebench.harness.utils import load_swebench_dataset from composio import Action, ComposioToolSet from composio.utils.logging import WithLogger -def group_task_instances(task_instances): +def group_task_instances(task_instances: list[SWEbenchInstance]): groups = {} for instance in task_instances: repo = instance["repo"] @@ -43,11 +44,25 @@ def __init__( self.outdir.mkdir() def generate(self): - task_instances = get_eval_refs(data_path_or_name=self.dataset) - task_instance_groups = group_task_instances(task_instances.values()) + task_instances = load_swebench_dataset(name=self.dataset) + task_instance_groups = group_task_instances(task_instances) for repo, versions in task_instance_groups.items(): self.logger.info(f"Repo {repo} with {set(versions.keys())} versions") for version, instances in versions.items(): + outname = _repo_name(repo) + docker_outdir = Path("generated") / outname / version + + # Check if files in generated directory are complete + if ( + docker_outdir.exists() + and (docker_outdir / "deeplake").exists() + and (docker_outdir / "fqdn_cache.json").exists() + ): + self.logger.info( + f"Skipping {repo} {version} - files already exist in generated directory" + ) + continue + self.logger.info(f"\tGenerating for version - {version}") self.create_index( repository=repo, version=version, setup_ref_instance=instances[0] @@ -58,8 +73,8 @@ def create_index( ): outname = _repo_name(repository) outdir = self.outdir / outname / version - if outdir.exists(): - return + docker_outdir = Path("generated") / outname / version + repo_url = f"https://github.com/{repository}.git" base_commit = setup_ref_instance["base_commit"] if not (outdir / outname).exists(): @@ -76,10 +91,16 @@ def create_index( ["git", "checkout", base_commit], cwd=outdir / outname, check=True ) - composio_toolset = ComposioToolSet() + composio_toolset = ComposioToolSet( + metadata={ + Action.CODE_ANALYSIS_TOOL_CREATE_CODE_MAP: { + "dir_to_index_path": str(outdir / outname), + }, + }, + ) composio_toolset.execute_action( action=Action.CODE_ANALYSIS_TOOL_CREATE_CODE_MAP, - params={"dir_to_index_path": str(outdir / outname)}, + params={}, ) with open(f"{Path.home()}/.composio/tmp/{outname}/fqdn_cache.json") as f: fqdn_index = json.load(f) @@ -91,8 +112,6 @@ def create_index( ) fqdn_index[k] = v - docker_outdir = Path("generated") / outname / version - # docker_outdir.mkdir(exist_ok=True, parents=True) with open( docker_outdir / "fqdn_cache.json", "w", @@ -100,7 +119,6 @@ def create_index( json.dump(fqdn_index, f, indent=4) DEEPLAKE_PATH = docker_outdir / "deeplake" - # DEEPLAKE_PATH.mkdir(exist_ok=True, parents=True) if not DEEPLAKE_PATH.exists(): shutil.copytree( f"{Path.home()}/.composio/tmp/{outname}/deeplake", diff --git a/python/tox.ini b/python/tox.ini index 63e47c7905e..576da38d263 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -7,7 +7,7 @@ basepython = python deps = isort==5.12.0 commands = - isort composio/ scripts/ tests/ swe/ --profile black + isort composio/ scripts/ tests/ swe/ --profile black --skip swe/dockerfiles/generated --skip swe/dockerfiles/indexed isort plugins/ --profile black [testenv:isort-check] @@ -15,7 +15,7 @@ basepython = python3 deps = isort==5.12.0 commands = - isort composio/ scripts/ tests/ swe/ --check --profile black + isort composio/ scripts/ tests/ swe/ --check --profile black --skip swe/dockerfiles/generated --skip swe/dockerfiles/indexed isort plugins/ --check --profile black [testenv:black] @@ -23,7 +23,7 @@ basepython = python deps = black==24.10.0 commands = - black composio/ scripts/ tests/ swe/ + black composio/ scripts/ tests/ swe/ --exclude "swe/dockerfiles/(generated|indexed)" black plugins/ [testenv:black-check] @@ -31,7 +31,7 @@ basepython = python3 deps = black==24.10.0 commands = - black composio/ scripts/ tests/ swe/ --check + black composio/ scripts/ tests/ swe/ --check --exclude "swe/dockerfiles/(generated|indexed)" black plugins/ --check [testenv:black-diff] @@ -39,7 +39,7 @@ basepython = python3 deps = black==24.10.0 commands = - black composio/ scripts/ tests/ swe/ --check --diff + black composio/ scripts/ tests/ swe/ --check --diff --exclude "swe/dockerfiles/(generated|indexed)" black plugins/ --check --diff [testenv:mypy] @@ -171,7 +171,7 @@ sections=FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,PLUGINS,PACKAGES,LOCALFOLDER [flake8] max_line_length = 200 -exclude= **/build, **/dist +exclude = **/build, **/dist, swe/dockerfiles/generated/*, swe/dockerfiles/indexed/* per-file-ignores = __init__.py:F401,W503 tests/**:E501 @@ -179,7 +179,7 @@ ignore = E231, W291, W503, E704 [mypy] strict_optional = True -exclude=plugins/.*/setup\.py|plugins/.*/build/lib/|swe/build/lib/ +exclude = plugins/.*/setup\.py|plugins/.*/build/lib/|swe/build/lib/|swe/dockerfiles/generated/.*|swe/dockerfiles/indexed/.* ignore_missing_imports = True [mypy-requests.*] From 7480c26d01166309403069b9702b45937b09f3dc Mon Sep 17 00:00:00 2001 From: Siddharth Date: Fri, 7 Feb 2025 20:00:57 +0530 Subject: [PATCH 2/2] remove unused module --- python/swe/dockerfiles/build.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/swe/dockerfiles/build.py b/python/swe/dockerfiles/build.py index d9e60a0c8ca..cb4b1183b80 100644 --- a/python/swe/dockerfiles/build.py +++ b/python/swe/dockerfiles/build.py @@ -1,6 +1,5 @@ import signal import subprocess -import sys import threading import typing as t from concurrent.futures import ThreadPoolExecutor