diff --git a/.env.example b/.env.example index 3f7871fd..0d66c807 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,9 @@ +# cp .env.example .env +# Edit your .env file with your own values +# Don't commit your .env file to git/push to GitHub! +# Don't modify/delete .env.example unless adding extensions to the project +# which require new variable to be added to the .env file + # API CONFIG OPENAI_API_KEY= OPENAI_API_MODEL=gpt-3.5-turbo # alternatively, gpt-4, text-davinci-003, etc @@ -9,4 +15,12 @@ TABLE_NAME=test-table # PROJECT CONFIG OBJECTIVE=Solve world hunger -FIRST_TASK=Develop a task list +# For backwards compatibility +# FIRST_TASK can be used instead of INITIAL_TASK +INITIAL_TASK=Develop a task list + +# Extensions +# List additional extensions to load (except .env.example!) +DOTENV_EXTENSIONS= +# Set to true to enable command line args support +ENABLE_COMMAND_LINE_ARGS=false diff --git a/.gitignore b/.gitignore index e58e5a29..37a4938c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,12 @@ +__pycache__/ +*.py[cod] +*$py.class + .env -.idea +.env.* +env/ +.venv +venv/ + +.vscode/ +.idea/ diff --git a/CNAME b/CNAME new file mode 100644 index 00000000..a43bb23b --- /dev/null +++ b/CNAME @@ -0,0 +1 @@ +babyagi.org \ No newline at end of file diff --git a/README.md b/README.md index c7c0b55b..23a9734b 100644 --- a/README.md +++ b/README.md @@ -36,13 +36,12 @@ To use the script, you will need to follow these steps: 3. Set your OpenAI and Pinecone API keys in the OPENAI_API_KEY, OPENAPI_API_MODEL, and PINECONE_API_KEY variables. 4. Set the Pinecone environment in the PINECONE_ENVIRONMENT variable. 5. Set the name of the table where the task results will be stored in the TABLE_NAME variable. -6. Set the objective of the task management system in the OBJECTIVE variable. Alternatively you can pass it to the script as a quote argument. -``` -./babyagi.py [""] -``` -7. Set the first task of the system in the FIRST_TASK variable. +6. (Optional) Set the objective of the task management system in the OBJECTIVE variable. +7. (Optional) Set the first task of the system in the INITIAL_TASK variable. 8. Run the script. +All optional values above can also be specified on the command line. + # Warning This script is designed to be run continuously as part of a task management system. Running this script continuously can result in high API usage, so please use it responsibly. Additionally, the script requires the OpenAI and Pinecone APIs to be set up correctly, so make sure you have set up the APIs before running the script. diff --git a/babyagi.py b/babyagi.py index c34bb9e5..a9efd8e2 100755 --- a/babyagi.py +++ b/babyagi.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import os -import sys import time from collections import deque from typing import Dict, List @@ -9,7 +8,7 @@ import pinecone from dotenv import load_dotenv -# Set Variables +# Load default environment variables (.env) load_dotenv() # Set API Keys @@ -29,7 +28,7 @@ PINECONE_API_KEY = os.getenv("PINECONE_API_KEY", "") assert PINECONE_API_KEY, "PINECONE_API_KEY environment variable is missing from .env" -PINECONE_ENVIRONMENT = os.getenv("PINECONE_ENVIRONMENT", "us-east1-gcp") +PINECONE_ENVIRONMENT = os.getenv("PINECONE_ENVIRONMENT", "") assert ( PINECONE_ENVIRONMENT ), "PINECONE_ENVIRONMENT environment variable is missing from .env" @@ -38,16 +37,44 @@ YOUR_TABLE_NAME = os.getenv("TABLE_NAME", "") assert YOUR_TABLE_NAME, "TABLE_NAME environment variable is missing from .env" -# Project config -OBJECTIVE = sys.argv[1] if len(sys.argv) > 1 else os.getenv("OBJECTIVE", "") -assert OBJECTIVE, "OBJECTIVE environment variable is missing from .env" +# Goal configuation +OBJECTIVE = os.getenv("OBJECTIVE", "") +INITIAL_TASK = os.getenv("INITIAL_TASK", os.getenv("FIRST_TASK", "")) -YOUR_FIRST_TASK = os.getenv("FIRST_TASK", "") -assert YOUR_FIRST_TASK, "FIRST_TASK environment variable is missing from .env" +DOTENV_EXTENSIONS = os.getenv("DOTENV_EXTENSIONS", "").split(" ") + +# Command line arguments extension +# Can override any of the above environment variables +ENABLE_COMMAND_LINE_ARGS = ( + os.getenv("ENABLE_COMMAND_LINE_ARGS", "false").lower() == "true" +) +if ENABLE_COMMAND_LINE_ARGS: + from extensions.argparseext import parse_arguments + + OBJECTIVE, INITIAL_TASK, OPENAI_API_MODEL, DOTENV_EXTENSIONS = parse_arguments() + +# Load additional environment variables for enabled extensions +if DOTENV_EXTENSIONS: + from extensions.dotenvext import load_dotenv_extensions + + load_dotenv_extensions(DOTENV_EXTENSIONS) + +# TODO: There's still work to be done here to enable people to get +# defaults from dotenv extensions # but also provide command line +# arguments to override them + +if "gpt-4" in OPENAI_API_MODEL.lower(): + print( + "\033[91m\033[1m" + + "\n*****USING GPT-4. POTENTIALLY EXPENSIVE. MONITOR YOUR COSTS*****" + + "\033[0m\033[0m" + ) # Print OBJECTIVE -print("\033[96m\033[1m" + "\n*****OBJECTIVE*****\n" + "\033[0m\033[0m") -print(OBJECTIVE) +print("\033[94m\033[1m" + "\n*****OBJECTIVE*****\n" + "\033[0m\033[0m") +print(f"{OBJECTIVE}") + +print("\033[93m\033[1m" + "\nInitial task:" + "\033[0m\033[0m" + f" {INITIAL_TASK}") # Configure OpenAI and Pinecone openai.api_key = OPENAI_API_KEY @@ -87,30 +114,39 @@ def openai_call( temperature: float = 0.5, max_tokens: int = 100, ): - if not model.startswith("gpt-"): - # Use completion API - response = openai.Completion.create( - engine=model, - prompt=prompt, - temperature=temperature, - max_tokens=max_tokens, - top_p=1, - frequency_penalty=0, - presence_penalty=0, - ) - return response.choices[0].text.strip() - else: - # Use chat completion API - messages = [{"role": "user", "content": prompt}] - response = openai.ChatCompletion.create( - model=model, - messages=messages, - temperature=temperature, - max_tokens=max_tokens, - n=1, - stop=None, - ) - return response.choices[0].message.content.strip() + while True: + try: + if not model.startswith("gpt-"): + # Use completion API + response = openai.Completion.create( + engine=model, + prompt=prompt, + temperature=temperature, + max_tokens=max_tokens, + top_p=1, + frequency_penalty=0, + presence_penalty=0, + ) + return response.choices[0].text.strip() + else: + # Use chat completion API + messages = [{"role": "user", "content": prompt}] + response = openai.ChatCompletion.create( + model=model, + messages=messages, + temperature=temperature, + max_tokens=max_tokens, + n=1, + stop=None, + ) + return response.choices[0].message.content.strip() + except openai.error.RateLimitError: + print( + "The OpenAI API rate limit has been exceeded. Waiting 10 seconds and trying again." + ) + time.sleep(10) # Wait 10 seconds and try again + else: + break def task_creation_agent( @@ -123,7 +159,7 @@ def task_creation_agent( Based on the result, create new tasks to be completed by the AI system that do not overlap with incomplete tasks. Return the tasks as an array.""" response = openai_call(prompt) - new_tasks = response.split("\n") + new_tasks = response.split("\n") if "\n" in response else [response] return [{"task_name": task_name} for task_name in new_tasks] @@ -154,8 +190,8 @@ def execution_agent(objective: str, task: str) -> str: # print("\n*******RELEVANT CONTEXT******\n") # print(context) prompt = f""" - You are an AI who performs one task based on the following objective: {objective}. - Take into account these previously completed tasks: {context}. + You are an AI who performs one task based on the following objective: {objective}\n. + Take into account these previously completed tasks: {context}\n. Your task: {task}\nResponse:""" return openai_call(prompt, temperature=0.7, max_tokens=2000) @@ -170,7 +206,7 @@ def context_agent(query: str, n: int): # Add the first task -first_task = {"task_id": 1, "task_name": YOUR_FIRST_TASK} +first_task = {"task_id": 1, "task_name": INITIAL_TASK} add_task(first_task) # Main loop @@ -198,31 +234,25 @@ def context_agent(query: str, n: int): "data": result } # This is where you should enrich the result if needed result_id = f"result_{task['task_id']}" - vector = enriched_result[ - "data" - ] # extract the actual result from the dictionary + vector = get_ada_embedding( + enriched_result["data"] + ) # get vector of the actual result extracted from the dictionary index.upsert( - [ - ( - result_id, - get_ada_embedding(vector), - {"task": task["task_name"], "result": result}, - ) - ] + [(result_id, vector, {"task": task["task_name"], "result": result})] ) - # Step 3: Create new tasks and reprioritize task list - new_tasks = task_creation_agent( - OBJECTIVE, - enriched_result, - task["task_name"], - [t["task_name"] for t in task_list], - ) + # Step 3: Create new tasks and reprioritize task list + new_tasks = task_creation_agent( + OBJECTIVE, + enriched_result, + task["task_name"], + [t["task_name"] for t in task_list], + ) - for new_task in new_tasks: - task_id_counter += 1 - new_task.update({"task_id": task_id_counter}) - add_task(new_task) - prioritization_agent(this_task_id) + for new_task in new_tasks: + task_id_counter += 1 + new_task.update({"task_id": task_id_counter}) + add_task(new_task) + prioritization_agent(this_task_id) time.sleep(1) # Sleep before checking the task list again diff --git a/extensions/argparseext.py b/extensions/argparseext.py new file mode 100644 index 00000000..49d8266b --- /dev/null +++ b/extensions/argparseext.py @@ -0,0 +1,76 @@ +import os +import sys +import argparse + +# Extract the env filenames in the -e flag only +# Ignore any other arguments +def parse_dotenv_extensions(argv): + env_argv = [] + if '-e' in argv: + tmp_argv = argv[argv.index('-e') + 1:] + parsed_args = [] + for arg in tmp_argv: + if arg.startswith('-'): + break + parsed_args.append(arg) + env_argv = ['-e'] + parsed_args + + parser = argparse.ArgumentParser() + parser.add_argument('-e', '--env', nargs='+', help=''' + filenames for additional env variables to load + ''', default=os.getenv("DOTENV_EXTENSIONS", "").split(' ')) + + return parser.parse_args(env_argv).env + +def parse_arguments(): + dotenv_extensions = parse_dotenv_extensions(sys.argv) + # Check if we need to load any additional env files + # This allows us to override the default .env file + # and update the default values for any command line arguments + if dotenv_extensions: + from extensions.dotenvext import load_dotenv_extensions + load_dotenv_extensions(parse_dotenv_extensions(sys.argv)) + + # Now parse the full command line arguments + parser = argparse.ArgumentParser( + add_help=False, + ) + parser.add_argument('objective', nargs='*', metavar='', help=''' + main objective description. Doesn\'t need to be quoted. + if not specified, get objective from environment. + ''', default=[os.getenv("OBJECTIVE", "")]) + parser.add_argument('-t', '--task', metavar='', help=''' + initial task description. must be quoted. + if not specified, get initial_task from environment. + ''', default=os.getenv("INITIAL_TASK", os.getenv("FIRST_TASK", ""))) + parser.add_argument('-4', '--gpt-4', dest='openai_api_model', action='store_const', const="gpt-4", help=''' + use GPT-4 instead of the default GPT-3 model. + ''', default=os.getenv("OPENAI_API_MODEL", "gpt-3.5-turbo")) + # This will parse -e again, which we want, because we need + # to load those in the main file later as well + parser.add_argument('-e', '--env', nargs='+', help=''' + filenames for additional env variables to load + ''', default=os.getenv("DOTENV_EXTENSIONS", "").split(' ')) + parser.add_argument('-h', '-?', '--help', action='help', help=''' + show this help message and exit + ''') + + args = parser.parse_args() + + openai_api_model = args.openai_api_model + + dotenv_extensions = args.env + + objective = ' '.join(args.objective).strip() + if not objective: + print("\033[91m\033[1m"+"No objective specified or found in environment.\n"+"\033[0m\033[0m") + parser.print_help() + parser.exit() + + initial_task = args.task + if not initial_task: + print("\033[91m\033[1m"+"No initial task specified or found in environment.\n"+"\033[0m\033[0m") + parser.print_help() + parser.exit() + + return objective, initial_task, openai_api_model, dotenv_extensions \ No newline at end of file diff --git a/extensions/dotenvext.py b/extensions/dotenvext.py new file mode 100644 index 00000000..0db80aea --- /dev/null +++ b/extensions/dotenvext.py @@ -0,0 +1,5 @@ +from dotenv import load_dotenv + +def load_dotenv_extensions(dotenv_files): + for dotenv_file in dotenv_files: + load_dotenv(dotenv_file) diff --git a/requirements.txt b/requirements.txt index d6531bcc..b41bc108 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -openai==0.27.2 +argparse==1.4.0 +openai==0.27.4 pinecone-client==2.2.1 pre-commit>=3.2.0 python-dotenv==1.0.0 diff --git a/tools/results_browser.py b/tools/results_browser.py new file mode 100644 index 00000000..028a6f19 --- /dev/null +++ b/tools/results_browser.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +import os +import curses +import argparse +import openai +import pinecone +from dotenv import load_dotenv +import textwrap + +load_dotenv() + +OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "") +assert OPENAI_API_KEY, "OPENAI_API_KEY environment variable is missing from .env" + +PINECONE_API_KEY = os.getenv("PINECONE_API_KEY", "") +assert PINECONE_API_KEY, "PINECONE_API_KEY environment variable is missing from .env" + +PINECONE_ENVIRONMENT = os.getenv("PINECONE_ENVIRONMENT", "us-east1-gcp") +assert PINECONE_ENVIRONMENT, "PINECONE_ENVIRONMENT environment variable is missing from .env" + +# Table config +PINECONE_TABLE_NAME = os.getenv("TABLE_NAME", "") +assert PINECONE_TABLE_NAME, "TABLE_NAME environment variable is missing from .env" + +# Function to query records from the Pinecone index +def query_records(index, query, top_k=1000): + results = index.query(query, top_k=top_k, include_metadata=True) + return [{"name": f"{task.metadata['task']}", "result": f"{task.metadata['result']}"} for task in results.matches] + +# Get embedding for the text +def get_ada_embedding(text): + return openai.Embedding.create(input=[text], model="text-embedding-ada-002")["data"][0]["embedding"] + +def draw_tasks(stdscr, tasks, scroll_pos, selected): + y = 0 + h, w = stdscr.getmaxyx() + for idx, task in enumerate(tasks[scroll_pos:], start=scroll_pos): + if y >= h: + break + task_name = f'{task["name"]}' + truncated_str = task_name[:w-1] + if idx == selected: + stdscr.addstr(y, 0, truncated_str, curses.A_REVERSE) + else: + stdscr.addstr(y, 0, truncated_str) + y += 1 + +def draw_result(stdscr, task): + task_name = f'Task: {task["name"]}' + task_result = f'Result: {task["result"]}' + + _, w = stdscr.getmaxyx() + task_name_wrapped = textwrap.wrap(task_name, width=w) + + for i, line in enumerate(task_name_wrapped): + stdscr.addstr(i, 0, line) + + y, _ = stdscr.getyx() + stdscr.addstr(y+1, 0, '------------------') + stdscr.addstr(y+2, 0, task_result) + +def draw_summary(stdscr, objective, tasks, start, num): + stdscr.box() + summary_text = f'{len(tasks)} tasks ({start}-{num}) | {objective}' + stdscr.addstr(1, 1, summary_text[:stdscr.getmaxyx()[1] - 2]) + +def main(stdscr): + # Initialize Pinecone + pinecone.init(api_key=PINECONE_API_KEY) + + # Connect to the objective index + index = pinecone.Index(PINECONE_TABLE_NAME) + + curses.curs_set(0) + stdscr.timeout(1000) + + h, w = stdscr.getmaxyx() + left_w = w // 2 + visible_lines = h - 3 + + scroll_pos = 0 + selected = 0 + + # Parse command-line arguments + parser = argparse.ArgumentParser(description="Query Pinecone index using a string.") + parser.add_argument('objective', nargs='*', metavar='', help=''' + main objective description. Doesn\'t need to be quoted. + if not specified, get objective from environment. + ''', default=[os.getenv("OBJECTIVE", "")]) + args = parser.parse_args() + + # Query records from the index + objective = ' '.join(args.objective).strip().replace("\n", " ") + retrieved_tasks = query_records(index, get_ada_embedding(objective)) + + while True: + stdscr.clear() + draw_tasks(stdscr.subwin(h-3, left_w, 0, 0), retrieved_tasks, scroll_pos, selected) + draw_result(stdscr.subwin(h, w - left_w, 0, left_w), retrieved_tasks[selected]) + draw_summary(stdscr.subwin(3, left_w, h - 3, 0), objective, retrieved_tasks, scroll_pos+1, scroll_pos+h-3) + + stdscr.refresh() + key = stdscr.getch() + + if key == ord('q') or key == 27: + break + elif key == curses.KEY_UP and selected > 0: + selected -= 1 + if selected < scroll_pos: + scroll_pos -= 1 + elif key == curses.KEY_DOWN and selected < len(retrieved_tasks) - 1: + selected += 1 + if selected - scroll_pos >= visible_lines: + scroll_pos += 1 + +curses.wrapper(main) \ No newline at end of file