diff --git a/src/dbally/iql_generator/prompt.py b/src/dbally/iql_generator/prompt.py index f2c29d62..8c838ba6 100644 --- a/src/dbally/iql_generator/prompt.py +++ b/src/dbally/iql_generator/prompt.py @@ -194,17 +194,17 @@ def __init__( "role": "system", "content": ( "You have access to an API that lets you query a database:\n" - "\n{methods}\n" + "\n{methods}\n\n" "Suggest which one(s) to call and how they should be joined with logic operators (AND, OR, NOT).\n" - "Remember! Don't give any comments, just the function calls.\n" - "The output will look like this:\n" - 'filter1("arg1") AND (NOT filter2(120) OR filter3(True))\n' - "DO NOT INCLUDE arguments names in your response. Only the values.\n" - "You MUST use only these methods:\n" - "\n{methods}\n" - "It is VERY IMPORTANT not to use methods other than those listed above." + "Remember! Don't return any comments, just the function calls.\n" + "The output should look like Python function calls with positional arguments, joined by logic operators:\n" + 'some_filter("foo") AND (NOT filter2(120) OR filter3(True))\n\n' + "DO NOT INCLUDE arguments names in your response. Only the values. Strings must be quoted.\n" + "You MUST use only these functions:\n" + "\n{methods}\n\n" + "It is VERY IMPORTANT not to use functions other than those listed above." """If you DON'T KNOW HOW TO ANSWER DON'T SAY anything other than `UNSUPPORTED QUERY`""" - "This is CRUCIAL, otherwise the system will crash. " + "This is CRUCIAL, otherwise the system will crash." ), }, { @@ -220,17 +220,17 @@ def __init__( { "role": "system", "content": ( - "You have access to an API that lets you query a database supporting a SINGLE aggregation.\n" - "When prompted for an aggregation, use the following methods: \n" + "You have access to an API that lets you query a database, supporting a SINGLE aggregation.\n" + "When prompted for an aggregation, use one of the following functions: \n" "{methods}" - "DO NOT INCLUDE arguments names in your response. Only the values.\n" - "You MUST use only these methods:\n" + "The output should look like a single Python function call with positional arguments:\n" + 'some_aggregation("foo", True)\n' + "DO NOT INCLUDE arguments names in your response. Only the values. Strings must be quoted.\n\n" + "You MUST only choose from these functions:\n" "\n{methods}\n" "It is VERY IMPORTANT not to use methods other than those listed above." """If you DON'T KNOW HOW TO ANSWER DON'T SAY anything other than `UNSUPPORTED QUERY`""" "This is CRUCIAL to put `UNSUPPORTED QUERY` text only, otherwise the system will crash. " - "Structure output to resemble the following pattern:\n" - 'aggregation1("arg1", arg2)\n' ), }, { diff --git a/tests/unit/test_iql_format.py b/tests/unit/test_iql_format.py index 3a21a1fe..a497f9cd 100644 --- a/tests/unit/test_iql_format.py +++ b/tests/unit/test_iql_format.py @@ -1,11 +1,24 @@ +from typing import List + from dbally.iql_generator.prompt import FILTERS_GENERATION_TEMPLATE, IQLGenerationPromptFormat from dbally.prompt.elements import FewShotExample +from dbally.views.exposed_functions import ExposedFunction, MethodParamWithTyping async def test_iql_prompt_format_default() -> None: prompt_format = IQLGenerationPromptFormat( - question="", - methods=[], + question="Some question", + methods=[ + ExposedFunction( + name="filter_by_name", description="", parameters=[MethodParamWithTyping(name="name", type=List[str])] + ), + ExposedFunction( + name="filter_by_city", description="", parameters=[MethodParamWithTyping(name="city", type=str)] + ), + ExposedFunction( + name="filter_by_company", description="", parameters=[MethodParamWithTyping(name="company", type=str)] + ), + ], examples=[], ) formatted_prompt = FILTERS_GENERATION_TEMPLATE.format_prompt(prompt_format) @@ -13,29 +26,37 @@ async def test_iql_prompt_format_default() -> None: assert formatted_prompt.chat == [ { "role": "system", - "content": "You have access to an API that lets you query a database:\n" - "\n\n" + "content": "You have access to an API that lets you query a database:\n\n" + + "filter_by_name(name: List[str])\n" + + "filter_by_city(city: str)\n" + + "filter_by_company(company: str)\n\n" "Suggest which one(s) to call and how they should be joined with logic operators (AND, OR, NOT).\n" - "Remember! Don't give any comments, just the function calls.\n" - "The output will look like this:\n" - 'filter1("arg1") AND (NOT filter2(120) OR filter3(True))\n' - "DO NOT INCLUDE arguments names in your response. Only the values.\n" - "You MUST use only these methods:\n" - "\n\n" - "It is VERY IMPORTANT not to use methods other than those listed above." + "Remember! Don't return any comments, just the function calls.\n" + "The output should look like Python function calls with positional arguments, joined by logic operators:\n" + 'some_filter("foo") AND (NOT filter2(120) OR filter3(True))\n\n' + "DO NOT INCLUDE arguments names in your response. Only the values. Strings must be quoted.\n" + "You MUST use only these functions:\n\n" + + "filter_by_name(name: List[str])\n" + + "filter_by_city(city: str)\n" + + "filter_by_company(company: str)\n\n" + "It is VERY IMPORTANT not to use functions other than those listed above." """If you DON'T KNOW HOW TO ANSWER DON'T SAY anything other than `UNSUPPORTED QUERY`""" - "This is CRUCIAL, otherwise the system will crash. ", + "This is CRUCIAL, otherwise the system will crash.", "is_example": False, }, - {"role": "user", "content": "", "is_example": False}, + {"role": "user", "content": "Some question", "is_example": False}, ] async def test_iql_prompt_format_few_shots_injected() -> None: examples = [FewShotExample("q1", "a1")] prompt_format = IQLGenerationPromptFormat( - question="", - methods=[], + question="Some question", + methods=[ + ExposedFunction( + name="filter_by_name", description="", parameters=[MethodParamWithTyping(name="name", type=List[str])] + ) + ], examples=examples, ) formatted_prompt = FILTERS_GENERATION_TEMPLATE.format_prompt(prompt_format) @@ -43,23 +64,22 @@ async def test_iql_prompt_format_few_shots_injected() -> None: assert formatted_prompt.chat == [ { "role": "system", - "content": "You have access to an API that lets you query a database:\n" - "\n\n" + "content": "You have access to an API that lets you query a database:\n\n" + + "filter_by_name(name: List[str])\n\n" "Suggest which one(s) to call and how they should be joined with logic operators (AND, OR, NOT).\n" - "Remember! Don't give any comments, just the function calls.\n" - "The output will look like this:\n" - 'filter1("arg1") AND (NOT filter2(120) OR filter3(True))\n' - "DO NOT INCLUDE arguments names in your response. Only the values.\n" - "You MUST use only these methods:\n" - "\n\n" - "It is VERY IMPORTANT not to use methods other than those listed above." + "Remember! Don't return any comments, just the function calls.\n" + "The output should look like Python function calls with positional arguments, joined by logic operators:\n" + 'some_filter("foo") AND (NOT filter2(120) OR filter3(True))\n\n' + "DO NOT INCLUDE arguments names in your response. Only the values. Strings must be quoted.\n" + "You MUST use only these functions:\n\n" + "filter_by_name(name: List[str])\n\n" + "It is VERY IMPORTANT not to use functions other than those listed above." """If you DON'T KNOW HOW TO ANSWER DON'T SAY anything other than `UNSUPPORTED QUERY`""" - "This is CRUCIAL, otherwise the system will crash. ", + "This is CRUCIAL, otherwise the system will crash.", "is_example": False, }, {"role": "user", "content": examples[0].question, "is_example": True}, {"role": "assistant", "content": examples[0].answer, "is_example": True}, - {"role": "user", "content": "", "is_example": False}, + {"role": "user", "content": "Some question", "is_example": False}, ]