Feature/updating prompt (#1050)

Editing prompt.py so that adaptation of testset_generator to dutch is
possible;
Added a output_parser for the question_answer_prompt, so that there are
less nans with testset_generation

Author: louky123

src/ragas/llms/prompt.py

@@ -3,6 +3,7 @@
 import json
 import logging
 import os
+import ast
 import typing as t
 
 from langchain_core.messages import BaseMessage, HumanMessage
@@ -228,12 +229,14 @@ def get_all_keys(nested_json):
             example_dict.update(
                 {k: v for k, v in zip(self.input_keys, example[: len(self.input_keys)])}
             )
-            example_dict[self.output_key] = (
-                json_loader._safe_load(example[-1], llm)
-                if self.output_type.lower() == "json"
-                else example[-1]
-            )
-
+            if self.output_type.lower() == "json":
+                example_dict[self.output_key] = json_loader._safe_load(example[-1], llm)
+                if example_dict[self.output_key] == {}:
+                    # Extracting the dictionary part using string slicing
+                    dict_str = example[-1].split('(')[0].strip()
+                    example_dict[self.output_key ] = ast.literal_eval(dict_str)
+                else:
+                    example_dict[self.output_key] = example[-1]
             if self.output_type.lower() == "json":
                 output = example_dict[self.output_key]
                 if isinstance(output, dict):

src/ragas/testset/prompts.py

@@ -3,6 +3,15 @@
 from ragas.llms.output_parser import RagasoutputParser, get_json_format_instructions
 from ragas.llms.prompt import Prompt
 
+
+class AnswerFormat(BaseModel):
+    answer: str
+    verdict: int
+
+
+question_answer_parser = RagasoutputParser(pydantic_object=AnswerFormat)
+
+
 reasoning_question_prompt = Prompt(
     name="reasoning_question",
     instruction="""Complicate the given question by rewriting question into a multi-hop reasoning question based on the provided context.
@@ -137,30 +146,28 @@
 question_answer_prompt = Prompt(
     name="answer_formulate",
     instruction="""Answer the question using the information from the given context. Output verdict as '1' if answer is present '-1' if answer is not present in the context.""",
+    output_format_instruction=get_json_format_instructions(AnswerFormat),
     examples=[
         {
             "context": """Climate change is significantly influenced by human activities, notably the emission of greenhouse gases from burning fossil fuels. The increased greenhouse gas concentration in the atmosphere traps more heat, leading to global warming and changes in weather patterns.""",
             "question": "How do human activities contribute to climate change?",
-            "answer": {
+            "answer": AnswerFormat.parse_obj({
                 "answer": "Human activities contribute to climate change primarily through the emission of greenhouse gases from burning fossil fuels. These emissions increase the concentration of greenhouse gases in the atmosphere, which traps more heat and leads to global warming and altered weather patterns.",
-                "verdict": "1",
-            },
+                "verdict": "1",}).dict(),
         },
         {
             "context": """The concept of artificial intelligence (AI) has evolved over time, but it fundamentally refers to machines designed to mimic human cognitive functions. AI can learn, reason, perceive, and, in some instances, react like humans, making it pivotal in fields ranging from healthcare to autonomous vehicles.""",
             "question": "What are the key capabilities of artificial intelligence?",
-            "answer": {
+            "answer": AnswerFormat.parse_obj({ 
                 "answer": "Artificial intelligence is designed to mimic human cognitive functions, with key capabilities including learning, reasoning, perception, and reacting to the environment in a manner similar to humans. These capabilities make AI pivotal in various fields, including healthcare and autonomous driving.",
-                "verdict": "1",
-            },
+                "verdict": "1",}).dict(),
         },
         {
             "context": """The novel "Pride and Prejudice" by Jane Austen revolves around the character Elizabeth Bennet and her family. The story is set in the 19th century in rural England and deals with issues of marriage, morality, and misconceptions.""",
             "question": "What year was 'Pride and Prejudice' published?",
-            "answer": {
+            "answer": AnswerFormat.parse_obj({
                 "answer": "The answer to given question is not present in context",
-                "verdict": "-1",
-            },
+                "verdict": "-1",}).dict(),
         },
     ],
     input_keys=["context", "question"],