Initialize components with interfaces and stub implementations

.gitignore

@@ -3,3 +3,5 @@ build
 prompt2model.egg-info
 .vscode
 .mypy_cache
+*.pyc
+

prompt2model/dataset_generator/__init__.py

@@ -0,0 +1,5 @@
+"""Import DatasetGenerator classes."""
+from prompt2model.dataset_generator.base import DatasetGenerator, DatasetSplit
+from prompt2model.dataset_generator.mock import MockDatasetGenerator
+
+__all__ = ("MockDatasetGenerator", "DatasetGenerator", "DatasetSplit")

prompt2model/dataset_generator/base.py

@@ -0,0 +1,68 @@
+"""An interface for dataset generation."""
+
+from __future__ import annotations  # noqa FI58
+
+from abc import ABC, abstractmethod
+from enum import Enum
+
+import datasets
+
+from prompt2model.prompt_parser import PromptSpec
+
+
+class DatasetSplit(Enum):
+    """The split of a dataset."""
+
+    TRAIN = "train"
+    VAL = "val"
+    TEST = "test"
+
+
+class DatasetGenerator(ABC):
+    """A class for generating datasets from a prompt specification."""
+
+    @abstractmethod
+    def generate_examples(
+        self,
+        prompt_spec: PromptSpec,
+        num_examples: int,
+        split: DatasetSplit,
+    ) -> datasets.Dataset:
+        """Generate data for a single named split of data.
+
+        Args:
+            prompt_spec: A prompt spec (containing a system description).
+            num_examples: Number of examples in split.
+            split: Name of dataset split to generate.)
+
+        Returns:
+            A single dataset split.
+
+        """
+
+    def generate_datasets(
+        self,
+        prompt_spec: PromptSpec,
+        num_examples: dict[DatasetSplit, int],
+        output_dir: str | None = None,
+    ) -> datasets.DatasetDict:
+        """Generate training/validation/testing datasets from a prompt.
+
+        Args:
+            prompt_spec: A prompt specification.
+            num_examples: Number of examples per split (train/val/test/etc).
+
+        Returns:
+            A DatasetDict containing train, validation, and test splits.
+        """
+        dataset_dict = datasets.DatasetDict(
+            {
+                split: self.generate_examples(prompt_spec, num, split=split)
+                for split, num in num_examples.items()
+            }
+        )
+
+        if output_dir:
+            dataset_dict.save_to_disk(output_dir)
+
+        return dataset_dict

prompt2model/dataset_generator/mock.py

@@ -0,0 +1,34 @@
+"""A class for generating empty datasets (for testing purposes)."""
+
+import datasets
+import pandas as pd
+
+from prompt2model.dataset_generator.base import DatasetGenerator, DatasetSplit
+from prompt2model.prompt_parser import PromptSpec
+
+
+class MockDatasetGenerator(DatasetGenerator):
+    """A class for generating empty datasets (for testing purposes)."""
+
+    def generate_examples(
+        self,
+        prompt_spec: PromptSpec,
+        num_examples: int,
+        split: DatasetSplit,
+    ) -> datasets.Dataset:
+        """Create empty versions of the datasets, for testing.
+
+        Args:
+            prompt_spec: A prompt specification.
+            num_examples: Number of examples in split.
+            split: Name of dataset split to generate.
+
+        Returns:
+            A single dataset split.
+
+        """
+        _ = prompt_spec, split  # suppress unused variable warnings
+        col_values = ["" for i in range(num_examples)]
+        # Construct empty-valued dataframe with length matching num_examples.
+        df = pd.DataFrame.from_dict({"input_col": col_values, "output_col": col_values})
+        return datasets.Dataset.from_pandas(df)

prompt2model/dataset_retriever/__init__.py

@@ -0,0 +1,5 @@
+"""Import DatasetRetriever classes."""
+from prompt2model.dataset_retriever.base import DatasetRetriever
+from prompt2model.dataset_retriever.mock import MockRetriever
+
+__all__ = ("DatasetRetriever", "MockRetriever")

prompt2model/dataset_retriever/base.py

@@ -0,0 +1,24 @@
+"""An interface for dataset retrieval."""
+
+from abc import ABC, abstractmethod
+
+import datasets
+
+from prompt2model.prompt_parser import PromptSpec
+
+
+# pylint: disable=too-few-public-methods
+class DatasetRetriever(ABC):
+    """A class for retrieving datasets."""
+
+    @abstractmethod
+    def retrieve_datasets(self, prompt_spec: PromptSpec) -> list[datasets.Dataset]:
+        """Retrieve datasets from a prompt specification.
+
+        Args:
+            prompt_spec: A prompt spec (containing a system description).
+
+        Returns:
+            A list of retrieved datasets.
+
+        """

prompt2model/dataset_retriever/mock.py

@@ -0,0 +1,20 @@
+"""A mock dataset retriever for testing purposes."""
+
+import datasets
+import pandas as pd
+
+from prompt2model.dataset_retriever.base import DatasetRetriever
+from prompt2model.prompt_parser import PromptSpec
+
+
+class MockRetriever(DatasetRetriever):
+    """A class for retrieving datasets."""
+
+    def __init__(self):
+        """Construct a mock dataset retriever."""
+
+    def retrieve_datasets(self, prompt_spec: PromptSpec) -> list[datasets.Dataset]:
+        """Return a single empty dataset for testing purposes."""
+        _ = prompt_spec  # suppress unused vaiable warning
+        test_df = pd.DataFrame.from_dict({"input_col": [""], "output_col": [""]})
+        return [datasets.Dataset.from_pandas(test_df)]

prompt2model/demo_creator/gradio_creator.py

@@ -0,0 +1,24 @@
+"""An interface for creating Gradio demos automatically."""
+
+import gradio as gr
+import transformers
+
+from prompt2model.prompt_parser.base import PromptSpec
+
+
+def create_gradio(
+    model: transformers.PreTrainedModel, prompt_spec: PromptSpec
+) -> gr.Interface:
+    """Create a Gradio interface automatically.
+
+    Args:
+        model: A trained model to expose via a Gradio interface.
+        prompt_spec: A PromptSpec to help choose the visual interface.
+
+    Returns:
+        A Gradio interface for interacting with the model.
+
+    """
+    _ = model, prompt_spec  # suppress unused variable warnings
+    dummy_interface = gr.Interface(lambda input: None, "textbox", "label")
+    return dummy_interface

prompt2model/evaluator/__init__.py

@@ -0,0 +1,5 @@
+"""Import evaluator classes."""
+from prompt2model.evaluator.base import Evaluator
+from prompt2model.evaluator.mock import MockEvaluator
+
+__all__ = ("MockEvaluator", "Evaluator")

prompt2model/evaluator/base.py

@@ -0,0 +1,49 @@
+"""An interface for automatic model evaluation."""
+
+from __future__ import annotations  # noqa FI58
+
+import json
+from abc import ABC, abstractmethod
+from typing import Any
+
+import datasets
+
+from prompt2model.model_executor import ModelOutput
+from prompt2model.prompt_parser import PromptSpec
+
+
+class Evaluator(ABC):
+    """An interface for automatic model evaluation."""
+
+    @abstractmethod
+    def evaluate_model(
+        self,
+        dataset: datasets.Dataset,
+        gt_column: str,
+        predictions: list[ModelOutput],
+        metrics: list[datasets.Metric] | None = None,
+        prompt_spec: PromptSpec | None = None,
+    ) -> dict[str, Any]:
+        """Evaluate a model on a test set..
+
+        Args:
+            dataset: The dataset to evaluate metrics on.
+            gt_column: The dataset column to use as ground truth.
+            predictions: Model outputs to evaluate.
+            metrics: (Optional) The metrics to use.
+            prompt_spec: (Optional) A PromptSpec to infer the metrics from.
+
+        Returns:
+            A dictionary of metric values to return.
+        """
+
+    def write_metrics(self, metrics_dict: dict[str, Any], metrics_path: str) -> None:
+        """This function writes metrics to a file.
+
+        Args:
+            metrics_dict: A dictionary of metrics to write.
+            metrics_path: The file path to write metrics to.
+
+        """
+        with open(metrics_path, "w") as f:
+            json.dump(metrics_dict, f)

prompt2model/evaluator/mock.py

@@ -0,0 +1,39 @@
+"""A dummy evaluator for testing purposes."""
+from __future__ import annotations  # noqa FI58
+
+from typing import Any
+
+import datasets
+
+from prompt2model.evaluator.base import Evaluator
+from prompt2model.model_executor import ModelOutput
+from prompt2model.prompt_parser import PromptSpec
+
+
+class MockEvaluator(Evaluator):
+    """A dummy evaluator that always returns the same metric value."""
+
+    def __init__(self) -> None:
+        """Initialize the evaluation setting."""
+
+    def evaluate_model(
+        self,
+        dataset: datasets.Dataset,
+        gt_column: str,
+        predictions: list[ModelOutput],
+        metrics: list[datasets.Metric] | None = None,
+        prompt_spec: PromptSpec | None = None,
+    ) -> dict[str, Any]:
+        """Return empty metrics dictionary.
+
+        Args:
+            dataset: The dataset to evaluate metrics on.
+            gt_column: The dataset column to use as ground truth.
+            predictions: Corresponding model outputs to evaluate.
+            metrics: (Optional) The metrics to use.
+            prompt_spec: (Optional) A PromptSpec to infer the metrics from.
+
+        Returns:
+            An empty dictionary (for testing purposes).
+        """
+        return {}

prompt2model/model_executor/__init__.py

@@ -0,0 +1,6 @@
+"""Import all the model executor classes."""
+
+from prompt2model.model_executor.base import ModelExecutor, ModelOutput
+from prompt2model.model_executor.mock import MockModelExecutor
+
+__all__ = ("ModelExecutor", "ModelOutput", "MockModelExecutor")

prompt2model/model_executor/base.py

@@ -0,0 +1,47 @@
+"""An interface for generating model outputs."""
+
+from __future__ import annotations  # noqa FI58
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any
+
+import datasets
+import transformers
+
+
+@dataclass(frozen=True)
+class ModelOutput:
+    """A model output for a single example.
+
+    Attributes:
+        prediction: The prediction by the model
+        confidence: A confidence value in the prediction (or None)
+        auxiliary_info: Any other auxiliary information provided by the model
+    """
+
+    prediction: Any
+    confidence: float | None
+    auxiliary_info: dict[str, Any]
+
+
+class ModelExecutor(ABC):
+    """An interface for automatic model evaluation."""
+
+    @abstractmethod
+    def make_predictions(
+        self,
+        model: transformers.PreTrainedModel,
+        test_set: datasets.Dataset,
+        input_column: str,
+    ) -> list[ModelOutput]:
+        """Evaluate a model on a test set.
+
+        Args:
+            model: The model to evaluate.
+            test_set: The dataset to make predictions on.
+            input_column: The dataset column to use as input to the model.
+
+        Returns:
+            A list of model outputs, one for each element in the test set.
+        """