Initialize components with interfaces and stub implementations

.gitignore

@@ -3,3 +3,5 @@ build
 prompt2model.egg-info
 .vscode
 .mypy_cache
+*.pyc
+

prompt2model/dataset_generator/__init__.py

@@ -0,0 +1,3 @@
+"""Import DatasetGenerator classes."""
+from dataset_generator.base import EmptyDatasetGenerator  # noqa: F401
+from dataset_generator.base import DatasetGenerator, DatasetSplit  # noqa: F401

prompt2model/dataset_generator/base.py

@@ -0,0 +1,125 @@
+"""An interface for dataset generation."""
+
+from __future__ import annotations  # noqa FI58
+
+from abc import ABC, abstractmethod
+from enum import Enum
+
+import datasets
+import pandas as pd
+from prompt_parser import PromptSpec
+from utils.rng import ConstantSeedGenerator
+
+
+class DatasetSplit(Enum):
+    """The split of a dataset."""
+
+    TRAIN = "train"
+    VAL = "val"
+    TEST = "test"
+
+
+class DatasetGenerator(ABC):
+    """A class for generating datasets from a prompt specification."""
+
+    def __init__(
+        self,
+        model_config: dict | None = None,
+        output_dir: str | None = None,
+    ):
+        """Construct a dataset generator."""
+        self.model_config = model_config
+        self.output_dir = output_dir
+        self.seed_generator = ConstantSeedGenerator()
+
+    @abstractmethod
+    def generate_examples(
+        self,
+        prompt_spec: PromptSpec,
+        num_examples: int | None,
+        split: DatasetSplit,
+    ) -> datasets.Dataset:
+        """Generate data for a single named split of data.
+
+        Args:
+            prompt_spec: A prompt spec (containing a system description).
+            num_examples: Number of examples in split.
+            split: Name of dataset split to generate.)
+
+        Returns:
+            A single dataset split.
+
+        """
+
+    def generate_datasets(
+        self,
+        prompt_spec: PromptSpec,
+        num_examples: dict[DatasetSplit, int],
+    ) -> datasets.DatasetDict:
+        """Generate training/validation/testing datasets from a prompt.
+
+        Args:
+            prompt_spec: A prompt specification.
+            num_examples: Number of examples per split (train/val/test/etc).
+
+        Returns:
+            A DatasetDict containing train, validation, and test splits.
+        """
+        assert num_examples.keys() == {
+            DatasetSplit.TRAIN,
+            DatasetSplit.VAL,
+            DatasetSplit.TEST,
+        }
+
+        train_examples = self.generate_examples(
+            prompt_spec, num_examples[DatasetSplit.TRAIN], split=DatasetSplit.TRAIN
+        )
+        val_examples = self.generate_examples(
+            prompt_spec, num_examples[DatasetSplit.VAL], split=DatasetSplit.VAL
+        )
+        test_examples = self.generate_examples(
+            prompt_spec, num_examples[DatasetSplit.TEST], split=DatasetSplit.TEST
+        )
+
+        dataset_dict = datasets.DatasetDict(
+            {
+                DatasetSplit.TRAIN: train_examples,
+                DatasetSplit.VAL: val_examples,
+                DatasetSplit.TEST: test_examples,
+            }
+        )
+
+        if self.output_dir:
+            dataset_dict.save_to_disk(self.output_dir)
+
+        return dataset_dict
+
+
+class EmptyDatasetGenerator(DatasetGenerator):
+    """A class for generating empty datasets (for testing purposes)."""
+
+    def generate_examples(
+        self,
+        prompt_spec: PromptSpec,
+        num_examples: int | None,
+        split: DatasetSplit,
+    ) -> datasets.Dataset:
+        """Create empty versions of the datasets, for testing.
+
+        Args:
+            prompt_spec: A prompt specification.
+            num_examples: Number of examples in split.
+            split: Name of dataset split to generate.)
+
+        Returns:
+            A single dataset split.
+
+        """
+        _ = prompt_spec, split  # suppress unused variable warnings
+        if num_examples is None:
+            raise NotImplementedError
+        else:
+            col_values = ["" for i in range(num_examples)]
+        # Construct empty-valued dataframe with length matching num_examples.
+        df = pd.DataFrame.from_dict({"test_col": col_values})
+        return datasets.Dataset.from_pandas(df)

prompt2model/dataset_retriever/__init__.py

@@ -0,0 +1,3 @@
+"""Import DatasetRetriever classes."""
+from dataset_retriever.base import BaseRetriever  # noqa: F401
+from dataset_retriever.base import DatasetRetriever  # noqa: F401

prompt2model/dataset_retriever/base.py

@@ -0,0 +1,41 @@
+"""An interface for dataset retrieval."""
+
+from abc import ABC, abstractmethod
+
+import datasets
+import pandas as pd
+from prompt_parser import PromptSpec
+
+
+# pylint: disable=too-few-public-methods
+class DatasetRetriever(ABC):
+    """A class for retrieving datasets.
+
+    TO IMPLEMENT IN SUBCLASSES:
+    def __init__(self):
+    '''Construct a search index from HuggingFace Datasets.'''
+    """
+
+    @abstractmethod
+    def retrieve_datasets(self, prompt_spec: PromptSpec) -> list[datasets.Dataset]:
+        """Retrieve datasets from a prompt specification.
+
+        Args:
+            prompt_spec: A prompt spec (containing a system description).
+
+        Returns:
+            A list of retrieved datasets.
+
+        """
+
+
+class BaseRetriever(DatasetRetriever):
+    """A class for retrieving datasets."""
+
+    def __init__(self):
+        """Construct a mock dataset retriever."""
+
+    def retrieve_datasets(self, prompt_spec: PromptSpec) -> list[datasets.Dataset]:
+        """Return a single empty dataset for testing purposes."""
+        _ = prompt_spec  # suppress unused variable warning
+        return [datasets.Dataset.from_pandas(pd.DataFrame({}))]

prompt2model/demo_creator/gradio_creator.py

@@ -0,0 +1,23 @@
+"""An interface for creating Gradio demos automatically."""
+
+import gradio as gr
+import transformers
+from prompt_parser.base import PromptSpec
+
+
+def create_gradio(
+    model: transformers.PreTrainedModel, prompt_spec: PromptSpec
+) -> gr.Interface:
+    """Create a Gradio interface automatically.
+
+    Args:
+        model: A trained model to expose via a Gradio interface.
+        prompt_spec: A PromptSpec to help choose the visual interface.
+
+    Returns:
+        A Gradio interface for interacting with the model.
+
+    """
+    _ = model, prompt_spec  # suppress unused variable warnings
+    dummy_interface = gr.Interface(lambda input: None, "textbox", "label")
+    return dummy_interface

prompt2model/evaluator/__init__.py

@@ -0,0 +1,2 @@
+"""Import evaluator classes."""
+from evaluator.base import BaseEvaluator, Evaluator  # noqa: F401

prompt2model/evaluator/base.py

@@ -0,0 +1,69 @@
+"""An interface for automatic model evaluation."""
+
+from __future__ import annotations  # noqa FI58
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+import datasets
+import transformers
+from prompt_parser.base import PromptSpec
+
+
+class Evaluator(ABC):
+    """An interface for automatic model evaluation."""
+
+    @abstractmethod
+    def evaluate_model(self, model: transformers.PreTrainedModel) -> dict[str, Any]:
+        """Evaluate a model on a test set.
+
+        Args:
+            model: The model to evaluate.
+
+        Returns:
+            A dictionary of metric values to return.
+
+        """
+
+    @abstractmethod
+    def write_metrics(self, metrics_dict: dict[str, Any], metrics_path: str) -> None:
+        """Write or display metrics to a file.
+
+        Args:
+            metrics_dict: A dictionary of metrics to write.
+            metrics_path: The file path to write metrics to.
+
+        """
+
+
+class BaseEvaluator(Evaluator):
+    """A dummy evaluator that always returns the same metric value."""
+
+    def __init__(
+        self,
+        dataset: datasets.Dataset,
+        metrics: list[datasets.Metric] | None = None,
+        prompt_spec: PromptSpec | None = None,
+    ) -> None:
+        """Initialize the evaluation setting.
+
+        Args:
+            dataset: The dataset to evaluate metrics on.
+            metrics: (Optional) The metrics to use.
+            prompt_spec: (Optional) A PromptSpec to infer the metrics from.
+
+        """
+        self.test_data = dataset
+        self.metrics = metrics
+        self.prompt_spec = prompt_spec
+
+    def evaluate_model(
+        self,
+        model: transformers.PreTrainedModel,
+    ) -> dict[str, Any]:
+        """Return empty metrics dictionary."""
+        return {}
+
+    def write_metrics(self, metrics_dict: dict[str, Any], metrics_path: str) -> None:
+        """Do nothing."""
+        _ = metrics_dict, metrics_path  # suppress unused variable warnings

prompt2model/model_selector/__init__.py

@@ -0,0 +1,3 @@
+"""Import model selector classes."""
+from model_selector.base import DefaultParameterSelector  # noqa: F401
+from model_selector.base import ModelSelector  # noqa: F401