[FEAT] Add LoReTTA

src/peft/__init__.py

@@ -76,6 +76,7 @@
     LoraConfig,
     LoraModel,
     LoraRuntimeConfig,
+    LORETTaConfig,
     MissConfig,
     MissModel,
     MultitaskPromptTuningConfig,
@@ -171,6 +172,7 @@
     "LoraConfig",
     "LoraModel",
     "LoraRuntimeConfig",
+    "LORETTaConfig",
     "MissConfig",
     "MissModel",
     "MultitaskPromptTuningConfig",

src/peft/tuners/__init__.py

@@ -35,6 +35,7 @@
     get_eva_state_dict,
     initialize_lora_eva_weights,
 )
+from .loretta import LORETTaConfig
 from .miss import MissConfig, MissModel
 from .mixed import MixedModel
 from .multitask_prompt_tuning import MultitaskPromptEmbedding, MultitaskPromptTuningConfig, MultitaskPromptTuningInit
@@ -84,6 +85,7 @@
     "LoraConfig",
     "LoraModel",
     "LoraRuntimeConfig",
+    "LORETTaConfig",
     "MissConfig",
     "MissModel",
     "MixedModel",

src/peft/tuners/loretta/__init__.py

@@ -0,0 +1,3 @@
+from .config import LORETTaConfig
+
+__all__ = ["LORETTaConfig"]

src/peft/tuners/loretta/config.py

@@ -0,0 +1,6 @@
+from dataclasses import dataclass
+from peft.config import PeftConfig
+
+@dataclass
+class LORETTaConfig(PeftConfig):
+    pass

tests/test_loretta.py

@@ -0,0 +1,16 @@
+"""
+Relevant links used to create this test
+1. https://heidloff.net/article/efficient-fine-tuning-lora/
+2.
+"""
+from peft import LORETTaConfig
+
+
+class TestLORETTA:
+    @staticmethod
+    def config_test():
+        loretta_config = LORETTaConfig()
+
+if __name__=="__main__":
+    TestLORETTA.config_test()
+    pass