remove pkg_resources and distutils (#1052)

* removed pkg_resources

* add reqs

* remove duplicate

* preserve note from duplicate

* format

* fix for py3.8

* format

* nit

* remove distutils as well and add notes

* notes

* nits

* fix static_resource for py38 again

format.sh

@@ -10,6 +10,7 @@ else
     exit 1
 fi
 
+echo "Sorting imports in $FORMAT_PATH"
+isort $FORMAT_PATH -s .conda -s trulens_eval/.conda
 echo "Formatting $FORMAT_PATH"
-isort $FORMAT_PATH
-yapf --style .style.yapf -r -i --verbose --parallel -r -i $FORMAT_PATH -e .conda
+yapf --style .style.yapf -r -i --verbose --parallel -r -i $FORMAT_PATH -e .conda -e trulens_eval/.conda

trulens_eval/.gitignore

@@ -0,0 +1 @@
+dist

trulens_eval/setup.py

@@ -1,34 +1,50 @@
-from distutils import log
-from distutils.command.build import build
+"""
+# _TruLens-Eval_ build script
+
+To build:
+
+```bash
+python setup.py bdist_wheel
+```
+
+TODO: It is more standard to configure a lot of things we configure
+here in a setup.cfg file instead. It is unclear whether we can do everything
+with a config file though so we may need to keep this script or parts of it.
+"""
+
 import os
-from pathlib import Path
 
-from pkg_resources import parse_requirements
+from pip._internal.req import parse_requirements
 from setuptools import find_namespace_packages
 from setuptools import setup
+from setuptools.command.build import build
+from setuptools.logging import logging
 
 required_packages = list(
     map(
-        str,
-        parse_requirements(Path("trulens_eval/requirements.txt").read_text())
+        lambda pip_req: str(pip_req.requirement),
+        parse_requirements("trulens_eval/requirements.txt", session=None)
     )
 )
 optional_packages = list(
     map(
-        str,
+        lambda pip_req: str(pip_req.requirement),
         parse_requirements(
-            Path("trulens_eval/requirements.optional.txt").read_text()
+            "trulens_eval/requirements.optional.txt", session=None
         )
     )
 )
 
-
-class javascript_build(build):
-
+class BuildJavascript(build):
     def run(self):
-        log.info("running npm i")
+        """Custom build command to run npm commands before building the package.
+    
+        This builds the record timeline component for the dashboard.
+        """
+
+        logging.info("running npm i")
         os.system("npm i --prefix trulens_eval/react_components/record_viewer")
-        log.info("running npm run build")
+        logging.info("running npm run build")
         os.system(
             "npm run --prefix trulens_eval/react_components/record_viewer build"
         )
@@ -38,7 +54,7 @@ def run(self):
 setup(
     name="trulens_eval",
     cmdclass={
-        'build': javascript_build,
+        'build': BuildJavascript,
     },
     include_package_data=True,  # includes things specified in MANIFEST.in
     packages=find_namespace_packages(

trulens_eval/tests/unit/test_feedback_score_generation.py

@@ -3,7 +3,6 @@
 pattern matching of feedback scores from LLM responses.
 """
 
-
 import pytest
 
 from trulens_eval.utils.generated import ParseError

trulens_eval/trulens_eval/Leaderboard.py

@@ -20,8 +20,6 @@
 from trulens_eval import Tru
 from trulens_eval.ux import styles
 from trulens_eval.ux.components import draw_metadata
-
-
 from trulens_eval.ux.page_config import set_page_config
 
 set_page_config(page_title="Leaderboard")

trulens_eval/trulens_eval/__init__.py

@@ -85,7 +85,6 @@
 `utils/asynchro.py`
 `utils/text.py`
 `utils/__init__.py`
-
 """
 
 __version_info__ = (0, 27, 0)

trulens_eval/trulens_eval/database/sqlalchemy_db.py

@@ -3,8 +3,9 @@
 import json
 import logging
 from sqlite3 import OperationalError
-from typing import (Any, ClassVar, Dict, Iterable, List, Optional, Sequence,
-                    Tuple, Union)
+from typing import (
+    Any, ClassVar, Dict, Iterable, List, Optional, Sequence, Tuple, Union
+)
 import warnings
 
 import numpy as np

trulens_eval/trulens_eval/feedback/feedback.py

@@ -24,11 +24,12 @@
 
 from trulens_eval.feedback.provider.base import LLMProvider
 from trulens_eval.feedback.provider.endpoint.base import Endpoint
-from trulens_eval.schema import AppDefinition, FeedbackOnMissingParameters
+from trulens_eval.schema import AppDefinition
 from trulens_eval.schema import Cost
 from trulens_eval.schema import FeedbackCall
 from trulens_eval.schema import FeedbackCombinations
 from trulens_eval.schema import FeedbackDefinition
+from trulens_eval.schema import FeedbackOnMissingParameters
 from trulens_eval.schema import FeedbackResult
 from trulens_eval.schema import FeedbackResultID
 from trulens_eval.schema import FeedbackResultStatus
@@ -64,10 +65,13 @@
 AggCallable = Callable[[Iterable[float]], float]
 """Signature of aggregation functions."""
 
+
 class InvalidSelector(Exception):
     """Raised when a selector names something that is missing in a record/app."""
 
-    def __init__(self, selector: Lens, source_data: Optional[Dict[str, Any]] = None):
+    def __init__(
+        self, selector: Lens, source_data: Optional[Dict[str, Any]] = None
+    ):
         self.selector = selector
         self.source_data = source_data
 
@@ -77,6 +81,7 @@ def __str__(self):
     def __repr__(self):
         return f"InvalidSelector({self.selector})"
 
+
 def rag_triad(
     provider: LLMProvider,
     question: Optional[Lens] = None,
@@ -817,16 +822,16 @@ def run(
         except InvalidSelector as e:
             # Handle the cases where a selector named something that does not
             # exist in source data.
-            
+
             if self.if_missing == FeedbackOnMissingParameters.ERROR:
                 feedback_result.status = FeedbackResultStatus.FAILED
                 raise e
 
             if self.if_missing == FeedbackOnMissingParameters.WARN:
                 feedback_result.status = FeedbackResultStatus.SKIPPED
                 logger.warning(
-                    "Feedback %s cannot run as %s does not exist in record or app.", self.name,
-                    e.selector
+                    "Feedback %s cannot run as %s does not exist in record or app.",
+                    self.name, e.selector
                 )
                 return feedback_result
 

trulens_eval/trulens_eval/feedback/groundedness.py

@@ -1,6 +1,8 @@
 import logging
 from typing import Dict, List, Optional, Tuple
 
+import nltk
+from nltk.tokenize import sent_tokenize
 import numpy as np
 from tqdm.auto import tqdm
 
@@ -17,9 +19,6 @@
 from trulens_eval.utils.pyschema import WithClassInfo
 from trulens_eval.utils.serial import SerialModel
 
-from nltk.tokenize import sent_tokenize
-import nltk
-
 with OptionalImports(messages=REQUIREMENT_BEDROCK):
     from trulens_eval.feedback.provider.bedrock import Bedrock
 
@@ -75,7 +74,7 @@ def __init__(
         if groundedness_provider is None:
             logger.warning("Provider not provided. Using OpenAI.")
             groundedness_provider = OpenAI()
-            
+
         nltk.download('punkt')
         super().__init__(groundedness_provider=groundedness_provider, **kwargs)
 

trulens_eval/trulens_eval/feedback/provider/base.py

@@ -1244,4 +1244,3 @@ def stereotypes_with_cot_reasons(self, prompt: str,
         )
 
         return self.generate_score_and_reasons(system_prompt, user_prompt)
-

trulens_eval/trulens_eval/pages/Apps.py

@@ -15,7 +15,6 @@
 import streamlit as st
 from ux.page_config import set_page_config
 
-
 st.runtime.legacy_caching.clear_cache()
 
 set_page_config(page_title="App Runner")

trulens_eval/trulens_eval/pages/Evaluations.py

@@ -39,13 +39,11 @@
 from trulens_eval.ux.components import write_or_json
 from trulens_eval.ux.styles import cellstyle_jscode
 
-
 st.runtime.legacy_caching.clear_cache()
 
 set_page_config(page_title="Evaluations")
 st.title("Evaluations")
 
-
 tru = Tru()
 lms = tru.db
 
@@ -65,9 +63,7 @@
 
 
 def render_component(
-    query: Lens,
-    component: ComponentView,
-    header: bool=True
+    query: Lens, component: ComponentView, header: bool = True
 ) -> None:
     """Render the accessor/path within the wrapped app of the component."""
 
@@ -107,12 +103,11 @@ def render_component(
 
 
 def render_record_metrics(
-    app_df: pd.DataFrame,
-    selected_rows: pd.DataFrame
+    app_df: pd.DataFrame, selected_rows: pd.DataFrame
 ) -> None:
     """Render record level metrics (e.g. total tokens, cost, latency) compared
     to the average when appropriate."""
-    
+
     app_specific_df = app_df[app_df["app_id"] == selected_rows["app_id"][0]]
 
     token_col, cost_col, latency_col = st.columns(3)

trulens_eval/trulens_eval/requirements.txt

@@ -7,7 +7,11 @@ nltk               >= 3.8.1     # groundedness.py
 requests           >= 2.31.0
 nest-asyncio       >= 1.5.8
 typing_extensions  >= 4.9.0
-psutil             >= 5.9.8     # tru.py
+psutil             >= 5.9.8  # tru.py dashboard starting/ending
+pip                >= 24.0   # for requirements management
+
+packaging          >= 23.2   # for requirements, resources management
+# also: resolves version conflict with langchain-core
 
 # Secrets/env management
 python-dotenv      >= 1.0.0
@@ -20,7 +24,6 @@ merkle-json        >= 1.0.0
 langchain          >= 0.1.14  # required for cost tracking even outside of langchain
 langchain-core     >= 0.1.6  # required by langchain
 typing-inspect     >= 0.8.0  # fixes bug with langchain on python < 3.9 
-packaging          == 23.2 # resolves version conflict with langchain-core
 
 # Models
 # All models are optional.

trulens_eval/trulens_eval/schema.py

@@ -211,7 +211,7 @@ class Record(serial.SerialModel, Hashable):
 
     model_config: ClassVar[dict] = {
         # for `Future[FeedbackResult]`
-        'arbitrary_types_allowed' : True
+        'arbitrary_types_allowed': True
     }
 
     record_id: RecordID
@@ -529,6 +529,7 @@ class FeedbackOnMissingParameters(str, Enum):
     [SKIPPED][trulens_eval.schema.FeedbackResultStatus.SKIPPED].
     """
 
+
 class FeedbackCall(serial.SerialModel):
     """Invocations of feedback function results in one of these instances.
     
@@ -754,7 +755,8 @@ def __init__(
                                        pyschema.Method]] = None,
         aggregator: Optional[Union[pyschema.Function, pyschema.Method]] = None,
         if_exists: Optional[serial.Lens] = None,
-        if_missing: FeedbackOnMissingParameters = FeedbackOnMissingParameters.ERROR,
+        if_missing: FeedbackOnMissingParameters = FeedbackOnMissingParameters.
+        ERROR,
         selectors: Optional[Dict[str, serial.Lens]] = None,
         name: Optional[str] = None,
         higher_is_better: Optional[bool] = None,

trulens_eval/trulens_eval/tru.py

@@ -21,7 +21,6 @@
 
 import humanize
 import pandas
-import pkg_resources
 from tqdm.auto import tqdm
 from typing_extensions import Annotated
 from typing_extensions import Doc
@@ -30,12 +29,12 @@
 from trulens_eval import schema
 from trulens_eval.database import sqlalchemy_db
 from trulens_eval.feedback import feedback
-from trulens_eval.utils import imports
 from trulens_eval.utils import notebook_utils
 from trulens_eval.utils import python
 from trulens_eval.utils import serial
 from trulens_eval.utils import text
 from trulens_eval.utils import threading as tru_threading
+from trulens_eval.utils.imports import static_resource
 from trulens_eval.utils.python import Future  # code style exception
 
 pp = PrettyPrinter()
@@ -955,9 +954,7 @@ def run_dashboard(
             print("Credentials file already exists. Skipping writing process.")
 
         #run leaderboard with subprocess
-        leaderboard_path = pkg_resources.resource_filename(
-            'trulens_eval', 'Leaderboard.py'
-        )
+        leaderboard_path = static_resource('Leaderboard.py')
 
         if Tru._dashboard_proc is not None:
             print("Dashboard already running at path:", Tru._dashboard_urls)

trulens_eval/trulens_eval/tru_llama.py

@@ -8,7 +8,6 @@
 from pprint import PrettyPrinter
 from typing import Any, Callable, ClassVar, Dict, Optional, Union
 
-from pkg_resources import parse_version
 from pydantic import Field
 
 from trulens_eval.app import App
@@ -18,6 +17,7 @@
 from trulens_eval.utils.imports import Dummy
 from trulens_eval.utils.imports import get_package_version
 from trulens_eval.utils.imports import OptionalImports
+from trulens_eval.utils.imports import parse_version
 from trulens_eval.utils.imports import REQUIREMENT_LLAMA
 from trulens_eval.utils.pyschema import Class
 from trulens_eval.utils.pyschema import FunctionOrMethod