Made test stubs for multimedia processor. Split up llm optimizer tests for LLM document. Created factory classes for unified test data.

Author: the-ride-never-ends

.vscode/settings.json

@@ -18,11 +18,15 @@
         "Jaccard",
         "kylerose",
         "LAION",
+        "ndarray",
+        "ndarrays",
         "psutil",
         "pydantic",
+        "pytest",
         "sparql",
         "tesseract",
-        "uvicorn"
+        "uvicorn",
+        "ytdlp"
     ],
     "cSpell.ignoreWords": [
         "Chunker",

_example_test_format.md

@@ -15,10 +15,12 @@ from resource_monitor import (
     ResourceMonitor,
 )
 
-from _test_utils import (
-    _has_good_callable_metadata,
-    _raise_on_fake_code,
-    _raise_on_mocked_code,
+from tests._test_utils import (
+    has_good_callable_metadata,
+    raise_on_bad_callable_code_quality,
+    get_ast_tree,
+    BadDocumentationError,
+    BadSignatureError
 )
 
 # Check if the ResourceMonitor class has the required attributes
@@ -40,7 +42,7 @@ class TestErrorMonitorInitialization(unittest.TestCase):
         Ensure that the docstring of the ErrorMonitor class meets the standards set forth in `_example_docstring_format.md`.
         """
         try:
-            _has_good_callable_metadata(ResourceMonitor)
+            has_good_callable_metadata(ResourceMonitor)
         except Exception as e:
             self.fail(f"Callable metadata in ResourceMonitor does not meet standards: {e}")
 
@@ -101,7 +103,7 @@ class TestErrorMonitorInitialization:
         Ensure that the docstring of the ErrorMonitor class meets the standards set forth in `_example_docstring_format.md`.
         """
         try:
-            _has_good_callable_metadata(ResourceMonitor)
+            has_good_callable_metadata(ResourceMonitor)
         except Exception as e:
             self.fail(f"Callable metadata in ResourceMonitor does not meet standards: {e}")
 
@@ -117,12 +119,6 @@ class TestErrorMonitorInitialization:
         WHEN ErrorMonitor is initialized
         THEN expect:
             - Instance created successfully
-            - _logger is set from resources['logger']
-            - _suppress_errors is set from configs.processing.suppress_errors
-            - _root_dir is set from configs.paths.ROOT_DIR
-            - _error_counters initialized as empty dict
-            - _error_types initialized as empty set
-            - traceback and datetime attributes are set from resources
         """
         raise NotImplementedError("test_init_with_valid_resources_and_configs test needs to be implemented")
 

ipfs_datasets_py/mcp_server/tools/vector_tools/vector_store_management.py

@@ -467,13 +467,13 @@ async def search_vector_index(
     Returns:
         Dictionary with search results
     """
-    try:
+    try: # TODO _search_qdrant_index and _search_elasticsearch_index are hallucinated. Make them or remove them.
         if backend == "faiss":
             return await _search_faiss_index(index_name, query, top_k, config)
         elif backend == "qdrant":
-            return await _search_qdrant_index(index_name, query, top_k, filters, config)
+            pass # return await _search_qdrant_index(index_name, query, top_k, filters, config)
         elif backend == "elasticsearch":
-            return await _search_elasticsearch_index(index_name, query, top_k, filters, config)
+            pass # return await _search_elasticsearch_index(index_name, query, top_k, filters, config)
         else:
             return {
                 "status": "error",

ipfs_datasets_py/multimedia/media_processor.py

@@ -4,16 +4,50 @@
 This module provides a unified interface for processing multimedia content
 using various backends like FFmpeg and yt-dlp.
 """
-
-import asyncio
 import logging
-from typing import Dict, List, Any, Optional, Union
+from typing import Dict, Any, Optional
 from pathlib import Path
 
 from .ytdlp_wrapper import YtDlpWrapper, YTDLP_AVAILABLE
 from .ffmpeg_wrapper import FFmpegWrapper, FFMPEG_AVAILABLE
 
-logger = logging.getLogger(__name__)
+
+
+def make_media_processor(
+    default_output_dir: Optional[str] = None,
+    enable_logging: bool = True,
+    logger: logging.Logger = logging.getLogger(__name__),
+    ytdlp: Optional[YtDlpWrapper] = None,
+    ffmpeg: Optional[FFmpegWrapper] = None
+) -> 'MediaProcessor':
+    """
+    Factory function to create a MediaProcessor instance.
+
+    Args:
+        default_output_dir (Optional[str], optional): Default directory path for output files.
+            If not provided, defaults to the current working directory. The directory will
+            be created if it doesn't exist. Defaults to None.
+        enable_logging (bool, optional): Enable detailed logging for debugging and monitoring.
+            When True, logs initialization status, operation progress, and error details.
+            Defaults to True.
+
+    Returns:
+        MediaProcessor: Configured MediaProcessor instance.
+    """
+    input_dict = {
+        "default_output_dir": default_output_dir,
+        "enable_logging": enable_logging,
+        "logger": logger,
+        "ytdlp": ytdlp,
+        "ffmpeg": ffmpeg
+    }
+    if input_dict["ytdlp"] is None and YTDLP_AVAILABLE:
+        input_dict["ytdlp"] = YtDlpWrapper(default_output_dir, enable_logging)
+
+    if input_dict["ffmpeg"] is None and FFMPEG_AVAILABLE:
+        input_dict["ffmpeg"] = FFmpegWrapper(default_output_dir, enable_logging)
+
+    return MediaProcessor(**input_dict)
 
 
 class MediaProcessor:
@@ -96,8 +130,12 @@ class MediaProcessor:
     """
 
     def __init__(self, 
-                 default_output_dir: Optional[str] = None,
-                 enable_logging: bool = True) -> None:
+                default_output_dir: Optional[str] = None,
+                enable_logging: bool = True,
+                logger: logging.Logger =  logging.getLogger(__name__),
+                ytdlp: Optional[YtDlpWrapper] = None,
+                ffmpeg: Optional[FFmpegWrapper] = None
+                ) -> None:
         """
         Initialize the MediaProcessor with specified configuration options.
 
@@ -150,12 +188,12 @@ def __init__(self,
             The initialization process logs the availability status of backend libraries
             (yt-dlp and FFmpeg) when logging is enabled.
         """
-        self.default_output_dir = Path(default_output_dir) if default_output_dir else Path.cwd()
-        self.enable_logging = enable_logging
-        
+        self.default_output_dir: Path = Path(default_output_dir) if default_output_dir else Path.cwd()
+        self.enable_logging: bool = enable_logging
+
         # Initialize component wrappers
-        self.ytdlp = YtDlpWrapper(default_output_dir, enable_logging) if YTDLP_AVAILABLE else None
-        self.ffmpeg = FFmpegWrapper(default_output_dir, enable_logging) if FFMPEG_AVAILABLE else None
+        self.ytdlp: Optional[YtDlpWrapper] = ytdlp
+        self.ffmpeg: Optional[FFmpegWrapper] = ffmpeg
 
         logger.info(f"MediaProcessor initialized - YT-DLP: {YTDLP_AVAILABLE}, FFmpeg: {FFMPEG_AVAILABLE}")
 
@@ -273,7 +311,7 @@ async def download_and_convert(self,
             return download_result
 
         except Exception as e:
-            logger.error(f"Error in download_and_convert: {e}")
+            self.logger.error(f"Error in download_and_convert: {e}")
             return {
                 "status": "error",
                 "error": str(e)

ipfs_datasets_py/pdf_processing/batch_processor.py

@@ -607,7 +607,7 @@ async def process_batch(self,
             failed_jobs=0,
             pending_jobs=len(pdf_paths),
             processing_jobs=0,
-            start_time=datetime.utcnow().isoformat()
+            start_time=datetime.now().isoformat()
         )
         self.active_batches[batch_id] = batch_status