diff --git a/take-away/config/inventory.json b/take-away/config/inventory.json index 423ab343..878b7061 100755 --- a/take-away/config/inventory.json +++ b/take-away/config/inventory.json @@ -8,7 +8,7 @@ }, { "id": "002", - "name": "apple", + "name": "red apple", "category": "fruit", "aliases": ["red apple", "gala apple"] }, @@ -41,6 +41,30 @@ "name": "pepsi can", "category": "beverage", "aliases": ["pepsi", "pepsi cola can"] + }, + { + "id": "007", + "name": "pepsi can", + "category": "beverage", + "aliases": ["pepsi", "pepsi cola can"] + }, + { + "id": "007", + "name": "pepsi can", + "category": "beverage", + "aliases": ["pepsi", "pepsi cola can"] + }, + { + "id": "008", + "name": "tomato local", + "category": "fruit", + "aliases": ["tomato local"] + }, + { + "id": "009", + "name": "tomato hybrid", + "category": "fruit", + "aliases": ["tomato hybrid"] } ] } diff --git a/take-away/config/orders.json b/take-away/config/orders.json index c99686e8..e6fcb0cd 100755 --- a/take-away/config/orders.json +++ b/take-away/config/orders.json @@ -1,27 +1,28 @@ { "925": [ - { "name": "green apple", "quantity": 2 }, - { "name": "water bottle", "quantity": 1 } + { "name": "green apple", "quantity": 1 }, + { "name": "water bottle", "quantity": 1 }, + { "name": "tomato local", "quantity": 3 } ], "539": [ - { "name": "apple", "quantity": 2 }, - { "name": "water", "quantity": 1 }, - { "name": "pepsi can", "quantity": 1 }, - { "name": "coke 2 liter bottle", "quantity": 1 }, - { "name": "coke bottle", "quantity": 1 } + { "name": "red apple", "quantity": 1 }, + { "name": "biscuit", "quantity": 1 }, + { "name": "coffee", "quantity": 1 } ], "100": [ - { "name": "coke bottle", "quantity": 2 }, + { "name": "pepsi can", "quantity": 1 }, { "name": "banana", "quantity": 2 }, - { "name": "water bottle", "quantity": 1 } + { "name": "water bottle", "quantity": 1 }, + { "name": "tomato hybrid", "quantity": 3 } ], "384": [ - { "name": "pepsi can", "quantity": 1 }, - { "name": "yellow banana", "quantity": 1 } + { "name": "coke bottle", "quantity": 1 }, + { "name": "banana", "quantity": 2 }, + { "name": "red apple", "quantity": 1 } ], "651": [ - { "name": "banana", "quantity": 2 }, - { "name": "coke 2 liter bottle", "quantity": 1 }, + { "name": "red apple", "quantity": 2 }, + { "name": "pepsi can", "quantity": 1 }, { "name": "water bottle", "quantity": 1 } ], "892": [ @@ -29,5 +30,12 @@ { "name": "banana", "quantity": 1 }, { "name": "coke 2 liter bottle", "quantity": 1 }, { "name": "coke bottle", "quantity": 1 } + ], + "789": [ + { "name": "banana", "quantity": 1 }, + { "name": "green apple", "quantity": 1 }, + { "name": "biscuit", "quantity": 1 }, + { "name": "coke bottle", "quantity": 1 }, + { "name": "coffee", "quantity": 1 } ] } diff --git a/take-away/gradio-ui/gradio_app.py b/take-away/gradio-ui/gradio_app.py index bfd23a59..f5417b9c 100755 --- a/take-away/gradio-ui/gradio_app.py +++ b/take-away/gradio-ui/gradio_app.py @@ -1460,6 +1460,14 @@ def _load_mode_status(): with gr.Row(): refresh_btn = gr.Button("Refresh Results", variant="secondary") clear_history_btn = gr.Button("Clear History", variant="secondary") + + # Auto-refresh detected orders every 5 seconds. + demo.load( + fn=refresh_history, + inputs=None, + outputs=results_history_display, + every=5, + ) refresh_btn.click( fn=refresh_history, @@ -1571,4 +1579,4 @@ def _load_mode_status(): share=False, show_api=False, show_error=True, - ) \ No newline at end of file + ) diff --git a/take-away/src/core/inventory_narrower.py b/take-away/src/core/inventory_narrower.py new file mode 100644 index 00000000..edc9ec0b --- /dev/null +++ b/take-away/src/core/inventory_narrower.py @@ -0,0 +1,208 @@ +""" +Inventory Narrowing Utility + +Narrows the full inventory list to only include items relevant to a specific order. +This reduces VLM cognitive load when catalogs are large (10k+ items). + +Architecture: +1. Expected items: Items required by the order (from orders.json) +2. Confusable neighbors: Items in same category or with similar names +3. Aliases: Alternative names from inventory.json metadata +4. Fallback set: Full inventory if narrowing fails or order not found +""" + +import json +import logging +from typing import List, Dict, Set, Optional + +logger = logging.getLogger(__name__) + + +def load_inventory_metadata(inventory_json_path: str) -> Dict[str, Dict]: + """ + Load inventory metadata with aliases and categories. + + Args: + inventory_json_path: Path to inventory.json file + + Returns: + Dict mapping item names to their metadata (category, aliases, id) + Example: { + "green apple": {"id": "001", "category": "fruit", "aliases": ["granny smith apple", ...]}, + ... + } + """ + try: + with open(inventory_json_path, "r") as f: + data = json.load(f) + + # Convert items list to name-keyed dict for faster lookup + metadata = {} + for item in data.get("items", []): + name = item.get("name", "").lower() + if name: + metadata[name] = { + "id": item.get("id", ""), + "category": item.get("category", "unknown"), + "aliases": [alias.lower() for alias in item.get("aliases", [])] + } + + logger.debug(f"Loaded metadata for {len(metadata)} items from {inventory_json_path}") + return metadata + except Exception as e: + logger.error(f"Failed to load inventory metadata from {inventory_json_path}: {e}") + return {} + + +def narrow_inventory( + full_inventory: List[str], + expected_items: List[Dict], + inventory_metadata: Optional[Dict[str, Dict]] = None, + include_confusable: bool = True, + max_size: int = 50 +) -> List[str]: + """ + Narrow inventory to items relevant to a specific order. + + Strategy: + 1. Include all expected items (exact match from order) + 2. Include confusable neighbors (same category, similar names) + 3. Include aliases (alternative names for expected items) + 4. Cap at max_size to avoid prompt bloat + + Args: + full_inventory: Complete inventory list from config + expected_items: Items expected in order (from orders.json) + Format: [{"name": "apple", "quantity": 2}, ...] + inventory_metadata: Metadata with aliases and categories (from inventory.json) + include_confusable: Whether to include items in same category + max_size: Maximum narrowed inventory size + + Returns: + Narrowed inventory list suitable for VLM prompt + """ + + if not expected_items: + logger.warning("No expected items provided; returning full inventory") + return full_inventory + + # Convert full inventory to lowercase for matching + full_inventory_lower = {item.lower(): item for item in full_inventory} + + # Initialize result with all expected items + narrowed = set() + expected_names = set() + + # Stage 1: Add all expected items (with fuzzy matching) + for item in expected_items: + item_name = item.get("name", "").lower() + if not item_name: + continue + + expected_names.add(item_name) + + # Try exact match + if item_name in full_inventory_lower: + narrowed.add(full_inventory_lower[item_name]) + else: + # Try substring match (e.g., "water" matches "water bottle") + matches = [ + inv_item for inv_lower, inv_item in full_inventory_lower.items() + if item_name in inv_lower or inv_lower in item_name + ] + if matches: + narrowed.add(matches[0]) # Add first match + logger.debug(f"Fuzzy matched '{item_name}' to '{matches[0]}'") + else: + logger.warning(f"Expected item '{item_name}' not found in inventory") + + # Stage 2: Add confusable neighbors (same category) + if include_confusable and inventory_metadata: + for item in expected_items: + item_name = item.get("name", "").lower() + if not item_name or item_name not in inventory_metadata: + continue + + category = inventory_metadata[item_name].get("category", "") + if not category: + continue + + # Find all items in same category + neighbors = [ + inv_lower for inv_lower, meta in inventory_metadata.items() + if meta.get("category") == category and inv_lower not in expected_names + ] + + # Add up to 2 neighbors per expected item to limit growth + for neighbor_lower in neighbors[:2]: + # Map back to original inventory capitalization + if neighbor_lower in full_inventory_lower: + narrowed.add(full_inventory_lower[neighbor_lower]) + + # Stage 3: Add aliases for all narrowed items + if inventory_metadata: + items_to_check = list(narrowed) + for item in items_to_check: + item_lower = item.lower() + if item_lower in inventory_metadata: + aliases = inventory_metadata[item_lower].get("aliases", []) + for alias in aliases: + # Try to find alias in full inventory + for inv_lower, inv_item in full_inventory_lower.items(): + if alias in inv_lower: + narrowed.add(inv_item) + break + + # Convert to sorted list for consistent ordering + result = sorted(list(narrowed)) + + # Cap size + if len(result) > max_size: + logger.warning(f"Narrowed inventory size {len(result)} exceeds max {max_size}; truncating") + result = result[:max_size] + + logger.info( + f"Narrowed inventory from {len(full_inventory)} to {len(result)} items " + f"(expected: {len(expected_names)}, confusable: {len(result) - len(expected_names)})" + ) + logger.debug(f"Narrowed inventory: {result}") + + return result + + +def build_narrowed_inventory_text( + full_inventory: List[str], + expected_items: List[Dict], + inventory_metadata: Optional[Dict[str, Dict]] = None, + fallback_to_full: bool = True +) -> str: + """ + Build inventory text for VLM prompt using narrowed list. + + Args: + full_inventory: Complete inventory list + expected_items: Expected items from order + inventory_metadata: Optional metadata for aliases + fallback_to_full: If narrowing fails, use full inventory + + Returns: + Formatted inventory text for prompt (newline-separated list with dashes) + """ + try: + # Try to narrow inventory + if expected_items: + narrowed = narrow_inventory(full_inventory, expected_items, inventory_metadata) + else: + narrowed = full_inventory + + # Format as bullet list for prompt + inventory_text = "\n".join(f"- {item}" for item in narrowed) + return inventory_text + + except Exception as e: + logger.error(f"Error building narrowed inventory: {e}") + if fallback_to_full: + logger.warning("Falling back to full inventory") + return "\n".join(f"- {item}" for item in full_inventory) + else: + raise diff --git a/take-away/src/core/vlm_service.py b/take-away/src/core/vlm_service.py index b24134d2..48e38cde 100755 --- a/take-away/src/core/vlm_service.py +++ b/take-away/src/core/vlm_service.py @@ -11,6 +11,7 @@ from .order_results import add_result from .validation_agent import validate_order from .vlm_backend_factory import VLMBackendFactory +from .inventory_narrower import load_inventory_metadata, build_narrowed_inventory_text import json # Configure logging @@ -75,6 +76,13 @@ logger.error(f"Failed to load orders.json: {e}") EXPECTED_ORDERS = {} +# ============================================================ +# LOAD INVENTORY METADATA (for aliases and categories) +# ============================================================ + +INVENTORY_METADATA_FILE = "/config/inventory.json" +INVENTORY_METADATA = load_inventory_metadata(INVENTORY_METADATA_FILE) + # ============================================================ # MINIO CLIENT # ============================================================ @@ -183,12 +191,25 @@ def process(self, images: list[np.ndarray], unique_id: str = None, expected_item else: order_hint = "" + # ===== Build inventory text: narrowed for specific orders, full as fallback ===== + if expected_items: + # Narrow inventory to order-relevant items for better VLM focus + inventory_text_for_prompt = build_narrowed_inventory_text( + INVENTORY, + expected_items, + inventory_metadata=INVENTORY_METADATA, + fallback_to_full=True + ) + else: + # No specific order context; use full inventory + inventory_text_for_prompt = INVENTORY_TEXT + # ===== Inventory-aware + order-aware prompt ===== prompt = ( f"You will receive {num_frames} frames from a grocery order packing station.\n\n" f"{order_hint}" f"Recognize products ONLY from this inventory list:\n" - f"{INVENTORY_TEXT}\n\n" + f"{inventory_text_for_prompt}\n\n" f"Rules:\n" f"- Always choose the closest matching inventory item name.\n" f"- Never invent new product names outside the list.\n" diff --git a/take-away/src/parallel/vlm_scheduler.py b/take-away/src/parallel/vlm_scheduler.py index 36e2bded..e5336946 100755 --- a/take-away/src/parallel/vlm_scheduler.py +++ b/take-away/src/parallel/vlm_scheduler.py @@ -36,6 +36,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'core')) from ovms_client import OVMSVLMClient # type: ignore from config_loader import load_config # type: ignore +from inventory_narrower import load_inventory_metadata, build_narrowed_inventory_text # type: ignore logger = logging.getLogger(__name__) @@ -51,6 +52,20 @@ INVENTORY_TEXT = "\n".join(f"- {item}" for item in INVENTORY) +# Load inventory metadata (for aliases and categories) +INVENTORY_METADATA_FILE = "/config/inventory.json" +INVENTORY_METADATA = load_inventory_metadata(INVENTORY_METADATA_FILE) + +# Load expected orders +ORDERS_FILE = "/config/orders.json" +try: + with open(ORDERS_FILE, "r") as f: + EXPECTED_ORDERS = json.load(f) + logger.info(f"VLM Scheduler loaded {len(EXPECTED_ORDERS)} expected orders from {ORDERS_FILE}") +except Exception as e: + logger.error(f"Failed to load orders.json: {e}") + EXPECTED_ORDERS = {} + class VLMScheduler: """ @@ -443,8 +458,8 @@ def _send_to_ovms(self, request: VLMRequest) -> VLMResponse: if not images: raise Exception("No valid images in request") - # Build prompt using existing format from vlm_service.py - prompt = self._build_vlm_prompt(len(images)) + # Build prompt using narrowed inventory for this order + prompt = self._build_vlm_prompt(len(images), order_id=request.order_id) # Call OVMS VLM client with unique_id for metrics logging output = self._vlm_client.generate( @@ -492,19 +507,45 @@ def _send_to_ovms(self, request: VLMRequest) -> VLMResponse: except Exception as e: raise Exception(f"VLM inference error: {e}") - def _build_vlm_prompt(self, num_images: int) -> str: + def _build_vlm_prompt(self, num_images: int, order_id: str = None) -> str: """ Build VLM prompt with inventory constraints. - Matches the EXACT prompt format used in vlm_service.py. + When order_id is provided, uses narrowed inventory for better VLM focus. + Otherwise uses full inventory as fallback. + + Args: + num_images: Number of frames in the request + order_id: Order ID to lookup expected items for narrowing + + Returns: + Formatted VLM prompt """ # OVMS automatically associates images with the prompt img_tags = "" + # Get expected items for this order (if available) + expected_items = None + if order_id: + expected_items = EXPECTED_ORDERS.get(str(order_id)) + if expected_items: + logger.debug(f"Found {len(expected_items)} expected items for order {order_id}") + + # Build inventory text: narrowed for specific orders, full as fallback + if expected_items: + inventory_text_for_prompt = build_narrowed_inventory_text( + INVENTORY, + expected_items, + inventory_metadata=INVENTORY_METADATA, + fallback_to_full=True + ) + else: + inventory_text_for_prompt = INVENTORY_TEXT + prompt = ( f"You will receive {num_images} frames.\n\n" f"Recognize products ONLY from this inventory list:\n" - f"{INVENTORY_TEXT}\n\n" + f"{inventory_text_for_prompt}\n\n" f"Rules:\n" f"- Always choose the closest matching inventory item name.\n" f"- Never invent new product names outside the list.\n"