LLM integration

.gitignore

@@ -83,3 +83,4 @@ secrets.yaml
 official.csv
 fact.png
 jupyterhub/.env-dev
+/.run/

backend/app/main.py

@@ -52,6 +52,7 @@
     keycloak,
     licenses,
     listeners,
+    llm,
     metadata,
     metadata_datasets,
     metadata_files,
@@ -261,6 +262,9 @@
 )
 api_router.include_router(status.router, prefix="/status", tags=["status"])
 api_router.include_router(keycloak.router, prefix="/auth", tags=["auth"])
+
+api_router.include_router(llm.router, prefix="/llm", tags=["llm"])
+
 app.include_router(api_router, prefix=settings.API_V2_STR)
 
 

backend/app/routers/llm.py

@@ -0,0 +1,81 @@
+import os
+
+from fastapi import APIRouter, HTTPException
+from openai import OpenAI
+from openai.types.chat import ChatCompletionMessage
+from pydantic import BaseModel
+from qdrant_client import QdrantClient
+
+router = APIRouter()
+
+
+class PromptRequest(BaseModel):
+    prompt: str
+
+
+class PromptResponse(BaseModel):
+    response: ChatCompletionMessage
+
+
+@router.post("", response_model=PromptResponse)
+async def llm(request: PromptRequest):
+    def get_embedding(text, model="text-embedding-ada-002"):
+        text = text.replace("\n", " ")
+        return client.embeddings.create(input=[text], model=model).data[0].embedding
+
+    client = OpenAI(
+        api_key=os.getenv("OPENAI_API_KEY"),
+    )
+    qdrant = QdrantClient(url="http://localhost:6333")
+
+    embedding = get_embedding(request.prompt)
+
+    search_result = qdrant.search(
+        collection_name="txt_vector_collection", query_vector=embedding, limit=3
+    )
+
+    # Extract context from search results
+    # context_list = [point.payload['page_content'] for point in search_result]
+    # context = "\n\n".join(context_list)
+
+    # messages = [
+    #     {"role": "system", "content": "You are a helpful assistant."},
+    #     {"role": "system", "content": f"Context: {context}"},
+    #     {"role": "user", "content": request.prompt},
+    # ]
+
+    # Extract context from search results
+    context_list = [point.payload["page_content"] for point in search_result]
+    context = "\n\n".join(
+        [f"... doc {i + 1} ...\n{doc}" for i, doc in enumerate(context_list)]
+    )
+
+    # Format the message according to the specified structure
+    formatted_message = f"""Please answer the question using the provided context.
+    <Context>
+    {context}
+    <Context/>
+
+    Now answer the user question:
+    <user question>
+    {request.prompt}
+    <user question/>"""
+
+    print("formatted message", formatted_message)
+
+    # Include the formatted message in the messages
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": formatted_message},
+    ]
+
+    try:
+        completion = client.chat.completions.create(
+            model="gpt-4o-mini",
+            messages=messages,
+            max_tokens=150,
+            temperature=0.7,
+        )
+        return PromptResponse(response=completion.choices[0].message)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))

docker-compose.extractors.yml

@@ -1,7 +1,19 @@
 version: '3.7'
+services:
 
 
-services:
+  detr-resnet-50-panoptic:
+    image: clowder/extractors-detr-resnet-50-panoptic:${CLOWDER_VERSION:-latest}
+    networks:
+      - clowder2
+    depends_on:
+      - rabbitmq
+      - extractors-messages
+      - extractors-heartbeat
+    environment:
+      CLOWDER_VERSION: 2
+      RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F
+
   name-entity-recognition:
     image: socialmediamacroscope/name_entity_recognition_extractor:latest
     environment:

docker-compose.llm.yml

@@ -0,0 +1,15 @@
+version: '3'
+services:
+  qdrant:
+    image: qdrant/qdrant:latest
+    networks:
+      - clowder2
+    volumes:
+        - qdrant-data:/qdrant/storage
+        - qdrant-snapshots:/qdrant/snapshots
+    ports:
+      - "6333:6333"
+
+volumes:
+    qdrant-data:
+    qdrant-snapshots: