Adding short circuit condition and preventing model from hallucinatin…

…g. Added log conditions too

llm-server/requirements.txt

@@ -114,6 +114,7 @@ pydantic==2.4.2
 pydantic_core==2.10.1
 pymongo==4.5.0
 PyMySQL==1.1.0
+pypdfium2==4.24.0
 PyPika==0.48.9
 PySocks==1.7.1
 python-dateutil==2.8.2

llm-server/routes/root_service.py

@@ -47,6 +47,34 @@
 
 chat = get_chat_model(CHAT_MODELS.gpt_3_5_turbo_16k)
 
+def validate_steps(steps: List[str], swagger_doc: ResolvingParser):
+    try:
+        paths = swagger_doc.specification.get("paths", {})
+        operationIds: List[str] = []
+
+        for path in paths:
+            operations = paths[path]
+            for method in operations:
+                operation = operations[method]
+                operationId = operation.get("operationId")
+                if operationId:
+                    operationIds.append(operationId)
+
+        if not operationIds:
+            logger.warn("No operationIds found in the Swagger document.")
+            return False
+
+        if all(x in operationIds for x in steps):
+            return True
+        else:
+            logger.warn("Model has hallucinated, made up operation id", steps=steps, operationIds=operationIds)
+            return False
+
+    except Exception as e:
+        logger.error(f"An error occurred: {str(e)}")
+        return False
+
+
 
 async def handle_request(
     text: str,
@@ -94,9 +122,15 @@ async def handle_request(
             }
 
         if len(step.ids) > 0:
+            swagger_doc = get_swagger_doc(swagger_url)
+            fl = validate_steps(step.ids, swagger_doc)
+
+            if fl is False:
+                return {"error": None, "response": step.bot_message}
+
             response = await handle_api_calls(
                 ids=step.ids,
-                swagger_doc=get_swagger_doc(swagger_url),
+                swagger_doc=swagger_doc,
                 app=app,
                 bot_id=bot_id,
                 headers=headers,

llm-server/routes/workflow/utils/process_conversation_step.py

@@ -30,16 +30,9 @@ def process_conversation_step(
     bot_id: str,
     base_prompt: str
 ):
+    logger.info("planner data", context=context, api_summaries=api_summaries, prev_conversations=prev_conversations, flows=flows)
     if not session_id:
         raise ValueError("Session id must be defined for chat conversations")
-    # prompt_templates = load_prompts(bot_id)
-
-    logger.debug(
-        "System message classification",
-        incident="system_message_classifier",
-        app=app,
-        context=context,
-    )
     messages: List[BaseMessage] = []
     messages.append(SystemMessage(content=base_prompt))
 
@@ -78,7 +71,7 @@ def process_conversation_step(
 
     messages.append(
         HumanMessage(
-            content="""Based on the information provided to you I want you to answer the questions that follow. Your should respond with a json that looks like the following - 
+            content="""Based on the information provided to you I want you to answer the questions that follow. Your should respond with a json that looks like the following, you must always use the operationIds provided in api summaries. Do not make up an operation id - 
     {{
         "ids": ["list", "of", "operationIds", "for apis to be called"],
         "bot_message": "your response based on the instructions provided at the beginning",
@@ -92,6 +85,9 @@ def process_conversation_step(
     )
 
     messages.append(HumanMessage(content=user_requirement))
+
+
+    logger.info("messages array", messages=messages)
 
     content = cast(str, chat(messages=messages).content)
 

llm-server/routes/workflow/utils/run_openapi_ops.py

@@ -67,7 +67,7 @@ async def run_openapi_operations(
                 # so we don't necessarily have to defined mappers for all api endpoints
                 partial_json = load_json_config(app, operation_id)
                 if not partial_json:
-                    logger.error(
+                    logger.warn(
                         "Config map is not defined for this operationId",
                         incident="config_map_undefined",
                         operation_id=operation_id,

llm-server/shared/utils/opencopilot_utils/embedding_type.py

@@ -6,4 +6,5 @@ class EmbeddingProvider(Enum):
     BARD = "bard"
     azure = "azure"
     llama2 = "llama2"
+    openchat = "openchat"
 

llm-server/shared/utils/opencopilot_utils/get_embeddings.py

@@ -1,11 +1,17 @@
+from functools import lru_cache
 from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.embeddings.ollama import OllamaEmbeddings
 from .embedding_type import EmbeddingProvider
 import os
 from langchain.embeddings.base import Embeddings
-
+from utils.get_logger import CustomLogger
 import os, warnings
 
 
+logger = CustomLogger(module_name=__name__)
+
+LOCAL_IP = os.getenv("LOCAL_IP", "host.docker.internal")
+
 def get_embedding_provider():
     """Gets the chosen embedding provider from environment variables."""
     return os.environ.get("EMBEDDING_PROVIDER")
@@ -31,7 +37,7 @@ def get_openai_embedding():
     """Gets embeddings using the OpenAI embedding provider."""
     openai_api_key = os.environ.get("OPENAI_API_KEY")
 
-    return OpenAIEmbeddings(openai_api_key=openai_api_key, chunk_size=1)
+    return OpenAIEmbeddings(openai_api_key=openai_api_key)
 
 def choose_embedding_provider():
     """Chooses and returns the appropriate embedding provider instance."""
@@ -40,6 +46,10 @@ def choose_embedding_provider():
     if embedding_provider == EmbeddingProvider.azure.value:
         return get_azure_embedding()
 
+    elif embedding_provider == EmbeddingProvider.openchat.value:
+        logger.info("Got ollama embedding provider", provider=embedding_provider)
+        return OllamaEmbeddings(base_url=f"{LOCAL_IP}:11434", model="openchat")
+
     elif embedding_provider == EmbeddingProvider.OPENAI.value or embedding_provider is None:
         if embedding_provider is None:
             warnings.warn("No embedding provider specified. Defaulting to OpenAI.")
@@ -53,6 +63,7 @@ def choose_embedding_provider():
         )
 
 # Main function to get embeddings
+@lru_cache(maxsize=1)
 def get_embeddings() -> Embeddings:
     """Gets embeddings using the chosen embedding provider."""
     return choose_embedding_provider()

llm-server/shared/utils/opencopilot_utils/get_vector_store.py

@@ -27,10 +27,10 @@ def get_vector_store(options: StoreOptions) -> VectorStore:
         vector_store = Qdrant(
             client, collection_name=options.namespace, embeddings=embedding
         )
-        
+
         # vector_store = Qdrant.from_documents([], embedding, url='http://localhost:6333', collection=options.namespace)
 
     else:
         raise ValueError("Invalid STORE environment variable value")
 
-    return vector_store
+    return vector_store

llm-server/utils/get_chat_model.py

@@ -1,3 +1,4 @@
+from typing import Dict
 from langchain.chat_models import ChatOpenAI
 from langchain.chat_models.base import BaseChatModel
 from langchain.chat_models import ChatOllama, ChatAnthropic
@@ -8,45 +9,55 @@
 localip = os.getenv("LOCAL_IP", "localhost")
 
 
+# Create a dictionary to store cached instances
+model_cache: Dict[str, BaseChatModel] = {}
+
+
 def get_chat_model(prop: str) -> BaseChatModel:
+    # Check if the model is already cached
+    if prop in model_cache:
+        return model_cache[prop]
+
     if prop == CHAT_MODELS.gpt_3_5_turbo:
-        return ChatOpenAI(
+        model = ChatOpenAI(
             openai_api_key=os.getenv("OPENAI_API_KEY"),
             model=CHAT_MODELS.gpt_3_5_turbo,
             temperature=0,
         )
-
     elif prop == CHAT_MODELS.gpt_3_5_turbo_16k:
-        return ChatOpenAI(
+        model = ChatOpenAI(
             openai_api_key=os.getenv("OPENAI_API_KEY"),
             model=CHAT_MODELS.gpt_3_5_turbo_16k,
             temperature=0,
         )
     elif prop == CHAT_MODELS.claude_2_0:
-        return ChatAnthropic(
-            anthropic_api_key=os.getenv("CLAUDE_API_KEY", "CLAUDE_API_KEY")  # type: ignore
+        model = ChatAnthropic(
+            anthropic_api_key=os.getenv("CLAUDE_API_KEY", "CLAUDE_API_KEY")
         )
     elif prop == CHAT_MODELS.mistral_openorca:
-        return ChatOllama(
+        model = ChatOllama(
             base_url=f"{localip}:11434",
             model=CHAT_MODELS.mistral_openorca,
             temperature=0,
         )
     elif prop == CHAT_MODELS.nous_hermes:
-        return ChatOllama(
+        model = ChatOllama(
             base_url=f"{localip}:11434",
             model=CHAT_MODELS.nous_hermes,
             temperature=0,
         )
     elif prop == CHAT_MODELS.xwinlm:
-        return ChatOllama(
+        model = ChatOllama(
             base_url=f"{localip}:11434",
             model=CHAT_MODELS.xwinlm,
             temperature=0,
         )
     elif prop == "llama2":
-        return ChatOpenAI(model="llama2", temperature=0)
+        model = ChatOpenAI(model="llama2", temperature=0)
     else:
-        raise ValueError(
-            "Couldn't match one of the supported models, please refer to llm-server/readme.md"
-        )
+        raise ValueError("Couldn't match one of the supported models.")
+
+    # Cache the initialized model
+    model_cache[prop] = model
+
+    return model

llm-server/utils/vector_db/add_workflow.py

@@ -1,12 +1,7 @@
 from routes.workflow.dto.workflow_dto import Workflow
 from typing import Any , List
 from opencopilot_types.workflow_type import WorkflowDataType
-from shared.utils.opencopilot_utils import (
-    StoreOptions,
-    get_embeddings,
-    init_vector_store,
-    get_vector_store
-)
+from shared.utils.opencopilot_utils import (StoreOptions,get_vector_store)
 from langchain.docstore.document import Document
 
 
@@ -27,3 +22,4 @@ def add_workflow_data_to_qdrant(
 
     vector_ids = vector_store.add_documents(docs)
     return vector_ids
+

llm-server/workers/tasks/process_pdfs.py

@@ -4,20 +4,29 @@
 from langchain.document_loaders import PyPDFium2Loader
 from shared.models.opencopilot_db.pdf_data_sources import insert_pdf_data_source, update_pdf_data_source_status
 from shared.utils.opencopilot_utils import get_embeddings, init_vector_store, StoreOptions, get_file_path
+from shared.utils.opencopilot_utils import get_vector_store
+from utils.get_logger import CustomLogger
 
+logger = CustomLogger(module_name=__name__)
+
+embeddings = get_embeddings()
+kb_vector_store = get_vector_store(StoreOptions("knowledgebase"))
 @shared_task
 def process_pdf(file_name: str, bot_id: str):
     try:
+        logger.info("Pdf task picked up", file_name=file_name, bot_id=bot_id)
         insert_pdf_data_source(chatbot_id=bot_id, file_name=file_name, status="PENDING")
         loader = PyPDFium2Loader(get_file_path(file_name))
         raw_docs = loader.load()
         text_splitter = RecursiveCharacterTextSplitter(
             chunk_size=1000, chunk_overlap=200, length_function=len
         )
         docs = text_splitter.split_documents(raw_docs)
-        embeddings = get_embeddings()
-        init_vector_store(docs, embeddings, StoreOptions(namespace="knowledgebase", metadata={"bot_id": bot_id}))
-
+
+        for doc in docs:
+            doc.metadata["bot_id"] = bot_id
+
+        kb_vector_store.add_documents(docs)
         update_pdf_data_source_status(chatbot_id=bot_id, file_name=file_name, status="COMPLETED")
     except Exception as e:
         update_pdf_data_source_status(chatbot_id=bot_id, file_name=file_name, status="FAILED")