adjusted embedding now to e5-base

app/embed_documents.py

@@ -6,6 +6,7 @@ from langchain_core.documents import Document
 from langchain_qdrant import QdrantVectorStore
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community.document_loaders import PyPDFLoader
 
 import os
 from pathlib import Path
@@ -13,6 +14,7 @@ from uuid import uuid4
 
 # %%
 QDRANT_URL = os.getenv('QDRANT_URL')
+QDRANT_API_KEY = os.getenv('QDRANT_API_KEY')
 
 # %%
 FAQ_COLLECTION = "faqs"
@@ -23,12 +25,12 @@ SUPPORT_COLLECTION = "support"
 PRODUCT_COLLECTION = "product"
 
 # %%
-client = QdrantClient(url=QDRANT_URL, port=6333)
-embedding_model = "intfloat/e5-large-v2"
+client = QdrantClient(url=QDRANT_URL, api_key=QDRANT_API_KEY)
+embedding_model = "intfloat/e5-base-v2"
 embeddings = HuggingFaceEmbeddings(model_name=embedding_model)
 
 # %%
-data_directory = Path("app/data")
+data_directory = Path(__file__).parent / "data"
 text_splitter = RecursiveCharacterTextSplitter(chunk_size=512, chunk_overlap=64)
 
 # %%
@@ -39,6 +41,7 @@ def delete_collection(collection_name):
         print(f"Collection '{collection_name}' deleted.")
 
 # %%
+#Create Collection
 def create_collection(collection_name):
     if not client.collection_exists(collection_name):
         client.create_collection(
@@ -70,6 +73,17 @@ def load_documents_from_folder(folder_path):
                             'topic': topic}
                 )
                 documents.append(doc)
+
+    for file_path in folder_path.rglob("*.pdf"):
+        try:
+            loader = PyPDFLoader(file_path)
+            docs = loader.load()
+            for doc in docs:
+                doc.metadata["topic"] = file_path.parent.name
+            documents.extend(docs)
+        except Exception as e:
+            print(f"Failed to load PDF {file_path}: {e}")
+
     return documents
 
 # %%
@@ -105,7 +119,7 @@ for topic in sub_folders:
     print('\n')
 
 # %%
-collection_name = 'wellness_docs'
+"""collection_name = 'wellness_docs'
 delete_collection(collection_name)
 create_collection(collection_name)
 
@@ -118,13 +132,6 @@ for topic in sub_folders:
     if docs:
         split_and_upload_to_qdrant(collection_name, docs)
 
-    print('\n')
-
-# %%
-print(client.get_collections())
-
-
-# %%
-
+    print('\n')"""
 
 

app/notebooks/embed_documents.ipynb

@@ -10,7 +10,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "/Users/vishalpatel/Documents/Internship/Auro/chatbot/auro_chatbot_backend/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "c:\\Users\\vip11\\Documents\\Projects\\Auro_Chatbot\\auro_chatbot_backend\\.venv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
       "  from .autonotebook import tqdm as notebook_tqdm\n"
      ]
     }
@@ -64,7 +64,7 @@
    "outputs": [],
    "source": [
     "client = QdrantClient(url=QDRANT_URL, api_key=QDRANT_API_KEY)\n",
-    "embedding_model = \"intfloat/e5-large-v2\"\n",
+    "embedding_model = \"intfloat/e5-base-v2\"\n",
     "embeddings = HuggingFaceEmbeddings(model_name=embedding_model)"
    ]
   },
@@ -105,7 +105,7 @@
     "    if not client.collection_exists(collection_name):\n",
     "        client.create_collection(\n",
     "            collection_name=collection_name,\n",
-    "            vectors_config=VectorParams(size=1024, distance=Distance.COSINE),\n",
+    "            vectors_config=VectorParams(size=768, distance=Distance.COSINE),\n",
     "        )\n",
     "        print(f\"Created Collection: {collection_name}\")"
    ]
@@ -187,57 +187,44 @@
       "Processing: blogs\n",
       "Collection 'blogs' deleted.\n",
       "Created Collection: blogs\n",
-      "Loaded 105 docs from ../data/blogs\n",
+      "Loaded 105 docs from ..\\data\\blogs\n",
       "Uploaded 1045 chunks to blogs\n",
       "\n",
       "\n",
-      "Processing: technology\n",
-      "Collection 'technology' deleted.\n",
-      "Created Collection: technology\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Ignoring wrong pointing object 6 0 (offset 0)\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Loaded 3 docs from ../data/technology\n",
-      "Uploaded 11 chunks to technology\n",
-      "\n",
-      "\n",
-      "Processing: revolution\n",
-      "Collection 'revolution' deleted.\n",
-      "Created Collection: revolution\n",
-      "Loaded 274 docs from ../data/revolution\n",
-      "Uploaded 1415 chunks to revolution\n",
+      "Processing: faqs\n",
+      "Collection 'faqs' deleted.\n",
+      "Created Collection: faqs\n",
+      "Loaded 1 docs from ..\\data\\faqs\n",
+      "Uploaded 14 chunks to faqs\n",
       "\n",
       "\n",
       "Processing: product\n",
       "Collection 'product' deleted.\n",
       "Created Collection: product\n",
-      "Loaded 19 docs from ../data/product\n",
+      "Loaded 19 docs from ..\\data\\product\n",
       "Uploaded 132 chunks to product\n",
       "\n",
       "\n",
-      "Processing: faqs\n",
-      "Collection 'faqs' deleted.\n",
-      "Created Collection: faqs\n",
-      "Loaded 1 docs from ../data/faqs\n",
-      "Uploaded 14 chunks to faqs\n",
+      "Processing: revolution\n",
+      "Collection 'revolution' deleted.\n",
+      "Created Collection: revolution\n",
+      "Loaded 1 docs from ..\\data\\revolution\n",
+      "Uploaded 32 chunks to revolution\n",
       "\n",
       "\n",
       "Processing: support\n",
       "Collection 'support' deleted.\n",
       "Created Collection: support\n",
-      "Loaded 2 docs from ../data/support\n",
+      "Loaded 2 docs from ..\\data\\support\n",
       "Uploaded 15 chunks to support\n",
       "\n",
+      "\n",
+      "Processing: technology\n",
+      "Collection 'technology' deleted.\n",
+      "Created Collection: technology\n",
+      "Loaded 1 docs from ..\\data\\technology\n",
+      "Uploaded 8 chunks to technology\n",
+      "\n",
       "\n"
      ]
     }

app/notebooks/embed_documents.py

@@ -1,137 +0,0 @@
-# %%
-from qdrant_client import QdrantClient
-from qdrant_client.models import VectorParams, Distance
-
-from langchain_core.documents import Document
-from langchain_qdrant import QdrantVectorStore
-from langchain_text_splitters import RecursiveCharacterTextSplitter
-from langchain_huggingface import HuggingFaceEmbeddings
-from langchain_community.document_loaders import PyPDFLoader
-
-import os
-from pathlib import Path
-from uuid import uuid4
-
-# %%
-QDRANT_URL = os.getenv('QDRANT_URL')
-QDRANT_API_KEY = os.getenv('QDRANT_API_KEY')
-
-# %%
-FAQ_COLLECTION = "faqs"
-BLOGS_COLLECTION = "blogs"
-TECHNOLOGY_COLLECTION = "technology"
-REVOLUTION_COLLECTION = "revolution"
-SUPPORT_COLLECTION = "support"
-PRODUCT_COLLECTION = "product"
-
-# %%
-client = QdrantClient(url=QDRANT_URL, api_key=QDRANT_API_KEY)
-embedding_model = "intfloat/e5-large-v2"
-embeddings = HuggingFaceEmbeddings(model_name=embedding_model)
-
-# %%
-data_directory = Path("../data")
-text_splitter = RecursiveCharacterTextSplitter(chunk_size=512, chunk_overlap=64)
-
-# %%
-#Delete Collection
-def delete_collection(collection_name):
-    if client.collection_exists(collection_name):
-        client.delete_collection(collection_name)
-        print(f"Collection '{collection_name}' deleted.")
-
-# %%
-#Create Collection
-def create_collection(collection_name):
-    if not client.collection_exists(collection_name):
-        client.create_collection(
-            collection_name=collection_name,
-            vectors_config=VectorParams(size=1024, distance=Distance.COSINE),
-        )
-        print(f"Created Collection: {collection_name}")
-
-# %%
-def load_documents_from_folder(folder_path):
-    documents = []
-
-    for file_path in folder_path.rglob("*.txt"):
-        with open(file_path, 'r', encoding='utf-8') as f:
-            lines = f.readlines()
-
-            if not lines:
-                print(f"{file_path} is empty")
-                continue
-
-            source_url = lines[0].replace("Source URL:","").strip()
-            content = "".join(lines[1:]).strip()
-            topic = file_path.parent.name
-
-            if content:
-                doc = Document(
-                    page_content=content,
-                    metadata={'source': source_url,
-                            'topic': topic}
-                )
-                documents.append(doc)
-
-    for file_path in folder_path.rglob("*.pdf"):
-        try:
-            loader = PyPDFLoader(file_path)
-            docs = loader.load()
-            for doc in docs:
-                doc.metadata["topic"] = file_path.parent.name
-            documents.extend(docs)
-        except Exception as e:
-            print(f"Failed to load PDF {file_path}: {e}")
-
-    return documents
-
-# %%
-def split_and_upload_to_qdrant(collection_name, documents):
-    splits = text_splitter.split_documents(documents)
-    uuids = [str(uuid4()) for _ in range(len(splits))]
-
-    vector_store = QdrantVectorStore(
-    client=client,
-    collection_name=collection_name,
-    embedding=embeddings
-    )
-
-    vector_store.add_documents(documents=splits, ids=uuids)
-    print(f"Uploaded {len(splits)} chunks to {collection_name}")
-
-# %%
-sub_folders = [sub_folder for sub_folder in data_directory.iterdir() if sub_folder.is_dir()]
-
-for topic in sub_folders:
-    collection_name = topic.name
-    print(f"Processing: {topic.name}")
-
-    delete_collection(collection_name)
-    create_collection(collection_name)
-
-    docs = load_documents_from_folder(topic)
-    print(f"Loaded {len(docs)} docs from {topic}")
-
-    if docs:
-        split_and_upload_to_qdrant(collection_name, docs)
-
-    print('\n')
-
-# %%
-"""collection_name = 'wellness_docs'
-delete_collection(collection_name)
-create_collection(collection_name)
-
-sub_folders = [sub_folder for sub_folder in data_directory.iterdir() if sub_folder.is_dir()]
-for topic in sub_folders:
-    print(f"Processing: {topic.name}")
-    docs = load_documents_from_folder(topic)
-    print(f"Loaded {len(docs)} docs from {topic}")
-
-    if docs:
-        split_and_upload_to_qdrant(collection_name, docs)
-
-    print('\n')"""
-
-

app/notebooks/rag_original.py

@@ -23,7 +23,7 @@ console = Console()
 
 client = QdrantClient(url="localhost", port=6333)
 COLLECTION_NAME = "wellness_docs"
-embedding_model = "intfloat/e5-large-v2"
+embedding_model = "intfloat/e5-base-v2"
 
 
 embeddings = HuggingFaceEmbeddings(model_name=embedding_model)

app/rag.py

@@ -27,7 +27,7 @@ session_histories: dict[str, list] = {}
 LLM_MODEL = "mistral-medium-latest"
 OPENROUTER_API_KEY = os.getenv('OPENROUTER_API_KEY')
 COLLECTION_NAME = "wellness_docs"
-EMBEDDING_MODEL = "intfloat/e5-large-v2"
+EMBEDDING_MODEL = "intfloat/e5-base-v2"
 QDRANT_URL = os.getenv('QDRANT_URL')
 QDRANT_API_KEY = os.getenv('QDRANT_API_KEY')
 SUPABASE_URL = os.getenv('SUPABASE_URL')