embed documents to include csv

app/notebooks/embed_documents.ipynb

@@ -10,7 +10,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "/Users/vishalpatel/Documents/Internship/Auro/chatbot/auro_chatbot_backend/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "c:\\Users\\vip11\\Documents\\Projects\\Auro_Chatbot\\auro_chatbot_backend\\.venv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
       "  from .autonotebook import tqdm as notebook_tqdm\n"
      ]
     }
@@ -23,7 +23,7 @@
     "from langchain_qdrant import QdrantVectorStore\n",
     "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
     "from langchain_huggingface import HuggingFaceEmbeddings\n",
-    "from langchain_community.document_loaders import PyPDFLoader\n",
+    "from langchain_community.document_loaders import PyPDFLoader, CSVLoader\n",
     "\n",
     "import os\n",
     "from pathlib import Path\n",
@@ -150,6 +150,16 @@
     "        except Exception as e:\n",
     "            print(f\"Failed to load PDF {file_path}: {e}\")\n",
     "\n",
+    "    for file_path in folder_path.rglob(\"*.csv\"):\n",
+    "        try:\n",
+    "            loader = CSVLoader(file_path, encoding='utf-8')\n",
+    "            docs = loader.load()\n",
+    "            for doc in docs:\n",
+    "                doc.metadata[\"topic\"] = file_path.parent.name\n",
+    "            documents.extend(docs)\n",
+    "        except Exception as e:\n",
+    "            print(f\"Failed to load PDF {file_path}: {e}\")\n",
+    "\n",
     "    return documents"
    ]
   },
@@ -187,52 +197,44 @@
       "Processing: blogs\n",
       "Collection 'auro_blogs' deleted.\n",
       "Created Collection: auro_blogs\n",
-      "Loaded 105 docs from ../data/blogs\n",
+      "Loaded 105 docs from ..\\data\\blogs\n",
       "Uploaded 1045 chunks to auro_blogs\n",
       "\n",
       "\n",
-      "Processing: technology\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Ignoring wrong pointing object 6 0 (offset 0)\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Created Collection: auro_technology\n",
-      "Loaded 3 docs from ../data/technology\n",
-      "Uploaded 11 chunks to auro_technology\n",
-      "\n",
-      "\n",
-      "Processing: revolution\n",
-      "Created Collection: auro_revolution\n",
-      "Loaded 274 docs from ../data/revolution\n",
-      "Uploaded 1415 chunks to auro_revolution\n",
+      "Processing: faqs\n",
+      "Collection 'auro_faqs' deleted.\n",
+      "Created Collection: auro_faqs\n",
+      "Loaded 108 docs from ..\\data\\faqs\n",
+      "Uploaded 150 chunks to auro_faqs\n",
       "\n",
       "\n",
       "Processing: product\n",
+      "Collection 'auro_product' deleted.\n",
       "Created Collection: auro_product\n",
-      "Loaded 19 docs from ../data/product\n",
+      "Loaded 19 docs from ..\\data\\product\n",
       "Uploaded 132 chunks to auro_product\n",
       "\n",
       "\n",
-      "Processing: faqs\n",
-      "Created Collection: auro_faqs\n",
-      "Loaded 1 docs from ../data/faqs\n",
-      "Uploaded 14 chunks to auro_faqs\n",
+      "Processing: revolution\n",
+      "Collection 'auro_revolution' deleted.\n",
+      "Created Collection: auro_revolution\n",
+      "Loaded 1 docs from ..\\data\\revolution\n",
+      "Uploaded 32 chunks to auro_revolution\n",
       "\n",
       "\n",
       "Processing: support\n",
+      "Collection 'auro_support' deleted.\n",
       "Created Collection: auro_support\n",
-      "Loaded 2 docs from ../data/support\n",
+      "Loaded 2 docs from ..\\data\\support\n",
       "Uploaded 15 chunks to auro_support\n",
       "\n",
+      "\n",
+      "Processing: technology\n",
+      "Collection 'auro_technology' deleted.\n",
+      "Created Collection: auro_technology\n",
+      "Loaded 1 docs from ..\\data\\technology\n",
+      "Uploaded 8 chunks to auro_technology\n",
+      "\n",
       "\n"
      ]
     }

app/rag.py

@@ -432,5 +432,7 @@ async def get_response(query: str, name, email, config) -> dict:
         latency_ms= latency_ms,
         metadata=metadata
         )
+
+
     return result
 # %%