Tested rag.py

app/rag.py

@@ -21,7 +21,8 @@ from dotenv import load_dotenv
 
 session_histories: dict[str, list] = {}
 # %%
-LLM_MODEL_PATH = "models/mistral.gguf"
+LLM_MODEL_PATH = "TheBloke/Mistral-7B-Instruct-v0.1-GGUF"
+LLM_MODEL = "mistral-7b-instruct-v0.1.Q2_K.gguf"
 COLLECTION_NAME = "wellness_docs"
 EMBEDDING_MODEL = "intfloat/e5-large-v2"
 QDRANT_URL = os.getenv('QDRANT_URL')
@@ -34,8 +35,8 @@ REVOLUTION_COLLECTION = "revolution"
 
 # %%
 llm = AutoModelForCausalLM.from_pretrained(
-    model_path_or_repo_id="TheBloke/Mistral-7B-Instruct-v0.1-GGUF",
-    model_file="mistral-7b-instruct-v0.1.Q2_K.gguf",
+    model_path_or_repo_id=LLM_MODEL_PATH,
+    model_file=LLM_MODEL,
     model_type="mistral",
     gpu_layers=0  # or tweak this if you want GPU acceleration
 )