Fix TokenizersBackend error by using slow tokenizer and adding mistral-common

chat_gui.py

@@ -300,10 +300,16 @@ if st.session_state.messages and st.session_state.messages[-1]["role"] == "user"
                         model_class = AutoModelForCausalLM
                     
                     # Use AutoTokenizer for all (even VLMs have tokenizers)
+                    # Setting use_fast=False to avoid "TokenizersBackend" errors on some environments
                     processor_class = AutoTokenizer
                     
                     st.info(f"Loading {st.session_state.current_model} via Transformers...")
-                    tokenizer = processor_class.from_pretrained(conf["path"], token=hf_token, trust_remote_code=True)
+                    tokenizer = processor_class.from_pretrained(
+                        conf["path"], 
+                        token=hf_token, 
+                        trust_remote_code=True,
+                        use_fast=False
+                    )
                     # Use 4-bit quantization if on low-memory cloud
                     model = model_class.from_pretrained(
                         conf["path"], 

requirements.txt

@@ -11,7 +11,8 @@ altair==5.3.0
 # Inference (MLX for local Mac, Transformers for Cloud/Linux)
 mlx; platform_system == "Darwin"
 mlx-vlm; platform_system == "Darwin"
-transformers==4.48.2
+transformers>=4.48.0
+mistral-common>=1.8.6
 torch
 peft
 accelerate