Create app.py

app.py

@@ -0,0 +1,122 @@
+import gradio as gr
+import torch
+import spaces
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from threading import Thread
+
+# --- 1. Model and Tokenizer Loading ---
+# Load your fine-tuned model and tokenizer from the Hugging Face Hub.
+model_name = "gitglubber/Ntfy"
+print(f"Loading tokenizer from {model_name}...")
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+print(f"Loading model from {model_name}...")
+model = AutoModelForCausalLM.from_pretrained(
+    model_name,
+    torch_dtype="auto",
+    device_map="auto"
+)
+print("Model and tokenizer loaded successfully.")
+
+# --- 2. System Message ---
+# Define the persona and rules for the NTFY bot.
+system_message = """You are an expert technical assistant for ntfy, a publish-subscribe notification service. Your purpose is to provide users with accurate, clear, and helpful information about the ntfy project.
+
+**Your Capabilities:**
+
+* You can explain what ntfy is, its key features, and what platforms it supports.
+* You provide detailed instructions on how to install, configure, and self-host the ntfy server.
+* You can answer specific questions about all configuration options found in server.yml.
+* You offer step-by-step troubleshooting for common problems, such as notification delivery issues, SSL errors, and authentication problems.
+* You provide accurate curl commands and other code snippets to demonstrate how to publish messages and use the API.
+
+**People to know:** binweiderhier is the developer of NTFY. wunter8 is the expert and moderator of the discord channel. Support occurs primarily from the discord channel. 
+
+Free to use (with restrictions) server is available at ntfy.sh - pro plans are available. NTFY client is available for desktop (PWA), iphone (appstore), android (f-droid and google play).
+
+**Rules You Must Follow:**
+
+1.  **Stick to Your Expertise:** Your knowledge is strictly limited to the ntfy project. If asked about unrelated topics (e.g., other software, general programming, personal opinions), politely state that you are an ntfy expert and cannot answer questions outside of that domain.
+2.  **Structure for Clarity:** Always use Markdown to format your answers. Use lists for steps or features, bold text for emphasis, and code blocks for all commands, URLs, and configuration examples. This is crucial for user readability.
+3.  **Be Accurate and Concise:** Provide factual information based on the ntfy documentation and community knowledge. Get straight to the point and avoid unnecessary conversational filler.
+4.  **Prioritize Troubleshooting:** When a user has a problem, your primary goal is to help them solve it. Provide clear, actionable steps.
+5.  **Be a Helpful Assistant:** Your tone should always be helpful, patient, and professional. You are the go-to resource for all things ntfy."""
+
+# --- 3. Gradio Interface with Streaming ---
+with gr.Blocks(fill_height=True, theme=gr.themes.Soft()) as demo:
+    gr.Markdown("# NTFY Expert Chat Bot")
+    chatbot = gr.Chatbot(
+        [],
+        elem_id="chatbot",
+        bubble_full_width=False,
+        avatar_images=(None, "https://ntfy.sh/img/ntfy.png"),
+        scale=1
+    )
+    msg = gr.Textbox(label="Input", scale=0, placeholder="Ask me a question about ntfy...")
+    clear = gr.Button("Clear")
+
+    # Use a generator function to handle streaming
+    @spaces.GPU(duration=120)
+    def respond(message, chat_history):
+        """
+        Gradio response function that streams model output.
+        """
+        if not message.strip():
+            yield "", chat_history
+            return
+
+        # Append the new user message to the history for immediate display
+        chat_history.append((message, ""))
+        yield "", chat_history
+
+        # --- Prepare model input ---
+        messages = [{"role": "system", "content": system_message}]
+        for user_msg, assistant_msg in chat_history[:-1]:
+            messages.append({"role": "user", "content": user_msg})
+            if assistant_msg is not None:
+                messages.append({"role": "assistant", "content": assistant_msg})
+        messages.append({"role": "user", "content": message})
+
+        # Apply the chat template
+        text = tokenizer.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=True,
+        )
+        model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
+
+        # --- Setup the streamer and generation thread ---
+        streamer = TextIteratorStreamer(
+            tokenizer,
+            skip_prompt=True,
+            skip_special_tokens=True
+        )
+
+        generation_kwargs = dict(
+            **model_inputs,
+            streamer=streamer,
+            max_new_tokens=8192,
+            do_sample=True,
+            top_p=0.95,
+            top_k=50,
+            temperature=0.7,
+        )
+
+        # Start the generation in a separate thread
+        thread = Thread(target=model.generate, kwargs=generation_kwargs)
+        thread.start()
+
+        # --- Yield tokens as they become available ---
+        bot_response = ""
+        for new_text in streamer:
+            bot_response += new_text
+            chat_history[-1] = (message, bot_response)
+            yield "", chat_history
+
+    # Wire up the Gradio components
+    msg.submit(respond, [msg, chatbot], [msg, chatbot])
+    clear.click(lambda: [], None, chatbot, queue=False)
+
+# Launch the app
+demo.queue().launch(debug=True)
+