add GA resolution search

.gitignore

@@ -0,0 +1,2 @@
+/small_scripts/
+.gitignore

app.py

@@ -2,61 +2,223 @@ import gradio as gr
 from FlagEmbedding import BGEM3FlagModel
 import numpy as np
 import json
+import os # Import os to handle potential path issues
 
-model = BGEM3FlagModel('BAAI/bge-m3',
-                       use_fp16=True) # Setting use_fp16 to True speeds up computation with a slight performance degradation
-issue_embeddings = np.load('ns_issues_dense_bge-m3.npy')
+# --- Configuration and Global Data Loading ---
 
-with open("issue_titles.json") as file:
-    issue_titles = json.load(file)
+# Determine the directory of the script to load files relative to it
+script_dir = os.path.dirname(os.path.abspath(__file__))
+issue_embeddings_path = os.path.join(script_dir, 'ns_issues_dense_bge-m3.npy')
+issue_titles_path = os.path.join(script_dir, 'issue_titles.json')
+ga_embeddings_path = os.path.join(script_dir, 'ns_ga_resolutions_dense_bge-m3.npy')
+ga_resolutions_path = os.path.join(script_dir, 'parsed_ga_resolutions.json')
 
-def get_similarity(search_term):
-    search_embedding = model.encode([search_term])['dense_vecs']
-    similarity = search_embedding @ issue_embeddings.T
-    return similarity
 
+print("Loading BGE-M3 model...")
+try:
+    # Use a local path if the model is downloaded, or let it download from Hugging Face
+    model = BGEM3FlagModel('BAAI/bge-m3', use_fp16=True)
+    print("Model loaded successfully.")
+except Exception as e:
+    print(f"Error loading model: {e}")
+    print("Please ensure you have an internet connection or the model is cached locally.")
+    # Consider exiting or handling the error appropriately
+    exit()
 
-def get_similarity_rankings(search_term):
-    similarity = get_similarity(search_term)
 
-    results = []
-    for i, search_query_similarities in enumerate(similarity):
-        indexed_similarities = []
-        for issue_index, sim_score in enumerate(search_query_similarities):
-            indexed_similarities.append((issue_index, sim_score))
+print("Loading issue data...")
+try:
+    issue_embeddings = np.load(issue_embeddings_path)
+    with open(issue_titles_path) as file:
+        issue_titles = json.load(file)
+    print(f"Issue data loaded: {len(issue_titles)} issues.")
+except FileNotFoundError as e:
+    print(f"Error loading issue data: {e}")
+    print("Please ensure 'ns_issues_dense_bge-m3.npy' and 'issue_titles.json' are in the same directory as app.py")
+    # Consider exiting or handling the error appropriately
+    exit()
+except Exception as e:
+    print(f"Error loading issue data: {e}")
+    exit()
+
+
+print("Loading GA resolution data...")
+try:
+    ga_embeddings = np.load(ga_embeddings_path)
+    with open(ga_resolutions_path) as file:
+        ga_resolutions_data = json.load(file) # List of dictionaries
+    print(f"GA resolution data loaded: {len(ga_resolutions_data)} resolutions.")
+except FileNotFoundError as e:
+    print(f"Error loading GA resolution data: {e}")
+    print("Please ensure 'ns_ga_resolutions_dense_bge-m3.npy' and 'parsed_ga_resolutions.json' are in the same directory as app.py")
+    # Consider exiting or handling the error appropriately
+    # If the file is not found, the GA search tab won't work, but the app might still launch with just the issue search
+    ga_embeddings = None # Indicate that GA data is not available
+    ga_resolutions_data = []
+except Exception as e:
+    print(f"Error loading GA resolution data: {e}")
+    ga_embeddings = None
+    ga_resolutions_data = []
+
+
+# --- Search Functions ---
+
+def get_issue_similarity_rankings(search_term):
+    """Searches issues and returns formatted results."""
+    if not search_term:
+        return "Please enter a search term."
+    if issue_embeddings is None or not issue_titles:
+         return "Issue data not loaded. Cannot perform search."
+
+    try:
+        # Encode the search term
+        search_embedding = model.encode([search_term])['dense_vecs']
+
+        # Calculate similarity (dot product)
+        similarity = search_embedding @ issue_embeddings.T # Shape: (1, num_issues)
+
+        # Get similarities for the single search term
+        search_query_similarities = similarity[0] # Shape: (num_issues,)
+
+        # Pair index with similarity score
+        indexed_similarities = [(i, score) for i, score in enumerate(search_query_similarities)]
+
+        # Sort by similarity score in descending order
+        sorted_similarities = sorted(indexed_similarities, key=lambda item: item[1], reverse=True)
+
+        # Format results as text
+        similarity_text = "# Top 20 Issue Search Results\n"
+        search_ranking = 1
+        # Get top 20 results
+        for index, sim_score in sorted_similarities[:20]:
+            # issue_titles is a dict, needs string key
+            issue_title = issue_titles.get(str(index), f"Unknown Issue (Index {index})")
+            similarity_text += f"{search_ranking}. {issue_title}, Similarity: {sim_score:.4f}\n"
+            search_ranking += 1
+
+        return similarity_text
+    except Exception as e:
+        return f"An error occurred during issue search: {e}"
+
+
+def search_ga_resolutions(search_term):
+    """Searches GA resolutions and returns formatted results with links."""
+    if not search_term:
+        return "Please enter a search term."
+    if ga_embeddings is None or not ga_resolutions_data:
+        return "GA Resolution data not loaded. Cannot perform search."
+
+    try:
+        # Encode the search term
+        search_embedding = model.encode([search_term])['dense_vecs']
+
+        # Calculate similarity (dot product)
+        similarity = search_embedding @ ga_embeddings.T # Shape: (1, num_resolutions)
+
+        # Get similarities for the single search term
+        search_query_similarities = similarity[0] # Shape: (num_resolutions,)
+
+        # Pair index with similarity score
+        indexed_similarities = [(i, score) for i, score in enumerate(search_query_similarities)]
+
+        # Sort by similarity score in descending order
         sorted_similarities = sorted(indexed_similarities, key=lambda item: item[1], reverse=True)
-        results.append(sorted_similarities)
 
-    similarity_text = ""
-    for i, search_query_result in enumerate(results):
-        similarity_text += f"# Search Results"
+        # Format results as Markdown with links
+        similarity_text = "# Top 20 GA Resolution Search Results\n"
         search_ranking = 1
-        for index, sim_score in search_query_result[:20]:
-            similarity_text += f"\n{search_ranking}. {issue_titles[str(index)]}, Similarity: {sim_score:.4f}"
+        # Get top 20 results
+        for index, sim_score in sorted_similarities[:20]:
+            if index < len(ga_resolutions_data):
+                resolution = ga_resolutions_data[index]
+                title = resolution.get('title', 'Untitled Resolution')
+                res_id = resolution.get('id', 'N/A')
+                council = resolution.get('council', 1) # Default to council 1 if not specified
+                # Construct the NationStates URL
+                url = f"https://www.nationstates.net/page=WA_past_resolution/id={res_id}/council={council}"
+                # Format as Markdown link
+                similarity_text += f"{search_ranking}. [{title}]({url}), Similarity: {sim_score:.4f}\n"
+            else:
+                 # Fallback if index is unexpectedly out of range
+                 similarity_text += f"{search_ranking}. Error retrieving resolution data for index {index}, Similarity: {sim_score:.4f}\n"
+
             search_ranking += 1
 
-    return similarity_text
+        return similarity_text
+    except Exception as e:
+         return f"An error occurred during GA resolution search: {e}"
 
 
+# --- Gradio Interface ---
+
 """
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
+For information on how to customize the Gradio Blocks and Tabs, peruse the gradio docs:
+https://www.gradio.app/docs/blocks
+https://www.gradio.app/docs/tabs
+https://www.gradio.app/docs/interface (used within tabs)
 """
-demo = gr.Interface(
-    get_similarity_rankings,
-    inputs=gr.Textbox(label="Search term", placeholder="What are you looking for?"),
-    outputs=gr.Markdown(container=True),
-    examples=["coffee",
-              "land value tax",
-              "Elon Musk",
-              "After an corrupted election, citizens demand the real results, after discovering it was rigged. ",
-              "Eureka! A new scientific law regarding the universe's expansion may have just been discovered at the University of @@CAPITAL@@. Unfortunately, tempers are flaring over who should get naming credit. Maxtopian grad student Georgie Bubble claims the work alone while his boss Dr.@@RANDOMNAME1@@ claims that all work in the University is @@NAME@@’s collectively. Your Minister of Education has elevated this to your desk.",
-              ],
-    title="NationStates Issue Search",
-    description="Search through all 1660 issues in [NationStates](https://www.nationstates.net/). This is like semantic search, not keyword search. You can input words, sentences, paragraphs, or even full issues. For finding duplicates, I recommend only searching with the issue description.",
-    article="Made by [Jiangbei](www.nationstates.net/nation=jiangbei). [Issues](https://github.com/Clarissa-Valentine-Z/NationStates-Issue-Megathread) downloaded from [Valentine Z](https://www.nationstates.net/nation=valentine_z) and written by respective authors. Powered by [BAAI/bge-m3](https://huggingface.co/BAAI/bge-m3).",
-    submit_btn="Search",
-)
+
+with gr.Blocks() as demo:
+    gr.Markdown("""
+    # NationStates Semantic Search
+    Search through NationStates content using semantic search powered by BGE-M3.
+    """)
+
+    with gr.Tabs() as tabs:
+        with gr.TabItem("Issue Search"):
+            gr.Markdown("""
+            ### Search NationStates Issues
+            Search through all 1660 issues. Semantic search allows finding related concepts or paraphrased ideas, not just keywords.
+            """)
+            issue_search_interface = gr.Interface(
+                fn=get_issue_similarity_rankings,
+                inputs=gr.Textbox(label="Search term", placeholder="What issue are you looking for?"),
+                outputs=gr.Markdown(container=True),
+                examples=[
+                    "coffee",
+                    "land value tax",
+                    "Elon Musk",
+                    "After an corrupted election, citizens demand the real results, after discovering it was rigged. ",
+                    "Eureka! A new scientific law regarding the universe's expansion may have just been discovered at the University of @@CAPITAL@@. Unfortunately, tempers are flaring over who should get naming credit. Maxtopian grad student Georgie Bubble claims the work alone while his boss Dr.@@RANDOMNAME1@@ claims that all work in the University is @@NAME@@’s collectively. Your Minister of Education has elevated this to your desk.",
+                ],
+                title=None, # Title is now handled by the Markdown within the tab
+                description=None, # Description is now handled by the Markdown within the tab
+                submit_btn="Search Issues",
+                # No live=True as it's computationally intensive
+            )
+
+        with gr.TabItem("GA Resolution Search"):
+             gr.Markdown("""
+             ### Search NationStates General Assembly Resolutions
+             Search through General Assembly resolutions. The results include clickable links to the resolutions on NationStates.
+             """)
+             ga_search_interface = gr.Interface(
+                 fn=search_ga_resolutions,
+                 inputs=gr.Textbox(label="Search term", placeholder="What GA resolution are you looking for?"),
+                 outputs=gr.Markdown(container=True),
+                 examples=[
+                    "repeal process", # Common term related to resolutions
+                    "condemn genocide",
+                    "rights of animals",
+                    "regulating space mining",
+                    "limit weapons production",
+                    "World Assembly neutrality", # Example from Resolution 2 description/body
+                    "founding of the World Assembly", # Example from Resolution 1 description/body
+                    "recognition of new nations" # Example of a common WA topic
+                 ],
+                  title=None, # Title handled by Markdown
+                  description=None, # Description handled by Markdown
+                  submit_btn="Search Resolutions",
+                  # No live=True
+             )
+
+    gr.Markdown("""
+    <p>Made by [Jiangbei](www.nationstates.net/nation=jiangbei). Issue data from [Valentine Z](https://www.nationstates.net/nation=valentine_z). GA Resolution data parsed from NationStates. Powered by [BAAI/bge-m3](https://huggingface.co/BAAI/bge-m3).</p>
+    """)
 
 
+# --- Launch App ---
 if __name__ == "__main__":
-    demo.launch()
+    # Set share=True to make the app accessible externally (requires ngrok)
+    # share=False is default and runs locally
+    demo.launch()

ns_ga_resolutions_dense_bge-m3.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a3e1e4def2ec2cd87a3c11fabb8af6e2457251c120619ffc29940c938c100e4
+size 1595520