Update app.py

app.py

@@ -35,7 +35,6 @@ except Exception as e:
 
 # Download the entire index folder for better performance
 try:
-    # Use snapshot_download to get the entire data directory with indices
     CACHE_DIR = snapshot_download(
         repo_id=REPO_ID,
         repo_type="dataset",
@@ -44,7 +43,6 @@ try:
     INDEX_PATH = os.path.join(CACHE_DIR, INDEX_FOLDER)
     print(f"Index files downloaded to: {INDEX_PATH}")
     
-    # Count index files
     if os.path.exists(INDEX_PATH):
         index_files = list(Path(INDEX_PATH).glob("*.ldb"))
         print(f"Found {len(index_files)} index files (.ldb)")
@@ -64,22 +62,22 @@ def get_db_connection():
         
         # Apply PRAGMA optimizations for read performance
         pragmas = [
-            "PRAGMA query_only = ON",           # Read-only mode
-            "PRAGMA temp_store = MEMORY",        # Use memory for temp tables
-            "PRAGMA cache_size = -128000",       # 128MB cache (negative = KB)
-            "PRAGMA page_size = 8192",           # Larger page size for better I/O
-            "PRAGMA mmap_size = 2147483648",     # 2GB memory-mapped I/O
-            "PRAGMA synchronous = OFF",          # Safe for read-only
-            "PRAGMA journal_mode = OFF",         # No journal needed for read-only
-            "PRAGMA locking_mode = NORMAL",      # Allow multiple readers
-            "PRAGMA threads = 4",                # Use multiple threads
+            ("PRAGMA query_only = ON", True),
+            ("PRAGMA temp_store = MEMORY", True),
+            ("PRAGMA cache_size = -128000", True),      # 128MB cache
+            ("PRAGMA page_size = 8192", False),         # Can't change on existing DB
+            ("PRAGMA mmap_size = 2147483648", True),    # 2GB memory-mapped I/O
+            ("PRAGMA synchronous = OFF", True),
+            ("PRAGMA locking_mode = NORMAL", True),
+            ("PRAGMA threads = 4", True),
         ]
         
-        for pragma in pragmas:
+        for pragma, critical in pragmas:
             try:
                 conn.execute(pragma)
             except sqlite3.OperationalError as e:
-                print(f"Warning: Could not apply {pragma}: {e}")
+                if critical:
+                    print(f"Warning: Could not apply {pragma}: {e}")
         
         return conn
     except Exception as e:
@@ -117,9 +115,9 @@ def verify_indices():
                         cursor.execute(f"PRAGMA index_info({idx_name})")
                         idx_cols = cursor.fetchall()
                         
-                        cols = [col[2] for col in idx_cols]
+                        cols = [col[2] for col in idx_cols if col[2]]
                         unique_str = "UNIQUE" if unique else "NON-UNIQUE"
-                        print(f"  ├─ {idx_name} ({unique_str}) on columns: {', '.join(cols)}")
+                        print(f"  ├─ {idx_name} ({unique_str}) on columns: {', '.join(cols) if cols else 'id'}")
                         total_indices += 1
             
             print(f"\n✅ Total indices found: {total_indices}")
@@ -130,7 +128,13 @@ def verify_indices():
             if fts:
                 print(f"✅ Full-Text Search enabled: {[f[0] for f in fts]}")
             
-            # Check database page size and cache
+            # Check for FTS tables
+            cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '%fts%'")
+            fts_tables = cursor.fetchall()
+            if fts_tables:
+                print(f"✅ FTS tables found: {[t[0] for t in fts_tables]}")
+            
+            # Check database stats
             cursor.execute("PRAGMA page_size")
             page_size = cursor.fetchone()[0]
             cursor.execute("PRAGMA cache_size")
@@ -138,7 +142,6 @@ def verify_indices():
             print(f"\n📈 Page size: {page_size} bytes")
             print(f"📈 Cache size: {abs(cache_size)} KB" if cache_size < 0 else f"📈 Cache size: {cache_size} pages")
             
-            # Get database size
             cursor.execute("SELECT page_count * page_size as size FROM pragma_page_count(), pragma_page_size()")
             db_size = cursor.fetchone()[0]
             print(f"📦 Database size: {db_size / 1024 / 1024 / 1024:.2f} GB")
@@ -205,7 +208,7 @@ def get_schema_info():
                         
                         unique_badge = "🔒 UNIQUE" if unique else "📑 INDEX"
                         schema_md += f"- **{idx_name}** {unique_badge}\n"
-                        schema_md += f"  - Columns: `{', '.join(cols) if cols else 'N/A'}`\n"
+                        schema_md += f"  - Columns: `{', '.join(cols) if cols else 'id'}`\n"
                         schema_md += f"  - Origin: {origin}\n"
                 
                 schema_md += "\n---\n\n"
@@ -244,13 +247,12 @@ def run_query(start_node, relation, end_node, limit):
     params = []
     
     try:
-        # Build WHERE conditions leveraging indices
+        # Build WHERE conditions
         if start_node:
             if "%" in start_node:
                 where_conditions.append("s.id LIKE ?")
                 params.append(start_node)
             else:
-                # Exact match or prefix match
                 where_conditions.append("s.id LIKE ?")
                 params.append(f"%{start_node}%")
         
@@ -259,7 +261,6 @@ def run_query(start_node, relation, end_node, limit):
                 where_conditions.append("r.label LIKE ?")
                 params.append(relation)
             else:
-                # Exact match is faster
                 where_conditions.append("r.label = ?")
                 params.append(relation)
 
@@ -274,24 +275,12 @@ def run_query(start_node, relation, end_node, limit):
         if where_conditions:
             query += " WHERE " + " AND ".join(where_conditions)
             
-        # Order by weight to get most relevant results first
         query += " ORDER BY e.weight DESC LIMIT ?"
         params.append(limit)
         
         print(f"Executing SQL with {len(params)} parameters")
 
         with get_db_connection() as conn:
-            # Use EXPLAIN QUERY PLAN to verify index usage (for debugging)
-            explain_query = "EXPLAIN QUERY PLAN " + query
-            try:
-                explain_result = conn.execute(explain_query, params).fetchall()
-                print("Query Plan:")
-                for row in explain_result:
-                    print(f"  {row}")
-            except:
-                pass
-            
-            # Execute actual query
             df = pd.read_sql_query(query, conn, params=params)
         
         if df.empty:
@@ -316,7 +305,7 @@ def run_raw_query(sql_query):
         
     try:
         with get_db_connection() as conn:
-            # Show query plan
+            # Show query plan for debugging
             try:
                 explain_result = conn.execute("EXPLAIN QUERY PLAN " + sql_query).fetchall()
                 print("Query Plan:")
@@ -339,7 +328,6 @@ def run_raw_query(sql_query):
 def get_semantic_profile(word, lang='en'):
     """
     HIGHLY OPTIMIZED: Single query with UNION ALL for all relations at once.
-    Uses indexed columns for maximum speed.
     """
     if not word:
         return "⚠️ Please enter a word."
@@ -374,7 +362,6 @@ def get_semantic_profile(word, lang='en'):
                         en.label as target_label,
                         e.weight as weight
                     FROM edge e
-                    INDEXED BY (SELECT name FROM pragma_index_list('edge') LIMIT 1)
                     INNER JOIN node s ON e.start_id = s.id
                     INNER JOIN node en ON e.end_id = en.id
                     INNER JOIN relation r ON e.rel_id = r.id
@@ -485,9 +472,14 @@ verify_indices()
 
 with gr.Blocks(title="ConceptNet SQLite Explorer", theme=gr.themes.Soft()) as demo:
     gr.Markdown("# 🧠 ConceptNet SQLite Explorer")
+    
+    # Status info
+    db_size_gb = os.path.getsize(DB_PATH) / 1024 / 1024 / 1024 if os.path.exists(DB_PATH) else 0
+    index_status = '✅ Indices Loaded' if INDEX_PATH and os.path.exists(INDEX_PATH) else '⚠️ No Index Cache'
+    
     gr.Markdown(
-        f"**Database:** `{os.path.basename(DB_PATH)}` ({os.path.getsize(DB_PATH) / 1024 / 1024 / 1024:.2f} GB) | "
-        f"**Status:** {'✅ Indices Loaded' if INDEX_PATH and os.path.exists(INDEX_PATH) else '⚠️ No Index Cache'}"
+        f"**Database:** `{os.path.basename(DB_PATH)}` ({db_size_gb:.2f} GB) | "
+        f"**Status:** {index_status}"
     )
     gr.Markdown("*Explore semantic relationships in ConceptNet with optimized indexed queries*")
     
@@ -585,7 +577,7 @@ with gr.Blocks(title="ConceptNet SQLite Explorer", theme=gr.themes.Soft()) as de
             
             raw_sql_input = gr.Textbox(
                 label="SQL Query", 
-                placeholder="SELECT s.label, r.label, e.label FROM edge e JOIN node s ON e.start_id = s.id JOIN relation r ON e.rel_id = r.id JOIN node e ON e.end_id = e.id WHERE s.label = 'dog' LIMIT 10",
+                placeholder="SELECT s.label, r.label, en.label FROM edge e JOIN node s ON e.start_id = s.id JOIN relation r ON e.rel_id = r.id JOIN node en ON e.end_id = en.id WHERE s.label = 'dog' LIMIT 10",
                 lines=6,
                 info="Write SELECT query"
             )
@@ -602,11 +594,11 @@ with gr.Blocks(title="ConceptNet SQLite Explorer", theme=gr.themes.Soft()) as de
                     "```\n\n"
                     "**Find strongest connections:**\n"
                     "```sql\n"
-                    "SELECT s.label, r.label, e.label, edge.weight\n"
-                    "FROM edge \n"
-                    "JOIN node s ON edge.start_id = s.id\n"
-                    "JOIN relation r ON edge.rel_id = r.id\n"
-                    "JOIN node e ON edge.end_id = e.id\n"
+                    "SELECT s.label, r.label, en.label, e.weight\n"
+                    "FROM edge e\n"
+                    "JOIN node s ON e.start_id = s.id\n"
+                    "JOIN relation r ON e.rel_id = r.id\n"
+                    "JOIN node en ON e.end_id = en.id\n"
                     "ORDER BY weight DESC LIMIT 20\n"
                     "```\n\n"
                     "**Check index usage:**\n"
@@ -630,10 +622,11 @@ with gr.Blocks(title="ConceptNet SQLite Explorer", theme=gr.themes.Soft()) as de
             schema_output = gr.Markdown("*Click button to load schema...*")
             
     gr.Markdown("---")
+    index_count = len(list(Path(INDEX_PATH).glob("*.ldb"))) if INDEX_PATH and os.path.exists(INDEX_PATH) else 0
     gr.Markdown(
-        "💡 **Performance:** Queries use database indices for fast lookups. "
-        "Exact matches are faster than wildcards. "
-        f"{'✅ Index files loaded from HuggingFace cache.' if INDEX_PATH else '⚠️ Running without index cache - queries may be slower.'}"
+        f"💡 **Performance:** Queries use database indices for fast lookups. "
+        f"Exact matches are faster than wildcards. "
+        f"{'✅ ' + str(index_count) + ' index files loaded.' if index_count > 0 else '⚠️ Running without index cache.'}"
     )
             
     # Connect UI to functions