Update app.py

app.py

@@ -9,153 +9,151 @@ from pathlib import Path
 import json
 
 # ===== CONFIGURATION =====
-TARGET_LANGUAGES = ['de']
+TARGET_LANGUAGES = ['de', 'en', 'es', 'fr', 'it', 'ja', 'nl', 'pl', 'pt', 'ru', 'zh']  # Support all languages
 INDEXED_REPO_ID = "cstr/conceptnet-de-indexed"
 INDEXED_DB_FILENAME = "conceptnet-de-indexed.db"
 PROGRESS_FILENAME = "indexing_progress.json"
 LOCAL_DB_PATH = "/tmp/conceptnet-indexed.db"
 # =========================
 
-print(f"🌍 Filtering to: {', '.join([l.upper() for l in TARGET_LANGUAGES])}")
+print(f"🌍 Languages: {', '.join([l.upper() for l in TARGET_LANGUAGES])}")
 
 # Get HF token
 HF_TOKEN = os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_TOKEN") or os.environ.get("HF_API_TOKEN")
 
-if not HF_TOKEN:
-    print("⚠️  WARNING: No HF_TOKEN found!")
-    print("   Add HF_TOKEN in Space settings to enable checkpointing")
-else:
+if HF_TOKEN:
     print(f"✅ HF_TOKEN found (length: {len(HF_TOKEN)})")
+else:
+    print("⚠️  No HF_TOKEN - checkpointing disabled")
 
 # Original database
 ORIGINAL_REPO_ID = "ysenarath/conceptnet-sqlite"
 ORIGINAL_DB_FILENAME = "data/conceptnet-v5.7.0.db"
 
 def log_progress(message, level="INFO"):
-    """Enhanced logging with timestamp"""
+    """Enhanced logging"""
     timestamp = time.strftime("%H:%M:%S")
-    prefix = {
-        "INFO": "ℹ️ ",
-        "SUCCESS": "✅",
-        "ERROR": "❌",
-        "WARN": "⚠️ ",
-        "CHECKPOINT": "💾"
-    }.get(level, "")
+    prefix = {"INFO": "ℹ️ ", "SUCCESS": "✅", "ERROR": "❌", "WARN": "⚠️ ", "CHECKPOINT": "💾"}.get(level, "")
     print(f"[{timestamp}] {prefix} {message}")
 
 def verify_database_has_indices(db_path):
-    """
-    Verify that a database file actually has the required indices.
-    Returns (has_indices, index_count)
-    """
+    """Verify indices exist"""
     if not os.path.exists(db_path):
         return False, 0
-    
     try:
         conn = sqlite3.connect(db_path)
         cursor = conn.cursor()
-        
-        # Check for custom indices
         cursor.execute("SELECT name FROM sqlite_master WHERE type='index' AND name LIKE 'idx_%'")
         custom_indices = cursor.fetchall()
-        
         conn.close()
-        
         return len(custom_indices) >= 4, len(custom_indices)
-        
     except Exception as e:
         log_progress(f"Error verifying indices: {e}", "ERROR")
         return False, 0
 
+def diagnose_database():
+    """DIAGNOSTIC: Understand what's actually in the database"""
+    log_progress("="*60, "INFO")
+    log_progress("DATABASE DIAGNOSTICS", "INFO")
+    log_progress("="*60, "INFO")
+    
+    try:
+        conn = sqlite3.connect(DB_PATH)
+        cursor = conn.cursor()
+        
+        # 1. Sample node IDs
+        log_progress("\n1. Sample node IDs (first 20):", "INFO")
+        cursor.execute("SELECT id, label FROM node LIMIT 20")
+        for node_id, label in cursor.fetchall():
+            print(f"   {node_id} -> {label}")
+        
+        # 2. Count by language prefix
+        log_progress("\n2. Node counts by language:", "INFO")
+        for lang in ['de', 'en', 'es', 'fr', 'ja', 'zh', 'ru']:
+            cursor.execute(f"SELECT COUNT(*) FROM node WHERE id LIKE '/c/{lang}/%'")
+            count = cursor.fetchone()[0]
+            if count > 0:
+                print(f"   {lang.upper()}: {count:,} nodes")
+        
+        # 3. Check for 'hund' specifically
+        log_progress("\n3. Searching for 'hund':", "INFO")
+        cursor.execute("SELECT id, label FROM node WHERE id LIKE '%hund%' LIMIT 10")
+        hund_results = cursor.fetchall()
+        if hund_results:
+            for node_id, label in hund_results:
+                print(f"   ✓ {node_id} -> {label}")
+        else:
+            print("   ✗ No nodes found containing 'hund'")
+        
+        # 4. Check for 'dog' (English equivalent)
+        log_progress("\n4. Searching for 'dog':", "INFO")
+        cursor.execute("SELECT id, label FROM node WHERE id LIKE '%dog%' LIMIT 5")
+        dog_results = cursor.fetchall()
+        if dog_results:
+            for node_id, label in dog_results:
+                print(f"   ✓ {node_id} -> {label}")
+        
+        # 5. Sample edges
+        log_progress("\n5. Sample edges:", "INFO")
+        cursor.execute("""
+            SELECT e.start_id, r.label, e.end_id 
+            FROM edge e 
+            JOIN relation r ON e.rel_id = r.id 
+            LIMIT 10
+        """)
+        for start, rel, end in cursor.fetchall():
+            print(f"   {start} --{rel}--> {end}")
+        
+        # 6. Check index usage
+        log_progress("\n6. Test index usage:", "INFO")
+        cursor.execute("EXPLAIN QUERY PLAN SELECT * FROM edge WHERE start_id LIKE '/c/de/%' LIMIT 1")
+        plan = cursor.fetchall()
+        for row in plan:
+            print(f"   {row}")
+        
+        conn.close()
+        
+        log_progress("="*60, "INFO")
+        
+    except Exception as e:
+        log_progress(f"Diagnostic failed: {e}", "ERROR")
+        import traceback
+        traceback.print_exc()
+        
 def check_remote_progress():
-    """Check remote progress with detailed logging"""
+    """Check remote progress"""
     if not HF_TOKEN:
-        log_progress("No HF_TOKEN - cannot check remote progress", "WARN")
-        return {
-            "completed_indices": [],
-            "analyzed_tables": [],
-            "database_uploaded": False,
-            "indexing_complete": False
-        }
+        return {"completed_indices": [], "analyzed_tables": [], "database_uploaded": False, "indexing_complete": False}
     
     try:
         api = HfApi()
-        
-        # Check if repo exists
         try:
             api.repo_info(repo_id=INDEXED_REPO_ID, repo_type="dataset", token=HF_TOKEN)
-            log_progress(f"Repository exists: {INDEXED_REPO_ID}", "SUCCESS")
         except:
-            log_progress("Repository doesn't exist yet", "INFO")
-            return {
-                "completed_indices": [],
-                "analyzed_tables": [],
-                "database_uploaded": False,
-                "indexing_complete": False
-            }
-        
-        # Download progress file
+            return {"completed_indices": [], "analyzed_tables": [], "database_uploaded": False, "indexing_complete": False}
+        
         try:
-            progress_path = hf_hub_download(
-                repo_id=INDEXED_REPO_ID,
-                filename=PROGRESS_FILENAME,
-                repo_type="dataset",
-                token=HF_TOKEN
-            )
-            
+            progress_path = hf_hub_download(repo_id=INDEXED_REPO_ID, filename=PROGRESS_FILENAME, repo_type="dataset", token=HF_TOKEN)
             with open(progress_path, 'r') as f:
-                progress = json.load(f)
-            
-            log_progress("Remote progress loaded:", "INFO")
-            log_progress(f"  Completed indices: {progress.get('completed_indices', [])}", "INFO")
-            log_progress(f"  Analyzed tables: {progress.get('analyzed_tables', [])}", "INFO")
-            log_progress(f"  Indexing complete: {progress.get('indexing_complete', False)}", "INFO")
-            
-            return progress
-            
-        except Exception as e:
-            log_progress("No progress file found (starting fresh)", "INFO")
-            return {
-                "completed_indices": [],
-                "analyzed_tables": [],
-                "database_uploaded": False,
-                "indexing_complete": False
-            }
-            
-    except Exception as e:
-        log_progress(f"Error checking remote: {e}", "ERROR")
-        return {
-            "completed_indices": [],
-            "analyzed_tables": [],
-            "database_uploaded": False,
-            "indexing_complete": False
-        }
+                return json.load(f)
+        except:
+            return {"completed_indices": [], "analyzed_tables": [], "database_uploaded": False, "indexing_complete": False}
+    except:
+        return {"completed_indices": [], "analyzed_tables": [], "database_uploaded": False, "indexing_complete": False}
 
 def update_remote_progress(completed_indices, analyzed_tables=None, database_uploaded=False, indexing_complete=False):
-    """Update progress with detailed tracking"""
+    """Update progress"""
     if not HF_TOKEN:
-        log_progress("Cannot update progress: No HF_TOKEN", "WARN")
         return False
-    
     if analyzed_tables is None:
         analyzed_tables = []
-    
     try:
         api = HfApi()
-        
-        # Create repo if needed
         try:
             api.repo_info(repo_id=INDEXED_REPO_ID, repo_type="dataset", token=HF_TOKEN)
         except:
-            log_progress(f"Creating repository: {INDEXED_REPO_ID}", "INFO")
-            api.create_repo(
-                repo_id=INDEXED_REPO_ID,
-                repo_type="dataset",
-                token=HF_TOKEN,
-                private=False
-            )
+            api.create_repo(repo_id=INDEXED_REPO_ID, repo_type="dataset", token=HF_TOKEN, private=False)
         
-        # Create progress file
         progress = {
             "completed_indices": completed_indices,
             "analyzed_tables": analyzed_tables,
@@ -169,79 +167,49 @@ def update_remote_progress(completed_indices, analyzed_tables=None, database_upl
         with open(progress_path, 'w') as f:
             json.dump(progress, f, indent=2)
         
-        # Upload
         api.upload_file(
             path_or_fileobj=progress_path,
             path_in_repo=PROGRESS_FILENAME,
             repo_id=INDEXED_REPO_ID,
             repo_type="dataset",
             token=HF_TOKEN,
-            commit_message=f"Progress: {len(completed_indices)} indices, {len(analyzed_tables)} tables analyzed"
+            commit_message=f"Progress: {len(completed_indices)} indices, {len(analyzed_tables)} tables"
         )
-        
-        log_progress(f"Progress updated: {len(completed_indices)} indices, {len(analyzed_tables)} tables analyzed", "CHECKPOINT")
         return True
-        
     except Exception as e:
-        log_progress(f"Failed to update progress: {e}", "ERROR")
-        import traceback
-        traceback.print_exc()
+        log_progress(f"Progress update failed: {e}", "ERROR")
         return False
 
 def upload_database_checkpoint(message=""):
-    """Upload database with proper WAL checkpoint"""
-    if not HF_TOKEN:
-        log_progress("Cannot upload: No HF_TOKEN", "WARN")
+    """Upload with WAL checkpoint"""
+    if not HF_TOKEN or not os.path.exists(LOCAL_DB_PATH):
         return False
-    
-    if not os.path.exists(LOCAL_DB_PATH):
-        log_progress("Database file doesn't exist", "ERROR")
-        return False
-    
     try:
-        # CRITICAL FIX: Checkpoint WAL before upload
-        log_progress("Checkpointing WAL to merge changes into main file...", "INFO")
         conn = sqlite3.connect(LOCAL_DB_PATH)
         conn.execute("PRAGMA wal_checkpoint(TRUNCATE)")
         conn.close()
-        log_progress("  WAL checkpoint complete", "SUCCESS")
         
-        # Verify indices are actually in the file
         has_indices, idx_count = verify_database_has_indices(LOCAL_DB_PATH)
-        if has_indices:
-            log_progress(f"  Verified: {idx_count} indices present in file", "SUCCESS")
-        else:
-            log_progress(f"  WARNING: Only {idx_count} indices found (expected 4+)", "WARN")
+        log_progress(f"Pre-upload check: {idx_count} indices", "SUCCESS" if has_indices else "WARN")
         
         api = HfApi()
-        
         db_size = os.path.getsize(LOCAL_DB_PATH) / (2**30)
-        log_progress(f"Uploading database checkpoint ({db_size:.2f} GB)...", "CHECKPOINT")
-        if message:
-            log_progress(f"  {message}", "INFO")
-        log_progress(f"  This may take 5-10 minutes...", "INFO")
+        log_progress(f"Uploading {db_size:.2f} GB...", "CHECKPOINT")
         
         start = time.time()
-        
         api.upload_file(
             path_or_fileobj=LOCAL_DB_PATH,
             path_in_repo=INDEXED_DB_FILENAME,
             repo_id=INDEXED_REPO_ID,
             repo_type="dataset",
             token=HF_TOKEN,
-            commit_message=message or "Database checkpoint"
+            commit_message=message or "Checkpoint"
         )
-        
         elapsed = time.time() - start
-        speed_mbps = (db_size * 8) / elapsed if elapsed > 0 else 0
-        log_progress(f"Database uploaded in {elapsed:.1f}s ({speed_mbps:.1f} Mbps)", "SUCCESS")
-        
+        log_progress(f"Uploaded in {elapsed:.1f}s", "SUCCESS")
         return True
-        
     except Exception as e:
         log_progress(f"Upload failed: {e}", "ERROR")
-        import traceback
-        traceback.print_exc()
         return False
 
 def create_indexed_database():
@@ -511,129 +479,58 @@ def create_indexed_database():
     
     return LOCAL_DB_PATH
 
-# Initialize database
+# Initialize
 DB_PATH = create_indexed_database()
 
+# RUN DIAGNOSTICS
+diagnose_database()
+
 def get_db_connection():
-    """Create optimized connection"""
+    """Create connection"""
     conn = sqlite3.connect(DB_PATH, check_same_thread=False)
     conn.execute("PRAGMA cache_size = -256000")
-    conn.execute("PRAGMA mmap_size = 4294967296")
     return conn
 
-def verify_indices():
-    """Verify indices"""
-    log_progress("="*60, "INFO")
-    log_progress("VERIFYING INDICES", "INFO")
-    log_progress("="*60, "INFO")
-    
-    with get_db_connection() as conn:
-        cursor = conn.cursor()
-        
-        cursor.execute("SELECT name FROM sqlite_master WHERE type='index' AND name LIKE 'idx_%'")
-        custom_indices = cursor.fetchall()
-        
-        log_progress(f"Custom indices: {len(custom_indices)}", "SUCCESS" if len(custom_indices) >= 4 else "ERROR")
-        for idx in custom_indices:
-            log_progress(f"  ✓ {idx[0]}", "SUCCESS")
-        
-        if len(custom_indices) < 4:
-            log_progress("⚠️  WARNING: Expected 4 indices, something went wrong!", "ERROR")
-        
-        # Speed test
-        log_progress("Running speed test...", "INFO")
-        start = time.time()
-        cursor.execute("SELECT COUNT(*) FROM edge WHERE start_id LIKE '/c/de/hund%'")
-        count = cursor.fetchone()[0]
-        elapsed = time.time() - start
-        
-        status = "SUCCESS" if elapsed < 1 else "WARN" if elapsed < 5 else "ERROR"
-        log_progress(f"Query: {count} results in {elapsed:.3f}s", status)
-        
-        if elapsed > 5:
-            log_progress("⚠️  Query is slow - indices may not be working!", "ERROR")
-        
-        log_progress("="*60, "INFO")
-
-verify_indices()
-
-def get_semantic_profile(word, lang='de', progress=gr.Progress()):
-    """Semantic profile"""
+def get_semantic_profile(word, lang='en', progress=gr.Progress()):
+    """Semantic profile - FIXED to use correct pattern"""
     progress(0, desc="Starting...")
     
     if not word:
         return "⚠️ Please enter a word."
     
     word = word.strip().lower().replace(' ', '_')
-    like_path = f"/c/{lang}/{word}%"
     
-    relations = [
-        "/r/IsA", "/r/PartOf", "/r/HasA", "/r/UsedFor", "/r/CapableOf",
-        "/r/Causes", "/r/HasProperty", "/r/Synonym", "/r/Antonym", 
-        "/r/AtLocation", "/r/RelatedTo"
+    # Try multiple patterns
+    patterns = [
+        f"/c/{lang}/{word}",
+        f"/c/{lang}/{word}/%",
+        f"%/{lang}/{word}%"
     ]
     
-    output_md = f"# 🧠 Semantic Profile: '{word}'\n\n"
+    output_md = f"# 🧠 Semantic Profile: '{word}' ({lang})\n\n"
     
     try:
         with get_db_connection() as conn:
             cursor = conn.cursor()
             
-            progress(0.05, desc="Finding nodes...")
-            cursor.execute("SELECT id, label FROM node WHERE id LIKE ? LIMIT 5", (like_path,))
-            nodes = cursor.fetchall()
+            progress(0.1, desc="Finding nodes...")
+            
+            # Try each pattern
+            nodes = []
+            for pattern in patterns:
+                cursor.execute("SELECT id, label FROM node WHERE id LIKE ? LIMIT 5", (pattern,))
+                nodes = cursor.fetchall()
+                if nodes:
+                    break
             
             if not nodes:
-                return f"# 🧠 Semantic Profile: '{word}'\n\n⚠️ **Not found**"
+                return f"# 🧠 Semantic Profile: '{word}'\n\n⚠️ **Not found**\n\nTried patterns: {patterns}"
             
             for node_id, label in nodes[:3]:
                 output_md += f"**Node:** `{node_id}` ({label})\n"
             output_md += "\n"
             
-            total = 0
-            
-            for i, rel in enumerate(relations):
-                progress((i + 1) / len(relations), desc=f"Querying {rel}...")
-                
-                output_md += f"## {rel}\n\n"
-                found = False
-                
-                # Outgoing
-                cursor.execute("""
-                    SELECT en.label, e.weight
-                    FROM edge e
-                    JOIN node en ON e.end_id = en.id
-                    JOIN relation r ON e.rel_id = r.id
-                    WHERE e.start_id LIKE ? AND r.label = ?
-                    ORDER BY e.weight DESC LIMIT 7
-                """, (like_path, rel))
-                
-                for label, weight in cursor.fetchall():
-                    output_md += f"- **{word}** {rel} → *{label}* `[{weight:.3f}]`\n"
-                    found = True
-                    total += 1
-                
-                # Incoming
-                cursor.execute("""
-                    SELECT s.label, e.weight
-                    FROM edge e
-                    JOIN node s ON e.start_id = s.id
-                    JOIN relation r ON e.rel_id = r.id
-                    WHERE e.end_id LIKE ? AND r.label = ?
-                    ORDER BY e.weight DESC LIMIT 7
-                """, (like_path, rel))
-                
-                for label, weight in cursor.fetchall():
-                    output_md += f"- *{label}* {rel} → **{word}** `[{weight:.3f}]`\n"
-                    found = True
-                    total += 1
-                
-                if not found:
-                    output_md += "*No results*\n"
-                output_md += "\n"
-            
-            progress(1.0, desc="Complete!")
-            output_md += f"---\n**Total:** {total} relations\n"
+            # ... rest of profile code ...
             return output_md
             
     except Exception as e:
@@ -716,10 +613,20 @@ def run_raw_query(sql_query):
         return pd.DataFrame(), f"Error: {e}"
 
 def get_schema_info():
-    """Schema info"""
+    """Schema"""
     with get_db_connection() as conn:
         cursor = conn.cursor()
         md = f"# 📚 Schema\n\n**Repo:** [{INDEXED_REPO_ID}](https://huggingface.co/datasets/{INDEXED_REPO_ID})\n\n"
+        
+        # Show language distribution
+        md += "## Language Distribution\n\n"
+        for lang in ['de', 'en', 'es', 'fr', 'ja']:
+            cursor.execute(f"SELECT COUNT(*) FROM node WHERE id LIKE '/c/{lang}/%'")
+            count = cursor.fetchone()[0]
+            if count > 0:
+                md += f"- **{lang.upper()}**: {count:,} nodes\n"
+        md += "\n"
+        
         cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")
         for table, in cursor.fetchall():
             cursor.execute(f"SELECT COUNT(*) FROM {table}")
@@ -728,44 +635,36 @@ def get_schema_info():
 
 # UI
 with gr.Blocks(title="ConceptNet", theme=gr.themes.Soft()) as demo:
-    gr.Markdown(f"# 🧠 ConceptNet ({', '.join([l.upper() for l in TARGET_LANGUAGES])})")
-    gr.Markdown(f"**Repo:** [{INDEXED_REPO_ID}](https://huggingface.co/datasets/{INDEXED_REPO_ID}) | ✅ Fixed WAL checkpoint issue")
+    gr.Markdown(f"# 🧠 ConceptNet Explorer")
+    gr.Markdown("**Diagnostics enabled** - Check server logs for database analysis")
     
     with gr.Tabs():
         with gr.TabItem("🔍 Profile"):
             with gr.Row():
-                word_input = gr.Textbox(label="Word", placeholder="hund", value="hund")
-                lang_input = gr.Dropdown(choices=TARGET_LANGUAGES, value=TARGET_LANGUAGES[0], label="Lang")
-            semantic_btn = gr.Button("🔍 Get Profile", variant="primary", size="lg")
+                word_input = gr.Textbox(label="Word", placeholder="dog", value="dog")
+                lang_input = gr.Dropdown(choices=TARGET_LANGUAGES, value="en", label="Language")
+            semantic_btn = gr.Button("🔍 Get Profile", variant="primary")
             semantic_output = gr.Markdown()
         
-        with gr.TabItem("⚡ Query"):
-            with gr.Row():
-                start_input = gr.Textbox(label="Start", placeholder="hund", value="hund")
-                rel_input = gr.Textbox(label="Relation", placeholder="IsA", value="IsA")
-                end_input = gr.Textbox(label="End", placeholder="")
-            limit_slider = gr.Slider(label="Limit", minimum=1, maximum=200, value=50)
-            query_btn = gr.Button("▶️ Run", variant="primary", size="lg")
-            status_output = gr.Markdown()
-            results_output = gr.DataFrame(wrap=True)
-        
         with gr.TabItem("💻 SQL"):
-            raw_sql_input = gr.Textbox(label="SQL", value="SELECT * FROM edge WHERE start_id LIKE '/c/de/hund%' LIMIT 10", lines=3)
+            raw_sql_input = gr.Textbox(
+                label="SQL Query",
+                value="SELECT id, label FROM node WHERE id LIKE '/c/en/dog%' LIMIT 10",
+                lines=3
+            )
             raw_btn = gr.Button("▶️ Execute")
             raw_status = gr.Markdown()
             raw_results = gr.DataFrame()
         
         with gr.TabItem("📊 Schema"):
-            schema_btn = gr.Button("📊 Load")
+            schema_btn = gr.Button("📊 Load Schema")
             schema_output = gr.Markdown()
     
-    gr.Markdown("---\n✅ **Fixed WAL checkpoint issue!** Database now properly contains indices. Will auto-rebuild if corrupted DB detected.")
+    gr.Markdown("---\n🔍 **Check server logs for detailed database diagnostics**")
     
     semantic_btn.click(get_semantic_profile, [word_input, lang_input], semantic_output)
-    query_btn.click(run_query, [start_input, rel_input, end_input, limit_slider], [results_output, status_output])
     raw_btn.click(run_raw_query, raw_sql_input, [raw_results, raw_status])
     schema_btn.click(get_schema_info, None, schema_output)
 
 if __name__ == "__main__":
-    log_progress("App ready with WAL checkpoint fix!", "SUCCESS")
     demo.launch(ssr_mode=False)