Upload 5 files

api.py

@@ -13,6 +13,7 @@ from enum import Enum
 import os
 
 from video_processing import process_video_pipeline
+from audio_tools import process_audio_for_video
 from casting_loader import ensure_chroma, build_faces_index, build_voices_index
 from narration_system import NarrationSystem
 from llm_router import load_yaml, LLMRouter
@@ -172,13 +173,17 @@ def process_video_job(job_id: str):
                 output_base=str(base),
                 epsilon=epsilon,
                 min_cluster_size=min_cluster_size,
-                video_name=video_name
+                video_name=video_name,
+                start_offset_sec=5.0,
+                extract_every_sec=0.5
             )
             
             print(f"[{job_id}] DEBUG - result completo: {result}")
             
             characters = result.get("characters", [])
             analysis_path = result.get("analysis_path", "")
+            face_labels = result.get("face_labels", [])
+            num_face_embeddings = int(result.get("num_face_embeddings", 0))
             
             print(f"[{job_id}] Personajes detectados: {len(characters)}")
             for char in characters:
@@ -216,12 +221,44 @@ def process_video_job(job_id: str):
             except Exception as _e:
                 print(f"[{job_id}] WARN - No se pudo enumerar face_files: {_e}")
 
+            # Procesamiento de audio: diarización, ASR y embeddings de voz
+            try:
+                cfg = load_yaml("config.yaml")
+                audio_segments, srt_unmod, full_txt = process_audio_for_video(video_path, base, cfg, voice_collection=None)
+            except Exception as e_audio:
+                import traceback
+                print(f"[{job_id}] WARN - Audio pipeline failed: {e_audio}\n{traceback.format_exc()}")
+                audio_segments, srt_unmod, full_txt = [], None, ""
+
+            # Clustering de voces (DBSCAN sobre embeddings válidos)
+            from sklearn.cluster import DBSCAN
+            import numpy as np
+            voice_embeddings = [seg.get("voice_embedding") for seg in audio_segments if seg.get("voice_embedding")]
+            if voice_embeddings:
+                try:
+                    Xv = np.array(voice_embeddings)
+                    v_eps = 1.3
+                    v_min = 1
+                    v_labels = DBSCAN(eps=v_eps, min_samples=v_min, metric='euclidean').fit(Xv).labels_.tolist()
+                except Exception as _e:
+                    print(f"[{job_id}] WARN - Voice clustering failed: {_e}")
+                    v_labels = []
+            else:
+                v_labels = []
+
             # Guardar resultados primero y luego marcar como completado (evita carreras)
             job["results"] = {
                 "characters": characters,
                 "num_characters": len(characters),
                 "analysis_path": analysis_path,
-                "base_dir": str(base)
+                "base_dir": str(base),
+                "face_labels": face_labels,
+                "num_face_embeddings": num_face_embeddings,
+                "audio_segments": audio_segments,
+                "srt_unmodified": srt_unmod,
+                "full_transcription": full_txt,
+                "voice_labels": v_labels,
+                "num_voice_embeddings": len(voice_embeddings),
             }
             job["status"] = JobStatus.DONE
             

character_detection.py

@@ -54,7 +54,9 @@ class CharacterDetector:
         for d in [self.faces_dir, self.voices_dir, self.scenes_dir]:
             d.mkdir(parents=True, exist_ok=True)
     
-    def extract_faces_embeddings(self) -> List[Dict[str, Any]]:
+    def extract_faces_embeddings(self, *, start_offset_sec: float = 3.0, extract_every_sec: float = 0.5,
+                                 detector_backend: str = 'retinaface', min_face_area: int = 900,
+                                 enforce_detection: bool = True) -> List[Dict[str, Any]]:
         """
         Extrae caras del vídeo y calcula sus embeddings usando DeepFace directamente.
         
@@ -67,13 +69,14 @@ class CharacterDetector:
         
         logger.info("Extrayendo caras del vídeo con DeepFace...")
         
-        extract_every = 1.0  # segundos
+        extract_every = float(extract_every_sec)
         video = cv2.VideoCapture(self.video_path)
         fps = int(video.get(cv2.CAP_PROP_FPS))
         total_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
         frame_interval = int(fps * extract_every)
         frame_count = 0
         saved_count = 0
+        start_frame = int(max(0.0, start_offset_sec) * (fps if fps > 0 else 25))
         
         embeddings_caras = []
         
@@ -84,6 +87,10 @@ class CharacterDetector:
             if not ret:
                 break
             
+            if frame_count < start_frame:
+                frame_count += 1
+                continue
+
             if frame_count % frame_interval == 0:
                 temp_path = self.faces_dir / "temp_frame.jpg"
                 cv2.imwrite(str(temp_path), frame)
@@ -94,14 +101,21 @@ class CharacterDetector:
                     face_objs = DeepFace.represent(
                         img_path=str(temp_path),
                         model_name='Facenet512',
-                        detector_backend='opencv',
-                        enforce_detection=False
+                        detector_backend=detector_backend,
+                        enforce_detection=enforce_detection
                     )
                     
                     if face_objs:
                         for i, face_obj in enumerate(face_objs):
                             embedding = face_obj['embedding']
                             facial_area = face_obj.get('facial_area', {})
+                            try:
+                                w = int(facial_area.get('w', 0))
+                                h = int(facial_area.get('h', 0))
+                                if w * h < int(min_face_area):
+                                    continue
+                            except Exception:
+                                pass
                             
                             # Guardar el frame completo
                             save_path = self.faces_dir / f"frame_{saved_count:04d}.jpg"
@@ -270,7 +284,8 @@ class CharacterDetector:
         
         return analysis_path
     
-    def detect_characters(self, epsilon: float = 0.5, min_cluster_size: int = 2) -> Tuple[List[Dict], Path]:
+    def detect_characters(self, epsilon: float = 0.5, min_cluster_size: int = 2,
+                          *, start_offset_sec: float = 3.0, extract_every_sec: float = 0.5) -> Tuple[List[Dict], Path, np.ndarray, List[Dict[str, Any]]]:
         """
         Pipeline completo de detección de personajes.
         
@@ -282,7 +297,7 @@ class CharacterDetector:
             Tuple de (lista de personajes, path al analysis.json)
         """
         # 1. Extraer caras y embeddings
-        embeddings_caras = self.extract_faces_embeddings()
+        embeddings_caras = self.extract_faces_embeddings(start_offset_sec=start_offset_sec, extract_every_sec=extract_every_sec)
         
         # 2. Extraer voces y embeddings (opcional, por ahora)
         embeddings_voices = self.extract_voices_embeddings()
@@ -299,13 +314,14 @@ class CharacterDetector:
         # 6. Crear carpetas de personajes
         characters = self.create_character_folders(embeddings_caras, labels)
         
-        return characters, analysis_path
+        return characters, analysis_path, labels, embeddings_caras
 
 
 # Función de conveniencia para usar en el API
 def detect_characters_from_video(video_path: str, output_base: str, 
                                  epsilon: float = 0.5, min_cluster_size: int = 2,
-                                 video_name: str = None) -> Dict[str, Any]:
+                                 video_name: str = None,
+                                 *, start_offset_sec: float = 3.0, extract_every_sec: float = 0.5) -> Dict[str, Any]:
     """
     Función de alto nivel para detectar personajes en un vídeo.
     
@@ -320,10 +336,14 @@ def detect_characters_from_video(video_path: str, output_base: str,
         Dict con resultados: {"characters": [...], "analysis_path": "..."}
     """
     detector = CharacterDetector(video_path, Path(output_base), video_name=video_name)
-    characters, analysis_path = detector.detect_characters(epsilon, min_cluster_size)
+    characters, analysis_path, labels, embeddings_caras = detector.detect_characters(epsilon, min_cluster_size,
+                                                                                    start_offset_sec=start_offset_sec,
+                                                                                    extract_every_sec=extract_every_sec)
     
     return {
         "characters": characters,
         "analysis_path": str(analysis_path),
-        "num_characters": len(characters)
+        "num_characters": len(characters),
+        "face_labels": labels.tolist() if isinstance(labels, np.ndarray) else list(labels),
+        "num_face_embeddings": len(embeddings_caras)
     }