dataset model json en

app.py

@@ -533,14 +533,21 @@ class ModeloDataset:
             special_tokens = self.tokenizer.all_special_tokens
             filtered_tokens = []
             filtered_labels = []
-
-            for token, label in zip(new_tokens, new_identificadores):
-                if token not in special_tokens:
-                    filtered_tokens.append(token)
-                    filtered_labels.append(label)
-                        
             
-            return filtered_labels,filtered_tokens#new_identificadores, new_tokens
+            
+            for token_linea, label_linea in zip(new_tokens, new_identificadores):
+                for token, label in zip(token_linea, label_linea):
+                    if token not in special_tokens:
+                        print('token:*********************************', token)
+                        filtered_tokens.append(token)
+                        filtered_labels.append(label)
+            print('filtered_tokens')
+            print(filtered_tokens)
+            print('filtered_labels')
+            print(filtered_labels)
+                                
+            
+            return filtered_labels,filtered_tokens #new_identificadores, new_tokens
         
   ###
   ### Procesa los tokens generados del texto de entradas con los tokens predichos, para generar los tokens por palabra