dataset model csv ingles

app.py

@@ -477,56 +477,67 @@ class ModeloDataset:
             print('idioma:',idioma)            
             self.tokenizer = AutoTokenizer.from_pretrained("dayannex/distilbert-tuned-4labels")
             self.model = AutoModelForTokenClassification.from_pretrained("dayannex/distilbert-tuned-4labels")     
-   #tokenized_text=[self.tokenizer.tokenize(sentence[:500]) for sentence in _sentences]
-            inputs=[self.tokenizer(sentence[:500], return_tensors="pt") for sentence in _sentences]
-            print('inputs',inputs)
-            #ids = [self.tokenizer.convert_tokens_to_ids(x) for x in tokenized_text]
-            
+   
+            inputs = self.tokenizer(_sentences, padding=True, truncation=True, return_tensors="pt", max_length=512)
+            #model.eval()
 
-            #MAX_LEN=128
-            #ids=pad_sequences(ids,maxlen=MAX_LEN,dtype="long",truncating="post", padding="post")
-            #input_ids = torch.tensor(ids) 
-            
             with torch.no_grad():
-                outputs = self.model(inputs)
-            logits = outputs.logits   
-            predicted_token_class_ids = torch.argmax(logits, dim=2)     
+                outputs = model(**inputs)
+
+
+            logits = outputs.logits
+            predictions = torch.argmax(logits, dim=2)
+
+            id2label = model.config.id2label
+
+
+            all_tokens = []
+            all_label_ids = []
+            all_labels = []
+            for i, sentence in enumerate(_sentences):
             
-            #predicted_token_class_ids = predicted_token_class_ids[0].tolist()
-            i=0
-            _predicted_tokens_classes=[]
-            for a in predicted_token_class_ids:
-                    
-                    _predicted_tokens_classes.append( [self.model.config.id2label[label_id] for label_id in predicted_token_class_ids[i]])
-                    i=i+1 
-            print('_predicted_tokens_classes:',_predicted_tokens_classes[0])
-            #with torch.no_grad():
-            # logits = self.model(input_ids).logits
-            #predicted_token_class_ids = logits.argmax(-1)
-            #i=0
-            #_predicted_tokens_classes=[]
-            #for a in predicted_token_class_ids:
-                    
-            #        _predicted_tokens_classes.append([self.model.config.id2label[t.item()] for t in predicted_token_class_ids[i]])
-            #        i=i+1  
-            #labels = predicted_token_class_ids
-            #loss = self.model(input_ids, labels=labels).loss
+                tokens = self.tokenizer.convert_ids_to_tokens(inputs.input_ids[i])
+                label_ids = predictions[i].tolist()
+                labels = [id2label[label_id] for label_id in label_ids]
+                
+                
+                all_tokens.append(tokens)
+                all_label_ids.append(label_ids)
+                all_labels.append(labels)
+
+            #se eliminan el primer y ultimo elemento
+            for item in all_tokens:
+                item.pop(0)
+                item.pop(len(item)-1)
+            for item in all_labels:
+                item.pop(0)
+                item.pop(len(item)-1)
+
 
             new_tokens=[]
             ig_tok=[]
             i=0
             new_identificadores=[]
-            for item in tokenized_text:
-               
-               aux1, aux2= self.reordenacion_tokens(item,"#")
-               new_tokens.append(aux1)
-               ig_tok.append(aux2)
-             
-            
-            for items in _predicted_tokens_classes:
-                aux=self.reordenacion_identificadores(ig_tok[i],items,len(new_tokens[i]))
-                new_identificadores.append(aux)
-                i=i+1
+            for item in all_tokens:
+                aux1, aux2= self.reordenacion_tokens(item,"#")
+                new_tokens.append(aux1)
+                ig_tok.append(aux2)
+
+            print('ig_tok')
+            print(ig_tok)
+            #print('all_tokens')
+            #print(all_tokens)
+            i=0
+            for items in all_labels:
+                            aux=self.reordenacion_identificadores(ig_tok[i],items,len(new_tokens[i]))
+                            new_identificadores.append(aux)
+                            i=i+1
+            print('new_tokens') 
+            print(new_tokens[1])
+            print(all_tokens[1])
+
+            print(len(new_tokens[1]))
+            print(len(new_identificadores[1]))
             
             
             return new_identificadores, new_tokens