Ensire works on CUDA for extra speed

supporting/RAGAS_Service/train_roberta.py

@@ -113,104 +113,115 @@ def compute_metrics(eval_pred):
         "recall": recall_score(labels, preds, average="weighted", zero_division=0),
     }
 
-texts, labels = load_dataset_from_csv("../../data/classify.csv")
+def main():
+    torch.multiprocessing.set_start_method('fork')
+    print("CUDA available:", torch.cuda.is_available())
+    print("CUDA device count:", torch.cuda.device_count())
+    print("Current device:", torch.cuda.current_device() if torch.cuda.is_available() else "CPU")
+    texts, labels = load_dataset_from_csv("../../data/classify.csv")
 
-tokenizer = RobertaTokenizer.from_pretrained(model_name)
+    tokenizer = RobertaTokenizer.from_pretrained(model_name)
-model = RobertaForSequenceClassification.from_pretrained(
+    model = RobertaForSequenceClassification.from_pretrained(
-    model_name,
+        model_name,
-    num_labels=NUM_CLASSES
+        num_labels=NUM_CLASSES
-)
+    )
 
-for param in model.roberta.parameters():
+    for param in model.roberta.parameters():
-    param.requires_grad = False
+        param.requires_grad = False
 
-for param in model.roberta.encoder.layer[-3:].parameters():
+    for param in model.roberta.encoder.layer[-3:].parameters():
-    param.requires_grad = True
+        param.requires_grad = True
 
-print("Dataset size:", len(texts))
+    print("Dataset size:", len(texts))
-print("Label distribution:")
+    print("Label distribution:")
-print(Counter(labels))
+    print(Counter(labels))
 
-train_texts, val_texts, train_labels, val_labels = train_test_split(
+    train_texts, val_texts, train_labels, val_labels = train_test_split(
-    texts,
+        texts,
-    labels,
+        labels,
-    test_size=0.2,
+        test_size=0.2,
-    random_state=42
+        random_state=42
-)
+    )
 
 
-class_weights = compute_class_weight(
+    class_weights = compute_class_weight(
-    class_weight="balanced",
+        class_weight="balanced",
-    classes=np.unique(train_labels),
+        classes=np.unique(train_labels),
-    y=train_labels
+        y=train_labels
-)
+    )
 
-class_weights = torch.tensor(class_weights, dtype=torch.float)
+    class_weights = torch.tensor(class_weights, dtype=torch.float)
-print("Class weights:", class_weights)
+    print("Class weights:", class_weights)
 
-train_encodings = tokenizer(
+    train_encodings = tokenizer(
-    train_texts,
+        train_texts,
-    truncation=True,
+        truncation=True,
-    padding=True,
+        padding=True,
-    max_length=256
+        max_length=256
-)
+    )
 
-val_encodings = tokenizer(
+    val_encodings = tokenizer(
-    val_texts,
+        val_texts,
-    truncation=True,
+        truncation=True,
-    padding=True,
+        padding=True,
-    max_length=256
+        max_length=256
-)
+    )
 
-class TextDataset(torch.utils.data.Dataset):
+    class TextDataset(torch.utils.data.Dataset):
-    def __init__(self, encodings, labels):
+        def __init__(self, encodings, labels):
-        self.encodings = encodings
+            self.encodings = encodings
-        self.labels = labels
+            self.labels = labels
 
-    def __getitem__(self, idx):
+        def __getitem__(self, idx):
-        item = {
+            item = {
-            key: torch.tensor(val[idx])
+                key: torch.tensor(val[idx])
-            for key, val in self.encodings.items()
+                for key, val in self.encodings.items()
-        }
+            }
-        item["labels"] = torch.tensor(self.labels[idx])
+            item["labels"] = torch.tensor(self.labels[idx])
-        return item
+            return item
 
-    def __len__(self):
+        def __len__(self):
-        return len(self.labels)
+            return len(self.labels)
 
-training_args = TrainingArguments(
+    training_args = TrainingArguments(
-    output_dir="./results",
+        output_dir="./results",
-    learning_rate=1e-5,
+        learning_rate=1e-5,
-    per_device_train_batch_size=8,
+        per_device_train_batch_size=8,
-    num_train_epochs=15,
+        num_train_epochs=15,
-    weight_decay=0.01,
+        weight_decay=0.01,
-    load_best_model_at_end=True,
+        load_best_model_at_end=True,
-    eval_strategy="epoch",
+        eval_strategy="epoch",
-    save_strategy="epoch",
+        save_strategy="epoch",
-    metric_for_best_model="f1",
+        metric_for_best_model="f1",
-    greater_is_better=True,
+        greater_is_better=True,
-    dataloader_pin_memory=False
+        dataloader_num_workers=4,
-)
+        dataloader_pin_memory=True
+    )
 
-train_dataset = TextDataset(train_encodings, train_labels)
+    train_dataset = TextDataset(train_encodings, train_labels)
 
-val_dataset = TextDataset(val_encodings, val_labels)
+    val_dataset = TextDataset(val_encodings, val_labels)
 
-trainer = WeightedTrainer(
+    trainer = WeightedTrainer(
-    model=model,
+        model=model,
-    args=training_args,
+        args=training_args,
-    train_dataset=train_dataset,
+        train_dataset=train_dataset,
-    eval_dataset=val_dataset,
+        eval_dataset=val_dataset,
-    compute_metrics=compute_metrics,
+        compute_metrics=compute_metrics,
-    class_weights=class_weights
+        class_weights=class_weights
-)
+    )
 
-trainer.train()
+    trainer.train()
 
-metrics = trainer.evaluate()
+    metrics = trainer.evaluate()
-print("Final evaluation metrics:")
+    print("Final evaluation metrics:")
-for k, v in metrics.items():
+    for k, v in metrics.items():
-    print(f"{k}: {v}")
+        print(f"{k}: {v}")
 
-trainer.save_model("./roberta_classifier")
+    trainer.save_model("./roberta_classifier")
-tokenizer.save_pretrained("./roberta_classifier")
+    tokenizer.save_pretrained("./roberta_classifier")
+
+
+
+if __name__ == "__main__":
+    main()