jill/LLMsForDisinformationAnalysis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Actually we need to go the other way

Browse Source
This commit is contained in:
William Jeynes
2026-03-23 14:03:06 +00:00
parent bff5423f3d
commit 070aab6a5c
1 changed files with 24 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
supporting/RAGAS_Service/train_roberta.py
+24 -20
View File
@@ -10,7 +10,7 @@ import csv
import numpy as np import numpy as np
NUM_CLASSES = 3 NUM_CLASSES = 3
model_name = "microsoft/deberta-v3-base" model_name = "distilbert/distilroberta-base"
LABEL_PRIORITY = [ LABEL_PRIORITY = [
("PERFECT", 0), ("PERFECT", 0),
@@ -29,19 +29,20 @@ class WeightedTrainer(Trainer):
def compute_loss(self, model, inputs, return_outputs=False, **kwargs): def compute_loss(self, model, inputs, return_outputs=False, **kwargs):
labels = inputs.get("labels") labels = inputs.get("labels")
print("Before forward") # print("DBG: Before forward")
outputs = model(**inputs) outputs = model(**inputs)
print("After forward") # print("DBG: After forward")
logits = outputs.get("logits") logits = outputs.get("logits")
# loss_fct = CrossEntropyLoss(weight=self.class_weights.to(logits.device)) # loss_fct = CrossEntropyLoss(weight=self.class_weights.to(logits.device))
loss_fct = CrossEntropyLoss( # loss_fct = CrossEntropyLoss(
weight=self.class_weights.to(logits.device).to(logits.dtype) # weight=self.class_weights.to(logits.device).to(logits.dtype)
) # )
print("Before loss") loss_fct = CrossEntropyLoss()
# print("DBG: Before loss")
loss = loss_fct(logits, labels) loss = loss_fct(logits, labels)
# loss.backward() # loss.backward()
print("After loss") # print("DBG: After loss")
return (loss, outputs) if return_outputs else loss return (loss, outputs) if return_outputs else loss
def label_to_int(extra_info: str) -> int: def label_to_int(extra_info: str) -> int:
@@ -121,7 +122,7 @@ def compute_metrics(eval_pred):
} }
def main(): def main():
# torch.multiprocessing.set_start_method('fork') torch.multiprocessing.set_start_method('fork')
print("CUDA available:", torch.cuda.is_available()) print("CUDA available:", torch.cuda.is_available())
print("CUDA device count:", torch.cuda.device_count()) print("CUDA device count:", torch.cuda.device_count())
print("Current device:", torch.cuda.current_device() if torch.cuda.is_available() else "CPU") print("Current device:", torch.cuda.current_device() if torch.cuda.is_available() else "CPU")
@@ -139,11 +140,11 @@ def main():
num_labels=NUM_CLASSES num_labels=NUM_CLASSES
) )
for param in model.deberta.parameters(): # for param in model.deberta.parameters():
param.requires_grad = False # param.requires_grad = True
for param in model.deberta.encoder.layer[-1:].parameters(): # for param in model.deberta.encoder.layer[-6:].parameters():
param.requires_grad = True # param.requires_grad = True
print("Dataset size:", len(texts)) print("Dataset size:", len(texts))
print("Label distribution:") print("Label distribution:")
@@ -153,7 +154,8 @@ def main():
texts, texts,
labels, labels,
test_size=0.2, test_size=0.2,
random_state=42 random_state=42,
stratify=labels
) )
@@ -170,14 +172,14 @@ def main():
train_texts, train_texts,
truncation=True, truncation=True,
padding=True, padding=True,
max_length=256 max_length=512
) )
val_encodings = tokenizer( val_encodings = tokenizer(
val_texts, val_texts,
truncation=True, truncation=True,
padding=True, padding=True,
max_length=256 max_length=512
) )
class TextDataset(torch.utils.data.Dataset): class TextDataset(torch.utils.data.Dataset):
@@ -186,7 +188,7 @@ def main():
self.labels = labels self.labels = labels
def __getitem__(self, idx): def __getitem__(self, idx):
print(f"Loading item {idx}") # print(f"DBG: Loading item {idx}")
item = { item = {
key: torch.tensor(val[idx]) key: torch.tensor(val[idx])
for key, val in self.encodings.items() for key, val in self.encodings.items()
@@ -200,8 +202,9 @@ def main():
training_args = TrainingArguments( training_args = TrainingArguments(
output_dir="./results", output_dir="./results",
learning_rate=2e-5, learning_rate=2e-5,
per_device_train_batch_size=32, per_device_train_batch_size=16,
num_train_epochs=5, gradient_accumulation_steps=2,
num_train_epochs=10,
weight_decay=0.01, weight_decay=0.01,
load_best_model_at_end=True, load_best_model_at_end=True,
eval_strategy="epoch", eval_strategy="epoch",
@@ -209,7 +212,8 @@ def main():
metric_for_best_model="f1", metric_for_best_model="f1",
greater_is_better=True, greater_is_better=True,
dataloader_num_workers=4, dataloader_num_workers=4,
dataloader_pin_memory=True dataloader_pin_memory=True,
# warmup_steps=100,
) )
train_dataset = TextDataset(train_encodings, train_labels) train_dataset = TextDataset(train_encodings, train_labels)