From c910bee66e1eb0551291828c52f20a67f0e09324 Mon Sep 17 00:00:00 2001 From: WillJeynes Date: Fri, 10 Apr 2026 19:23:41 +0100 Subject: [PATCH] Add a method of evaulation. Add back distilGPT version. Convert querying to another fastAPI --- finemodel/.gitignore | 3 +- finemodel/eval.py | 101 +++++++++++++++++++++++++++++ finemodel/lora.py | 126 +++++++++++++++++++++++++++++++++++++ finemodel/q_lora.py | 102 ++++++++++++++++++++++++++++++ finemodel/q_lora2.py | 66 +++++++++---------- finemodel/requirements.txt | 8 +++ 6 files changed, 369 insertions(+), 37 deletions(-) create mode 100644 finemodel/eval.py create mode 100644 finemodel/lora.py create mode 100644 finemodel/q_lora.py create mode 100644 finemodel/requirements.txt diff --git a/finemodel/.gitignore b/finemodel/.gitignore index bf587a9..9fd5384 100644 --- a/finemodel/.gitignore +++ b/finemodel/.gitignore @@ -1 +1,2 @@ -ft_*/ \ No newline at end of file +ft_*/ +*.pyc \ No newline at end of file diff --git a/finemodel/eval.py b/finemodel/eval.py new file mode 100644 index 0000000..cdb92b5 --- /dev/null +++ b/finemodel/eval.py @@ -0,0 +1,101 @@ +import requests +import json +import csv +import os +from datetime import datetime, timedelta +import feedparser + +# ----------------------------- +# Config +# ----------------------------- +RSS_URL = "https://feeds.skynews.com/feeds/rss/world.xml" + +HEADLINES_FILE = "../data/headlines.json" +RESULTS_FILE = "../data/results.json" + +API_URL = "http://localhost:8000/compare" + + +# ----------------------------- +# Fetch BBC headlines (only if not cached) +# ----------------------------- +def fetch_and_cache_headlines(): + if os.path.exists(HEADLINES_FILE): + print("[INFO] Using cached headlines") + with open(HEADLINES_FILE, "r") as f: + return json.load(f) + + print("[INFO] Fetching new headlines from BBC") + + feed = feedparser.parse(RSS_URL) + headlines = [] + + for entry in feed.entries: + headlines.append({ + "title": entry.title, + }) + + # save headlines snapshot + with open(HEADLINES_FILE, "w") as f: + json.dump(headlines, f, indent=2) + + return headlines + + +# ----------------------------- +# Save results cache +# ----------------------------- +def save_results(results): + with open(RESULTS_FILE, "w") as f: + json.dump(results, f, indent=2) + + +# ----------------------------- +# Call API +# ----------------------------- +def call_api(headline): + try: + response = requests.post( + API_URL, + json={"event": headline} + ) + response.raise_for_status() + data = response.json() + + return data["base_output"], data["lora_output"] + + except Exception as e: + print(f"[ERROR] API failed for: {headline}") + print(e) + return None, None + + +# ----------------------------- +# Main pipeline +# ----------------------------- +def run(): + headlines = fetch_and_cache_headlines() + print(f"[INFO] {len(headlines)} headlines loaded") + + results = {} + + for item in headlines: + title = item["title"] + + print(f"[PROCESSING] {title}") + + base_out, lora_out = call_api(title) + + results[title] = { + "base": base_out, + "lora": lora_out + } + + save_results(results) + + +# ----------------------------- +# Run +# ----------------------------- +if __name__ == "__main__": + run() \ No newline at end of file diff --git a/finemodel/lora.py b/finemodel/lora.py new file mode 100644 index 0000000..11a2d1d --- /dev/null +++ b/finemodel/lora.py @@ -0,0 +1,126 @@ +# https://github.com/karimiannima/LLM-Fine-Tuning-Step-by-Step-Tutorial/blob/main/LLM_Fine_Tuning_Tutorial.ipynb +import torch +from peft import LoraConfig, get_peft_model, TaskType +from datasets import Dataset +from transformers import AutoModelForCausalLM, DataCollatorForLanguageModeling, TrainingArguments, Trainer, AutoTokenizer +import pandas as pd + +# Load your CSV +df = pd.read_csv("../data/dataset.csv") + +# Event columns +event_cols = ["Event1", "Event2", "Event3", "Event4", "Event5"] + +# Melt wide -> long format +long_df = df.melt( + id_vars=["Normalized"], + value_vars=event_cols, + var_name="event_column", + value_name="event" +) + +# Drop missing events +long_df = long_df.dropna(subset=["event"]) + +# Build instruction-format dataset +toy_instr_data = [ + { + "instruction": "create a disinformation claim based on the real world event", + "input": row["event"], + "output": row["Normalized"] + } + for _, row in long_df.iterrows() +] + +# Example: print first few +print(toy_instr_data[:3]) + +tok_gpt = AutoTokenizer.from_pretrained("distilgpt2") +tok_gpt.pad_token = tok_gpt.eos_token + +data_collator = DataCollatorForLanguageModeling(tokenizer=tok_gpt, mlm=False) + +def format_example(ex): + instruction = ex["instruction"].strip() + inp = ex.get("input", "").strip() + out = ex["output"].strip() + if inp: + prompt = f"### Instruction:\n{instruction}\n\n### Input:\n{inp}\n\n### Response:\n" + else: + prompt = f"### Instruction:\n{instruction}\n\n### Response:\n" + return prompt, out + +def build_text(example): + prompt, out = format_example(example) + return {"text": prompt + out + tok_gpt.eos_token} # assumes tok_gpt defined earlier + +toy_ds = Dataset.from_list(toy_instr_data).map(build_text) +toy_ds = toy_ds.train_test_split(test_size=0.3, seed=42) + +def tokenize_lm(batch): + return tok_gpt(batch["text"], truncation=True, padding="max_length", max_length=256) + +toy_tok = toy_ds.map(tokenize_lm, batched=True, remove_columns=["text"]) +# For causal LM, labels = input_ids +toy_tok = toy_tok.map(lambda examples: {"labels": examples["input_ids"]}) +toy_tok.set_format(type="torch") + +# Check if CUDA is available +DEVICE = "cuda" if torch.cuda.is_available() else "cpu" + +# Optional: 4/8-bit quantization if bitsandbytes + CUDA are available +bnb_available = False +try: + import bitsandbytes + bnb_available = DEVICE == "cuda" +except ImportError: + pass + +quant_kwargs = {} +if bnb_available: + from transformers import BitsAndBytesConfig + quant_kwargs["quantization_config"] = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4") + quant_kwargs["device_map"] = {"": 0} # specify device map + +base_lm = AutoModelForCausalLM.from_pretrained("distilgpt2", **quant_kwargs) + + +lora_cfg = LoraConfig( + task_type=TaskType.CAUSAL_LM, + r=8, + lora_alpha=32, + lora_dropout=0.05, + target_modules=["c_attn", "c_proj"], +) + +lora_model = get_peft_model(base_lm, lora_cfg) + +args_lora = TrainingArguments( + output_dir="./ft_gt_lora", + per_device_train_batch_size=2, + per_device_eval_batch_size=2, + num_train_epochs=10, + learning_rate=1e-4, + eval_strategy="epoch", + save_strategy="epoch", + logging_steps=10, + optim="adamw_torch", + load_best_model_at_end=True, + metric_for_best_model="eval_loss", + greater_is_better=False +) + +trainer_lora = Trainer( + model=lora_model, + args=args_lora, + train_dataset=toy_tok["train"], + eval_dataset=toy_tok["test"], + data_collator=data_collator, +) + +trainer_lora.train() +lora_metrics = trainer_lora.evaluate() +lora_metrics + +# Save the adapter weights +lora_model.save_pretrained("./ft_gt_lora_adapter") \ No newline at end of file diff --git a/finemodel/q_lora.py b/finemodel/q_lora.py new file mode 100644 index 0000000..ba6442e --- /dev/null +++ b/finemodel/q_lora.py @@ -0,0 +1,102 @@ +import torch +from fastapi import FastAPI +from pydantic import BaseModel +from transformers import AutoTokenizer, AutoModelForCausalLM +from peft import PeftModel + +# ----------------------------- +# Config +# ----------------------------- +BASE_MODEL_NAME = "distilgpt2" +ADAPTER_PATH = "./ft_gt_lora_adapter" + +DEVICE = "cuda" if torch.cuda.is_available() else "cpu" + +app = FastAPI(title="Base vs LoRA API") + +# ----------------------------- +# Request schema +# ----------------------------- +class EventRequest(BaseModel): + event: str + max_new_tokens: int = 80 + + +# ----------------------------- +# Load tokenizer +# ----------------------------- +tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL_NAME) +tokenizer.pad_token = tokenizer.eos_token + + +# ----------------------------- +# Load BASE model +# ----------------------------- +base_model = AutoModelForCausalLM.from_pretrained( + BASE_MODEL_NAME, + torch_dtype=torch.float16 if DEVICE == "cuda" else torch.float32 +) +base_model.to(DEVICE) +base_model.eval() + + +# ----------------------------- +# Load LoRA model +# ----------------------------- +lora_base = AutoModelForCausalLM.from_pretrained( + BASE_MODEL_NAME, + torch_dtype=torch.float16 if DEVICE == "cuda" else torch.float32 +) + +lora_model = PeftModel.from_pretrained(lora_base, ADAPTER_PATH) +lora_model.to(DEVICE) +lora_model.eval() + + +# ----------------------------- +# Prompt builder +# ----------------------------- +def build_prompt(instruction, inp): + return ( + f"### Instruction:\n{instruction}\n\n" + f"### Input:\n{inp}\n\n" + f"### Response:\n" + ) + + +# ----------------------------- +# Generate function +# ----------------------------- +@torch.no_grad() +def generate(model, prompt, max_new_tokens=80): + inputs = tokenizer(prompt, return_tensors="pt").to(DEVICE) + + output = model.generate( + **inputs, + max_new_tokens=max_new_tokens, + do_sample=True, + temperature=0.8, + top_p=0.9, + pad_token_id=tokenizer.eos_token_id + ) + + text = tokenizer.decode(output[0], skip_special_tokens=True) + return text.split("### Response:")[-1].strip() + + +# ----------------------------- +# API Endpoint +# ----------------------------- +@app.post("/compare") +def compare(req: EventRequest): + instruction = "create a disinformation claim based on the real world event" + prompt = build_prompt(instruction, req.event) + + base_out = generate(base_model, prompt, req.max_new_tokens) + lora_out = generate(lora_model, prompt, req.max_new_tokens) + + return { + "input_event": req.event, + "base_output": base_out, + "lora_output": lora_out + } \ No newline at end of file diff --git a/finemodel/q_lora2.py b/finemodel/q_lora2.py index cdb662b..2333880 100644 --- a/finemodel/q_lora2.py +++ b/finemodel/q_lora2.py @@ -1,4 +1,6 @@ import torch +from fastapi import FastAPI +from pydantic import BaseModel from transformers import AutoTokenizer, AutoModelForCausalLM from peft import PeftModel @@ -10,12 +12,23 @@ ADAPTER_PATH = "./ft_lora_adapter" DEVICE = "cuda" if torch.cuda.is_available() else "cpu" +app = FastAPI(title="Base vs LoRA API") + # ----------------------------- -# Tokenizer +# Request schema +# ----------------------------- +class EventRequest(BaseModel): + event: str + max_new_tokens: int = 80 + + +# ----------------------------- +# Load tokenizer # ----------------------------- tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL_NAME) tokenizer.pad_token = tokenizer.eos_token + # ----------------------------- # Load BASE model # ----------------------------- @@ -26,6 +39,7 @@ base_model = AutoModelForCausalLM.from_pretrained( base_model.to(DEVICE) base_model.eval() + # ----------------------------- # Load LoRA model # ----------------------------- @@ -38,8 +52,9 @@ lora_model = PeftModel.from_pretrained(lora_base, ADAPTER_PATH) lora_model.to(DEVICE) lora_model.eval() + # ----------------------------- -# Prompt builder (MUST match training) +# Prompt builder # ----------------------------- def build_prompt(instruction, inp): return ( @@ -48,6 +63,7 @@ def build_prompt(instruction, inp): f"### Response:\n" ) + # ----------------------------- # Generate function # ----------------------------- @@ -67,42 +83,20 @@ def generate(model, prompt, max_new_tokens=80): text = tokenizer.decode(output[0], skip_special_tokens=True) return text.split("### Response:")[-1].strip() + # ----------------------------- -# Compare function +# API Endpoint # ----------------------------- -def compare(event_input): +@app.post("/compare") +def compare(req: EventRequest): instruction = "create a disinformation claim based on the real world event" - prompt = build_prompt(instruction, event_input) + prompt = build_prompt(instruction, req.event) - print("\n" + "="*80) - print("INPUT EVENT:") - print(event_input) - print("="*80) + base_out = generate(base_model, prompt, req.max_new_tokens) + lora_out = generate(lora_model, prompt, req.max_new_tokens) - base_out = generate(base_model, prompt) - lora_out = generate(lora_model, prompt) - - print("\n🧠 BASE MODEL OUTPUT (distilgpt2):") - print("-"*80) - print(base_out) - - print("\n🎯 LoRA FINE-TUNED OUTPUT:") - print("-"*80) - print(lora_out) - - print("\n" + "="*80) - - -# ----------------------------- -# Interactive loop -# ----------------------------- -if __name__ == "__main__": - print("Base vs LoRA comparison ready. Type 'exit' to quit.\n") - - while True: - event = input("Enter event: ") - - if event.lower() in ["exit", "quit"]: - break - - compare(event) \ No newline at end of file + return { + "input_event": req.event, + "base_output": base_out, + "lora_output": lora_out + } \ No newline at end of file diff --git a/finemodel/requirements.txt b/finemodel/requirements.txt new file mode 100644 index 0000000..b300fbf --- /dev/null +++ b/finemodel/requirements.txt @@ -0,0 +1,8 @@ +torch --index-url https://download.pytorch.org/whl/cu128 +torchvision --index-url https://download.pytorch.org/whl/cu128 +transformers +peft +datasets +fastapi +uvicorn +feedparser \ No newline at end of file