Add a method of evaulation. Add back distilGPT version. Convert querying to another fastAPI

finemodel/.gitignore

@@ -1 +1,2 @@
-ft_*/
+ft_*/
+*.pyc

finemodel/eval.py

@@ -0,0 +1,101 @@
+import requests
+import json
+import csv
+import os
+from datetime import datetime, timedelta
+import feedparser
+
+# -----------------------------
+# Config
+# -----------------------------
+RSS_URL = "https://feeds.skynews.com/feeds/rss/world.xml"
+
+HEADLINES_FILE = "../data/headlines.json"
+RESULTS_FILE = "../data/results.json"
+
+API_URL = "http://localhost:8000/compare"
+
+
+# -----------------------------
+# Fetch BBC headlines (only if not cached)
+# -----------------------------
+def fetch_and_cache_headlines():
+    if os.path.exists(HEADLINES_FILE):
+        print("[INFO] Using cached headlines")
+        with open(HEADLINES_FILE, "r") as f:
+            return json.load(f)
+
+    print("[INFO] Fetching new headlines from BBC")
+
+    feed = feedparser.parse(RSS_URL)
+    headlines = []
+
+    for entry in feed.entries:
+        headlines.append({
+            "title": entry.title,
+        })
+
+    # save headlines snapshot
+    with open(HEADLINES_FILE, "w") as f:
+        json.dump(headlines, f, indent=2)
+
+    return headlines
+
+
+# -----------------------------
+# Save results cache
+# -----------------------------
+def save_results(results):
+    with open(RESULTS_FILE, "w") as f:
+        json.dump(results, f, indent=2)
+
+
+# -----------------------------
+# Call API
+# -----------------------------
+def call_api(headline):
+    try:
+        response = requests.post(
+            API_URL,
+            json={"event": headline}
+        )
+        response.raise_for_status()
+        data = response.json()
+
+        return data["base_output"], data["lora_output"]
+
+    except Exception as e:
+        print(f"[ERROR] API failed for: {headline}")
+        print(e)
+        return None, None
+
+
+# -----------------------------
+# Main pipeline
+# -----------------------------
+def run():
+    headlines = fetch_and_cache_headlines()
+    print(f"[INFO] {len(headlines)} headlines loaded")
+
+    results = {}
+
+    for item in headlines:
+        title = item["title"]
+
+        print(f"[PROCESSING] {title}")
+
+        base_out, lora_out = call_api(title)
+
+        results[title] = {
+            "base": base_out,
+            "lora": lora_out
+        }
+
+    save_results(results)
+
+
+# -----------------------------
+# Run
+# -----------------------------
+if __name__ == "__main__":
+    run()

finemodel/lora.py

@@ -0,0 +1,126 @@
+# https://github.com/karimiannima/LLM-Fine-Tuning-Step-by-Step-Tutorial/blob/main/LLM_Fine_Tuning_Tutorial.ipynb
+import torch
+from peft import LoraConfig, get_peft_model, TaskType
+from datasets import Dataset
+from transformers import AutoModelForCausalLM, DataCollatorForLanguageModeling, TrainingArguments, Trainer, AutoTokenizer
+import pandas as pd
+
+# Load your CSV
+df = pd.read_csv("../data/dataset.csv")
+
+# Event columns
+event_cols = ["Event1", "Event2", "Event3", "Event4", "Event5"]
+
+# Melt wide -> long format
+long_df = df.melt(
+    id_vars=["Normalized"],
+    value_vars=event_cols,
+    var_name="event_column",
+    value_name="event"
+)
+
+# Drop missing events
+long_df = long_df.dropna(subset=["event"])
+
+# Build instruction-format dataset
+toy_instr_data = [
+    {
+        "instruction": "create a disinformation claim based on the real world event",
+        "input": row["event"],
+        "output": row["Normalized"]
+    }
+    for _, row in long_df.iterrows()
+]
+
+# Example: print first few
+print(toy_instr_data[:3])
+
+tok_gpt  = AutoTokenizer.from_pretrained("distilgpt2")
+tok_gpt.pad_token = tok_gpt.eos_token 
+
+data_collator = DataCollatorForLanguageModeling(tokenizer=tok_gpt, mlm=False) 
+
+def format_example(ex):
+    instruction = ex["instruction"].strip()
+    inp = ex.get("input", "").strip()
+    out = ex["output"].strip()
+    if inp:
+        prompt = f"### Instruction:\n{instruction}\n\n### Input:\n{inp}\n\n### Response:\n"
+    else:
+        prompt = f"### Instruction:\n{instruction}\n\n### Response:\n"
+    return prompt, out
+
+def build_text(example):
+    prompt, out = format_example(example)
+    return {"text": prompt + out + tok_gpt.eos_token}   # assumes tok_gpt defined earlier
+
+toy_ds = Dataset.from_list(toy_instr_data).map(build_text)
+toy_ds = toy_ds.train_test_split(test_size=0.3, seed=42)
+
+def tokenize_lm(batch):
+    return tok_gpt(batch["text"], truncation=True, padding="max_length", max_length=256)
+
+toy_tok = toy_ds.map(tokenize_lm, batched=True, remove_columns=["text"])
+# For causal LM, labels = input_ids
+toy_tok = toy_tok.map(lambda examples: {"labels": examples["input_ids"]})
+toy_tok.set_format(type="torch")
+
+# Check if CUDA is available
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+# Optional: 4/8-bit quantization if bitsandbytes + CUDA are available
+bnb_available = False
+try:
+    import bitsandbytes
+    bnb_available = DEVICE == "cuda"
+except ImportError:
+    pass
+
+quant_kwargs = {}
+if bnb_available:
+    from transformers import BitsAndBytesConfig
+    quant_kwargs["quantization_config"] = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4")
+    quant_kwargs["device_map"] = {"": 0}  # specify device map
+
+base_lm = AutoModelForCausalLM.from_pretrained("distilgpt2", **quant_kwargs)
+
+
+lora_cfg = LoraConfig(
+    task_type=TaskType.CAUSAL_LM,
+    r=8,
+    lora_alpha=32,
+    lora_dropout=0.05,
+    target_modules=["c_attn", "c_proj"],
+)
+
+lora_model = get_peft_model(base_lm, lora_cfg)
+
+args_lora = TrainingArguments(
+    output_dir="./ft_gt_lora",
+    per_device_train_batch_size=2,
+    per_device_eval_batch_size=2,
+    num_train_epochs=10,
+    learning_rate=1e-4,
+    eval_strategy="epoch",
+    save_strategy="epoch",
+    logging_steps=10,
+    optim="adamw_torch",
+    load_best_model_at_end=True,
+    metric_for_best_model="eval_loss",
+    greater_is_better=False
+)
+
+trainer_lora = Trainer(
+    model=lora_model,
+    args=args_lora,
+    train_dataset=toy_tok["train"],
+    eval_dataset=toy_tok["test"],
+    data_collator=data_collator,
+)
+
+trainer_lora.train()
+lora_metrics = trainer_lora.evaluate()
+lora_metrics
+
+# Save the adapter weights
+lora_model.save_pretrained("./ft_gt_lora_adapter")

finemodel/q_lora.py

@@ -0,0 +1,102 @@
+import torch
+from fastapi import FastAPI
+from pydantic import BaseModel
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from peft import PeftModel
+
+# -----------------------------
+# Config
+# -----------------------------
+BASE_MODEL_NAME = "distilgpt2"
+ADAPTER_PATH = "./ft_gt_lora_adapter"
+
+DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+app = FastAPI(title="Base vs LoRA API")
+
+# -----------------------------
+# Request schema
+# -----------------------------
+class EventRequest(BaseModel):
+    event: str
+    max_new_tokens: int = 80
+
+
+# -----------------------------
+# Load tokenizer
+# -----------------------------
+tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL_NAME)
+tokenizer.pad_token = tokenizer.eos_token
+
+
+# -----------------------------
+# Load BASE model
+# -----------------------------
+base_model = AutoModelForCausalLM.from_pretrained(
+    BASE_MODEL_NAME,
+    torch_dtype=torch.float16 if DEVICE == "cuda" else torch.float32
+)
+base_model.to(DEVICE)
+base_model.eval()
+
+
+# -----------------------------
+# Load LoRA model
+# -----------------------------
+lora_base = AutoModelForCausalLM.from_pretrained(
+    BASE_MODEL_NAME,
+    torch_dtype=torch.float16 if DEVICE == "cuda" else torch.float32
+)
+
+lora_model = PeftModel.from_pretrained(lora_base, ADAPTER_PATH)
+lora_model.to(DEVICE)
+lora_model.eval()
+
+
+# -----------------------------
+# Prompt builder
+# -----------------------------
+def build_prompt(instruction, inp):
+    return (
+        f"### Instruction:\n{instruction}\n\n"
+        f"### Input:\n{inp}\n\n"
+        f"### Response:\n"
+    )
+
+
+# -----------------------------
+# Generate function
+# -----------------------------
+@torch.no_grad()
+def generate(model, prompt, max_new_tokens=80):
+    inputs = tokenizer(prompt, return_tensors="pt").to(DEVICE)
+
+    output = model.generate(
+        **inputs,
+        max_new_tokens=max_new_tokens,
+        do_sample=True,
+        temperature=0.8,
+        top_p=0.9,
+        pad_token_id=tokenizer.eos_token_id
+    )
+
+    text = tokenizer.decode(output[0], skip_special_tokens=True)
+    return text.split("### Response:")[-1].strip()
+
+
+# -----------------------------
+# API Endpoint
+# -----------------------------
+@app.post("/compare")
+def compare(req: EventRequest):
+    instruction = "create a disinformation claim based on the real world event"
+    prompt = build_prompt(instruction, req.event)
+
+    base_out = generate(base_model, prompt, req.max_new_tokens)
+    lora_out = generate(lora_model, prompt, req.max_new_tokens)
+
+    return {
+        "input_event": req.event,
+        "base_output": base_out,
+        "lora_output": lora_out
+    }

finemodel/q_lora2.py

@@ -1,4 +1,6 @@
 import torch
+from fastapi import FastAPI
+from pydantic import BaseModel
 from transformers import AutoTokenizer, AutoModelForCausalLM
 from peft import PeftModel
 
@@ -10,12 +12,23 @@ ADAPTER_PATH = "./ft_lora_adapter"
 
 DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
 
+app = FastAPI(title="Base vs LoRA API")
+
 # -----------------------------
-# Tokenizer
+# Request schema
+# -----------------------------
+class EventRequest(BaseModel):
+    event: str
+    max_new_tokens: int = 80
+
+
+# -----------------------------
+# Load tokenizer
 # -----------------------------
 tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL_NAME)
 tokenizer.pad_token = tokenizer.eos_token
 
+
 # -----------------------------
 # Load BASE model
 # -----------------------------
@@ -26,6 +39,7 @@ base_model = AutoModelForCausalLM.from_pretrained(
 base_model.to(DEVICE)
 base_model.eval()
 
+
 # -----------------------------
 # Load LoRA model
 # -----------------------------
@@ -38,8 +52,9 @@ lora_model = PeftModel.from_pretrained(lora_base, ADAPTER_PATH)
 lora_model.to(DEVICE)
 lora_model.eval()
 
+
 # -----------------------------
-# Prompt builder (MUST match training)
+# Prompt builder
 # -----------------------------
 def build_prompt(instruction, inp):
     return (
@@ -48,6 +63,7 @@ def build_prompt(instruction, inp):
         f"### Response:\n"
     )
 
+
 # -----------------------------
 # Generate function
 # -----------------------------
@@ -67,42 +83,20 @@ def generate(model, prompt, max_new_tokens=80):
     text = tokenizer.decode(output[0], skip_special_tokens=True)
     return text.split("### Response:")[-1].strip()
 
+
 # -----------------------------
-# Compare function
+# API Endpoint
 # -----------------------------
-def compare(event_input):
+@app.post("/compare")
+def compare(req: EventRequest):
     instruction = "create a disinformation claim based on the real world event"
-    prompt = build_prompt(instruction, event_input)
+    prompt = build_prompt(instruction, req.event)
 
-    print("\n" + "="*80)
+    base_out = generate(base_model, prompt, req.max_new_tokens)
-    print("INPUT EVENT:")
+    lora_out = generate(lora_model, prompt, req.max_new_tokens)
-    print(event_input)
-    print("="*80)
 
-    base_out = generate(base_model, prompt)
+    return {
-    lora_out = generate(lora_model, prompt)
+        "input_event": req.event,
-
+        "base_output": base_out,
-    print("\n🧠 BASE MODEL OUTPUT (distilgpt2):")
+        "lora_output": lora_out
-    print("-"*80)
+    }
-    print(base_out)
-
-    print("\n🎯 LoRA FINE-TUNED OUTPUT:")
-    print("-"*80)
-    print(lora_out)
-
-    print("\n" + "="*80)
-
-
-# -----------------------------
-# Interactive loop
-# -----------------------------
-if __name__ == "__main__":
-    print("Base vs LoRA comparison ready. Type 'exit' to quit.\n")
-
-    while True:
-        event = input("Enter event: ")
-
-        if event.lower() in ["exit", "quit"]:
-            break
-
-        compare(event)

finemodel/requirements.txt

@@ -0,0 +1,8 @@
+torch --index-url https://download.pytorch.org/whl/cu128
+torchvision --index-url https://download.pytorch.org/whl/cu128
+transformers 
+peft 
+datasets
+fastapi
+uvicorn
+feedparser