From 2417efbeca80666b29761854371705a29705cf2f Mon Sep 17 00:00:00 2001
From: WillJeynes <william.jeynes@hotmail.com>
Date: Fri, 10 Apr 2026 17:34:36 +0100
Subject: [PATCH] Use mini llama, getting more interesting results

---
 finemodel/lora2.py   | 15 ++++++++++-----
 finemodel/q_lora2.py |  2 +-
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/finemodel/lora2.py b/finemodel/lora2.py
index 2a1914f..83fa97d 100644
--- a/finemodel/lora2.py
+++ b/finemodel/lora2.py
@@ -35,7 +35,7 @@ toy_instr_data = [
 # Example: print first few
 print(toy_instr_data[:3])
 
-tok_gpt  = AutoTokenizer.from_pretrained("distilgpt2")
+tok_gpt  = AutoTokenizer.from_pretrained("TinyLlama/TinyLlama-1.1B-Chat-v1.0")
 tok_gpt.pad_token = tok_gpt.eos_token 
 
 data_collator = DataCollatorForLanguageModeling(tokenizer=tok_gpt, mlm=False) 
@@ -82,15 +82,20 @@ if bnb_available:
     quant_kwargs["quantization_config"] = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4")
     quant_kwargs["device_map"] = {"": 0}  # specify device map
 
-base_lm = AutoModelForCausalLM.from_pretrained("distilgpt2", **quant_kwargs)
+base_lm = AutoModelForCausalLM.from_pretrained("TinyLlama/TinyLlama-1.1B-Chat-v1.0", **quant_kwargs)
+
 
 lora_cfg = LoraConfig(
     task_type=TaskType.CAUSAL_LM,
-    r=8,             # rank
+    r=8,
     lora_alpha=32,
     lora_dropout=0.05,
-    target_modules=["c_attn", "c_proj", "c_fc"],
-    fan_in_fan_out=True,
+    target_modules=[
+        "q_proj",
+        "k_proj",
+        "v_proj",
+        "o_proj"
+    ]
 )
 
 lora_model = get_peft_model(base_lm, lora_cfg)
diff --git a/finemodel/q_lora2.py b/finemodel/q_lora2.py
index d16df57..cdb662b 100644
--- a/finemodel/q_lora2.py
+++ b/finemodel/q_lora2.py
@@ -5,7 +5,7 @@ from peft import PeftModel
 # -----------------------------
 # Config
 # -----------------------------
-BASE_MODEL_NAME = "distilgpt2"
+BASE_MODEL_NAME = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
 ADAPTER_PATH = "./ft_lora_adapter"
 
 DEVICE = "cuda" if torch.cuda.is_available() else "cpu"