remove context examples

.gitignore

@@ -1,2 +1,3 @@
 # TEMP
+literature/
 backup.tar.gz

README.md

@@ -1,55 +1,20 @@
 # AI models for identifying trigger events in disinformation analysis
+Final Dissertation Submission Repository
 
-## Abstract
-Disinformation on the internet has become a significant and growing challenge, driven by the sheer volume and speed of content generation across digital platforms. 
-
-While prior work has thoroughly investigated tasks such as automatic disinformation detection, especially using large language models, comparatively little attention has been paid to applying these techniques to generative analyses of claims.  In particular, the retrieval of structured contextual background information that explains why disinformation spreads remains underexplored.
- 
-This paper introduces a dataset of trigger events, defined as actions or statements that influence the creation, spread or believability of disinformation claims. 
-The dataset is created using large language models as few-shot retrievers and subsequently validated through an ensemble classifier to ensure consistency.
-
-We find that we can successfully retrieve a large dataset of trigger events with high accuracy, and that the dataset proves useful in applications that can be useful in the real-world scenarios of debunking and prebunking. 
-
-
-[Project Writeup](/literature/WillJeynes-Dissertation.pdf)
-
-[Project Presentation](https://jillweynes.github.io/LLMsForDisinformationPrediction-GraphVizBuilt/presentation)
-
-## Repository Information
-
-This repository contains the code for main dataset generation, as a [langchain](https://www.langchain.com/) project. 
-
-![diagram](literature/design.png)
-
-In **green** is claim normalisation based on [this paper](https://arxiv.org/abs/2508.17402)
-
-In **red** is the main AI processing, based on a simple tool invocation setup
-
-![tool invocation](literature/prompting.png)
-
-Finally, in **yellow** is an ensemble model to quantify the usefullness of the AI generated responses, compared against a human-labelled dataset
-
-## Generated Dataset Link
-
-A dataset of this pipeline ran against 2000 input claims from [StopFake](https://www.stopfake.org/en/main/) and [ScienceFeedback](https://science.feedback.org/) is available on [HuggingFace](https://huggingface.co/datasets/WillJeynes/LLMsForDisinformationAnalysis-Dataset)
-
-
-## More information
-A Graph-Based Dataset Visualisation tool is available [online](https://jillweynes.github.io/LLMsForDisinformationPrediction-GraphVizBuilt/), to show the potential usability of the content.
-
-This and other usage experiments can be seen on the [sister repository](https://git.host.jeynes.uk/jill/LLMsForDisinformationPrediction)
-
-
-
-# This repository:
+## Project Description
+-- todo --
 
+## Solution Diagram
+-- todo --
 
 ## Classifier Refinement
-To see more detail about the ensembkle classifier, see [this folder](/supporting/RAGAS_Service/)
+[See RAGAS_Service](/supporting/RAGAS_Service/)
 
 ## Agent Refinement
-To see more detail about experiments to improve the output quality, see [this folder](/agent/)
+[See agent](/agent/)
 
+## Generated Database Link and Usage Experiments
+-- todo --
 
 ## Repository Structure
 ```
@@ -64,6 +29,8 @@ To see more detail about experiments to improve the output quality, see [this fo
 |   ├── input.jsonl                 # Response in cleaned format to give as context to agent
 |   ├── ranked.jsonl                # Cleaned trigger event response from scorer frontend
 |   └── results.jsonl               # Output from wrapper script, read and modified by scorer
+├── literature/
+|   └── report.pdf                  # Final submission report
 ├── agent/                          # Code for main project pipeline
 |   ├── agent.ts                    # Graph definition file
 |   ├── conditionals/               # Conditional translations

agent/README.md

@@ -1,32 +1,3 @@
 ## Refining the agent output
 
-Experiments modifying pipeline
-
-| Model            | % Correct | % Change |
-|------------------|----------:|---------:|
-| BASELINE         | 33        | 0        |
-| Improv Prompt    | 39.96     | 0.21     |
-| Add Examples     | 44.67     | 0.35     |
-| Date             | 45.51     | 0.38     |
-| Chain of Thought | 43.38     | 0.31     |
-| Self-Critique    | 44.36     | 0.34     |
-
-Experiments with different model types:
-| Model                         | % Correct | % Change |
-|-------------------------------|----------:|---------:|
-| gpt-5-mini                    | 45.51     |          |
-| gpt-5.4-mini                  | 32.4      |          |
-| gpt-5.4-nano                  | 23.28     |          |
-| gpt-4.1-mini                  | 27.85     |          |
-| gpt-4o-mini                   | 32.47     |          |
-| llama3.1:8b-instruct-q4_K_M   | ?         |          |
-| qwen3.5:9b                    | 0         |          |
-
-%age valid URLS
-| Model                         | Number    | % Age    |
-|-------------------------------|----------:|---------:|
-| gpt-5-mini                    | 22/405    | 5.43     |
-| gpt-5.4-mini                  | 29/278    | 10.43    |
-| gpt-5.4-nano                  | 6/210     | 2.85     |
-| gpt-4.1-mini                  | 15/269    | 5.57     |
-| gpt-4o-mini                   | 27/287    | 9.407    |
+TODO: Table and document experiments

agent/nodes/triggerEventSetup.ts

@@ -7,11 +7,11 @@ export const triggerEventSetup: GraphNode<typeof MessagesState> = async (state)
   let nc = state?.messages?.at(-1)?.content ?? "" //keep a copy of normalized trigger event. Again two things, womp womp
   
   //Now give in-context examples. hopwfully we can self-teach?
-  let similarityResults = await rankExampleTriggerEvents(state.disinformationTitle)
+  // let similarityResults = await rankExampleTriggerEvents(state.disinformationTitle)
 
-  let messages : BaseMessage[] = similarityResults.map((item) => {
-    return new AIMessage(`- Event: ${item.rawtext} \n\n - Claims and given scores: ${item.cleantext}`)
-  })
+  // let messages : BaseMessage[] = similarityResults.map((item) => {
+  //   return new AIMessage(`- Event: ${item.rawtext} \n\n - Claims and given scores: ${item.cleantext}`)
+  // })
   
-  return { messages: messages, disinformationTitle: state.disinformationTitle, normalizedClaim: nc };
+  return { disinformationTitle: state.disinformationTitle, normalizedClaim: nc };
 };

agent/nodes/verificationSetup.ts

@@ -1,7 +1,8 @@
 import { GraphNode } from "@langchain/langgraph";
 import { MessagesState, ProposedTriggerEventArray } from "../state";
 import { logger } from "../utils/logger";
-import { jsonrepair } from 'jsonrepair'
+import { queryScraper } from "../tools/webSearch";
+import { rankAndDisplayData } from "../tools/triggerEventTools";
 
 export const verificationSetup: GraphNode<typeof MessagesState> = async (state) => {
   //this is kinda doing two things, but having two nodes for it seems overkill
@@ -10,29 +11,15 @@ export const verificationSetup: GraphNode<typeof MessagesState> = async (state)
     logger.warn("No trigger events in memory, parsing")
 
     let genResponse = state.messages.at(-1)?.content.toString() ?? "";
+    const parsed = ProposedTriggerEventArray.parse(JSON.parse(genResponse));
 
-    const repaired = jsonrepair(genResponse);
+    for (let i = 0; i < parsed.length; i++) {
+      const search = parsed[i].SearchQuery
+      // const data = await queryScraper(search);
+      // const output = await rankAndDisplayData(data, search);
 
-    let parsed;
-
-    try {
-      const json = JSON.parse(repaired);
-
-      if (Array.isArray(json)) {
-        parsed = ProposedTriggerEventArray.parse(json);
-      } else {
-        // try grab first value
-        const firstValue = Object.values(json)[0];
-
-        if (Array.isArray(firstValue)) {
-          parsed = ProposedTriggerEventArray.parse(firstValue);
-        } else {
-          throw new Error("No array found in JSON");
-        }
-      }
-    } catch (err: any) {
-      logger.error(`Failed to parse LLM response: ${err.message}`);
-      throw new Error(`Failed to parse LLM response: ${err}`);
+      // parsed[i].context = output;
+      parsed[i].context = "NONE"
     }
     
     return { proposedTriggerEvent: parsed, proposedTriggerEventIndex: 0 };

agent/package-lock.json

@@ -20,7 +20,6 @@
         "dotenv": "^17.2.3",
         "exponential-backoff": "^3.1.3",
         "fs": "^0.0.1-security",
-        "jsonrepair": "^3.13.3",
         "langchain": "^1.2.14",
         "selenium-webdriver": "^4.40.0",
         "tldts": "^7.0.23",
@@ -2076,15 +2075,6 @@
       "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==",
       "license": "ISC"
     },
-    "node_modules/jsonrepair": {
-      "version": "3.13.3",
-      "resolved": "https://registry.npmjs.org/jsonrepair/-/jsonrepair-3.13.3.tgz",
-      "integrity": "sha512-BTznj0owIt2CBAH/LTo7+1I5pMvl1e1033LRl/HUowlZmJOIhzC0zbX5bxMngLkfT4WnzPP26QnW5wMr2g9tsQ==",
-      "license": "ISC",
-      "bin": {
-        "jsonrepair": "bin/cli.js"
-      }
-    },
     "node_modules/jszip": {
       "version": "3.10.1",
       "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz",

agent/package.json

@@ -24,7 +24,6 @@
     "dotenv": "^17.2.3",
     "exponential-backoff": "^3.1.3",
     "fs": "^0.0.1-security",
-    "jsonrepair": "^3.13.3",
     "langchain": "^1.2.14",
     "selenium-webdriver": "^4.40.0",
     "tldts": "^7.0.23",

agent/prompts/relation.txt

@@ -0,0 +1,9 @@
+Could the following real-world event:
+###TECLAIM###
+
+Be a trigger for the following disinformation:
+###TITLE###
+
+Respond with "RELATION", followed by : followed by a confidence score (VERYHIGH, HIGH, MEDIUM, LOW, VERYLOW) followed by : followed by the reason. Use no other words, just return the score and reason in format.
+
+Ignore wether the event happened or not, purely consider the likiness of causation

agent/prompts/trigger.txt

@@ -14,9 +14,7 @@ Include a concise but specific search query that can be looked up on a search en
 
 Include a url to a source for your trigger event (not a web search, a specific url from a reputuable source). Do not use OAI cite, include url as text in response.
 
-Include the date that the event happened ("March 2022" for exmaple)
-
-Use a JSON format with each entry containing "Event,ReasoningWhyRelevant,SearchQuery,Url,Date".
+Use a JSON format with each entry containing "Event,ReasoningWhyRelevant,SearchQuery,Url".
 
 Multiple tool invocations should be requested at once, if applicable.
 Use your abilities to look between the lines and produce some insightful analysis, thinking both short and long term.

agent/prompts/verify.txt

@@ -0,0 +1,8 @@
+Do the search results cited below
+###TESEARCH###
+Support the idea that the following happened:
+###TECLAIM###
+
+Respond with "CONFIDENCE", followed by : followed by a confidence score (VERYHIGH, HIGH, MEDIUM, LOW, VERYLOW) followed by : followed by the reason. Use no other words, just return the score and reason in format.
+
+Dates can be off by a few days, that would still be valid

agent/state.ts

@@ -9,7 +9,6 @@ export const ProposedTriggerEvent = z.object({
   ReasoningWhyRelevant: z.string(),
   SearchQuery: z.string(),
   Url: z.url(),
-  Date: z.string(),
   context: z.string().optional(),
   score: z.number().optional()
 })

agent/tools/retreiveExamples.ts

@@ -15,8 +15,6 @@ const CACHE_PATH = "../data/csv.cache.json";
 
 const JSONL_PATH = "../data/input.jsonl"
 
-const BM25_MIN_DOCS = 3;
-
 type EmbeddingCache = {
   rawtexts: string[];
   cleantexts: string[];
@@ -289,20 +287,8 @@ async function embedText(text: string): Promise<number[]> {
 }
 
 function buildBM25(texts: string[]) {
-  let paddedTexts = texts;
+  logger.info("Building BM25 index (%s docs)...", texts.length);
 
-  if (texts.length < BM25_MIN_DOCS) {
-    const needed = BM25_MIN_DOCS - texts.length;
-    logger.error(
-      "Corpus too small for BM25 (%s docs, need %s+), padding with %s dummy doc(s)",
-      texts.length,
-      BM25_MIN_DOCS,
-      needed
-    );
-    paddedTexts = [...texts, ...Array(needed).fill("placeholder dummy document")];
-  }
-
-  logger.info("Building BM25 index (%s docs)...", paddedTexts.length);
   const bm25 = bm25Factory();
 
   bm25.defineConfig({
@@ -316,7 +302,7 @@ function buildBM25(texts: string[]) {
     nlp.tokens.removeWords,
   ]);
 
-  paddedTexts.forEach((text, i) => {
+  texts.forEach((text, i) => {
     bm25.addDoc({ text }, i);
   });
 

agent/tools/webpageFetch.ts

@@ -1,92 +1,32 @@
 import { Builder, Browser } from "selenium-webdriver";
 import firefox from "selenium-webdriver/firefox";
-import { backOff } from "exponential-backoff";
-import { logger } from "../utils/logger";
 
-export async function extractWebpageContent(url: string): Promise<string[]> {
-  try {
-    const response = await backOff(async () => {
-      return await extractWebpageContentWorker(url);
-    }, {
-      numOfAttempts: 10,
-      startingDelay: 500,
-      timeMultiple: 2,
-      jitter: "full",
-      maxDelay: 50000,
-    });
-    return response;
-  } catch (err: any) {
-    logger.error(`Failed out of retry loop for URL "${url}", returning placeholder to pipeline`);
-    return ["API EXCEPTION"];
-  }
-}
-
-async function extractWebpageContentWorker(url: string): Promise<string[]> {
-  let driver;
-  try {
+export async function extractWebpageContent(url: string) : Promise<string[]>{
     const options = new firefox.Options();
     options.addArguments("--headless");
-    driver = await new Builder()
-      .forBrowser(Browser.FIREFOX)
-      .setFirefoxOptions(options)
-      .build();
-  } catch (err: any) {
-    const desc = `Failed to launch Firefox driver: ${err.message}`;
-    logger.error(desc);
-    throw new Error(desc);
-  }
 
-  try {
+    let driver = await new Builder().forBrowser(Browser.FIREFOX).setFirefoxOptions(options).build()
     try {
-      await driver.get(url);
-    } catch (err: any) {
-      const desc = `Failed to navigate to URL "${url}": ${err.message}`;
-      logger.error(desc);
-      throw new Error(desc);
-    }
+        await driver.get(url)
+        await driver.wait(async () => {
+            return await driver.executeScript(
+                "return document.readyState === 'complete'"
+            );
+        }, 5000);
 
-    try {
-      await driver.wait(async () => {
-        return await driver.executeScript(
-          "return document.readyState === 'complete'"
-        );
-      }, 5000);
-    } catch (err: any) {
-      logger.error(`Page load timed out for "${url}", attempting to read partial content: ${err.message}`);
-      // do not throw, attempt to read
-    }
+        const readableText = await driver.executeScript(
+            "return document.body.innerText;"
+        ) as string;
 
-    let readableText: string;
-    try {
-      readableText = await driver.executeScript(
-        "return document.body.innerText;"
-      ) as string;
-    } catch (err: any) {
-      const desc = `Failed to extract page text from "${url}": ${err.message}`;
-      logger.error(desc);
-      throw new Error(desc);
+        const filteredLines = readableText
+            .split(/\r?\n/)
+            .map(line => line.trim())
+            .filter(line => line.split(/\s+/).length > 1); 
+        
+        return filteredLines;
+    } finally {
+        await driver.quit()
     }
-
-    const filteredLines = readableText
-      .split(/\r?\n/)
-      .map(line => line.trim())
-      .filter(line => line.split(/\s+/).length > 1);
-
-    if (filteredLines.length === 0) {
-      const desc = `No content extracted from "${url}"`;
-      logger.error(desc);
-      throw new Error(desc);
-    }
-
-    return filteredLines;
-  } finally {
-    try {
-      await driver.quit();
-    } catch (err: any) {
-      logger.error(`Failed to quit Firefox driver cleanly: ${err.message}`);
-    }
-  }
 }
 
-// console.log(await extractWebpageContent("https://www.bbc.co.uk/news/live/c74wd01egvyt"))
-// console.log(await extractWebpageContent("https://badcertificate.int.jeynes.uk/"))
+//console.log(await extractWebpageContent("https://www.bbc.co.uk/news/live/c74wd01egvyt"))

run.sh

@@ -5,7 +5,7 @@ set -e
 run_agent () {
     echo "Starting LangGraph agent..."
     cd agent
-    npx @langchain/langgraph-cli@1.1.17 dev
+    npx @langchain/langgraph-cli dev
 }
 
 run_ensemble_service () {

supporting/RAGAS_Service/README.md

@@ -2,32 +2,14 @@
 
 Made using a dataset of 1000 labeled claims from MVP pipeline.
 
-# Roberta model 
+Roberta model trained on an augmented dataset with LLM generated adversarial examples for low frequency labels.
 
-Trained on an augmented dataset with LLM generated adversarial examples for low frequency labels.
+Flan model trained using raw labelled claims, inherrent natural language ability allows for pattern recognition without the need for fake data.
 
-![ROBERTA Diagram](/literature/classifierROBERTA.png)
+Regression model trained using the roberta dataset.
 
-# Flan model 
-
-Trained using raw labelled claims, inherrent natural language ability allows for pattern recognition without the need for fake data.
-
-![FLAN Diagram](/literature/classifierFLAN.png)
-
-# NN Model
-
-Regression model trained like roberta.
-
-![NN Diagram](/literature/classifierNN.png)
-
-
-# Ensemble 
 Used ensemble model in the final version, with the component models available on Hugging Face. 
 
-
-![Ensemble Diagram](/literature/classifierOverall.png)
-
-
 | Model                                                      | % Correct | % Valid taken forward|Used in ensemble|Link
 |------------------------------------------------------------|-----------|----------------------|----------------|-
 | Original                                                   | 53.22     | 61.72                |

supporting/RAGAS_Service/requirements.txt

@@ -9,7 +9,6 @@ datasets
 # ROBERTA
 scikit-learn
 transformers[torch]
-sentence_transformers
 
 # Utils
 numpy

supporting/Wrapper/run.ts

@@ -19,9 +19,6 @@ const MODE = process.env.MODE ?? "claim";
 
 const MAX_CONCURRENCY = 5;
 
-const OFFSET = parseInt(process.env.OFFSET ?? "0", 10);
-const LIMIT = process.env.LIMIT ? parseInt(process.env.LIMIT, 10) : null;
-
 const client = new Client({ apiUrl: API_URL });
 
 
@@ -121,7 +118,7 @@ async function processRecord(record: any): Promise<ResultRecord> {
       input: buildAgentInput(record),
       streamMode: "values",
       config: {
-        recursion_limit: 100
+        recursion_limit: 50
       }
     });
 
@@ -167,19 +164,10 @@ async function processRecord(record: any): Promise<ResultRecord> {
 async function main() {
   console.log("Reading input file...");
 
-  const allRecords = await loadInputs();
+  const records = await loadInputs();
 
-  console.log(`Loaded ${allRecords.length} records`);
+  console.log(`Loaded ${records.length} records`);
 
-  const records = allRecords.slice(
-    OFFSET,
-    LIMIT !== null ? OFFSET + LIMIT : undefined
-  );
-
-  console.log(
-    `Processing ${records.length} records (offset=${OFFSET}, limit=${LIMIT ?? "∞"})`
-  );
-  
   fs.writeFileSync(OUTPUT_FILE, "", { flag: "a" });
 
   const limit = pLimit(MAX_CONCURRENCY);

supporting/checker/checker.py

@@ -1,119 +0,0 @@
-import json
-import argparse
-from urllib.parse import urlparse
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-from selenium.common.exceptions import WebDriverException, TimeoutException, StaleElementReferenceException
-from tqdm import tqdm
-
-def init_driver():
-    options = Options()
-    options.headless = True
-    options.add_argument("--disable-gpu")
-    options.add_argument("--no-sandbox")
-    options.add_argument("--headless")
-    options.add_argument("--disable-blink-features=AutomationControlled")
-    options.add_argument("--window-size=1920,1080")
-    prefs = {
-        "profile.managed_default_content_settings.images": 2,  # block images
-        "profile.default_content_setting_values.stylesheets": 2,  # block CSS
-        "profile.managed_default_content_settings.cookies": 2,  # optional
-    }
-    options.add_experimental_option("prefs", prefs)
-
-    driver = webdriver.Chrome(options=options)
-    driver.set_page_load_timeout(30)
-    return driver
-
-def is_root_url(url):
-    parsed = urlparse(url)
-    return parsed.path in ("", "/")
-
-def is_404_page(driver):
-    """Safely check for 404, handling stale elements."""
-    try:
-        title = driver.title.lower()
-        body_text = driver.find_element("tag name", "body").text.lower()
-        return "404" in title or "404" in body_text
-    except StaleElementReferenceException:
-        return False
-    except Exception:
-        return False
-
-def check_url_selenium(url):
-    driver = None
-    try:
-        driver = init_driver()
-        driver.get(url)
-        # 404 check
-        if is_404_page(driver):
-            return False, "404 page detected"
-        # Root URL after redirects
-        final_url = driver.current_url
-        if is_root_url(final_url):
-            return False, f"Redirected to root URL ({final_url})"
-        return True, None
-    except (WebDriverException, TimeoutException) as e:
-        return False, str(e)
-    finally:
-        if driver:
-            driver.quit()
-
-def process_event(event):
-    """Process an event only if score > 0.4."""
-    score = event.get("score", 0)
-    if score <= 0.4:
-        return None, False, "Score too low"
-    url = event.get("Url")
-    if not url:
-        return None, False, "No URL"
-    is_valid, error_msg = check_url_selenium(url)
-    event["url_valid"] = is_valid
-    return url, is_valid, error_msg
-
-def process_jsonl_file(file_path, max_workers=4):
-    invalid_urls = []
-    valid_urls = 0
-
-    # Gather events with score > 0.4
-    urls_to_check = []
-    with open(file_path, "r", encoding="utf-8") as f:
-        for line in f:
-            line_data = json.loads(line)
-            if line_data.get("status") != "success":
-                continue
-            for event in line_data.get("events", []):
-                if event.get("score", 0) > 0.4:
-                    urls_to_check.append(event)
-
-    total_urls = len(urls_to_check)
-
-    # ThreadPoolExecutor with tqdm progress bar
-    with ThreadPoolExecutor(max_workers=max_workers) as executor:
-        future_to_event = {executor.submit(process_event, e): e for e in urls_to_check}
-        for future in tqdm(as_completed(future_to_event), total=total_urls, desc="Checking URLs"):
-            url, is_valid, error_msg = future.result()
-            if not is_valid and url:
-                invalid_urls.append((url, error_msg))
-            else:
-                valid_urls += 1
-
-    # Summary
-    if invalid_urls:
-        print("\nList of invalid URLs and reasons:")
-        for url, err in invalid_urls:
-            print(f"{url} --> {err}")
-    print("\n=== URL Validation Summary ===")
-    print(f"Total URLs processed: {total_urls}")
-    print(f"Valid URLs (loaded successfully): {valid_urls}")
-    print(f"Invalid URLs: {len(invalid_urls)}")
-    
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Validate URLs in JSONL file events using Selenium")
-    parser.add_argument("file_path", type=str, help="Path to the JSONL file")
-    parser.add_argument("--workers", type=int, default=4, help="Number of parallel Selenium workers")
-    args = parser.parse_args()
-
-    process_jsonl_file(args.file_path, max_workers=args.workers)

supporting/dbkf/fetch.py

@@ -27,7 +27,7 @@ DEFAULT_PARAMS = [
     ("organization", "http://weverify.eu/resource/Organization/3727f7b2aa90ec0716693e5464b28d18"), # StopFake
 ]
 
-NUM_RANDOM_CLAIMS = 2000
+NUM_RANDOM_CLAIMS = 200
 
 INPUT_FILE = "../../data/input.jsonl"
 OUTPUT_FILE = "../../data/claims.json"