Improve response extraction

agent/.env.example

@@ -4,3 +4,4 @@ LANGSMITH_API_KEY=123456
 LANGSMITH_ENDPOINT=https://eu.api.smith.langchain.com
 SCRAPER_INSTANCE=https://example.com
 SCRAPER_PARAM_ANYTHING=else
+RANKING_URL=http://localhost:8000/evaluate

agent/README.md

@@ -1,31 +1,3 @@
 ## Refining the agent output
 
-Experiments modifying pipeline
+TODO: Table and document experiments
-
-| Model            | % Correct | % Change |
-|------------------|----------:|---------:|
-| BASELINE         | 33        | 0        |
-| Improv Prompt    | 39.96     | 0.21     |
-| Add Examples     | 44.67     | 0.35     |
-| Date             | 45.51     | 0.38     |
-| Chain of Thought | 43.38     | 0.31     |
-| Self-Critique    | 44.36     | 0.34     |
-
-Experiments with different model types:
-| Model                         | % Correct | % Change |
-|-------------------------------|----------:|---------:|
-| gpt-5-mini                    | 45.51     |          |
-| gpt-5.4-mini                  | 32.4      |          |
-| gpt-5.4-nano                  | 23.28     |          |
-| gpt-4.1-mini                  | 27.85     |          |
-| gpt-4o-mini                   | 32.47     |          |
-| llama3.1:8b-instruct-q4_K_M   | ?         |          |
-| qwen3.5:9b                    | 0         |          |
-
-%age valid URLS
-| Model                         | Number    | % Age    |
-|-------------------------------|----------:|---------:|
-| gpt-5-mini                    | 22/405    | 5.43     |
-| gpt-5.4-mini                  | 29/278    | 10.43    |
-| llama3.1:8b-instruct-q4_K_M   | ?         | ?        |
-| qwen3.5:9b                    | 0         | 0        |

agent/nodes/model.ts

@@ -1,25 +1,28 @@
-import { HumanMessage, SystemMessage } from "@langchain/core/messages";
+import { SystemMessage } from "@langchain/core/messages";
 import { GraphNode } from "@langchain/langgraph";
 import { MessagesState } from "../state";
-import { ChatOpenAI } from "@langchain/openai"
+import { ChatOllama } from "@langchain/ollama";
 import { hydratePrompt } from "../prompts/hydratePrompt";
+import { logger } from "../utils/logger";
 
 export function createModelNode(tools: any, promptPath: string): GraphNode<typeof MessagesState> {
     return async (state) => {
         const sysPrompt = await hydratePrompt(promptPath, state);
 
-        const model = new ChatOpenAI({
+        const model = new ChatOllama({
-            model: "gpt-4o-mini"
+            model: "llama3.1:8b-instruct-q4_K_M",
+            temperature: 0.3
         });
+
         const modelWithTools = model.bindTools(Object.values(tools));
 
         const response = await modelWithTools.invoke([
-            new SystemMessage(
+            new SystemMessage(sysPrompt),
-                sysPrompt
-            ),
             ...state.messages,
         ]);
 
+        logger.error(response);
+
         return {
             messages: [response]
         };

agent/nodes/triggerEventSetup.ts

@@ -3,8 +3,16 @@ import { MessagesState } from "../state";
 import { AIMessage, BaseMessage } from "@langchain/core/messages";
 import { rankExampleTriggerEvents } from "../tools/retreiveExamples";
 
+function extractTE(text: string) {
+  const match = text.match(/<norm>([\s\S]*?)<\/norm>/);
+  if (!match) throw new Error("Nothing found between <norm> tags");
+  return match[1].trim();
+}
+
+
 export const triggerEventSetup: GraphNode<typeof MessagesState> = async (state) => {
-  let nc = state?.messages?.at(-1)?.content ?? "" //keep a copy of normalized trigger event. Again two things, womp womp
+  let raw = state?.messages?.at(-1)?.content ?? "" //keep a copy of normalized trigger event. Again two things, womp womp
+  let nc = extractTE(raw.toString())
 
   //Now give in-context examples. hopwfully we can self-teach?
   let similarityResults = await rankExampleTriggerEvents(state.disinformationTitle)

agent/nodes/verificationSetup.ts

@@ -1,20 +1,31 @@
 import { GraphNode } from "@langchain/langgraph";
 import { MessagesState, ProposedTriggerEventArray } from "../state";
 import { logger } from "../utils/logger";
-import { jsonrepair } from 'jsonrepair'
+import { jsonrepair } from 'jsonrepair';
+
+function extractJSON(text: string) {
+  const match = text.match(/<json>([\s\S]*?)<\/json>/);
+  if (!match) throw new Error("No JSON found between <json> tags");
+  return match[1].trim();
+}
 
 export const verificationSetup: GraphNode<typeof MessagesState> = async (state) => {
-  //this is kinda doing two things, but having two nodes for it seems overkill
-
   if (state.proposedTriggerEvent == undefined) {
-    logger.warn("No trigger events in memory, parsing")
+    logger.warn("No trigger events in memory, parsing");
 
-    let genResponse = state.messages.at(-1)?.content.toString() ?? "";
+    const genResponse = state.messages.at(-1)?.content.toString() ?? "";
 
-    const repaired = jsonrepair(genResponse);
+    let repaired: string;
+    try {
+      let extracted = extractJSON(genResponse)
+      repaired = jsonrepair(extracted);
+    } catch (repairErr: any) {
+      logger.error("Failed to repair JSON from LLM response.");
+      logger.error("Original LLM response:\n%s", genResponse);
+      throw new Error(`JSON repair failed: ${repairErr.message}`);
+    }
 
     let parsed;
-
     try {
       const json = JSON.parse(repaired);
 
@@ -27,19 +38,23 @@ export const verificationSetup: GraphNode<typeof MessagesState> = async (state)
         if (Array.isArray(firstValue)) {
           parsed = ProposedTriggerEventArray.parse(firstValue);
         } else {
-          throw new Error("No array found in JSON");
+          logger.error("No array found in JSON after parsing.");
+          logger.error("Repaired JSON:\n%s", repaired);
+          logger.error("Original LLM response:\n%s", genResponse);
+          throw new Error("No array found in JSON structure");
         }
       }
-    } catch (err: any) {
+    } catch (parseErr: any) {
-      logger.error(`Failed to parse LLM response: ${err.message}`);
+      logger.error("Failed to parse LLM response to JSON or validate array.");
-      throw new Error(`Failed to parse LLM response: ${err}`);
+      logger.error("Repaired JSON:\n%s", repaired);
+      logger.error("Original LLM response:\n%s", genResponse);
+      throw new Error(`Parsing failed: ${parseErr.message}`);
     }
 
     return { proposedTriggerEvent: parsed, proposedTriggerEventIndex: 0 };
-  }
+  } else {
-  else {
+    logger.info("Trigger event index %s", state.proposedTriggerEventIndex + 1);
-    logger.info("Trigger event index %s", state.proposedTriggerEventIndex+1)
 
-    return { proposedTriggerEvent: state.proposedTriggerEvent, proposedTriggerEventIndex: state.proposedTriggerEventIndex+1 };
+    return { proposedTriggerEvent: state.proposedTriggerEvent, proposedTriggerEventIndex: state.proposedTriggerEventIndex + 1 };
   }
 };

agent/package.json

@@ -17,6 +17,7 @@
     "@langchain/core": "^1.1.17",
     "@langchain/langgraph": "^1.1.2",
     "@langchain/langgraph-sdk": "^1.5.5",
+    "@langchain/ollama": "^1.2.6",
     "@langchain/openai": "^1.2.3",
     "axios": "^1.13.5",
     "compute-cosine-similarity": "^1.1.0",

agent/prompts/normalization.txt

@@ -16,4 +16,7 @@ Relevent examples are included in preceeding messages, use these as exact inspir
 The claim to normalize is:
 ###TITLE###
 
-Produce no other text other than the condensed claim.
+Produce no other text other than the condensed claim, surrounded <norm></norm>
+
+For example: BREAKING: the sky is green!
+Becomes: <norm>The sky is green</norm>

agent/prompts/relation.txt

@@ -1,9 +0,0 @@
-Could the following real-world event:
-###TECLAIM###
-
-Be a trigger for the following disinformation:
-###TITLE###
-
-Respond with "RELATION", followed by : followed by a confidence score (VERYHIGH, HIGH, MEDIUM, LOW, VERYLOW) followed by : followed by the reason. Use no other words, just return the score and reason in format.
-
-Ignore wether the event happened or not, purely consider the likiness of causation

agent/prompts/trigger.txt

@@ -8,10 +8,6 @@ Produce up-to 5 specific "trigger events" that happened that could have led to t
 Remember the time frame of the disinformation campaign: ###CDATE###
 Include no information or events that would not have been available at the time. 
 
-You MEED TO use the tools available to you in order to produce up to date information on URL and search query, else you will be wrong and the analysis invalid.
-You NEED TO use the web search and open URL tools to ensure page validity or else all work upto this point will have to be discarded.
-
-
 Produce no more text other than the json.
 
 Include a concise but specific search query that can be looked up on a search engine in order to allow for the verification. 
@@ -21,6 +17,15 @@ Include a url to a source for your trigger event (not a web search, a specific u
 Include the date that the event happened ("March 2022" for exmaple)
 
 Use a JSON format with each entry containing "Event,ReasoningWhyRelevant,SearchQuery,Url,Date".
+Return ONLY JSON, no extra text. Wrap it like this:
+<json>
+[
+    {
+        "Event": "Example"
+        ...
+    }
+]
+</json>
 
 Multiple tool invocations should be requested at once, if applicable.
 Use your abilities to look between the lines and produce some insightful analysis, thinking both short and long term.
@@ -30,9 +35,8 @@ Events will be reordered as part of processing, each statement must stand alone
 The preceeding messages act as examples of previous responses to potentially ficitonal events and scores given.
 Analysis should only be completed for proposed events that would graner >0.7 points
 
-This pipeline is running well pasy your knowledge cutoff. 
+Since URLs change frequently, use tools to retreive up to date informaiton everytime, provided examples or existing knowledge will be wrong or out of date.
-Any URLs will change signigicantly over time. 
+
-You MEED TO use the tools available to you in order to produce up to date information on URL and search query, else you will be wrong and the analysis invalid.
+Remember to return just json enclosed by <json></json>
-You NEED TO use the web search and open URL tools to ensure page validity or else all work upto this point will have to be discarded.
 
 Lets go through it step by step

agent/prompts/verify.txt

@@ -1,8 +0,0 @@
-Do the search results cited below
-###TESEARCH###
-Support the idea that the following happened:
-###TECLAIM###
-
-Respond with "CONFIDENCE", followed by : followed by a confidence score (VERYHIGH, HIGH, MEDIUM, LOW, VERYLOW) followed by : followed by the reason. Use no other words, just return the score and reason in format.
-
-Dates can be off by a few days, that would still be valid

agent/tools/ensembleCall.ts

@@ -7,7 +7,7 @@ export async function evaluateWithEnsemble({
   answer: string;
   method: string
 }): Promise<{ validProb: number; invalidProb: number; }> {
-  const res = await axios.post("http://localhost:8000/evaluate", {
+    const res = await axios.post(process.env.RANKING_URL ?? "http://localhost:8000/evaluate", {
     answer,
     method
   }, {timeout: 0});
@@ -18,11 +18,15 @@ export async function evaluateWithEnsemble({
   return {validProb, invalidProb};
 }
 
-// let res = await evaluateWithRoberta({answer: "High-profile political downplaying of COVID-19 (examples: President Trump saying 'it will go away' in March–August 2020)"});
+// import dotenv from "dotenv";
+
+// dotenv.config();
+
+// let res = await evaluateWithEnsemble({method:"flan" ,answer: "High-profile political downplaying of COVID-19 (examples: President Trump saying 'it will go away' in March–August 2020)"});
 // console.log(res)
 
-// res = await evaluateWithRoberta({answer: "Multiple mirrored reuploads (2020–2023) put the clip on other channels with titles implying it was a genuine 1970s public information film."});
+// res = await evaluateWithEnsemble({method:"roberta" ,answer: "Multiple mirrored reuploads (2020–2023) put the clip on other channels with titles implying it was a genuine 1970s public information film."});
 // console.log(res)
 
-// res = await evaluateWithRoberta({answer: "The COVID-19 Pandemic"});
+// res = await evaluateWithEnsemble({method:"logreg" ,answer: "The COVID-19 Pandemic"});
 // console.log(res)

agent/tools/ragasCall.ts

@@ -1,22 +0,0 @@
-import axios from "axios";
-
-export async function evaluateWithRagas({
-  question,
-  answer,
-  contexts,
-}: {
-  question: string;
-  answer: string;
-  contexts: string[];
-}) {
-  const res = await axios.post("http://localhost:8001/evaluate", {
-    question,
-    answer,
-    contexts,
-  });
-
-  return res.data;
-}
-
-// let res = await evaluateWithRagas({question: "Who was Bill Nye", answer: "Bill Nye was a Scientist", contexts: ["Bill nye was a Scientist"]});
-// console.log(res)

agent/tools/webpageFetch.ts

@@ -26,6 +26,9 @@ async function extractWebpageContentWorker(url: string): Promise<string[]> {
   try {
     const options = new firefox.Options();
     options.addArguments("--headless");
+    options.addArguments("--disable-gpu");
+    options.addArguments("--no-sandbox"); // Linux sandbox issues
+    options.addArguments("--disable-dev-shm-usage"); // /dev/shm issues
     driver = await new Builder()
       .forBrowser(Browser.FIREFOX)
       .setFirefoxOptions(options)

run.sh

@@ -5,7 +5,7 @@ set -e
 run_agent () {
     echo "Starting LangGraph agent..."
     cd agent
-    npx @langchain/langgraph-cli dev --host 127.0.0.1
+    npx @langchain/langgraph-cli dev
 }
 
 run_ensemble_service () {

supporting/RAGAS_Service/ensemble_service.py

@@ -92,7 +92,7 @@ LABEL_TO_INT = {v: k for k, v in INT_TO_LABEL.items()}
 flan_tokenizer = AutoTokenizer.from_pretrained(FLAN_PATH)
 flan_model = AutoModelForSeq2SeqLM.from_pretrained(FLAN_PATH)
 
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+device = torch.device("cpu")
 flan_model.to(device)
 flan_model.eval()
 

supporting/Wrapper/run.ts

@@ -17,7 +17,7 @@ const AGENT_NAME = process.env.AGENT ?? "agent";
  */
 const MODE = process.env.MODE ?? "claim";
 
-const MAX_CONCURRENCY = 5;
+const MAX_CONCURRENCY = 1;
 
 const client = new Client({ apiUrl: API_URL });
 

supporting/checker/checker.py

@@ -1,119 +0,0 @@
-import json
-import argparse
-from urllib.parse import urlparse
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-from selenium.common.exceptions import WebDriverException, TimeoutException, StaleElementReferenceException
-from tqdm import tqdm
-
-def init_driver():
-    options = Options()
-    options.headless = True
-    options.add_argument("--disable-gpu")
-    options.add_argument("--no-sandbox")
-    options.add_argument("--headless")
-    options.add_argument("--disable-blink-features=AutomationControlled")
-    options.add_argument("--window-size=1920,1080")
-    prefs = {
-        "profile.managed_default_content_settings.images": 2,  # block images
-        "profile.default_content_setting_values.stylesheets": 2,  # block CSS
-        "profile.managed_default_content_settings.cookies": 2,  # optional
-    }
-    options.add_experimental_option("prefs", prefs)
-
-    driver = webdriver.Chrome(options=options)
-    driver.set_page_load_timeout(30)
-    return driver
-
-def is_root_url(url):
-    parsed = urlparse(url)
-    return parsed.path in ("", "/")
-
-def is_404_page(driver):
-    """Safely check for 404, handling stale elements."""
-    try:
-        title = driver.title.lower()
-        body_text = driver.find_element("tag name", "body").text.lower()
-        return "404" in title or "404" in body_text
-    except StaleElementReferenceException:
-        return False
-    except Exception:
-        return False
-
-def check_url_selenium(url):
-    driver = None
-    try:
-        driver = init_driver()
-        driver.get(url)
-        # 404 check
-        if is_404_page(driver):
-            return False, "404 page detected"
-        # Root URL after redirects
-        final_url = driver.current_url
-        if is_root_url(final_url):
-            return False, f"Redirected to root URL ({final_url})"
-        return True, None
-    except (WebDriverException, TimeoutException) as e:
-        return False, str(e)
-    finally:
-        if driver:
-            driver.quit()
-
-def process_event(event):
-    """Process an event only if score > 0.4."""
-    score = event.get("score", 0)
-    if score <= 0.4:
-        return None, False, "Score too low"
-    url = event.get("Url")
-    if not url:
-        return None, False, "No URL"
-    is_valid, error_msg = check_url_selenium(url)
-    event["url_valid"] = is_valid
-    return url, is_valid, error_msg
-
-def process_jsonl_file(file_path, max_workers=4):
-    invalid_urls = []
-    valid_urls = 0
-
-    # Gather events with score > 0.4
-    urls_to_check = []
-    with open(file_path, "r", encoding="utf-8") as f:
-        for line in f:
-            line_data = json.loads(line)
-            if line_data.get("status") != "success":
-                continue
-            for event in line_data.get("events", []):
-                if event.get("score", 0) > 0.4:
-                    urls_to_check.append(event)
-
-    total_urls = len(urls_to_check)
-
-    # ThreadPoolExecutor with tqdm progress bar
-    with ThreadPoolExecutor(max_workers=max_workers) as executor:
-        future_to_event = {executor.submit(process_event, e): e for e in urls_to_check}
-        for future in tqdm(as_completed(future_to_event), total=total_urls, desc="Checking URLs"):
-            url, is_valid, error_msg = future.result()
-            if not is_valid and url:
-                invalid_urls.append((url, error_msg))
-            else:
-                valid_urls += 1
-
-    # Summary
-    if invalid_urls:
-        print("\nList of invalid URLs and reasons:")
-        for url, err in invalid_urls:
-            print(f"{url} --> {err}")
-    print("\n=== URL Validation Summary ===")
-    print(f"Total URLs processed: {total_urls}")
-    print(f"Valid URLs (loaded successfully): {valid_urls}")
-    print(f"Invalid URLs: {len(invalid_urls)}")
-    
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Validate URLs in JSONL file events using Selenium")
-    parser.add_argument("file_path", type=str, help="Path to the JSONL file")
-    parser.add_argument("--workers", type=int, default=4, help="Number of parallel Selenium workers")
-    args = parser.parse_args()
-
-    process_jsonl_file(args.file_path, max_workers=args.workers)